@oneuptime/common 12.0.3 → 12.0.5

package/Server/Services/OpenTelemetryIngestService.ts

@@ -168,6 +168,66 @@ const PROJECT_RETENTION_CACHE_TTL_SECONDS: number = 5 * 60;
 const projectRetentionInProcessCache: Map<string, CachedRetentionContext> =
   new Map();
 
+/*
+ * Everything `findOrCreateTelemetryService` needs from the Service row. Cached
+ * instead of the model itself so the payload is a flat, JSON-round-trippable
+ * shape with no TypeORM entity semantics riding along.
+ */
+interface ServiceResolution {
+  serviceId: string;
+  retainTelemetryDataForDays: number | null;
+  telemetryRetentionConfig: TelemetryRetentionConfig | null;
+}
+
+interface CachedServiceResolution {
+  resolution: ServiceResolution;
+  expiresAtMs: number;
+}
+
+/*
+ * Per-process memo for `service.name` -> Service resolution, backed by the same
+ * L1-Map + L2-Redis pair as the project retention context above.
+ *
+ * This is the hot read on the whole ingest path and it was the one ungated
+ * operation in the function. `resolveTelemetryResource` calls it once per OTLP
+ * *resource*, not per request — the shipped host-metrics collector config
+ * produces hundreds per batch — and again for every signal type a collector
+ * exports, from every ingest pod. The per-batch `serviceDictionary` in the four
+ * Otel*IngestService files cannot help: it is written AFTER this resolves and
+ * never read before it.
+ *
+ * The uncached read was also the most expensive shape it could have been.
+ * `QueryHelper.findWithSameText` emits `LOWER("Service"."name") = $2`, and the
+ * live indexes on Service are all over the RAW name column
+ * (IDX_80cd6d06b0ab8fbce9fea6d3bc on ("projectId","name")), so Postgres can use
+ * only the `projectId` prefix as an index qual and must then heap-fetch and
+ * filter every service in the project. Cost per call is O(services in project),
+ * not O(log n). The matching expression index existed once —
+ * `("projectId", LOWER("name")) WHERE "deletedAt" IS NULL` in migration
+ * 1774559064921 — and was dropped as collateral damage by autogenerated
+ * migration 1775735059360. Restoring it is a separate change; this cache is
+ * what removes the calls in the first place.
+ */
+const SERVICE_RESOLUTION_CACHE_NAMESPACE: string = "service-resolution";
+
+/*
+ * Matches the project-retention TTL deliberately: the two are read together on
+ * every resolution and there is no value in them expiring at different times.
+ *
+ * Staleness is bounded rather than invalidated, exactly as it is for project
+ * retention. A retention change made in the UI takes up to this long to affect
+ * newly ingested rows, and a service deleted in the UI keeps resolving for up
+ * to this long — which stamps `primaryEntityId` values into ClickHouse that no
+ * longer have a Postgres parent. Those rows are orphaned, not corrupt (the
+ * analytics side has no FK to Service) and they age out via retention; this is
+ * the same trade-off the dedupe migration already documents for its own
+ * ClickHouse rows. Cross-process invalidation would need pub/sub, which
+ * GlobalCache has no primitive for.
+ */
+const SERVICE_RESOLUTION_CACHE_TTL_SECONDS: number = 5 * 60;
+const serviceResolutionInProcessCache: Map<string, CachedServiceResolution> =
+  new Map();
+
 export default class OTelIngestService {
   /*
    * Read a single string-valued OTel resource attribute out of the raw
@@ -459,6 +519,114 @@ export default class OTelIngestService {
     return result;
   }
 
+  /*
+   * Normalises exactly as `QueryHelper.findWithSameText` does
+   * (`toLowerCase().trim()`). This has to stay in lockstep with that helper:
+   * if the key kept casing or surrounding whitespace that the SQL predicate
+   * folds away, two spellings of one `service.name` would occupy two cache
+   * entries pointing at the same row, and a batch could miss the entry the
+   * previous batch just wrote.
+   */
+  private static getServiceResolutionCacheKey(
+    projectId: ObjectID,
+    serviceName: string,
+  ): string {
+    return `${projectId.toString()}:${serviceName.toLowerCase().trim()}`;
+  }
+
+  private static async getCachedServiceResolution(
+    cacheKey: string,
+  ): Promise<ServiceResolution | null> {
+    const now: number = Date.now();
+
+    // L1: in-process memo. Zero network cost.
+    const memoed: CachedServiceResolution | undefined =
+      serviceResolutionInProcessCache.get(cacheKey);
+    if (memoed && memoed.expiresAtMs > now) {
+      return memoed.resolution;
+    }
+
+    // L2: Redis. Single round-trip; shared across workers.
+    try {
+      const cached: JSONObject | null = await GlobalCache.getJSONObject(
+        SERVICE_RESOLUTION_CACHE_NAMESPACE,
+        cacheKey,
+      );
+
+      /*
+       * `serviceId` is the load-bearing field — a cached entry without one
+       * cannot produce a `primaryEntityId`, so treat it as a miss and let the
+       * caller re-resolve rather than stamping rows with a broken id.
+       */
+      if (cached && cached["serviceId"]) {
+        const resolution: ServiceResolution = {
+          serviceId: String(cached["serviceId"]),
+          retainTelemetryDataForDays:
+            (cached["retainTelemetryDataForDays"] as number | null) ?? null,
+          telemetryRetentionConfig:
+            (cached[
+              "telemetryRetentionConfig"
+            ] as TelemetryRetentionConfig | null) ?? null,
+        };
+
+        serviceResolutionInProcessCache.set(cacheKey, {
+          resolution,
+          expiresAtMs: now + SERVICE_RESOLUTION_CACHE_TTL_SECONDS * 1000,
+        });
+
+        return resolution;
+      }
+    } catch (err) {
+      // Cache outage must never fail ingest. Fall through to Postgres.
+      logger.warn(
+        `Service resolution cache read failed for "${cacheKey}"; falling back to Postgres: ${
+          err instanceof Error ? err.message : String(err)
+        }`,
+      );
+    }
+
+    return null;
+  }
+
+  private static async cacheServiceResolution(
+    cacheKey: string,
+    resolution: ServiceResolution,
+  ): Promise<void> {
+    serviceResolutionInProcessCache.set(cacheKey, {
+      resolution,
+      expiresAtMs: Date.now() + SERVICE_RESOLUTION_CACHE_TTL_SECONDS * 1000,
+    });
+
+    try {
+      await GlobalCache.setJSON(
+        SERVICE_RESOLUTION_CACHE_NAMESPACE,
+        cacheKey,
+        {
+          serviceId: resolution.serviceId,
+          retainTelemetryDataForDays: resolution.retainTelemetryDataForDays,
+          telemetryRetentionConfig: (resolution.telemetryRetentionConfig ??
+            null) as unknown as JSONObject,
+        },
+        { expiresInSeconds: SERVICE_RESOLUTION_CACHE_TTL_SECONDS },
+      );
+    } catch (err) {
+      // Best-effort warm. Don't fail the request.
+      logger.warn(
+        `Service resolution cache write failed for "${cacheKey}": ${
+          err instanceof Error ? err.message : String(err)
+        }`,
+      );
+    }
+  }
+
+  private static toServiceResolution(service: Service): ServiceResolution {
+    return {
+      serviceId: service.id!.toString(),
+      retainTelemetryDataForDays: service.retainTelemetryDataForDays ?? null,
+      telemetryRetentionConfig: service.telemetryRetentionConfig ?? null,
+    };
+  }
+
   @CaptureSpan()
   private static async findOrCreateTelemetryService(data: {
     serviceName: string;
@@ -481,23 +649,38 @@ export default class OTelIngestService {
      * Stored casing is preserved: service-detail pages key telemetry by the
      * stable primaryEntityId, and the name is user-facing.
      */
-    const service: Service | null = await ServiceService.findOneBy({
-      query: {
-        projectId: data.projectId,
-        name: QueryHelper.findWithSameText(data.serviceName),
-      },
-      select: {
-        _id: true,
-        retainTelemetryDataForDays: true,
-        telemetryRetentionConfig: true,
-      },
-      sort: {
-        createdAt: SortOrder.Ascending,
-      },
-      props: {
-        isRoot: true,
-      },
-    });
+    const resolutionCacheKey: string = this.getServiceResolutionCacheKey(
+      data.projectId,
+      data.serviceName,
+    );
+
+    let resolution: ServiceResolution | null =
+      await this.getCachedServiceResolution(resolutionCacheKey);
+
+    if (!resolution) {
+      const service: Service | null = await ServiceService.findOneBy({
+        query: {
+          projectId: data.projectId,
+          name: QueryHelper.findWithSameText(data.serviceName),
+        },
+        select: {
+          _id: true,
+          retainTelemetryDataForDays: true,
+          telemetryRetentionConfig: true,
+        },
+        sort: {
+          createdAt: SortOrder.Ascending,
+        },
+        props: {
+          isRoot: true,
+        },
+      });
+
+      if (service) {
+        resolution = this.toServiceResolution(service);
+        await this.cacheServiceResolution(resolutionCacheKey, resolution);
+      }
+    }
 
     const projectContext: ProjectRetentionContext =
       await this.getProjectRetentionContext(data.projectId);
@@ -513,14 +696,16 @@ export default class OTelIngestService {
       data.serviceName,
     );
 
-    const buildMetadata: (svc: Service) => TelemetryServiceMetadata = (
-      svc: Service,
+    const buildMetadata: (
+      resolved: ServiceResolution,
+    ) => TelemetryServiceMetadata = (
+      resolved: ServiceResolution,
     ): TelemetryServiceMetadata => {
       const serviceLevelRetention: number | null =
-        svc.retainTelemetryDataForDays ?? null;
+        resolved.retainTelemetryDataForDays ?? null;
       return {
         serviceName: data.serviceName,
-        primaryEntityId: svc.id!,
+        primaryEntityId: new ObjectID(resolved.serviceId),
         primaryEntityType: ServiceType.OpenTelemetry,
         entityKeys: [serviceEntityKey],
         /*
@@ -534,14 +719,14 @@ export default class OTelIngestService {
         },
         dataRententionInDays:
           serviceLevelRetention || projectContext.projectRetentionInDays,
-        serviceRetentionConfig: svc.telemetryRetentionConfig ?? null,
+        serviceRetentionConfig: resolved.telemetryRetentionConfig ?? null,
         serviceRetentionInDays: serviceLevelRetention,
         projectRetentionConfig: projectContext.projectRetentionConfig,
         projectRetentionInDays: projectContext.projectRetentionInDays,
       };
     };
 
-    if (!service) {
+    if (!resolution) {
       try {
         const newService: Service = new Service();
         newService.projectId = data.projectId;
@@ -555,11 +740,23 @@ export default class OTelIngestService {
           },
         });
 
-        return buildMetadata(createdService);
+        const createdResolution: ServiceResolution =
+          this.toServiceResolution(createdService);
+        await this.cacheServiceResolution(
+          resolutionCacheKey,
+          createdResolution,
+        );
+
+        return buildMetadata(createdResolution);
       } catch {
         /*
          * Race condition: another request created the service concurrently.
          * Re-fetch the existing service (oldest wins, see sort above).
+         *
+         * This refetch emits SQL byte-identical to the lookup above, so it is
+         * indistinguishable from it in pg_stat_activity — caching the result
+         * here matters as much as caching it there. A first-contact stampede
+         * on a brand-new service.name is exactly when this branch runs hot.
          */
         const existingService: Service | null = await ServiceService.findOneBy({
           query: {
@@ -580,7 +777,14 @@ export default class OTelIngestService {
         });
 
         if (existingService) {
-          return buildMetadata(existingService);
+          const existingResolution: ServiceResolution =
+            this.toServiceResolution(existingService);
+          await this.cacheServiceResolution(
+            resolutionCacheKey,
+            existingResolution,
+          );
+
+          return buildMetadata(existingResolution);
         }
 
         throw new Error(
@@ -589,7 +793,7 @@ export default class OTelIngestService {
       }
     }
 
-    return buildMetadata(service);
+    return buildMetadata(resolution);
   }
 
   /*

package/Server/Services/UserService.ts

@@ -24,6 +24,9 @@ import URL from "../../Types/API/URL";
 import DatabaseCommonInteractionProps from "../../Types/BaseDatabase/DatabaseCommonInteractionProps";
 import LIMIT_MAX from "../../Types/Database/LimitMax";
 import OneUptimeDate from "../../Types/Date";
+import PositiveNumber from "../../Types/PositiveNumber";
+import ProjectService from "./ProjectService";
+import Semaphore, { SemaphoreMutex } from "../Infrastructure/Semaphore";
 import Email from "../../Types/Email";
 import EmailTemplateType from "../../Types/Email/EmailTemplateType";
 import HashedString from "../../Types/HashedString";
@@ -45,6 +48,37 @@ import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
 import Timezone from "../../Types/Timezone";
 import InMemoryTTLCache from "../Infrastructure/InMemoryTTLCache";
 
+/*
+ * Names the Redis mutex that serializes the first-Master-Admin election across
+ * every process on this instance. Namespaced the same way as the other
+ * cross-process mutexes in this codebase (see ProjectService's counter locks)
+ * so the key is self-documenting in Redis and cannot collide with another lock.
+ */
+const FIRST_MASTER_ADMIN_ELECTION_LOCK_NAMESPACE: string =
+  "UserService.firstMasterAdminElection";
+const FIRST_MASTER_ADMIN_ELECTION_LOCK_KEY: string = "instance";
+
+/*
+ * How long the mutex survives without being refreshed. This is a crash bound,
+ * NOT a work budget: redis-semaphore refreshes an held lock every
+ * 0.8 * lockTimeout for as long as the holder is alive, so a slow COUNT+INSERT
+ * does not expire the lock out from under itself. It only matters when a holder
+ * dies mid-election (SIGKILL, eviction) — the lock then frees itself after this
+ * long instead of wedging every subsequent signup on the instance forever.
+ */
+const FIRST_MASTER_ADMIN_ELECTION_LOCK_TIMEOUT_MS: number = 10_000;
+
+/*
+ * How long a signup waits for the lock before giving up. Signup is a
+ * user-facing request, so the wait has to be bounded rather than indefinite —
+ * this replaces the bound that Postgres `lock_timeout` used to give us.
+ *
+ * Failing the signup is the correct outcome when it fires: refusing to serve
+ * one request is safe, serving it unserialized is exactly the bug this lock
+ * exists to prevent (GHSA-3qqq-hprx-g2jw).
+ */
+const FIRST_MASTER_ADMIN_ELECTION_ACQUIRE_TIMEOUT_MS: number = 5_000;
+
 export class Service extends DatabaseService<Model> {
   /*
    * Suppresses repeated `lastActive` UPDATEs from a single API node. 60s of
@@ -499,6 +533,156 @@ export class Service extends DatabaseService<Model> {
     return onUpdate;
   }
 
+  /**
+   * Creates a user who is signing themselves up, and decides — atomically —
+   * whether they are the instance's very first Master Admin.
+   *
+   * Every signup MUST come through here rather than calling `create` directly,
+   * because this method is the only place that is allowed to set
+   * `isMasterAdmin` on a self-service signup. It always assigns the column, so
+   * an `isMasterAdmin: true` smuggled into the request body is overwritten
+   * rather than honoured (signup creates with `isRoot: true`, which bypasses
+   * the column's empty `create: []` access control).
+   */
+  @CaptureSpan()
+  public async createUserOnSignup(data: {
+    user: Model;
+    props: DatabaseCommonInteractionProps;
+  }): Promise<Model> {
+    // Never inherited from the caller. Decided below, or not at all.
+    data.user.isMasterAdmin = false;
+
+    if (IsBillingEnabled) {
+      /*
+       * On the hosted service there is no first-user bootstrap: master admins
+       * are provisioned out of band, so there is nothing to elect and no reason
+       * to take a lock.
+       */
+      return await this.create({
+        data: data.user,
+        props: data.props,
+      });
+    }
+
+    /*
+     * Self-hosted: the first user to sign up bootstraps the instance as Master
+     * Admin. The eligibility check reads the User table and the create writes to
+     * it, which is a read-then-write and therefore not atomic on its own — two
+     * signups landing together both counted zero users and both became Master
+     * Admin (GHSA-3qqq-hprx-g2jw). Serialize the check and the insert
+     * instance-wide so the second request observes the first one's row.
+     *
+     * The mutex is the Redis one every other cross-process critical section in
+     * this codebase uses (Semaphore), not a Postgres advisory lock. It holds no
+     * database connection and no open transaction, so serializing signups
+     * cannot pin a pooled backend `idle in transaction`, and it is unaffected
+     * by how PgBouncer is pooling.
+     */
+    let mutex: SemaphoreMutex;
+
+    try {
+      mutex = await Semaphore.lock({
+        key: FIRST_MASTER_ADMIN_ELECTION_LOCK_KEY,
+        namespace: FIRST_MASTER_ADMIN_ELECTION_LOCK_NAMESPACE,
+        lockTimeout: FIRST_MASTER_ADMIN_ELECTION_LOCK_TIMEOUT_MS,
+        acquireTimeout: FIRST_MASTER_ADMIN_ELECTION_ACQUIRE_TIMEOUT_MS,
+      });
+    } catch (err) {
+      /*
+       * Redis unreachable, or the lock stayed contended past the acquire
+       * timeout. Fail the signup rather than fall back to an unserialized
+       * election — the fallback is the vulnerability.
+       */
+      logger.error(
+        "Refusing to create a signup: could not acquire the first-Master-Admin election lock.",
+      );
+      logger.error(err);
+
+      throw err;
+    }
+
+    /*
+     * The critical section deliberately covers the INSERT, not just the count:
+     * releasing after the count would let the next holder count a table the
+     * winner had not written to yet, which is the same race with extra steps.
+     *
+     * `finally` so a failed create still frees the lock immediately instead of
+     * blocking every other signup until lockTimeout elapses.
+     */
+    try {
+      data.user.isMasterAdmin = await this.isInstanceAwaitingFirstUser();
+
+      return await this.create({
+        data: data.user,
+        props: data.props,
+      });
+    } finally {
+      try {
+        await Semaphore.release(mutex);
+      } catch (err) {
+        /*
+         * Best-effort and non-masking: an error here must not replace the
+         * outcome of the signup, and an unreleased lock still expires on its
+         * own after lockTimeout.
+         */
+        logger.error(
+          "Failed to release the first-Master-Admin election lock; it will expire on its own.",
+        );
+        logger.error(err);
+      }
+    }
+  }
+
+  /**
+   * Is this a brand-new instance that nobody has ever signed up to — i.e. may
+   * the next user through the door take Master Admin?
+   *
+   * Only ever called while holding the first-Master-Admin election mutex.
+   */
+  private async isInstanceAwaitingFirstUser(): Promise<boolean> {
+    const userCount: PositiveNumber = await this.countBy({
+      props: {
+        isRoot: true,
+      },
+      query: {},
+    });
+
+    if (!userCount.isZero()) {
+      return false;
+    }
+
+    /*
+     * Zero users, but projects exist: this is not a fresh install, it is an
+     * instance whose User table was emptied out from under it — a bad restore, a
+     * cleanup script, a cascade. Handing Master Admin to whoever signs up next
+     * would be a free promotion for a stranger, so refuse and make the operator
+     * grant it deliberately.
+     *
+     * This cannot fire on any supported path: a user who belongs to a project
+     * cannot be deleted (see onBeforeDelete), so "projects with no users at all"
+     * is only reachable by writing to the database directly. It does not get in
+     * the way of the ordinary recovery either — someone who signs up, has no
+     * projects yet, deletes their account and signs up again still bootstraps
+     * normally.
+     */
+    const projectCount: PositiveNumber = await ProjectService.countBy({
+      props: {
+        isRoot: true,
+      },
+      query: {},
+    });
+
+    if (!projectCount.isZero()) {
+      logger.warn(
+        "Refusing to grant Master Admin to a new signup: the User table is empty but this instance already has projects. Grant Master Admin explicitly instead.",
+      );
+
+      return false;
+    }
+
+    return true;
+  }
+
   @CaptureSpan()
   public async createByEmail(data: {
     email: Email;

package/Server/Utils/AI/Chat/ObservabilityAssistant.ts

@@ -59,6 +59,9 @@ export interface ObservabilityAssistantStep {
 export interface ObservabilityAssistantRequest {
   projectId: ObjectID;
   userId?: ObjectID | undefined;
+  incidentId?: ObjectID | undefined;
+  alertId?: ObjectID | undefined;
+  aiRunId?: ObjectID | undefined;
   // The requesting user's real permission props — tools run under these.
   props: DatabaseCommonInteractionProps;
   question: string;
@@ -230,6 +233,9 @@ export default class ObservabilityAssistant {
       const response: AILogResponse = await AIService.executeWithLogging({
         projectId: request.projectId,
         userId: request.userId,
+        incidentId: request.incidentId,
+        alertId: request.alertId,
+        aiRunId: request.aiRunId,
         llmProviderId: request.llmProviderId,
         feature: request.feature,
         messages: messages,

package/Server/Utils/AI/CodeFix/CodeFixAgentCompletion.ts

@@ -130,6 +130,8 @@ export default class CodeFixAgentCompletion {
         runType: true,
         status: true,
         aiAgentId: true,
+        triggeredByIncidentId: true,
+        triggeredByAlertId: true,
       },
       props: {
         isRoot: true,
@@ -198,6 +200,8 @@ export default class CodeFixAgentCompletion {
       projectId: run.projectId,
       feature: AI_CODE_FIX_FEATURE,
       aiRunId: request.aiRunId,
+      incidentId: run.triggeredByIncidentId,
+      alertId: run.triggeredByAlertId,
       llmProviderId: llmProvider.id,
       messages: request.messages,
       tools: request.tools,

package/Server/Utils/AI/CodeFix/CodeFixReadiness.ts

@@ -102,9 +102,12 @@ export default class CodeFixReadiness {
      * self-hosted too, where a project-owned provider makes the balance gate
      * moot and this becomes the ONLY thing that can kill a run.
      *
-     * A limit of 0 is a documented kill-switch ("pause AI entirely"), i.e.
-     * durable config — so without this a paused project would read "ready"
-     * forever while every run died at its first completion call.
+     * These readiness surfaces create subjectless exception/general tasks, so
+     * they use the "Other AI Workload" token lane. Incident/alert fix paths
+     * enforce their own lane when they are triggered and on every completion.
+     * A limit of 0 is a documented kill-switch for this lane, so without this
+     * a paused project would read "ready" forever while every subjectless run
+     * died at its first completion call.
      */
     const budget: AutonomousBudgetStatus =
       await AIService.getAutonomousDailyBudgetStatus(params.projectId);
@@ -116,8 +119,8 @@ export default class CodeFixReadiness {
         title: "LLM provider",
         detail:
           budget.limitInTokens !== null && budget.limitInTokens <= 0
-            ? "AI is paused for this project: the daily autonomous AI token limit is set to 0. Raise or unset it in the AI settings pages to let fix tasks run."
-            : `The daily autonomous AI token budget is exhausted (${budget.usedTokensToday.toLocaleString()} of ${budget.limitInTokens?.toLocaleString()} tokens used today). Fix tasks resume tomorrow (UTC) — raise or unset the limit in the AI settings pages.`,
+            ? "Other autonomous AI work is paused for this project: the daily background AI token limit is set to 0. Raise or unset it under Project Settings > AI > AI Guardrails to let subjectless fix tasks run."
+            : `The daily background AI token budget is exhausted (${budget.usedTokensToday.toLocaleString()} of ${budget.limitInTokens?.toLocaleString()} tokens used today). Subjectless fix tasks resume tomorrow (UTC) — raise or unset the limit under Project Settings > AI > AI Guardrails.`,
       };
     }