@oneuptime/common 12.0.3 → 12.0.4

package/Server/Services/DatabaseService.ts

@@ -30,6 +30,7 @@ import UpdateByID from "../Types/Database/UpdateByID";
 import UpdateByIDAndFetch from "../Types/Database/UpdateByIDAndFetch";
 import UpdateOneBy from "../Types/Database/UpdateOneBy";
 import Encryption from "../Utils/Encryption";
+import PasswordHash from "../Utils/PasswordHash";
 import PostgresErrorTranslator from "../Utils/Database/PostgresErrorTranslator";
 import logger, { LogAttributes } from "../Utils/Logger";
 import BaseService from "./BaseService";
@@ -321,6 +322,55 @@ class DatabaseService<TBaseModel extends BaseModel> extends BaseService {
     return data;
   }
 
+  /*
+   * Hash one HashedString column in place.
+   *
+   * Declaring `hashSaltColumn` on a column is what marks it a CREDENTIAL —
+   * a low-entropy, human-chosen secret. Those go through scrypt (see
+   * PasswordHash): deliberately slow and memory-hard, with a fresh per-record
+   * salt minted into the sibling column named by the metadata.
+   *
+   * Columns without a salt column are high-entropy values the server itself
+   * generated — session refresh tokens and the like. They keep the fast
+   * SHA-256 hash, which is the right tool for them: there is nothing to guess,
+   * and they have to stay searchable by hash because that is how they are
+   * looked up.
+   *
+   * The salt is written onto the SAME payload as the hash, so the pair can
+   * never be persisted out of step with each other.
+   *
+   * The salt column grants nobody create or update rights, so it has to get
+   * past the column-permission check some other way on each path. On update
+   * it does so by timing — sanitizeCreateOrUpdate runs after the check. On
+   * create the check runs after `hash()`, so the salt column relies on being
+   * declared `computed: true`, which checkDataColumnPermissions skips on
+   * create. A salt column that forgets that flag breaks every non-root
+   * create of a row carrying a password.
+   */
+  private async hashColumnValue(data: {
+    payload: TBaseModel | PartialEntity<TBaseModel>;
+    columnName: string;
+    columnValue: HashedString;
+  }): Promise<void> {
+    const saltColumnName: string | undefined =
+      this.model.getTableColumnMetadata(data.columnName)?.hashSaltColumn;
+
+    if (!saltColumnName) {
+      await data.columnValue.hashValue(EncryptionSecret);
+      return;
+    }
+
+    const salt: string = PasswordHash.generateSalt();
+    (data.payload as any)[saltColumnName] = salt;
+
+    data.columnValue.setHashedValue(
+      await PasswordHash.hash({
+        plainValue: data.columnValue.toString(),
+        salt: salt,
+      }),
+    );
+  }
+
   protected async hash(data: TBaseModel): Promise<TBaseModel> {
     const columns: Columns = data.getHashedColumns();
 
@@ -329,13 +379,110 @@ class DatabaseService<TBaseModel extends BaseModel> extends BaseService {
         data.hasValue(key) &&
         !(data.getValue(key) as HashedString).isValueHashed()
       ) {
-        await ((data as any)[key] as HashedString).hashValue(EncryptionSecret);
+        await this.hashColumnValue({
+          payload: data,
+          columnName: key,
+          columnValue: (data as any)[key] as HashedString,
+        });
       }
     }
 
     return data;
   }
 
+  /*
+   * Check a plaintext secret (a password) against a hashed column on a row
+   * that has already been read out of the database, and re-hash the stored
+   * value in place when it was not written by today's scheme.
+   *
+   * The row must have been selected with BOTH the hashed column and its salt
+   * column. Without the salt a scrypt hash cannot be checked at all, and an
+   * older SHA-256 one would be checked against the wrong scheme — either way
+   * a perfectly good password gets rejected. PasswordHash.verify throws
+   * rather than silently returning false for the scrypt case, because a
+   * missing SELECT is a bug and must not look like a bad password.
+   *
+   * Why the caller cannot just query by the hash any more: with a per-user
+   * salt the hash is not computable until the row's own salt is known, so
+   * `WHERE password = <hash>` has nothing to bind. Read the row by its
+   * identity (email, id) and verify here.
+   *
+   * THE UPGRADE. Anything that is not exactly what PasswordHash writes today
+   * is re-hashed once the password is known to be correct: the two SHA-256
+   * schemes that preceded scrypt, and any scrypt hash whose cost parameters
+   * have since been raised. That is what makes raising the cost a one-line
+   * change — users migrate as they log in, with no reset and no migration.
+   *
+   * The write is deliberately hook-free and leaves updatedAt alone: the user
+   * did not change their password, the server changed how it stores it.
+   * Firing the password-change hooks would revoke every session and mail the
+   * user about a change they did not make. A failed upgrade is logged and
+   * swallowed — a correct password must not be rejected because a
+   * bookkeeping write lost a race.
+   */
+  @CaptureSpan()
+  public async verifyHashedColumnValue(data: {
+    item: TBaseModel;
+    columnName: string;
+    plainValue: string;
+  }): Promise<boolean> {
+    const storedHash: string | undefined = (data.item as any)[
+      data.columnName
+    ]?.toString();
+
+    if (!storedHash || !data.plainValue) {
+      return false;
+    }
+
+    const saltColumnName: string | undefined =
+      this.model.getTableColumnMetadata(data.columnName)?.hashSaltColumn;
+
+    const salt: string | null = saltColumnName
+      ? ((data.item as any)[saltColumnName] as string | undefined) || null
+      : null;
+
+    const isValid: boolean = await PasswordHash.verify({
+      plainValue: data.plainValue,
+      storedValue: storedHash,
+      salt: salt,
+    });
+
+    if (!isValid) {
+      return false;
+    }
+
+    if (
+      saltColumnName &&
+      data.item.id &&
+      PasswordHash.needsUpgrade(storedHash)
+    ) {
+      try {
+        const newSalt: string = PasswordHash.generateSalt();
+
+        await this.updateColumnsByIdWithoutHooks({
+          id: data.item.id,
+          data: {
+            [data.columnName]: await PasswordHash.hash({
+              plainValue: data.plainValue,
+              salt: newSalt,
+            }),
+            [saltColumnName]: newSalt,
+          } as unknown as PartialEntity<TBaseModel>,
+          skipUpdateDateColumn: true,
+        });
+      } catch (err) {
+        /*
+         * `data.item` is deliberately left holding the old hash and old salt,
+         * so it stays an internally consistent pair whether or not the write
+         * above landed.
+         */
+        logger.error(err);
+      }
+    }
+
+    return true;
+  }
+
   protected async decrypt(data: TBaseModel): Promise<TBaseModel> {
     for (const key of data.getEncryptedColumns().columns) {
       if (!data.hasValue(key)) {
@@ -552,6 +699,26 @@ class DatabaseService<TBaseModel extends BaseModel> extends BaseService {
       if (this.model.isHashedStringColumn(columnName)) {
         const columnValue: JSONValue = (data as any)[columnName];
 
+        /*
+         * A salted column's hash and its salt are only ever correct as a
+         * pair. A write that supplies the column as a bare string skips the
+         * hashing branch below, leaving the row's existing salt attached to
+         * a value it was not computed from — which locks the account out
+         * silently, at the next login, far from whatever wrote it. Refuse
+         * instead. Callers pass a HashedString; the write path hashes it.
+         */
+        if (
+          data &&
+          columnName &&
+          columnValue &&
+          !(columnValue instanceof HashedString) &&
+          this.model.getTableColumnMetadata(columnName)?.hashSaltColumn
+        ) {
+          throw new BadDataException(
+            `${columnName} is a salted column and must be supplied as a HashedString, not a pre-hashed value.`,
+          );
+        }
+
         if (
           data &&
           columnName &&
@@ -559,7 +726,23 @@ class DatabaseService<TBaseModel extends BaseModel> extends BaseService {
           columnValue instanceof HashedString
         ) {
           if (!columnValue.isValueHashed()) {
-            await columnValue.hashValue(EncryptionSecret);
+            /*
+             * The update path never runs `hash()`, so this is where an
+             * updated password gets both hashed and salted. The salt lands
+             * on `data`, which the caller turns into the SET list, so the
+             * new hash and the salt it was computed with are written by the
+             * same statement.
+             *
+             * An update matching several rows writes the same salt to all of
+             * them, which is correct: they are all being set to the same
+             * value by one statement. Per-row salts come from per-row
+             * writes.
+             */
+            await this.hashColumnValue({
+              payload: data,
+              columnName: columnName,
+              columnValue: columnValue,
+            });
           }
 
           (data as any)[columnName] = columnValue.toString();

package/Server/Services/OpenTelemetryIngestService.ts

@@ -168,6 +168,66 @@ const PROJECT_RETENTION_CACHE_TTL_SECONDS: number = 5 * 60;
 const projectRetentionInProcessCache: Map<string, CachedRetentionContext> =
   new Map();
 
+/*
+ * Everything `findOrCreateTelemetryService` needs from the Service row. Cached
+ * instead of the model itself so the payload is a flat, JSON-round-trippable
+ * shape with no TypeORM entity semantics riding along.
+ */
+interface ServiceResolution {
+  serviceId: string;
+  retainTelemetryDataForDays: number | null;
+  telemetryRetentionConfig: TelemetryRetentionConfig | null;
+}
+
+interface CachedServiceResolution {
+  resolution: ServiceResolution;
+  expiresAtMs: number;
+}
+
+/*
+ * Per-process memo for `service.name` -> Service resolution, backed by the same
+ * L1-Map + L2-Redis pair as the project retention context above.
+ *
+ * This is the hot read on the whole ingest path and it was the one ungated
+ * operation in the function. `resolveTelemetryResource` calls it once per OTLP
+ * *resource*, not per request — the shipped host-metrics collector config
+ * produces hundreds per batch — and again for every signal type a collector
+ * exports, from every ingest pod. The per-batch `serviceDictionary` in the four
+ * Otel*IngestService files cannot help: it is written AFTER this resolves and
+ * never read before it.
+ *
+ * The uncached read was also the most expensive shape it could have been.
+ * `QueryHelper.findWithSameText` emits `LOWER("Service"."name") = $2`, and the
+ * live indexes on Service are all over the RAW name column
+ * (IDX_80cd6d06b0ab8fbce9fea6d3bc on ("projectId","name")), so Postgres can use
+ * only the `projectId` prefix as an index qual and must then heap-fetch and
+ * filter every service in the project. Cost per call is O(services in project),
+ * not O(log n). The matching expression index existed once —
+ * `("projectId", LOWER("name")) WHERE "deletedAt" IS NULL` in migration
+ * 1774559064921 — and was dropped as collateral damage by autogenerated
+ * migration 1775735059360. Restoring it is a separate change; this cache is
+ * what removes the calls in the first place.
+ */
+const SERVICE_RESOLUTION_CACHE_NAMESPACE: string = "service-resolution";
+
+/*
+ * Matches the project-retention TTL deliberately: the two are read together on
+ * every resolution and there is no value in them expiring at different times.
+ *
+ * Staleness is bounded rather than invalidated, exactly as it is for project
+ * retention. A retention change made in the UI takes up to this long to affect
+ * newly ingested rows, and a service deleted in the UI keeps resolving for up
+ * to this long — which stamps `primaryEntityId` values into ClickHouse that no
+ * longer have a Postgres parent. Those rows are orphaned, not corrupt (the
+ * analytics side has no FK to Service) and they age out via retention; this is
+ * the same trade-off the dedupe migration already documents for its own
+ * ClickHouse rows. Cross-process invalidation would need pub/sub, which
+ * GlobalCache has no primitive for.
+ */
+const SERVICE_RESOLUTION_CACHE_TTL_SECONDS: number = 5 * 60;
+const serviceResolutionInProcessCache: Map<string, CachedServiceResolution> =
+  new Map();
+
 export default class OTelIngestService {
   /*
    * Read a single string-valued OTel resource attribute out of the raw
@@ -459,6 +519,114 @@ export default class OTelIngestService {
     return result;
   }
 
+  /*
+   * Normalises exactly as `QueryHelper.findWithSameText` does
+   * (`toLowerCase().trim()`). This has to stay in lockstep with that helper:
+   * if the key kept casing or surrounding whitespace that the SQL predicate
+   * folds away, two spellings of one `service.name` would occupy two cache
+   * entries pointing at the same row, and a batch could miss the entry the
+   * previous batch just wrote.
+   */
+  private static getServiceResolutionCacheKey(
+    projectId: ObjectID,
+    serviceName: string,
+  ): string {
+    return `${projectId.toString()}:${serviceName.toLowerCase().trim()}`;
+  }
+
+  private static async getCachedServiceResolution(
+    cacheKey: string,
+  ): Promise<ServiceResolution | null> {
+    const now: number = Date.now();
+
+    // L1: in-process memo. Zero network cost.
+    const memoed: CachedServiceResolution | undefined =
+      serviceResolutionInProcessCache.get(cacheKey);
+    if (memoed && memoed.expiresAtMs > now) {
+      return memoed.resolution;
+    }
+
+    // L2: Redis. Single round-trip; shared across workers.
+    try {
+      const cached: JSONObject | null = await GlobalCache.getJSONObject(
+        SERVICE_RESOLUTION_CACHE_NAMESPACE,
+        cacheKey,
+      );
+
+      /*
+       * `serviceId` is the load-bearing field — a cached entry without one
+       * cannot produce a `primaryEntityId`, so treat it as a miss and let the
+       * caller re-resolve rather than stamping rows with a broken id.
+       */
+      if (cached && cached["serviceId"]) {
+        const resolution: ServiceResolution = {
+          serviceId: String(cached["serviceId"]),
+          retainTelemetryDataForDays:
+            (cached["retainTelemetryDataForDays"] as number | null) ?? null,
+          telemetryRetentionConfig:
+            (cached[
+              "telemetryRetentionConfig"
+            ] as TelemetryRetentionConfig | null) ?? null,
+        };
+
+        serviceResolutionInProcessCache.set(cacheKey, {
+          resolution,
+          expiresAtMs: now + SERVICE_RESOLUTION_CACHE_TTL_SECONDS * 1000,
+        });
+
+        return resolution;
+      }
+    } catch (err) {
+      // Cache outage must never fail ingest. Fall through to Postgres.
+      logger.warn(
+        `Service resolution cache read failed for "${cacheKey}"; falling back to Postgres: ${
+          err instanceof Error ? err.message : String(err)
+        }`,
+      );
+    }
+
+    return null;
+  }
+
+  private static async cacheServiceResolution(
+    cacheKey: string,
+    resolution: ServiceResolution,
+  ): Promise<void> {
+    serviceResolutionInProcessCache.set(cacheKey, {
+      resolution,
+      expiresAtMs: Date.now() + SERVICE_RESOLUTION_CACHE_TTL_SECONDS * 1000,
+    });
+
+    try {
+      await GlobalCache.setJSON(
+        SERVICE_RESOLUTION_CACHE_NAMESPACE,
+        cacheKey,
+        {
+          serviceId: resolution.serviceId,
+          retainTelemetryDataForDays: resolution.retainTelemetryDataForDays,
+          telemetryRetentionConfig: (resolution.telemetryRetentionConfig ??
+            null) as unknown as JSONObject,
+        },
+        { expiresInSeconds: SERVICE_RESOLUTION_CACHE_TTL_SECONDS },
+      );
+    } catch (err) {
+      // Best-effort warm. Don't fail the request.
+      logger.warn(
+        `Service resolution cache write failed for "${cacheKey}": ${
+          err instanceof Error ? err.message : String(err)
+        }`,
+      );
+    }
+  }
+
+  private static toServiceResolution(service: Service): ServiceResolution {
+    return {
+      serviceId: service.id!.toString(),
+      retainTelemetryDataForDays: service.retainTelemetryDataForDays ?? null,
+      telemetryRetentionConfig: service.telemetryRetentionConfig ?? null,
+    };
+  }
+
   @CaptureSpan()
   private static async findOrCreateTelemetryService(data: {
     serviceName: string;
@@ -481,23 +649,38 @@ export default class OTelIngestService {
      * Stored casing is preserved: service-detail pages key telemetry by the
      * stable primaryEntityId, and the name is user-facing.
      */
-    const service: Service | null = await ServiceService.findOneBy({
-      query: {
-        projectId: data.projectId,
-        name: QueryHelper.findWithSameText(data.serviceName),
-      },
-      select: {
-        _id: true,
-        retainTelemetryDataForDays: true,
-        telemetryRetentionConfig: true,
-      },
-      sort: {
-        createdAt: SortOrder.Ascending,
-      },
-      props: {
-        isRoot: true,
-      },
-    });
+    const resolutionCacheKey: string = this.getServiceResolutionCacheKey(
+      data.projectId,
+      data.serviceName,
+    );
+
+    let resolution: ServiceResolution | null =
+      await this.getCachedServiceResolution(resolutionCacheKey);
+
+    if (!resolution) {
+      const service: Service | null = await ServiceService.findOneBy({
+        query: {
+          projectId: data.projectId,
+          name: QueryHelper.findWithSameText(data.serviceName),
+        },
+        select: {
+          _id: true,
+          retainTelemetryDataForDays: true,
+          telemetryRetentionConfig: true,
+        },
+        sort: {
+          createdAt: SortOrder.Ascending,
+        },
+        props: {
+          isRoot: true,
+        },
+      });
+
+      if (service) {
+        resolution = this.toServiceResolution(service);
+        await this.cacheServiceResolution(resolutionCacheKey, resolution);
+      }
+    }
 
     const projectContext: ProjectRetentionContext =
       await this.getProjectRetentionContext(data.projectId);
@@ -513,14 +696,16 @@ export default class OTelIngestService {
       data.serviceName,
     );
 
-    const buildMetadata: (svc: Service) => TelemetryServiceMetadata = (
-      svc: Service,
+    const buildMetadata: (
+      resolved: ServiceResolution,
+    ) => TelemetryServiceMetadata = (
+      resolved: ServiceResolution,
     ): TelemetryServiceMetadata => {
       const serviceLevelRetention: number | null =
-        svc.retainTelemetryDataForDays ?? null;
+        resolved.retainTelemetryDataForDays ?? null;
       return {
         serviceName: data.serviceName,
-        primaryEntityId: svc.id!,
+        primaryEntityId: new ObjectID(resolved.serviceId),
         primaryEntityType: ServiceType.OpenTelemetry,
         entityKeys: [serviceEntityKey],
         /*
@@ -534,14 +719,14 @@ export default class OTelIngestService {
         },
         dataRententionInDays:
           serviceLevelRetention || projectContext.projectRetentionInDays,
-        serviceRetentionConfig: svc.telemetryRetentionConfig ?? null,
+        serviceRetentionConfig: resolved.telemetryRetentionConfig ?? null,
         serviceRetentionInDays: serviceLevelRetention,
         projectRetentionConfig: projectContext.projectRetentionConfig,
         projectRetentionInDays: projectContext.projectRetentionInDays,
       };
     };
 
-    if (!service) {
+    if (!resolution) {
       try {
         const newService: Service = new Service();
         newService.projectId = data.projectId;
@@ -555,11 +740,23 @@ export default class OTelIngestService {
           },
         });
 
-        return buildMetadata(createdService);
+        const createdResolution: ServiceResolution =
+          this.toServiceResolution(createdService);
+        await this.cacheServiceResolution(
+          resolutionCacheKey,
+          createdResolution,
+        );
+
+        return buildMetadata(createdResolution);
       } catch {
         /*
          * Race condition: another request created the service concurrently.
          * Re-fetch the existing service (oldest wins, see sort above).
+         *
+         * This refetch emits SQL byte-identical to the lookup above, so it is
+         * indistinguishable from it in pg_stat_activity — caching the result
+         * here matters as much as caching it there. A first-contact stampede
+         * on a brand-new service.name is exactly when this branch runs hot.
          */
         const existingService: Service | null = await ServiceService.findOneBy({
           query: {
@@ -580,7 +777,14 @@ export default class OTelIngestService {
         });
 
         if (existingService) {
-          return buildMetadata(existingService);
+          const existingResolution: ServiceResolution =
+            this.toServiceResolution(existingService);
+          await this.cacheServiceResolution(
+            resolutionCacheKey,
+            existingResolution,
+          );
+
+          return buildMetadata(existingResolution);
         }
 
         throw new Error(
@@ -589,7 +793,7 @@ export default class OTelIngestService {
       }
     }
 
-    return buildMetadata(service);
+    return buildMetadata(resolution);
   }
 
   /*