@oneuptime/common 11.7.4 → 12.0.1

package/Server/Services/RumApplicationService.ts

@@ -7,9 +7,10 @@ import QueryHelper from "../Types/Database/QueryHelper";
 import OneUptimeDate from "../../Types/Date";
 import LIMIT_MAX from "../../Types/Database/LimitMax";
 import GlobalCache from "../Infrastructure/GlobalCache";
+import SessionReplayGateCacheStore from "../Utils/SessionReplay/SessionReplayGateCacheStore";
 import logger, { LogAttributes } from "../Utils/Logger";
 import crypto from "crypto";
-import { OnCreate } from "../Types/Database/Hooks";
+import { OnCreate, OnDelete, OnUpdate } from "../Types/Database/Hooks";
 import RumApplicationLabelRuleEngineService from "./RumApplicationLabelRuleEngineService";
 import RumApplicationOwnerRuleEngineService from "./RumApplicationOwnerRuleEngineService";
 
@@ -19,6 +20,14 @@ const LAST_SEEN_THROTTLE_SECONDS: number = 60;
 const LABELS_APPLIED_CACHE_NAMESPACE: string = "rum-application-labels-applied";
 const LABELS_APPLIED_CACHE_TTL_SECONDS: number = 60;
 
+const REPLAY_CHUNK_RECEIVED_CACHE_NAMESPACE: string =
+  "rum-application-replay-chunk-received";
+const REPLAY_CHUNK_RECEIVED_THROTTLE_SECONDS: number = 60;
+
+const REPLAY_BUDGET_EXCEEDED_CACHE_NAMESPACE: string =
+  "rum-application-replay-budget-exceeded";
+const REPLAY_BUDGET_EXCEEDED_THROTTLE_SECONDS: number = 300;
+
 export class Service extends DatabaseService<Model> {
   public constructor() {
     super(Model);
@@ -54,6 +63,144 @@ export class Service extends DatabaseService<Model> {
     return createdItem;
   }
 
+  /*
+   * Session replay policy invalidation.
+   *
+   * The ingest gate caches the resolved policy for POLICY_CACHE_TTL_MS per
+   * pod, and the config endpoint is cached in the browser for 300s on top of
+   * that. Without these hooks, switching replay off for an application would
+   * keep being accepted for up to ~6 minutes across the fleet - so the kill
+   * key exists precisely to be written here.
+   *
+   * The kill key is per PROJECT, which is coarser than the per-application
+   * change that triggered it: disabling one application in a multi-app
+   * project briefly refuses the project's other applications too. That is
+   * accepted deliberately. The window is KILL_KEY_CACHE_TTL_MS plus the
+   * policy TTL, the failure mode is "a few recordings not taken", and the
+   * alternative - keying on projectId:appIdentifier - would leave the
+   * project-level `isSessionReplayAllowed` switch with no fast path at all.
+   */
+  @CaptureSpan()
+  protected override async onUpdateSuccess(
+    onUpdate: OnUpdate<Model>,
+    updatedItemIds: Array<ObjectID>,
+  ): Promise<OnUpdate<Model>> {
+    await this.invalidateSessionReplayPolicy({
+      updateData: onUpdate.updateBy.data as Record<string, unknown>,
+      itemIds: updatedItemIds,
+      tenantId: onUpdate.updateBy.props.tenantId,
+    });
+
+    return onUpdate;
+  }
+
+  @CaptureSpan()
+  protected override async onDeleteSuccess(
+    onDelete: OnDelete<Model>,
+    itemIdsBeforeDelete: Array<ObjectID>,
+  ): Promise<OnDelete<Model>> {
+    /*
+     * A deleted application can never be re-enabled, so this always takes
+     * the "turn it off now" path rather than only clearing the cache.
+     */
+    await this.invalidateSessionReplayPolicy({
+      updateData: { isSessionReplayEnabled: false },
+      itemIds: itemIdsBeforeDelete,
+      tenantId: onDelete.deleteBy.props.tenantId,
+    });
+
+    return onDelete;
+  }
+
+  private async invalidateSessionReplayPolicy(data: {
+    updateData: Record<string, unknown>;
+    itemIds: Array<ObjectID>;
+    tenantId: ObjectID | undefined;
+  }): Promise<void> {
+    /*
+     * Only replay-relevant edits invalidate. A lastSeenAt heartbeat or a
+     * label change must not write a kill key, and updateLastSeen bypasses
+     * hooks entirely for the same reason.
+     */
+    const touchesReplayPolicy: boolean = Object.keys(data.updateData).some(
+      (key: string): boolean => {
+        return (
+          key.startsWith("sessionReplay") || key === "isSessionReplayEnabled"
+        );
+      },
+    );
+
+    if (!touchesReplayPolicy) {
+      return;
+    }
+
+    const projectIds: Array<ObjectID> = await this.resolveProjectIds(data);
+
+    const isTurningOff: boolean =
+      data.updateData["isSessionReplayEnabled"] === false;
+
+    for (const projectId of projectIds) {
+      try {
+        SessionReplayGateCacheStore.clearCache(projectId);
+
+        if (isTurningOff) {
+          await SessionReplayGateCacheStore.markProjectDisabled(projectId);
+        } else {
+          /*
+           * Any other replay edit (including turning it back ON) must also
+           * drop a kill key left over from an earlier disable, or the
+           * project stays refused for the remainder of the key's TTL.
+           */
+          await SessionReplayGateCacheStore.clearProjectDisabled(projectId);
+        }
+      } catch (err) {
+        /*
+         * Best effort. The ordinary cache expiry still applies, so a Redis
+         * outage degrades the response time of a settings change rather
+         * than failing the settings change itself.
+         */
+        logger.warn(
+          `RumApplicationService: could not invalidate the session replay gate cache for project ${projectId.toString()}`,
+        );
+        logger.warn(err);
+      }
+    }
+  }
+
+  private async resolveProjectIds(data: {
+    itemIds: Array<ObjectID>;
+    tenantId: ObjectID | undefined;
+  }): Promise<Array<ObjectID>> {
+    if (data.tenantId) {
+      return [data.tenantId];
+    }
+
+    /*
+     * A root-props update carries no tenantId, so the project has to be read
+     * back off the rows that were touched.
+     */
+    const projectIds: Map<string, ObjectID> = new Map();
+
+    for (const id of data.itemIds) {
+      const app: Model | null = await this.findOneById({
+        id: id,
+        select: {
+          _id: true,
+          projectId: true,
+        },
+        props: {
+          isRoot: true,
+        },
+      });
+
+      if (app?.projectId) {
+        projectIds.set(app.projectId.toString(), app.projectId);
+      }
+    }
+
+    return Array.from(projectIds.values());
+  }
+
   @CaptureSpan()
   public async findOrCreateByAppIdentifier(data: {
     projectId: ObjectID;
@@ -196,6 +343,98 @@ export class Service extends DatabaseService<Model> {
     });
   }
 
+  /*
+   * Session replay ingest health markers.
+   *
+   * Both write columns whose names start with "sessionReplay", which the
+   * onUpdateSuccess hook treats as a policy edit (cache clear + kill-key
+   * churn) - so they MUST go through the hook-free single-statement path,
+   * exactly like updateLastSeen. They also skip the updatedAt refresh:
+   * configEpoch is derived from updatedAt, and a passive health timestamp
+   * broadcasting "configuration changed" to every live recorder once a
+   * minute would be self-inflicted config-fetch load.
+   *
+   * Throttled through GlobalCache so 20k chunks/min becomes at most one
+   * Postgres write per application per window across the whole fleet.
+   * Failures are swallowed after a warn: this bookkeeping must never make
+   * an ingest request fail.
+   */
+  @CaptureSpan()
+  public async markSessionReplayChunkReceived(
+    rumApplicationId: ObjectID,
+  ): Promise<void> {
+    await this.writeReplayHealthColumn({
+      rumApplicationId: rumApplicationId,
+      cacheNamespace: REPLAY_CHUNK_RECEIVED_CACHE_NAMESPACE,
+      throttleSeconds: REPLAY_CHUNK_RECEIVED_THROTTLE_SECONDS,
+      column: "sessionReplayLastChunkReceivedAt",
+    });
+  }
+
+  @CaptureSpan()
+  public async markSessionReplayBudgetExceeded(
+    rumApplicationId: ObjectID,
+  ): Promise<void> {
+    await this.writeReplayHealthColumn({
+      rumApplicationId: rumApplicationId,
+      cacheNamespace: REPLAY_BUDGET_EXCEEDED_CACHE_NAMESPACE,
+      throttleSeconds: REPLAY_BUDGET_EXCEEDED_THROTTLE_SECONDS,
+      column: "sessionReplayBudgetExceededAt",
+    });
+  }
+
+  private async writeReplayHealthColumn(data: {
+    rumApplicationId: ObjectID;
+    cacheNamespace: string;
+    throttleSeconds: number;
+    column:
+      | "sessionReplayLastChunkReceivedAt"
+      | "sessionReplayBudgetExceededAt";
+  }): Promise<void> {
+    const cacheKey: string = data.rumApplicationId.toString();
+
+    try {
+      const cached: string | null = await GlobalCache.getString(
+        data.cacheNamespace,
+        cacheKey,
+      );
+
+      if (cached) {
+        return; // written recently, across all pods
+      }
+    } catch {
+      /*
+       * Cache unavailable - fail open and write anyway, same stance as
+       * updateLastSeen: a stale health column is worse than an occasional
+       * extra single-row UPDATE.
+       */
+    }
+
+    try {
+      await GlobalCache.setString(data.cacheNamespace, cacheKey, "1", {
+        expiresInSeconds: data.throttleSeconds,
+      });
+    } catch {
+      // Best-effort throttle write; proceed with the DB update regardless.
+    }
+
+    try {
+      await this.updateColumnsByIdWithoutHooks({
+        id: data.rumApplicationId,
+        data: {
+          [data.column]: OneUptimeDate.getCurrentDate(),
+          // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        } as any,
+        skipUpdateDateColumn: true,
+      });
+    } catch (err) {
+      logger.warn(
+        `RumApplicationService: could not write ${data.column} for application ${cacheKey}`,
+      );
+      logger.warn(err);
+    }
+  }
+
   @CaptureSpan()
   public async attachLabels(data: {
     rumApplicationId: ObjectID;

package/Server/Services/RumSessionChunkService.ts

@@ -0,0 +1,11 @@
+import ClickhouseDatabase from "../Infrastructure/ClickhouseDatabase";
+import AnalyticsDatabaseService from "./AnalyticsDatabaseService";
+import RumSessionChunk from "../../Models/AnalyticsModels/RumSessionChunk";
+
+export class RumSessionChunkService extends AnalyticsDatabaseService<RumSessionChunk> {
+  public constructor(clickhouseDatabase?: ClickhouseDatabase | undefined) {
+    super({ modelType: RumSessionChunk, database: clickhouseDatabase });
+  }
+}
+
+export default new RumSessionChunkService();

package/Server/Services/RumSessionErasureRequestService.ts

@@ -0,0 +1,197 @@
+import DatabaseService from "./DatabaseService";
+import Model, {
+  RumSessionErasureRequestStatus,
+  RumSessionErasureRequestType,
+} from "../../Models/DatabaseModels/RumSessionErasureRequest";
+import BadDataException from "../../Types/Exception/BadDataException";
+import ObjectID from "../../Types/ObjectID";
+import OneUptimeDate from "../../Types/Date";
+import SortOrder from "../../Types/BaseDatabase/SortOrder";
+import CreateBy from "../Types/Database/CreateBy";
+import { OnCreate } from "../Types/Database/Hooks";
+import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
+
+/*
+ * Session replay erasure requests.
+ *
+ * The service owns the two invariants the model cannot express on its
+ * own: a request is always born Pending with zeroed counters no matter
+ * what the client sent, and a request must actually be actionable by the
+ * worker before it is accepted. A request that is accepted but can never
+ * be executed is worse than a rejected one - the requester believes their
+ * data is gone.
+ */
+export class Service extends DatabaseService<Model> {
+  public constructor() {
+    super(Model);
+  }
+
+  @CaptureSpan()
+  protected override async onBeforeCreate(
+    createBy: CreateBy<Model>,
+  ): Promise<OnCreate<Model>> {
+    const requestType: RumSessionErasureRequestType | undefined =
+      createBy.data.requestType;
+
+    if (!requestType) {
+      throw new BadDataException(
+        "requestType is required on a session replay erasure request.",
+      );
+    }
+
+    /*
+     * Validated here rather than only in the UI because the worker cannot
+     * do anything sensible with a request whose target is missing, and it
+     * would sit Pending forever looking like an erasure in progress.
+     */
+    if (requestType === RumSessionErasureRequestType.ByDateRange) {
+      if (!createBy.data.startDate || !createBy.data.endDate) {
+        throw new BadDataException(
+          "startDate and endDate are required for a ByDateRange erasure request.",
+        );
+      }
+
+      if (
+        OneUptimeDate.isAfter(createBy.data.startDate, createBy.data.endDate)
+      ) {
+        throw new BadDataException(
+          "startDate must not be after endDate on a ByDateRange erasure request.",
+        );
+      }
+    } else if (!createBy.data.targetValue) {
+      throw new BadDataException(
+        `targetValue is required for a ${requestType} erasure request.`,
+      );
+    }
+
+    /*
+     * A ByRumApplication request stores the application id in targetValue
+     * as well as in the relation, because rumApplicationId is nulled out
+     * (not cascaded) when the application is deleted - which is precisely
+     * when the erasure still needs to run.
+     */
+    if (
+      requestType === RumSessionErasureRequestType.ByRumApplication &&
+      createBy.data.rumApplicationId &&
+      !createBy.data.targetValue
+    ) {
+      createBy.data.targetValue = createBy.data.rumApplicationId.toString();
+    }
+
+    /*
+     * All five are overwritten unconditionally, never merged with what the
+     * client sent. Every one of them is declared `computed` on the model,
+     * which is what lets these assignments survive the create-column
+     * permission check that runs after this hook.
+     */
+    createBy.data.status = RumSessionErasureRequestStatus.Pending;
+    createBy.data.requestedAt = OneUptimeDate.getCurrentDate();
+    createBy.data.sessionsDeleted = 0;
+    createBy.data.chunksDeleted = 0;
+
+    if (createBy.props.userId) {
+      createBy.data.requestedByUserId = createBy.props.userId;
+    } else {
+      /*
+       * No authenticated user (an API key, say). Deleting rather than
+       * leaving the field alone matters: a client-supplied value would
+       * otherwise persist and attribute the erasure to somebody who never
+       * asked for it.
+       */
+      delete createBy.data.requestedByUserId;
+    }
+
+    return { createBy, carryForward: null };
+  }
+
+  /*
+   * Outstanding requests for the erasure worker, oldest first so a
+   * request cannot be starved by newer ones.
+   *
+   * sessionsDeleted / chunksDeleted are in the select list because a
+   * requeued request has to ACCUMULATE its counters across runs; omitting
+   * them is what made the worker stop using this helper and inline its own
+   * copy of the query.
+   */
+  @CaptureSpan()
+  public async getPendingRequests(data: {
+    limit: number;
+  }): Promise<Array<Model>> {
+    return await this.findBy({
+      query: {
+        status: RumSessionErasureRequestStatus.Pending,
+      },
+      select: {
+        _id: true,
+        projectId: true,
+        rumApplicationId: true,
+        requestType: true,
+        targetValue: true,
+        startDate: true,
+        endDate: true,
+        requestedAt: true,
+        sessionsDeleted: true,
+        chunksDeleted: true,
+      },
+      sort: {
+        requestedAt: SortOrder.Ascending,
+      },
+      skip: 0,
+      limit: data.limit,
+      props: { isRoot: true },
+    });
+  }
+
+  @CaptureSpan()
+  public async markInProgress(data: { requestId: ObjectID }): Promise<void> {
+    await this.updateOneById({
+      id: data.requestId,
+      data: {
+        status: RumSessionErasureRequestStatus.InProgress,
+      },
+      props: { isRoot: true },
+    });
+  }
+
+  @CaptureSpan()
+  public async markCompleted(data: {
+    requestId: ObjectID;
+    sessionsDeleted: number;
+    chunksDeleted: number;
+  }): Promise<void> {
+    await this.updateOneById({
+      id: data.requestId,
+      data: {
+        status: RumSessionErasureRequestStatus.Completed,
+        completedAt: OneUptimeDate.getCurrentDate(),
+        sessionsDeleted: data.sessionsDeleted,
+        chunksDeleted: data.chunksDeleted,
+      },
+      props: { isRoot: true },
+    });
+  }
+
+  /*
+   * completedAt is set on failure too. It means "the attempt finished",
+   * not "the data is gone" - status is what says whether it worked, and
+   * leaving completedAt null would make a failed request indistinguishable
+   * from one still running.
+   */
+  @CaptureSpan()
+  public async markFailed(data: {
+    requestId: ObjectID;
+    failureReason: string;
+  }): Promise<void> {
+    await this.updateOneById({
+      id: data.requestId,
+      data: {
+        status: RumSessionErasureRequestStatus.Failed,
+        completedAt: OneUptimeDate.getCurrentDate(),
+        failureReason: data.failureReason,
+      },
+      props: { isRoot: true },
+    });
+  }
+}
+
+export default new Service();

package/Server/Services/RumSessionPinService.ts

@@ -0,0 +1,213 @@
+import DatabaseService from "./DatabaseService";
+import Model from "../../Models/DatabaseModels/RumSessionPin";
+import BadDataException from "../../Types/Exception/BadDataException";
+import ObjectID from "../../Types/ObjectID";
+import OneUptimeDate from "../../Types/Date";
+import CreateBy from "../Types/Database/CreateBy";
+import { OnCreate } from "../Types/Database/Hooks";
+import QueryHelper from "../Types/Database/QueryHelper";
+import CaptureSpan from "../Utils/Telemetry/CaptureSpan";
+
+/* Postgres unique_violation. https://www.postgresql.org/docs/current/errcodes-appendix.html */
+const UNIQUE_VIOLATION: string = "23505";
+
+type IsUniqueViolationFunction = (error: unknown) => boolean;
+
+/*
+ * TypeORM wraps driver failures in QueryFailedError, which hoists the pg
+ * fields onto itself but also keeps the original under `driverError`.
+ * Check both rather than assuming which one carries the code.
+ */
+const isUniqueViolation: IsUniqueViolationFunction = (
+  error: unknown,
+): boolean => {
+  if (!error || typeof error !== "object") {
+    return false;
+  }
+
+  if ((error as { code?: string | undefined }).code === UNIQUE_VIOLATION) {
+    return true;
+  }
+
+  const driverError: unknown = (error as { driverError?: unknown }).driverError;
+
+  return Boolean(
+    driverError &&
+      typeof driverError === "object" &&
+      (driverError as { code?: string | undefined }).code === UNIQUE_VIOLATION,
+  );
+};
+
+/*
+ * Pins that keep a session recording past its retention window.
+ *
+ * The unique index on (projectId, rumApplicationId, sessionId) is the real
+ * guard against duplicate copies; this service turns the resulting
+ * constraint violation into an idempotent "already pinned" so that
+ * clicking Pin twice, or pinning the same recording from two incidents,
+ * is not an error the user has to interpret.
+ */
+export class Service extends DatabaseService<Model> {
+  public constructor() {
+    super(Model);
+  }
+
+  @CaptureSpan()
+  protected override async onBeforeCreate(
+    createBy: CreateBy<Model>,
+  ): Promise<OnCreate<Model>> {
+    if (!createBy.data.projectId) {
+      throw new BadDataException("projectId is required");
+    }
+
+    if (!createBy.data.rumApplicationId) {
+      throw new BadDataException("rumApplicationId is required");
+    }
+
+    if (!createBy.data.sessionId) {
+      throw new BadDataException("sessionId is required");
+    }
+
+    /*
+     * materializedAt is never client-supplied: it asserts that the
+     * recording has actually been copied out of the expiring table, and
+     * a client that could set it would make an unprotected recording look
+     * protected.
+     */
+    delete createBy.data.materializedAt;
+
+    /*
+     * Assigned or removed, never left as the client sent it. pinnedByUserId
+     * is declared `computed` on the model, which is what lets this
+     * assignment survive the create-column permission check that runs after
+     * this hook; the delete is what stops an API-key caller (no
+     * props.userId) attributing the pin to a colleague.
+     */
+    if (createBy.props.userId) {
+      createBy.data.pinnedByUserId = createBy.props.userId;
+    } else {
+      delete createBy.data.pinnedByUserId;
+    }
+
+    return { createBy, carryForward: null };
+  }
+
+  /*
+   * Pinning is idempotent, which is the whole reason this override exists.
+   * Clicking Pin twice, or pinning the same recording from two incidents,
+   * hits the unique index on (projectId, rumApplicationId, sessionId) and
+   * would otherwise surface as a raw 500 carrying a Postgres constraint
+   * name: checkForUniqueValues only understands single-column `unique`
+   * metadata, not a composite @Index, so nothing else catches it.
+   *
+   * The pre-check handles the common case and the catch closes the race
+   * between two concurrent pins of the same session.
+   */
+  @CaptureSpan()
+  public override async create(createBy: CreateBy<Model>): Promise<Model> {
+    const projectId: ObjectID | undefined = createBy.data.projectId;
+    const rumApplicationId: ObjectID | undefined =
+      createBy.data.rumApplicationId;
+    const sessionId: string | undefined = createBy.data.sessionId;
+
+    if (!projectId || !rumApplicationId || !sessionId) {
+      /* Let onBeforeCreate produce the specific validation message. */
+      return await super.create(createBy);
+    }
+
+    const existingPin: Model | null = await this.getPinForSession({
+      projectId: projectId,
+      rumApplicationId: rumApplicationId,
+      sessionId: sessionId,
+    });
+
+    if (existingPin) {
+      return existingPin;
+    }
+
+    try {
+      return await super.create(createBy);
+    } catch (err) {
+      if (!isUniqueViolation(err)) {
+        throw err;
+      }
+
+      const racedPin: Model | null = await this.getPinForSession({
+        projectId: projectId,
+        rumApplicationId: rumApplicationId,
+        sessionId: sessionId,
+      });
+
+      if (racedPin) {
+        return racedPin;
+      }
+
+      throw err;
+    }
+  }
+
+  /* Null when the recording is not pinned. */
+  @CaptureSpan()
+  public async getPinForSession(data: {
+    projectId: ObjectID;
+    rumApplicationId: ObjectID;
+    sessionId: string;
+  }): Promise<Model | null> {
+    return await this.findOneBy({
+      query: {
+        projectId: data.projectId,
+        rumApplicationId: data.rumApplicationId,
+        sessionId: data.sessionId,
+      },
+      select: {
+        _id: true,
+        reason: true,
+        incidentId: true,
+        alertId: true,
+        expiresAt: true,
+        materializedAt: true,
+        pinnedByUserId: true,
+      },
+      props: { isRoot: true },
+    });
+  }
+
+  /*
+   * Pins whose copy has not happened yet. This is the worker's queue, and
+   * it is also the honest answer to "is this recording safe?" - a pin
+   * without a materializedAt protects nothing.
+   */
+  @CaptureSpan()
+  public async getUnmaterializedPins(data: {
+    limit: number;
+  }): Promise<Array<Model>> {
+    return await this.findBy({
+      query: {
+        materializedAt: QueryHelper.isNull(),
+      },
+      select: {
+        _id: true,
+        projectId: true,
+        rumApplicationId: true,
+        sessionId: true,
+        expiresAt: true,
+      },
+      skip: 0,
+      limit: data.limit,
+      props: { isRoot: true },
+    });
+  }
+
+  @CaptureSpan()
+  public async markMaterialized(data: { pinId: ObjectID }): Promise<void> {
+    await this.updateOneById({
+      id: data.pinId,
+      data: {
+        materializedAt: OneUptimeDate.getCurrentDate(),
+      },
+      props: { isRoot: true },
+    });
+  }
+}
+
+export default new Service();