@oneuptime/common 11.7.3 → 12.0.0

package/Server/Utils/SessionReplay/SessionReplayRetention.ts

@@ -0,0 +1,114 @@
+import OneUptimeDate from "../../../Types/Date";
+import {
+  DEFAULT_SESSION_REPLAY_RETENTION_IN_DAYS,
+  SESSION_REPLAY_ALLOWED_RETENTION_DAYS,
+} from "../../../Types/Rum/SessionReplay";
+import TelemetryRetentionConfig from "../../../Types/Telemetry/TelemetryRetentionConfig";
+import SessionIdentity from "../../../Utils/Rum/SessionIdentity";
+
+/*
+ * Retention resolution for session replay.
+ *
+ * Deliberately NOT resolveTelemetryRetentionInDays. Its candidate order is
+ *
+ *   service[pillar].default -> service.retainTelemetryDataForDays
+ *     -> project[pillar].default -> project.defaultTelemetryRetentionInDays
+ *     -> 15
+ *
+ * and service.retainTelemetryDataForDays is RumApplication's single
+ * telemetry retention knob, sitting ABOVE the project pillar default. An
+ * application set to keep traces for 90 days would therefore silently keep
+ * recordings of real end users' screens for 90 days too, and an
+ * application that had never been configured at all would get 15 days
+ * rather than replay's 7. Neither is something anybody asked for.
+ *
+ * So replay reads only its own pillar, falls back to 7, and then clamps.
+ */
+
+export interface SessionReplayRetentionInput {
+  /*
+   * RumApplication.sessionReplayRetentionInDays - the explicit per-app
+   * setting, and the highest-priority candidate.
+   */
+  applicationRetentionInDays?: number | null | undefined;
+
+  /* RumApplication.telemetryRetentionConfig */
+  applicationConfig?: TelemetryRetentionConfig | null | undefined;
+
+  /* Project.telemetryRetentionConfig */
+  projectConfig?: TelemetryRetentionConfig | null | undefined;
+}
+
+function pickPositive(value: number | null | undefined): number | null {
+  return typeof value === "number" && value > 0 ? value : null;
+}
+
+/*
+ * The clamp is load-bearing twice over, which is why it is not optional
+ * and not configurable:
+ *
+ *  - Replay chunks are partitioned by expiry date, so every distinct
+ *    retention value in a project creates its own partition per ingest
+ *    day. A free integer would mean unbounded partitions on the fattest
+ *    table in the system.
+ *  - It bounds the blast radius of a mis-set value: a typo of 900 becomes
+ *    90, not two and a half years of end-user screen recordings.
+ *
+ * SessionIdentity.clampRetentionDays rounds UP to the next allowed value,
+ * because silently shortening a customer's configured retention destroys
+ * evidence they expected to have, whereas keeping it a little longer costs
+ * only disk.
+ */
+export function resolveSessionReplayRetentionInDays(
+  input: SessionReplayRetentionInput,
+): number {
+  const { applicationRetentionInDays, applicationConfig, projectConfig } =
+    input;
+
+  const candidates: Array<number | null> = [
+    pickPositive(applicationRetentionInDays),
+    pickPositive(applicationConfig?.sessionReplay?.default),
+    pickPositive(projectConfig?.sessionReplay?.default),
+  ];
+
+  let requestedDays: number = DEFAULT_SESSION_REPLAY_RETENTION_IN_DAYS;
+
+  for (const candidate of candidates) {
+    if (candidate !== null) {
+      requestedDays = candidate;
+      break;
+    }
+  }
+
+  return SessionIdentity.clampRetentionDays(
+    requestedDays,
+    SESSION_REPLAY_ALLOWED_RETENTION_DAYS,
+    DEFAULT_SESSION_REPLAY_RETENTION_IN_DAYS,
+  );
+}
+
+/*
+ * The retentionDate written on both the session header and every chunk of
+ * that session.
+ *
+ * Derived from the SERVER-CLAMPED session start, not from the ingest date
+ * the way every other pillar does it. A chunk buffered offline and flushed
+ * hours later would otherwise get full retention from the moment it
+ * arrived, so one session's chunks would expire on different days, TTL
+ * would drop them mid-session, and what remained would be an unplayable
+ * fragment. Session-start derivation makes a session expire atomically.
+ *
+ * It also keeps the chunk retentionDate equal to the header's, which
+ * matters because reads have `AND retentionDate >= now()` appended
+ * silently - a mismatch produces sessions that are listable but
+ * unplayable, or playable but invisible.
+ */
+export function getSessionReplayRetentionDate(data: {
+  serverClampedSessionStart: Date;
+  retentionInDays: number;
+}): Date {
+  return OneUptimeDate.addRemoveDays(
+    data.serverClampedSessionStart,
+    data.retentionInDays,
+  );
+}

package/Server/Utils/SessionReplay/SessionReplayTargeting.ts

@@ -0,0 +1,144 @@
+import crypto from "crypto";
+import { EncryptionSecret } from "../../EnvironmentConfig";
+import Redis, { ClientType } from "../../Infrastructure/Redis";
+import logger from "../Logger";
+import ObjectID from "../../../Types/ObjectID";
+import {
+  SESSION_REPLAY_MAX_USER_REF_LENGTH,
+  SESSION_REPLAY_TARGET_TTL_SECONDS,
+} from "../../../Types/Rum/SessionReplay";
+
+/*
+ * "Record the next session for user X."
+ *
+ * A dashboard user names an end-user reference; the next recorder boot that
+ * identifies as that user gets isTargeted on its config response and
+ * records from its first event. The whole mechanism is one Redis key:
+ *
+ *   session-replay:target:<projectId>:<HMAC(userRef)>  ->  "1", 24h TTL
+ *
+ * The member is an HMAC (keyed on the server's EncryptionSecret, scoped by
+ * project + application) rather than the raw reference, because end-user
+ * references are routinely emails: a Redis KEYS listing on a shared
+ * cluster must not be a directory of the customer's users. The HMAC also
+ * makes key length independent of reference length.
+ *
+ * consume() is a plain DEL used as an atomic take: 1 deleted means this
+ * config fetch owns the target, so two tabs racing produce exactly one
+ * targeted session. And every read path here FAILS TO "not targeted" -
+ * targeting is a diagnostic convenience, and a Redis blip must degrade to
+ * "the session was not specially recorded", never to a config error that
+ * stops normal recording.
+ */
+
+const TARGET_KEY_PREFIX: string = "session-replay:target:";
+
+export interface SessionReplayTargetRef {
+  projectId: ObjectID;
+  appIdentifier: string;
+  userRef: string;
+}
+
+export default class SessionReplayTargeting {
+  /*
+   * A reference is usable when non-empty and within the shared length
+   * cap. The cap is enforced on BOTH the dashboard write and the config
+   * read, so an over-long ref simply can never match rather than being
+   * truncated differently on each side.
+   */
+  public static isUsableUserRef(userRef: unknown): userRef is string {
+    return (
+      typeof userRef === "string" &&
+      userRef.trim().length > 0 &&
+      userRef.length <= SESSION_REPLAY_MAX_USER_REF_LENGTH
+    );
+  }
+
+  public static buildTargetKey(data: SessionReplayTargetRef): string {
+    const digest: string = crypto
+      .createHmac("sha256", EncryptionSecret.toString())
+      .update(
+        `${data.projectId.toString()}:${data.appIdentifier
+          .trim()
+          .toLowerCase()}:${data.userRef.trim()}`,
+      )
+      .digest("hex");
+
+    return `${TARGET_KEY_PREFIX}${data.projectId.toString()}:${digest}`;
+  }
+
+  /* Dashboard write path. Throws so the caller can answer 5xx honestly. */
+  public static async setTarget(data: SessionReplayTargetRef): Promise<void> {
+    const client: ClientType = SessionReplayTargeting.requireClient();
+
+    await client.set(
+      SessionReplayTargeting.buildTargetKey(data),
+      "1",
+      "EX",
+      SESSION_REPLAY_TARGET_TTL_SECONDS,
+    );
+  }
+
+  /* Dashboard cancel path. Throws for the same reason setTarget does. */
+  public static async clearTarget(data: SessionReplayTargetRef): Promise<void> {
+    const client: ClientType = SessionReplayTargeting.requireClient();
+
+    await client.del(SessionReplayTargeting.buildTargetKey(data));
+  }
+
+  /* Dashboard status read. Throws; the panel shows an error, not a lie. */
+  public static async isTargetPending(
+    data: SessionReplayTargetRef,
+  ): Promise<boolean> {
+    const client: ClientType = SessionReplayTargeting.requireClient();
+
+    const found: number = await client.exists(
+      SessionReplayTargeting.buildTargetKey(data),
+    );
+
+    return found === 1;
+  }
+
+  /*
+   * The config endpoint's atomic take. Returns true exactly once per
+   * target - DEL's return value is the arbiter - and NEVER throws: this
+   * sits on the hot config path, where a Redis error must cost us the
+   * targeting nicety and nothing else.
+   */
+  public static async consumeTarget(
+    data: SessionReplayTargetRef,
+  ): Promise<boolean> {
+    if (!SessionReplayTargeting.isUsableUserRef(data.userRef)) {
+      return false;
+    }
+
+    try {
+      const client: ClientType | null = Redis.getClient();
+
+      if (!client || !Redis.isConnected()) {
+        return false;
+      }
+
+      const deleted: number = await client.del(
+        SessionReplayTargeting.buildTargetKey(data),
+      );
+
+      return deleted === 1;
+    } catch (err) {
+      logger.error("SessionReplayTargeting: consumeTarget failed:");
+      logger.error(err);
+
+      return false;
+    }
+  }
+
+  private static requireClient(): ClientType {
+    const client: ClientType | null = Redis.getClient();
+
+    if (!client || !Redis.isConnected()) {
+      throw new Error("Redis is not connected.");
+    }
+
+    return client;
+  }
+}

package/Server/Utils/SessionReplay/SessionReplayUsage.ts

@@ -0,0 +1,92 @@
+import Redis, { ClientType } from "../../Infrastructure/Redis";
+import logger from "../Logger";
+import ObjectID from "../../../Types/ObjectID";
+
+/*
+ * Session replay byte-budget bookkeeping shared between the two sides that
+ * need it: the App-tier ingest gate CONSUMES these counters (INCRBY, in
+ * App/FeatureSet/Telemetry/Utils/SessionReplayRateLimiter.ts) and the
+ * Dashboard-facing TelemetryAPI READS them so a customer can see "you have
+ * used 800MB of today's budget" instead of inferring exhaustion from
+ * recordings that silently stop appearing.
+ *
+ * The key builders live here, in Common, precisely so those two callers
+ * cannot drift: the ingest gate charging one key while the dashboard reads
+ * another would report a healthy budget forever.
+ */
+
+const DAILY_PROJECT_BYTE_KEY_PREFIX: string = "replay:rate:bytes:";
+const MONTHLY_APP_BYTE_KEY_PREFIX: string = "replay:rate:bytes-month:";
+
+export default class SessionReplayUsage {
+  /*
+   * UTC rather than local, so the budget window is the same for every pod
+   * regardless of container timezone.
+   */
+  public static getUtcDayBucket(): string {
+    return new Date().toISOString().substring(0, 10);
+  }
+
+  /* "YYYY-MM", UTC, for the per-application monthly budget window. */
+  public static getUtcMonthBucket(): string {
+    return new Date().toISOString().substring(0, 7);
+  }
+
+  public static getDailyProjectByteKey(projectId: ObjectID): string {
+    return `${DAILY_PROJECT_BYTE_KEY_PREFIX}${projectId.toString()}:${this.getUtcDayBucket()}`;
+  }
+
+  public static getMonthlyApplicationByteKey(data: {
+    projectId: ObjectID;
+    rumApplicationId: ObjectID;
+  }): string {
+    return `${MONTHLY_APP_BYTE_KEY_PREFIX}${data.projectId.toString()}:${data.rumApplicationId.toString()}:${this.getUtcMonthBucket()}`;
+  }
+
+  /*
+   * Bytes consumed from the project's daily budget so far. Read-only.
+   *
+   * null means "unknown" (Redis unavailable), which callers must render as
+   * unknown rather than as zero: a dashboard telling a customer their usage
+   * is 0 while the gate is refusing chunks would be worse than no number.
+   */
+  public static async getProjectBytesUsedToday(
+    projectId: ObjectID,
+  ): Promise<number | null> {
+    return this.readCounter(this.getDailyProjectByteKey(projectId));
+  }
+
+  /* Bytes consumed from the application's monthly budget. Read-only. */
+  public static async getApplicationBytesUsedThisMonth(data: {
+    projectId: ObjectID;
+    rumApplicationId: ObjectID;
+  }): Promise<number | null> {
+    return this.readCounter(this.getMonthlyApplicationByteKey(data));
+  }
+
+  private static async readCounter(key: string): Promise<number | null> {
+    const client: ClientType | null = Redis.getClient();
+
+    if (!client || !Redis.isConnected()) {
+      return null;
+    }
+
+    try {
+      const value: string | null = await client.get(key);
+
+      if (value === null) {
+        return 0;
+      }
+
+      const parsed: number = parseInt(value, 10);
+
+      return isNaN(parsed) ? 0 : parsed;
+    } catch (err) {
+      logger.warn(
+        `SessionReplayUsage: could not read the byte counter at ${key}`,
+      );
+      logger.warn(err);
+      return null;
+    }
+  }
+}

package/Server/Utils/StartServer.ts

@@ -8,6 +8,10 @@ import {
 } from "../EnvironmentConfig";
 import LocalCache from "../Infrastructure/LocalCache";
 import HttpMetricsMiddleware from "../Middleware/HttpMetricsMiddleware";
+import CorsOptions, {
+  CORS_EXPOSED_HEADERS,
+  CORS_PREFLIGHT_MAX_AGE_SECONDS,
+} from "./CorsOptions";
 import "./Environment";
 import Express, {
   ExpressApplication,
@@ -99,15 +103,33 @@ const setDefaultHeaders: RequestHandler = (
   res.header("Access-Control-Allow-Credentials", "true");
   res.header("Access-Control-Allow-Origin", "*");
   res.header("Access-Control-Allow-Methods", "GET,PUT,POST,DELETE,OPTIONS");
+  /*
+   * x-oneuptime-token and x-oneuptime-app-identifier are listed explicitly
+   * because the session-replay recorder runs on a customer's own origin and
+   * so its POSTs are genuinely cross-origin and preflighted. Browser ingest
+   * has worked so far only because `app.use(cors())` runs before this
+   * handler; relying on that ordering for a required custom header would be
+   * a silent, hard-to-diagnose failure the first time it changed.
+   */
   res.header(
     "Access-Control-Allow-Headers",
-    "X-Requested-With, X-HTTP-Method-Override, Content-Type, Accept, Authorization, DNT, X-CustomHeader, Keep-Alive, User-Agent, If-Modified-Since, Cache-Control, Content-Type",
+    "X-Requested-With, X-HTTP-Method-Override, Content-Type, Accept, Authorization, DNT, X-CustomHeader, Keep-Alive, User-Agent, If-Modified-Since, Cache-Control, Content-Type, x-oneuptime-token, x-oneuptime-app-identifier",
   );
 
+  /*
+   * Repeated on the simple (non-preflight) response as well. The value that
+   * actually governs preflight caching is the one configured on the cors
+   * middleware below - this handler never runs for an OPTIONS request,
+   * because cors answers preflights itself and calls res.end().
+   */
+  res.header("Access-Control-Max-Age", String(CORS_PREFLIGHT_MAX_AGE_SECONDS));
+
+  res.header("Access-Control-Expose-Headers", CORS_EXPOSED_HEADERS.join(", "));
+
   next();
 };
 
-app.use(cors());
+app.use(cors(CorsOptions));
 app.use(HttpMetricsMiddleware);
 app.use(setDefaultHeaders);
 
@@ -140,7 +162,20 @@ const protobufBodyParserMiddleware: RequestHandler = ExpressRaw({
 });
 
 app.use((req: OneUptimeRequest, res: ExpressResponse, next: NextFunction) => {
-  if (req.path.includes("/otlp/v1/")) {
+  /*
+   * `includes`, not `startsWith`. Both of these routers are mounted on
+   * several prefixes, so /telemetry/otlp/v1/... and
+   * /telemetry/session-replay/v1/... are equally live. A prefix-anchored
+   * test would let the prefixed path fall into the gzip fast-path below,
+   * which has NO size limit and no content-length pre-check (a zip-bomb
+   * vector on a public browser endpoint) and which sets req.body to the
+   * DECOMPRESSED buffer — that in turn trips the ingest middleware's
+   * "already parsed" early-out, so its byte cap would never run at all.
+   */
+  if (
+    req.path.includes("/otlp/v1/") ||
+    req.path.includes("/session-replay/v1/")
+  ) {
     return next();
   }
 
@@ -190,8 +225,17 @@ app.use((req: OneUptimeRequest, res: ExpressResponse, next: NextFunction) => {
 });
 
 app.use((req: ExpressRequest, res: ExpressResponse, next: NextFunction) => {
+  /*
+   * The urlencoded twin of the bypass above. It must carry the same
+   * session-replay exemption: a chunk POST carries a
+   * vnd.oneuptime.session-replay content type but the recorder's identity
+   * fallback path sends no Content-Encoding, so without this predicate the
+   * urlencoded parser would consume the stream before the replay body
+   * reader ever saw it.
+   */
   if (
     req.path.includes("/otlp/v1/") ||
+    req.path.includes("/session-replay/v1/") ||
     headerValueToString(req.headers["content-encoding"])?.includes("gzip")
   ) {
     next();

package/Server/Utils/Telemetry/AppMetrics.ts

@@ -208,4 +208,35 @@ export default class AppMetrics {
 
     return this.ingestPayloadBytes;
   }
+
+  // -- Telemetry discarded by project drop filters -----------------------
+
+  private static ingestDroppedCounter: TelemetryCounter | null = null;
+
+  /*
+   * Records telemetry a project's own drop filters discarded at ingest.
+   *
+   * Before this existed, a drop filter deleted records with no trace
+   * anywhere — no log line, no metric, no counter — so "my logs are
+   * missing" tickets could not be distinguished from a broken pipeline
+   * without reading the customer's filter rows by hand.
+   *
+   * Attributes carry `oneuptime.project.id` and `oneuptime.drop_filter.id`
+   * on purpose: without them the counter says only "something dropped
+   * telemetry", which is exactly the unanswerable state this replaces.
+   * Cardinality stays bounded — projects times a handful of filters each,
+   * and only projects that configured a filter ever emit a point.
+   */
+  public static getIngestDroppedCounter(): TelemetryCounter {
+    if (!this.ingestDroppedCounter) {
+      this.ingestDroppedCounter = Telemetry.getCounter({
+        name: "oneuptime.telemetry.ingest.dropped.count",
+        description:
+          "Number of telemetry records discarded by a project's drop filters, partitioned by signal, action, project and filter.",
+        unit: "1",
+      });
+    }
+
+    return this.ingestDroppedCounter;
+  }
 }

package/Server/Utils/Telemetry/TelemetryFanInWriter.ts

@@ -111,6 +111,16 @@ export type FanInInsertTransport = (
 export interface FanInWriterOptions {
   /** Target rows per ClickHouse insert. Buffers flush as soon as they reach this. */
   maxBatchRows: number;
+  /**
+   * Per-table override of maxBatchRows, keyed on the ClickHouse table name.
+   *
+   * The batcher counts ROWS, not bytes, so one global number cannot be right
+   * for tables whose rows differ by three orders of magnitude. A session
+   * replay chunk row carries the whole decompressed rrweb payload in a single
+   * column (100 KB+ is ordinary), so the 100,000 default would attempt a
+   * multi-hundred-MB insert.
+   */
+  maxBatchRowsByTable?: Record<string, number> | undefined;
   /** Max time rows sit buffered before a flush is forced, ms. */
   maxWaitMs: number;
   /** Per-pod cap on concurrent ClickHouse inserts across all tables. */
@@ -275,6 +285,18 @@ const readPositiveInt: EnvIntReader = (
 export function readFanInWriterOptionsFromEnv(): FanInWriterOptions {
   return {
     maxBatchRows: readPositiveInt("TELEMETRY_FANIN_MAX_BATCH_ROWS", 100_000),
+    /*
+     * Session replay is the one signal whose row size makes the global row
+     * count dangerous. At the ingest gate's own permitted rate a 5s maxWaitMs
+     * window buffers well over a thousand 100 KB+ rows, so the default here
+     * is deliberately two orders of magnitude below the global one.
+     */
+    maxBatchRowsByTable: {
+      RumSessionChunkV1: readPositiveInt(
+        "TELEMETRY_FANIN_MAX_BATCH_ROWS_SESSION_REPLAY",
+        2000,
+      ),
+    },
     maxWaitMs: readPositiveInt("TELEMETRY_FANIN_MAX_WAIT_MS", 5000),
     maxConcurrentInserts: readPositiveInt(
       "TELEMETRY_FANIN_MAX_CONCURRENT_INSERTS",
@@ -438,7 +460,7 @@ export class TelemetryFanInWriter {
       buffer.rowCount += rows.length;
       this.pendingRows += rows.length;
 
-      if (buffer.rowCount >= this.options.maxBatchRows) {
+      if (buffer.rowCount >= this.getMaxBatchRows(tableName)) {
         this.cutAndDispatch(tableName, false);
       } else if (!buffer.timer) {
         buffer.timer = setTimeout(() => {
@@ -557,31 +579,43 @@ export class TelemetryFanInWriter {
    * (a single oversized submission forms its own batch), so an insert failure
    * maps 1:1 onto the submissions it contained.
    */
+  /*
+   * Batch size for one table: its override if it has one, the global
+   * maxBatchRows otherwise. A non-positive override is ignored rather than
+   * honoured, because a zero would make cutAndDispatch spin.
+   */
+  private getMaxBatchRows(tableName: string): number {
+    const override: number | undefined =
+      this.options.maxBatchRowsByTable?.[tableName];
+
+    if (typeof override === "number" && override > 0) {
+      return override;
+    }
+
+    return this.options.maxBatchRows;
+  }
+
   private cutAndDispatch(tableName: string, drain: boolean): void {
     const buffer: TableBuffer | undefined = this.buffers.get(tableName);
     if (!buffer) {
       return;
     }
 
-    while (
-      buffer.rowCount >= this.options.maxBatchRows ||
-      (drain && buffer.rowCount > 0)
-    ) {
+    const maxBatchRows: number = this.getMaxBatchRows(tableName);
+
+    while (buffer.rowCount >= maxBatchRows || (drain && buffer.rowCount > 0)) {
       const batch: Array<PendingSubmission> = [];
       let batchRows: number = 0;
 
       while (buffer.submissions.length > 0) {
         const next: PendingSubmission = buffer.submissions[0]!;
-        if (
-          batch.length > 0 &&
-          batchRows + next.rows.length > this.options.maxBatchRows
-        ) {
+        if (batch.length > 0 && batchRows + next.rows.length > maxBatchRows) {
           break;
         }
         buffer.submissions.shift();
         batch.push(next);
         batchRows += next.rows.length;
-        if (batchRows >= this.options.maxBatchRows) {
+        if (batchRows >= maxBatchRows) {
           break;
         }
       }