@oneuptime/common 11.7.3 → 12.0.0

package/Utils/Rum/ChunkMath.ts

@@ -0,0 +1,211 @@
+import {
+  SessionReplayChunkManifestEntry,
+  SessionReplayGap,
+} from "../../Types/Rum/SessionReplay";
+
+/*
+ * Pure chunk-sequence arithmetic shared by the ingest finalizer and the
+ * Dashboard player: seek-anchor selection, gap detection, and paging
+ * windows.
+ *
+ * Kept free of any I/O so it can be exhaustively unit-tested, which
+ * matters because the failure mode of getting this wrong is not a crash
+ * — it is a player that renders a plausible DOM the user never saw.
+ */
+
+export default class ChunkMath {
+  /*
+   * The greatest full-snapshot chunk index at or before the target.
+   *
+   * Seeking cannot simply jump to the target chunk: rrweb resolves
+   * mutations against node ids established by a prior snapshot, so
+   * playback must restart from a snapshot boundary. Returns null when no
+   * anchor exists at or before the target, which the caller must treat as
+   * "cannot seek there" rather than "start from zero".
+   *
+   * anchors need not be sorted; they are copied and sorted here so
+   * callers can pass a raw groupArray() result straight from ClickHouse.
+   */
+  public static findSeekAnchor(
+    anchors: Array<number>,
+    targetChunkIndex: number,
+  ): number | null {
+    if (anchors.length === 0) {
+      return null;
+    }
+
+    const sorted: Array<number> = [...anchors].sort(
+      (a: number, b: number): number => {
+        return a - b;
+      },
+    );
+
+    let low: number = 0;
+    let high: number = sorted.length - 1;
+    let best: number | null = null;
+
+    while (low <= high) {
+      const mid: number = Math.floor((low + high) / 2);
+      const candidate: number | undefined = sorted[mid];
+
+      if (candidate === undefined) {
+        break;
+      }
+
+      if (candidate <= targetChunkIndex) {
+        best = candidate;
+        low = mid + 1;
+      } else {
+        high = mid - 1;
+      }
+    }
+
+    return best;
+  }
+
+  /*
+   * Holes in the chunk sequence for a single tab.
+   *
+   * A gap means chunks were lost — the browser was closed mid-flush, a
+   * request 413'd, the byte budget cut in. Playback must jump forward to
+   * the next full snapshot rather than applying mutations across the
+   * hole, and the scrubber must draw the gap. Silently interpolating is
+   * the failure that destroys trust in the whole feature.
+   *
+   * missingMs is measured from the end of the chunk before the gap to the
+   * start of the chunk after it, so it reflects wall-clock time the
+   * viewer will not see.
+   */
+  public static detectGaps(
+    entries: Array<SessionReplayChunkManifestEntry>,
+  ): Array<SessionReplayGap> {
+    if (entries.length < 2) {
+      return [];
+    }
+
+    const sorted: Array<SessionReplayChunkManifestEntry> = [...entries].sort(
+      (
+        a: SessionReplayChunkManifestEntry,
+        b: SessionReplayChunkManifestEntry,
+      ): number => {
+        return a.chunkIndex - b.chunkIndex;
+      },
+    );
+
+    const gaps: Array<SessionReplayGap> = [];
+
+    for (let i: number = 1; i < sorted.length; i++) {
+      const previous: SessionReplayChunkManifestEntry | undefined =
+        sorted[i - 1];
+      const current: SessionReplayChunkManifestEntry | undefined = sorted[i];
+
+      if (!previous || !current) {
+        continue;
+      }
+
+      if (current.chunkIndex === previous.chunkIndex + 1) {
+        continue;
+      }
+
+      gaps.push({
+        fromIndex: previous.chunkIndex,
+        toIndex: current.chunkIndex,
+        missingMs: Math.max(
+          0,
+          current.chunkStartOffsetMs - previous.chunkEndOffsetMs,
+        ),
+      });
+    }
+
+    return gaps;
+  }
+
+  /*
+   * Chunk indexes missing from a contiguous 0..maxChunkIndex run.
+   *
+   * Used by the finalizer to record missingChunkCount without holding the
+   * whole index list. Derived from the set difference rather than from a
+   * counter, so a duplicate delivery cannot make it drift.
+   */
+  public static findMissingChunkIndexes(
+    presentIndexes: Array<number>,
+    maxChunkIndex: number,
+  ): Array<number> {
+    const present: Set<number> = new Set(presentIndexes);
+    const missing: Array<number> = [];
+
+    for (let i: number = 0; i <= maxChunkIndex; i++) {
+      if (!present.has(i)) {
+        missing.push(i);
+      }
+    }
+
+    return missing;
+  }
+
+  /*
+   * The next window of chunk indexes to fetch, bounded by both a count
+   * cap and the total byte cap.
+   *
+   * Both caps are needed: a page of 8 tiny chunks is cheap, but 8 chunks
+   * each holding a large full snapshot would be tens of megabytes in one
+   * response. Always returns at least one index when any candidate
+   * remains, so an oversized single chunk still loads rather than
+   * deadlocking the player at a zero-length page.
+   */
+  public static planChunkPage(
+    entries: Array<SessionReplayChunkManifestEntry>,
+    fromChunkIndex: number,
+    maxChunks: number,
+    maxBytes: number,
+  ): Array<number> {
+    const candidates: Array<SessionReplayChunkManifestEntry> = entries
+      .filter((entry: SessionReplayChunkManifestEntry): boolean => {
+        return entry.chunkIndex >= fromChunkIndex;
+      })
+      .sort(
+        (
+          a: SessionReplayChunkManifestEntry,
+          b: SessionReplayChunkManifestEntry,
+        ): number => {
+          return a.chunkIndex - b.chunkIndex;
+        },
+      );
+
+    const page: Array<number> = [];
+    let bytes: number = 0;
+
+    for (const entry of candidates) {
+      if (page.length >= maxChunks) {
+        break;
+      }
+
+      if (page.length > 0 && bytes + entry.payloadBytes > maxBytes) {
+        break;
+      }
+
+      page.push(entry.chunkIndex);
+      bytes += entry.payloadBytes;
+    }
+
+    return page;
+  }
+
+  /*
+   * Total duration a session actually covers, excluding gaps. Used to
+   * label the player honestly when chunks were lost: a 5-minute session
+   * missing 90 seconds should not present itself as 5 minutes of footage.
+   */
+  public static getCoveredDurationMs(
+    entries: Array<SessionReplayChunkManifestEntry>,
+  ): number {
+    return entries.reduce(
+      (total: number, entry: SessionReplayChunkManifestEntry): number => {
+        return (
+          total + Math.max(0, entry.chunkEndOffsetMs - entry.chunkStartOffsetMs)
+        );
+      },
+      0,
+    );
+  }
+}

package/Utils/Rum/Masking.ts

@@ -0,0 +1,168 @@
+/*
+ * Text and input masking for session replay.
+ *
+ * Applied in the browser, before an event is buffered — the server never
+ * receives the unmasked value, so nothing here can be repaired after the
+ * fact.
+ *
+ * The central correctness point: rrweb's built-in mask replaces a value
+ * with `'*'.repeat(value.length)`, which preserves the exact character
+ * count. For a password, a one-time code, or a card number that is a
+ * length oracle — it narrows a password search space, and it identifies a
+ * 16-digit PAN as a PAN. So this module never emits length-preserving
+ * output for input values.
+ *
+ * Must stay dependency-free — this is bundled into the browser recorder.
+ */
+
+const MASK_CHARACTER: string = "•";
+
+/*
+ * Input values collapse to a constant-width block regardless of input
+ * length. Three characters is enough to render as "something was typed
+ * here" without hinting at how much.
+ */
+const FIXED_INPUT_MASK_LENGTH: number = 3;
+
+/*
+ * Static text nodes get bucketed rather than fixed width, because a
+ * paragraph collapsed to three characters destroys the layout that makes
+ * a wireframe replay readable at all. The bucket leaks only which of
+ * three coarse size classes the text fell into.
+ */
+const TEXT_BUCKET_SHORT_MAX: number = 8;
+const TEXT_BUCKET_MEDIUM_MAX: number = 32;
+
+const TEXT_BUCKET_SHORT_LENGTH: number = 4;
+const TEXT_BUCKET_MEDIUM_LENGTH: number = 16;
+const TEXT_BUCKET_LONG_LENGTH: number = 40;
+
+/*
+ * autocomplete tokens that mark a field as high-sensitivity regardless of
+ * its input type. A "show password" toggle flips type="password" to
+ * type="text", so type alone is not a durable signal — see
+ * isStickySensitiveAutocomplete and the recorder's per-node WeakSet.
+ */
+const SENSITIVE_AUTOCOMPLETE_TOKENS: Array<string> = [
+  "current-password",
+  "new-password",
+  "one-time-code",
+  "cc-number",
+  "cc-csc",
+  "cc-exp",
+  "cc-exp-month",
+  "cc-exp-year",
+  "cc-name",
+];
+
+/*
+ * Input types whose *value* is masked in every mode. Kept separate from
+ * the mode-dependent set so a MaskInputsOnly application still cannot
+ * leak a password.
+ */
+const ALWAYS_SENSITIVE_INPUT_TYPES: Array<string> = ["password"];
+
+export default class Masking {
+  /*
+   * Mask an input value. Constant width — never derived from the input
+   * length. Returns the mask even for an empty value so that "field was
+   * cleared" and "field was never touched" are not distinguishable in a
+   * way that leaks.
+   */
+  public static maskInputValue(): string {
+    return MASK_CHARACTER.repeat(FIXED_INPUT_MASK_LENGTH);
+  }
+
+  /*
+   * Mask a static text node, preserving a coarse size class so layout
+   * survives. Whitespace-only text is returned untouched: masking it
+   * would collapse spacing without protecting anything.
+   */
+  public static maskText(value: string): string {
+    if (!value) {
+      return value;
+    }
+
+    if (value.trim().length === 0) {
+      return value;
+    }
+
+    const length: number = value.trim().length;
+
+    let maskedLength: number = TEXT_BUCKET_LONG_LENGTH;
+
+    if (length <= TEXT_BUCKET_SHORT_MAX) {
+      maskedLength = TEXT_BUCKET_SHORT_LENGTH;
+    } else if (length <= TEXT_BUCKET_MEDIUM_MAX) {
+      maskedLength = TEXT_BUCKET_MEDIUM_LENGTH;
+    }
+
+    return MASK_CHARACTER.repeat(maskedLength);
+  }
+
+  /*
+   * True when a file input's value must be blanked. Always true: the DOM
+   * value of a file input is `C:\fakepath\<real filename>`, and filenames
+   * are routinely personal ("passport-scan.pdf", "payslip-march.pdf").
+   * The recorder records that a file was chosen, never which one.
+   */
+  public static maskFileInputValue(): string {
+    return "";
+  }
+
+  /* Does this input type carry a value that must be masked in every mode? */
+  public static isAlwaysSensitiveInputType(inputType: string): boolean {
+    return ALWAYS_SENSITIVE_INPUT_TYPES.includes(
+      (inputType || "").toLowerCase(),
+    );
+  }
+
+  /*
+   * Does this autocomplete attribute mark the field as permanently
+   * sensitive? Once true for a node, the recorder keeps masking it even
+   * if the attribute or the input type later changes, and suppresses the
+   * type mutation itself so playback cannot reveal the switch.
+   */
+  public static isStickySensitiveAutocomplete(
+    autocompleteValue: string,
+  ): boolean {
+    if (!autocompleteValue) {
+      return false;
+    }
+
+    /*
+     * autocomplete accepts space-separated tokens with optional section
+     * and billing/shipping prefixes ("section-payment billing cc-number"),
+     * so every token is checked rather than the whole string compared.
+     */
+    const tokens: Array<string> = autocompleteValue
+      .toLowerCase()
+      .split(/\s+/)
+      .filter((token: string): boolean => {
+        return token.length > 0;
+      });
+
+    return tokens.some((token: string): boolean => {
+      return SENSITIVE_AUTOCOMPLETE_TOKENS.includes(token);
+    });
+  }
+
+  /*
+   * Quantise an input event timestamp into coarse buckets.
+   *
+   * A masked password field still leaks inter-keystroke timing, which is
+   * a published side channel for inferring typed content. Bucketing the
+   * timestamps removes the signal while keeping playback smooth enough
+   * that typing still looks like typing.
+   */
+  public static quantiseTimestamp(
+    timestampMs: number,
+    bucketMs: number = 250,
+  ): number {
+    if (!Number.isFinite(timestampMs) || bucketMs <= 0) {
+      return timestampMs;
+    }
+
+    return Math.floor(timestampMs / bucketMs) * bucketMs;
+  }
+}

package/Utils/Rum/SessionIdentity.ts

@@ -0,0 +1,230 @@
+import {
+  SESSION_REPLAY_IDLE_ROLLOVER_MS,
+  SESSION_REPLAY_MAX_SESSION_MS,
+} from "../../Types/Rum/SessionReplay";
+
+/*
+ * Session identity lifecycle, expressed as pure functions of timestamps
+ * so it can be tested exhaustively without a browser, a clock, or storage.
+ *
+ * The recorder owns the storage (localStorage for the session, sessionStorage
+ * for the tab); this module owns the *decisions*.
+ *
+ * Must stay dependency-free apart from the shared constants — bundled
+ * into the browser recorder.
+ */
+
+export enum SessionRotationReason {
+  /* No prior session, or storage was cleared. */
+  New = "new",
+
+  /* More than the idle window elapsed since the last recorded activity. */
+  Idle = "idle",
+
+  /* The session hit its absolute duration ceiling. */
+  DurationCap = "duration-cap",
+
+  /* Stored state was unreadable or inconsistent; fail into a fresh session. */
+  Corrupt = "corrupt",
+}
+
+export interface StoredSessionState {
+  sessionId: string;
+  sessionStartUnixMs: number;
+  lastActivityUnixMs: number;
+}
+
+export interface SessionRotationDecision {
+  shouldRotate: boolean;
+  reason?: SessionRotationReason;
+}
+
+export default class SessionIdentity {
+  /*
+   * Should the recorder mint a new session id?
+   *
+   * Two independent conditions, checked in this order because the idle
+   * check is the common case and the duration cap is the backstop:
+   *
+   *  - idle: the user walked away and came back. A single "session" that
+   *    spans a lunch break is not a session, and stitching one produces a
+   *    recording with a 40-minute dead zone in the middle.
+   *  - duration cap: a tab left open for days would otherwise accumulate
+   *    an unbounded chunk sequence under one sort-key prefix.
+   *
+   * Inconsistent stored state (missing id, non-finite or future-dated
+   * timestamps, start after last activity) rotates rather than being
+   * repaired — a fresh session loses at most a little context, whereas
+   * trusting corrupt state can attribute one user's chunks to another
+   * user's session id.
+   */
+  public static shouldRotateSession(
+    stored: StoredSessionState | null,
+    nowUnixMs: number,
+    idleRolloverMs: number = SESSION_REPLAY_IDLE_ROLLOVER_MS,
+    maxSessionMs: number = SESSION_REPLAY_MAX_SESSION_MS,
+  ): SessionRotationDecision {
+    if (!stored) {
+      return { shouldRotate: true, reason: SessionRotationReason.New };
+    }
+
+    if (!SessionIdentity.isStoredStateValid(stored, nowUnixMs)) {
+      return { shouldRotate: true, reason: SessionRotationReason.Corrupt };
+    }
+
+    if (nowUnixMs - stored.lastActivityUnixMs >= idleRolloverMs) {
+      return { shouldRotate: true, reason: SessionRotationReason.Idle };
+    }
+
+    if (nowUnixMs - stored.sessionStartUnixMs >= maxSessionMs) {
+      return {
+        shouldRotate: true,
+        reason: SessionRotationReason.DurationCap,
+      };
+    }
+
+    return { shouldRotate: false };
+  }
+
+  public static isStoredStateValid(
+    stored: StoredSessionState,
+    nowUnixMs: number,
+  ): boolean {
+    if (!stored.sessionId) {
+      return false;
+    }
+
+    if (
+      !Number.isFinite(stored.sessionStartUnixMs) ||
+      !Number.isFinite(stored.lastActivityUnixMs)
+    ) {
+      return false;
+    }
+
+    if (stored.sessionStartUnixMs <= 0 || stored.lastActivityUnixMs <= 0) {
+      return false;
+    }
+
+    /* Activity cannot precede the session start. */
+    if (stored.lastActivityUnixMs < stored.sessionStartUnixMs) {
+      return false;
+    }
+
+    /*
+     * Tolerate a small amount of forward skew (the device clock may have
+     * been corrected between writes) but reject state stamped far in the
+     * future, which indicates either a clock jump or tampering.
+     */
+    const forwardToleranceMs: number = 5 * 60 * 1000;
+
+    if (stored.sessionStartUnixMs > nowUnixMs + forwardToleranceMs) {
+      return false;
+    }
+
+    return true;
+  }
+
+  /*
+   * Clamp a client-reported session start against the server's own clock.
+   *
+   * End-user device clocks are wrong constantly — by minutes, by hours,
+   * occasionally by years. An unclamped client timestamp lands in the
+   * partition key and the TTL expression, so a device clock set to 2035
+   * would create a partition that never expires, and one set to 2015
+   * would create a row that is deleted on arrival.
+   *
+   * The clamp is one-directional in effect: the returned value is always
+   * within [serverNow - maxBackwardMs, serverNow]. A session cannot start
+   * in the future, and cannot claim to have started before the window we
+   * are willing to accept.
+   */
+  public static clampSessionStart(
+    clientReportedStartUnixMs: number,
+    serverNowUnixMs: number,
+    maxBackwardMs: number = SESSION_REPLAY_MAX_SESSION_MS,
+  ): number {
+    if (!Number.isFinite(clientReportedStartUnixMs)) {
+      return serverNowUnixMs;
+    }
+
+    if (clientReportedStartUnixMs > serverNowUnixMs) {
+      return serverNowUnixMs;
+    }
+
+    const earliestAcceptable: number = serverNowUnixMs - maxBackwardMs;
+
+    if (clientReportedStartUnixMs < earliestAcceptable) {
+      return earliestAcceptable;
+    }
+
+    return clientReportedStartUnixMs;
+  }
+
+  /*
+   * Observed clock skew, stored on the session header so a support
+   * engineer can see "this recording's timings are 40 minutes off"
+   * instead of being quietly misled by them.
+   *
+   * Positive means the client clock runs ahead of the server.
+   */
+  public static getClockSkewMs(
+    clientSendUnixMs: number,
+    serverReceiveUnixMs: number,
+  ): number {
+    if (
+      !Number.isFinite(clientSendUnixMs) ||
+      !Number.isFinite(serverReceiveUnixMs)
+    ) {
+      return 0;
+    }
+
+    return clientSendUnixMs - serverReceiveUnixMs;
+  }
+
+  /*
+   * Clamp a retention day count to the allowed closed set, rounding *up*
+   * to the next allowed value.
+   *
+   * Rounding up rather than to the nearest is deliberate: silently
+   * shortening a customer's configured retention destroys evidence they
+   * expected to keep, whereas keeping it slightly longer costs only disk.
+   */
+  public static clampRetentionDays(
+    requestedDays: number,
+    allowedDays: Array<number>,
+    fallbackDays: number,
+  ): number {
+    if (allowedDays.length === 0) {
+      return fallbackDays;
+    }
+
+    const sorted: Array<number> = [...allowedDays].sort(
+      (a: number, b: number): number => {
+        return a - b;
+      },
+    );
+
+    const smallest: number | undefined = sorted[0];
+    const largest: number | undefined = sorted[sorted.length - 1];
+
+    if (smallest === undefined || largest === undefined) {
+      return fallbackDays;
+    }
+
+    if (!Number.isFinite(requestedDays) || requestedDays <= 0) {
+      return fallbackDays;
+    }
+
+    if (requestedDays >= largest) {
+      return largest;
+    }
+
+    for (const allowed of sorted) {
+      if (requestedDays <= allowed) {
+        return allowed;
+      }
+    }
+
+    return fallbackDays;
+  }
+}

package/Utils/Rum/SessionSampling.ts

@@ -0,0 +1,108 @@
+/*
+ * Deterministic session sampling.
+ *
+ * The decision MUST be a pure function of the session id, computed
+ * identically in the browser and at the ingest gate. A per-event or
+ * per-chunk random draw (which is what the log drop-filter path does)
+ * would shred a single session into an unplayable set of fragments —
+ * some chunks kept, some dropped, no full snapshot where the player
+ * needs one.
+ *
+ * This file must stay dependency-free: it is bundled into the browser
+ * recorder as well as imported by the server.
+ */
+
+/*
+ * FNV-1a, 32-bit. Chosen over a cryptographic hash because it is ~20
+ * lines, has no platform dependency (no SubtleCrypto, no node:crypto),
+ * is synchronous, and distributes hex session ids evenly. Sampling is
+ * not a security boundary — see the note on tampering below.
+ */
+const FNV_OFFSET_BASIS_32: number = 0x811c9dc5;
+const FNV_PRIME_32: number = 0x01000193;
+
+/* Resolution of the sample percentage: two decimal places. */
+const SAMPLING_BUCKETS: number = 10000;
+
+export default class SessionSampling {
+  /*
+   * 32-bit FNV-1a hash of a string, returned as an unsigned integer.
+   *
+   * Math.imul is used for the multiply so the intermediate stays in
+   * 32-bit integer space; a plain `*` would overflow into a float and
+   * lose the low bits, which is the classic way this hash silently
+   * stops distributing.
+   */
+  public static fnv1a32(value: string): number {
+    let hash: number = FNV_OFFSET_BASIS_32;
+
+    for (let i: number = 0; i < value.length; i++) {
+      hash ^= value.charCodeAt(i);
+      hash = Math.imul(hash, FNV_PRIME_32);
+    }
+
+    /* Coerce to unsigned 32-bit. */
+    return hash >>> 0;
+  }
+
+  /*
+   * Which of the 10,000 sampling buckets a session falls into.
+   * Stable for the lifetime of the session id.
+   */
+  public static getBucket(sessionId: string): number {
+    return SessionSampling.fnv1a32(sessionId) % SAMPLING_BUCKETS;
+  }
+
+  /*
+   * True when this session is inside the sample.
+   *
+   * samplePercentage is 0..100 and may carry two decimal places. 0 means
+   * nothing is sampled; 100 means everything is. Values outside the range
+   * are clamped rather than rejected, because a bad config value must not
+   * be able to turn sampling into an exception on the ingest hot path.
+   *
+   * Note honestly: because the input is a client-generated id, a
+   * determined client can regenerate ids until one passes. This function
+   * protects against misconfiguration and gives stable, reproducible
+   * decisions — it is not an anti-abuse control. The origin allowlist,
+   * the rate limiter and the per-project byte budget are.
+   */
+  public static isSampled(
+    sessionId: string,
+    samplePercentage: number,
+  ): boolean {
+    const clamped: number = SessionSampling.clampPercentage(samplePercentage);
+
+    if (clamped <= 0) {
+      return false;
+    }
+
+    if (clamped >= 100) {
+      return true;
+    }
+
+    /*
+     * Rounded rather than truncated so a percentage like 33.335 maps to
+     * the nearer bucket boundary instead of always downward.
+     */
+    const threshold: number = Math.round(clamped * (SAMPLING_BUCKETS / 100));
+
+    return SessionSampling.getBucket(sessionId) < threshold;
+  }
+
+  public static clampPercentage(samplePercentage: number): number {
+    if (!Number.isFinite(samplePercentage)) {
+      return 0;
+    }
+
+    if (samplePercentage < 0) {
+      return 0;
+    }
+
+    if (samplePercentage > 100) {
+      return 100;
+    }
+
+    return samplePercentage;
+  }
+}