@obsrviq/tracker 0.5.0 → 0.6.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@obsrviq/tracker",
-  "version": "0.5.0",
+  "version": "0.6.0",
   "license": "MIT",
   "publishConfig": {
     "access": "public"
@@ -27,7 +27,7 @@
     "@rrweb/types": "2.0.1",
     "fflate": "^0.8.2",
     "rrweb": "2.0.1",
-    "@obsrviq/types": "0.4.1"
+    "@obsrviq/types": "0.5.0"
   },
   "devDependencies": {
     "tsup": "^8.3.5",

package/src/index.ts

@@ -1,7 +1,7 @@
 import type { IngestSessionMeta, ObsrviqEvent, ObsrviqInitConfig } from '@obsrviq/types';
 import type { InstrumentCtx, ResolvedConfig, Teardown } from './context.js';
 import { Transport } from './transport.js';
-import { instrumentDom } from './recorder.js';
+import { instrumentDom, takeFullSnapshot } from './recorder.js';
 import { instrumentConsole } from './console.js';
 import { instrumentNetwork } from './network.js';
 import { instrumentWebSocket } from './websocket.js';
@@ -215,6 +215,10 @@ class ObsrviqClient {
         this.nextSeq = next;
         this.persistSession();
       },
+      // Backpressure dropped DOM deltas → re-anchor the rrweb stream immediately
+      // so replay stays coherent (a delta stream with a hole is corrupt until
+      // the next 30s checkout).
+      onDomDrop: () => takeFullSnapshot(),
       onFlush: ({ bytes, events, ok }) => {
         if (ok) {
           this.diag.flushes++;

package/src/network.ts

@@ -1,9 +1,12 @@
 import type { NetworkEvent, NetworkResourceType, NetworkTiming } from '@obsrviq/types';
 import type { InstrumentCtx, Teardown } from './context.js';
 import { maskString } from './mask.js';
+import { tokenBucket } from './ratelimit.js';
 import { uuid } from './util.js';
 
 const BODY_CAP = 4 * 1024; // 4 KB body preview cap
+const NET_RATE = 50; // sustained network events/sec per page
+const NET_BURST = 100; // burst capacity
 // Response/request header allowlist — everything else is dropped.
 const HEADER_ALLOW = new Set([
   'content-type',
@@ -56,16 +59,56 @@ function bodyPreview(
   contentType: string | undefined,
 ): NetworkEvent['requestBody'] {
   if (text == null) return undefined;
-  const masked = maskString(text);
-  const truncated = masked.length > BODY_CAP;
+  // Pre-cut giant bodies (2× slack) so masking never regex-scans a multi-MB string.
+  const cut = text.length > BODY_CAP * 2 ? text.slice(0, BODY_CAP * 2) : text;
+  const masked = maskString(cut);
+  const truncated = cut !== text || masked.length > BODY_CAP;
   return {
     contentType,
     preview: truncated ? masked.slice(0, BODY_CAP) : masked,
     truncated,
-    redacted: masked !== text,
+    redacted: masked !== cut,
   };
 }
 
+/** Read at most `cap`+1 bytes of a Response body via its stream reader (then
+ *  cancel), so a multi-MB response is never fully buffered just to keep a 4 KB
+ *  preview. `overflow` = the body had more bytes than `cap`. Falls back to
+ *  text() where a stream reader isn't available (older fetch polyfills). */
+async function readTextCapped(res: Response, cap: number): Promise<{ text: string; overflow: boolean }> {
+  const body = res.body;
+  if (!body || typeof body.getReader !== 'function') {
+    return { text: await res.text(), overflow: false };
+  }
+  const reader = body.getReader();
+  const decoder = new TextDecoder(); // Response.text() decodes UTF-8 — match it
+  let text = '';
+  let bytes = 0;
+  let overflow = false;
+  try {
+    for (;;) {
+      const { done, value } = await reader.read();
+      if (done || !value) {
+        text += decoder.decode(); // flush any trailing multi-byte sequence
+        break;
+      }
+      const remaining = cap + 1 - bytes;
+      if (value.byteLength >= remaining) {
+        // Keep only up to cap+1 bytes total — the +1 detects truncation.
+        text += decoder.decode(value.subarray(0, remaining), { stream: true });
+        overflow = true;
+        break;
+      }
+      text += decoder.decode(value, { stream: true });
+      bytes += value.byteLength;
+    }
+  } finally {
+    // Release the tee'd clone branch so it doesn't buffer the rest of the response.
+    void reader.cancel().catch(() => {});
+  }
+  return { text, overflow };
+}
+
 function perfNow(): number {
   return typeof performance !== 'undefined' ? performance.now() : 0;
 }
@@ -239,8 +282,26 @@ function createTimingResolver(): TimingResolver {
   };
 }
 
+/** A rate-gated NetworkEvent emitter shared by all capture paths on the page. */
+type EmitNet = (ev: NetworkEvent) => void;
+
 export function instrumentNetwork(ctx: InstrumentCtx): Teardown {
   const timing = createTimingResolver();
+  // Per-page event throttle — a polling loop / request storm must not flood the
+  // buffer. Dropped events are counted and reported on the next captured one.
+  const bucket = tokenBucket(NET_RATE, NET_BURST);
+  let dropped = 0;
+  const emitNet: EmitNet = (ev) => {
+    if (!bucket.take()) {
+      dropped++;
+      return;
+    }
+    if (dropped > 0) {
+      ev.droppedEvents = dropped;
+      dropped = 0;
+    }
+    ctx.emit(ev);
+  };
   // On page hide, settle any request whose Resource Timing hasn't arrived yet so
   // its event lands in the transport buffer before the unload flush.
   const onPageHide = () => timing.flushPending();
@@ -251,10 +312,10 @@ export function instrumentNetwork(ctx: InstrumentCtx): Teardown {
   if (typeof document !== 'undefined') document.addEventListener('visibilitychange', onVisibility);
 
   const teardowns: Teardown[] = [];
-  teardowns.push(wrapFetch(ctx, timing));
-  teardowns.push(wrapXhr(ctx, timing));
-  teardowns.push(wrapBeacon(ctx));
-  teardowns.push(observeResources(ctx));
+  teardowns.push(wrapFetch(ctx, timing, emitNet));
+  teardowns.push(wrapXhr(ctx, timing, emitNet));
+  teardowns.push(wrapBeacon(ctx, emitNet));
+  teardowns.push(observeResources(ctx, emitNet));
   teardowns.push(() => {
     if (typeof window !== 'undefined') window.removeEventListener('pagehide', onPageHide);
     if (typeof document !== 'undefined') document.removeEventListener('visibilitychange', onVisibility);
@@ -264,7 +325,7 @@ export function instrumentNetwork(ctx: InstrumentCtx): Teardown {
 }
 
 // ───────────────────────────── fetch ────────────────────────────────
-function wrapFetch(ctx: InstrumentCtx, tr: TimingResolver): Teardown {
+function wrapFetch(ctx: InstrumentCtx, tr: TimingResolver, emitNet: EmitNet): Teardown {
   const orig = window.fetch;
   if (!orig) return () => {};
   window.fetch = async function (input: RequestInfo | URL, init?: RequestInit) {
@@ -304,7 +365,7 @@ function wrapFetch(ctx: InstrumentCtx, tr: TimingResolver): Teardown {
           timing,
           ...partial,
         };
-        ctx.emit(ev);
+        emitNet(ev);
       });
     };
 
@@ -316,8 +377,10 @@ function wrapFetch(ctx: InstrumentCtx, tr: TimingResolver): Teardown {
       let responseBody: NetworkEvent['responseBody'];
       if (ctx.config.captureNetworkBodies && isTexty(res.headers.get('content-type'))) {
         try {
-          const text = await res.clone().text();
+          // Stream-read the clone up to the preview cap — never buffer a huge body.
+          const { text, overflow } = await readTextCapped(res.clone(), BODY_CAP);
           responseBody = bodyPreview(text, res.headers.get('content-type') || undefined);
+          if (responseBody && overflow) responseBody.truncated = true;
         } catch {
           /* ignore */
         }
@@ -364,7 +427,7 @@ interface XhrMeta {
   body?: string;
 }
 
-function wrapXhr(ctx: InstrumentCtx, tr: TimingResolver): Teardown {
+function wrapXhr(ctx: InstrumentCtx, tr: TimingResolver, emitNet: EmitNet): Teardown {
   const XHR = XMLHttpRequest.prototype;
   const origOpen = XHR.open;
   const origSend = XHR.send;
@@ -445,7 +508,7 @@ function wrapXhr(ctx: InstrumentCtx, tr: TimingResolver): Teardown {
             timing,
             error: error || classifyHttp(status),
           };
-          ctx.emit(ev);
+          emitNet(ev);
         });
       };
       this.addEventListener('loadend', () => finish());
@@ -484,7 +547,7 @@ function parseRawHeaders(ctx: InstrumentCtx, raw: string): Record<string, string
 }
 
 // ──────────────────────────── sendBeacon ────────────────────────────
-function wrapBeacon(ctx: InstrumentCtx): Teardown {
+function wrapBeacon(ctx: InstrumentCtx, emitNet: EmitNet): Teardown {
   const orig = navigator.sendBeacon;
   if (!orig) return () => {};
   navigator.sendBeacon = function (url: string | URL, data?: BodyInit | null): boolean {
@@ -511,7 +574,7 @@ function wrapBeacon(ctx: InstrumentCtx): Teardown {
           timing: { startT, endT: ctx.clock.now(), duration: 0 },
           error: ok ? undefined : 'network',
         };
-        ctx.emit(ev);
+        emitNet(ev);
       }
     }
     return ok;
@@ -534,7 +597,7 @@ const RESOURCE_TYPE_MAP: Record<string, NetworkResourceType> = {
   track: 'media',
 };
 
-function observeResources(ctx: InstrumentCtx): Teardown {
+function observeResources(ctx: InstrumentCtx, emitNet: EmitNet): Teardown {
   if (typeof PerformanceObserver === 'undefined') return () => {};
   let obs: PerformanceObserver | undefined;
   try {
@@ -571,7 +634,7 @@ function observeResources(ctx: InstrumentCtx): Teardown {
           responseSize: entry.transferSize || entry.encodedBodySize || undefined,
           timing,
         };
-        ctx.emit(ev);
+        emitNet(ev);
       }
     });
     obs.observe({ type: 'resource', buffered: true });

package/src/ratelimit.ts

@@ -0,0 +1,24 @@
+// Shared token-bucket rate limiter for capture paths (network / WebSocket
+// event floods). Kept tiny on purpose — callers count their own drops.
+
+export interface TokenBucket {
+  /** Spend one token if available; false = the caller should drop the event. */
+  take(): boolean;
+}
+
+/** A token bucket: refills at `rate` tokens/sec (sustained), holds at most
+ *  `burst` tokens (burst capacity). */
+export function tokenBucket(rate: number, burst: number): TokenBucket {
+  let tokens = burst;
+  let last = Date.now();
+  return {
+    take() {
+      const now = Date.now();
+      tokens = Math.min(burst, tokens + ((now - last) / 1000) * rate);
+      last = now;
+      if (tokens < 1) return false;
+      tokens -= 1;
+      return true;
+    },
+  };
+}

package/src/recorder.ts

@@ -10,11 +10,18 @@ const INCREMENTAL = 3;
 export function instrumentDom(ctx: InstrumentCtx): Teardown {
   const stop = record({
     emit(event: eventWithTime) {
+      const t = ctx.clock.fromPerf(performanceNow());
+      // ONE CLOCK: rrweb stamps events with Date.now(), while everything else in
+      // the session runs on the monotonic perf clock (`t`). Rewrite the rrweb
+      // timestamp onto that same clock (startedAt + t) so an NTP step / manual
+      // clock change mid-session can't desync replay from the console/network
+      // lanes — the player only ever uses timestamp DELTAS, so this is safe.
+      event.timestamp = ctx.clock.startedAt + t;
       const dom: DomEvent = {
         id: uuid(),
         sessionId: ctx.sessionId,
         type: 'dom',
-        t: ctx.clock.fromPerf(performanceNow()),
+        t,
         ts: ctx.clock.wall(),
         data: event,
       };
@@ -48,6 +55,18 @@ export function instrumentDom(ctx: InstrumentCtx): Teardown {
   };
 }
 
+/** Force a fresh FullSnapshot checkpoint NOW. Used after transport backpressure
+ *  had to drop buffered DOM deltas — the new snapshot re-anchors the stream so
+ *  replay stays coherent instead of playing broken deltas until the next 30s
+ *  checkout. Safe no-op when recording isn't active. */
+export function takeFullSnapshot(): void {
+  try {
+    record.takeFullSnapshot(true);
+  } catch {
+    /* not recording */
+  }
+}
+
 function performanceNow(): number {
   return typeof performance !== 'undefined' ? performance.now() : Date.now();
 }

package/src/serialize.ts

@@ -1,7 +1,8 @@
 import type { SerializedArg } from '@obsrviq/types';
-import { maskString, maskValue } from './mask.js';
+import { maskString, isSensitiveKeyName } from './mask.js';
 
 const DEFAULT_MAX_BYTES = 8 * 1024; // 8 KB string cap (§6.1)
+const MAX_WALK_NODES = 2000; // deep-walk node-visit cap (§6.1)
 
 function byteLen(s: string): number {
   return new Blob([s]).size;
@@ -24,9 +25,12 @@ export function serializeArg(arg: unknown, maxBytes = DEFAULT_MAX_BYTES): Serial
   }
 
   if (type === 'string') {
-    const masked = maskString(arg as string);
+    // Pre-cut giant strings (2× slack) so masking never regex-scans a multi-MB arg.
+    const s = arg as string;
+    const cut = s.length > maxBytes * 2 ? s.slice(0, maxBytes * 2) : s;
+    const masked = maskString(cut);
     const { value, truncated } = truncate(masked, maxBytes);
-    return { kind: 'string', value, truncated };
+    return { kind: 'string', value, truncated: truncated || cut !== s };
   }
 
   if (arg instanceof Error) {
@@ -43,24 +47,76 @@ export function serializeArg(arg: unknown, maxBytes = DEFAULT_MAX_BYTES): Serial
   }
 
   if (type === 'object' || type === 'function' || type === 'symbol' || type === 'bigint') {
-    const masked = maskValue(arg);
+    // Budgets are enforced DURING the walk (node-visit count + ~2× the byte cap)
+    // so a huge object never burns CPU/memory before the final truncate applies.
+    const budget: WalkBudget = { nodes: MAX_WALK_NODES, bytes: maxBytes * 2, exhausted: false };
+    const masked = maskValueBudgeted(arg, budget);
     let json: string | undefined;
-    let truncated = false;
+    let truncated = budget.exhausted;
     try {
       const full = JSON.stringify(masked, safeReplacer());
       const cut = truncate(full ?? 'undefined', maxBytes);
       json = cut.value;
-      truncated = cut.truncated;
+      truncated = truncated || cut.truncated;
     } catch {
       json = undefined;
     }
-    return { kind: 'object', preview: previewOf(masked), json, truncated };
+    return { kind: 'object', preview: previewOf(arg), json, truncated };
   }
 
   // undefined / fallback
   return { kind: 'primitive', value: String(arg) };
 }
 
+interface WalkBudget {
+  nodes: number; // node visits left
+  bytes: number; // approximate serialized bytes left
+  exhausted: boolean;
+}
+
+/** `maskValue` with hard budgets enforced DURING the walk — a node-visit cap and a
+ *  running approximate byte budget — so a huge arg (e.g. a multi-MB store) is never
+ *  fully cloned/stringified just to be truncated afterwards. When a budget runs out
+ *  we stop descending and substitute a '«truncated»' placeholder. Byte-identical to
+ *  the unbudgeted walk for args that fit the budget. */
+function maskValueBudgeted(value: unknown, budget: WalkBudget, depth = 0): unknown {
+  if (budget.exhausted) return '«truncated»';
+  if (depth > 6) return '«depth»';
+  if (--budget.nodes <= 0) budget.exhausted = true;
+  if (typeof value === 'string') {
+    // Pre-cut giant strings so masking never regex-scans more than the budget.
+    const cut = value.length > budget.bytes ? value.slice(0, Math.max(0, budget.bytes)) : value;
+    if (cut !== value) budget.exhausted = true;
+    const masked = maskString(cut);
+    if ((budget.bytes -= masked.length + 2) <= 0) budget.exhausted = true;
+    return masked;
+  }
+  if (value === null || typeof value !== 'object') return value;
+  if (Array.isArray(value)) {
+    const out: unknown[] = [];
+    for (const v of value) {
+      if (budget.exhausted) {
+        out.push('«truncated»');
+        break;
+      }
+      out.push(maskValueBudgeted(v, budget, depth + 1));
+    }
+    return out;
+  }
+  const out: Record<string, unknown> = {};
+  for (const k of Object.keys(value)) {
+    if (budget.exhausted) {
+      out['…'] = '«truncated»';
+      break;
+    }
+    if ((budget.bytes -= k.length + 4) <= 0) budget.exhausted = true;
+    out[k] = isSensitiveKeyName(k)
+      ? '«redacted»'
+      : maskValueBudgeted((value as Record<string, unknown>)[k], budget, depth + 1);
+  }
+  return out;
+}
+
 function safeReplacer() {
   const seen = new WeakSet();
   return (_key: string, value: unknown) => {