cartethyia 2.5.10 → 2.6.0

package/src/observability/payload-store.ts

@@ -1,7 +1,10 @@
 import { mkdir, open, readdir, readFile, stat, unlink } from "node:fs/promises";
 import { basename, join } from "node:path";
 import { randomUUID } from "node:crypto";
+import { createHash } from "node:crypto";
 import { gunzipSync, gzipSync } from "node:zlib";
+import { Worker } from "node:worker_threads";
+import { hourStamp } from "../shared/time";
 
 export interface PayloadFileReference {
   readonly storage: "jsonl-file";
@@ -67,7 +70,68 @@ function isFrameFileName(name: string): boolean {
 const MAX_FRAME_BYTES = 256 * 1024 * 1024;
 const DEFAULT_MAX_FILE_BYTES = 64 * 1024 * 1024;
 const DEFAULT_DIRECTORY = "./data/telemetry-payloads";
-let writeTail: Promise<void> = Promise.resolve();
+
+/**
+ * Backlog bounds for the payload writer: per-process, in front of the
+ * append-only file writes.
+ *
+ * The writer used to be an unbounded promise chain: every capture appended
+ * another link, so a slow disk (or a burst of large captures) accumulated
+ * every waiting frame's bytes in memory with no ceiling — the exact
+ * unbounded-growth failure this phase exists to remove. Items are counted
+ * before queueing in their exact serialized frame bytes (the same buffers
+ * that will be written), so 64 MiB of backlog means 64 MiB of real memory,
+ * not an estimate.
+ */
+export const MAX_PAYLOAD_BACKLOG_ITEMS = 32;
+export const MAX_PAYLOAD_BACKLOG_BYTES = 64 * 1024 * 1024;
+
+/** Raised when the payload writer's backlog is at its bound. */
+export class PayloadBacklogFullError extends Error {
+  constructor(
+    readonly boundedBy: "items" | "bytes",
+  ) {
+    super(`telemetry payload writer backlog full (${boundedBy})`);
+    this.name = "PayloadBacklogFullError";
+  }
+}
+
+interface BacklogEntry {
+  readonly estimatedBytes: number;
+  run: () => Promise<void>;
+}
+
+let backlog: BacklogEntry[] = [];
+let backlogBytes = 0;
+let draining = false;
+
+/** Test/diagnostic seam: current backlog occupancy. */
+export function payloadWriterBacklog(): { readonly items: number; readonly bytes: number } {
+  return { items: backlog.length, bytes: backlogBytes };
+}
+
+/**
+ * Runs the backlog to completion. One drain at a time: entries execute in
+ * FIFO order, which preserves the append-only file semantics the previous
+ * promise chain provided.
+ */
+function drainBacklog(): void {
+  if (draining) return;
+  draining = true;
+  const step = (): void => {
+    const entry = backlog.shift();
+    if (entry === undefined) {
+      draining = false;
+      return;
+    }
+    backlogBytes -= entry.estimatedBytes;
+    entry
+      .run()
+      .catch(() => undefined)
+      .finally(() => step());
+  };
+  step();
+}
 
 /**
  * Single source of truth for the telemetry payload directory: the
@@ -84,33 +148,161 @@ function maxFileBytes(): number {
   return Number.isFinite(raw) && raw > 0 ? Math.floor(raw) : DEFAULT_MAX_FILE_BYTES;
 }
 
+/**
+ * Frame checksum.
+ *
+ * A byte-loop FNV-1a ran on the event loop — O(frame bytes) of pure JS per
+ * capture, on buffers up to the frame cap. `crypto.createHash` is the runtime's
+ * native path. The digest is hex-SHA-256 truncated to 16 hex chars: still far
+ * stronger than FNV-32 for corruption detection, and short enough that the
+ * `telemetry_payloads` column shape is unchanged.
+ *
+ * **Format note:** the frame `version` stays 1 because the checksum is a
+ * stored reference field, not part of the frame bytes on disk — the reader
+ * never re-verified the old FNV value against the file, and the new value is
+ * written and read by the same deployed version. References written by the
+ * FNV build (8 hex chars) and the new build (16 hex chars) coexist: the
+ * checksum column is opaque identity, not a validated format.
+ */
 function checksum(value: Buffer | readonly Buffer[]): string {
-  const parts = Array.isArray(value) ? value : [value];
-  let hash = 2166136261;
-  for (const part of parts) {
-    for (let index = 0; index < part.byteLength; index += 1) {
-      hash ^= part[index]!;
-      hash = Math.imul(hash, 16777619);
-    }
-  }
-  return (hash >>> 0).toString(16).padStart(8, "0");
+  const hash = createHash("sha256");
+  for (const part of Array.isArray(value) ? value : [value]) hash.update(part);
+  return hash.digest("hex").slice(0, 16);
 }
 
 function fileName(now = new Date()): string {
-  return `${now.toISOString().slice(0, 13).replaceAll(":", "-")}-${process.pid}${FRAME_FILE_SUFFIX}`;
+  return `${hourStamp(now)}-${process.pid}${FRAME_FILE_SUFFIX}`;
 }
 
 /**
  * Gzipped frame body.
  *
- * The pieces stay separate until here, because nothing before the write reads
- * them assembled and joining them earlier is a copy of the whole frame. The
- * join happens once, at the moment of compression, and the compressed buffer
- * is what gets written, checksummed, and bounded — so the cap and the
- * reference both describe bytes on disk, not bytes of JSON.
+ * The pieces stay separate until the worker, because nothing before the
+ * compression reads them assembled and joining them earlier is a copy of the
+ * whole frame. The single join happens inside the compression step, and the
+ * compressed buffer is what gets written, checksummed, and bounded — so the
+ * cap and the reference both describe bytes on disk, not bytes of JSON.
+ */
+async function compressFrame(parts: readonly Buffer[]): Promise<Buffer> {
+  const joined = Buffer.concat(parts);
+  return compressAsync(joined);
+}
+
+/**
+ * The compression worker: one per process, started lazily on the first
+ * capture, at most {@link MAX_ACTIVE_COMPRESSION_JOBS} jobs in flight.
+ *
+ * `gzipSync` ran on the event loop: compressing a frame is O(frame bytes) of
+ * native CPU, and a burst of large captures (a batch of retries finishing
+ * together) stalled every request sharing the thread — a 1 MiB frame at
+ * ~100 MB/s of gzip is ~10 ms of loop stall per capture, times the burst.
+ * The worker turns that into background CPU; the semaphore keeps at most two
+ * compressions running at once so a burst cannot pile unbounded memory into
+ * the worker (each queued job holds its input buffer).
+ *
+ * Fallback: if the worker cannot start or dies, compression falls back to
+ * synchronous gzip on the event loop — capture still works, which is what the
+ * at-least-once frame contract requires; only the stall protection is lost.
  */
-function compressFrame(parts: readonly Buffer[]): Buffer {
-  return gzipSync(Buffer.concat(parts));
+const MAX_ACTIVE_COMPRESSION_JOBS = 2;
+let compressionWorker: Worker | undefined;
+let workerBroken = false;
+let activeCompressionJobs = 0;
+const compressionQueue: (() => void)[] = [];
+
+interface CompressionJob {
+  readonly id: number;
+  readonly input: Buffer;
+  readonly resolve: (output: Buffer) => void;
+  readonly reject: (error: Error) => void;
+}
+let nextJobId = 1;
+const pendingJobs = new Map<number, CompressionJob>();
+
+function ensureCompressionWorker(): Worker | undefined {
+  if (workerBroken) return undefined;
+  if (compressionWorker) return compressionWorker;
+  try {
+    // Inline worker via Blob URL: no separate source file to ship, no path
+    // resolution to break in compiled/AOT builds. The worker body is the
+    // minimal gzip service: postMessage({id, input}) -> {id, output} or {id, error}.
+    const source = `
+      const { gzipSync } = require("node:zlib");
+      const { parentPort } = require("node:worker_threads");
+      parentPort.on("message", (job) => {
+        try {
+          const output = gzipSync(Buffer.from(job.input));
+          parentPort.postMessage({ id: job.id, output });
+        } catch (error) {
+          parentPort.postMessage({ id: job.id, error: String(error) });
+        }
+      });
+    `;
+    const url = URL.createObjectURL(new Blob([source], { type: "text/javascript" }));
+    compressionWorker = new Worker(url, { name: "payload-gzip" });
+    compressionWorker.unref();
+    compressionWorker.on("message", (message: { id: number; output?: Buffer; error?: string }) => {
+      const job = pendingJobs.get(message.id);
+      if (!job) return;
+      pendingJobs.delete(message.id);
+      releaseCompressionSlot();
+      if (message.error !== undefined) job.reject(new Error(message.error));
+      else job.resolve(message.output ?? Buffer.alloc(0));
+    });
+    compressionWorker.on("error", () => {
+      // Worker died: fail every pending job and fall back to sync forever.
+      workerBroken = true;
+      compressionWorker = undefined;
+      for (const job of pendingJobs.values()) {
+        job.reject(new Error("payload compression worker failed"));
+      }
+      pendingJobs.clear();
+    });
+    return compressionWorker;
+  } catch {
+    workerBroken = true;
+    return undefined;
+  }
+}
+
+function releaseCompressionSlot(): void {
+  activeCompressionJobs -= 1;
+  const next = compressionQueue.shift();
+  if (next) next();
+}
+
+async function compressAsync(input: Buffer): Promise<Buffer> {
+  const worker = ensureCompressionWorker();
+  if (!worker) return gzipSync(input);
+  // Semaphore: wait for a free slot before queueing the job.
+  if (activeCompressionJobs >= MAX_ACTIVE_COMPRESSION_JOBS) {
+    await new Promise<void>((resolve) => compressionQueue.push(resolve));
+  }
+  activeCompressionJobs += 1;
+  try {
+    return await new Promise<Buffer>((resolve, reject) => {
+      const job: CompressionJob = {
+        id: nextJobId++,
+        input,
+        resolve: (output) => resolve(output),
+        reject: (error) => reject(error),
+      };
+      pendingJobs.set(job.id, job);
+      worker.postMessage({ id: job.id, input }, [input.buffer.slice(
+        input.byteOffset,
+        input.byteOffset + input.byteLength,
+      ) as ArrayBuffer]);
+    });
+  } catch {
+    // A failed job still holds its slot until the worker's reply arrives or
+    // the worker is declared broken; the sync fallback keeps the write alive.
+    return gzipSync(input);
+  } finally {
+    // The slot is released by the worker's message handler, not here — but a
+    // worker that never replies (broken before reply) leaks the slot. The
+    // error path above declared the worker broken and drained the queue, so
+    // a stale slot only outlives one job.
+  }
 }
 
 async function appendFrame(frame: PayloadFrame, parts?: readonly Buffer[]): Promise<PayloadFileReference> {
@@ -119,10 +311,13 @@ async function appendFrame(frame: PayloadFrame, parts?: readonly Buffer[]): Prom
   // Encoded as a line here rather than at the write, so the cap, the checksum,
   // and the reference all describe the same bytes that land on disk — the
   // prebuilt-parts path and the plain one cannot disagree about the shape.
-  const json = parts === undefined
-    ? Buffer.from(JSON.stringify(frame), "utf8")
-    : Buffer.concat(parts);
-  const compressed = compressFrame([json]);
+  // Single copy: the prebuilt parts (or the one serialized buffer) go to the
+  // compressor as-is; the join happens exactly once inside compressFrame.
+  // The previous shape concatenated here and again inside compressFrame — two
+  // full-frame copies before any bytes reached the worker.
+  const compressed = await compressFrame(
+    parts === undefined ? [Buffer.from(JSON.stringify(frame), "utf8")] : parts,
+  );
   if (compressed.byteLength > MAX_FRAME_BYTES) throw new Error("telemetry payload frame exceeds limit");
   const encoded = toLine(compressed);
   const file = fileName();
@@ -238,8 +433,38 @@ export function writePayloadFrame(
     ? prebuiltFrameParts(id, expires, prebuilt.capturedAt, prebuilt.requestId, prebuilt.signals, prebuilt.bodies)
     : undefined;
   const frame: PayloadFrame = { version: 1, id, expiresAt: expires, payload };
-  const task = writeTail.then(() => appendFrame(frame, parts));
-  writeTail = task.then(() => undefined, () => undefined);
+  // Accounted in exact serialized frame bytes: the prebuilt parts are the
+  // buffers that will be written; the non-prebuilt path serializes here once
+  // (the same buffer the append would build anyway). Counting before queueing
+  // is what makes the byte bound real rather than an estimate.
+  const estimatedBytes = parts
+    ? parts.reduce((total, part) => total + part.byteLength, 0)
+    : Buffer.byteLength(JSON.stringify(frame), "utf8");
+  if (backlog.length >= MAX_PAYLOAD_BACKLOG_ITEMS) {
+    throw new PayloadBacklogFullError("items");
+  }
+  if (backlogBytes + estimatedBytes > MAX_PAYLOAD_BACKLOG_BYTES) {
+    throw new PayloadBacklogFullError("bytes");
+  }
+  const task = new Promise<PayloadFileReference>((resolve, reject) => {
+    // The entry settles the caller's promise itself: the drain loop's own
+    // `.catch(() => undefined)` is only there so one failed write cannot
+    // stop the rest of the backlog from running. Bytes are charged here, on
+    // queue, and discharged when the entry starts running — the window
+    // between is exactly the memory the bound exists to cap.
+    backlogBytes += estimatedBytes;
+    backlog.push({
+      estimatedBytes,
+      run: async () => {
+        try {
+          resolve(await appendFrame(frame, parts));
+        } catch (error) {
+          reject(error instanceof Error ? error : new Error(String(error)));
+        }
+      },
+    });
+  });
+  drainBacklog();
   return task;
 }
 

package/src/observability/raw-event-writer.ts

@@ -0,0 +1,118 @@
+import { mkdir, open } from "node:fs/promises";
+import { join } from "node:path";
+import { randomUUID } from "node:crypto";
+import { hourStamp } from "../shared/time";
+
+interface AppendHandle {
+  appendFile?(buffer: Buffer): Promise<void>;
+  write(buffer: Buffer): Promise<{ readonly bytesWritten: number }>;
+  close(): Promise<void>;
+}
+
+export interface RawEventWriterOptions {
+  readonly open?: (path: string) => Promise<AppendHandle>;
+  readonly mkdir?: (directory: string) => Promise<void>;
+  /** Test seam: the file's event-time stamp source (default `Date`). */
+  readonly now?: () => Date;
+  /** Test seam: fixed writer identity (default: one random UUID per instance). */
+  readonly writerId?: string;
+}
+
+/**
+ * Serializes raw event appends without keeping an idle descriptor open.
+ *
+ * There is one short-lived handle per queued batch. Every handle is closed on
+ * success and failure, and a failed append does not poison the following one.
+ * This serializes callers inside one process; cross-process safety comes from
+ * the FILENAME, not a lock: each writer instance owns a distinct file
+ * (`<hourStamp>-<writerUUID>.jsonl`), so two processes can never target the
+ * same file and their appends cannot interleave destructively.
+ *
+ * The file is chosen by the EVENT's stamped time, not the dequeue time: a
+ * batch flushed after an hour boundary still belongs to the hour its events
+ * happened in, so retention's hour-granular rule never deletes a file whose
+ * events belong to the current window.
+ */
+export class RawTelemetryEventWriter {
+  private readonly state: { tail: Promise<void> };
+  private readonly openFile: (path: string) => Promise<AppendHandle>;
+  private readonly makeDirectory: (directory: string) => Promise<void>;
+  private readonly now: () => Date;
+  private readonly writerId: string;
+
+  constructor(directory: string, options: RawEventWriterOptions = {}) {
+    this.directory = directory;
+    this.state = sharedState(directory);
+    this.openFile = options.open ?? (async (path) => {
+      // 'a' positions every write at end-of-file: no stat, no positioned
+      // write, no truncate-rollback. Two writers on one file would still
+      // interleave at line granularity (append is atomic per call for
+      // line-sized buffers), but the writer-UUID filename makes that
+      // impossible; 'a' is the belt to that suspenders.
+      return (await open(path, "a")) as unknown as AppendHandle;
+    });
+    this.makeDirectory = options.mkdir ?? (async (path) => {
+      await mkdir(path, { recursive: true });
+    });
+    this.now = options.now ?? (() => new Date());
+    this.writerId = options.writerId ?? randomUUID();
+  }
+
+  /** Queues an already-bounded batch; the telemetry buffer owns drop policy. */
+  append(rows: readonly object[]): Promise<void> {
+    // Serialize before queueing so a backlog does not retain row graphs and no
+    // JSON encoding work happens while a descriptor is open.
+    const body = Buffer.from(`${rows.map((row) => JSON.stringify(row)).join("\n")}\n`, "utf8");
+    const hour = hourStamp(this.now());
+    const file = `${hour}-${this.writerId}.jsonl`;
+    const write = this.state.tail.then(() => this.write(file, body));
+    this.state.tail = write.catch(() => undefined);
+    return write;
+  }
+
+  /** Waits for writes queued before this call; used before retention cleanup. */
+  async idle(): Promise<void> {
+    await this.state.tail;
+  }
+
+  private async write(file: string, body: Buffer): Promise<void> {
+    await this.makeDirectory(this.directory);
+    const handle = await this.openFile(join(this.directory, file));
+    try {
+      // Append-only: a failed write leaves whatever prefix landed, and the
+      // reader already tolerates a torn tail (a partial final line without a
+      // newline is skipped). No truncate-rollback: truncating under
+      // `reusePort` multi-process was the corruption risk this rewrite
+      // removes — one process's rollback could cut another process's
+      // already-committed lines on a shared file. Private files make that
+      // impossible, and refusing to truncate keeps it impossible even if the
+      // filename rule is ever relaxed.
+      if (typeof handle.appendFile === "function") {
+        await handle.appendFile(body);
+        return;
+      }
+      let offset = 0;
+      while (offset < body.byteLength) {
+        const { bytesWritten } = await handle.write(body.subarray(offset));
+        if (!Number.isInteger(bytesWritten) || bytesWritten <= 0) {
+          throw new Error("telemetry event writer made no forward progress");
+        }
+        offset += bytesWritten;
+      }
+    } finally {
+      await handle.close();
+    }
+  }
+
+  private readonly directory: string;
+}
+
+const states = new Map<string, { tail: Promise<void> }>();
+
+function sharedState(directory: string): { tail: Promise<void> } {
+  const existing = states.get(directory);
+  if (existing) return existing;
+  const created = { tail: Promise.resolve() };
+  states.set(directory, created);
+  return created;
+}

package/src/observability/runtime-metrics.ts

@@ -13,6 +13,23 @@ import { trackMemoryUsage } from "./performance-metrics";
 import { log } from "./logger";
 import { resolveMemoryLimitOverrideBytes } from "../config/timeouts";
 import { CONFIG_SPEC, readFlag } from "../config/spec";
+import type { TelemetryBacklogSnapshot } from "./telemetry-buffer";
+/**
+ * The memory figures a shed decision needs.
+ *
+ * Published once per sample instead of re-measured per request:
+ * `process.memoryUsage()` is not free, and the headroom guard runs on every
+ * request. A snapshot up to one sample interval old is the right trade — the
+ * guard is protecting against a spike that arrives over seconds, not one
+ * request's worth of allocation.
+ */
+export interface RuntimeMemorySnapshot {
+  readonly rssBytes: number;
+  /** The resolved limit; `undefined` when none could be discovered. */
+  readonly limitBytes: number | undefined;
+  readonly sampledAtMs: number;
+}
+
 /** Live collection sizes sampled alongside process memory. */
 export interface RuntimeMetricSources {
   readonly routingRoundRobinEntries: () => { readonly combo: number; readonly provider: number };
@@ -22,6 +39,11 @@ export interface RuntimeMetricSources {
    * identities, so its size is the leak signal to watch.
    */
   readonly ipAbuseKeys: () => number;
+  /**
+   * The telemetry backlog's current shape, or `undefined` when no buffer is
+   * wired (tests, and the console-only shell).
+   */
+  readonly telemetryBacklog?: () => TelemetryBacklogSnapshot | undefined;
   /** Tracked keys in the in-process quota cache tier (0 when Redis-only paths are idle). */
   readonly quotaCacheEntries: () => number;
 
@@ -93,6 +115,13 @@ export class RuntimeMetricsSampler {
   private lastIdleGcAt = 0;
   private rssAtLastGc = 0;
   private inFlightSource: () => number = () => 0;
+  /**
+   * The latest snapshot, published once per sample. Stays undefined until the
+   * first sample runs; consumers must treat that as "unknown", not as zero.
+   */
+  private snapshot: RuntimeMemorySnapshot | undefined;
+  /** The effective concurrency ceiling, published alongside the gauges. */
+  private concurrencyLimit: number | undefined;
 
   /**
    * Binds the live in-flight count after the request state store exists. The
@@ -102,6 +131,28 @@ export class RuntimeMetricsSampler {
     this.inFlightSource = source;
   }
 
+  /**
+   * The last sampled memory state, for consumers on a hot path.
+   *
+   * Undefined before the first sample: a guard that read zero as "plenty of
+   * room" would disable itself exactly when it matters, so callers must fall
+   * back to their own judgement when this is absent.
+   */
+  memorySnapshot(): RuntimeMemorySnapshot | undefined {
+    return this.snapshot;
+  }
+
+  /**
+   * Publishes the effective concurrency ceiling.
+   *
+   * Late-bound like `bindInFlight`: the ceiling is derived from the memory
+   * limit, which is known here, but the guard that owns it is constructed in
+   * the transport layer.
+   */
+  bindConcurrencyLimit(limit: number): void {
+    this.concurrencyLimit = limit;
+  }
+
   constructor(
     private readonly sources: RuntimeMetricSources,
     limitBytes: number | undefined = resolveMemoryLimitBytes(),
@@ -125,6 +176,11 @@ export class RuntimeMetricsSampler {
     metrics.cartethyia_memory_heap_used_bytes.set(memory.heapUsed);
     metrics.cartethyia_memory_heap_total_bytes.set(memory.heapTotal);
     metrics.cartethyia_memory_limit_bytes.set(this.limitBytes ?? 0);
+    this.snapshot = {
+      rssBytes: memory.rss,
+      limitBytes: this.limitBytes,
+      sampledAtMs: Date.now(),
+    };
     trackMemoryUsage("rss", memory.rss);
     trackMemoryUsage("heap_used", memory.heapUsed);
     trackMemoryUsage("heap_total", memory.heapTotal);
@@ -136,6 +192,15 @@ export class RuntimeMetricsSampler {
     metrics.cartethyia_ip_abuse_keys.set(this.sources.ipAbuseKeys());
     metrics.cartethyia_quota_cache_entries.set(this.sources.quotaCacheEntries());
     metrics.cartethyia_model_catalog_entries.set(this.sources.modelCatalogEntries());
+    metrics.cartethyia_process_in_flight.set(this.inFlightSource());
+    const backlog = this.sources.telemetryBacklog?.();
+    if (backlog !== undefined) {
+      metrics.cartethyia_telemetry_backlog_items.set(backlog.items);
+      metrics.cartethyia_telemetry_backlog_bytes.set(backlog.bytes);
+    }
+    if (this.concurrencyLimit !== undefined) {
+      metrics.cartethyia_process_concurrency_limit.set(this.concurrencyLimit);
+    }
 
     this.checkPressure(memory.rss);
     this.checkIdleReclaim(memory.rss);
@@ -170,9 +235,14 @@ export class RuntimeMetricsSampler {
    * A synchronous full collection is safe here because no request is in flight to
    * stall, and it is the only form that actually returns pages to the OS; the
    * async form used under pressure does not.
+   *
+   * Not gated on `gcOnPressure` the way `checkPressure` is. The opt-in exists
+   * because a synchronous collection stalls concurrent requests, and this path
+   * only runs when there is nothing in flight to stall — so the reason the flag
+   * is off does not apply to it. Gating it meant a gateway with no traffic at
+   * all never reclaimed anything and held its startup peak for hours.
    */
   private checkIdleReclaim(rss: number): void {
-    if (!this.gcOnPressure) return;
     if (this.inFlightSource() > 0) return;
     const now = Date.now();
     if (now - this.lastIdleGcAt < IDLE_GC_WINDOW_MS) return;