@milaboratories/pl-crash-recorder 0.3.0

package/src/redact.ts

@@ -0,0 +1,213 @@
+import crypto from "node:crypto";
+import { summarizeData, type DataSummary } from "./data_summary";
+
+/**
+ * Structure-preserving redaction for anything a driver seam is handed.
+ *
+ * The definition a block model builds is recorded by *shape* rather than by a
+ * hand-written digest per definition type. Keys, numbers and the small set of
+ * strings that are schema survive verbatim; every other string is replaced by a
+ * hash and a length. That keeps the record useful for diagnosis, keeps customer
+ * data out of it by default rather than by enumeration, and works unchanged for
+ * definition shapes this code has never seen — the V2 query API included.
+ *
+ * The default for an unrecognised string is to hash it. A new field can
+ * therefore make a report less informative, but never make it leak.
+ */
+
+/** Keys whose string value is schema, kept as written. */
+const SCHEMA_KEYS = new Set(["type", "name", "valueType", "kind", "operator", "mode"]);
+
+/** Keys under which every string is schema, at any depth (axis identity). */
+const SCHEMA_SUBTREE_KEYS = new Set(["domain", "contextDomain"]);
+
+/** Keys never descended into; summarised by counts instead. */
+const SUMMARISED_KEYS = new Set(["data", "dataInfo"]);
+
+/** Keys reduced to a cardinality, because their contents are values. */
+const COUNTED_KEYS = new Set(["references", "parts"]);
+
+export type RedactionStats = {
+  hashedStrings: number;
+  truncatedArrays: number;
+  omittedItems: number;
+  depthCapped: number;
+  opaqueObjects: number;
+  budgetExhausted: boolean;
+};
+
+type RedactOptions = {
+  maxDepth?: number;
+  maxArrayItems?: number;
+  maxStringLength?: number;
+  /** Ceiling on emitted values, so one pathological definition cannot fill the log. */
+  maxNodes?: number;
+};
+
+type HashedString = { h: string; n: number };
+
+/** Redacts a definition, returning the new value and what had to be elided. */
+export function redact(
+  value: unknown,
+  options: RedactOptions = {},
+): { value: unknown; stats: RedactionStats } {
+  const limits = {
+    maxDepth: options.maxDepth ?? 32,
+    maxArrayItems: options.maxArrayItems ?? 64,
+    maxStringLength: options.maxStringLength ?? 128,
+    maxNodes: options.maxNodes ?? 20_000,
+  };
+  const stats: RedactionStats = {
+    hashedStrings: 0,
+    truncatedArrays: 0,
+    omittedItems: 0,
+    depthCapped: 0,
+    opaqueObjects: 0,
+    budgetExhausted: false,
+  };
+  const state = { nodes: 0, seen: new WeakSet<object>() };
+  return {
+    value: walk(value, { key: undefined, schemaSubtree: false, depth: 0 }, limits, stats, state),
+    stats,
+  };
+}
+
+/** Stable short hash plus the original length. Never reversible to the value. */
+function hashString(value: string): HashedString {
+  return {
+    h: crypto.createHash("sha256").update(value).digest("hex").slice(0, 12),
+    n: value.length,
+  };
+}
+
+/** True for the object form produced in place of a redacted string. */
+export function isHashedString(value: unknown): value is HashedString {
+  return typeof value === "object" && value !== null && "h" in value && "n" in value;
+}
+
+// Internals
+
+type Position = { key: string | undefined; schemaSubtree: boolean; depth: number };
+type Limits = Required<RedactOptions>;
+type State = { nodes: number; seen: WeakSet<object> };
+
+function walk(
+  value: unknown,
+  at: Position,
+  limits: Limits,
+  stats: RedactionStats,
+  state: State,
+): unknown {
+  if (state.nodes++ > limits.maxNodes) {
+    stats.budgetExhausted = true;
+    return { $budget: true };
+  }
+
+  if (value === null || value === undefined) return value ?? null;
+  if (typeof value === "bigint") return Number(value);
+  if (typeof value === "number" || typeof value === "boolean") return value;
+  if (typeof value === "string") return redactString(value, at, limits, stats);
+  if (typeof value !== "object") return { $type: typeof value };
+
+  if (at.depth >= limits.maxDepth) {
+    stats.depthCapped++;
+    return { $depth: at.depth };
+  }
+  // A definition can carry live accessors and other class instances whose
+  // internals reference each other; walking those is neither safe nor useful.
+  if (state.seen.has(value)) return { $cycle: true };
+
+  if (Array.isArray(value)) {
+    state.seen.add(value);
+    return walkArray(value, at, limits, stats, state);
+  }
+  if (!isPlainObject(value)) {
+    stats.opaqueObjects++;
+    return { $opaque: className(value) };
+  }
+
+  state.seen.add(value);
+  const out: Record<string, unknown> = {};
+  for (const [key, child] of Object.entries(value)) {
+    if (SUMMARISED_KEYS.has(key)) {
+      out[key] = summarizeData(child);
+      continue;
+    }
+    if (COUNTED_KEYS.has(key)) {
+      out[key] = { $count: countOf(child) };
+      continue;
+    }
+    out[key] = walk(
+      child,
+      {
+        key,
+        schemaSubtree: at.schemaSubtree || SCHEMA_SUBTREE_KEYS.has(key),
+        depth: at.depth + 1,
+      },
+      limits,
+      stats,
+      state,
+    );
+  }
+  return out;
+}
+
+function walkArray(
+  value: unknown[],
+  at: Position,
+  limits: Limits,
+  stats: RedactionStats,
+  state: State,
+): unknown[] {
+  const kept = value
+    .slice(0, limits.maxArrayItems)
+    .map((item) =>
+      walk(
+        item,
+        { key: at.key, schemaSubtree: at.schemaSubtree, depth: at.depth + 1 },
+        limits,
+        stats,
+        state,
+      ),
+    );
+  const omitted = value.length - kept.length;
+  if (omitted <= 0) return kept;
+  // Arrays stay arrays so the rules can still walk join entries; the loss is
+  // recorded in the array itself rather than in a side channel.
+  stats.truncatedArrays++;
+  stats.omittedItems += omitted;
+  return [...kept, { $omitted: omitted }];
+}
+
+function redactString(
+  value: string,
+  at: Position,
+  limits: Limits,
+  stats: RedactionStats,
+): string | HashedString {
+  if (at.schemaSubtree || (at.key !== undefined && SCHEMA_KEYS.has(at.key))) {
+    return value.length > limits.maxStringLength
+      ? `${value.slice(0, limits.maxStringLength)}…`
+      : value;
+  }
+  stats.hashedStrings++;
+  return hashString(value);
+}
+
+function countOf(value: unknown): number | undefined {
+  if (Array.isArray(value)) return value.length;
+  if (isPlainObject(value)) return Object.keys(value).length;
+  return undefined;
+}
+
+function isPlainObject(value: unknown): value is Record<string, unknown> {
+  if (typeof value !== "object" || value === null || Array.isArray(value)) return false;
+  const proto = Object.getPrototypeOf(value) as object | null;
+  return proto === Object.prototype || proto === null;
+}
+
+function className(value: object): string {
+  return (value as { constructor?: { name?: string } }).constructor?.name ?? "unknown";
+}
+
+export type { DataSummary };

package/src/sampler.ts

@@ -0,0 +1,40 @@
+import path from "node:path";
+import { Worker } from "node:worker_threads";
+import { SAMPLER_FILE_PREFIX } from "./events";
+
+type MemorySamplerOptions = {
+  dir: string;
+  sessionId: string;
+  /** Sampling period; 250 ms is roughly four short appends per second. */
+  intervalMs?: number;
+};
+
+export type MemorySampler = {
+  /** Sibling log the sampler appends to. */
+  readonly file: string;
+  stop(): void;
+};
+
+/**
+ * Starts the out-of-band memory sampler for a session.
+ *
+ * The sampler runs on a thread of its own with a small heap of its own, so it
+ * keeps producing readings when the observed thread is blocked and when the
+ * observed thread's heap is the thing that is full.
+ */
+export function startMemorySampler(options: MemorySamplerOptions): MemorySampler {
+  const { dir, sessionId, intervalMs = 250 } = options;
+  const file = path.join(dir, `${SAMPLER_FILE_PREFIX}-${sessionId}.ndjson`);
+  const worker = new Worker(new URL("./sampler_thread.js", import.meta.url), {
+    workerData: { file, intervalMs },
+    resourceLimits: { maxOldGenerationSizeMb: 32 },
+  });
+  // Unreferenced so a sampler that is never stopped cannot hold the process open.
+  worker.unref();
+  return {
+    file,
+    stop: () => {
+      void worker.terminate();
+    },
+  };
+}

package/src/sampler_thread.ts

@@ -0,0 +1,60 @@
+/**
+ * Memory sampler, run on its own worker thread.
+ *
+ * It exists because the thread worth watching is the one that blocks. While the
+ * middle layer sits inside a synchronous pframes call its own timers do not
+ * fire, so its memory series goes dark exactly while memory is growing fastest.
+ * This thread stays responsive and keeps the resident-size curve intact right up
+ * to the moment the process dies.
+ *
+ * `rss` and `freeMemory` are process- and machine-wide and so are meaningful
+ * from here. Heap figures are per-isolate and would describe only this thread,
+ * so they are deliberately not recorded; the observed thread reports its own.
+ *
+ * Resident size alone understates a process under pressure, because the OS moves
+ * its pages into the compressor or out to swap and they stop being resident. The
+ * machine's own account of where memory went is therefore sampled alongside it,
+ * less often because it costs a subprocess.
+ */
+
+import fs from "node:fs";
+import os from "node:os";
+import { workerData } from "node:worker_threads";
+import type { SamplerRecord } from "./events";
+import { readMachineMemory } from "./machine_memory";
+
+type SamplerWorkerData = { file: string; intervalMs: number; machineIntervalMs?: number };
+
+const { file, intervalMs, machineIntervalMs = 1000 } = workerData as SamplerWorkerData;
+const fd = fs.openSync(file, "a");
+let seq = 0;
+let peakRss = 0;
+let machineDueAt = 0;
+
+setInterval(() => {
+  const rss = process.memoryUsage.rss();
+  if (rss > peakRss) peakRss = rss;
+  const now = Date.now();
+  // Taken on the first tick and then on its own schedule, so the curve keeps its
+  // sampling rate while the costlier reading stays occasional.
+  const machine = now >= machineDueAt ? readMachineMemory() : undefined;
+  if (machine) machineDueAt = now + machineIntervalMs;
+  const record: SamplerRecord = {
+    seq: ++seq,
+    t: Math.round(performance.now() * 1000) / 1000,
+    wall: now,
+    type: "mem-sampler",
+    rss,
+    peakRss,
+    // The kernel's own high-water mark, which no sampling interval can miss.
+    maxRss: process.resourceUsage().maxRSS * 1024,
+    freeMemory: os.freemem(),
+    totalMemory: os.totalmem(),
+    ...(machine ? { machine } : {}),
+  };
+  try {
+    fs.writeSync(fd, `${JSON.stringify(record)}\n`);
+  } catch {
+    // Sampling must never take the application down.
+  }
+}, intervalMs);

package/src/session.ts

@@ -0,0 +1,71 @@
+import { openRecorder, startSelfSampler, type Recorder } from "./recorder";
+import { startMemorySampler, type MemorySampler } from "./sampler";
+import { createHandleRegistry, type HandleRegistry } from "./instrument";
+
+/** Environment variable naming the directory crash logs are written to. */
+export const CRASH_DIR_ENV = "MI_CRASH_RECORDER_DIR";
+
+/**
+ * Environment variable carrying the session id a supervising parent assigned.
+ * Set it alongside {@link CRASH_DIR_ENV} when spawning the worker and pass the
+ * same id to `superviseWorker`.
+ */
+export const CRASH_SESSION_ENV = "MI_CRASH_RECORDER_SESSION";
+
+export type RecordingSessionOptions = {
+  /** Overrides the directory from the environment. */
+  dir?: string;
+  /** Overrides the session id from the environment. */
+  sessionId?: string;
+  role?: string;
+  meta?: Record<string, unknown>;
+  samplerIntervalMs?: number;
+  selfSamplerIntervalMs?: number;
+};
+
+export type RecordingSession = {
+  readonly recorder: Recorder;
+  readonly sampler: MemorySampler;
+  /** Shared so create calls and later data calls agree on handle identity. */
+  readonly registry: HandleRegistry;
+  close(reason?: string): void;
+};
+
+/**
+ * Opens a recording session, or returns undefined when recording is not enabled.
+ *
+ * Recording is opt-in for now: it appends synchronously on every recorded
+ * operation, and that cost has not been measured against a real project, so it
+ * is switched on by pointing {@link CRASH_DIR_ENV} at a directory rather than
+ * being on by default.
+ */
+export function openRecordingSession(
+  options: RecordingSessionOptions = {},
+): RecordingSession | undefined {
+  const dir = options.dir ?? process.env[CRASH_DIR_ENV];
+  if (!dir) return undefined;
+
+  const recorder = openRecorder({
+    dir,
+    role: options.role,
+    meta: options.meta,
+    sessionId: options.sessionId ?? process.env[CRASH_SESSION_ENV] ?? undefined,
+  });
+  const sampler = startMemorySampler({
+    dir,
+    sessionId: recorder.sessionId,
+    intervalMs: options.samplerIntervalMs,
+  });
+  const stopSelfSampler = startSelfSampler(recorder, options.selfSamplerIntervalMs);
+
+  return {
+    recorder,
+    sampler,
+    registry: createHandleRegistry(),
+    close(reason) {
+      stopSelfSampler();
+      sampler.stop();
+      recorder.close(reason);
+    },
+  };
+}

package/src/supervisor.ts

@@ -0,0 +1,183 @@
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { DEATH_FILE_PREFIX, type CrashMarker, type CrashReason } from "./events";
+import { listSessions, sessionIdFromFile } from "./recorder";
+
+type CrashMarkerInput = {
+  /** Session id the parent assigned to the worker. Omitted, the marker carries no identity. */
+  sessionId?: string;
+  reason?: CrashReason;
+  error?: (Error & { code?: string }) | unknown;
+  code?: number;
+  signal?: string;
+  stderrTail?: string;
+};
+
+export type SuperviseOptions = {
+  /**
+   * The session id handed to the worker at spawn (see `CRASH_SESSION_ENV`).
+   * With it the marker names the dying session with certainty. Without it the
+   * analyzer has to attribute the marker by timing, and will decline to
+   * attribute it at all when more than one session looks dead.
+   */
+  sessionId?: string;
+  onCrash?: (info: {
+    kind: "error" | "exit";
+    markerFile: string;
+    error?: unknown;
+    code?: number;
+  }) => void;
+};
+
+/** Minimal view of a worker, so callers are not forced to import worker_threads. */
+export type SupervisedWorker = {
+  on(event: "error", listener: (error: Error) => void): unknown;
+  on(event: "exit", listener: (code: number) => void): unknown;
+};
+
+/**
+ * Records an abnormal end observed from outside the dying thread.
+ *
+ * A thread that runs out of heap cannot describe its own death: the last reading
+ * it wrote predates the blow-up, and when the blow-up is synchronous no sampler
+ * tick of its own lands either. The parent is the only place where the cause is
+ * known rather than inferred — Node reports `ERR_WORKER_OUT_OF_MEMORY` to it —
+ * so the parent writes the verdict down on the dead thread's behalf.
+ */
+export function writeCrashMarker(dir: string, input: CrashMarkerInput = {}): string {
+  fs.mkdirSync(dir, { recursive: true });
+  const error = input.error as (Error & { code?: string }) | undefined;
+  // Only an id the parent handed to the worker is certain, and only a certain
+  // id goes in `sessionId`. Reading the newest open crash log names whichever
+  // session wrote last, which a concurrent live session makes wrong; recorded
+  // as identity that would misattribute the death and, worse, stop the session
+  // that actually died from claiming the marker. So it is advisory only.
+  const marker: CrashMarker = {
+    type: "external-crash",
+    wall: Date.now(),
+    sessionId: input.sessionId,
+    guessedSessionId: input.sessionId === undefined ? newestOpenSessionId(dir) : undefined,
+    reason: input.reason ?? classifyReason(input),
+    errorCode: error?.code,
+    errorName: error?.name,
+    message: truncate(String(error?.message ?? input.error ?? ""), 2000),
+    exitCode: input.code,
+    signal: input.signal,
+    stderrTail: truncate(input.stderrTail ?? "", 4000),
+    memoryAtDeath: memoryNow(),
+  };
+  const file = path.join(dir, `${DEATH_FILE_PREFIX}-${marker.wall}.ndjson`);
+  fs.writeFileSync(file, `${JSON.stringify(marker)}\n`);
+  return file;
+}
+
+/** Crash markers in a directory, oldest first. */
+export function readCrashMarkers(dir: string): CrashMarker[] {
+  let names: string[];
+  try {
+    names = fs.readdirSync(dir);
+  } catch {
+    return [];
+  }
+  const markers: CrashMarker[] = [];
+  for (const name of names) {
+    if (!name.startsWith(`${DEATH_FILE_PREFIX}-`) || !name.endsWith(".ndjson")) continue;
+    try {
+      const first = fs.readFileSync(path.join(dir, name), "utf8").split("\n")[0];
+      markers.push(JSON.parse(first) as CrashMarker);
+    } catch {
+      // A marker that cannot be parsed is skipped; it is one line of evidence,
+      // not the report.
+    }
+  }
+  return markers.sort((lhs, rhs) => lhs.wall - rhs.wall);
+}
+
+/**
+ * Attaches crash recording to a middle-layer worker thread.
+ *
+ * A worker whose isolate exhausts its heap dies alone and the parent receives
+ * `ERR_WORKER_OUT_OF_MEMORY`, with or without `resourceLimits`. What
+ * `resourceLimits.maxOldGenerationSizeMb` adds is a chosen ceiling: V8's default
+ * is several gigabytes, so on a small machine the OS can run out of memory and
+ * kill the whole process before V8 ever reports the worker's heap as full — and
+ * then there is no parent left to write anything.
+ */
+export function superviseWorker(
+  worker: SupervisedWorker,
+  dir: string,
+  options: SuperviseOptions = {},
+): void {
+  // One death fires `error` and then `exit`. Only `error` carries the cause, so
+  // a later `exit` must not overwrite it with a bare exit code.
+  let recorded = false;
+  worker.on("error", (error: Error) => {
+    recorded = true;
+    const markerFile = writeCrashMarker(dir, { error, sessionId: options.sessionId });
+    options.onCrash?.({ kind: "error", error, markerFile });
+  });
+  worker.on("exit", (code: number) => {
+    if (code === 0 || recorded) return;
+    const markerFile = writeCrashMarker(dir, {
+      reason: "worker-exit",
+      code,
+      sessionId: options.sessionId,
+    });
+    options.onCrash?.({ kind: "exit", code, markerFile });
+  });
+}
+
+// Internals
+
+/**
+ * The parent's view of memory at the moment it saw the death.
+ *
+ * The dying thread cannot take this reading, and the sampler's last one predates
+ * the end by up to its interval. Taken here it is contemporaneous with the exit
+ * code it sits beside, which is what stops an exhausted machine from reading as
+ * an ordinary failure.
+ *
+ * Every reading here is a syscall. The machine's compressor and swap totals are
+ * deliberately not among them: on macOS they cost a subprocess, and this runs on
+ * the parent's event loop inside the worker's error handler, before the marker
+ * is written and before the caller learns of the death. A fork is exactly what
+ * becomes slow or impossible on the exhausted machine this code exists for, so
+ * the fuller picture is left to the sampler, whose last reading is at most one
+ * interval old and sits in the same bundle.
+ */
+function memoryNow(): CrashMarker["memoryAtDeath"] {
+  try {
+    return {
+      rss: process.memoryUsage.rss(),
+      maxRss: process.resourceUsage().maxRSS * 1024,
+      freeMemory: os.freemem(),
+      totalMemory: os.totalmem(),
+    };
+  } catch {
+    // A marker without memory is still a marker; failing to take the reading
+    // must never cost the record of the death itself.
+    return undefined;
+  }
+}
+
+// Advisory only, for a human reading a directory by hand: the dying session has
+// no terminating record, so among the sessions that look dead this names the one
+// that wrote last. Never used as identity — see `CrashMarker.guessedSessionId`.
+function newestOpenSessionId(dir: string): string | undefined {
+  const open = listSessions(dir).find((session) => session.crashed);
+  return open ? sessionIdFromFile(open.file) : undefined;
+}
+
+function classifyReason({ error, code, signal }: CrashMarkerInput): CrashReason {
+  const errorCode = (error as { code?: string } | undefined)?.code;
+  if (errorCode === "ERR_WORKER_OUT_OF_MEMORY") return "js-heap-out-of-memory";
+  if (signal === "SIGKILL") return "killed-by-os";
+  if (signal === "SIGABRT" || code === 134) return "abort-or-fatal-allocation-failure";
+  if (typeof code === "number" && code !== 0) return "nonzero-exit";
+  return "unknown";
+}
+
+function truncate(value: string, limit: number): string {
+  return value.length > limit ? `${value.slice(0, limit)}…` : value;
+}