@milaboratories/pl-crash-recorder 0.3.0

package/src/recorder.ts

@@ -0,0 +1,435 @@
+import fs from "node:fs";
+import path from "node:path";
+import os from "node:os";
+import v8 from "node:v8";
+import {
+  SESSION_FILE_PREFIX,
+  MEM_BASELINE_RECORD,
+  SESSION_END_RECORD,
+  SESSION_RECORD,
+  type LogRecord,
+  type MemorySnapshot,
+  type SessionEnvironment,
+} from "./events";
+
+export type RecorderOptions = {
+  /** Directory holding crash logs; created if absent. */
+  dir: string;
+  /** Which part of the app is recording, e.g. `middle-layer`. */
+  role?: string;
+  /** Free-form context stored in the session header (app version, project id). */
+  meta?: Record<string, unknown>;
+  /** Log is rotated past this size so the tail, which explains the crash, survives. */
+  maxFileBytes?: number;
+  /**
+   * Session id assigned by a supervising parent, so the crash marker the parent
+   * writes names this session with certainty rather than by inference.
+   * Generated when absent.
+   */
+  sessionId?: string;
+};
+
+export type Recorder = {
+  readonly sessionId: string;
+  readonly file: string;
+  /** Appends one record and returns its sequence number. Never throws. */
+  event(type: string, payload?: Record<string, unknown>): number;
+  /**
+   * Appends a record that every later segment of the log gets again.
+   *
+   * For a fact stated once that the rest of the log is unreadable without — what
+   * a block id stands for, say. Written once per `key`; a rotation that discards
+   * the original rewrites it, so a long session cannot outlive its own legend.
+   *
+   * Past a cap on how many such facts a session may carry, the record is still
+   * written once and marked `notRetained`: it then survives only until the
+   * rotation that discards it.
+   */
+  sticky(key: string, type: string, payload?: Record<string, unknown>): void;
+  /** Memory reading for the calling thread; `rss` is process-wide. */
+  memorySnapshot(): MemorySnapshot;
+  /** Writes the terminating record. Its absence is how a crash is detected. */
+  close(reason?: string): void;
+};
+
+export type SessionFileInfo = {
+  file: string;
+  mtimeMs: number;
+  bytes: number;
+  /** True when the log has no terminating record, i.e. the process died. */
+  crashed: boolean;
+};
+
+type ParsedSession = {
+  file: string;
+  records: LogRecord[];
+  /** True when the last line was cut mid-write by the kill. */
+  truncatedTail: boolean;
+};
+
+/**
+ * Opens a crash log for this process and writes the session header.
+ *
+ * Records are appended with a synchronous write rather than through a stream:
+ * the process being recorded dies without warning — V8 fatal out-of-memory, a
+ * failed native allocation, the OS out-of-memory killer — so no exit hook, no
+ * flush and no `finally` block runs. Anything still sitting in a userspace
+ * buffer is exactly the part that would have explained the death.
+ */
+export function openRecorder(options: RecorderOptions): Recorder {
+  const {
+    dir,
+    role = "middle-layer",
+    meta = {},
+    maxFileBytes = 32 * 1024 * 1024,
+    sessionId = newSessionId(),
+  } = options;
+  fs.mkdirSync(dir, { recursive: true });
+
+  const file = path.join(dir, `${SESSION_FILE_PREFIX}-${sessionId}.ndjson`);
+  const state: WriterState = {
+    fd: fs.openSync(file, "a"),
+    bytes: 0,
+    preambleBytes: 0,
+    seq: 0,
+    closed: false,
+    header: undefined,
+    baselineMem: undefined,
+    openBegins: new Map(),
+    sticky: new Map(),
+  };
+
+  const memorySnapshot = (): MemorySnapshot => {
+    const usage = process.memoryUsage();
+    return {
+      rss: usage.rss,
+      heapUsed: usage.heapUsed,
+      heapTotal: usage.heapTotal,
+      external: usage.external,
+      arrayBuffers: usage.arrayBuffers,
+      heapLimit: v8.getHeapStatistics().heap_size_limit,
+    };
+  };
+
+  const event = (type: string, payload: Record<string, unknown> = {}): number => {
+    if (state.closed) return -1;
+    const seq = ++state.seq;
+    const record: LogRecord = {
+      seq,
+      t: monotonic(),
+      wall: Date.now(),
+      type,
+      ...payload,
+    };
+    if (state.baselineMem === undefined && record.mem) state.baselineMem = record.mem;
+    trackOpenOperation(state, record);
+    writeLine(state, file, maxFileBytes, record);
+    return seq;
+  };
+
+  const sticky = (key: string, type: string, payload?: Record<string, unknown>): void => {
+    if (state.sticky.has(key)) return;
+    // The preamble is rewritten in full at every rotation, so it cannot grow
+    // without bound. Past the cap the fact is still written once — losing it
+    // entirely would leave records naming a block nothing can identify — and
+    // only its survival across rotation is given up. The record says so, so a
+    // reader can tell a missing legend from one that was never written.
+    const retained = state.sticky.size < MAX_STICKY_RECORDS;
+    const seq = event(type, retained ? payload : { ...payload, notRetained: true });
+    if (!retained) return;
+    const record = { ...payload, seq, type } as LogRecord;
+    state.sticky.set(key, record);
+  };
+
+  const recorder: Recorder = {
+    sessionId,
+    file,
+    event,
+    sticky,
+    memorySnapshot,
+    close(reason = "normal") {
+      if (state.closed) return;
+      event(SESSION_END_RECORD, { reason, mem: memorySnapshot() });
+      state.closed = true;
+      try {
+        fs.closeSync(state.fd);
+      } catch {
+        // Closing an already-dead descriptor must not fail shutdown.
+      }
+    },
+  };
+
+  // Kept so a rotated log can be given the same header again: the active file
+  // must describe its own session even if the parked segment is lost.
+  state.header = { role, pid: process.pid, meta, env: describeEnvironment() };
+  event(SESSION_RECORD, { ...state.header, mem: memorySnapshot() });
+
+  return recorder;
+}
+
+/**
+ * Periodic memory record written by the recorded thread itself.
+ *
+ * Doubles as a stall detector. This timer cannot fire while its thread is inside
+ * a long synchronous call, so a gap here that the independent sampler thread
+ * does not share means the thread was blocked — which is also why the heap
+ * reading nearest a synchronous blow-up is always stale.
+ */
+export function startSelfSampler(recorder: Recorder, intervalMs = 500): () => void {
+  let last = Date.now();
+  const timer = setInterval(() => {
+    const now = Date.now();
+    recorder.event("mem-self", {
+      mem: recorder.memorySnapshot(),
+      stallMs: Math.max(0, now - last - intervalMs),
+    });
+    last = now;
+  }, intervalMs);
+  timer.unref();
+  return () => clearInterval(timer);
+}
+
+/** Crash logs in a directory, newest first, each flagged as crashed or clean. */
+export function listSessions(dir: string): SessionFileInfo[] {
+  let names: string[];
+  try {
+    names = fs.readdirSync(dir);
+  } catch {
+    return [];
+  }
+  return names
+    .filter((name) => name.startsWith(`${SESSION_FILE_PREFIX}-`) && name.endsWith(".ndjson"))
+    .map((name) => {
+      const file = path.join(dir, name);
+      const stat = fs.statSync(file);
+      return { file, mtimeMs: stat.mtimeMs, bytes: stat.size, crashed: !hasSessionEnd(file) };
+    })
+    .sort((lhs, rhs) => rhs.mtimeMs - lhs.mtimeMs);
+}
+
+/**
+ * Parses a crash log, tolerating a final line cut short by a hard kill.
+ *
+ * A rotated session spans two files: the parked `.1` segment holds the original
+ * header and the earlier operations, the active file holds the tail. Both are
+ * read so begin records in one segment can be paired with end records in the
+ * other; sequence numbers run across the boundary.
+ */
+export function readSession(file: string): ParsedSession {
+  const records: LogRecord[] = [];
+  let truncatedTail = false;
+  const parked = `${file}.1`;
+  if (fs.existsSync(parked)) {
+    for (const line of fs.readFileSync(parked, "utf8").split("\n")) {
+      if (line === "") continue;
+      try {
+        records.push(JSON.parse(line) as LogRecord);
+      } catch {
+        // A damaged line in the parked segment costs one record, not the session.
+      }
+    }
+  }
+  const lines = fs.readFileSync(file, "utf8").split("\n");
+  for (const [index, line] of lines.entries()) {
+    if (line === "") continue;
+    try {
+      records.push(JSON.parse(line) as LogRecord);
+    } catch {
+      if (index >= lines.length - 2) truncatedTail = true;
+    }
+  }
+  return { file, records, truncatedTail };
+}
+
+/**
+ * Mints a session id. A parent that supervises a worker calls this, hands the id
+ * to the worker, and keeps it for the crash marker, so both sides agree on the
+ * session by construction. The leading timestamp is what
+ * {@link sessionStartFromId} reads back.
+ */
+export function newSessionId(): string {
+  return `${Date.now()}-${process.pid}-${randomTag()}`;
+}
+
+/** Wall-clock start of a session, taken from the id its file name carries. */
+export function sessionStartFromId(sessionId: string): number {
+  const start = Number(sessionId.split("-")[0]);
+  return Number.isFinite(start) ? start : 0;
+}
+
+/** Session id embedded in a crash log's file name. */
+export function sessionIdFromFile(file: string): string {
+  const match = path
+    .basename(file)
+    .match(new RegExp(`^${SESSION_FILE_PREFIX}-(.+)\\.ndjson(\\.1)?$`));
+  return match ? match[1] : path.basename(file);
+}
+
+// Internals
+
+/** How many sticky records a session may keep, bounding the rewritten preamble. */
+const MAX_STICKY_RECORDS = 64;
+
+type WriterState = {
+  fd: number;
+  bytes: number;
+  /** Bytes of the carried-forward preamble, which do not count toward the limit. */
+  preambleBytes: number;
+  seq: number;
+  closed: boolean;
+  header: Record<string, unknown> | undefined;
+  /** Earliest memory reading of the session, so growth stays measurable. */
+  baselineMem: MemorySnapshot | undefined;
+  /** Begin records with no end yet, keyed by their sequence number. */
+  openBegins: Map<number, LogRecord>;
+  /** Records rewritten into every later segment, keyed by the caller's key. */
+  sticky: Map<string, LogRecord>;
+};
+
+/** Cap on carried-forward begins, so a leak cannot make the preamble unbounded. */
+const MAX_CARRIED_BEGINS = 256;
+
+function writeLine(
+  state: WriterState,
+  file: string,
+  maxFileBytes: number,
+  record: LogRecord,
+): void {
+  let line: string;
+  try {
+    line = `${JSON.stringify(record, bigintSafe)}\n`;
+  } catch {
+    line = `${JSON.stringify({ seq: record.seq, type: "record-serialization-failed" })}\n`;
+  }
+  try {
+    // The preamble is not charged against the limit, so a large preamble cannot
+    // trigger another rotation on the very next write.
+    if (state.bytes - state.preambleBytes + line.length > maxFileBytes) {
+      rotate(state, file);
+      writePreamble(state);
+    }
+    fs.writeSync(state.fd, line);
+    state.bytes += line.length;
+  } catch {
+    // A recorder that cannot write stays silent rather than cascading into the
+    // application it is only supposed to observe.
+  }
+}
+
+// The tail is the only part that explains a crash, so the old file is parked
+// beside the new one instead of the new writes being dropped. Exactly one parked
+// segment is kept, which bounds a session's disk use at twice the file limit.
+function rotate(state: WriterState, file: string): void {
+  fs.closeSync(state.fd);
+  try {
+    fs.renameSync(file, `${file}.1`);
+  } catch {
+    // If the parked slot cannot be written, recording simply continues.
+  }
+  state.fd = fs.openSync(file, "a");
+  state.bytes = 0;
+  state.preambleBytes = 0;
+}
+
+/**
+ * Rewrites into the new segment the three things a report cannot be produced
+ * without, so repeated rotation costs only completed operations.
+ *
+ * Losing an *open* begin record would remove that operation from pairing
+ * entirely, and the operation still running at the moment of death is the one
+ * the report exists to name. Losing the earliest memory reading would leave the
+ * heap series starting mid-session while the sampler's resident series still
+ * starts at zero, which biases the classifier toward blaming native memory.
+ */
+function writePreamble(state: WriterState): void {
+  if (state.header) {
+    emitPreambleRecord(state, {
+      seq: ++state.seq,
+      t: monotonic(),
+      wall: Date.now(),
+      type: SESSION_RECORD,
+      ...state.header,
+      continuation: true,
+    });
+  }
+  if (state.baselineMem) {
+    emitPreambleRecord(state, {
+      seq: ++state.seq,
+      t: monotonic(),
+      wall: Date.now(),
+      type: MEM_BASELINE_RECORD,
+      mem: state.baselineMem,
+      carriedForward: true,
+    });
+  }
+  for (const record of state.sticky.values()) {
+    emitPreambleRecord(state, { ...record, carriedForward: true });
+  }
+  // Original sequence numbers are kept, which is what lets an end record in a
+  // later segment pair with a begin first written in an overwritten one.
+  for (const begin of state.openBegins.values()) {
+    emitPreambleRecord(state, { ...begin, carriedForward: true });
+  }
+}
+
+function emitPreambleRecord(state: WriterState, record: LogRecord): void {
+  try {
+    const line = `${JSON.stringify(record, bigintSafe)}\n`;
+    fs.writeSync(state.fd, line);
+    state.bytes += line.length;
+    state.preambleBytes += line.length;
+  } catch {
+    // A preamble that cannot be written must not stop the session.
+  }
+}
+
+// Open operations are tracked by the same suffix convention the analyzer pairs
+// on, so the recorder needs no separate vocabulary for them.
+function trackOpenOperation(state: WriterState, record: LogRecord): void {
+  if (record.type.endsWith("-begin")) {
+    if (state.openBegins.size < MAX_CARRIED_BEGINS) state.openBegins.set(record.seq, record);
+    return;
+  }
+  if (record.type.endsWith("-end") || record.type.endsWith("-error")) {
+    if (typeof record.begin === "number") state.openBegins.delete(record.begin);
+  }
+}
+
+function hasSessionEnd(file: string): boolean {
+  const size = fs.statSync(file).size;
+  if (size === 0) return false;
+  const window = Math.min(size, 8192);
+  const buffer = Buffer.alloc(window);
+  const fd = fs.openSync(file, "r");
+  try {
+    fs.readSync(fd, buffer, 0, window, size - window);
+  } finally {
+    fs.closeSync(fd);
+  }
+  return buffer.toString("utf8").includes(`"type":"${SESSION_END_RECORD}"`);
+}
+
+function describeEnvironment(): SessionEnvironment {
+  const maxOldSpaceFlag = process.execArgv.find((arg) => arg.startsWith("--max-old-space-size"));
+  return {
+    node: process.version,
+    platform: `${process.platform}-${process.arch}`,
+    cpus: os.cpus().length,
+    totalMemory: os.totalmem(),
+    heapLimit: v8.getHeapStatistics().heap_size_limit,
+    execArgv: [...process.execArgv],
+    maxOldSpaceSize: maxOldSpaceFlag ? Number(maxOldSpaceFlag.split("=")[1]) : undefined,
+  };
+}
+
+function bigintSafe(_key: string, value: unknown): unknown {
+  return typeof value === "bigint" ? Number(value) : value;
+}
+
+function monotonic(): number {
+  return Math.round(performance.now() * 1000) / 1000;
+}
+
+function randomTag(): string {
+  return Math.random().toString(36).slice(2, 8);
+}

package/src/redact.test.ts

@@ -0,0 +1,155 @@
+import { describe, expect, test } from "vitest";
+import { digestDef } from "./digest";
+import { isHashedString, redact } from "./redact";
+
+const SECRET = "CASSLGQGAETQYF";
+
+describe("redaction", () => {
+  test("no cell value, filter reference or annotation value survives", () => {
+    const digest = digestDef("PTableDef", {
+      src: {
+        type: "inner",
+        entries: [
+          {
+            type: "inlineColumn",
+            column: {
+              id: "inline",
+              spec: {
+                kind: "PColumn",
+                name: "x",
+                valueType: "String",
+                annotations: { "pl7.app/label": SECRET },
+                axesSpec: [{ type: "String", name: "s" }],
+              },
+              data: [{ key: [SECRET], value: SECRET }],
+            },
+          },
+        ],
+      },
+      partitionFilters: [],
+      filters: [
+        {
+          type: "bySingleColumnV2",
+          column: { type: "axis", id: { type: "String", name: "s" } },
+          predicate: { operator: "Equal", reference: SECRET },
+        },
+      ],
+      sorting: [],
+    });
+    expect(JSON.stringify(digest)).not.toContain(SECRET);
+  });
+
+  test("schema survives verbatim so a report stays readable", () => {
+    const { value } = redact({
+      spec: {
+        kind: "PColumn",
+        name: "pl7.app/vdj/readCount",
+        valueType: "Long",
+        domain: { "pl7.app/vdj/chain": "IGH" },
+        axesSpec: [{ type: "String", name: "pl7.app/sampleId" }],
+      },
+    });
+    const spec = (value as { spec: Record<string, unknown> }).spec;
+    expect(spec.name).toBe("pl7.app/vdj/readCount");
+    expect(spec.valueType).toBe("Long");
+    // Domain values carry join identity, so they are kept at any depth.
+    expect(spec.domain).toEqual({ "pl7.app/vdj/chain": "IGH" });
+  });
+
+  test("annotation keys are kept and their values hashed", () => {
+    const { value } = redact({ annotations: { "pl7.app/label": SECRET } });
+    const annotations = (value as { annotations: Record<string, unknown> }).annotations;
+    expect(Object.keys(annotations)).toEqual(["pl7.app/label"]);
+    expect(isHashedString(annotations["pl7.app/label"])).toBe(true);
+    expect(annotations["pl7.app/label"]).toMatchObject({ n: SECRET.length });
+  });
+
+  test("the same value hashes the same way, so two labels can be compared", () => {
+    const a = redact({ note: SECRET }).value as { note: { h: string } };
+    const b = redact({ note: SECRET }).value as { note: { h: string } };
+    const c = redact({ note: `${SECRET}x` }).value as { note: { h: string } };
+    expect(a.note.h).toBe(b.note.h);
+    expect(a.note.h).not.toBe(c.note.h);
+  });
+
+  test("a class instance is named, not walked", () => {
+    class TreeAccessor {
+      constructor(public readonly secret = SECRET) {}
+    }
+    const { value, stats } = redact({ data: { type: "x" }, accessor: new TreeAccessor() });
+    expect(JSON.stringify(value)).not.toContain(SECRET);
+    expect((value as { accessor: unknown }).accessor).toEqual({ $opaque: "TreeAccessor" });
+    expect(stats.opaqueObjects).toBe(1);
+  });
+
+  test("a cycle is marked instead of hanging", () => {
+    const node: Record<string, unknown> = { type: "column" };
+    node.self = node;
+    const { value } = redact(node);
+    expect((value as { self: unknown }).self).toEqual({ $cycle: true });
+  });
+
+  test("a long array keeps its head, stays an array, and records the loss", () => {
+    const { value, stats } = redact(
+      { entries: Array.from({ length: 200 }, (_, i) => ({ i })) },
+      {
+        maxArrayItems: 8,
+      },
+    );
+    const entries = (value as { entries: unknown[] }).entries;
+    expect(Array.isArray(entries)).toBe(true);
+    expect(entries).toHaveLength(9);
+    expect(entries.at(-1)).toEqual({ $omitted: 192 });
+    expect(stats.omittedItems).toBe(192);
+  });
+
+  test("a column payload is replaced by counts and never descended into", () => {
+    const digest = digestDef("PTableDef", {
+      src: {
+        type: "column",
+        column: {
+          id: "c",
+          spec: { kind: "PColumn", name: "c", valueType: "Int", axesSpec: [] },
+          data: {
+            type: "ParquetPartitioned",
+            partitionKeyLength: 1,
+            parts: {
+              [`["${SECRET}"]`]: { data: SECRET, stats: { numberOfRows: 7, size: { column: 3 } } },
+            },
+          },
+        },
+      },
+      partitionFilters: [],
+      filters: [],
+      sorting: [],
+    });
+    expect(JSON.stringify(digest)).not.toContain(SECRET);
+    const data = (digest.def as { src: { column: { data: Record<string, unknown> } } }).src.column
+      .data;
+    expect(data).toMatchObject({ kind: "ParquetPartitioned", parts: 1, rows: 7 });
+  });
+
+  test("an inline payload is reduced to a count and a sampled size", () => {
+    const values = Array.from({ length: 1000 }, (_, i) => ({ key: [`k${i}`], value: SECRET }));
+    const { value } = redact({ data: values });
+    const data = (value as { data: Record<string, unknown> }).data;
+    expect(JSON.stringify(data)).not.toContain(SECRET);
+    expect(data.kind).toBe("inline");
+    expect(data.entries).toBe(1000);
+    expect(data.approxBytes as number).toBeGreaterThan(0);
+  });
+
+  test("a deep definition is cut rather than followed forever", () => {
+    let node: Record<string, unknown> = { type: "leaf" };
+    for (let i = 0; i < 50; i++) node = { type: "wrap", input: node };
+    const { value, stats } = redact(node, { maxDepth: 6 });
+    expect(stats.depthCapped).toBeGreaterThan(0);
+    expect(JSON.stringify(value)).toContain("$depth");
+  });
+
+  test("the node budget bounds one record", () => {
+    const wide = { entries: Array.from({ length: 500 }, (_, i) => ({ type: "column", i })) };
+    const { stats } = redact(wide, { maxNodes: 50, maxArrayItems: 500 });
+    expect(stats.budgetExhausted).toBe(true);
+  });
+});