@milaboratories/pl-crash-recorder 0.3.0

package/dist/session.js

@@ -0,0 +1,50 @@
+import { openRecorder, startSelfSampler } from "./recorder.js";
+import { startMemorySampler } from "./sampler.js";
+import { createHandleRegistry } from "./instrument.js";
+//#region src/session.ts
+/** Environment variable naming the directory crash logs are written to. */
+const CRASH_DIR_ENV = "MI_CRASH_RECORDER_DIR";
+/**
+* Environment variable carrying the session id a supervising parent assigned.
+* Set it alongside {@link CRASH_DIR_ENV} when spawning the worker and pass the
+* same id to `superviseWorker`.
+*/
+const CRASH_SESSION_ENV = "MI_CRASH_RECORDER_SESSION";
+/**
+* Opens a recording session, or returns undefined when recording is not enabled.
+*
+* Recording is opt-in for now: it appends synchronously on every recorded
+* operation, and that cost has not been measured against a real project, so it
+* is switched on by pointing {@link CRASH_DIR_ENV} at a directory rather than
+* being on by default.
+*/
+function openRecordingSession(options = {}) {
+	const dir = options.dir ?? process.env["MI_CRASH_RECORDER_DIR"];
+	if (!dir) return void 0;
+	const recorder = openRecorder({
+		dir,
+		role: options.role,
+		meta: options.meta,
+		sessionId: options.sessionId ?? process.env["MI_CRASH_RECORDER_SESSION"] ?? void 0
+	});
+	const sampler = startMemorySampler({
+		dir,
+		sessionId: recorder.sessionId,
+		intervalMs: options.samplerIntervalMs
+	});
+	const stopSelfSampler = startSelfSampler(recorder, options.selfSamplerIntervalMs);
+	return {
+		recorder,
+		sampler,
+		registry: createHandleRegistry(),
+		close(reason) {
+			stopSelfSampler();
+			sampler.stop();
+			recorder.close(reason);
+		}
+	};
+}
+//#endregion
+export { CRASH_DIR_ENV, CRASH_SESSION_ENV, openRecordingSession };
+
+//# sourceMappingURL=session.js.map

package/dist/session.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"session.js","names":[],"sources":["../src/session.ts"],"sourcesContent":["import { openRecorder, startSelfSampler, type Recorder } from \"./recorder\";\nimport { startMemorySampler, type MemorySampler } from \"./sampler\";\nimport { createHandleRegistry, type HandleRegistry } from \"./instrument\";\n\n/** Environment variable naming the directory crash logs are written to. */\nexport const CRASH_DIR_ENV = \"MI_CRASH_RECORDER_DIR\";\n\n/**\n * Environment variable carrying the session id a supervising parent assigned.\n * Set it alongside {@link CRASH_DIR_ENV} when spawning the worker and pass the\n * same id to `superviseWorker`.\n */\nexport const CRASH_SESSION_ENV = \"MI_CRASH_RECORDER_SESSION\";\n\nexport type RecordingSessionOptions = {\n  /** Overrides the directory from the environment. */\n  dir?: string;\n  /** Overrides the session id from the environment. */\n  sessionId?: string;\n  role?: string;\n  meta?: Record<string, unknown>;\n  samplerIntervalMs?: number;\n  selfSamplerIntervalMs?: number;\n};\n\nexport type RecordingSession = {\n  readonly recorder: Recorder;\n  readonly sampler: MemorySampler;\n  /** Shared so create calls and later data calls agree on handle identity. */\n  readonly registry: HandleRegistry;\n  close(reason?: string): void;\n};\n\n/**\n * Opens a recording session, or returns undefined when recording is not enabled.\n *\n * Recording is opt-in for now: it appends synchronously on every recorded\n * operation, and that cost has not been measured against a real project, so it\n * is switched on by pointing {@link CRASH_DIR_ENV} at a directory rather than\n * being on by default.\n */\nexport function openRecordingSession(\n  options: RecordingSessionOptions = {},\n): RecordingSession | undefined {\n  const dir = options.dir ?? process.env[CRASH_DIR_ENV];\n  if (!dir) return undefined;\n\n  const recorder = openRecorder({\n    dir,\n    role: options.role,\n    meta: options.meta,\n    sessionId: options.sessionId ?? process.env[CRASH_SESSION_ENV] ?? undefined,\n  });\n  const sampler = startMemorySampler({\n    dir,\n    sessionId: recorder.sessionId,\n    intervalMs: options.samplerIntervalMs,\n  });\n  const stopSelfSampler = startSelfSampler(recorder, options.selfSamplerIntervalMs);\n\n  return {\n    recorder,\n    sampler,\n    registry: createHandleRegistry(),\n    close(reason) {\n      stopSelfSampler();\n      sampler.stop();\n      recorder.close(reason);\n    },\n  };\n}\n"],"mappings":";;;;;AAKA,MAAa,gBAAgB;;;;;;AAO7B,MAAa,oBAAoB;;;;;;;;;AA6BjC,SAAgB,qBACd,UAAmC,CAAC,GACN;CAC9B,MAAM,MAAM,QAAQ,OAAO,QAAQ,IAAA;CACnC,IAAI,CAAC,KAAK,OAAO,KAAA;CAEjB,MAAM,WAAW,aAAa;EAC5B;EACA,MAAM,QAAQ;EACd,MAAM,QAAQ;EACd,WAAW,QAAQ,aAAa,QAAQ,IAAA,gCAA0B,KAAA;CACpE,CAAC;CACD,MAAM,UAAU,mBAAmB;EACjC;EACA,WAAW,SAAS;EACpB,YAAY,QAAQ;CACtB,CAAC;CACD,MAAM,kBAAkB,iBAAiB,UAAU,QAAQ,qBAAqB;CAEhF,OAAO;EACL;EACA;EACA,UAAU,qBAAqB;EAC/B,MAAM,QAAQ;GACZ,gBAAgB;GAChB,QAAQ,KAAK;GACb,SAAS,MAAM,MAAM;EACvB;CACF;AACF"}

package/dist/supervisor.d.ts

@@ -0,0 +1,37 @@
+import { CrashMarker } from "./events.js";
+//#region src/supervisor.d.ts
+export type SuperviseOptions = {
+  /**
+   * The session id handed to the worker at spawn (see `CRASH_SESSION_ENV`).
+   * With it the marker names the dying session with certainty. Without it the
+   * analyzer has to attribute the marker by timing, and will decline to
+   * attribute it at all when more than one session looks dead.
+   */
+  sessionId?: string;
+  onCrash?: (info: {
+    kind: "error" | "exit";
+    markerFile: string;
+    error?: unknown;
+    code?: number;
+  }) => void;
+};
+/** Minimal view of a worker, so callers are not forced to import worker_threads. */
+export type SupervisedWorker = {
+  on(event: "error", listener: (error: Error) => void): unknown;
+  on(event: "exit", listener: (code: number) => void): unknown;
+};
+/** Crash markers in a directory, oldest first. */
+export declare function readCrashMarkers(dir: string): CrashMarker[];
+/**
+ * Attaches crash recording to a middle-layer worker thread.
+ *
+ * A worker whose isolate exhausts its heap dies alone and the parent receives
+ * `ERR_WORKER_OUT_OF_MEMORY`, with or without `resourceLimits`. What
+ * `resourceLimits.maxOldGenerationSizeMb` adds is a chosen ceiling: V8's default
+ * is several gigabytes, so on a small machine the OS can run out of memory and
+ * kill the whole process before V8 ever reports the worker's heap as full — and
+ * then there is no parent left to write anything.
+ */
+export declare function superviseWorker(worker: SupervisedWorker, dir: string, options?: SuperviseOptions): void;
+//#endregion
+//# sourceMappingURL=supervisor.d.ts.map

package/dist/supervisor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"supervisor.d.ts","names":[],"sources":["../src/supervisor.ts"],"mappings":";;YAgBY;;;;;;;EAOV;EACA,WAAW;IACT;IACA;IACA;IACA;;;;YAKQ;EACV,GAAG,gBAAgB,WAAW,OAAO;EACrC,GAAG,eAAe,WAAW;;;wBAwCf,iBAAiB,cAAc;;;;;;;;;;;wBA+B/B,gBACd,QAAQ,kBACR,aACA,UAAS"}

package/dist/supervisor.js

@@ -0,0 +1,138 @@
+import { DEATH_FILE_PREFIX } from "./events.js";
+import { listSessions, sessionIdFromFile } from "./recorder.js";
+import fs from "node:fs";
+import path from "node:path";
+import os from "node:os";
+//#region src/supervisor.ts
+/**
+* Records an abnormal end observed from outside the dying thread.
+*
+* A thread that runs out of heap cannot describe its own death: the last reading
+* it wrote predates the blow-up, and when the blow-up is synchronous no sampler
+* tick of its own lands either. The parent is the only place where the cause is
+* known rather than inferred — Node reports `ERR_WORKER_OUT_OF_MEMORY` to it —
+* so the parent writes the verdict down on the dead thread's behalf.
+*/
+function writeCrashMarker(dir, input = {}) {
+	fs.mkdirSync(dir, { recursive: true });
+	const error = input.error;
+	const marker = {
+		type: "external-crash",
+		wall: Date.now(),
+		sessionId: input.sessionId,
+		guessedSessionId: input.sessionId === void 0 ? newestOpenSessionId(dir) : void 0,
+		reason: input.reason ?? classifyReason(input),
+		errorCode: error?.code,
+		errorName: error?.name,
+		message: truncate(String(error?.message ?? input.error ?? ""), 2e3),
+		exitCode: input.code,
+		signal: input.signal,
+		stderrTail: truncate(input.stderrTail ?? "", 4e3),
+		memoryAtDeath: memoryNow()
+	};
+	const file = path.join(dir, `${DEATH_FILE_PREFIX}-${marker.wall}.ndjson`);
+	fs.writeFileSync(file, `${JSON.stringify(marker)}\n`);
+	return file;
+}
+/** Crash markers in a directory, oldest first. */
+function readCrashMarkers(dir) {
+	let names;
+	try {
+		names = fs.readdirSync(dir);
+	} catch {
+		return [];
+	}
+	const markers = [];
+	for (const name of names) {
+		if (!name.startsWith(`death-`) || !name.endsWith(".ndjson")) continue;
+		try {
+			const first = fs.readFileSync(path.join(dir, name), "utf8").split("\n")[0];
+			markers.push(JSON.parse(first));
+		} catch {}
+	}
+	return markers.sort((lhs, rhs) => lhs.wall - rhs.wall);
+}
+/**
+* Attaches crash recording to a middle-layer worker thread.
+*
+* A worker whose isolate exhausts its heap dies alone and the parent receives
+* `ERR_WORKER_OUT_OF_MEMORY`, with or without `resourceLimits`. What
+* `resourceLimits.maxOldGenerationSizeMb` adds is a chosen ceiling: V8's default
+* is several gigabytes, so on a small machine the OS can run out of memory and
+* kill the whole process before V8 ever reports the worker's heap as full — and
+* then there is no parent left to write anything.
+*/
+function superviseWorker(worker, dir, options = {}) {
+	let recorded = false;
+	worker.on("error", (error) => {
+		recorded = true;
+		const markerFile = writeCrashMarker(dir, {
+			error,
+			sessionId: options.sessionId
+		});
+		options.onCrash?.({
+			kind: "error",
+			error,
+			markerFile
+		});
+	});
+	worker.on("exit", (code) => {
+		if (code === 0 || recorded) return;
+		const markerFile = writeCrashMarker(dir, {
+			reason: "worker-exit",
+			code,
+			sessionId: options.sessionId
+		});
+		options.onCrash?.({
+			kind: "exit",
+			code,
+			markerFile
+		});
+	});
+}
+/**
+* The parent's view of memory at the moment it saw the death.
+*
+* The dying thread cannot take this reading, and the sampler's last one predates
+* the end by up to its interval. Taken here it is contemporaneous with the exit
+* code it sits beside, which is what stops an exhausted machine from reading as
+* an ordinary failure.
+*
+* Every reading here is a syscall. The machine's compressor and swap totals are
+* deliberately not among them: on macOS they cost a subprocess, and this runs on
+* the parent's event loop inside the worker's error handler, before the marker
+* is written and before the caller learns of the death. A fork is exactly what
+* becomes slow or impossible on the exhausted machine this code exists for, so
+* the fuller picture is left to the sampler, whose last reading is at most one
+* interval old and sits in the same bundle.
+*/
+function memoryNow() {
+	try {
+		return {
+			rss: process.memoryUsage.rss(),
+			maxRss: process.resourceUsage().maxRSS * 1024,
+			freeMemory: os.freemem(),
+			totalMemory: os.totalmem()
+		};
+	} catch {
+		return;
+	}
+}
+function newestOpenSessionId(dir) {
+	const open = listSessions(dir).find((session) => session.crashed);
+	return open ? sessionIdFromFile(open.file) : void 0;
+}
+function classifyReason({ error, code, signal }) {
+	if (error?.code === "ERR_WORKER_OUT_OF_MEMORY") return "js-heap-out-of-memory";
+	if (signal === "SIGKILL") return "killed-by-os";
+	if (signal === "SIGABRT" || code === 134) return "abort-or-fatal-allocation-failure";
+	if (typeof code === "number" && code !== 0) return "nonzero-exit";
+	return "unknown";
+}
+function truncate(value, limit) {
+	return value.length > limit ? `${value.slice(0, limit)}…` : value;
+}
+//#endregion
+export { readCrashMarkers, superviseWorker, writeCrashMarker };
+
+//# sourceMappingURL=supervisor.js.map

package/dist/supervisor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"supervisor.js","names":[],"sources":["../src/supervisor.ts"],"sourcesContent":["import fs from \"node:fs\";\nimport os from \"node:os\";\nimport path from \"node:path\";\nimport { DEATH_FILE_PREFIX, type CrashMarker, type CrashReason } from \"./events\";\nimport { listSessions, sessionIdFromFile } from \"./recorder\";\n\ntype CrashMarkerInput = {\n  /** Session id the parent assigned to the worker. Omitted, the marker carries no identity. */\n  sessionId?: string;\n  reason?: CrashReason;\n  error?: (Error & { code?: string }) | unknown;\n  code?: number;\n  signal?: string;\n  stderrTail?: string;\n};\n\nexport type SuperviseOptions = {\n  /**\n   * The session id handed to the worker at spawn (see `CRASH_SESSION_ENV`).\n   * With it the marker names the dying session with certainty. Without it the\n   * analyzer has to attribute the marker by timing, and will decline to\n   * attribute it at all when more than one session looks dead.\n   */\n  sessionId?: string;\n  onCrash?: (info: {\n    kind: \"error\" | \"exit\";\n    markerFile: string;\n    error?: unknown;\n    code?: number;\n  }) => void;\n};\n\n/** Minimal view of a worker, so callers are not forced to import worker_threads. */\nexport type SupervisedWorker = {\n  on(event: \"error\", listener: (error: Error) => void): unknown;\n  on(event: \"exit\", listener: (code: number) => void): unknown;\n};\n\n/**\n * Records an abnormal end observed from outside the dying thread.\n *\n * A thread that runs out of heap cannot describe its own death: the last reading\n * it wrote predates the blow-up, and when the blow-up is synchronous no sampler\n * tick of its own lands either. The parent is the only place where the cause is\n * known rather than inferred — Node reports `ERR_WORKER_OUT_OF_MEMORY` to it —\n * so the parent writes the verdict down on the dead thread's behalf.\n */\nexport function writeCrashMarker(dir: string, input: CrashMarkerInput = {}): string {\n  fs.mkdirSync(dir, { recursive: true });\n  const error = input.error as (Error & { code?: string }) | undefined;\n  // Only an id the parent handed to the worker is certain, and only a certain\n  // id goes in `sessionId`. Reading the newest open crash log names whichever\n  // session wrote last, which a concurrent live session makes wrong; recorded\n  // as identity that would misattribute the death and, worse, stop the session\n  // that actually died from claiming the marker. So it is advisory only.\n  const marker: CrashMarker = {\n    type: \"external-crash\",\n    wall: Date.now(),\n    sessionId: input.sessionId,\n    guessedSessionId: input.sessionId === undefined ? newestOpenSessionId(dir) : undefined,\n    reason: input.reason ?? classifyReason(input),\n    errorCode: error?.code,\n    errorName: error?.name,\n    message: truncate(String(error?.message ?? input.error ?? \"\"), 2000),\n    exitCode: input.code,\n    signal: input.signal,\n    stderrTail: truncate(input.stderrTail ?? \"\", 4000),\n    memoryAtDeath: memoryNow(),\n  };\n  const file = path.join(dir, `${DEATH_FILE_PREFIX}-${marker.wall}.ndjson`);\n  fs.writeFileSync(file, `${JSON.stringify(marker)}\\n`);\n  return file;\n}\n\n/** Crash markers in a directory, oldest first. */\nexport function readCrashMarkers(dir: string): CrashMarker[] {\n  let names: string[];\n  try {\n    names = fs.readdirSync(dir);\n  } catch {\n    return [];\n  }\n  const markers: CrashMarker[] = [];\n  for (const name of names) {\n    if (!name.startsWith(`${DEATH_FILE_PREFIX}-`) || !name.endsWith(\".ndjson\")) continue;\n    try {\n      const first = fs.readFileSync(path.join(dir, name), \"utf8\").split(\"\\n\")[0];\n      markers.push(JSON.parse(first) as CrashMarker);\n    } catch {\n      // A marker that cannot be parsed is skipped; it is one line of evidence,\n      // not the report.\n    }\n  }\n  return markers.sort((lhs, rhs) => lhs.wall - rhs.wall);\n}\n\n/**\n * Attaches crash recording to a middle-layer worker thread.\n *\n * A worker whose isolate exhausts its heap dies alone and the parent receives\n * `ERR_WORKER_OUT_OF_MEMORY`, with or without `resourceLimits`. What\n * `resourceLimits.maxOldGenerationSizeMb` adds is a chosen ceiling: V8's default\n * is several gigabytes, so on a small machine the OS can run out of memory and\n * kill the whole process before V8 ever reports the worker's heap as full — and\n * then there is no parent left to write anything.\n */\nexport function superviseWorker(\n  worker: SupervisedWorker,\n  dir: string,\n  options: SuperviseOptions = {},\n): void {\n  // One death fires `error` and then `exit`. Only `error` carries the cause, so\n  // a later `exit` must not overwrite it with a bare exit code.\n  let recorded = false;\n  worker.on(\"error\", (error: Error) => {\n    recorded = true;\n    const markerFile = writeCrashMarker(dir, { error, sessionId: options.sessionId });\n    options.onCrash?.({ kind: \"error\", error, markerFile });\n  });\n  worker.on(\"exit\", (code: number) => {\n    if (code === 0 || recorded) return;\n    const markerFile = writeCrashMarker(dir, {\n      reason: \"worker-exit\",\n      code,\n      sessionId: options.sessionId,\n    });\n    options.onCrash?.({ kind: \"exit\", code, markerFile });\n  });\n}\n\n// Internals\n\n/**\n * The parent's view of memory at the moment it saw the death.\n *\n * The dying thread cannot take this reading, and the sampler's last one predates\n * the end by up to its interval. Taken here it is contemporaneous with the exit\n * code it sits beside, which is what stops an exhausted machine from reading as\n * an ordinary failure.\n *\n * Every reading here is a syscall. The machine's compressor and swap totals are\n * deliberately not among them: on macOS they cost a subprocess, and this runs on\n * the parent's event loop inside the worker's error handler, before the marker\n * is written and before the caller learns of the death. A fork is exactly what\n * becomes slow or impossible on the exhausted machine this code exists for, so\n * the fuller picture is left to the sampler, whose last reading is at most one\n * interval old and sits in the same bundle.\n */\nfunction memoryNow(): CrashMarker[\"memoryAtDeath\"] {\n  try {\n    return {\n      rss: process.memoryUsage.rss(),\n      maxRss: process.resourceUsage().maxRSS * 1024,\n      freeMemory: os.freemem(),\n      totalMemory: os.totalmem(),\n    };\n  } catch {\n    // A marker without memory is still a marker; failing to take the reading\n    // must never cost the record of the death itself.\n    return undefined;\n  }\n}\n\n// Advisory only, for a human reading a directory by hand: the dying session has\n// no terminating record, so among the sessions that look dead this names the one\n// that wrote last. Never used as identity — see `CrashMarker.guessedSessionId`.\nfunction newestOpenSessionId(dir: string): string | undefined {\n  const open = listSessions(dir).find((session) => session.crashed);\n  return open ? sessionIdFromFile(open.file) : undefined;\n}\n\nfunction classifyReason({ error, code, signal }: CrashMarkerInput): CrashReason {\n  const errorCode = (error as { code?: string } | undefined)?.code;\n  if (errorCode === \"ERR_WORKER_OUT_OF_MEMORY\") return \"js-heap-out-of-memory\";\n  if (signal === \"SIGKILL\") return \"killed-by-os\";\n  if (signal === \"SIGABRT\" || code === 134) return \"abort-or-fatal-allocation-failure\";\n  if (typeof code === \"number\" && code !== 0) return \"nonzero-exit\";\n  return \"unknown\";\n}\n\nfunction truncate(value: string, limit: number): string {\n  return value.length > limit ? `${value.slice(0, limit)}…` : value;\n}\n"],"mappings":";;;;;;;;;;;;;;;AA+CA,SAAgB,iBAAiB,KAAa,QAA0B,CAAC,GAAW;CAClF,GAAG,UAAU,KAAK,EAAE,WAAW,KAAK,CAAC;CACrC,MAAM,QAAQ,MAAM;CAMpB,MAAM,SAAsB;EAC1B,MAAM;EACN,MAAM,KAAK,IAAI;EACf,WAAW,MAAM;EACjB,kBAAkB,MAAM,cAAc,KAAA,IAAY,oBAAoB,GAAG,IAAI,KAAA;EAC7E,QAAQ,MAAM,UAAU,eAAe,KAAK;EAC5C,WAAW,OAAO;EAClB,WAAW,OAAO;EAClB,SAAS,SAAS,OAAO,OAAO,WAAW,MAAM,SAAS,EAAE,GAAG,GAAI;EACnE,UAAU,MAAM;EAChB,QAAQ,MAAM;EACd,YAAY,SAAS,MAAM,cAAc,IAAI,GAAI;EACjD,eAAe,UAAU;CAC3B;CACA,MAAM,OAAO,KAAK,KAAK,KAAK,GAAG,kBAAkB,GAAG,OAAO,KAAK,QAAQ;CACxE,GAAG,cAAc,MAAM,GAAG,KAAK,UAAU,MAAM,EAAE,GAAG;CACpD,OAAO;AACT;;AAGA,SAAgB,iBAAiB,KAA4B;CAC3D,IAAI;CACJ,IAAI;EACF,QAAQ,GAAG,YAAY,GAAG;CAC5B,QAAQ;EACN,OAAO,CAAC;CACV;CACA,MAAM,UAAyB,CAAC;CAChC,KAAK,MAAM,QAAQ,OAAO;EACxB,IAAI,CAAC,KAAK,WAAW,QAAuB,KAAK,CAAC,KAAK,SAAS,SAAS,GAAG;EAC5E,IAAI;GACF,MAAM,QAAQ,GAAG,aAAa,KAAK,KAAK,KAAK,IAAI,GAAG,MAAM,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC;GACxE,QAAQ,KAAK,KAAK,MAAM,KAAK,CAAgB;EAC/C,QAAQ,CAGR;CACF;CACA,OAAO,QAAQ,MAAM,KAAK,QAAQ,IAAI,OAAO,IAAI,IAAI;AACvD;;;;;;;;;;;AAYA,SAAgB,gBACd,QACA,KACA,UAA4B,CAAC,GACvB;CAGN,IAAI,WAAW;CACf,OAAO,GAAG,UAAU,UAAiB;EACnC,WAAW;EACX,MAAM,aAAa,iBAAiB,KAAK;GAAE;GAAO,WAAW,QAAQ;EAAU,CAAC;EAChF,QAAQ,UAAU;GAAE,MAAM;GAAS;GAAO;EAAW,CAAC;CACxD,CAAC;CACD,OAAO,GAAG,SAAS,SAAiB;EAClC,IAAI,SAAS,KAAK,UAAU;EAC5B,MAAM,aAAa,iBAAiB,KAAK;GACvC,QAAQ;GACR;GACA,WAAW,QAAQ;EACrB,CAAC;EACD,QAAQ,UAAU;GAAE,MAAM;GAAQ;GAAM;EAAW,CAAC;CACtD,CAAC;AACH;;;;;;;;;;;;;;;;;AAoBA,SAAS,YAA0C;CACjD,IAAI;EACF,OAAO;GACL,KAAK,QAAQ,YAAY,IAAI;GAC7B,QAAQ,QAAQ,cAAc,CAAC,CAAC,SAAS;GACzC,YAAY,GAAG,QAAQ;GACvB,aAAa,GAAG,SAAS;EAC3B;CACF,QAAQ;EAGN;CACF;AACF;AAKA,SAAS,oBAAoB,KAAiC;CAC5D,MAAM,OAAO,aAAa,GAAG,CAAC,CAAC,MAAM,YAAY,QAAQ,OAAO;CAChE,OAAO,OAAO,kBAAkB,KAAK,IAAI,IAAI,KAAA;AAC/C;AAEA,SAAS,eAAe,EAAE,OAAO,MAAM,UAAyC;CAE9E,IADmB,OAAyC,SAC1C,4BAA4B,OAAO;CACrD,IAAI,WAAW,WAAW,OAAO;CACjC,IAAI,WAAW,aAAa,SAAS,KAAK,OAAO;CACjD,IAAI,OAAO,SAAS,YAAY,SAAS,GAAG,OAAO;CACnD,OAAO;AACT;AAEA,SAAS,SAAS,OAAe,OAAuB;CACtD,OAAO,MAAM,SAAS,QAAQ,GAAG,MAAM,MAAM,GAAG,KAAK,EAAE,KAAK;AAC9D"}

package/package.json

@@ -0,0 +1,43 @@
+{
+  "name": "@milaboratories/pl-crash-recorder",
+  "version": "0.3.0",
+  "description": "Crash-survivable diagnostics for the block model layer: records join shapes and memory, and explains an out-of-memory death after the fact",
+  "keywords": [],
+  "license": "UNLICENSED",
+  "files": [
+    "./dist/**/*",
+    "./src/**/*"
+  ],
+  "type": "module",
+  "main": "./dist/index.js",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "dependencies": {
+    "@milaboratories/pl-model-common": "1.49.0"
+  },
+  "devDependencies": {
+    "@vitest/coverage-istanbul": "^4.1.3",
+    "typescript": "7.0.2",
+    "vitest": "^4.1.3",
+    "@milaboratories/build-configs": "2.0.1",
+    "@milaboratories/ts-configs": "1.4.0",
+    "@milaboratories/ts-builder": "1.7.2"
+  },
+  "scripts": {
+    "build": "ts-builder build --target node --build-config build.config.js",
+    "watch": "ts-builder build --target node --build-config build.config.js --watch",
+    "check": "ts-builder check --target node",
+    "formatter:check": "ts-builder formatter --check",
+    "linter:check": "ts-builder linter --check",
+    "types:check": "ts-builder type-check --target node",
+    "test": "vitest run --coverage",
+    "do-pack": "rm -f *.tgz && pnpm pack && mv *.tgz package.tgz",
+    "fmt": "ts-builder format"
+  }
+}

package/src/data_summary.ts

@@ -0,0 +1,163 @@
+/**
+ * Counts and sizes for a column payload, never the payload.
+ *
+ * This is the one place that has to know the shape of `DataInfo`, because the
+ * numbers that predict a join's cost — rows per partition and their byte sizes —
+ * live at type-specific positions inside it. Everything else about a definition
+ * is recorded structurally.
+ *
+ * Chunk statistics are optional: the producing workflow fills them in, so row
+ * counts are reported when present and left unknown otherwise rather than
+ * guessed.
+ */
+
+export type DataSummary = {
+  kind: string;
+  /** Entries for inline or JSON payloads. */
+  entries?: number;
+  approxBytes?: number;
+  keyLength?: number;
+  partitionKeyLength?: number;
+  parts?: number;
+  partsWithStats?: number;
+  rows?: number;
+  bytes?: number;
+  /**
+   * Distinct values per axis, in the order the column declares its axes.
+   *
+   * Counted independently, so multiplying them gives an upper bound on the
+   * distinct key tuples rather than their number: axes are usually correlated,
+   * and a product invents combinations that never occur. Use `distinctKeys`
+   * where the key in question is the whole tuple.
+   */
+  axisCardinality?: number[];
+  /**
+   * Distinct whole key tuples, which is exact where a join keys on every axis
+   * the column has — the ordinary case for two columns sharing their key.
+   */
+  distinctKeys?: number;
+  /** Set when the entries were too many to count distinct keys over. */
+  axisCardinalityUncounted?: boolean;
+};
+
+export function summarizeData(data: unknown): DataSummary {
+  if (data === null || data === undefined) return { kind: "absent" };
+  if (Array.isArray(data)) {
+    // Inline values, built inside the model sandbox.
+    return {
+      kind: "inline",
+      entries: data.length,
+      approxBytes: approxInlineBytes(data),
+      ...inlineAxisCardinality(data),
+    };
+  }
+  if (typeof data !== "object") return { kind: typeof data };
+
+  const info = data as { type?: string; [key: string]: unknown };
+  switch (info.type) {
+    case "Json":
+      return {
+        kind: "Json",
+        keyLength: numberOr(info.keyLength),
+        entries: countKeys(info.data),
+      };
+    case "JsonPartitioned":
+    case "BinaryPartitioned":
+      return {
+        kind: info.type,
+        partitionKeyLength: numberOr(info.partitionKeyLength),
+        parts: countKeys(info.parts),
+      };
+    case "ParquetPartitioned":
+      return summarizeParquet(info);
+    default:
+      return { kind: info.type ?? opaqueKind(data) };
+  }
+}
+
+// Internals
+
+function summarizeParquet(info: { [key: string]: unknown }): DataSummary {
+  const parts = Object.values((info.parts ?? {}) as Record<string, unknown>);
+  let rows = 0;
+  let bytes = 0;
+  let withStats = 0;
+  for (const part of parts) {
+    const stats = (
+      part as { stats?: { numberOfRows?: number; size?: { axes?: number[]; column?: number } } }
+    )?.stats;
+    if (!stats) continue;
+    withStats++;
+    if (typeof stats.numberOfRows === "number") rows += stats.numberOfRows;
+    if (stats.size) bytes += (stats.size.column ?? 0) + sum(stats.size.axes ?? []);
+  }
+  return {
+    kind: "ParquetPartitioned",
+    partitionKeyLength: numberOr(info.partitionKeyLength),
+    parts: parts.length,
+    partsWithStats: withStats,
+    rows: withStats > 0 ? rows : undefined,
+    bytes: withStats > 0 ? bytes : undefined,
+  };
+}
+
+/**
+ * How many entries may be walked to count distinct axis keys.
+ *
+ * Counting is exact and needs a set per axis, so it costs memory in proportion
+ * to the distinct keys it finds — which is the wrong thing to spend in the
+ * situation this code exists to diagnose. Past the cap the count is declined
+ * rather than approximated, so a number that is present is always true.
+ */
+const CARDINALITY_LIMIT = 100_000;
+
+function inlineAxisCardinality(values: unknown[]): {
+  axisCardinality?: number[];
+  distinctKeys?: number;
+  axisCardinalityUncounted?: boolean;
+} {
+  if (values.length > CARDINALITY_LIMIT) return { axisCardinalityUncounted: true };
+  const firstKey = (values[0] as { key?: unknown } | undefined)?.key;
+  if (!Array.isArray(firstKey)) return {};
+
+  const perAxis = firstKey.map(() => new Set<unknown>());
+  // Counted alongside the per-axis sets rather than derived from them: the two
+  // are equal only when the axes vary independently, which they rarely do.
+  const tuples = new Set<string>();
+  for (const entry of values) {
+    const key = (entry as { key?: unknown }).key;
+    if (!Array.isArray(key) || key.length !== perAxis.length) return {};
+    for (const [axis, value] of key.entries()) perAxis[axis].add(value);
+    tuples.add(key.map((value) => String(value)).join("\u0000"));
+  }
+  return { axisCardinality: perAxis.map((set) => set.size), distinctKeys: tuples.size };
+}
+
+// Sampled rather than measured: walking millions of entries to size them is
+// itself a memory risk in the situation this code exists to diagnose.
+function approxInlineBytes(values: unknown[]): number {
+  const sampleSize = Math.min(values.length, 64);
+  if (sampleSize === 0) return 0;
+  let bytes = 0;
+  for (let i = 0; i < sampleSize; i++) {
+    const value = values[Math.floor((i * values.length) / sampleSize)];
+    bytes += JSON.stringify(value ?? null)?.length ?? 0;
+  }
+  return Math.round((bytes / sampleSize) * values.length);
+}
+
+function countKeys(value: unknown): number | undefined {
+  return value && typeof value === "object" ? Object.keys(value).length : undefined;
+}
+
+function numberOr(value: unknown): number | undefined {
+  return typeof value === "number" ? value : undefined;
+}
+
+function opaqueKind(value: object): string {
+  return (value as { constructor?: { name?: string } }).constructor?.name ?? "opaque";
+}
+
+function sum(values: number[]): number {
+  return values.reduce((acc, value) => acc + value, 0);
+}

package/src/digest.ts

@@ -0,0 +1,36 @@
+import { redact, type RedactionStats } from "./redact";
+
+/**
+ * What a definition record carries.
+ *
+ * The definition is recorded structurally (see `redact`) rather than through a
+ * hand-written digest per definition type. Structural rules are not run here:
+ * they belong to the analyzer, so nothing is computed on the hot path and the
+ * rules can be revised against logs that already exist.
+ */
+
+export type DefKind = "PTableDef" | "PTableDefV2" | "PFrameDef";
+
+export type DefDigest = {
+  kind: DefKind;
+  /** Redacted definition, same shape as the original. */
+  def: unknown;
+  redaction: RedactionStats & { bytes: number };
+};
+
+/** Records one definition: redacted, measured, and tagged with its API shape. */
+export function digestDef(kind: DefKind, def: unknown): DefDigest {
+  const { value, stats } = redact(def);
+  const json = safeLength(value);
+  return { kind, def: value, redaction: { ...stats, bytes: json } };
+}
+
+// Internals
+
+function safeLength(value: unknown): number {
+  try {
+    return JSON.stringify(value)?.length ?? 0;
+  } catch {
+    return 0;
+  }
+}

package/src/events.ts

@@ -0,0 +1,166 @@
+/**
+ * Record types written to a crash log.
+ *
+ * The log is append-only NDJSON, one record per line, and is read back by
+ * tooling that may be older or newer than the writer, so every field beyond
+ * {@link LogRecordBase} is optional and unknown record types are skipped
+ * rather than rejected.
+ */
+
+/** Memory reading taken by the thread that wrote the record. */
+export type MemorySnapshot = {
+  /** Resident set size of the whole process. */
+  rss: number;
+  /** Heap in use by the writing thread's isolate. */
+  heapUsed: number;
+  heapTotal: number;
+  external: number;
+  arrayBuffers: number;
+  /** V8 heap ceiling for the writing thread's isolate. */
+  heapLimit: number;
+};
+
+type LogRecordBase = {
+  /** Monotonically increasing within one session; used to pair begin with end. */
+  seq: number;
+  /** Milliseconds since process start, for durations. */
+  t: number;
+  /** Wall clock, for correlating with the sampler series and crash markers. */
+  wall: number;
+  type: string;
+};
+
+export type LogRecord = LogRecordBase & {
+  mem?: MemorySnapshot;
+  /** Sequence number of the matching begin record, on end and error records. */
+  begin?: number;
+  [key: string]: unknown;
+};
+
+/** Session header, always the first record. */
+export type SessionEnvironment = {
+  node: string;
+  platform: string;
+  cpus: number;
+  totalMemory: number;
+  heapLimit: number;
+  execArgv: string[];
+  maxOldSpaceSize?: number;
+};
+
+/** Written by the sampler thread to its own sibling file. */
+export type SamplerRecord = LogRecordBase & {
+  type: "mem-sampler";
+  rss: number;
+  peakRss: number;
+  /** Highest resident size the kernel has seen for this process. */
+  maxRss?: number;
+  freeMemory: number;
+  totalMemory: number;
+  /** Where the machine's memory actually is, refreshed less often than `rss`. */
+  machine?: MachineMemory;
+};
+
+/**
+ * A reading only the application process can take.
+ *
+ * Resident size falls when the OS compresses or pages a process out, so it
+ * cannot say whether the memory was released or merely moved. Private bytes can:
+ * they are unshared and stay committed until the process actually gives them
+ * back. That is the difference between "our process is holding this" and "the
+ * machine is short of memory for some other reason", which nothing else here
+ * distinguishes.
+ */
+export type HostRecord = LogRecordBase & {
+  type: "mem-host";
+  /** Unshared, still-committed bytes of the process hosting the middle layer. */
+  private?: number;
+  /** What the machine has paged out, where the platform reports it. */
+  swapUsed?: number;
+  swapTotal?: number;
+  /** The same figure per process, where the host can enumerate its own. */
+  processes?: { pid: number; name?: string; private?: number }[];
+};
+
+/**
+ * The machine's own account of its memory.
+ *
+ * Resident size is not the whole story on a machine under pressure: macOS moves
+ * pages out of a process's resident set into the compressor, and both Unixes
+ * swap, so a process can appear to shrink while the memory it asked for is still
+ * held. These readings are what a resident-size curve has to be read against.
+ */
+export type MachineMemory = {
+  /** Bytes of process memory the compressor holds, counted before compression. */
+  compressedStored?: number;
+  /** Physical bytes the compressor itself occupies. */
+  compressedOccupied?: number;
+  swapUsed?: number;
+  swapTotal?: number;
+  anonymous?: number;
+  fileBacked?: number;
+  wired?: number;
+  /** Why the reading is missing, when it is. */
+  unavailable?: string;
+};
+
+/** Written by the parent when a supervised thread or process dies. */
+export type CrashMarker = {
+  type: "external-crash";
+  wall: number;
+  /**
+   * Session the marker belongs to. Present only when the parent assigned the id
+   * to the worker and therefore knows it; never inferred, because a wrong id
+   * here would both misattribute the death and stop the right session from
+   * claiming it.
+   */
+  sessionId?: string;
+  /**
+   * Advisory only: the newest open crash log at the moment of death. A
+   * concurrent live session can make this wrong, so it is never matched against
+   * — it exists to help a human read a directory by hand.
+   */
+  guessedSessionId?: string;
+  /** Written by an older recorder that put a guess in `sessionId`. */
+  sessionIdSource?: "assigned" | "guessed";
+  reason: CrashReason;
+  errorCode?: string;
+  errorName?: string;
+  message?: string;
+  exitCode?: number;
+  signal?: string;
+  stderrTail?: string;
+  /**
+   * What memory looked like when the death was observed, taken by the parent.
+   *
+   * Without it a marker reading `worker-exit` with exit code 1 is
+   * indistinguishable from an ordinary application error, and a reader who
+   * starts here would classify an exhausted machine as a bug in the code that
+   * happened to be running.
+   */
+  memoryAtDeath?: {
+    /** Resident size of the parent process, which hosts the dying worker. */
+    rss: number;
+    /** Highest resident size the kernel recorded for it. */
+    maxRss: number;
+    freeMemory: number;
+    totalMemory: number;
+  };
+};
+
+export type CrashReason =
+  | "js-heap-out-of-memory"
+  | "killed-by-os"
+  | "abort-or-fatal-allocation-failure"
+  | "worker-exit"
+  | "nonzero-exit"
+  | "unknown";
+
+export const SESSION_RECORD = "session";
+/** Earliest memory reading of a session, rewritten into every rotated segment. */
+export const MEM_BASELINE_RECORD = "mem-baseline";
+export const SESSION_END_RECORD = "session-end";
+export const HOST_FILE_PREFIX = "host";
+export const SESSION_FILE_PREFIX = "session";
+export const SAMPLER_FILE_PREFIX = "memory";
+export const DEATH_FILE_PREFIX = "death";