@milaboratories/pl-crash-recorder 0.3.0 → 0.3.2

package/dist/index.d.ts

@@ -3,5 +3,5 @@ import { Recorder, RecorderOptions, SessionFileInfo, listSessions, newSessionId,
 import { HandleRegistry, RenderInfo, createHandleRegistry, recordModelRenderSync, wrapDataDriver, wrapModelDriver } from "./instrument.js";
 import { CRASH_DIR_ENV, CRASH_SESSION_ENV, RecordingSession, RecordingSessionOptions, openRecordingSession } from "./session.js";
 import { HostReading, HostSampler, HostSamplerOptions, startHostSampler } from "./host_sampler.js";
-import { SuperviseOptions, SupervisedWorker, readCrashMarkers, superviseWorker } from "./supervisor.js";
-export { CRASH_DIR_ENV, CRASH_SESSION_ENV, type CrashMarker, type CrashReason, type HandleRegistry, type HostReading, type HostRecord, type HostSampler, type HostSamplerOptions, type LogRecord, type MachineMemory, type MemorySnapshot, type Recorder, type RecorderOptions, type RecordingSession, type RecordingSessionOptions, type RenderInfo, type SamplerRecord, type SessionEnvironment, type SessionFileInfo, type SuperviseOptions, type SupervisedWorker, createHandleRegistry, listSessions, newSessionId, openRecorder, openRecordingSession, readCrashMarkers, recordModelRenderSync, sessionIdFromFile, startHostSampler, superviseWorker, wrapDataDriver, wrapModelDriver };
+import { StoredCrashMarker, SuperviseOptions, SupervisedWorker, readCrashMarkers, superviseWorker } from "./supervisor.js";
+export { CRASH_DIR_ENV, CRASH_SESSION_ENV, type CrashMarker, type CrashReason, type HandleRegistry, type HostReading, type HostRecord, type HostSampler, type HostSamplerOptions, type LogRecord, type MachineMemory, type MemorySnapshot, type Recorder, type RecorderOptions, type RecordingSession, type RecordingSessionOptions, type RenderInfo, type SamplerRecord, type SessionEnvironment, type SessionFileInfo, type StoredCrashMarker, type SuperviseOptions, type SupervisedWorker, createHandleRegistry, listSessions, newSessionId, openRecorder, openRecordingSession, readCrashMarkers, recordModelRenderSync, sessionIdFromFile, startHostSampler, superviseWorker, wrapDataDriver, wrapModelDriver };

package/dist/machine_memory.js

@@ -13,11 +13,62 @@ function readMachineMemory() {
 	try {
 		if (process.platform === "darwin") return readDarwin();
 		if (process.platform === "linux") return readLinux();
-		return { unavailable: `not implemented for ${process.platform}` };
+		return { unavailable: machineMemoryUnsupported() };
 	} catch (error) {
 		return { unavailable: error instanceof Error ? error.message : String(error) };
 	}
 }
+/**
+* Decides what each sampler tick should record about machine-wide memory.
+*
+* Three things are being balanced. The reading costs a subprocess, so it is
+* taken on its own interval rather than with every sample. Repeating an
+* unchanged failure every second buries the resident-size curve it sits beside,
+* so a reason is written once and not again while it still holds. And a failure
+* is not the end of the matter: a `vm_stat` that timed out did so under load,
+* which is precisely the moment its figures are worth having a second later —
+* so the source keeps being tried, and a reading that comes back is recorded.
+*
+* Only an unsupported platform stops it for good, because that is the one
+* condition a running process cannot get out of.
+*/
+function createMachineMemoryReader(options) {
+	const read = options.read ?? readMachineMemory;
+	const unsupported = (options.unsupportedReason ?? machineMemoryUnsupportedReason)();
+	let pending = unsupported;
+	let dueAt = 0;
+	let statedFailure;
+	return (now) => {
+		if (pending !== void 0) {
+			const reason = pending;
+			pending = void 0;
+			return { unavailable: reason };
+		}
+		if (unsupported !== void 0 || now < dueAt) return void 0;
+		dueAt = now + options.intervalMs;
+		const reading = read();
+		if (reading.unavailable === void 0) {
+			statedFailure = void 0;
+			return reading;
+		}
+		if (reading.unavailable === statedFailure) return void 0;
+		statedFailure = reading.unavailable;
+		return reading;
+	};
+}
+/**
+* Names what the machine-wide reading would have contributed, for platforms
+* where it cannot be taken at all.
+*
+* A reader who finds no compressor or swap figures needs to know whether the
+* machine had none or the sampler never asked, and where the equivalent evidence
+* is instead. Callers record it once: it is a property of the platform and
+* repeating it every second buries the curve it was meant to explain.
+*/
+function machineMemoryUnsupportedReason() {
+	if (process.platform === "darwin" || process.platform === "linux") return void 0;
+	return machineMemoryUnsupported();
+}
 function readDarwin() {
 	const stat = execFileSync("vm_stat", {
 		encoding: "utf8",
@@ -62,7 +113,10 @@ function readLinux() {
 		wired: kb("Unevictable")
 	};
 }
+function machineMemoryUnsupported() {
+	return `no machine-wide memory source on ${process.platform}: compressor, swap and anonymous/file-backed totals are not sampled (vm_stat and sysctl are macOS-only, /proc/meminfo Linux-only). Per-process committed bytes are recorded instead, as \`private\` in the host log.`;
+}
 //#endregion
-export { readMachineMemory };
+export { createMachineMemoryReader, machineMemoryUnsupportedReason, readMachineMemory };
 
 //# sourceMappingURL=machine_memory.js.map

package/dist/machine_memory.js.map

@@ -1 +1 @@
-{"version":3,"file":"machine_memory.js","names":[],"sources":["../src/machine_memory.ts"],"sourcesContent":["import { execFileSync } from \"node:child_process\";\nimport fs from \"node:fs\";\nimport type { MachineMemory } from \"./events\";\n\n/**\n * Reads where the machine's memory currently is.\n *\n * Costs about two milliseconds on macOS and one file read on Linux, which is why\n * the sampler can afford it once a second. It never throws: a reading that cannot\n * be taken says so and the sampler carries on, because a missing number must not\n * cost the resident-size curve it accompanies.\n */\nexport function readMachineMemory(): MachineMemory {\n  try {\n    if (process.platform === \"darwin\") return readDarwin();\n    if (process.platform === \"linux\") return readLinux();\n    return { unavailable: `not implemented for ${process.platform}` };\n  } catch (error: unknown) {\n    return { unavailable: error instanceof Error ? error.message : String(error) };\n  }\n}\n\n// Internals\n\nfunction readDarwin(): MachineMemory {\n  const stat = execFileSync(\"vm_stat\", { encoding: \"utf8\", timeout: 2000 });\n  // The page size is stated in the header and is 16 KiB on Apple silicon against\n  // 4 KiB elsewhere, so every count below is meaningless without reading it.\n  const pageSize = Number(/page size of (\\d+) bytes/.exec(stat)?.[1] ?? 4096);\n  const pages = (label: string): number | undefined => {\n    const match = new RegExp(`${label}:\\\\s+(\\\\d+)\\\\.`).exec(stat);\n    return match ? Number(match[1]) * pageSize : undefined;\n  };\n\n  const swap = execFileSync(\"sysctl\", [\"-n\", \"vm.swapusage\"], { encoding: \"utf8\", timeout: 2000 });\n  const swapBytes = (label: string): number | undefined => {\n    const match = new RegExp(`${label} = ([\\\\d.]+)M`).exec(swap);\n    return match ? Number(match[1]) * 1024 * 1024 : undefined;\n  };\n\n  return {\n    // \"Stored\" counts the memory before compression and is what went missing from\n    // a process's resident set; \"occupied\" is what it costs the machine now.\n    compressedStored: pages(\"Pages stored in compressor\"),\n    compressedOccupied: pages(\"Pages occupied by compressor\"),\n    swapUsed: swapBytes(\"used\"),\n    swapTotal: swapBytes(\"total\"),\n    anonymous: pages(\"Anonymous pages\"),\n    fileBacked: pages(\"File-backed pages\"),\n    wired: pages(\"Pages wired down\"),\n  };\n}\n\nfunction readLinux(): MachineMemory {\n  const info = fs.readFileSync(\"/proc/meminfo\", \"utf8\");\n  const kb = (label: string): number | undefined => {\n    const match = new RegExp(`^${label}:\\\\s+(\\\\d+) kB`, \"m\").exec(info);\n    return match ? Number(match[1]) * 1024 : undefined;\n  };\n  const swapTotal = kb(\"SwapTotal\");\n  const swapFree = kb(\"SwapFree\");\n  return {\n    swapTotal,\n    swapUsed: swapTotal !== undefined && swapFree !== undefined ? swapTotal - swapFree : undefined,\n    anonymous: kb(\"AnonPages\"),\n    fileBacked: kb(\"Cached\"),\n    // Linux has no compressor of its own; zram, where present, reports as swap.\n    wired: kb(\"Unevictable\"),\n  };\n}\n"],"mappings":";;;;;;;;;;;AAYA,SAAgB,oBAAmC;CACjD,IAAI;EACF,IAAI,QAAQ,aAAa,UAAU,OAAO,WAAW;EACrD,IAAI,QAAQ,aAAa,SAAS,OAAO,UAAU;EACnD,OAAO,EAAE,aAAa,uBAAuB,QAAQ,WAAW;CAClE,SAAS,OAAgB;EACvB,OAAO,EAAE,aAAa,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK,EAAE;CAC/E;AACF;AAIA,SAAS,aAA4B;CACnC,MAAM,OAAO,aAAa,WAAW;EAAE,UAAU;EAAQ,SAAS;CAAK,CAAC;CAGxE,MAAM,WAAW,OAAO,2BAA2B,KAAK,IAAI,CAAC,GAAG,MAAM,IAAI;CAC1E,MAAM,SAAS,UAAsC;EACnD,MAAM,QAAQ,IAAI,OAAO,GAAG,MAAM,eAAe,CAAC,CAAC,KAAK,IAAI;EAC5D,OAAO,QAAQ,OAAO,MAAM,EAAE,IAAI,WAAW,KAAA;CAC/C;CAEA,MAAM,OAAO,aAAa,UAAU,CAAC,MAAM,cAAc,GAAG;EAAE,UAAU;EAAQ,SAAS;CAAK,CAAC;CAC/F,MAAM,aAAa,UAAsC;EACvD,MAAM,QAAQ,IAAI,OAAO,GAAG,MAAM,cAAc,CAAC,CAAC,KAAK,IAAI;EAC3D,OAAO,QAAQ,OAAO,MAAM,EAAE,IAAI,OAAO,OAAO,KAAA;CAClD;CAEA,OAAO;EAGL,kBAAkB,MAAM,4BAA4B;EACpD,oBAAoB,MAAM,8BAA8B;EACxD,UAAU,UAAU,MAAM;EAC1B,WAAW,UAAU,OAAO;EAC5B,WAAW,MAAM,iBAAiB;EAClC,YAAY,MAAM,mBAAmB;EACrC,OAAO,MAAM,kBAAkB;CACjC;AACF;AAEA,SAAS,YAA2B;CAClC,MAAM,OAAO,GAAG,aAAa,iBAAiB,MAAM;CACpD,MAAM,MAAM,UAAsC;EAChD,MAAM,QAAQ,IAAI,OAAO,IAAI,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,IAAI;EAClE,OAAO,QAAQ,OAAO,MAAM,EAAE,IAAI,OAAO,KAAA;CAC3C;CACA,MAAM,YAAY,GAAG,WAAW;CAChC,MAAM,WAAW,GAAG,UAAU;CAC9B,OAAO;EACL;EACA,UAAU,cAAc,KAAA,KAAa,aAAa,KAAA,IAAY,YAAY,WAAW,KAAA;EACrF,WAAW,GAAG,WAAW;EACzB,YAAY,GAAG,QAAQ;EAEvB,OAAO,GAAG,aAAa;CACzB;AACF"}
+{"version":3,"file":"machine_memory.js","names":[],"sources":["../src/machine_memory.ts"],"sourcesContent":["import { execFileSync } from \"node:child_process\";\nimport fs from \"node:fs\";\nimport type { MachineMemory } from \"./events\";\n\n/**\n * Reads where the machine's memory currently is.\n *\n * Costs about two milliseconds on macOS and one file read on Linux, which is why\n * the sampler can afford it once a second. It never throws: a reading that cannot\n * be taken says so and the sampler carries on, because a missing number must not\n * cost the resident-size curve it accompanies.\n */\nexport function readMachineMemory(): MachineMemory {\n  try {\n    if (process.platform === \"darwin\") return readDarwin();\n    if (process.platform === \"linux\") return readLinux();\n    return { unavailable: machineMemoryUnsupported() };\n  } catch (error: unknown) {\n    return { unavailable: error instanceof Error ? error.message : String(error) };\n  }\n}\n\n/**\n * Decides what each sampler tick should record about machine-wide memory.\n *\n * Three things are being balanced. The reading costs a subprocess, so it is\n * taken on its own interval rather than with every sample. Repeating an\n * unchanged failure every second buries the resident-size curve it sits beside,\n * so a reason is written once and not again while it still holds. And a failure\n * is not the end of the matter: a `vm_stat` that timed out did so under load,\n * which is precisely the moment its figures are worth having a second later —\n * so the source keeps being tried, and a reading that comes back is recorded.\n *\n * Only an unsupported platform stops it for good, because that is the one\n * condition a running process cannot get out of.\n */\nexport function createMachineMemoryReader(options: {\n  intervalMs: number;\n  read?: () => MachineMemory;\n  unsupportedReason?: () => string | undefined;\n}): (now: number) => MachineMemory | undefined {\n  const read = options.read ?? readMachineMemory;\n  const unsupported = (options.unsupportedReason ?? machineMemoryUnsupportedReason)();\n\n  let pending = unsupported;\n  let dueAt = 0;\n  let statedFailure: string | undefined;\n\n  return (now: number): MachineMemory | undefined => {\n    if (pending !== undefined) {\n      const reason = pending;\n      pending = undefined;\n      return { unavailable: reason };\n    }\n    if (unsupported !== undefined || now < dueAt) return undefined;\n\n    dueAt = now + options.intervalMs;\n    const reading = read();\n    if (reading.unavailable === undefined) {\n      // A recovery is worth seeing: the gap in the series ends where the\n      // figures resume.\n      statedFailure = undefined;\n      return reading;\n    }\n    if (reading.unavailable === statedFailure) return undefined;\n    statedFailure = reading.unavailable;\n    return reading;\n  };\n}\n\n/**\n * Names what the machine-wide reading would have contributed, for platforms\n * where it cannot be taken at all.\n *\n * A reader who finds no compressor or swap figures needs to know whether the\n * machine had none or the sampler never asked, and where the equivalent evidence\n * is instead. Callers record it once: it is a property of the platform and\n * repeating it every second buries the curve it was meant to explain.\n */\nexport function machineMemoryUnsupportedReason(): string | undefined {\n  if (process.platform === \"darwin\" || process.platform === \"linux\") return undefined;\n  return machineMemoryUnsupported();\n}\n\n// Internals\n\nfunction readDarwin(): MachineMemory {\n  const stat = execFileSync(\"vm_stat\", { encoding: \"utf8\", timeout: 2000 });\n  // The page size is stated in the header and is 16 KiB on Apple silicon against\n  // 4 KiB elsewhere, so every count below is meaningless without reading it.\n  const pageSize = Number(/page size of (\\d+) bytes/.exec(stat)?.[1] ?? 4096);\n  const pages = (label: string): number | undefined => {\n    const match = new RegExp(`${label}:\\\\s+(\\\\d+)\\\\.`).exec(stat);\n    return match ? Number(match[1]) * pageSize : undefined;\n  };\n\n  const swap = execFileSync(\"sysctl\", [\"-n\", \"vm.swapusage\"], { encoding: \"utf8\", timeout: 2000 });\n  const swapBytes = (label: string): number | undefined => {\n    const match = new RegExp(`${label} = ([\\\\d.]+)M`).exec(swap);\n    return match ? Number(match[1]) * 1024 * 1024 : undefined;\n  };\n\n  return {\n    // \"Stored\" counts the memory before compression and is what went missing from\n    // a process's resident set; \"occupied\" is what it costs the machine now.\n    compressedStored: pages(\"Pages stored in compressor\"),\n    compressedOccupied: pages(\"Pages occupied by compressor\"),\n    swapUsed: swapBytes(\"used\"),\n    swapTotal: swapBytes(\"total\"),\n    anonymous: pages(\"Anonymous pages\"),\n    fileBacked: pages(\"File-backed pages\"),\n    wired: pages(\"Pages wired down\"),\n  };\n}\n\nfunction readLinux(): MachineMemory {\n  const info = fs.readFileSync(\"/proc/meminfo\", \"utf8\");\n  const kb = (label: string): number | undefined => {\n    const match = new RegExp(`^${label}:\\\\s+(\\\\d+) kB`, \"m\").exec(info);\n    return match ? Number(match[1]) * 1024 : undefined;\n  };\n  const swapTotal = kb(\"SwapTotal\");\n  const swapFree = kb(\"SwapFree\");\n  return {\n    swapTotal,\n    swapUsed: swapTotal !== undefined && swapFree !== undefined ? swapTotal - swapFree : undefined,\n    anonymous: kb(\"AnonPages\"),\n    fileBacked: kb(\"Cached\"),\n    // Linux has no compressor of its own; zram, where present, reports as swap.\n    wired: kb(\"Unevictable\"),\n  };\n}\n\nfunction machineMemoryUnsupported(): string {\n  return (\n    `no machine-wide memory source on ${process.platform}: ` +\n    \"compressor, swap and anonymous/file-backed totals are not sampled \" +\n    \"(vm_stat and sysctl are macOS-only, /proc/meminfo Linux-only). \" +\n    \"Per-process committed bytes are recorded instead, as `private` in the host log.\"\n  );\n}\n"],"mappings":";;;;;;;;;;;AAYA,SAAgB,oBAAmC;CACjD,IAAI;EACF,IAAI,QAAQ,aAAa,UAAU,OAAO,WAAW;EACrD,IAAI,QAAQ,aAAa,SAAS,OAAO,UAAU;EACnD,OAAO,EAAE,aAAa,yBAAyB,EAAE;CACnD,SAAS,OAAgB;EACvB,OAAO,EAAE,aAAa,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK,EAAE;CAC/E;AACF;;;;;;;;;;;;;;;AAgBA,SAAgB,0BAA0B,SAIK;CAC7C,MAAM,OAAO,QAAQ,QAAQ;CAC7B,MAAM,eAAe,QAAQ,qBAAqB,+BAAA,CAAgC;CAElF,IAAI,UAAU;CACd,IAAI,QAAQ;CACZ,IAAI;CAEJ,QAAQ,QAA2C;EACjD,IAAI,YAAY,KAAA,GAAW;GACzB,MAAM,SAAS;GACf,UAAU,KAAA;GACV,OAAO,EAAE,aAAa,OAAO;EAC/B;EACA,IAAI,gBAAgB,KAAA,KAAa,MAAM,OAAO,OAAO,KAAA;EAErD,QAAQ,MAAM,QAAQ;EACtB,MAAM,UAAU,KAAK;EACrB,IAAI,QAAQ,gBAAgB,KAAA,GAAW;GAGrC,gBAAgB,KAAA;GAChB,OAAO;EACT;EACA,IAAI,QAAQ,gBAAgB,eAAe,OAAO,KAAA;EAClD,gBAAgB,QAAQ;EACxB,OAAO;CACT;AACF;;;;;;;;;;AAWA,SAAgB,iCAAqD;CACnE,IAAI,QAAQ,aAAa,YAAY,QAAQ,aAAa,SAAS,OAAO,KAAA;CAC1E,OAAO,yBAAyB;AAClC;AAIA,SAAS,aAA4B;CACnC,MAAM,OAAO,aAAa,WAAW;EAAE,UAAU;EAAQ,SAAS;CAAK,CAAC;CAGxE,MAAM,WAAW,OAAO,2BAA2B,KAAK,IAAI,CAAC,GAAG,MAAM,IAAI;CAC1E,MAAM,SAAS,UAAsC;EACnD,MAAM,QAAQ,IAAI,OAAO,GAAG,MAAM,eAAe,CAAC,CAAC,KAAK,IAAI;EAC5D,OAAO,QAAQ,OAAO,MAAM,EAAE,IAAI,WAAW,KAAA;CAC/C;CAEA,MAAM,OAAO,aAAa,UAAU,CAAC,MAAM,cAAc,GAAG;EAAE,UAAU;EAAQ,SAAS;CAAK,CAAC;CAC/F,MAAM,aAAa,UAAsC;EACvD,MAAM,QAAQ,IAAI,OAAO,GAAG,MAAM,cAAc,CAAC,CAAC,KAAK,IAAI;EAC3D,OAAO,QAAQ,OAAO,MAAM,EAAE,IAAI,OAAO,OAAO,KAAA;CAClD;CAEA,OAAO;EAGL,kBAAkB,MAAM,4BAA4B;EACpD,oBAAoB,MAAM,8BAA8B;EACxD,UAAU,UAAU,MAAM;EAC1B,WAAW,UAAU,OAAO;EAC5B,WAAW,MAAM,iBAAiB;EAClC,YAAY,MAAM,mBAAmB;EACrC,OAAO,MAAM,kBAAkB;CACjC;AACF;AAEA,SAAS,YAA2B;CAClC,MAAM,OAAO,GAAG,aAAa,iBAAiB,MAAM;CACpD,MAAM,MAAM,UAAsC;EAChD,MAAM,QAAQ,IAAI,OAAO,IAAI,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,IAAI;EAClE,OAAO,QAAQ,OAAO,MAAM,EAAE,IAAI,OAAO,KAAA;CAC3C;CACA,MAAM,YAAY,GAAG,WAAW;CAChC,MAAM,WAAW,GAAG,UAAU;CAC9B,OAAO;EACL;EACA,UAAU,cAAc,KAAA,KAAa,aAAa,KAAA,IAAY,YAAY,WAAW,KAAA;EACrF,WAAW,GAAG,WAAW;EACzB,YAAY,GAAG,QAAQ;EAEvB,OAAO,GAAG,aAAa;CACzB;AACF;AAEA,SAAS,2BAAmC;CAC1C,OACE,oCAAoC,QAAQ,SAAS;AAKzD"}

package/dist/sampler_thread.js

@@ -1,4 +1,4 @@
-import { readMachineMemory } from "./machine_memory.js";
+import { createMachineMemoryReader } from "./machine_memory.js";
 import fs from "node:fs";
 import os from "node:os";
 import { workerData } from "node:worker_threads";
@@ -25,13 +25,12 @@ const { file, intervalMs, machineIntervalMs = 1e3 } = workerData;
 const fd = fs.openSync(file, "a");
 let seq = 0;
 let peakRss = 0;
-let machineDueAt = 0;
+const readMachine = createMachineMemoryReader({ intervalMs: machineIntervalMs });
 setInterval(() => {
 	const rss = process.memoryUsage.rss();
 	if (rss > peakRss) peakRss = rss;
 	const now = Date.now();
-	const machine = now >= machineDueAt ? readMachineMemory() : void 0;
-	if (machine) machineDueAt = now + machineIntervalMs;
+	const machine = readMachine(now);
 	const record = {
 		seq: ++seq,
 		t: Math.round(performance.now() * 1e3) / 1e3,

package/dist/sampler_thread.js.map

@@ -1 +1 @@
-{"version":3,"file":"sampler_thread.js","names":[],"sources":["../src/sampler_thread.ts"],"sourcesContent":["/**\n * Memory sampler, run on its own worker thread.\n *\n * It exists because the thread worth watching is the one that blocks. While the\n * middle layer sits inside a synchronous pframes call its own timers do not\n * fire, so its memory series goes dark exactly while memory is growing fastest.\n * This thread stays responsive and keeps the resident-size curve intact right up\n * to the moment the process dies.\n *\n * `rss` and `freeMemory` are process- and machine-wide and so are meaningful\n * from here. Heap figures are per-isolate and would describe only this thread,\n * so they are deliberately not recorded; the observed thread reports its own.\n *\n * Resident size alone understates a process under pressure, because the OS moves\n * its pages into the compressor or out to swap and they stop being resident. The\n * machine's own account of where memory went is therefore sampled alongside it,\n * less often because it costs a subprocess.\n */\n\nimport fs from \"node:fs\";\nimport os from \"node:os\";\nimport { workerData } from \"node:worker_threads\";\nimport type { SamplerRecord } from \"./events\";\nimport { readMachineMemory } from \"./machine_memory\";\n\ntype SamplerWorkerData = { file: string; intervalMs: number; machineIntervalMs?: number };\n\nconst { file, intervalMs, machineIntervalMs = 1000 } = workerData as SamplerWorkerData;\nconst fd = fs.openSync(file, \"a\");\nlet seq = 0;\nlet peakRss = 0;\nlet machineDueAt = 0;\n\nsetInterval(() => {\n  const rss = process.memoryUsage.rss();\n  if (rss > peakRss) peakRss = rss;\n  const now = Date.now();\n  // Taken on the first tick and then on its own schedule, so the curve keeps its\n  // sampling rate while the costlier reading stays occasional.\n  const machine = now >= machineDueAt ? readMachineMemory() : undefined;\n  if (machine) machineDueAt = now + machineIntervalMs;\n  const record: SamplerRecord = {\n    seq: ++seq,\n    t: Math.round(performance.now() * 1000) / 1000,\n    wall: now,\n    type: \"mem-sampler\",\n    rss,\n    peakRss,\n    // The kernel's own high-water mark, which no sampling interval can miss.\n    maxRss: process.resourceUsage().maxRSS * 1024,\n    freeMemory: os.freemem(),\n    totalMemory: os.totalmem(),\n    ...(machine ? { machine } : {}),\n  };\n  try {\n    fs.writeSync(fd, `${JSON.stringify(record)}\\n`);\n  } catch {\n    // Sampling must never take the application down.\n  }\n}, intervalMs);\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;AA2BA,MAAM,EAAE,MAAM,YAAY,oBAAoB,QAAS;AACvD,MAAM,KAAK,GAAG,SAAS,MAAM,GAAG;AAChC,IAAI,MAAM;AACV,IAAI,UAAU;AACd,IAAI,eAAe;AAEnB,kBAAkB;CAChB,MAAM,MAAM,QAAQ,YAAY,IAAI;CACpC,IAAI,MAAM,SAAS,UAAU;CAC7B,MAAM,MAAM,KAAK,IAAI;CAGrB,MAAM,UAAU,OAAO,eAAe,kBAAkB,IAAI,KAAA;CAC5D,IAAI,SAAS,eAAe,MAAM;CAClC,MAAM,SAAwB;EAC5B,KAAK,EAAE;EACP,GAAG,KAAK,MAAM,YAAY,IAAI,IAAI,GAAI,IAAI;EAC1C,MAAM;EACN,MAAM;EACN;EACA;EAEA,QAAQ,QAAQ,cAAc,CAAC,CAAC,SAAS;EACzC,YAAY,GAAG,QAAQ;EACvB,aAAa,GAAG,SAAS;EACzB,GAAI,UAAU,EAAE,QAAQ,IAAI,CAAC;CAC/B;CACA,IAAI;EACF,GAAG,UAAU,IAAI,GAAG,KAAK,UAAU,MAAM,EAAE,GAAG;CAChD,QAAQ,CAER;AACF,GAAG,UAAU"}
+{"version":3,"file":"sampler_thread.js","names":[],"sources":["../src/sampler_thread.ts"],"sourcesContent":["/**\n * Memory sampler, run on its own worker thread.\n *\n * It exists because the thread worth watching is the one that blocks. While the\n * middle layer sits inside a synchronous pframes call its own timers do not\n * fire, so its memory series goes dark exactly while memory is growing fastest.\n * This thread stays responsive and keeps the resident-size curve intact right up\n * to the moment the process dies.\n *\n * `rss` and `freeMemory` are process- and machine-wide and so are meaningful\n * from here. Heap figures are per-isolate and would describe only this thread,\n * so they are deliberately not recorded; the observed thread reports its own.\n *\n * Resident size alone understates a process under pressure, because the OS moves\n * its pages into the compressor or out to swap and they stop being resident. The\n * machine's own account of where memory went is therefore sampled alongside it,\n * less often because it costs a subprocess.\n */\n\nimport fs from \"node:fs\";\nimport os from \"node:os\";\nimport { workerData } from \"node:worker_threads\";\nimport type { SamplerRecord } from \"./events\";\nimport { createMachineMemoryReader } from \"./machine_memory\";\n\ntype SamplerWorkerData = { file: string; intervalMs: number; machineIntervalMs?: number };\n\nconst { file, intervalMs, machineIntervalMs = 1000 } = workerData as SamplerWorkerData;\nconst fd = fs.openSync(file, \"a\");\nlet seq = 0;\nlet peakRss = 0;\nconst readMachine = createMachineMemoryReader({ intervalMs: machineIntervalMs });\n\nsetInterval(() => {\n  const rss = process.memoryUsage.rss();\n  if (rss > peakRss) peakRss = rss;\n  const now = Date.now();\n  // Taken on its own schedule, and only when it has something to say, so the\n  // curve keeps its sampling rate while the costlier reading stays occasional.\n  const machine = readMachine(now);\n  const record: SamplerRecord = {\n    seq: ++seq,\n    t: Math.round(performance.now() * 1000) / 1000,\n    wall: now,\n    type: \"mem-sampler\",\n    rss,\n    peakRss,\n    // The kernel's own high-water mark, which no sampling interval can miss.\n    maxRss: process.resourceUsage().maxRSS * 1024,\n    freeMemory: os.freemem(),\n    totalMemory: os.totalmem(),\n    ...(machine ? { machine } : {}),\n  };\n  try {\n    fs.writeSync(fd, `${JSON.stringify(record)}\\n`);\n  } catch {\n    // Sampling must never take the application down.\n  }\n}, intervalMs);\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;AA2BA,MAAM,EAAE,MAAM,YAAY,oBAAoB,QAAS;AACvD,MAAM,KAAK,GAAG,SAAS,MAAM,GAAG;AAChC,IAAI,MAAM;AACV,IAAI,UAAU;AACd,MAAM,cAAc,0BAA0B,EAAE,YAAY,kBAAkB,CAAC;AAE/E,kBAAkB;CAChB,MAAM,MAAM,QAAQ,YAAY,IAAI;CACpC,IAAI,MAAM,SAAS,UAAU;CAC7B,MAAM,MAAM,KAAK,IAAI;CAGrB,MAAM,UAAU,YAAY,GAAG;CAC/B,MAAM,SAAwB;EAC5B,KAAK,EAAE;EACP,GAAG,KAAK,MAAM,YAAY,IAAI,IAAI,GAAI,IAAI;EAC1C,MAAM;EACN,MAAM;EACN;EACA;EAEA,QAAQ,QAAQ,cAAc,CAAC,CAAC,SAAS;EACzC,YAAY,GAAG,QAAQ;EACvB,aAAa,GAAG,SAAS;EACzB,GAAI,UAAU,EAAE,QAAQ,IAAI,CAAC;CAC/B;CACA,IAAI;EACF,GAAG,UAAU,IAAI,GAAG,KAAK,UAAU,MAAM,EAAE,GAAG;CAChD,QAAQ,CAER;AACF,GAAG,UAAU"}

package/dist/supervisor.d.ts

@@ -1,5 +1,9 @@
 import { CrashMarker } from "./events.js";
 //#region src/supervisor.d.ts
+/** A crash marker together with the file holding it. */
+export type StoredCrashMarker = CrashMarker & {
+  file: string;
+};
 export type SuperviseOptions = {
   /**
    * The session id handed to the worker at spawn (see `CRASH_SESSION_ENV`).
@@ -20,8 +24,14 @@ export type SupervisedWorker = {
   on(event: "error", listener: (error: Error) => void): unknown;
   on(event: "exit", listener: (code: number) => void): unknown;
 };
-/** Crash markers in a directory, oldest first. */
-export declare function readCrashMarkers(dir: string): CrashMarker[];
+/**
+ * Crash markers in a directory, oldest first, each with the file it came from.
+ *
+ * The path travels with the marker so that a caller collecting the evidence — to
+ * attach to a report, say — never has to spell the file name itself and cannot
+ * drift from the one this module writes.
+ */
+export declare function readCrashMarkers(dir: string): StoredCrashMarker[];
 /**
  * Attaches crash recording to a middle-layer worker thread.
  *

package/dist/supervisor.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"supervisor.d.ts","names":[],"sources":["../src/supervisor.ts"],"mappings":";;YAgBY;;;;;;;EAOV;EACA,WAAW;IACT;IACA;IACA;IACA;;;;YAKQ;EACV,GAAG,gBAAgB,WAAW,OAAO;EACrC,GAAG,eAAe,WAAW;;;wBAwCf,iBAAiB,cAAc;;;;;;;;;;;wBA+B/B,gBACd,QAAQ,kBACR,aACA,UAAS"}
+{"version":3,"file":"supervisor.d.ts","names":[],"sources":["../src/supervisor.ts"],"mappings":";;;YAiBY,oBAAoB;EAAgB;;YAEpC;;;;;;;EAOV;EACA,WAAW;IACT;IACA;IACA;IACA;;;;YAKQ;EACV,GAAG,gBAAgB,WAAW,OAAO;EACrC,GAAG,eAAe,WAAW;;;;;;;;;wBA8Cf,iBAAiB,cAAc;;;;;;;;;;;wBAgC/B,gBACd,QAAQ,kBACR,aACA,UAAS"}

package/dist/supervisor.js

@@ -34,7 +34,13 @@ function writeCrashMarker(dir, input = {}) {
 	fs.writeFileSync(file, `${JSON.stringify(marker)}\n`);
 	return file;
 }
-/** Crash markers in a directory, oldest first. */
+/**
+* Crash markers in a directory, oldest first, each with the file it came from.
+*
+* The path travels with the marker so that a caller collecting the evidence — to
+* attach to a report, say — never has to spell the file name itself and cannot
+* drift from the one this module writes.
+*/
 function readCrashMarkers(dir) {
 	let names;
 	try {
@@ -45,9 +51,13 @@ function readCrashMarkers(dir) {
 	const markers = [];
 	for (const name of names) {
 		if (!name.startsWith(`death-`) || !name.endsWith(".ndjson")) continue;
+		const file = path.join(dir, name);
 		try {
-			const first = fs.readFileSync(path.join(dir, name), "utf8").split("\n")[0];
-			markers.push(JSON.parse(first));
+			const first = fs.readFileSync(file, "utf8").split("\n")[0];
+			markers.push({
+				...JSON.parse(first),
+				file
+			});
 		} catch {}
 	}
 	return markers.sort((lhs, rhs) => lhs.wall - rhs.wall);

package/dist/supervisor.js.map

@@ -1 +1 @@
-{"version":3,"file":"supervisor.js","names":[],"sources":["../src/supervisor.ts"],"sourcesContent":["import fs from \"node:fs\";\nimport os from \"node:os\";\nimport path from \"node:path\";\nimport { DEATH_FILE_PREFIX, type CrashMarker, type CrashReason } from \"./events\";\nimport { listSessions, sessionIdFromFile } from \"./recorder\";\n\ntype CrashMarkerInput = {\n  /** Session id the parent assigned to the worker. Omitted, the marker carries no identity. */\n  sessionId?: string;\n  reason?: CrashReason;\n  error?: (Error & { code?: string }) | unknown;\n  code?: number;\n  signal?: string;\n  stderrTail?: string;\n};\n\nexport type SuperviseOptions = {\n  /**\n   * The session id handed to the worker at spawn (see `CRASH_SESSION_ENV`).\n   * With it the marker names the dying session with certainty. Without it the\n   * analyzer has to attribute the marker by timing, and will decline to\n   * attribute it at all when more than one session looks dead.\n   */\n  sessionId?: string;\n  onCrash?: (info: {\n    kind: \"error\" | \"exit\";\n    markerFile: string;\n    error?: unknown;\n    code?: number;\n  }) => void;\n};\n\n/** Minimal view of a worker, so callers are not forced to import worker_threads. */\nexport type SupervisedWorker = {\n  on(event: \"error\", listener: (error: Error) => void): unknown;\n  on(event: \"exit\", listener: (code: number) => void): unknown;\n};\n\n/**\n * Records an abnormal end observed from outside the dying thread.\n *\n * A thread that runs out of heap cannot describe its own death: the last reading\n * it wrote predates the blow-up, and when the blow-up is synchronous no sampler\n * tick of its own lands either. The parent is the only place where the cause is\n * known rather than inferred — Node reports `ERR_WORKER_OUT_OF_MEMORY` to it —\n * so the parent writes the verdict down on the dead thread's behalf.\n */\nexport function writeCrashMarker(dir: string, input: CrashMarkerInput = {}): string {\n  fs.mkdirSync(dir, { recursive: true });\n  const error = input.error as (Error & { code?: string }) | undefined;\n  // Only an id the parent handed to the worker is certain, and only a certain\n  // id goes in `sessionId`. Reading the newest open crash log names whichever\n  // session wrote last, which a concurrent live session makes wrong; recorded\n  // as identity that would misattribute the death and, worse, stop the session\n  // that actually died from claiming the marker. So it is advisory only.\n  const marker: CrashMarker = {\n    type: \"external-crash\",\n    wall: Date.now(),\n    sessionId: input.sessionId,\n    guessedSessionId: input.sessionId === undefined ? newestOpenSessionId(dir) : undefined,\n    reason: input.reason ?? classifyReason(input),\n    errorCode: error?.code,\n    errorName: error?.name,\n    message: truncate(String(error?.message ?? input.error ?? \"\"), 2000),\n    exitCode: input.code,\n    signal: input.signal,\n    stderrTail: truncate(input.stderrTail ?? \"\", 4000),\n    memoryAtDeath: memoryNow(),\n  };\n  const file = path.join(dir, `${DEATH_FILE_PREFIX}-${marker.wall}.ndjson`);\n  fs.writeFileSync(file, `${JSON.stringify(marker)}\\n`);\n  return file;\n}\n\n/** Crash markers in a directory, oldest first. */\nexport function readCrashMarkers(dir: string): CrashMarker[] {\n  let names: string[];\n  try {\n    names = fs.readdirSync(dir);\n  } catch {\n    return [];\n  }\n  const markers: CrashMarker[] = [];\n  for (const name of names) {\n    if (!name.startsWith(`${DEATH_FILE_PREFIX}-`) || !name.endsWith(\".ndjson\")) continue;\n    try {\n      const first = fs.readFileSync(path.join(dir, name), \"utf8\").split(\"\\n\")[0];\n      markers.push(JSON.parse(first) as CrashMarker);\n    } catch {\n      // A marker that cannot be parsed is skipped; it is one line of evidence,\n      // not the report.\n    }\n  }\n  return markers.sort((lhs, rhs) => lhs.wall - rhs.wall);\n}\n\n/**\n * Attaches crash recording to a middle-layer worker thread.\n *\n * A worker whose isolate exhausts its heap dies alone and the parent receives\n * `ERR_WORKER_OUT_OF_MEMORY`, with or without `resourceLimits`. What\n * `resourceLimits.maxOldGenerationSizeMb` adds is a chosen ceiling: V8's default\n * is several gigabytes, so on a small machine the OS can run out of memory and\n * kill the whole process before V8 ever reports the worker's heap as full — and\n * then there is no parent left to write anything.\n */\nexport function superviseWorker(\n  worker: SupervisedWorker,\n  dir: string,\n  options: SuperviseOptions = {},\n): void {\n  // One death fires `error` and then `exit`. Only `error` carries the cause, so\n  // a later `exit` must not overwrite it with a bare exit code.\n  let recorded = false;\n  worker.on(\"error\", (error: Error) => {\n    recorded = true;\n    const markerFile = writeCrashMarker(dir, { error, sessionId: options.sessionId });\n    options.onCrash?.({ kind: \"error\", error, markerFile });\n  });\n  worker.on(\"exit\", (code: number) => {\n    if (code === 0 || recorded) return;\n    const markerFile = writeCrashMarker(dir, {\n      reason: \"worker-exit\",\n      code,\n      sessionId: options.sessionId,\n    });\n    options.onCrash?.({ kind: \"exit\", code, markerFile });\n  });\n}\n\n// Internals\n\n/**\n * The parent's view of memory at the moment it saw the death.\n *\n * The dying thread cannot take this reading, and the sampler's last one predates\n * the end by up to its interval. Taken here it is contemporaneous with the exit\n * code it sits beside, which is what stops an exhausted machine from reading as\n * an ordinary failure.\n *\n * Every reading here is a syscall. The machine's compressor and swap totals are\n * deliberately not among them: on macOS they cost a subprocess, and this runs on\n * the parent's event loop inside the worker's error handler, before the marker\n * is written and before the caller learns of the death. A fork is exactly what\n * becomes slow or impossible on the exhausted machine this code exists for, so\n * the fuller picture is left to the sampler, whose last reading is at most one\n * interval old and sits in the same bundle.\n */\nfunction memoryNow(): CrashMarker[\"memoryAtDeath\"] {\n  try {\n    return {\n      rss: process.memoryUsage.rss(),\n      maxRss: process.resourceUsage().maxRSS * 1024,\n      freeMemory: os.freemem(),\n      totalMemory: os.totalmem(),\n    };\n  } catch {\n    // A marker without memory is still a marker; failing to take the reading\n    // must never cost the record of the death itself.\n    return undefined;\n  }\n}\n\n// Advisory only, for a human reading a directory by hand: the dying session has\n// no terminating record, so among the sessions that look dead this names the one\n// that wrote last. Never used as identity — see `CrashMarker.guessedSessionId`.\nfunction newestOpenSessionId(dir: string): string | undefined {\n  const open = listSessions(dir).find((session) => session.crashed);\n  return open ? sessionIdFromFile(open.file) : undefined;\n}\n\nfunction classifyReason({ error, code, signal }: CrashMarkerInput): CrashReason {\n  const errorCode = (error as { code?: string } | undefined)?.code;\n  if (errorCode === \"ERR_WORKER_OUT_OF_MEMORY\") return \"js-heap-out-of-memory\";\n  if (signal === \"SIGKILL\") return \"killed-by-os\";\n  if (signal === \"SIGABRT\" || code === 134) return \"abort-or-fatal-allocation-failure\";\n  if (typeof code === \"number\" && code !== 0) return \"nonzero-exit\";\n  return \"unknown\";\n}\n\nfunction truncate(value: string, limit: number): string {\n  return value.length > limit ? `${value.slice(0, limit)}…` : value;\n}\n"],"mappings":";;;;;;;;;;;;;;;AA+CA,SAAgB,iBAAiB,KAAa,QAA0B,CAAC,GAAW;CAClF,GAAG,UAAU,KAAK,EAAE,WAAW,KAAK,CAAC;CACrC,MAAM,QAAQ,MAAM;CAMpB,MAAM,SAAsB;EAC1B,MAAM;EACN,MAAM,KAAK,IAAI;EACf,WAAW,MAAM;EACjB,kBAAkB,MAAM,cAAc,KAAA,IAAY,oBAAoB,GAAG,IAAI,KAAA;EAC7E,QAAQ,MAAM,UAAU,eAAe,KAAK;EAC5C,WAAW,OAAO;EAClB,WAAW,OAAO;EAClB,SAAS,SAAS,OAAO,OAAO,WAAW,MAAM,SAAS,EAAE,GAAG,GAAI;EACnE,UAAU,MAAM;EAChB,QAAQ,MAAM;EACd,YAAY,SAAS,MAAM,cAAc,IAAI,GAAI;EACjD,eAAe,UAAU;CAC3B;CACA,MAAM,OAAO,KAAK,KAAK,KAAK,GAAG,kBAAkB,GAAG,OAAO,KAAK,QAAQ;CACxE,GAAG,cAAc,MAAM,GAAG,KAAK,UAAU,MAAM,EAAE,GAAG;CACpD,OAAO;AACT;;AAGA,SAAgB,iBAAiB,KAA4B;CAC3D,IAAI;CACJ,IAAI;EACF,QAAQ,GAAG,YAAY,GAAG;CAC5B,QAAQ;EACN,OAAO,CAAC;CACV;CACA,MAAM,UAAyB,CAAC;CAChC,KAAK,MAAM,QAAQ,OAAO;EACxB,IAAI,CAAC,KAAK,WAAW,QAAuB,KAAK,CAAC,KAAK,SAAS,SAAS,GAAG;EAC5E,IAAI;GACF,MAAM,QAAQ,GAAG,aAAa,KAAK,KAAK,KAAK,IAAI,GAAG,MAAM,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC;GACxE,QAAQ,KAAK,KAAK,MAAM,KAAK,CAAgB;EAC/C,QAAQ,CAGR;CACF;CACA,OAAO,QAAQ,MAAM,KAAK,QAAQ,IAAI,OAAO,IAAI,IAAI;AACvD;;;;;;;;;;;AAYA,SAAgB,gBACd,QACA,KACA,UAA4B,CAAC,GACvB;CAGN,IAAI,WAAW;CACf,OAAO,GAAG,UAAU,UAAiB;EACnC,WAAW;EACX,MAAM,aAAa,iBAAiB,KAAK;GAAE;GAAO,WAAW,QAAQ;EAAU,CAAC;EAChF,QAAQ,UAAU;GAAE,MAAM;GAAS;GAAO;EAAW,CAAC;CACxD,CAAC;CACD,OAAO,GAAG,SAAS,SAAiB;EAClC,IAAI,SAAS,KAAK,UAAU;EAC5B,MAAM,aAAa,iBAAiB,KAAK;GACvC,QAAQ;GACR;GACA,WAAW,QAAQ;EACrB,CAAC;EACD,QAAQ,UAAU;GAAE,MAAM;GAAQ;GAAM;EAAW,CAAC;CACtD,CAAC;AACH;;;;;;;;;;;;;;;;;AAoBA,SAAS,YAA0C;CACjD,IAAI;EACF,OAAO;GACL,KAAK,QAAQ,YAAY,IAAI;GAC7B,QAAQ,QAAQ,cAAc,CAAC,CAAC,SAAS;GACzC,YAAY,GAAG,QAAQ;GACvB,aAAa,GAAG,SAAS;EAC3B;CACF,QAAQ;EAGN;CACF;AACF;AAKA,SAAS,oBAAoB,KAAiC;CAC5D,MAAM,OAAO,aAAa,GAAG,CAAC,CAAC,MAAM,YAAY,QAAQ,OAAO;CAChE,OAAO,OAAO,kBAAkB,KAAK,IAAI,IAAI,KAAA;AAC/C;AAEA,SAAS,eAAe,EAAE,OAAO,MAAM,UAAyC;CAE9E,IADmB,OAAyC,SAC1C,4BAA4B,OAAO;CACrD,IAAI,WAAW,WAAW,OAAO;CACjC,IAAI,WAAW,aAAa,SAAS,KAAK,OAAO;CACjD,IAAI,OAAO,SAAS,YAAY,SAAS,GAAG,OAAO;CACnD,OAAO;AACT;AAEA,SAAS,SAAS,OAAe,OAAuB;CACtD,OAAO,MAAM,SAAS,QAAQ,GAAG,MAAM,MAAM,GAAG,KAAK,EAAE,KAAK;AAC9D"}
+{"version":3,"file":"supervisor.js","names":[],"sources":["../src/supervisor.ts"],"sourcesContent":["import fs from \"node:fs\";\nimport os from \"node:os\";\nimport path from \"node:path\";\nimport { DEATH_FILE_PREFIX, type CrashMarker, type CrashReason } from \"./events\";\nimport { listSessions, sessionIdFromFile } from \"./recorder\";\n\ntype CrashMarkerInput = {\n  /** Session id the parent assigned to the worker. Omitted, the marker carries no identity. */\n  sessionId?: string;\n  reason?: CrashReason;\n  error?: (Error & { code?: string }) | unknown;\n  code?: number;\n  signal?: string;\n  stderrTail?: string;\n};\n\n/** A crash marker together with the file holding it. */\nexport type StoredCrashMarker = CrashMarker & { file: string };\n\nexport type SuperviseOptions = {\n  /**\n   * The session id handed to the worker at spawn (see `CRASH_SESSION_ENV`).\n   * With it the marker names the dying session with certainty. Without it the\n   * analyzer has to attribute the marker by timing, and will decline to\n   * attribute it at all when more than one session looks dead.\n   */\n  sessionId?: string;\n  onCrash?: (info: {\n    kind: \"error\" | \"exit\";\n    markerFile: string;\n    error?: unknown;\n    code?: number;\n  }) => void;\n};\n\n/** Minimal view of a worker, so callers are not forced to import worker_threads. */\nexport type SupervisedWorker = {\n  on(event: \"error\", listener: (error: Error) => void): unknown;\n  on(event: \"exit\", listener: (code: number) => void): unknown;\n};\n\n/**\n * Records an abnormal end observed from outside the dying thread.\n *\n * A thread that runs out of heap cannot describe its own death: the last reading\n * it wrote predates the blow-up, and when the blow-up is synchronous no sampler\n * tick of its own lands either. The parent is the only place where the cause is\n * known rather than inferred — Node reports `ERR_WORKER_OUT_OF_MEMORY` to it —\n * so the parent writes the verdict down on the dead thread's behalf.\n */\nexport function writeCrashMarker(dir: string, input: CrashMarkerInput = {}): string {\n  fs.mkdirSync(dir, { recursive: true });\n  const error = input.error as (Error & { code?: string }) | undefined;\n  // Only an id the parent handed to the worker is certain, and only a certain\n  // id goes in `sessionId`. Reading the newest open crash log names whichever\n  // session wrote last, which a concurrent live session makes wrong; recorded\n  // as identity that would misattribute the death and, worse, stop the session\n  // that actually died from claiming the marker. So it is advisory only.\n  const marker: CrashMarker = {\n    type: \"external-crash\",\n    wall: Date.now(),\n    sessionId: input.sessionId,\n    guessedSessionId: input.sessionId === undefined ? newestOpenSessionId(dir) : undefined,\n    reason: input.reason ?? classifyReason(input),\n    errorCode: error?.code,\n    errorName: error?.name,\n    message: truncate(String(error?.message ?? input.error ?? \"\"), 2000),\n    exitCode: input.code,\n    signal: input.signal,\n    stderrTail: truncate(input.stderrTail ?? \"\", 4000),\n    memoryAtDeath: memoryNow(),\n  };\n  const file = path.join(dir, `${DEATH_FILE_PREFIX}-${marker.wall}.ndjson`);\n  fs.writeFileSync(file, `${JSON.stringify(marker)}\\n`);\n  return file;\n}\n\n/**\n * Crash markers in a directory, oldest first, each with the file it came from.\n *\n * The path travels with the marker so that a caller collecting the evidence — to\n * attach to a report, say — never has to spell the file name itself and cannot\n * drift from the one this module writes.\n */\nexport function readCrashMarkers(dir: string): StoredCrashMarker[] {\n  let names: string[];\n  try {\n    names = fs.readdirSync(dir);\n  } catch {\n    return [];\n  }\n  const markers: StoredCrashMarker[] = [];\n  for (const name of names) {\n    if (!name.startsWith(`${DEATH_FILE_PREFIX}-`) || !name.endsWith(\".ndjson\")) continue;\n    const file = path.join(dir, name);\n    try {\n      const first = fs.readFileSync(file, \"utf8\").split(\"\\n\")[0];\n      markers.push({ ...(JSON.parse(first) as CrashMarker), file });\n    } catch {\n      // A marker that cannot be parsed is skipped; it is one line of evidence,\n      // not the report.\n    }\n  }\n  return markers.sort((lhs, rhs) => lhs.wall - rhs.wall);\n}\n\n/**\n * Attaches crash recording to a middle-layer worker thread.\n *\n * A worker whose isolate exhausts its heap dies alone and the parent receives\n * `ERR_WORKER_OUT_OF_MEMORY`, with or without `resourceLimits`. What\n * `resourceLimits.maxOldGenerationSizeMb` adds is a chosen ceiling: V8's default\n * is several gigabytes, so on a small machine the OS can run out of memory and\n * kill the whole process before V8 ever reports the worker's heap as full — and\n * then there is no parent left to write anything.\n */\nexport function superviseWorker(\n  worker: SupervisedWorker,\n  dir: string,\n  options: SuperviseOptions = {},\n): void {\n  // One death fires `error` and then `exit`. Only `error` carries the cause, so\n  // a later `exit` must not overwrite it with a bare exit code.\n  let recorded = false;\n  worker.on(\"error\", (error: Error) => {\n    recorded = true;\n    const markerFile = writeCrashMarker(dir, { error, sessionId: options.sessionId });\n    options.onCrash?.({ kind: \"error\", error, markerFile });\n  });\n  worker.on(\"exit\", (code: number) => {\n    if (code === 0 || recorded) return;\n    const markerFile = writeCrashMarker(dir, {\n      reason: \"worker-exit\",\n      code,\n      sessionId: options.sessionId,\n    });\n    options.onCrash?.({ kind: \"exit\", code, markerFile });\n  });\n}\n\n// Internals\n\n/**\n * The parent's view of memory at the moment it saw the death.\n *\n * The dying thread cannot take this reading, and the sampler's last one predates\n * the end by up to its interval. Taken here it is contemporaneous with the exit\n * code it sits beside, which is what stops an exhausted machine from reading as\n * an ordinary failure.\n *\n * Every reading here is a syscall. The machine's compressor and swap totals are\n * deliberately not among them: on macOS they cost a subprocess, and this runs on\n * the parent's event loop inside the worker's error handler, before the marker\n * is written and before the caller learns of the death. A fork is exactly what\n * becomes slow or impossible on the exhausted machine this code exists for, so\n * the fuller picture is left to the sampler, whose last reading is at most one\n * interval old and sits in the same bundle.\n */\nfunction memoryNow(): CrashMarker[\"memoryAtDeath\"] {\n  try {\n    return {\n      rss: process.memoryUsage.rss(),\n      maxRss: process.resourceUsage().maxRSS * 1024,\n      freeMemory: os.freemem(),\n      totalMemory: os.totalmem(),\n    };\n  } catch {\n    // A marker without memory is still a marker; failing to take the reading\n    // must never cost the record of the death itself.\n    return undefined;\n  }\n}\n\n// Advisory only, for a human reading a directory by hand: the dying session has\n// no terminating record, so among the sessions that look dead this names the one\n// that wrote last. Never used as identity — see `CrashMarker.guessedSessionId`.\nfunction newestOpenSessionId(dir: string): string | undefined {\n  const open = listSessions(dir).find((session) => session.crashed);\n  return open ? sessionIdFromFile(open.file) : undefined;\n}\n\nfunction classifyReason({ error, code, signal }: CrashMarkerInput): CrashReason {\n  const errorCode = (error as { code?: string } | undefined)?.code;\n  if (errorCode === \"ERR_WORKER_OUT_OF_MEMORY\") return \"js-heap-out-of-memory\";\n  if (signal === \"SIGKILL\") return \"killed-by-os\";\n  if (signal === \"SIGABRT\" || code === 134) return \"abort-or-fatal-allocation-failure\";\n  if (typeof code === \"number\" && code !== 0) return \"nonzero-exit\";\n  return \"unknown\";\n}\n\nfunction truncate(value: string, limit: number): string {\n  return value.length > limit ? `${value.slice(0, limit)}…` : value;\n}\n"],"mappings":";;;;;;;;;;;;;;;AAkDA,SAAgB,iBAAiB,KAAa,QAA0B,CAAC,GAAW;CAClF,GAAG,UAAU,KAAK,EAAE,WAAW,KAAK,CAAC;CACrC,MAAM,QAAQ,MAAM;CAMpB,MAAM,SAAsB;EAC1B,MAAM;EACN,MAAM,KAAK,IAAI;EACf,WAAW,MAAM;EACjB,kBAAkB,MAAM,cAAc,KAAA,IAAY,oBAAoB,GAAG,IAAI,KAAA;EAC7E,QAAQ,MAAM,UAAU,eAAe,KAAK;EAC5C,WAAW,OAAO;EAClB,WAAW,OAAO;EAClB,SAAS,SAAS,OAAO,OAAO,WAAW,MAAM,SAAS,EAAE,GAAG,GAAI;EACnE,UAAU,MAAM;EAChB,QAAQ,MAAM;EACd,YAAY,SAAS,MAAM,cAAc,IAAI,GAAI;EACjD,eAAe,UAAU;CAC3B;CACA,MAAM,OAAO,KAAK,KAAK,KAAK,GAAG,kBAAkB,GAAG,OAAO,KAAK,QAAQ;CACxE,GAAG,cAAc,MAAM,GAAG,KAAK,UAAU,MAAM,EAAE,GAAG;CACpD,OAAO;AACT;;;;;;;;AASA,SAAgB,iBAAiB,KAAkC;CACjE,IAAI;CACJ,IAAI;EACF,QAAQ,GAAG,YAAY,GAAG;CAC5B,QAAQ;EACN,OAAO,CAAC;CACV;CACA,MAAM,UAA+B,CAAC;CACtC,KAAK,MAAM,QAAQ,OAAO;EACxB,IAAI,CAAC,KAAK,WAAW,QAAuB,KAAK,CAAC,KAAK,SAAS,SAAS,GAAG;EAC5E,MAAM,OAAO,KAAK,KAAK,KAAK,IAAI;EAChC,IAAI;GACF,MAAM,QAAQ,GAAG,aAAa,MAAM,MAAM,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC;GACxD,QAAQ,KAAK;IAAE,GAAI,KAAK,MAAM,KAAK;IAAmB;GAAK,CAAC;EAC9D,QAAQ,CAGR;CACF;CACA,OAAO,QAAQ,MAAM,KAAK,QAAQ,IAAI,OAAO,IAAI,IAAI;AACvD;;;;;;;;;;;AAYA,SAAgB,gBACd,QACA,KACA,UAA4B,CAAC,GACvB;CAGN,IAAI,WAAW;CACf,OAAO,GAAG,UAAU,UAAiB;EACnC,WAAW;EACX,MAAM,aAAa,iBAAiB,KAAK;GAAE;GAAO,WAAW,QAAQ;EAAU,CAAC;EAChF,QAAQ,UAAU;GAAE,MAAM;GAAS;GAAO;EAAW,CAAC;CACxD,CAAC;CACD,OAAO,GAAG,SAAS,SAAiB;EAClC,IAAI,SAAS,KAAK,UAAU;EAC5B,MAAM,aAAa,iBAAiB,KAAK;GACvC,QAAQ;GACR;GACA,WAAW,QAAQ;EACrB,CAAC;EACD,QAAQ,UAAU;GAAE,MAAM;GAAQ;GAAM;EAAW,CAAC;CACtD,CAAC;AACH;;;;;;;;;;;;;;;;;AAoBA,SAAS,YAA0C;CACjD,IAAI;EACF,OAAO;GACL,KAAK,QAAQ,YAAY,IAAI;GAC7B,QAAQ,QAAQ,cAAc,CAAC,CAAC,SAAS;GACzC,YAAY,GAAG,QAAQ;GACvB,aAAa,GAAG,SAAS;EAC3B;CACF,QAAQ;EAGN;CACF;AACF;AAKA,SAAS,oBAAoB,KAAiC;CAC5D,MAAM,OAAO,aAAa,GAAG,CAAC,CAAC,MAAM,YAAY,QAAQ,OAAO;CAChE,OAAO,OAAO,kBAAkB,KAAK,IAAI,IAAI,KAAA;AAC/C;AAEA,SAAS,eAAe,EAAE,OAAO,MAAM,UAAyC;CAE9E,IADmB,OAAyC,SAC1C,4BAA4B,OAAO;CACrD,IAAI,WAAW,WAAW,OAAO;CACjC,IAAI,WAAW,aAAa,SAAS,KAAK,OAAO;CACjD,IAAI,OAAO,SAAS,YAAY,SAAS,GAAG,OAAO;CACnD,OAAO;AACT;AAEA,SAAS,SAAS,OAAe,OAAuB;CACtD,OAAO,MAAM,SAAS,QAAQ,GAAG,MAAM,MAAM,GAAG,KAAK,EAAE,KAAK;AAC9D"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@milaboratories/pl-crash-recorder",
-  "version": "0.3.0",
+  "version": "0.3.2",
   "description": "Crash-survivable diagnostics for the block model layer: records join shapes and memory, and explains an out-of-memory death after the fact",
   "keywords": [],
   "license": "UNLICENSED",
@@ -19,13 +19,13 @@
     }
   },
   "dependencies": {
-    "@milaboratories/pl-model-common": "1.49.0"
+    "@milaboratories/pl-model-common": "1.50.0"
   },
   "devDependencies": {
     "@vitest/coverage-istanbul": "^4.1.3",
     "typescript": "7.0.2",
     "vitest": "^4.1.3",
-    "@milaboratories/build-configs": "2.0.1",
+    "@milaboratories/build-configs": "2.0.2",
     "@milaboratories/ts-configs": "1.4.0",
     "@milaboratories/ts-builder": "1.7.2"
   },

package/src/index.ts

@@ -27,6 +27,7 @@ export {
   readCrashMarkers,
   superviseWorker,
   type SupervisedWorker,
+  type StoredCrashMarker,
   type SuperviseOptions,
 } from "./supervisor";
 

package/src/machine_memory.test.ts

@@ -0,0 +1,78 @@
+import { describe, expect, test } from "vitest";
+import type { MachineMemory } from "./events";
+import { createMachineMemoryReader } from "./machine_memory";
+
+describe("machine memory sampling", () => {
+  test("a platform that cannot take the reading says so once", () => {
+    const read = (): MachineMemory => {
+      throw new Error("must not be attempted where there is no source");
+    };
+    const next = createMachineMemoryReader({
+      intervalMs: 1000,
+      read,
+      unsupportedReason: () => "no machine-wide memory source on win32: ...",
+    });
+
+    // Windows is most of the installed base, and a thirty-five second session
+    // once carried thirty-four copies of this line between the samples that
+    // hold the memory curve.
+    expect(next(0)?.unavailable).toContain("win32");
+    for (let now = 1; now <= 5000; now += 250) expect(next(now)).toBeUndefined();
+  });
+
+  test("a source that fails is stated once and kept on", () => {
+    let outcome: MachineMemory = { unavailable: "vm_stat timed out" };
+    const next = createMachineMemoryReader({
+      intervalMs: 1000,
+      read: () => outcome,
+      unsupportedReason: () => undefined,
+    });
+
+    expect(next(0)?.unavailable).toBe("vm_stat timed out");
+    // Stated, not repeated: the same failure every second would bury the curve.
+    expect(next(1000)).toBeUndefined();
+    expect(next(2000)).toBeUndefined();
+
+    // A reading that fails under memory pressure is the one worth having a
+    // moment later, so the source is still being asked.
+    outcome = { swapUsed: 7 };
+    expect(next(3000)).toEqual({ swapUsed: 7 });
+
+    // And a failure after a recovery is news again.
+    outcome = { unavailable: "vm_stat timed out" };
+    expect(next(4000)?.unavailable).toBe("vm_stat timed out");
+  });
+
+  test("the reading is taken on its own interval, not with every sample", () => {
+    let calls = 0;
+    const next = createMachineMemoryReader({
+      intervalMs: 1000,
+      read: () => {
+        calls++;
+        return { swapUsed: calls };
+      },
+      unsupportedReason: () => undefined,
+    });
+
+    next(0);
+    for (let now = 250; now < 1000; now += 250) expect(next(now)).toBeUndefined();
+    expect(next(1000)).toEqual({ swapUsed: 2 });
+    expect(calls).toBe(2);
+  });
+
+  test("a different failure is worth a record of its own", () => {
+    const outcomes = ["vm_stat timed out", "sysctl: no such file", "sysctl: no such file"];
+    let tick = 0;
+    const next = createMachineMemoryReader({
+      intervalMs: 1000,
+      read: () => ({ unavailable: outcomes[tick++] }),
+      unsupportedReason: () => undefined,
+    });
+
+    // Suppression is per reason, not a blanket silence: a source failing a new
+    // way is evidence the previous record does not carry.
+    expect(next(0)?.unavailable).toBe("vm_stat timed out");
+    expect(next(1000)?.unavailable).toBe("sysctl: no such file");
+    expect(next(2000)).toBeUndefined();
+  });
+});

package/src/machine_memory.ts

@@ -14,12 +14,74 @@ export function readMachineMemory(): MachineMemory {
   try {
     if (process.platform === "darwin") return readDarwin();
     if (process.platform === "linux") return readLinux();
-    return { unavailable: `not implemented for ${process.platform}` };
+    return { unavailable: machineMemoryUnsupported() };
   } catch (error: unknown) {
     return { unavailable: error instanceof Error ? error.message : String(error) };
   }
 }
 
+/**
+ * Decides what each sampler tick should record about machine-wide memory.
+ *
+ * Three things are being balanced. The reading costs a subprocess, so it is
+ * taken on its own interval rather than with every sample. Repeating an
+ * unchanged failure every second buries the resident-size curve it sits beside,
+ * so a reason is written once and not again while it still holds. And a failure
+ * is not the end of the matter: a `vm_stat` that timed out did so under load,
+ * which is precisely the moment its figures are worth having a second later —
+ * so the source keeps being tried, and a reading that comes back is recorded.
+ *
+ * Only an unsupported platform stops it for good, because that is the one
+ * condition a running process cannot get out of.
+ */
+export function createMachineMemoryReader(options: {
+  intervalMs: number;
+  read?: () => MachineMemory;
+  unsupportedReason?: () => string | undefined;
+}): (now: number) => MachineMemory | undefined {
+  const read = options.read ?? readMachineMemory;
+  const unsupported = (options.unsupportedReason ?? machineMemoryUnsupportedReason)();
+
+  let pending = unsupported;
+  let dueAt = 0;
+  let statedFailure: string | undefined;
+
+  return (now: number): MachineMemory | undefined => {
+    if (pending !== undefined) {
+      const reason = pending;
+      pending = undefined;
+      return { unavailable: reason };
+    }
+    if (unsupported !== undefined || now < dueAt) return undefined;
+
+    dueAt = now + options.intervalMs;
+    const reading = read();
+    if (reading.unavailable === undefined) {
+      // A recovery is worth seeing: the gap in the series ends where the
+      // figures resume.
+      statedFailure = undefined;
+      return reading;
+    }
+    if (reading.unavailable === statedFailure) return undefined;
+    statedFailure = reading.unavailable;
+    return reading;
+  };
+}
+
+/**
+ * Names what the machine-wide reading would have contributed, for platforms
+ * where it cannot be taken at all.
+ *
+ * A reader who finds no compressor or swap figures needs to know whether the
+ * machine had none or the sampler never asked, and where the equivalent evidence
+ * is instead. Callers record it once: it is a property of the platform and
+ * repeating it every second buries the curve it was meant to explain.
+ */
+export function machineMemoryUnsupportedReason(): string | undefined {
+  if (process.platform === "darwin" || process.platform === "linux") return undefined;
+  return machineMemoryUnsupported();
+}
+
 // Internals
 
 function readDarwin(): MachineMemory {
@@ -68,3 +130,12 @@ function readLinux(): MachineMemory {
     wired: kb("Unevictable"),
   };
 }
+
+function machineMemoryUnsupported(): string {
+  return (
+    `no machine-wide memory source on ${process.platform}: ` +
+    "compressor, swap and anonymous/file-backed totals are not sampled " +
+    "(vm_stat and sysctl are macOS-only, /proc/meminfo Linux-only). " +
+    "Per-process committed bytes are recorded instead, as `private` in the host log."
+  );
+}

package/src/recorder.test.ts

@@ -253,6 +253,17 @@ describe("crash markers", () => {
     expect(marker.guessedSessionId).toBeDefined();
   });
 
+  test("a marker carries the file it was read from", () => {
+    const written = writeCrashMarker(dir, { reason: "worker-exit", code: 1 });
+
+    // A consumer collecting evidence attaches this path. Spelling the name a
+    // second time on its own is how a rename silently drops the one record the
+    // parent contributes.
+    const [marker] = readCrashMarkers(dir);
+    expect(marker.file).toBe(written);
+    expect(fs.existsSync(marker.file)).toBe(true);
+  });
+
   test("an unparseable marker is skipped rather than failing the rest", () => {
     writeCrashMarker(dir, { reason: "worker-exit" });
     // Named like a marker, so it reaches the parse rather than being filtered

package/src/sampler_thread.ts

@@ -21,7 +21,7 @@ import fs from "node:fs";
 import os from "node:os";
 import { workerData } from "node:worker_threads";
 import type { SamplerRecord } from "./events";
-import { readMachineMemory } from "./machine_memory";
+import { createMachineMemoryReader } from "./machine_memory";
 
 type SamplerWorkerData = { file: string; intervalMs: number; machineIntervalMs?: number };
 
@@ -29,16 +29,15 @@ const { file, intervalMs, machineIntervalMs = 1000 } = workerData as SamplerWork
 const fd = fs.openSync(file, "a");
 let seq = 0;
 let peakRss = 0;
-let machineDueAt = 0;
+const readMachine = createMachineMemoryReader({ intervalMs: machineIntervalMs });
 
 setInterval(() => {
   const rss = process.memoryUsage.rss();
   if (rss > peakRss) peakRss = rss;
   const now = Date.now();
-  // Taken on the first tick and then on its own schedule, so the curve keeps its
-  // sampling rate while the costlier reading stays occasional.
-  const machine = now >= machineDueAt ? readMachineMemory() : undefined;
-  if (machine) machineDueAt = now + machineIntervalMs;
+  // Taken on its own schedule, and only when it has something to say, so the
+  // curve keeps its sampling rate while the costlier reading stays occasional.
+  const machine = readMachine(now);
   const record: SamplerRecord = {
     seq: ++seq,
     t: Math.round(performance.now() * 1000) / 1000,

package/src/supervisor.ts

@@ -14,6 +14,9 @@ type CrashMarkerInput = {
   stderrTail?: string;
 };
 
+/** A crash marker together with the file holding it. */
+export type StoredCrashMarker = CrashMarker & { file: string };
+
 export type SuperviseOptions = {
   /**
    * The session id handed to the worker at spawn (see `CRASH_SESSION_ENV`).
@@ -72,20 +75,27 @@ export function writeCrashMarker(dir: string, input: CrashMarkerInput = {}): str
   return file;
 }
 
-/** Crash markers in a directory, oldest first. */
-export function readCrashMarkers(dir: string): CrashMarker[] {
+/**
+ * Crash markers in a directory, oldest first, each with the file it came from.
+ *
+ * The path travels with the marker so that a caller collecting the evidence — to
+ * attach to a report, say — never has to spell the file name itself and cannot
+ * drift from the one this module writes.
+ */
+export function readCrashMarkers(dir: string): StoredCrashMarker[] {
   let names: string[];
   try {
     names = fs.readdirSync(dir);
   } catch {
     return [];
   }
-  const markers: CrashMarker[] = [];
+  const markers: StoredCrashMarker[] = [];
   for (const name of names) {
     if (!name.startsWith(`${DEATH_FILE_PREFIX}-`) || !name.endsWith(".ndjson")) continue;
+    const file = path.join(dir, name);
     try {
-      const first = fs.readFileSync(path.join(dir, name), "utf8").split("\n")[0];
-      markers.push(JSON.parse(first) as CrashMarker);
+      const first = fs.readFileSync(file, "utf8").split("\n")[0];
+      markers.push({ ...(JSON.parse(first) as CrashMarker), file });
     } catch {
       // A marker that cannot be parsed is skipped; it is one line of evidence,
       // not the report.