@milaboratories/pl-flight-recorder 0.2.0

package/dist/supervisor.js

@@ -0,0 +1,108 @@
+import { CRASH_FILE_PREFIX } from "./events.js";
+import { listSessions, sessionIdFromFile } from "./recorder.js";
+import fs from "node:fs";
+import path from "node:path";
+//#region src/supervisor.ts
+/**
+* Records an abnormal end observed from outside the dying thread.
+*
+* A thread that runs out of heap cannot describe its own death: the last reading
+* it wrote predates the blow-up, and when the blow-up is synchronous no sampler
+* tick of its own lands either. The parent is the only place where the cause is
+* known rather than inferred — Node reports `ERR_WORKER_OUT_OF_MEMORY` to it —
+* so the parent writes the verdict down on the dead thread's behalf.
+*/
+function writeCrashMarker(dir, input = {}) {
+	fs.mkdirSync(dir, { recursive: true });
+	const error = input.error;
+	const marker = {
+		type: "external-crash",
+		wall: Date.now(),
+		sessionId: input.sessionId,
+		guessedSessionId: input.sessionId === void 0 ? newestOpenSessionId(dir) : void 0,
+		reason: input.reason ?? classifyReason(input),
+		errorCode: error?.code,
+		errorName: error?.name,
+		message: truncate(String(error?.message ?? input.error ?? ""), 2e3),
+		exitCode: input.code,
+		signal: input.signal,
+		stderrTail: truncate(input.stderrTail ?? "", 4e3)
+	};
+	const file = path.join(dir, `${CRASH_FILE_PREFIX}-${marker.wall}.ndjson`);
+	fs.writeFileSync(file, `${JSON.stringify(marker)}\n`);
+	return file;
+}
+/** Crash markers in a directory, oldest first. */
+function readCrashMarkers(dir) {
+	let names;
+	try {
+		names = fs.readdirSync(dir);
+	} catch {
+		return [];
+	}
+	const markers = [];
+	for (const name of names) {
+		if (!name.startsWith(`crash-`) || !name.endsWith(".ndjson")) continue;
+		try {
+			const first = fs.readFileSync(path.join(dir, name), "utf8").split("\n")[0];
+			markers.push(JSON.parse(first));
+		} catch {}
+	}
+	return markers.sort((lhs, rhs) => lhs.wall - rhs.wall);
+}
+/**
+* Attaches crash recording to a middle-layer worker thread.
+*
+* A worker whose isolate exhausts its heap dies alone and the parent receives
+* `ERR_WORKER_OUT_OF_MEMORY`, with or without `resourceLimits`. What
+* `resourceLimits.maxOldGenerationSizeMb` adds is a chosen ceiling: V8's default
+* is several gigabytes, so on a small machine the OS can run out of memory and
+* kill the whole process before V8 ever reports the worker's heap as full — and
+* then there is no parent left to write anything.
+*/
+function superviseWorker(worker, dir, options = {}) {
+	let recorded = false;
+	worker.on("error", (error) => {
+		recorded = true;
+		const markerFile = writeCrashMarker(dir, {
+			error,
+			sessionId: options.sessionId
+		});
+		options.onCrash?.({
+			kind: "error",
+			error,
+			markerFile
+		});
+	});
+	worker.on("exit", (code) => {
+		if (code === 0 || recorded) return;
+		const markerFile = writeCrashMarker(dir, {
+			reason: "worker-exit",
+			code,
+			sessionId: options.sessionId
+		});
+		options.onCrash?.({
+			kind: "exit",
+			code,
+			markerFile
+		});
+	});
+}
+function newestOpenSessionId(dir) {
+	const open = listSessions(dir).find((session) => session.crashed);
+	return open ? sessionIdFromFile(open.file) : void 0;
+}
+function classifyReason({ error, code, signal }) {
+	if (error?.code === "ERR_WORKER_OUT_OF_MEMORY") return "js-heap-out-of-memory";
+	if (signal === "SIGKILL") return "killed-by-os";
+	if (signal === "SIGABRT" || code === 134) return "abort-or-fatal-allocation-failure";
+	if (typeof code === "number" && code !== 0) return "nonzero-exit";
+	return "unknown";
+}
+function truncate(value, limit) {
+	return value.length > limit ? `${value.slice(0, limit)}…` : value;
+}
+//#endregion
+export { readCrashMarkers, superviseWorker, writeCrashMarker };
+
+//# sourceMappingURL=supervisor.js.map

package/dist/supervisor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"supervisor.js","names":[],"sources":["../src/supervisor.ts"],"sourcesContent":["import fs from \"node:fs\";\nimport path from \"node:path\";\nimport { CRASH_FILE_PREFIX, type CrashMarker, type CrashReason } from \"./events\";\nimport { listSessions, sessionIdFromFile } from \"./recorder\";\n\nexport type CrashMarkerInput = {\n  /** Session id the parent assigned to the worker. Omitted, the marker carries no identity. */\n  sessionId?: string;\n  reason?: CrashReason;\n  error?: (Error & { code?: string }) | unknown;\n  code?: number;\n  signal?: string;\n  stderrTail?: string;\n};\n\nexport type SuperviseOptions = {\n  /**\n   * The session id handed to the worker at spawn (see `FLIGHT_SESSION_ENV`).\n   * With it the marker names the dying session with certainty. Without it the\n   * analyzer has to attribute the marker by timing, and will decline to\n   * attribute it at all when more than one session looks dead.\n   */\n  sessionId?: string;\n  onCrash?: (info: {\n    kind: \"error\" | \"exit\";\n    markerFile: string;\n    error?: unknown;\n    code?: number;\n  }) => void;\n};\n\n/** Minimal view of a worker, so callers are not forced to import worker_threads. */\nexport type SupervisedWorker = {\n  on(event: \"error\", listener: (error: Error) => void): unknown;\n  on(event: \"exit\", listener: (code: number) => void): unknown;\n};\n\n/**\n * Records an abnormal end observed from outside the dying thread.\n *\n * A thread that runs out of heap cannot describe its own death: the last reading\n * it wrote predates the blow-up, and when the blow-up is synchronous no sampler\n * tick of its own lands either. The parent is the only place where the cause is\n * known rather than inferred — Node reports `ERR_WORKER_OUT_OF_MEMORY` to it —\n * so the parent writes the verdict down on the dead thread's behalf.\n */\nexport function writeCrashMarker(dir: string, input: CrashMarkerInput = {}): string {\n  fs.mkdirSync(dir, { recursive: true });\n  const error = input.error as (Error & { code?: string }) | undefined;\n  // Only an id the parent handed to the worker is certain, and only a certain\n  // id goes in `sessionId`. Reading the newest open flight log names whichever\n  // session wrote last, which a concurrent live session makes wrong; recorded\n  // as identity that would misattribute the death and, worse, stop the session\n  // that actually died from claiming the marker. So it is advisory only.\n  const marker: CrashMarker = {\n    type: \"external-crash\",\n    wall: Date.now(),\n    sessionId: input.sessionId,\n    guessedSessionId: input.sessionId === undefined ? newestOpenSessionId(dir) : undefined,\n    reason: input.reason ?? classifyReason(input),\n    errorCode: error?.code,\n    errorName: error?.name,\n    message: truncate(String(error?.message ?? input.error ?? \"\"), 2000),\n    exitCode: input.code,\n    signal: input.signal,\n    stderrTail: truncate(input.stderrTail ?? \"\", 4000),\n  };\n  const file = path.join(dir, `${CRASH_FILE_PREFIX}-${marker.wall}.ndjson`);\n  fs.writeFileSync(file, `${JSON.stringify(marker)}\\n`);\n  return file;\n}\n\n/** Crash markers in a directory, oldest first. */\nexport function readCrashMarkers(dir: string): CrashMarker[] {\n  let names: string[];\n  try {\n    names = fs.readdirSync(dir);\n  } catch {\n    return [];\n  }\n  const markers: CrashMarker[] = [];\n  for (const name of names) {\n    if (!name.startsWith(`${CRASH_FILE_PREFIX}-`) || !name.endsWith(\".ndjson\")) continue;\n    try {\n      const first = fs.readFileSync(path.join(dir, name), \"utf8\").split(\"\\n\")[0];\n      markers.push(JSON.parse(first) as CrashMarker);\n    } catch {\n      // A marker that cannot be parsed is skipped; it is one line of evidence,\n      // not the report.\n    }\n  }\n  return markers.sort((lhs, rhs) => lhs.wall - rhs.wall);\n}\n\n/**\n * Attaches crash recording to a middle-layer worker thread.\n *\n * A worker whose isolate exhausts its heap dies alone and the parent receives\n * `ERR_WORKER_OUT_OF_MEMORY`, with or without `resourceLimits`. What\n * `resourceLimits.maxOldGenerationSizeMb` adds is a chosen ceiling: V8's default\n * is several gigabytes, so on a small machine the OS can run out of memory and\n * kill the whole process before V8 ever reports the worker's heap as full — and\n * then there is no parent left to write anything.\n */\nexport function superviseWorker(\n  worker: SupervisedWorker,\n  dir: string,\n  options: SuperviseOptions = {},\n): void {\n  // One death fires `error` and then `exit`. Only `error` carries the cause, so\n  // a later `exit` must not overwrite it with a bare exit code.\n  let recorded = false;\n  worker.on(\"error\", (error: Error) => {\n    recorded = true;\n    const markerFile = writeCrashMarker(dir, { error, sessionId: options.sessionId });\n    options.onCrash?.({ kind: \"error\", error, markerFile });\n  });\n  worker.on(\"exit\", (code: number) => {\n    if (code === 0 || recorded) return;\n    const markerFile = writeCrashMarker(dir, {\n      reason: \"worker-exit\",\n      code,\n      sessionId: options.sessionId,\n    });\n    options.onCrash?.({ kind: \"exit\", code, markerFile });\n  });\n}\n\n// Internals\n\n// Advisory only, for a human reading a directory by hand: the dying session has\n// no terminating record, so among the sessions that look dead this names the one\n// that wrote last. Never used as identity — see `CrashMarker.guessedSessionId`.\nfunction newestOpenSessionId(dir: string): string | undefined {\n  const open = listSessions(dir).find((session) => session.crashed);\n  return open ? sessionIdFromFile(open.file) : undefined;\n}\n\nfunction classifyReason({ error, code, signal }: CrashMarkerInput): CrashReason {\n  const errorCode = (error as { code?: string } | undefined)?.code;\n  if (errorCode === \"ERR_WORKER_OUT_OF_MEMORY\") return \"js-heap-out-of-memory\";\n  if (signal === \"SIGKILL\") return \"killed-by-os\";\n  if (signal === \"SIGABRT\" || code === 134) return \"abort-or-fatal-allocation-failure\";\n  if (typeof code === \"number\" && code !== 0) return \"nonzero-exit\";\n  return \"unknown\";\n}\n\nfunction truncate(value: string, limit: number): string {\n  return value.length > limit ? `${value.slice(0, limit)}…` : value;\n}\n"],"mappings":";;;;;;;;;;;;;;AA8CA,SAAgB,iBAAiB,KAAa,QAA0B,CAAC,GAAW;CAClF,GAAG,UAAU,KAAK,EAAE,WAAW,KAAK,CAAC;CACrC,MAAM,QAAQ,MAAM;CAMpB,MAAM,SAAsB;EAC1B,MAAM;EACN,MAAM,KAAK,IAAI;EACf,WAAW,MAAM;EACjB,kBAAkB,MAAM,cAAc,KAAA,IAAY,oBAAoB,GAAG,IAAI,KAAA;EAC7E,QAAQ,MAAM,UAAU,eAAe,KAAK;EAC5C,WAAW,OAAO;EAClB,WAAW,OAAO;EAClB,SAAS,SAAS,OAAO,OAAO,WAAW,MAAM,SAAS,EAAE,GAAG,GAAI;EACnE,UAAU,MAAM;EAChB,QAAQ,MAAM;EACd,YAAY,SAAS,MAAM,cAAc,IAAI,GAAI;CACnD;CACA,MAAM,OAAO,KAAK,KAAK,KAAK,GAAG,kBAAkB,GAAG,OAAO,KAAK,QAAQ;CACxE,GAAG,cAAc,MAAM,GAAG,KAAK,UAAU,MAAM,EAAE,GAAG;CACpD,OAAO;AACT;;AAGA,SAAgB,iBAAiB,KAA4B;CAC3D,IAAI;CACJ,IAAI;EACF,QAAQ,GAAG,YAAY,GAAG;CAC5B,QAAQ;EACN,OAAO,CAAC;CACV;CACA,MAAM,UAAyB,CAAC;CAChC,KAAK,MAAM,QAAQ,OAAO;EACxB,IAAI,CAAC,KAAK,WAAW,QAAuB,KAAK,CAAC,KAAK,SAAS,SAAS,GAAG;EAC5E,IAAI;GACF,MAAM,QAAQ,GAAG,aAAa,KAAK,KAAK,KAAK,IAAI,GAAG,MAAM,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC;GACxE,QAAQ,KAAK,KAAK,MAAM,KAAK,CAAgB;EAC/C,QAAQ,CAGR;CACF;CACA,OAAO,QAAQ,MAAM,KAAK,QAAQ,IAAI,OAAO,IAAI,IAAI;AACvD;;;;;;;;;;;AAYA,SAAgB,gBACd,QACA,KACA,UAA4B,CAAC,GACvB;CAGN,IAAI,WAAW;CACf,OAAO,GAAG,UAAU,UAAiB;EACnC,WAAW;EACX,MAAM,aAAa,iBAAiB,KAAK;GAAE;GAAO,WAAW,QAAQ;EAAU,CAAC;EAChF,QAAQ,UAAU;GAAE,MAAM;GAAS;GAAO;EAAW,CAAC;CACxD,CAAC;CACD,OAAO,GAAG,SAAS,SAAiB;EAClC,IAAI,SAAS,KAAK,UAAU;EAC5B,MAAM,aAAa,iBAAiB,KAAK;GACvC,QAAQ;GACR;GACA,WAAW,QAAQ;EACrB,CAAC;EACD,QAAQ,UAAU;GAAE,MAAM;GAAQ;GAAM;EAAW,CAAC;CACtD,CAAC;AACH;AAOA,SAAS,oBAAoB,KAAiC;CAC5D,MAAM,OAAO,aAAa,GAAG,CAAC,CAAC,MAAM,YAAY,QAAQ,OAAO;CAChE,OAAO,OAAO,kBAAkB,KAAK,IAAI,IAAI,KAAA;AAC/C;AAEA,SAAS,eAAe,EAAE,OAAO,MAAM,UAAyC;CAE9E,IADmB,OAAyC,SAC1C,4BAA4B,OAAO;CACrD,IAAI,WAAW,WAAW,OAAO;CACjC,IAAI,WAAW,aAAa,SAAS,KAAK,OAAO;CACjD,IAAI,OAAO,SAAS,YAAY,SAAS,GAAG,OAAO;CACnD,OAAO;AACT;AAEA,SAAS,SAAS,OAAe,OAAuB;CACtD,OAAO,MAAM,SAAS,QAAQ,GAAG,MAAM,MAAM,GAAG,KAAK,EAAE,KAAK;AAC9D"}

package/package.json

@@ -0,0 +1,43 @@
+{
+  "name": "@milaboratories/pl-flight-recorder",
+  "version": "0.2.0",
+  "description": "Crash-survivable diagnostics for the block model layer: records join shapes and memory, and explains an out-of-memory death after the fact",
+  "keywords": [],
+  "license": "UNLICENSED",
+  "files": [
+    "./dist/**/*",
+    "./src/**/*"
+  ],
+  "type": "module",
+  "main": "./dist/index.js",
+  "module": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "dependencies": {
+    "@milaboratories/pl-model-common": "1.48.0"
+  },
+  "devDependencies": {
+    "@vitest/coverage-istanbul": "^4.1.3",
+    "typescript": "7.0.2",
+    "vitest": "^4.1.3",
+    "@milaboratories/build-configs": "2.0.1",
+    "@milaboratories/ts-configs": "1.4.0",
+    "@milaboratories/ts-builder": "1.7.2"
+  },
+  "scripts": {
+    "build": "ts-builder build --target node --build-config build.config.js",
+    "watch": "ts-builder build --target node --build-config build.config.js --watch",
+    "check": "ts-builder check --target node",
+    "formatter:check": "ts-builder formatter --check",
+    "linter:check": "ts-builder linter --check",
+    "types:check": "ts-builder type-check --target node",
+    "test": "vitest run --coverage",
+    "do-pack": "rm -f *.tgz && pnpm pack && mv *.tgz package.tgz",
+    "fmt": "ts-builder format"
+  }
+}