@milaboratories/pl-crash-recorder 0.3.0

package/src/host_sampler.ts

@@ -0,0 +1,83 @@
+import fs from "node:fs";
+import path from "node:path";
+import { HOST_FILE_PREFIX, type HostRecord } from "./events";
+
+export type HostReading = Omit<HostRecord, "seq" | "t" | "wall" | "type">;
+
+export type HostSamplerOptions = {
+  dir: string;
+  sessionId: string;
+  /**
+   * Takes one reading. Supplied by the caller because the useful numbers come
+   * from the application framework rather than from Node, and this package must
+   * not depend on it.
+   */
+  read: () => HostReading | Promise<HostReading>;
+  /** Sampling period; these readings change slowly and cost more than resident size. */
+  intervalMs?: number;
+};
+
+export type HostSampler = {
+  /** Sibling log this sampler appends to. */
+  readonly file: string;
+  stop(): void;
+};
+
+/**
+ * Records what only the host process can measure, beside the session it hosts.
+ *
+ * Its own file rather than the session log: that log belongs to another thread
+ * and has its own descriptor and sequence, and two writers sharing them would
+ * corrupt both. Readers join the two by wall clock, as they already do with the
+ * memory sampler.
+ *
+ * This one runs wherever the caller runs, so a blocked host stops it — which is
+ * why it supplements the sampler thread rather than replacing it. What it adds
+ * is attribution, and attribution a second stale is still attribution.
+ */
+export function startHostSampler(options: HostSamplerOptions): HostSampler {
+  const { dir, sessionId, read, intervalMs = 1000 } = options;
+  fs.mkdirSync(dir, { recursive: true });
+  const file = path.join(dir, `${HOST_FILE_PREFIX}-${sessionId}.ndjson`);
+  const fd = fs.openSync(file, "a");
+  let seq = 0;
+  let writing = false;
+
+  const timer = setInterval(() => {
+    // A reading that outlives its interval must not queue up behind itself.
+    if (writing) return;
+    writing = true;
+    void Promise.resolve()
+      .then(read)
+      .then((reading) => {
+        const record: HostRecord = {
+          seq: ++seq,
+          t: Math.round(performance.now() * 1000) / 1000,
+          wall: Date.now(),
+          type: "mem-host",
+          ...reading,
+        };
+        fs.writeSync(fd, `${JSON.stringify(record)}\n`);
+      })
+      .catch(() => {
+        // Sampling must never take the application down.
+      })
+      .finally(() => {
+        writing = false;
+      });
+  }, intervalMs);
+  // Unreferenced so a sampler that is never stopped cannot hold the host open.
+  timer.unref();
+
+  return {
+    file,
+    stop: () => {
+      clearInterval(timer);
+      try {
+        fs.closeSync(fd);
+      } catch {
+        // Closing an already-dead descriptor must not fail shutdown.
+      }
+    },
+  };
+}

package/src/index.ts

@@ -0,0 +1,51 @@
+export {
+  openRecorder,
+  newSessionId,
+  listSessions,
+  sessionIdFromFile,
+  type Recorder,
+  type RecorderOptions,
+  type SessionFileInfo,
+} from "./recorder";
+
+export {
+  openRecordingSession,
+  CRASH_DIR_ENV,
+  CRASH_SESSION_ENV,
+  type RecordingSession,
+  type RecordingSessionOptions,
+} from "./session";
+
+export {
+  startHostSampler,
+  type HostSampler,
+  type HostSamplerOptions,
+  type HostReading,
+} from "./host_sampler";
+
+export {
+  readCrashMarkers,
+  superviseWorker,
+  type SupervisedWorker,
+  type SuperviseOptions,
+} from "./supervisor";
+
+export {
+  wrapModelDriver,
+  wrapDataDriver,
+  recordModelRenderSync,
+  createHandleRegistry,
+  type HandleRegistry,
+  type RenderInfo,
+} from "./instrument";
+
+export {
+  type LogRecord,
+  type MemorySnapshot,
+  type SamplerRecord,
+  type HostRecord,
+  type MachineMemory,
+  type CrashMarker,
+  type CrashReason,
+  type SessionEnvironment,
+} from "./events";

package/src/instrument.ts

@@ -0,0 +1,480 @@
+import type { PColumn, PTableDef, PTableDefV2 } from "@milaboratories/pl-model-common";
+import { digestDef, type DefDigest, type DefKind } from "./digest";
+import { redact } from "./redact";
+import type { Recorder } from "./recorder";
+
+/**
+ * Wrappers for the seams the model layer passes through.
+ *
+ * Every operation writes a begin record and an end record. That pairing is what
+ * makes a crash legible: when the process dies mid-operation the end record is
+ * missing, so the log names the exact call that was running when memory ran out
+ * — the question a post-crash report has to answer.
+ *
+ * The wrappers are structural rather than tied to one driver interface, because
+ * the same three creation methods appear twice with different return types: the
+ * model-facing driver hands back a bare handle, the internal one hands back a
+ * pool entry.
+ */
+
+type HandleOrigin = {
+  /** Sequence number of the record holding the definition. */
+  seq: number;
+  op: string;
+  observed?: { rows?: number; columns?: number };
+};
+
+export type HandleRegistry = {
+  put(handle: string, origin: HandleOrigin): void;
+  get(handle: string): HandleOrigin | undefined;
+  observe(handle: string, observed: { rows?: number; columns?: number }): void;
+};
+
+type ModelDriverLike<H> = {
+  createPFrame(def: never): H;
+  createPTable(def: never): H;
+  createPTableV2(def: never): H;
+};
+
+export type RenderInfo = {
+  blockId?: string;
+  /** What the block is, as `organization:name` — not the id it has in a project. */
+  block?: string;
+  blockVersion?: string;
+  /** Where the block came from: a registry, a local pack, a dev folder. */
+  blockSource?: string;
+  /** SDK the block's model was built against. */
+  sdkVersion?: string;
+  key?: string;
+  argsHash?: string;
+  /** Which lambda of the block's model is being rendered. */
+  lambda?: string;
+  /** Nth resumption of a deferred render, counted from one. */
+  recalculation?: number;
+  /** Read after the render, so sandbox counters cover the whole call. */
+  getStats?: () => unknown;
+  /** Any further context the call site wants on the record. */
+  [key: string]: unknown;
+};
+
+/** Maps driver handles back to the join that produced them. */
+export function createHandleRegistry(limit = 512): HandleRegistry {
+  const map = new Map<string, HandleOrigin>();
+  return {
+    put(handle, origin) {
+      if (map.size >= limit) {
+        const oldest = map.keys().next();
+        if (!oldest.done) map.delete(oldest.value);
+      }
+      map.set(handle, origin);
+    },
+    get(handle) {
+      return map.get(handle);
+    },
+    observe(handle, observed) {
+      const origin = map.get(handle);
+      if (origin) origin.observed = observed;
+    },
+  };
+}
+
+/**
+ * Wraps the driver that block models call to build frames and tables.
+ *
+ * Records the redacted join tree and its structural findings, then remembers
+ * which handle came from which join so later data calls can be attributed back
+ * to the definition that caused them.
+ */
+export function wrapModelDriver<D extends ModelDriverLike<unknown>>(
+  driver: D,
+  recorder: Recorder,
+  registry: HandleRegistry,
+  handleOf: (result: unknown) => string = defaultHandleOf,
+): D {
+  const wrapped = {
+    createPFrame(def: readonly PColumn<unknown>[]) {
+      return record(recorder, registry, handleOf, "createPFrame", "PFrameDef", def, () =>
+        (driver.createPFrame as (d: unknown) => unknown)(def),
+      );
+    },
+    createPTable(def: PTableDef<PColumn<unknown>>) {
+      return record(recorder, registry, handleOf, "createPTable", "PTableDef", def, () =>
+        (driver.createPTable as (d: unknown) => unknown)(def),
+      );
+    },
+    createPTableV2(def: PTableDefV2<PColumn<unknown>>) {
+      return record(recorder, registry, handleOf, "createPTableV2", "PTableDefV2", def, () =>
+        (driver.createPTableV2 as (d: unknown) => unknown)(def),
+      );
+    },
+  };
+  // The three creation methods are replaced and everything else is inherited,
+  // so the wrapper satisfies whichever driver interface the caller holds.
+  return Object.assign(Object.create(driver as object) as D, wrapped);
+}
+
+/**
+ * Wraps the asynchronous data-access driver.
+ *
+ * Adds the observed table shape, the size of what crossed back into JavaScript,
+ * and the amplification between input and output rows — the empirical
+ * counterpart to the structural findings taken from the join tree.
+ */
+export function wrapDataDriver<D extends object>(
+  driver: D,
+  recorder: Recorder,
+  registry: HandleRegistry,
+): D {
+  const source = driver as unknown as {
+    getShape(handle: string, ...rest: unknown[]): Promise<{ rows: number; columns: number }>;
+    getData(
+      handle: string,
+      columnIndices: number[],
+      range?: { offset: number; length: number },
+      ...rest: unknown[]
+    ): Promise<unknown[]>;
+    calculateTableData(handle: string, request: unknown, ...rest: unknown[]): Promise<unknown[]>;
+    getUniqueValues(
+      handle: string,
+      request: unknown,
+      ...rest: unknown[]
+    ): Promise<{ values?: { data?: unknown }; overflow?: boolean }>;
+    findColumns(
+      handle: string,
+      request: unknown,
+      ...rest: unknown[]
+    ): Promise<{ hits?: unknown[] }>;
+  };
+
+  const wrapped = {
+    async getShape(handle: string, ...rest: unknown[]) {
+      const origin = registry.get(handle);
+      return await span(
+        recorder,
+        "getShape",
+        { handle: shortHandle(handle), joinSeq: origin?.seq },
+        async () => {
+          const shape = await source.getShape(handle, ...rest);
+          registry.observe(handle, { rows: shape?.rows, columns: shape?.columns });
+          // How much this join amplified is judged in the analyzer, which can
+          // read the definition this handle came from without repeating the
+          // work here, on the hot path.
+          return { result: shape, detail: { rows: shape?.rows, columns: shape?.columns } };
+        },
+      );
+    },
+
+    async getData(
+      handle: string,
+      columnIndices: number[],
+      range?: { offset: number; length: number },
+      ...rest: unknown[]
+    ) {
+      const origin = registry.get(handle);
+      return await span(
+        recorder,
+        "getData",
+        {
+          handle: shortHandle(handle),
+          joinSeq: origin?.seq,
+          columnCount: columnIndices?.length,
+          range: range ? { offset: range.offset, length: range.length } : null,
+          // A fetch with no range pulls the whole table into the JS heap, which
+          // on a large table is an out-of-memory condition by itself.
+          unbounded: !range,
+          tableRows: origin?.observed?.rows,
+        },
+        async () => {
+          const data = await source.getData(handle, columnIndices, range, ...rest);
+          return { result: data, detail: { returnedBytes: vectorsBytes(data) } };
+        },
+      );
+    },
+
+    async calculateTableData(handle: string, request: unknown, ...rest: unknown[]) {
+      return await span(
+        recorder,
+        "calculateTableData",
+        { handle: shortHandle(handle), def: digestDef("PTableDef", request) },
+        async () => {
+          const data = await source.calculateTableData(handle, request, ...rest);
+          const vectors = (data ?? []).map((column) => (column as { data?: unknown })?.data);
+          return {
+            result: data,
+            detail: { columns: data?.length, returnedBytes: vectorsBytes(vectors) },
+          };
+        },
+      );
+    },
+
+    // Both of these reach the engine as well, and a filter that matches most of
+    // a large axis is a plausible place to run out of memory. The request is
+    // recorded redacted: axis identity survives, filter values become hashes.
+    async getUniqueValues(handle: string, request: unknown, ...rest: unknown[]) {
+      return await span(
+        recorder,
+        "getUniqueValues",
+        { handle: shortHandle(handle), request: redact(request).value },
+        async () => {
+          const response = await source.getUniqueValues(handle, request, ...rest);
+          return {
+            result: response,
+            detail: {
+              uniqueValues: vectorLength(response?.values),
+              overflow: response?.overflow,
+              returnedBytes: vectorsBytes([response?.values]),
+            },
+          };
+        },
+      );
+    },
+
+    async findColumns(handle: string, request: unknown, ...rest: unknown[]) {
+      return await span(
+        recorder,
+        "findColumns",
+        { handle: shortHandle(handle), request: redact(request).value },
+        async () => {
+          const response = await source.findColumns(handle, request, ...rest);
+          return { result: response, detail: { hits: response?.hits?.length } };
+        },
+      );
+    },
+  };
+
+  return Object.assign(Object.create(driver) as D, wrapped);
+}
+
+/**
+ * Records one block model render.
+ *
+ * `getStats` exposes the middle layer's own sandbox accounting, whose
+ * serialisation byte counts show how much data the model moved across the
+ * QuickJS boundary — the model-layer memory cost that no driver call reports.
+ */
+export async function recordModelRender<T>(
+  recorder: Recorder | undefined,
+  info: RenderInfo,
+  fn: () => Promise<T>,
+): Promise<T> {
+  if (!recorder) return await fn();
+  const { getStats, ...plain } = info;
+  const begin = recorder.event("render-begin", { ...plain, mem: recorder.memorySnapshot() });
+  const startedAt = performance.now();
+  try {
+    const result = await fn();
+    recorder.event("render-end", {
+      begin,
+      ...plain,
+      ms: round(performance.now() - startedAt),
+      stats: getStats?.(),
+      mem: recorder.memorySnapshot(),
+    });
+    return result;
+  } catch (error) {
+    recorder.event("render-error", {
+      begin,
+      ...plain,
+      ms: round(performance.now() - startedAt),
+      error: describeError(error),
+      mem: recorder.memorySnapshot(),
+    });
+    throw error;
+  }
+}
+
+/** Synchronous variant, for a render that is not driven by a promise. */
+export function recordModelRenderSync<T>(
+  recorder: Recorder | undefined,
+  info: RenderInfo,
+  fn: () => T,
+): T {
+  if (!recorder) return fn();
+  // The identity is written once as its own record and referred to by block id
+  // thereafter; repeating it on every render would be the same four fields over
+  // and over in the log a crash has to fit into.
+  const { getStats, block: _b, blockVersion: _v, blockSource: _s, sdkVersion: _k, ...plain } = info;
+  announceBlock(recorder, info);
+  const begin = recorder.event("render-begin", { ...plain, mem: recorder.memorySnapshot() });
+  const startedAt = performance.now();
+  // Driver calls the model makes while this render runs belong to this block.
+  // Recorded here rather than inferred later: an inference has to survive log
+  // rotation and cannot see a call made after the render that caused it returned.
+  openRenders.push(info.blockId);
+  try {
+    const result = fn();
+    recorder.event("render-end", {
+      begin,
+      ...plain,
+      ms: round(performance.now() - startedAt),
+      stats: getStats?.(),
+      mem: recorder.memorySnapshot(),
+    });
+    return result;
+  } catch (error) {
+    recorder.event("render-error", {
+      begin,
+      ...plain,
+      ms: round(performance.now() - startedAt),
+      error: describeError(error),
+      mem: recorder.memorySnapshot(),
+    });
+    throw error;
+  } finally {
+    openRenders.pop();
+  }
+}
+
+// Internals
+
+/** Blocks whose renders are open, innermost last. */
+const openRenders: (string | undefined)[] = [];
+
+/**
+ * Writes what a block actually is, once per session.
+ *
+ * A block id is unique to a project, so on its own it names nothing a reader can
+ * open: the package, its version and the SDK it was built against are what point
+ * at the code that ran. Written as a sticky record, so a session long enough to
+ * rotate its log away from the first render does not lose the legend for every
+ * id in what survives.
+ */
+function announceBlock(recorder: Recorder, info: RenderInfo): void {
+  const { blockId, block, blockVersion, blockSource, sdkVersion } = info;
+  if (blockId === undefined || block === undefined) return;
+  const identity = `${blockId}\u0000${block}\u0000${blockVersion}\u0000${sdkVersion}`;
+  recorder.sticky(identity, "block", { blockId, block, blockVersion, blockSource, sdkVersion });
+}
+
+function record<R>(
+  recorder: Recorder,
+  registry: HandleRegistry,
+  handleOf: (result: unknown) => string,
+  op: string,
+  kind: DefKind,
+  def: unknown,
+  call: () => R,
+): R {
+  let digest: DefDigest | { digestFailed: string };
+  try {
+    digest = digestDef(kind, def);
+  } catch (error) {
+    // Diagnostics must never be the reason a join fails to build.
+    digest = { digestFailed: describeError(error) };
+  }
+  // A creation call is synchronous but not free: it hands the definition to the
+  // native engine, which can allocate. It gets a begin/end pair like any other
+  // operation, so a death inside it is attributed to it and not to the render
+  // around it.
+  const seq = recorder.event(`${op}-begin`, {
+    def: digest,
+    blockId: openRenders.at(-1),
+    mem: recorder.memorySnapshot(),
+  });
+  const startedAt = performance.now();
+
+  let result: R;
+  try {
+    result = call();
+  } catch (error) {
+    recorder.event(`${op}-error`, {
+      begin: seq,
+      ms: round(performance.now() - startedAt),
+      error: describeError(error),
+      mem: recorder.memorySnapshot(),
+    });
+    throw error;
+  }
+
+  const handle = handleOf(result);
+  registry.put(handle, { seq, op });
+  recorder.event(`${op}-end`, {
+    begin: seq,
+    ms: round(performance.now() - startedAt),
+    handle: shortHandle(handle),
+    mem: recorder.memorySnapshot(),
+  });
+  return result;
+}
+
+async function span<T>(
+  recorder: Recorder,
+  op: string,
+  info: Record<string, unknown>,
+  fn: () => Promise<{ result: T; detail: Record<string, unknown> }>,
+): Promise<T> {
+  const begin = recorder.event(`${op}-begin`, { ...info, mem: recorder.memorySnapshot() });
+  const startedAt = performance.now();
+  try {
+    const { result, detail } = await fn();
+    recorder.event(`${op}-end`, {
+      begin,
+      ms: round(performance.now() - startedAt),
+      ...detail,
+      mem: recorder.memorySnapshot(),
+    });
+    return result;
+  } catch (error) {
+    recorder.event(`${op}-error`, {
+      begin,
+      ms: round(performance.now() - startedAt),
+      error: describeError(error),
+      mem: recorder.memorySnapshot(),
+    });
+    throw error;
+  }
+}
+
+/** Accepts both a bare handle string and a pool entry wrapping one. */
+function defaultHandleOf(result: unknown): string {
+  if (typeof result === "string") return result;
+  const key = (result as { key?: unknown } | null)?.key;
+  return typeof key === "string" ? key : String(result);
+}
+
+function vectorsBytes(vectors: unknown[]): number {
+  let bytes = 0;
+  for (const vector of vectors ?? []) {
+    const data = (vector as { data?: unknown; isNA?: { byteLength?: number } } | null)?.data;
+    if (!data) continue;
+    const byteLength = (data as { byteLength?: number }).byteLength;
+    if (typeof byteLength === "number") {
+      bytes += byteLength;
+    } else if (Array.isArray(data)) {
+      // String and Bytes columns arrive as plain arrays, and those are the ones
+      // that actually threaten the JS heap, so they are sampled not skipped.
+      bytes += sampledArrayBytes(data);
+    }
+    const isNA = (vector as { isNA?: { byteLength?: number } }).isNA;
+    if (typeof isNA?.byteLength === "number") bytes += isNA.byteLength;
+  }
+  return bytes;
+}
+
+function vectorLength(vector: unknown): number | undefined {
+  const data = (vector as { data?: { length?: number } } | undefined)?.data;
+  return typeof data?.length === "number" ? data.length : undefined;
+}
+
+function sampledArrayBytes(data: unknown[]): number {
+  const sampleSize = Math.min(data.length, 32);
+  if (sampleSize === 0) return 0;
+  let sampled = 0;
+  for (let i = 0; i < sampleSize; i++) {
+    const value = data[Math.floor((i * data.length) / sampleSize)];
+    sampled += typeof value === "string" ? value.length * 2 + 24 : 8;
+  }
+  return Math.round((sampled / sampleSize) * data.length);
+}
+
+function shortHandle(handle: string): string {
+  return typeof handle === "string" ? handle.slice(0, 24) : String(handle);
+}
+
+function describeError(error: unknown): string {
+  return String((error as { message?: unknown } | null)?.message ?? error);
+}
+
+function round(value: number): number {
+  return Math.round(value * 100) / 100;
+}

package/src/machine_memory.ts

@@ -0,0 +1,70 @@
+import { execFileSync } from "node:child_process";
+import fs from "node:fs";
+import type { MachineMemory } from "./events";
+
+/**
+ * Reads where the machine's memory currently is.
+ *
+ * Costs about two milliseconds on macOS and one file read on Linux, which is why
+ * the sampler can afford it once a second. It never throws: a reading that cannot
+ * be taken says so and the sampler carries on, because a missing number must not
+ * cost the resident-size curve it accompanies.
+ */
+export function readMachineMemory(): MachineMemory {
+  try {
+    if (process.platform === "darwin") return readDarwin();
+    if (process.platform === "linux") return readLinux();
+    return { unavailable: `not implemented for ${process.platform}` };
+  } catch (error: unknown) {
+    return { unavailable: error instanceof Error ? error.message : String(error) };
+  }
+}
+
+// Internals
+
+function readDarwin(): MachineMemory {
+  const stat = execFileSync("vm_stat", { encoding: "utf8", timeout: 2000 });
+  // The page size is stated in the header and is 16 KiB on Apple silicon against
+  // 4 KiB elsewhere, so every count below is meaningless without reading it.
+  const pageSize = Number(/page size of (\d+) bytes/.exec(stat)?.[1] ?? 4096);
+  const pages = (label: string): number | undefined => {
+    const match = new RegExp(`${label}:\\s+(\\d+)\\.`).exec(stat);
+    return match ? Number(match[1]) * pageSize : undefined;
+  };
+
+  const swap = execFileSync("sysctl", ["-n", "vm.swapusage"], { encoding: "utf8", timeout: 2000 });
+  const swapBytes = (label: string): number | undefined => {
+    const match = new RegExp(`${label} = ([\\d.]+)M`).exec(swap);
+    return match ? Number(match[1]) * 1024 * 1024 : undefined;
+  };
+
+  return {
+    // "Stored" counts the memory before compression and is what went missing from
+    // a process's resident set; "occupied" is what it costs the machine now.
+    compressedStored: pages("Pages stored in compressor"),
+    compressedOccupied: pages("Pages occupied by compressor"),
+    swapUsed: swapBytes("used"),
+    swapTotal: swapBytes("total"),
+    anonymous: pages("Anonymous pages"),
+    fileBacked: pages("File-backed pages"),
+    wired: pages("Pages wired down"),
+  };
+}
+
+function readLinux(): MachineMemory {
+  const info = fs.readFileSync("/proc/meminfo", "utf8");
+  const kb = (label: string): number | undefined => {
+    const match = new RegExp(`^${label}:\\s+(\\d+) kB`, "m").exec(info);
+    return match ? Number(match[1]) * 1024 : undefined;
+  };
+  const swapTotal = kb("SwapTotal");
+  const swapFree = kb("SwapFree");
+  return {
+    swapTotal,
+    swapUsed: swapTotal !== undefined && swapFree !== undefined ? swapTotal - swapFree : undefined,
+    anonymous: kb("AnonPages"),
+    fileBacked: kb("Cached"),
+    // Linux has no compressor of its own; zram, where present, reports as swap.
+    wired: kb("Unevictable"),
+  };
+}