@milaboratories/pl-flight-recorder 0.2.0

package/src/data_summary.ts

@@ -0,0 +1,110 @@
+/**
+ * Counts and sizes for a column payload, never the payload.
+ *
+ * This is the one place that has to know the shape of `DataInfo`, because the
+ * numbers that predict a join's cost — rows per partition and their byte sizes —
+ * live at type-specific positions inside it. Everything else about a definition
+ * is recorded structurally.
+ *
+ * Chunk statistics are optional: the producing workflow fills them in, so row
+ * counts are reported when present and left unknown otherwise rather than
+ * guessed.
+ */
+
+export type DataSummary = {
+  kind: string;
+  /** Entries for inline or JSON payloads. */
+  entries?: number;
+  approxBytes?: number;
+  keyLength?: number;
+  partitionKeyLength?: number;
+  parts?: number;
+  partsWithStats?: number;
+  rows?: number;
+  bytes?: number;
+};
+
+export function summarizeData(data: unknown): DataSummary {
+  if (data === null || data === undefined) return { kind: "absent" };
+  if (Array.isArray(data)) {
+    // Inline values, built inside the model sandbox.
+    return { kind: "inline", entries: data.length, approxBytes: approxInlineBytes(data) };
+  }
+  if (typeof data !== "object") return { kind: typeof data };
+
+  const info = data as { type?: string; [key: string]: unknown };
+  switch (info.type) {
+    case "Json":
+      return {
+        kind: "Json",
+        keyLength: numberOr(info.keyLength),
+        entries: countKeys(info.data),
+      };
+    case "JsonPartitioned":
+    case "BinaryPartitioned":
+      return {
+        kind: info.type,
+        partitionKeyLength: numberOr(info.partitionKeyLength),
+        parts: countKeys(info.parts),
+      };
+    case "ParquetPartitioned":
+      return summarizeParquet(info);
+    default:
+      return { kind: info.type ?? opaqueKind(data) };
+  }
+}
+
+// Internals
+
+function summarizeParquet(info: { [key: string]: unknown }): DataSummary {
+  const parts = Object.values((info.parts ?? {}) as Record<string, unknown>);
+  let rows = 0;
+  let bytes = 0;
+  let withStats = 0;
+  for (const part of parts) {
+    const stats = (
+      part as { stats?: { numberOfRows?: number; size?: { axes?: number[]; column?: number } } }
+    )?.stats;
+    if (!stats) continue;
+    withStats++;
+    if (typeof stats.numberOfRows === "number") rows += stats.numberOfRows;
+    if (stats.size) bytes += (stats.size.column ?? 0) + sum(stats.size.axes ?? []);
+  }
+  return {
+    kind: "ParquetPartitioned",
+    partitionKeyLength: numberOr(info.partitionKeyLength),
+    parts: parts.length,
+    partsWithStats: withStats,
+    rows: withStats > 0 ? rows : undefined,
+    bytes: withStats > 0 ? bytes : undefined,
+  };
+}
+
+// Sampled rather than measured: walking millions of entries to size them is
+// itself a memory risk in the situation this code exists to diagnose.
+function approxInlineBytes(values: unknown[]): number {
+  const sampleSize = Math.min(values.length, 64);
+  if (sampleSize === 0) return 0;
+  let bytes = 0;
+  for (let i = 0; i < sampleSize; i++) {
+    const value = values[Math.floor((i * values.length) / sampleSize)];
+    bytes += JSON.stringify(value ?? null)?.length ?? 0;
+  }
+  return Math.round((bytes / sampleSize) * values.length);
+}
+
+function countKeys(value: unknown): number | undefined {
+  return value && typeof value === "object" ? Object.keys(value).length : undefined;
+}
+
+function numberOr(value: unknown): number | undefined {
+  return typeof value === "number" ? value : undefined;
+}
+
+function opaqueKind(value: object): string {
+  return (value as { constructor?: { name?: string } }).constructor?.name ?? "opaque";
+}
+
+function sum(values: number[]): number {
+  return values.reduce((acc, value) => acc + value, 0);
+}

package/src/digest.ts

@@ -0,0 +1,49 @@
+import { redact, type RedactionStats } from "./redact";
+
+/**
+ * What a definition record carries.
+ *
+ * The definition is recorded structurally (see `redact`) rather than through a
+ * hand-written digest per definition type. Structural rules are not run here:
+ * they belong to the analyzer, so nothing is computed on the hot path and the
+ * rules can be revised against logs that already exist.
+ */
+
+export const REDACTION = {
+  kept: [
+    "definition shape",
+    "column and axis names",
+    "value types",
+    "axis domains",
+    "filter operators",
+    "row, byte and partition counts",
+  ],
+  hashed: ["filter reference values", "annotation values", "column ids", "every other string"],
+  dropped: ["cell values", "inline column payloads", "partition keys"],
+} as const;
+
+export type DefKind = "PTableDef" | "PTableDefV2" | "PFrameDef";
+
+export type DefDigest = {
+  kind: DefKind;
+  /** Redacted definition, same shape as the original. */
+  def: unknown;
+  redaction: RedactionStats & { bytes: number };
+};
+
+/** Records one definition: redacted, measured, and tagged with its API shape. */
+export function digestDef(kind: DefKind, def: unknown): DefDigest {
+  const { value, stats } = redact(def);
+  const json = safeLength(value);
+  return { kind, def: value, redaction: { ...stats, bytes: json } };
+}
+
+// Internals
+
+function safeLength(value: unknown): number {
+  try {
+    return JSON.stringify(value)?.length ?? 0;
+  } catch {
+    return 0;
+  }
+}

package/src/events.ts

@@ -0,0 +1,102 @@
+/**
+ * Record types written to a flight log.
+ *
+ * The log is append-only NDJSON, one record per line, and is read back by
+ * tooling that may be older or newer than the writer, so every field beyond
+ * {@link FlightRecordBase} is optional and unknown record types are skipped
+ * rather than rejected.
+ */
+
+/** Memory reading taken by the thread that wrote the record. */
+export type MemorySnapshot = {
+  /** Resident set size of the whole process. */
+  rss: number;
+  /** Heap in use by the writing thread's isolate. */
+  heapUsed: number;
+  heapTotal: number;
+  external: number;
+  arrayBuffers: number;
+  /** V8 heap ceiling for the writing thread's isolate. */
+  heapLimit: number;
+};
+
+export type FlightRecordBase = {
+  /** Monotonically increasing within one session; used to pair begin with end. */
+  seq: number;
+  /** Milliseconds since process start, for durations. */
+  t: number;
+  /** Wall clock, for correlating with the sampler series and crash markers. */
+  wall: number;
+  type: string;
+};
+
+export type FlightRecord = FlightRecordBase & {
+  mem?: MemorySnapshot;
+  /** Sequence number of the matching begin record, on end and error records. */
+  begin?: number;
+  [key: string]: unknown;
+};
+
+/** Session header, always the first record. */
+export type SessionEnvironment = {
+  node: string;
+  platform: string;
+  cpus: number;
+  totalMemory: number;
+  heapLimit: number;
+  execArgv: string[];
+  maxOldSpaceSize?: number;
+};
+
+/** Written by the sampler thread to its own sibling file. */
+export type SamplerRecord = FlightRecordBase & {
+  type: "mem-sampler";
+  rss: number;
+  peakRss: number;
+  freeMemory: number;
+  totalMemory: number;
+};
+
+/** Written by the parent when a supervised thread or process dies. */
+export type CrashMarker = {
+  type: "external-crash";
+  wall: number;
+  /**
+   * Session the marker belongs to. Present only when the parent assigned the id
+   * to the worker and therefore knows it; never inferred, because a wrong id
+   * here would both misattribute the death and stop the right session from
+   * claiming it.
+   */
+  sessionId?: string;
+  /**
+   * Advisory only: the newest open flight log at the moment of death. A
+   * concurrent live session can make this wrong, so it is never matched against
+   * — it exists to help a human read a directory by hand.
+   */
+  guessedSessionId?: string;
+  /** Written by an older recorder that put a guess in `sessionId`. */
+  sessionIdSource?: "assigned" | "guessed";
+  reason: CrashReason;
+  errorCode?: string;
+  errorName?: string;
+  message?: string;
+  exitCode?: number;
+  signal?: string;
+  stderrTail?: string;
+};
+
+export type CrashReason =
+  | "js-heap-out-of-memory"
+  | "killed-by-os"
+  | "abort-or-fatal-allocation-failure"
+  | "worker-exit"
+  | "nonzero-exit"
+  | "unknown";
+
+export const SESSION_RECORD = "session";
+/** Earliest memory reading of a session, rewritten into every rotated segment. */
+export const MEM_BASELINE_RECORD = "mem-baseline";
+export const SESSION_END_RECORD = "session-end";
+export const FLIGHT_FILE_PREFIX = "flight";
+export const SAMPLER_FILE_PREFIX = "mem";
+export const CRASH_FILE_PREFIX = "crash";

package/src/index.ts

@@ -0,0 +1,104 @@
+export {
+  openRecorder,
+  newSessionId,
+  startSelfSampler,
+  listSessions,
+  readSession,
+  sessionIdFromFile,
+  type Recorder,
+  type RecorderOptions,
+  type SessionFileInfo,
+  type ParsedSession,
+} from "./recorder";
+
+export { startMemorySampler, type MemorySampler, type MemorySamplerOptions } from "./sampler";
+
+export {
+  openFlightSession,
+  FLIGHT_DIR_ENV,
+  FLIGHT_SESSION_ENV,
+  type FlightSession,
+  type FlightSessionOptions,
+} from "./session";
+
+export {
+  writeCrashMarker,
+  readCrashMarkers,
+  superviseWorker,
+  type CrashMarkerInput,
+  type SupervisedWorker,
+  type SuperviseOptions,
+} from "./supervisor";
+
+export {
+  wrapModelDriver,
+  wrapDataDriver,
+  recordModelRender,
+  recordModelRenderSync,
+  createHandleRegistry,
+  type HandleRegistry,
+  type HandleOrigin,
+  type RenderInfo,
+} from "./instrument";
+
+export { digestDef, REDACTION, type DefDigest, type DefKind } from "./digest";
+
+export {
+  redact,
+  hashString,
+  isHashedString,
+  SCHEMA_KEYS,
+  SCHEMA_SUBTREE_KEYS,
+  SUMMARISED_KEYS,
+  COUNTED_KEYS,
+  type RedactionStats,
+  type RedactOptions,
+  type HashedString,
+} from "./redact";
+
+export { summarizeData, type DataSummary } from "./data_summary";
+
+export {
+  structuralFindings,
+  joinShapes,
+  inputRowsMax,
+  axesUnder,
+  axisKey,
+  axisNameKey,
+  isJoinNode,
+  joinChildren,
+  type StructuralFinding,
+  type FindingSeverity,
+  type JoinShape,
+  type AxisDescriptor,
+} from "./rules";
+
+export {
+  analyzeSession,
+  analyzeLatest,
+  formatBytes,
+  formatCount,
+  THRESHOLDS,
+  type SessionAnalysis,
+  type Finding,
+  type MemoryAnalysis,
+  type OperationSummary,
+  type RenderSummary,
+  type Verdict,
+} from "./analyze";
+
+export { renderReport } from "./report";
+
+export {
+  FLIGHT_FILE_PREFIX,
+  SAMPLER_FILE_PREFIX,
+  CRASH_FILE_PREFIX,
+  SESSION_RECORD,
+  SESSION_END_RECORD,
+  type FlightRecord,
+  type MemorySnapshot,
+  type SamplerRecord,
+  type CrashMarker,
+  type CrashReason,
+  type SessionEnvironment,
+} from "./events";