@manyoars/oars 0.2.0 → 0.4.0

package/dist/sessions/sources.d.ts

@@ -0,0 +1,50 @@
+import type { SubagentOrigin } from './extract.js';
+import type { Harness } from './schema.js';
+/** A session found in a source, before its transcript is read. */
+export interface SourceSession {
+    /** Names the skeleton (`<source>-<id>`), so it is a safe file name. */
+    id: string;
+    /** The working folder (a path); null when the source names none. */
+    folder: string | null;
+    /** With `mtimeMs`, the cursor `state.json` keeps to tell a changed session from one already deposited. */
+    size: number;
+    mtimeMs: number;
+    /** Set when the session is a spawned agent's: it is deposited as a subagent of its parent, so session counts stay main sessions. */
+    subagent?: SubagentOrigin;
+}
+export interface SourceStore {
+    sessions: SourceSession[];
+    /** The session as transcript text for `captureSession`; throws when it cannot be read. */
+    transcript: (session: SourceSession) => string | Promise<string>;
+    /** Sessions skipped because their id is not a safe file name, counted so the skip is never silent. */
+    rejected: number;
+    /** Items that could not be inspected, warned about once each and never fatal; `reason` names no path. */
+    unreadable: readonly {
+        file: string;
+        reason: string;
+    }[];
+    /** The file to blame when `transcript` throws for a session; the store's own file when a source keeps one. */
+    where: (session: SourceSession) => string;
+    close: () => void;
+}
+export interface Source {
+    /** The `--allow` name, the first segment of its `state.json` keys, and the prefix of its skeleton names. */
+    name: string;
+    harness: Harness;
+    /** How the deposit lines name it ("Cursor": `Read 3 Cursor sessions.`). */
+    label: string;
+    /** Where its store lives; the caller passes the environment so a test never reads the real one. */
+    root: (platform: NodeJS.Platform, env: {
+        [name: string]: string | undefined;
+    }, home: string) => string;
+    /** The file reported when the store as a whole cannot be opened. */
+    storeFile: (root: string) => string;
+    /** A session whose transcript cannot be read ends this source's deposit (Cursor's, as it was before the table) instead of skipping to the next. */
+    stopOnTranscriptError?: boolean;
+    /** Opens the store read-only; undefined when the machine has none, so a missing store is skipped silently. */
+    open: (root: string) => Promise<SourceStore | undefined>;
+}
+/** Every non-Claude source, in the order the deposit reads them. */
+export declare const SOURCES: readonly Source[];
+/** How a source is named in a deposit line; a name the table does not know (a test's) is its own label. */
+export declare const sourceLabel: (name: string, table?: readonly Source[]) => string;

package/dist/sessions/sources.js

@@ -0,0 +1,233 @@
+// The source table: one entry per non-Claude harness whose sessions `deposit` can read. `--allow <name>` opens an entry, the nightly hook
+// runtime, `allow.unmatched`, the deposit lines, `measure`, and `verify` read this table, so a new harness is one entry here plus its reader file.
+// Claude Code is not an entry: its transcripts are files the deposit walks itself (`depositLocked`), and its behavior does not change.
+import { join } from 'node:path';
+import { codexRoot, openCodexStore } from './codex.js';
+import { copilotRoot, openCopilotStore } from './copilot.js';
+import { cursorRoot, openCursorStore } from './cursor.js';
+import { grokRoot, openGrokStore } from './grok.js';
+import { geminiRoot, openGeminiStore } from './gemini.js';
+import { openPiStore, piRoot } from './pi.js';
+import { openOtelStore, otelRoot } from './otel.js';
+import { opencodeRoot, openOpencodeStore } from './opencode.js';
+import { openVscodeCopilotStore, vscodeAppData } from './vscode-copilot.js';
+/** A session handed back to its own store was listed by it, so a miss is a bug in the caller. */
+const need = (value) => {
+    if (value === undefined)
+        throw new Error('session not listed by its source');
+    return value;
+};
+const cursor = {
+    name: 'cursor',
+    harness: 'cursor',
+    label: 'Cursor',
+    stopOnTranscriptError: true,
+    root: (platform, env, home) => cursorRoot(platform, env, home),
+    storeFile: (root) => join(root, 'globalStorage', 'state.vscdb'),
+    open: async (root) => {
+        const store = await openCursorStore(root);
+        if (store === undefined)
+            return undefined;
+        const byId = new Map(store.sessions.map((session) => [session.composerId, session]));
+        return {
+            sessions: store.sessions.map((session) => ({ id: session.composerId, folder: session.folder, size: session.messages, mtimeMs: session.lastUpdatedAt })),
+            transcript: (session) => store.transcript(need(byId.get(session.id))),
+            rejected: store.rejected,
+            unreadable: [],
+            where: () => join(root, 'globalStorage', 'state.vscdb'),
+            close: store.close,
+        };
+    },
+};
+const codex = {
+    name: 'codex',
+    harness: 'codex',
+    label: 'Codex',
+    root: (_platform, env, home) => codexRoot(env, home),
+    storeFile: (root) => join(root, 'sessions'),
+    open: async (root) => {
+        const store = await openCodexStore(root);
+        if (store === undefined)
+            return undefined;
+        const byId = new Map(store.sessions.map((session) => [session.id, session]));
+        return {
+            sessions: store.sessions.map((session) => ({
+                id: session.id,
+                folder: session.folder,
+                size: session.size,
+                mtimeMs: session.mtimeMs,
+                ...(session.parent === null ? {} : { subagent: { parent: `codex-${session.parent}`, agentType: session.agentRole, spawnedBy: null } }),
+            })),
+            transcript: async (session) => await store.transcript(need(byId.get(session.id))),
+            rejected: store.rejected,
+            unreadable: store.unreadable,
+            where: (session) => byId.get(session.id)?.file ?? root,
+            close: () => undefined,
+        };
+    },
+};
+const copilot = {
+    name: 'copilot',
+    harness: 'copilot',
+    label: 'Copilot',
+    root: (_platform, env, home) => copilotRoot(env, home),
+    storeFile: (root) => join(root, 'session-state'),
+    open: async (root) => {
+        const store = await openCopilotStore(root);
+        if (store === undefined)
+            return undefined;
+        const byId = new Map(store.sessions.map((session) => [session.id, session]));
+        return {
+            sessions: store.sessions.map((session) => ({ id: session.id, folder: session.folder, size: session.size, mtimeMs: session.mtimeMs })),
+            transcript: async (session) => await store.transcript(need(byId.get(session.id))),
+            rejected: store.rejected,
+            unreadable: store.unreadable,
+            where: (session) => byId.get(session.id)?.file ?? root,
+            close: () => undefined,
+        };
+    },
+};
+const gemini = {
+    name: 'gemini',
+    harness: 'gemini',
+    label: 'Gemini',
+    root: (_platform, env, home) => geminiRoot(env, home),
+    storeFile: (root) => join(root, 'tmp'),
+    open: async (root) => {
+        const store = await openGeminiStore(root);
+        if (store === undefined)
+            return undefined;
+        const byId = new Map(store.sessions.map((session) => [session.id, session]));
+        return {
+            sessions: store.sessions.map((session) => ({
+                id: session.id,
+                folder: session.folder,
+                size: session.size,
+                mtimeMs: session.mtimeMs,
+                ...(session.parent === null ? {} : { subagent: { parent: `gemini-${session.parent}`, agentType: null, spawnedBy: null } }),
+            })),
+            transcript: async (session) => await store.transcript(need(byId.get(session.id))),
+            rejected: store.rejected,
+            unreadable: store.unreadable,
+            where: (session) => byId.get(session.id)?.file ?? root,
+            close: () => undefined,
+        };
+    },
+};
+const grok = {
+    name: 'grok',
+    harness: 'grok',
+    label: 'Grok Build',
+    root: (_platform, env, home) => grokRoot(env, home),
+    storeFile: (root) => join(root, 'sessions'),
+    open: async (root) => {
+        const store = await openGrokStore(root);
+        if (store === undefined)
+            return undefined;
+        const byId = new Map(store.sessions.map((session) => [session.id, session]));
+        return {
+            sessions: store.sessions.map((session) => ({ id: session.id, folder: session.folder, size: session.size, mtimeMs: session.mtimeMs })),
+            transcript: async (session) => await store.transcript(need(byId.get(session.id))),
+            rejected: store.rejected,
+            unreadable: store.unreadable,
+            where: (session) => join(byId.get(session.id)?.dir ?? root, 'updates.jsonl'),
+            close: () => undefined,
+        };
+    },
+};
+const opencode = {
+    name: 'opencode',
+    harness: 'opencode',
+    label: 'OpenCode',
+    root: (_platform, env, home) => opencodeRoot(env, home),
+    storeFile: (root) => join(root, 'opencode.db'),
+    open: async (root) => {
+        const store = await openOpencodeStore(root);
+        if (store === undefined)
+            return undefined;
+        const byId = new Map(store.sessions.map((session) => [session.id, session]));
+        return {
+            sessions: store.sessions.map((session) => ({
+                id: session.id,
+                folder: session.folder,
+                size: session.size,
+                mtimeMs: session.mtimeMs,
+                ...(session.parent === null ? {} : { subagent: { parent: `opencode-${session.parent}`, agentType: session.agent, spawnedBy: null } }),
+            })),
+            transcript: (session) => store.transcript(need(byId.get(session.id))),
+            rejected: store.rejected,
+            unreadable: [],
+            where: () => join(root, 'opencode.db'),
+            close: store.close,
+        };
+    },
+};
+const pi = {
+    name: 'pi',
+    harness: 'pi',
+    label: 'Pi',
+    root: (_platform, env, home) => piRoot(env, home),
+    storeFile: (root) => join(root, 'sessions'),
+    open: async (root) => {
+        const store = await openPiStore(root);
+        if (store === undefined)
+            return undefined;
+        const byId = new Map(store.sessions.map((session) => [session.id, session]));
+        return {
+            sessions: store.sessions.map((session) => ({ id: session.id, folder: session.folder, size: session.size, mtimeMs: session.mtimeMs })),
+            transcript: async (session) => await store.transcript(need(byId.get(session.id))),
+            rejected: store.rejected,
+            unreadable: store.unreadable,
+            where: (session) => byId.get(session.id)?.file ?? root,
+            close: () => undefined,
+        };
+    },
+};
+const vscodeCopilot = {
+    name: 'vscode-copilot',
+    harness: 'vscode-copilot',
+    label: 'VS Code Copilot Chat',
+    root: (platform, env, home) => vscodeAppData(platform, env, home),
+    storeFile: (root) => root,
+    open: async (root) => {
+        const store = await openVscodeCopilotStore(root);
+        if (store === undefined)
+            return undefined;
+        const byId = new Map(store.sessions.map((session) => [session.id, session]));
+        return {
+            sessions: store.sessions.map((session) => ({ id: session.id, folder: session.folder, size: session.size, mtimeMs: session.mtimeMs })),
+            transcript: async (session) => await store.transcript(need(byId.get(session.id))),
+            rejected: store.rejected,
+            unreadable: store.unreadable,
+            where: (session) => byId.get(session.id)?.files[0] ?? root,
+            close: () => undefined,
+        };
+    },
+};
+// A folder of OTLP JSON trace files (`$OARS_OTEL_TRACES`, else the config folder's `otel-traces`): a configured folder rather than an `import` command, so the cursor, the
+// allowlist, the nightly run, and the fail-open reads are the ones every other source has.
+const otel = {
+    name: 'otel',
+    harness: 'otel',
+    label: 'OpenTelemetry GenAI',
+    root: (platform, env, home) => otelRoot(env, home, platform),
+    storeFile: (root) => root,
+    open: async (root) => {
+        const store = await openOtelStore(root);
+        if (store === undefined)
+            return undefined;
+        const byId = new Map(store.sessions.map((session) => [session.id, session]));
+        return {
+            sessions: store.sessions.map((session) => ({ id: session.id, folder: session.folder, size: session.size, mtimeMs: session.mtimeMs, ...(session.subagent === undefined ? {} : { subagent: session.subagent }) })),
+            transcript: (session) => store.transcript(need(byId.get(session.id))),
+            rejected: store.rejected,
+            unreadable: store.unreadable,
+            where: (session) => byId.get(session.id)?.file ?? root,
+            close: () => undefined,
+        };
+    },
+};
+/** Every non-Claude source, in the order the deposit reads them. */
+export const SOURCES = [cursor, codex, copilot, gemini, grok, opencode, pi, vscodeCopilot, otel];
+/** How a source is named in a deposit line; a name the table does not know (a test's) is its own label. */
+export const sourceLabel = (name, table = SOURCES) => table.find((source) => source.name === name)?.label ?? name;

package/dist/sessions/steps.d.ts

@@ -0,0 +1,105 @@
+import { type Step, type StepCounts, type StepRow, type StepStatus, type Trigger } from './schema.js';
+import { type DepositTrace } from './store.js';
+/** Where one run's rows get their identity: the steps of one nightly or manual run share `run`. */
+export interface StepContext {
+    storeDir: string;
+    now: () => number;
+    run: string;
+    trigger: Trigger;
+}
+export declare const newContext: (storeDir: string, now: () => number, trigger: Trigger) => StepContext;
+/** What a step reports when it ends: the counts it read and wrote and its idempotency key (spec 4.2). */
+export interface StepReport {
+    in: StepCounts;
+    out: StepCounts;
+    key: string | null;
+}
+export declare const appendStep: (storeDir: string, row: StepRow) => Promise<void>;
+export declare const readSteps: (storeDir: string) => Promise<StepRow[]>;
+/** The first informative line of a failure, masked like every other text the store keeps (D1's text rule) and cut to 300 characters. */
+export declare const errorLine: (storeDir: string, error: unknown) => Promise<string>;
+/** Appends the row for a step that ended (or was skipped) now, having taken `ms`. */
+export declare const logStep: (context: StepContext, step: Step, status: StepStatus, ms: number, report: StepReport, error: string | null) => Promise<void>;
+/** A step an earlier step of the run left undone (spec 4.4): logged `skipped`, with no counts and no key. */
+export declare const logSkipped: (context: StepContext, step: Step) => Promise<void>;
+/** Writes a row without ever failing: a log that cannot be appended (or a failure line that cannot be redacted) must not change a step's outcome or hide its real error. */
+export declare const tryLog: (write: () => Promise<void>) => Promise<void>;
+/**
+ * Runs `work` and logs its row: `ok` with what it reports, or `error` with the failure's first line and `failedKey` (a key built from
+ * the step's inputs before the work, so a retry of the same work has the same key), after which the failure goes on to the caller.
+ */
+export declare const runStep: <T>(context: StepContext, step: Step, work: () => Promise<{
+    value: T;
+    report: StepReport;
+}>, failedKey: string) => Promise<T>;
+/** The runs of one step and key that failed in a row. Three is the line spec 4.4 draws. */
+export declare const STUCK_RUNS = 3;
+export interface StuckStep {
+    step: Step;
+    key: string;
+    /** Failed runs in a row, ending at the newest. */
+    runs: number;
+    /** The first error line of the newest failure. */
+    error: string;
+}
+/**
+ * Keys that failed on three runs in a row, per step and trigger (a nightly that fails every night is not cleared by the `stop` deposits
+ * between nights). A `skipped` or `deferred` row is no run of its step (a deferred deposit neither failed nor succeeded: it is waiting in the pending list); an `ok` row ends the open streaks of its own step and trigger, and of its step on the same key under another trigger, because a step's
+ * good run can carry a key (verify's high-water mark) that its failed run could not know before the work.
+ */
+export declare const stuckSteps: (rows: readonly StepRow[]) => StuckStep[];
+export interface StepLine {
+    step: Step;
+    /** The extractor version the runs ran at: spec 3.2 counts step time per `ev`. */
+    ev: string;
+    runs: number;
+    ok: number;
+    error: number;
+    skipped: number;
+    /** Stop deposits that found the store locked and went to the pending list: not failed runs, and not in `error`. */
+    deferred: number;
+    /** Mean wall time of the runs that ran (not the skipped or deferred ones), in ms. */
+    meanMs: number;
+}
+/** The spec 3.2 counters this pipeline can derive locally from the step log. */
+export interface StepCounters {
+    runs: number;
+    steps: StepLine[];
+    /** Capture records deposited, by kind, summed over every ok deposit. */
+    deposited: StepCounts;
+    /** The newest ok derive's output, by type. */
+    derived: StepCounts;
+    /** The newest ok index's output: the recurring signatures and projects it holds. */
+    indexed: StepCounts;
+    /** The newest ok verify's `out`: invariants checked and passed, `cv-behind` sessions, sessions failing the raw audit (`merkle`), and stuck steps. */
+    verify: StepCounts | null;
+    recall: {
+        queries: number;
+        matches: number;
+    };
+    /** Failed runs by `<step>@<ev>`. */
+    failures: StepCounts;
+    stuck: StuckStep[];
+}
+export declare const stepCounters: (rows: readonly StepRow[]) => StepCounters;
+export declare const formatStepCounters: (counters: StepCounters) => string;
+/** A key built from its inputs and `ev` (spec 4.2), hashed; a recall has none in the spec (a lookup writes nothing), so its query is the input. */
+export declare const stampKey: (...parts: readonly string[]) => string;
+/**
+ * A deposit's key from its inputs (spec 4.2 names the files' state, which is not known before the work): `cv`, `ev`, the trigger, the allow
+ * entries or transcript asked for, and the redact domains. The `stop` hook passes no domains (it reads none), so its key has none.
+ */
+export declare const inputKey: (trigger: Trigger, requested: readonly string[], domains: readonly string[]) => string;
+/**
+ * Runs a deposit and logs its rows: `deposit` with the records it wrote by kind, then, when it ended with the whole-store pass,
+ * `derive` and `index` (spec 4.1: one row per step at a nightly or manual run). `key` is built from the deposit's inputs (the trigger and the transcripts
+ * it was asked to deposit), so the failed row and the ok row of one deposit carry the same key and a failure that recovers is not stuck.
+ * A deposit that fails logs `error` and, when a derive was to follow, `skipped` for it and for the index (spec 4.4). A derive that fails
+ * after the deposit wrote its records logs `deposit` ok, `derive` error, and `index` skipped; an index that fails logs `derive` ok and then
+ * `index` error; the failure goes on to the caller. The SessionEnd hook runs
+ * `withDerive` false: derive follows at the nightly run.
+ */
+export declare const runDeposit: <T>(context: StepContext, withDerive: boolean, key: string, work: (trace: DepositTrace) => Promise<{
+    value: T;
+    in: StepCounts;
+}>) => Promise<T>;

package/dist/sessions/steps.js

@@ -0,0 +1,250 @@
+// The step log: `pipeline-runs.jsonl`, one row per run of a pipeline step, appended when the step ends (runtime-commons pipeline spec 3.1).
+// Local only: it sits in the store directory beside `hook-calls.jsonl`, and `measure` prints the spec 3.2 counters from it.
+import { randomUUID } from 'node:crypto';
+import { appendFile, mkdir, readFile } from 'node:fs/promises';
+import { homedir } from 'node:os';
+import { performance } from 'node:perf_hooks';
+import { CliError, ExitCode } from '../errors.js';
+import { makeRedactor } from './redact.js';
+import { CAPTURE_VERSION, EXTRACTOR_VERSION, parseStepRow } from './schema.js';
+import { digest, newTrace, paths, readRedactDomains } from './store.js';
+export const newContext = (storeDir, now, trigger) => ({ storeDir, now, run: randomUUID(), trigger });
+export const appendStep = async (storeDir, row) => {
+    await mkdir(storeDir, { recursive: true });
+    await appendFile(paths(storeDir).pipelineRuns, `${JSON.stringify(row)}\n`, { mode: 0o600 });
+};
+export const readSteps = async (storeDir) => {
+    let text = '';
+    try {
+        text = await readFile(paths(storeDir).pipelineRuns, 'utf8');
+    }
+    catch (error) {
+        if (!(error instanceof Error && 'code' in error && error.code === 'ENOENT'))
+            throw error;
+    }
+    const rows = [];
+    for (const line of text.split('\n')) {
+        if (line === '')
+            continue;
+        try {
+            const row = parseStepRow(JSON.parse(line));
+            if (row !== undefined)
+                rows.push(row);
+        }
+        catch {
+            // A torn last line is skipped.
+        }
+    }
+    return rows;
+};
+/**
+ * Reduces the home folder to `<home>` in whatever form a path takes: either separator, any letter case (a Windows drive letter, a
+ * `transcript_path` from hook stdin). An empty home matches nothing, since splitting on '' would garble the whole message.
+ */
+const scrubHome = (text) => {
+    const home = homedir();
+    if (home === '')
+        return text;
+    const pattern = home.replace(/[.*+?^${}()|[\]\\]/g, '\\$&').replace(/(?:\\\\|\/)/g, '[\\\\/]');
+    return text.replace(new RegExp(pattern, 'gi'), '<home>');
+};
+/** The first informative line of a failure, masked like every other text the store keeps (D1's text rule) and cut to 300 characters. */
+export const errorLine = async (storeDir, error) => {
+    const first = (error instanceof Error ? error.message : String(error)).split('\n').find((line) => line.trim() !== '') ?? 'unknown error';
+    // A path under the home folder is reduced to `<home>`, so the shared record never holds the user name (the class A rule in docs/sessions-store.md).
+    const scrubbed = scrubHome(first);
+    // A store whose state.json cannot be read still gets its failure row (spec 4.4): the line is masked with no domains then.
+    const domains = await readRedactDomains(storeDir).catch(() => []);
+    return makeRedactor(domains)(scrubbed).slice(0, 300);
+};
+/** Appends the row for a step that ended (or was skipped) now, having taken `ms`. */
+export const logStep = (context, step, status, ms, report, error) => appendStep(context.storeDir, {
+    at: new Date(context.now()).toISOString(),
+    run: context.run,
+    trigger: context.trigger,
+    step,
+    status,
+    ms: Math.round(ms),
+    in: report.in,
+    out: report.out,
+    cv: CAPTURE_VERSION,
+    ev: EXTRACTOR_VERSION,
+    key: report.key,
+    error,
+});
+/** A step an earlier step of the run left undone (spec 4.4): logged `skipped`, with no counts and no key. */
+export const logSkipped = (context, step) => logStep(context, step, 'skipped', 0, { in: {}, out: {}, key: null }, null);
+/** Writes a row without ever failing: a log that cannot be appended (or a failure line that cannot be redacted) must not change a step's outcome or hide its real error. */
+export const tryLog = async (write) => {
+    try {
+        await write();
+    }
+    catch {
+        // The step's value, or its own error, is what the caller gets.
+    }
+};
+/** Logs a failed step: `error`, the failure's first line, and the key the step had from its inputs. */
+const logFailure = (context, step, ms, key, failure) => tryLog(async () => logStep(context, step, 'error', ms, { in: {}, out: {}, key }, await errorLine(context.storeDir, failure)));
+/**
+ * Runs `work` and logs its row: `ok` with what it reports, or `error` with the failure's first line and `failedKey` (a key built from
+ * the step's inputs before the work, so a retry of the same work has the same key), after which the failure goes on to the caller.
+ */
+export const runStep = async (context, step, work, failedKey) => {
+    const started = performance.now();
+    let done;
+    try {
+        done = await work();
+    }
+    catch (error) {
+        await logFailure(context, step, performance.now() - started, failedKey, error);
+        throw error;
+    }
+    await tryLog(() => logStep(context, step, 'ok', performance.now() - started, done.report, null));
+    return done.value;
+};
+/** The runs of one step and key that failed in a row. Three is the line spec 4.4 draws. */
+export const STUCK_RUNS = 3;
+/**
+ * Keys that failed on three runs in a row, per step and trigger (a nightly that fails every night is not cleared by the `stop` deposits
+ * between nights). A `skipped` or `deferred` row is no run of its step (a deferred deposit neither failed nor succeeded: it is waiting in the pending list); an `ok` row ends the open streaks of its own step and trigger, and of its step on the same key under another trigger, because a step's
+ * good run can carry a key (verify's high-water mark) that its failed run could not know before the work.
+ */
+export const stuckSteps = (rows) => {
+    const streaks = new Map();
+    for (const row of rows) {
+        if (row.key === null || row.status === 'skipped' || row.status === 'deferred')
+            continue;
+        const id = `${row.step}:${row.trigger}:${row.key}`;
+        if (row.status === 'ok') {
+            // Its own trigger's streaks end whatever key the good run carries; another trigger's streak ends only on the same key, which now has output (spec 4.2).
+            for (const open of [...streaks.keys()])
+                if (open.startsWith(`${row.step}:${row.trigger}:`) || (open.startsWith(`${row.step}:`) && open.endsWith(`:${row.key}`)))
+                    streaks.delete(open);
+            continue;
+        }
+        streaks.set(id, { step: row.step, key: row.key, runs: (streaks.get(id)?.runs ?? 0) + 1, error: row.error ?? '' });
+    }
+    return [...streaks.values()].filter((streak) => streak.runs >= STUCK_RUNS);
+};
+const add = (counts, kind, count) => {
+    counts[kind] = (counts[kind] ?? 0) + count;
+};
+export const stepCounters = (rows) => {
+    const lines = new Map();
+    const deposited = {};
+    const failures = {};
+    let derived = {};
+    let indexed = {};
+    let verify = null;
+    const recall = { queries: 0, matches: 0 };
+    for (const row of rows) {
+        const line = lines.get(`${row.step}@${row.ev}`) ?? { step: row.step, ev: row.ev, runs: 0, ok: 0, error: 0, skipped: 0, deferred: 0, ms: 0 };
+        line.runs += 1;
+        line[row.status] += 1;
+        if (row.status !== 'skipped' && row.status !== 'deferred')
+            line.ms += row.ms;
+        lines.set(`${row.step}@${row.ev}`, line);
+        if (row.status === 'error')
+            add(failures, `${row.step}@${row.ev}`, 1);
+        if (row.status !== 'ok')
+            continue;
+        if (row.step === 'deposit')
+            for (const [kind, count] of Object.entries(row.out))
+                add(deposited, kind, count);
+        if (row.step === 'derive')
+            derived = row.out;
+        if (row.step === 'index')
+            indexed = row.out;
+        if (row.step === 'verify')
+            verify = row.out;
+        if (row.step === 'recall') {
+            recall.queries += 1;
+            recall.matches += row.out.matches ?? 0;
+        }
+    }
+    const steps = [...lines.values()].map((line) => ({ step: line.step, ev: line.ev, runs: line.runs, ok: line.ok, error: line.error, skipped: line.skipped, deferred: line.deferred, meanMs: line.runs === line.skipped + line.deferred ? 0 : Math.round(line.ms / (line.runs - line.skipped - line.deferred)) }));
+    return { runs: rows.length, steps, deposited, derived, indexed, verify, recall, failures, stuck: stuckSteps(rows) };
+};
+export const formatStepCounters = (counters) => {
+    if (counters.runs === 0)
+        return 'pipeline step log: no step has run yet';
+    const row = (label, value) => `${label.padEnd(40)} ${value.padStart(12)}`;
+    const by = (label, counts) => Object.entries(counts).sort(([a], [b]) => a.localeCompare(b)).map(([kind, count]) => row(`${label} ${kind}`, String(count)));
+    const verify = counters.verify;
+    return [
+        row('pipeline step log (all time)', 'value'),
+        ...counters.steps.flatMap((line) => [row(`${line.step}@${line.ev} runs (ok/error/skipped)`, `${String(line.ok)}/${String(line.error)}/${String(line.skipped)}`), row(`${line.step}@${line.ev} mean ms`, String(line.meanMs)), ...(line.deferred === 0 ? [] : [row(`${line.step}@${line.ev} deferred (store locked)`, String(line.deferred))])]),
+        ...by('records deposited,', counters.deposited),
+        ...by('records derived,', counters.derived),
+        ...by('index holds,', counters.indexed),
+        ...(verify === null ? [] : [row('invariants passed of checked', `${String(verify.passed ?? 0)}/${String(verify.checked ?? 0)}`), row('cv-behind sessions', String(verify.cvBehind ?? 0)), row('sessions failing the raw audit', String(verify.merkle ?? 0))]),
+        ...(counters.recall.queries === 0 ? [] : [row('recall queries', String(counters.recall.queries)), row('recall matches', String(counters.recall.matches))]),
+        ...by('failed runs,', counters.failures),
+        row('stuck steps', String(counters.stuck.length)),
+        ...counters.stuck.map((stuck) => `  stuck: ${stuck.step} key ${stuck.key}, ${String(stuck.runs)} runs in a row: ${stuck.error}`),
+    ].join('\n');
+};
+/** A key built from its inputs and `ev` (spec 4.2), hashed; a recall has none in the spec (a lookup writes nothing), so its query is the input. */
+export const stampKey = (...parts) => digest([EXTRACTOR_VERSION, ...parts]);
+/**
+ * A deposit's key from its inputs (spec 4.2 names the files' state, which is not known before the work): `cv`, `ev`, the trigger, the allow
+ * entries or transcript asked for, and the redact domains. The `stop` hook passes no domains (it reads none), so its key has none.
+ */
+export const inputKey = (trigger, requested, domains) => digest([CAPTURE_VERSION, EXTRACTOR_VERSION, trigger, ...[...requested].sort(), '--', ...[...domains].sort()]);
+/**
+ * Runs a deposit and logs its rows: `deposit` with the records it wrote by kind, then, when it ended with the whole-store pass,
+ * `derive` and `index` (spec 4.1: one row per step at a nightly or manual run). `key` is built from the deposit's inputs (the trigger and the transcripts
+ * it was asked to deposit), so the failed row and the ok row of one deposit carry the same key and a failure that recovers is not stuck.
+ * A deposit that fails logs `error` and, when a derive was to follow, `skipped` for it and for the index (spec 4.4). A derive that fails
+ * after the deposit wrote its records logs `deposit` ok, `derive` error, and `index` skipped; an index that fails logs `derive` ok and then
+ * `index` error; the failure goes on to the caller. The SessionEnd hook runs
+ * `withDerive` false: derive follows at the nightly run.
+ */
+export const runDeposit = async (context, withDerive, key, work) => {
+    const trace = newTrace();
+    const started = performance.now();
+    let done;
+    try {
+        done = await work(trace);
+    }
+    catch (error) {
+        // A usage error is the operator's mistake, not a failed run of the step: no row, so repeating it is never "stuck".
+        if (error instanceof CliError && error.exitCode === ExitCode.USAGE)
+            throw error;
+        await logFailure(context, 'deposit', performance.now() - started, key, error);
+        if (withDerive) {
+            await tryLog(() => logSkipped(context, 'derive'));
+            await tryLog(() => logSkipped(context, 'index'));
+        }
+        throw error;
+    }
+    // The pending sessions a deposit took first are inputs it was asked for beside its own transcripts.
+    const counted = { ...done.in, ...trace.pending };
+    // A Stop deposit that found the store locked wrote nothing and went to the pending list: its own outcome, with the refusal's line.
+    const refusal = trace.deferred;
+    if (refusal !== undefined) {
+        await tryLog(() => logStep(context, 'deposit', 'deferred', performance.now() - started, { in: counted, out: trace.kinds, key }, refusal));
+        return done.value;
+    }
+    const derive = trace.derive;
+    const failure = trace.deriveFailure;
+    const index = trace.index;
+    const indexFailure = trace.indexFailure;
+    // The deposit row keeps its own time: the derive and index passes ran inside its work and have rows of their own.
+    const passes = (derive?.ms ?? failure?.ms ?? 0) + (index?.ms ?? indexFailure?.ms ?? 0);
+    await tryLog(() => logStep(context, 'deposit', 'ok', performance.now() - started - passes, { in: counted, out: trace.kinds, key }, null));
+    if (derive !== undefined)
+        await tryLog(() => logStep(context, 'derive', 'ok', derive.ms, { in: derive.in, out: derive.out, key: derive.key }, null));
+    if (failure !== undefined) {
+        await logFailure(context, 'derive', failure.ms, failure.key, failure.error);
+        await tryLog(() => logSkipped(context, 'index'));
+        throw failure.error;
+    }
+    if (index !== undefined)
+        await tryLog(() => logStep(context, 'index', 'ok', index.ms, { in: index.in, out: index.out, key: index.key }, null));
+    if (indexFailure !== undefined) {
+        await logFailure(context, 'index', indexFailure.ms, indexFailure.key, indexFailure.error);
+        throw indexFailure.error;
+    }
+    return done.value;
+};