foottrace 0.0.0-stage → 1.0.1

package/dist/esm/lib/memory/backtrack.js

@@ -0,0 +1,466 @@
+/**
+ * backtrack.ts — Backward causal chain analysis on the commit log.
+ *
+ * Implements **backward program slicing** (Weiser 1984, thin-slice variant):
+ * given a starting execution step, walk backwards through read→write
+ * dependencies to build the causal DAG that produced the data at that step.
+ *
+ * ## Algorithm
+ *
+ * BFS on the implicit dependency graph where edges run from reader → writer.
+ *
+ * 1. Locate startId in commitLog → root node
+ * 2. Get keysRead for root via `getKeysRead` callback
+ * 3. For each key read, find who last wrote it before this step → parent commit
+ *    ("wrote" is the writer rule of `keyPaths.ts`, 9.33.0: a row on the key, inside it, or
+ *    around it that changed it — so a stage that read `cfg` links to the subflow seed or
+ *    merge-back that wrote `cfg␟b`)
+ * 4. Create parent CausalNode, link to root.parents
+ * 5. Enqueue parent. Repeat until queue empty or limits hit.
+ *
+ * Output is a **DAG** (not a linked list): a stage reading `creditScore` AND `dti`
+ * from different writers has two parents.
+ *
+ * ## Staged Optimization
+ *
+ * Two writer-lookup strategies, chosen automatically by commit log size:
+ *
+ * | Strategy | When | Complexity per lookup |
+ * |----------|------|----------------------|
+ * | Linear scan | N ≤ 256 | O(N) — simple backward scan |
+ * | Reverse index | N > 256 | O(K log N) — prebuilt key→[indices], binary search |
+ *
+ * The threshold (256) is chosen so the O(N) build cost of the reverse index
+ * is amortized over the BFS traversal. Below 256, linear scan wins because
+ * there's no index build overhead. The consumer never sees this — `causalChain()`
+ * picks the right strategy internally (like a query optimizer choosing between
+ * sequential scan vs index scan based on table size).
+ *
+ * ## Complexity
+ *
+ * - **Small logs (N ≤ 256):** O(V × K × N) total. V=visited, K=avg keys/node.
+ * - **Large logs (N > 256):** O(N × U) index build + O(V × K × log N) lookups.
+ *   U = unique keys. Amortized over all BFS hops.
+ *
+ * ## References
+ *
+ * - Weiser, M. (1984). "Program Slicing." IEEE TSE.
+ * - Sridharan, M. et al. (2007). "Thin Slicing." PLDI.
+ *
+ * @example
+ * ```typescript
+ * import { causalChain, flattenCausalDAG, formatCausalChain } from 'foottrace';
+ *
+ * const dag = causalChain(commitLog, 'decide#2', (id) => recorder.getKeysRead(id));
+ * const flat = flattenCausalDAG(dag);     // BFS-ordered flat list
+ * console.log(formatCausalChain(dag));     // human-readable
+ * ```
+ */
+import { buildCommitIndex, findLastWriter } from './commitLogUtils.js';
+import { queryWork, relation, writesOnlyInside } from './keyPaths.js';
+import { logModel } from './logModel.js';
+// ── Staged optimization: writer lookup strategies ──────────────────────
+/**
+ * Threshold for switching from linear scan to reverse index.
+ * Below this, O(N) scan is faster (no index build cost).
+ * Above this, O(log N) binary search wins.
+ */
+const REVERSE_INDEX_THRESHOLD = 256;
+/** Strategy 1: Linear scan — O(N) per lookup, zero setup cost. */
+function linearScanLookup(commitLog) {
+    return (key, beforeIdx) => findLastWriter(commitLog, key, beforeIdx);
+}
+/**
+ * Strategy 2: Reverse index — one O(rows) build, O(depth + log N) per lookup.
+ * The log's read model (`logModel.ts`: a path trie, each top-level key folded at
+ * most once) answers the writer rule — rows on, inside or around the key — by
+ * binary search for the last write on or inside it, then a verdict only for the
+ * around-candidates after it. The same answer `findLastWriter` gives, so the
+ * strategy switch below never changes a slice.
+ */
+function reverseIndexLookup(commitLog) {
+    const model = logModel(commitLog);
+    return (key, beforeIdx) => {
+        const at = model.lastWriterBefore(key, beforeIdx);
+        return at >= 0 ? commitLog[at] : undefined;
+    };
+}
+/**
+ * The trace entries of `commit` that make it a writer of `linkKey` — the entries whose read
+ * prefix (`TraceEntry.readKeys`) a per-write edge expands through. The FIRST entry on the exact
+ * path when there is one (the #P1 behaviour, unchanged); otherwise every entry inside or around
+ * the key (9.33.0: the commit wrote the key through a nested row — a subflow seed, an
+ * outputMapper merge-back — and all of those rows wrote it).
+ */
+function linkingEntries(commit, linkKey) {
+    const exact = commit.trace.find((t) => t.path === linkKey);
+    if (exact !== undefined)
+        return [exact];
+    return commit.trace.filter((t) => relation(t.path, linkKey) !== undefined);
+}
+/**
+ * Staged optimization: pick the right writer-lookup strategy based on data size.
+ *
+ * Like a database query optimizer choosing between sequential scan and index scan:
+ *
+ * - **Small log (≤ 256):** Linear scan wins. Zero setup cost, good cache locality.
+ *   The overhead of building a reverse index isn't worth it for short logs.
+ *
+ * - **Large log (> 256):** Reverse index wins. O(N×U) upfront build cost is amortized
+ *   across all BFS hops. Each lookup becomes O(log N) via binary search instead of O(N).
+ *   For an agent loop with 500 iterations and 5 keys per hop, this is 500×5×log(500)≈22K ops
+ *   vs 500×5×500=1.25M ops with linear scan.
+ *
+ * The caller never sees this — `causalChain()` picks automatically.
+ */
+function createWriterLookup(commitLog) {
+    if (commitLog.length <= REVERSE_INDEX_THRESHOLD) {
+        return linearScanLookup(commitLog);
+    }
+    return reverseIndexLookup(commitLog);
+}
+// ── Core algorithm ─────────────────────────────────────────────────────
+/**
+ * RFC-003 D2: the `incompleteSources` node fragment for a commit — `{}`
+ * when the stage consumed no untracked read paths, keeping the field
+ * ABSENT (not empty-array-valued) for fully-tracked stages.
+ */
+function incompleteSourcesFragment(commit) {
+    if (!commit.untrackedSources || commit.untrackedSources.length === 0)
+        return {};
+    return { incompleteSources: commit.untrackedSources };
+}
+/**
+ * Build the causal DAG rooted at `startId` by walking backwards
+ * through read→write dependencies in the commit log.
+ *
+ * Automatically selects the optimal writer lookup strategy:
+ * - Linear scan for small logs (≤ 256 commits)
+ * - Reverse index with binary search for large logs (> 256 commits)
+ *
+ * Produces a DAG (not a tree): if two children both read from the same
+ * parent, the parent node is shared (deduped by runtimeStageId).
+ *
+ * @param commitLog   Ordered commit bundles from executor.getSnapshot().commitLog
+ * @param startId     runtimeStageId to start backtracking from
+ * @param getKeysRead Callback returning keys read by a given execution step
+ * @param options     Depth and node limits
+ * @returns Root CausalNode with .parents forming the DAG, or undefined if startId not found
+ */
+export function causalChain(commitLog, startId, getKeysRead, options) {
+    return buildCausalChain(commitLog, startId, getKeysRead, options);
+}
+/**
+ * Internal slice composition: keep the actual bundle selected by the writer
+ * query (including a nonempty continuation). This is not a public entry point.
+ * The read position still belongs to the execution's first commit.
+ */
+export function causalChainFromCommit(commitLog, writer, getKeysRead, options) {
+    return buildCausalChain(commitLog, writer.runtimeStageId, getKeysRead, options, writer);
+}
+function buildCausalChain(commitLog, startId, getKeysRead, options, selectedWriter) {
+    const maxDepth = options?.maxDepth ?? 20;
+    const maxNodes = options?.maxNodes ?? 100;
+    const controlDeps = options?.controlDeps;
+    const weigh = options?.weigh;
+    const perWrite = options?.edgeAttribution === 'per-write';
+    // RFC-003 D4 — truncation visibility. Set only when a limit actually
+    // cuts the slice; surfaced on the root as `truncated` so a consumer can
+    // never mistake a truncated slice for a complete one.
+    let truncatedByDepth = false;
+    let truncatedByNodes = false;
+    // A continuation may write more data, but it is not another execution of
+    // the stage's reads. Use the shared FIRST-commit rule for every read cutoff;
+    // keep writer bundles separately for their write metadata and read prefixes.
+    const idxMap = buildCommitIndex(commitLog);
+    queryWork.units += commitLog.length;
+    const startIdx = idxMap.get(startId);
+    if (startIdx === undefined)
+        return undefined;
+    const startCommit = selectedWriter ?? commitLog[startIdx];
+    // Pick writer lookup strategy based on log size
+    const findWriter = createWriterLookup(commitLog);
+    // Node dedup map: runtimeStageId → CausalNode (ensures DAG, not tree)
+    const nodeMap = new Map();
+    const root = {
+        runtimeStageId: startId,
+        stageId: startCommit.stageId,
+        stageName: startCommit.stage,
+        keysWritten: startCommit.trace.map((t) => t.path),
+        linkedBy: '',
+        depth: 0,
+        parents: [],
+        parentEdges: [],
+        ...incompleteSourcesFragment(startCommit),
+    };
+    nodeMap.set(startId, root);
+    // ── #P1 per-write attribution machinery (inert under 'stage') ──────────
+    // expandedReads: per node, the reads already queued for expansion — late
+    // links via additional keys re-enqueue only the DELTA, so the slice grows
+    // monotonically toward the stage-level ceiling and terminates (each key
+    // expands at most once per node). fullyExpanded: nodes that fell back to
+    // stage level (no readKeys on a linking entry) — nothing left to add.
+    const expandedReads = new Map();
+    const fullyExpanded = new Set();
+    /**
+     * Resolve a node's expansion read-set for the given linking WRITTEN keys.
+     * Per-write mode with provenance present → union of the linking writes'
+     * temporal-prefix `readKeys`. Any linking entry WITHOUT `readKeys` (or no
+     * linkKeys at all) → honest stage-level fallback, flagged via the second
+     * tuple member so the caller marks the node fully expanded.
+     */
+    function readsForLinks(commit, linkKeys) {
+        if (!perWrite || !linkKeys || linkKeys.length === 0) {
+            return [getKeysRead(commit.runtimeStageId), true];
+        }
+        const union = new Set();
+        for (const linkKey of linkKeys) {
+            const entries = linkingEntries(commit, linkKey);
+            if (entries.length === 0 || entries.some((e) => e.readKeys === undefined)) {
+                // Mixed/dial-off log — degrade THIS node to stage level, honestly.
+                return [getKeysRead(commit.runtimeStageId), true];
+            }
+            for (const entry of entries)
+                for (const rk of entry.readKeys)
+                    union.add(rk);
+        }
+        return [[...union], false];
+    }
+    // BFS/worklist queue: [node, commitIdx, depth, keysToExpand]
+    const [rootReads, rootIsFull] = perWrite
+        ? readsForLinks(startCommit, options?.rootLinkKeys)
+        : [getKeysRead(startId), true];
+    expandedReads.set(startId, new Set(rootReads));
+    if (rootIsFull)
+        fullyExpanded.add(startId);
+    const queue = [[root, startIdx, 0, rootReads]];
+    let visited = 1;
+    /**
+     * Link `node → parent` (creating + enqueueing the parent when new).
+     * Shared by data-edge expansion (read→write) and control-edge expansion
+     * (D3). One CausalEdge per distinct (parent, kind, key) link; `parents`
+     * keeps its historical one-entry-per-parent dedup.
+     */
+    function linkParent(node, parentCommit, kind, key, depth) {
+        const parentId = parentCommit.runtimeStageId;
+        // the dedup scans below, and the parent's trace when it is new — counted for the scaling test
+        queryWork.units += 1 + node.parents.length + node.parentEdges.length;
+        // #P1: the parent's expansion reads, resolved LAZILY (only for new nodes
+        // or per-write re-expansion — duplicate links under 'stage' pay nothing).
+        // Data links expand through the reads that fed the parent's write of
+        // `key`; control links expand the decider at stage level (a decision
+        // depends on everything it read).
+        const resolveLinkReads = () => kind === 'data'
+            ? readsForLinks(parentCommit, key !== undefined ? [key] : undefined)
+            : [getKeysRead(parentId), true];
+        let parentNode = nodeMap.get(parentId);
+        if (!parentNode) {
+            // New node — create and enqueue (respecting the node budget)
+            if (visited >= maxNodes) {
+                truncatedByNodes = true; // D4: a discovered parent was dropped
+                return;
+            }
+            const parentIdx = idxMap.get(parentId);
+            if (parentIdx === undefined)
+                return;
+            queryWork.units += parentCommit.trace.length;
+            parentNode = {
+                runtimeStageId: parentId,
+                stageId: parentCommit.stageId,
+                stageName: parentCommit.stage,
+                keysWritten: parentCommit.trace.map((t) => t.path),
+                // linkedBy stays a DATA-key concept (back-compat) — control-linked
+                // nodes carry their label on the edge instead.
+                linkedBy: kind === 'data' ? key ?? '' : '',
+                depth: depth + 1,
+                parents: [],
+                parentEdges: [],
+                ...incompleteSourcesFragment(parentCommit),
+            };
+            nodeMap.set(parentId, parentNode);
+            visited++;
+            const [linkReads, linkIsFull] = resolveLinkReads();
+            expandedReads.set(parentId, new Set(linkReads));
+            if (linkIsFull)
+                fullyExpanded.add(parentId);
+            queue.push([parentNode, parentIdx, depth + 1, linkReads]);
+        }
+        else if (perWrite && !fullyExpanded.has(parentId)) {
+            // #P1 worklist: an EXISTING node linked via another key may owe more
+            // expansion — enqueue only the reads not yet expanded (monotone; the
+            // node budget is untouched, no node is created). Re-expansion keeps
+            // the node's ORIGINAL depth so its parents get a consistent depth+1.
+            const [linkReads, linkIsFull] = resolveLinkReads();
+            const expanded = expandedReads.get(parentId);
+            const delta = linkReads.filter((k) => !expanded.has(k));
+            if (linkIsFull)
+                fullyExpanded.add(parentId);
+            if (delta.length > 0) {
+                for (const k of delta)
+                    expanded.add(k);
+                const parentIdx = idxMap.get(parentId);
+                if (parentIdx !== undefined)
+                    queue.push([parentNode, parentIdx, parentNode.depth, delta]);
+            }
+        }
+        // DAG merge: one parents[] entry per distinct parent (historical shape)
+        if (!node.parents.some((p) => p.runtimeStageId === parentId)) {
+            node.parents.push(parentNode);
+        }
+        // One edge per distinct (parent, kind, key) link. The weigher (D4)
+        // stamps the weight at creation; `undefined` → 1.0 — the engine never
+        // computes weights itself.
+        if (!node.parentEdges.some((e) => e.parent.runtimeStageId === parentId && e.kind === kind && e.key === key)) {
+            // Error isolation (review finding): a consumer weigher that throws
+            // must degrade to the default weight, never crash the slice — the
+            // same contract every other consumer callback in the library gets.
+            let weight = 1.0;
+            if (weigh) {
+                try {
+                    weight = weigh(node, parentNode, key, kind) ?? 1.0;
+                }
+                catch {
+                    /* weigher threw — keep 1.0, the slice stays usable */
+                }
+            }
+            const nested = kind === 'data' && key !== undefined && writesOnlyInside(parentCommit, key);
+            node.parentEdges.push({
+                parent: parentNode,
+                kind,
+                key,
+                weight,
+                ...(nested && { basis: 'nested-rows' }),
+            });
+        }
+    }
+    while (queue.length > 0) {
+        const [node, commitIdx, depth, keysToExpand] = queue.shift();
+        queryWork.units += 1 + keysToExpand.length;
+        if (depth >= maxDepth) {
+            // D4: only a node that still HAD something to expand counts as a cut
+            // (a leaf at the horizon truncates nothing).
+            if (keysToExpand.length > 0 || controlDeps?.(node.runtimeStageId) !== undefined) {
+                truncatedByDepth = true;
+            }
+            continue;
+        }
+        // Data edges: for each key in this expansion's read-set, find who wrote
+        // it. Under 'stage' this is the node's full read-set (historical
+        // behavior); under 'per-write' it is the linking writes' temporal prefix
+        // (or a worklist delta on re-expansion).
+        const keysRead = keysToExpand;
+        for (const key of keysRead) {
+            const writer = findWriter(key, commitIdx);
+            if (!writer) {
+                // F4b: say why there is no edge — no commit before this node wrote the key.
+                node.preRunReads ??= { code: 'pre-run-origin', keys: [] };
+                if (!node.preRunReads.keys.includes(key))
+                    node.preRunReads.keys.push(key);
+                continue;
+            }
+            linkParent(node, writer, 'data', key, depth);
+        }
+        // Control edge (RFC-003 D3): link the governing decider, labeled by the
+        // decide() rule label when present. The decider node then expands
+        // normally through its own data reads (and its own control parent).
+        if (controlDeps) {
+            const dep = controlDeps(node.runtimeStageId);
+            if (dep) {
+                const deciderIdx = idxMap.get(dep.deciderId);
+                if (deciderIdx !== undefined) {
+                    linkParent(node, commitLog[deciderIdx], 'control', dep.label, depth);
+                }
+            }
+        }
+    }
+    // RFC-003 D4 — truncation visibility on the root (absent when complete). It is DATA, and only
+    // data: a reader reads `root.truncated` (and `formatCausalChain` prints it); the record writes no
+    // console line (the dev-mode warning went in C6 — the record imports nothing outside itself).
+    if (truncatedByDepth || truncatedByNodes) {
+        root.truncated = { byDepth: truncatedByDepth, byNodes: truncatedByNodes };
+    }
+    return root;
+}
+// ── Utilities ──────────────────────────────────────────────────────────
+/**
+ * Flatten the causal DAG into a BFS-ordered list of nodes.
+ * Each node appears exactly once (first occurrence by BFS order).
+ * Useful for linear display or iteration.
+ */
+export function flattenCausalDAG(root) {
+    const result = [];
+    const visited = new Set();
+    const queue = [root];
+    while (queue.length > 0) {
+        const node = queue.shift();
+        if (visited.has(node.runtimeStageId))
+            continue;
+        visited.add(node.runtimeStageId);
+        result.push(node);
+        for (const parent of node.parents) {
+            if (!visited.has(parent.runtimeStageId)) {
+                queue.push(parent);
+            }
+        }
+    }
+    return result;
+}
+/**
+ * Format a causal DAG as human-readable indented text.
+ * Shows the dependency chain with depth indentation and linked-by keys.
+ *
+ * RFC-003 D2: nodes that consumed untracked read paths render an extra
+ * `⚠ also consumed … — slice may be incomplete here` line, so a consumer
+ * (human or LLM) debugging from the slice is TOLD when it is incomplete.
+ *
+ * RFC-003 D3: control edges render as `← [control: <rule label>]`
+ * (label omitted when the decision carried none). Data rendering is
+ * byte-identical to the pre-D3 output — `← via <key>` from the node's
+ * discovery-time `linkedBy`.
+ *
+ * RFC-003 D4: edge weights from the `weigh` hook render as a suffix —
+ * `← via systemPrompt (0.18)` — only when ≠ 1.0, so unweighted output is
+ * unchanged. A truncated slice (root.truncated) appends a final
+ * `⚠ slice truncated …` line.
+ */
+export function formatCausalChain(root) {
+    const lines = [];
+    const visited = new Set();
+    const weightSuffix = (edge) => edge !== undefined && edge.weight !== 1 ? ` (${edge.weight})` : '';
+    function walk(node, indent, edgesFromChild) {
+        if (visited.has(node.runtimeStageId)) {
+            lines.push(`${'  '.repeat(indent)}↳ ${node.runtimeStageId} (see above)`);
+            return;
+        }
+        visited.add(node.runtimeStageId);
+        const linkParts = [];
+        if (node.linkedBy) {
+            const dataEdge = edgesFromChild?.find((e) => e.kind === 'data');
+            linkParts.push(`via ${node.linkedBy}${weightSuffix(dataEdge)}`);
+        }
+        const controlEdge = edgesFromChild?.find((e) => e.kind === 'control');
+        if (controlEdge) {
+            linkParts.push(`[control${controlEdge.key ? `: ${controlEdge.key}` : ''}]${weightSuffix(controlEdge)}`);
+        }
+        const link = linkParts.length > 0 ? ` ← ${linkParts.join(' ← ')}` : '';
+        const writes = node.keysWritten.length > 0 ? ` [wrote: ${node.keysWritten.join(', ')}]` : '';
+        lines.push(`${'  '.repeat(indent)}${node.stageName} (${node.runtimeStageId})${link}${writes}`);
+        if (node.incompleteSources && node.incompleteSources.length > 0) {
+            lines.push(`${'  '.repeat(indent + 1)}⚠ also consumed ${node.incompleteSources.join('/')} — slice may be incomplete here`);
+        }
+        for (const parent of node.parents) {
+            walk(parent, indent + 1, node.parentEdges.filter((e) => e.parent === parent));
+        }
+    }
+    walk(root, 0);
+    if (root.truncated) {
+        const causes = [root.truncated.byDepth && 'maxDepth reached', root.truncated.byNodes && 'maxNodes reached']
+            .filter(Boolean)
+            .join(', ');
+        lines.push(`⚠ slice truncated (${causes}) — older causes exist beyond this horizon`);
+    }
+    return lines.join('\n');
+}
+// ── Exported for testing (internal) ────────────────────────────────────
+/** @internal Exposed for testing the strategy selection. */
+export const _REVERSE_INDEX_THRESHOLD = REVERSE_INDEX_THRESHOLD;