@fitchmultz/pi-subagents 0.44.7 → 0.46.0

package/dist/history/ingest.js

@@ -2,238 +2,23 @@ import * as fs from "node:fs";
 import { createHash } from "node:crypto";
 import { JournalFrames } from "../shared/journal-reader.js";
 import { HistoryIndexError } from "./types.js";
-import { exactTextDigest, HistoryStore, safeText } from "./store.js";
-import { stripTerminalSequences } from "../shared/native-tui.js";
-import { stripAcceptanceReport } from "../runs/shared/acceptance-reports.js";
-export function identity(stat) { return `${stat.dev}:${stat.ino}`; }
-export function stamp(stat) { return `${identity(stat)}:${stat.size}:${stat.mtimeNs}:${stat.ctimeNs}`; }
-const metadata = new Set(["type", "id", "parentId", "timestamp", "version", "cwd", "provider", "model", "modelId", "thinkingLevel", "customType", "display", "summary", "content"]);
-const messageFields = new Set(["role", "content", "output", "summary", "toolCallId", "toolName", "isError", "timestamp", "model", "provider", "stopReason", "errorMessage", "customType", "display", "command", "exitCode", "cancelled"]);
-export const previewLimits = { depth: 64, nodes: 16_384, arrayLength: 4096, keyLength: 512 };
-export const previewProjection = (keys, root) => {
-    if (!keys.length)
-        return true;
-    if (keys[0] === "message") {
-        if (keys.length === 1)
-            return true;
-        if (!messageFields.has(String(keys[1])))
-            return false;
-        if (keys[1] === "content") {
-            if (keys.length <= 3)
-                return 512;
-            if (keys[3] === "arguments") {
-                if (keys.length === 4)
-                    return true;
-                if (keys.length === 5)
-                    return Object.keys(root?.message?.content?.[Number(keys[2])]?.arguments ?? {}).length < 32 ? 512 : false;
-                return keys.length === 6 && keys[5] === 0 ? 512 : false;
-            }
-            return keys.length === 4 && ["type", "text", "thinking", "id", "name", "mimeType"].includes(String(keys[3])) ? 512 : false;
-        }
-        return keys.length === 2 ? 512 : false;
-    }
-    if (keys[0] === "details")
-        return keys.length === 1 || keys.length === 2 && ["bodyText", "message"].includes(String(keys[1])) || keys.length === 3 && keys[1] === "message" && keys[2] === "id" ? 512 : false;
-    if (keys[0] === "content" && keys.length > 1)
-        return keys.length <= 2 || keys.length === 3 && ["type", "text", "mimeType"].includes(String(keys[2])) ? 512 : false;
-    return keys.length === 1 && metadata.has(String(keys[0])) ? 512 : false;
-};
-function candidate(keys) {
-    return ["summary", "content"].includes(String(keys[0])) && keys.length === 1
-        || keys.length === 2 && keys[0] === "details" && keys[1] === "bodyText"
-        || keys[0] === "message" && keys.length === 2 && ["content", "output", "summary"].includes(String(keys[1]))
-        || keys[0] === "message" && keys[1] === "content" && keys.length === 4 && keys[3] === "text"
-        || keys[0] === "content" && keys.length === 3 && keys[2] === "text";
-}
-function allowed(value, field) {
-    const keys = JSON.parse(field);
-    if (value.type === "custom_message") {
-        if (value.display === false)
-            return false;
-        if (keys[0] === "details")
-            return value.customType === "subagent-human-message";
-        if (value.customType === "subagent-human-message" && typeof value.details?.bodyText === "string")
-            return false;
-        return keys[0] === "content" && (keys.length === 1 || value.content?.[Number(keys[1])]?.type === "text");
-    }
-    if (["compaction", "branch_summary"].includes(value.type))
-        return keys.length === 1 && keys[0] === "summary";
-    if (value.type !== "message" || keys[0] !== "message")
-        return false;
-    const message = value.message;
-    if (!message)
-        return false;
-    if (["compactionSummary", "branchSummary"].includes(message.role))
-        return keys[1] === "summary";
-    if (message.role === "bashExecution")
-        return keys[1] === "output";
-    if (!["user", "assistant", "toolResult", "custom"].includes(message.role) || message.role === "custom" && message.display === false)
-        return false;
-    return keys[1] === "content" && (keys.length === 2 || message.content?.[Number(keys[2])]?.type === "text");
-}
-export function compactEntry(value) {
-    let remaining = 4096;
-    const clean = (input, key) => {
-        if (typeof input === "string") {
-            const text = safeText(input, ["id", "parentId", "type", "role", "toolCallId", "name", "toolName", "timestamp"].includes(key ?? "") ? 512 : Math.min(512, remaining));
-            remaining -= Math.min(remaining, text.length);
-            return text;
-        }
-        if (Array.isArray(input))
-            return input.slice(0, 32).map((item) => clean(item));
-        if (input && typeof input === "object")
-            return Object.fromEntries(Object.entries(input).map(([name, item]) => [name, clean(item, name)]));
-        return input;
-    };
-    return clean(value);
-}
-/** Exact matches never use previews. Oversized selected text has no fingerprint. */
-class AnswerTexts {
-    fields = new Map();
-    length = 0;
-    unavailable = false;
-    write(keys, text) {
-        if (this.unavailable || keys.length !== 4 || keys[0] !== "message" || keys[1] !== "content" || !["text", "thinking"].includes(String(keys[3])))
-            return;
-        this.length += text.length;
-        if (this.length > 16 * 1024 * 1024 || Number(keys[2]) >= 4096) {
-            this.unavailable = true;
-            this.fields.clear();
-            return;
-        }
-        const index = Number(keys[2]);
-        this.fields.set(index, (this.fields.get(index) ?? "") + text);
-    }
-    fingerprints(value, id) {
-        if (this.unavailable || value.type !== "message" || value.message?.role !== "assistant")
-            return [];
-        const content = Array.isArray(value.message.content) ? value.message.content : [];
-        const first = content.findIndex((part, index) => ["text", "thinking"].includes(part.type) && Boolean(this.fields.get(index)));
-        if (first < 0 || !content.some((part, index) => part.type === "text" && Boolean(this.fields.get(index))))
-            return [];
-        const item = `${id}:${first}`, parts = content.map((part, index) => ["text", "thinking"].includes(part.type) ? stripTerminalSequences(this.fields.get(index) ?? "") : part.type === "image" ? `[Image: ${part.mimeType ?? "image"}]` : "").filter(Boolean);
-        const texts = [parts.join("\n"), ...content.flatMap((part, index) => part.type === "text" ? [stripTerminalSequences(this.fields.get(index) ?? "")] : [])];
-        return [...new Set(texts.map((text) => stripAcceptanceReport(text).trim()).filter(Boolean))].map((text) => [exactTextDigest(text), item]);
-    }
-}
-function visibleId(value, id) {
-    if (["custom_message", "compaction", "branch_summary"].includes(value.type))
-        return id;
-    if (value.type !== "message")
-        return null;
-    if (["user", "toolResult", "bashExecution"].includes(value.message?.role))
-        return id;
-    if (value.message?.role !== "assistant")
-        return null;
-    const content = Array.isArray(value.message.content) ? value.message.content : [];
-    const last = content.findLastIndex((part) => part.type === "toolCall" || part.type === "text" && part.text || part.type === "thinking" && part.thinking);
-    return value.message.errorMessage ? `${id}:error` : last < 0 ? null : `${id}:${last}`;
-}
-/** Streaming ECMA-48 removal, retaining original UTF-16 positions without buffering control payloads. */
-class TerminalText {
-    state = "text";
-    write(text, visible) {
-        let position = 0;
-        while (position < text.length) {
-            if (this.state === "text") {
-                const next = text.slice(position).search(/[\x1b\x90\x98\x9b\x9d-\x9f]/);
-                if (next < 0) {
-                    visible(text.slice(position), position);
-                    return;
-                }
-                visible(text.slice(position, position + next), position);
-                position += next;
-                const control = text[position++];
-                this.state = control === "\x1b" ? "escape" : control === "\x9b" ? "csi" : "string";
-                continue;
-            }
-            const char = text[position++];
-            if (this.state === "escape")
-                this.state = char === "[" ? "csi" : "]PX^_".includes(char) ? "string" : char >= " " && char <= "/" ? "intermediate" : "text";
-            else if (this.state === "intermediate") {
-                if (char >= "0" && char <= "~")
-                    this.state = "text";
-            }
-            else if (this.state === "csi") {
-                if (char >= "@" && char <= "~")
-                    this.state = "text";
-            }
-            else if (char === "\x07" || char === "\x9c" || this.state === "stringEscape" && char === "\\")
-                this.state = "text";
-            else
-                this.state = char === "\x1b" ? "stringEscape" : "string";
-        }
-    }
-}
-/** Token windows overlap 12 tokens, including phrases across arbitrarily long whitespace. */
-class TextChunks {
-    fields = new Map();
-    staged = [];
-    emit(field, buffer, final = false) {
-        if (!buffer.tokens.length)
-            return;
-        this.staged.push({ field, start: buffer.tokens[0].start, end: buffer.tokens.at(-1).end, text: buffer.tokens.map((token) => token.text).join(" ") });
-        buffer.tokens = final ? [] : buffer.tokens.slice(-12);
-        buffer.length = buffer.tokens.reduce((length, token) => length + token.text.length + 1, 0);
-    }
-    token(field, buffer, end) {
-        if (!buffer.word)
-            return;
-        buffer.tokens.push({ text: buffer.word, start: buffer.wordStart, end });
-        buffer.length += buffer.word.length + 1;
-        buffer.word = "";
-        if (buffer.tokens.length >= 256 || buffer.length >= 8192 && buffer.tokens.length > 12)
-            this.emit(field, buffer);
-    }
-    write(keys, text) {
-        if (!candidate(keys))
-            return;
-        const field = JSON.stringify(keys);
-        let buffer = this.fields.get(field);
-        if (!buffer) {
-            if (this.fields.size >= 4096)
-                throw new HistoryIndexError("RECORD_COMPLEXITY", "Record exceeds the 4096 visible text-field indexing budget.");
-            this.fields.set(field, buffer = { word: "", wordStart: 0, received: 0, tokens: [], length: 0, terminal: new TerminalText() });
-        }
-        buffer.terminal.write(text, (visible, offset) => {
-            for (const part of visible.matchAll(/[\p{L}\p{N}\p{M}]+|[^\p{L}\p{N}\p{M}]+/gu)) {
-                const start = buffer.received + offset + part.index;
-                if (/^[\p{L}\p{N}\p{M}]/u.test(part[0])) {
-                    if (!buffer.word)
-                        buffer.wordStart = start;
-                    buffer.word += part[0];
-                    if (buffer.word.length > 65_536)
-                        throw new HistoryIndexError("RECORD_COMPLEXITY", "Record contains a lexical token exceeding the 64 KiB indexing budget.");
-                }
-                else
-                    this.token(field, buffer, start);
-            }
-        });
-        buffer.received += text.length;
-    }
-    flush(store, finish = false) {
-        if (finish) {
-            for (const [field, buffer] of this.fields) {
-                this.token(field, buffer, buffer.received);
-                this.emit(field, buffer, true);
-            }
-            this.fields.clear();
-        }
-        if (this.staged.length) {
-            const insert = store.db.prepare("INSERT INTO pending_text VALUES (?,?,?,?)");
-            for (const item of this.staged)
-                insert.run(item.field, item.start, item.end, item.text);
-            this.staged = [];
-        }
-    }
-}
+import { safeText } from "./store.js";
+import { previewProjection, previewLimits, allowed } from "./preview.js";
+import { TextChunks } from "./text-chunks.js";
+import { AnswerTexts } from "./answer-texts.js";
+import { object } from "./values.js";
+import { identity, stamp, openSource, requireSource } from "./source-file.js";
+export { identity, stamp } from "./source-file.js";
+import { number, text } from "./rows.js";
+import { stageEntry, publishTools } from "./entry-ingestion.js";
+export { previewProjection, previewLimits, compactEntry } from "./preview.js";
 /** One physical source, incremental byte framing and short row/cursor publication transactions. */
 export class SourceIngest {
     fd;
     initial;
     source;
     recordDigest = createHash("sha256");
-    /** prefix_digest is "start:sha256" of the last indexed line; earlier releases hashed the whole prefix. */
+    /** prefix_digest anchors the last indexed line; older releases hashed the whole prefix. */
     legacy = false;
     expected = null;
     verified = createHash("sha256");
@@ -257,70 +42,88 @@ export class SourceIngest {
     id;
     constructor(store, source, force = false) {
         this.store = store;
-        this.source = source;
         this.id = source.id;
-        store.beginIngest();
-        let opened;
+        const opened = openSource(store, source.id);
+        this.source = opened.source;
+        this.fd = opened.fd;
+        this.initial = opened.initial;
+        this.position = this.source.cursor;
+        this.headerSeen = this.source.cursor > 0;
+        this.prepareVerification();
+        this.phase = this.source.cursor > 0 ? "verify" : "read";
+        if (!force && this.source.stamp === stamp(this.initial)) {
+            this.phase = "done";
+        }
         try {
-            this.source = source = store.source(source.id);
-            if (!source)
-                throw new HistoryIndexError("SOURCE_CHANGED", "Source was removed before ingestion.");
-            store.operations.sourceChecks++;
-            store.operations.sourceOpens++;
-            this.fd = opened = fs.openSync(source.path, "r");
-            this.initial = fs.fstatSync(this.fd, { bigint: true });
-            if (!this.initial.isFile() || this.initial.size > BigInt(Number.MAX_SAFE_INTEGER))
-                throw new Error("Journal is not a supported regular file.");
-            if (source.identity && (source.identity !== identity(this.initial) || Number(this.initial.size) < source.cursor)) {
-                store.resetSource(source.id);
-                this.source = source = store.source(source.id);
+            if (this.phase !== "done") {
+                store.run("UPDATE sources SET state='indexing',error=NULL WHERE id=?", this.id);
             }
-            this.position = source.cursor;
-            this.headerSeen = source.cursor > 0;
-            store.clearUnpublished(source.id);
-            store.run("DELETE FROM pending_text");
-            // Appends re-hash only the last indexed line, so live sources cost their new bytes, not their size.
-            const anchor = /^(\d+):([0-9a-f]{64})$/.exec(source.prefix_digest ?? "");
-            this.legacy = !anchor;
-            this.expected = anchor ? anchor[2] : source.prefix_digest;
-            this.verifyAt = this.lineStart = anchor ? Number(anchor[1]) : 0;
-            this.phase = !force && source.stamp === stamp(this.initial) ? "done" : source.cursor ? "verify" : "read";
-            if (this.phase !== "done")
-                store.run("UPDATE sources SET state='indexing',error=NULL WHERE id=?", source.id);
             this.frames = new JournalFrames(previewProjection, (record) => {
-                this.record = { ...record, digest: this.recordDigest.copy().digest("hex") };
-            }, "inspect", source.cursor, (_start, _end, error) => { this.malformed = true; this.complexity = error instanceof RangeError || error instanceof HistoryIndexError && error.code === "RECORD_COMPLEXITY"; }, (keys, chunk) => { this.text.write(keys, chunk); this.answerTexts.write(keys, chunk); }, undefined, true, previewLimits);
+                this.record = {
+                    ...record,
+                    value: object(record.value),
+                    digest: this.recordDigest.copy().digest("hex"),
+                };
+            }, {
+                policy: "inspect",
+                start: this.source.cursor,
+                malformed: (_start, _end, error) => {
+                    this.malformed = true;
+                    this.complexity =
+                        error instanceof RangeError ||
+                            (error instanceof HistoryIndexError && error.code === "RECORD_COMPLEXITY");
+                },
+                stringChunk: (keys, chunk) => {
+                    this.text.write(keys, chunk);
+                    this.answerTexts.write(keys, chunk);
+                },
+                requireNewline: true,
+                limits: previewLimits,
+            });
         }
         catch (error) {
-            if (opened !== undefined)
-                fs.closeSync(opened);
+            fs.closeSync(this.fd);
             store.endIngest();
             throw error;
         }
     }
+    prepareVerification() {
+        const anchor = /^(\d+):([0-9a-f]{64})$/.exec(this.source.prefix_digest ?? "");
+        this.legacy = !anchor;
+        this.expected = anchor ? anchor[2] : this.source.prefix_digest;
+        this.lineStart = anchor ? Number(anchor[1]) : 0;
+        this.verifyAt = this.lineStart;
+    }
     published(end, work) {
         this.store.transaction(() => {
             const current = this.store.source(this.id);
-            if (!current || current.generation !== this.source.generation || current.cursor !== this.source.cursor)
+            if (!current ||
+                current.generation !== this.source.generation ||
+                current.cursor !== this.source.cursor) {
                 throw new HistoryIndexError("SOURCE_CHANGED", "Another writer advanced this source; reconcile before replaying.");
+            }
             work?.();
             this.store.run("UPDATE sources SET cursor=?,prefix_digest=?,identity=? WHERE id=?", end, `${this.source.cursor}:${this.recordDigest.copy().digest("hex")}`, identity(this.initial), this.id);
             this.store.bump();
         });
-        this.source.cursor = end;
+        this.source = { ...this.source, cursor: end };
         this.recordDigest = createHash("sha256");
         this.text = new TextChunks();
         this.answerTexts = new AnswerTexts();
         this.store.run("DELETE FROM pending_text");
         this.record = undefined;
-        this.malformed = this.complexity = false;
+        this.malformed = false;
+        this.complexity = false;
     }
     finishRecord(end) {
         this.text.flush(this.store, true);
         if (this.malformed) {
-            if (!this.headerSeen)
+            if (!this.headerSeen) {
                 throw new Error("Malformed native session header.");
-            this.published(end, () => this.store.run("UPDATE sources SET malformed=malformed+1,complexity=complexity+? WHERE id=?", this.complexity ? 1 : 0, this.id));
+            }
+            this.published(end, () => {
+                this.store.run("UPDATE sources SET malformed=malformed+1,complexity=complexity+? WHERE id=?", this.complexity ? 1 : 0, this.id);
+            });
             return;
         }
         const record = this.record;
@@ -330,58 +133,102 @@ export class SourceIngest {
         }
         const value = record.value;
         if (!this.headerSeen) {
-            if (value.type !== "session")
-                throw new Error("Linked source does not have a native session header.");
-            this.published(end, () => this.store.run("UPDATE sources SET session_id=?,header_digest=?,format_version=? WHERE id=?", typeof value.id === "string" ? value.id : null, record.digest, typeof value.version === "number" ? value.version : null, this.id));
-            this.source.format_version = typeof value.version === "number" ? value.version : null;
-            this.headerSeen = true;
+            this.publishHeader(record);
             return;
         }
-        if (value.type === "session")
+        if (value.type === "session") {
             throw new Error("Unexpected session header inside a native source.");
+        }
         const nativeId = typeof value.id === "string" ? value.id : null;
         const id = nativeId ?? `legacy-${record.start}`;
         if (this.store.get("SELECT rowid FROM entries WHERE source_id=? AND generation=? AND id=?", this.id, this.source.generation, id)) {
-            this.published(end, () => this.store.run("UPDATE sources SET malformed=malformed+1 WHERE id=?", this.id));
+            this.published(end, () => {
+                this.store.run("UPDATE sources SET malformed=malformed+1 WHERE id=?", this.id);
+            });
             return;
         }
-        const timestamp = typeof value.timestamp === "number" ? value.timestamp : Date.parse(value.timestamp);
-        const parent = this.source.format_version === 1 ? this.store.get("SELECT id FROM entries WHERE source_id=? AND generation=? AND published=1 ORDER BY start DESC LIMIT 1", this.id, this.source.generation)?.id ?? null : typeof value.parentId === "string" ? value.parentId : null;
-        const preview = compactEntry(value);
-        preview.id = id;
-        preview.parentId = parent;
-        const model = value.type === "model_change" && typeof value.modelId === "string" ? `${value.provider}/${value.modelId}` : value.type === "message" && value.message?.role === "assistant" && typeof value.message.model === "string" ? `${value.message.provider}/${value.message.model}` : null;
-        this.stagedEntry = Number(this.store.run("INSERT INTO entries(source_id,generation,id,native_id,parent_id,start,end,digest,timestamp,type,preview,role,assistant_text,visible_id,human_id,configuration_model,configuration_thinking) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", this.id, this.source.generation, id, nativeId, parent, record.start, end, record.digest, Number.isFinite(timestamp) ? timestamp : null, typeof value.type === "string" ? value.type : "unknown", JSON.stringify(preview), typeof value.message?.role === "string" ? value.message.role : null, value.message?.role === "assistant" && Array.isArray(value.message.content) && value.message.content.some((part) => part.type === "text" && part.text) ? 1 : 0, visibleId(value, id), value.type === "custom_message" && value.customType === "subagent-human-message" && typeof value.details?.message?.id === "string" ? value.details.message.id : null, model, value.type === "thinking_level_change" && typeof value.thinkingLevel === "string" ? value.thinkingLevel : null).lastInsertRowid);
+        this.stagedEntry = stageEntry(this.store, this.source, record);
         this.textCursor = 0;
         this.phase = "text";
     }
+    publishHeader(record) {
+        const value = record.value;
+        if (value.type !== "session") {
+            throw new Error("Linked source does not have a native session header.");
+        }
+        const version = typeof value.version === "number" ? value.version : null;
+        this.published(record.end, () => {
+            this.store.run("UPDATE sources SET session_id=?,header_digest=?,format_version=? WHERE id=?", typeof value.id === "string" ? value.id : null, record.digest, version, this.id);
+        });
+        this.source = { ...this.source, format_version: version };
+        this.headerSeen = true;
+    }
+    verifyPrefix() {
+        const bytes = Buffer.allocUnsafe(Math.min(64 * 1024, this.source.cursor - this.verifyAt));
+        const count = fs.readSync(this.fd, bytes, 0, bytes.length, this.verifyAt);
+        if (count !== bytes.length) {
+            throw new Error("Journal truncated during prefix verification.");
+        }
+        this.store.count("sourceBytesRead", count);
+        this.verified.update(bytes);
+        if (this.legacy) {
+            // A whole-prefix check also finds the last indexed line for later bounded checks.
+            const before = this.source.cursor - 2 - this.verifyAt;
+            const lf = before < 0 ? -1 : bytes.lastIndexOf(10, before);
+            if (lf < 0) {
+                this.line.update(bytes);
+            }
+            else {
+                this.lineStart = this.verifyAt + lf + 1;
+                this.line = createHash("sha256").update(bytes.subarray(lf + 1));
+            }
+        }
+        this.verifyAt += count;
+        if (this.verifyAt === this.source.cursor) {
+            if (this.verified.digest("hex") !== this.expected) {
+                this.store.resetSource(this.id);
+                throw new HistoryIndexError("SOURCE_CHANGED", "Journal prefix changed; a new physical generation was scheduled.");
+            }
+            if (this.legacy) {
+                this.store.run("UPDATE sources SET prefix_digest=? WHERE id=?", `${this.lineStart}:${this.line.digest("hex")}`, this.id);
+            }
+            this.phase = "read";
+        }
+    }
     indexText() {
+        const record = this.record;
+        const rowid = this.stagedEntry;
+        if (!record || rowid === undefined) {
+            throw new HistoryIndexError("CORRUPT", "Missing staged entry.");
+        }
         const chunks = this.store.all("SELECT rowid,* FROM pending_text WHERE rowid>? ORDER BY rowid LIMIT 32", this.textCursor);
-        if (chunks.length) {
+        if (chunks.length > 0) {
             this.store.transaction(() => {
                 for (const chunk of chunks) {
-                    if (!allowed(this.record.value, chunk.field))
+                    if (!allowed(record.value, text(chunk, "field"))) {
                         continue;
-                    const text = safeText(chunk.text, 1_048_576);
-                    const id = this.store.run("INSERT INTO documents(entry_rowid,field,text_start,text_end,preview) VALUES (?,?,?,?,?)", this.stagedEntry, chunk.field, chunk.position, chunk.end_position, safeText(text, 512)).lastInsertRowid;
-                    this.store.run("INSERT INTO corpus(rowid,text) VALUES (?,?)", id, text);
+                    }
+                    const body = safeText(text(chunk, "text"), 1_048_576);
+                    const id = this.store.run("INSERT INTO documents(entry_rowid,field,text_start,text_end,preview) VALUES (?,?,?,?,?)", rowid, text(chunk, "field"), number(chunk, "position"), number(chunk, "end_position"), safeText(body, 512)).lastInsertRowid;
+                    this.store.run("INSERT INTO corpus(rowid,text) VALUES (?,?)", id, body);
                 }
             });
-            this.textCursor = chunks.at(-1).rowid;
+            const last = chunks.at(-1);
+            if (last) {
+                this.textCursor = number(last, "rowid");
+            }
             return;
         }
-        const value = this.record.value, rowid = this.stagedEntry;
-        const answers = this.answerTexts.fingerprints(value, typeof value.id === "string" ? value.id : `legacy-${this.record.start}`);
-        this.published(this.record.end, () => {
-            if (this.store.run("UPDATE entries SET published=1 WHERE rowid=? AND source_id=? AND generation=? AND published=0", rowid, this.id, this.source.generation).changes !== 1)
+        const value = record.value;
+        const answers = this.answerTexts.fingerprints(value, typeof value.id === "string" ? value.id : `legacy-${record.start}`);
+        this.published(record.end, () => {
+            if (this.store.run("UPDATE entries SET published=1 WHERE rowid=? AND source_id=? AND generation=? AND published=0", rowid, this.id, this.source.generation).changes !== 1) {
                 throw new HistoryIndexError("SOURCE_CHANGED", "Staged entry disappeared before publication; reconcile before advancing the source.");
-            for (const [digest, item] of answers)
+            }
+            for (const [digest, item] of answers) {
                 this.store.run("INSERT OR IGNORE INTO answers VALUES (?,?,?)", rowid, digest, item);
-            for (const part of Array.isArray(value.message?.content) ? value.message.content : [])
-                if (part.type === "toolCall" && typeof part.id === "string")
-                    this.store.run("INSERT INTO tools VALUES (?,?,?,?,?)", rowid, this.id, this.source.generation, part.id, "call");
-            if (value.message?.role === "toolResult" && typeof value.message.toolCallId === "string")
-                this.store.run("INSERT INTO tools VALUES (?,?,?,?,?)", rowid, this.id, this.source.generation, value.message.toolCallId, "result");
+            }
+            publishTools(this.store, this.source, rowid, value);
         });
         this.stagedEntry = undefined;
         this.phase = "read";
@@ -397,43 +244,20 @@ export class SourceIngest {
             return false;
         }
         if (this.phase === "verify") {
-            const bytes = Buffer.allocUnsafe(Math.min(64 * 1024, this.source.cursor - this.verifyAt));
-            const count = fs.readSync(this.fd, bytes, 0, bytes.length, this.verifyAt);
-            if (count !== bytes.length)
-                throw new Error("Journal truncated during prefix verification.");
-            this.store.operations.sourceBytesRead += count;
-            this.verified.update(bytes);
-            if (this.legacy) {
-                // A whole-prefix check also finds the last indexed line, which anchors every later check.
-                const before = this.source.cursor - 2 - this.verifyAt, lf = before < 0 ? -1 : bytes.lastIndexOf(10, before);
-                if (lf < 0)
-                    this.line.update(bytes);
-                else {
-                    this.lineStart = this.verifyAt + lf + 1;
-                    this.line = createHash("sha256").update(bytes.subarray(lf + 1));
-                }
-            }
-            this.verifyAt += count;
-            if (this.verifyAt === this.source.cursor) {
-                if (this.verified.digest("hex") !== this.expected) {
-                    this.store.resetSource(this.id);
-                    throw new HistoryIndexError("SOURCE_CHANGED", "Journal prefix changed; a new physical generation was scheduled.");
-                }
-                if (this.legacy)
-                    this.store.run("UPDATE sources SET prefix_digest=? WHERE id=?", `${this.lineStart}:${this.line.digest("hex")}`, this.id);
-                this.phase = "read";
-            }
+            this.verifyPrefix();
             return false;
         }
-        if (!this.buffer.length) {
+        if (this.buffer.length === 0) {
             const remaining = Number(this.initial.size) - this.position;
-            if (remaining <= 0)
+            if (remaining <= 0) {
                 return this.finish();
+            }
             const bytes = Buffer.allocUnsafe(Math.min(64 * 1024, remaining));
             const count = fs.readSync(this.fd, bytes, 0, bytes.length, this.position);
-            if (count !== bytes.length)
+            if (count !== bytes.length) {
                 throw new Error("Journal truncated during ingestion.");
-            this.store.operations.sourceBytesRead += count;
+            }
+            this.store.count("sourceBytesRead", count);
             this.position += count;
             this.buffer = bytes;
         }
@@ -443,26 +267,39 @@ export class SourceIngest {
         this.frames.write(bytes);
         this.text.flush(this.store);
         this.buffer = this.buffer.subarray(bytes.length);
-        if (newline >= 0)
+        if (newline >= 0) {
             this.finishRecord(this.frames.offset);
+        }
         return false;
     }
     finish() {
         this.frames.finish();
         const now = fs.statSync(this.source.path, { bigint: true });
-        if (identity(now) !== identity(this.initial) || now.size < this.initial.size || now.size === this.initial.size && stamp(now) !== stamp(this.initial)) {
+        if (identity(now) !== identity(this.initial) ||
+            now.size < this.initial.size ||
+            (now.size === this.initial.size && stamp(now) !== stamp(this.initial))) {
             this.store.resetSource(this.id);
             throw new HistoryIndexError("SOURCE_CHANGED", "Journal changed during ingestion; retrying its new generation.");
         }
-        const { malformed, complexity } = this.store.source(this.id);
-        this.store.run("UPDATE sources SET identity=?,stamp=?,checked_at=?,state=?,error=? WHERE id=?", identity(this.initial), stamp(this.initial), Date.now(), malformed ? "degraded" : this.source.cursor < Number(this.initial.size) ? "partial" : "current", malformed ? `${malformed} malformed, duplicate or over-budget LF-published records were skipped.${complexity ? ` ${complexity} exceeded bounded history structure/text budgets.` : ""}` : null, this.id);
+        const { malformed, complexity } = requireSource(this.store, this.id);
+        this.store.run("UPDATE sources SET identity=?,stamp=?,checked_at=?,state=?,error=? WHERE id=?", identity(this.initial), stamp(this.initial), Date.now(), this.sourceState(malformed), malformed > 0
+            ? `${malformed} malformed, duplicate or over-budget LF-published records were skipped.${complexity > 0 ? ` ${complexity} exceeded bounded history structure/text budgets.` : ""}`
+            : null, this.id);
         this.phase = "done";
         this.close();
         return true;
     }
-    close() { if (!this.closed) {
-        fs.closeSync(this.fd);
-        this.store.endIngest();
-        this.closed = true;
-    } }
+    sourceState(malformed) {
+        if (malformed > 0) {
+            return "degraded";
+        }
+        return this.source.cursor < Number(this.initial.size) ? "partial" : "current";
+    }
+    close() {
+        if (!this.closed) {
+            fs.closeSync(this.fd);
+            this.store.endIngest();
+            this.closed = true;
+        }
+    }
 }

package/dist/history/output-digests.js

@@ -0,0 +1,40 @@
+import * as fs from "node:fs";
+import { exactTextDigest } from "./text.js";
+import { readSavedOutput } from "./canonical-result.js";
+/** Cached by physical file snapshot; compact/possibly truncated previews never become exact matches. */
+export class OutputDigests {
+    cached = new Map();
+    read(result) {
+        if (!result) {
+            return null;
+        }
+        if (result.fullOutputPath === undefined || result.fullOutputPath.length === 0) {
+            return typeof result.finalOutput === "string" &&
+                result.finalOutput.length !== 8192 &&
+                result.truncation?.truncated !== true
+                ? exactTextDigest(result.finalOutput)
+                : null;
+        }
+        return this.fileDigest(result.fullOutputPath);
+    }
+    fileDigest(file) {
+        try {
+            const stat = fs.statSync(file, { bigint: true });
+            if (!stat.isFile() || stat.size > 16n * 1024n * 1024n) {
+                return null;
+            }
+            const stamp = `${stat.dev}:${stat.ino}:${stat.size}:${stat.mtimeNs}:${stat.ctimeNs}`;
+            const cached = this.cached.get(file);
+            if (cached?.stamp === stamp) {
+                return cached.digest;
+            }
+            const digest = exactTextDigest(readSavedOutput(file));
+            this.cached.set(file, { stamp, digest });
+            return digest;
+        }
+        catch {
+            // An unavailable canonical file has no exact fingerprint; never fall back to its preview.
+            return null;
+        }
+    }
+}