@fitchmultz/pi-subagents 0.0.0-stage → 0.44.5

package/src/history/index.ts

@@ -0,0 +1,156 @@
+import { fork, type ChildProcess } from "node:child_process";
+import { fileURLToPath } from "node:url";
+import * as path from "node:path";
+import { HistoryIndexError } from "./types.ts";
+import { requirePiPackageRoot } from "../runs/shared/pi-spawn.ts";
+import type { ForegroundResumeRun, HistoryEntry, HistoryEntryInput, HistoryIndexStatus, HistoryOwner, HistoryPage, HistoryPageInput, HistoryRunOptions, HistoryRunPage, HistoryResult, HistorySearchInput, HistorySearchPage, OwnedRun, Request, Response } from "./types.ts";
+export * from "./types.ts";
+
+/** A disposable browse index. Authoritative owner/control/receipt checks remain outside this engine. */
+export class SubagentHistoryIndex {
+	private process?: ChildProcess;
+	private children = new Map<ChildProcess, Promise<void>>();
+	private owner?: HistoryOwner;
+	private ready?: Promise<void>;
+	private unavailable?: Error;
+	private closed = false;
+	private closing?: Promise<void>;
+	private sequence = 0;
+	private pending = new Map<number, { resolve: (value: any) => void; reject: (error: Error) => void; cleanup: () => void }>();
+	private listeners = new Set<() => void>();
+	private agentDir: string;
+	constructor(agentDir: string) { this.agentDir = path.resolve(agentDir); }
+	get failure(): Error | undefined { return this.unavailable; }
+
+	private spawn(): void {
+		const child = fork(fileURLToPath(new URL(`worker${import.meta.url.endsWith(".ts") ? ".ts" : ".js"}`, import.meta.url)), [this.agentDir], {
+			execArgv: [], stdio: ["ignore", "ignore", "ignore", "ipc"], serialization: "advanced",
+			env: { ...process.env, PI_PACKAGE_DIR: requirePiPackageRoot(), PI_CODING_AGENT_DIR: this.agentDir },
+		});
+		this.unavailable = undefined;
+		this.process = child;
+		this.children.set(child, new Promise<void>((resolve) => {
+			const exited = () => {
+				child.removeListener("exit", exited); child.removeListener("close", exited);
+				this.children.delete(child); resolve();
+			};
+			// Disconnected IPC children can exit without Node emitting close. Exit already
+			// releases their database and watch handles; failed spawns report close instead.
+			child.once("exit", exited); child.once("close", exited);
+		}));
+		child.on("message", (message: Response) => {
+			if (child !== this.process) return;
+			if ("changed" in message) {
+				for (const listener of this.listeners) { try { listener(); } catch { /* Consumer failures must not strand IPC requests. */ } }
+				return;
+			}
+			const pending = this.pending.get(message.id);
+			if (!pending) return;
+			this.pending.delete(message.id); pending.cleanup();
+			if (message.error) pending.reject(new HistoryIndexError(message.error.code, message.error.message));
+			else pending.resolve(message.value);
+		});
+		child.on("error", () => this.stop(new HistoryIndexError("UNAVAILABLE", "History process could not start."), child));
+		child.on("exit", () => this.stop(new HistoryIndexError("UNAVAILABLE", "History process exited; retry to reopen the disposable index."), child));
+	}
+	private stop(error: Error, child = this.process, kill = true): void {
+		if (!child || child !== this.process) return;
+		this.unavailable = error;
+		this.process = undefined; this.ready = undefined;
+		for (const pending of this.pending.values()) { pending.cleanup(); pending.reject(error); }
+		this.pending.clear();
+		if (kill && child.exitCode === null && child.signalCode === null) child.kill("SIGKILL");
+	}
+	private send<T>(method: string, input?: any, signal?: AbortSignal, timeout = 3000): Promise<T> {
+		if (this.closed) return Promise.reject(new HistoryIndexError("CLOSED", "History index is closed."));
+		if (signal?.aborted) return Promise.reject(new HistoryIndexError("CANCELLED", "History request cancelled."));
+		const child = this.process;
+		if (!child?.connected) return Promise.reject(new HistoryIndexError("UNAVAILABLE", "History process is unavailable."));
+		return new Promise<T>((resolve, reject) => {
+			const id = ++this.sequence;
+			const cancel = () => this.stop(new HistoryIndexError("CANCELLED", "History request cancelled; the process was stopped and can be reopened."), child);
+			const timer = setTimeout(() => this.stop(new HistoryIndexError("DEADLINE", "History request exceeded its process deadline; retry or narrow the query."), child), timeout);
+			timer.unref();
+			this.pending.set(id, { resolve, reject, cleanup: () => { clearTimeout(timer); signal?.removeEventListener("abort", cancel); } });
+			signal?.addEventListener("abort", cancel, { once: true });
+			child.send({ id, method, input } satisfies Request, (error) => { if (error) this.stop(new HistoryIndexError("UNAVAILABLE", "History IPC failed."), child); });
+		});
+	}
+	private async ensure(): Promise<void> {
+		if (this.closed) throw new HistoryIndexError("CLOSED", "History index is closed.");
+		if (this.unavailable) throw this.unavailable;
+		if (!this.owner) throw new HistoryIndexError("NO_OWNER", "Set genuine restored ownership before querying history.");
+		if (!this.process) {
+			await Promise.all(this.children.values());
+			if (this.closed) throw new HistoryIndexError("CLOSED", "History index is closed.");
+			if (!this.process) { this.spawn(); this.ready = this.send<void>("setOwner", this.owner, undefined, 15_000); }
+		}
+		await this.ready;
+	}
+	/** Arms an ordinary deadline only in the continuation that saw the current admission and worker settle, never behind a replacement admission. */
+	private async admitted<T>(method: string, input?: unknown, signal?: AbortSignal, timeout?: number): Promise<T> {
+		await this.ensure();
+		const ready = this.ready, child = this.process;
+		await ready;
+		// setOwner() or a restart may have replaced readiness during either await; wait for the replacement.
+		if (!child || ready !== this.ready || child !== this.process) return this.admitted<T>(method, input, signal, timeout);
+		return this.send<T>(method, input, signal, timeout);
+	}
+	async setOwner(input: HistoryOwner): Promise<void> {
+		if (this.closed) throw new HistoryIndexError("CLOSED", "History index is closed.");
+		if (!this.process) await Promise.all(this.children.values());
+		if (this.closed) throw new HistoryIndexError("CLOSED", "History index is closed.");
+		// Snapshot inputs so later caller mutation cannot silently change admission or replay.
+		const snapshot = structuredClone(input);
+		if (!snapshot.ownerSessionId || snapshot.runs.some((run) => run.ownerSessionId !== snapshot.ownerSessionId)) throw new HistoryIndexError("OWNERSHIP", "Every run must belong to the current owner session.");
+		if (new Set(snapshot.runs.map((run) => run.runId)).size !== snapshot.runs.length) throw new HistoryIndexError("INVALID", "Duplicate owned run IDs.");
+		this.owner = snapshot;
+		if (!this.process) this.spawn();
+		this.ready = this.send<void>("setOwner", snapshot, undefined, 15_000);
+		await this.ready;
+	}
+	async updateRun(run: OwnedRun, foreground?: ForegroundResumeRun): Promise<void> {
+		await this.ensure();
+		if (run.ownerSessionId !== this.owner!.ownerSessionId || foreground && foreground.runId !== run.runId) throw new HistoryIndexError("OWNERSHIP", "Run does not belong to the current owner.");
+		const copy = structuredClone(run), fg = foreground && structuredClone(foreground);
+		await this.admitted<void>("updateRun", { run: copy, foreground: fg });
+		this.owner!.runs = [...this.owner!.runs.filter((entry) => entry.runId !== run.runId), copy];
+		if (fg) this.owner!.foregroundRuns = [...(this.owner!.foregroundRuns ?? []).filter((entry) => entry.runId !== run.runId), fg];
+	}
+	private async query<T>(method: string, input: { signal?: AbortSignal }): Promise<T> {
+		const { signal, ...wire } = input;
+		return this.admitted<T>(method, wire, signal);
+	}
+	/** Tool discovery waits for run metadata only, not transcript backfill. */
+	needsControls(): Promise<boolean> { return this.admitted("needsControls", undefined, undefined, 15_000); }
+	listRuns(options: HistoryRunOptions = {}): Promise<HistoryRunPage> { return this.query("listRuns", options); }
+	historyPage(input: HistoryPageInput): Promise<HistoryPage> { return this.query("historyPage", input); }
+	search(input: HistorySearchInput): Promise<HistorySearchPage> { return this.query("search", input); }
+	status(): Promise<HistoryIndexStatus> { return this.query("status", {}); }
+	/** Validated bounded preview, including paired tools outside the current page. */
+	entry(input: HistoryEntryInput): Promise<HistoryEntry | null> { return this.query("entry", input); }
+	/** Explicit selected detail only; records over 16 MiB fail with RECORD_TOO_LARGE. */
+	record(input: HistoryEntryInput): Promise<Record<string, any> | null> { return this.query("record", input); }
+	/** Selected canonical output, never a transcript scan or a truncated-preview match. */
+	result(input: Pick<HistoryEntryInput, "runId" | "index" | "signal">): Promise<HistoryResult | null> { return this.query("result", input); }
+	async refresh(runId?: string, options: { signal?: AbortSignal } = {}): Promise<void> {
+		if (!this.closed) this.unavailable = undefined;
+		return this.admitted("refresh", { runId }, options.signal, 120_000);
+	}
+	onChanged(listener: () => void): () => void { this.listeners.add(listener); return () => this.listeners.delete(listener); }
+	/** Hard cancellation affects this instance's outstanding work, not native sessions or other indexers. */
+	cancel(): void { this.stop(new HistoryIndexError("CANCELLED", "History process cancelled; explicitly refresh or retry to reopen it.")); }
+	close(): Promise<void> {
+		if (this.closing) return this.closing;
+		const child = this.process;
+		this.closed = true; this.listeners.clear();
+		this.unavailable ??= new HistoryIndexError("CLOSED", "History index closed.");
+		const timers = [...this.children.keys()].map((owned) => { const timer = setTimeout(() => owned.kill("SIGKILL"), 1000); timer.unref(); return timer; });
+		this.closing = Promise.all(this.children.values()).then(() => { for (const timer of timers) clearTimeout(timer); });
+		if (child) {
+			this.stop(new HistoryIndexError("CLOSED", "History index closed."), child, false);
+			if (child.connected) child.disconnect(); else child.kill("SIGKILL");
+		}
+		return this.closing;
+	}
+}

package/src/history/ingest.ts

@@ -0,0 +1,349 @@
+import * as fs from "node:fs";
+import { createHash, type Hash } from "node:crypto";
+import { JournalFrames, type Projection } from "../shared/journal-reader.ts";
+import { HistoryIndexError } from "./types.ts";
+import { exactTextDigest, HistoryStore, safeText, type SourceRow } from "./store.ts";
+import { stripTerminalSequences } from "../shared/native-tui.ts";
+import { stripAcceptanceReport } from "../runs/shared/acceptance-reports.ts";
+
+export function identity(stat: fs.BigIntStats): string { return `${stat.dev}:${stat.ino}`; }
+export function stamp(stat: fs.BigIntStats): string { return `${identity(stat)}:${stat.size}:${stat.mtimeNs}:${stat.ctimeNs}`; }
+const metadata = new Set(["type", "id", "parentId", "timestamp", "version", "cwd", "provider", "model", "modelId", "thinkingLevel", "customType", "display", "summary", "content"]);
+const messageFields = new Set(["role", "content", "output", "summary", "toolCallId", "toolName", "isError", "timestamp", "model", "provider", "stopReason", "errorMessage", "customType", "display", "command", "exitCode", "cancelled"]);
+export const previewLimits = { depth: 64, nodes: 16_384, arrayLength: 4096, keyLength: 512 };
+export const previewProjection: Projection = (keys, root) => {
+	if (!keys.length) return true;
+	if (keys[0] === "message") {
+		if (keys.length === 1) return true;
+		if (!messageFields.has(String(keys[1]))) return false;
+		if (keys[1] === "content") {
+			if (keys.length <= 3) return 512;
+			if (keys[3] === "arguments") {
+				if (keys.length === 4) return true;
+				if (keys.length === 5) return Object.keys(root?.message?.content?.[Number(keys[2])]?.arguments ?? {}).length < 32 ? 512 : false;
+				return keys.length === 6 && keys[5] === 0 ? 512 : false;
+			}
+			return keys.length === 4 && ["type", "text", "thinking", "id", "name", "mimeType"].includes(String(keys[3])) ? 512 : false;
+		}
+		return keys.length === 2 ? 512 : false;
+	}
+	if (keys[0] === "details") return keys.length === 1 || keys.length === 2 && ["bodyText", "message"].includes(String(keys[1])) || keys.length === 3 && keys[1] === "message" && keys[2] === "id" ? 512 : false;
+	if (keys[0] === "content" && keys.length > 1) return keys.length <= 2 || keys.length === 3 && ["type", "text", "mimeType"].includes(String(keys[2])) ? 512 : false;
+	return keys.length === 1 && metadata.has(String(keys[0])) ? 512 : false;
+};
+function candidate(keys: readonly (string | number)[]): boolean {
+	return ["summary", "content"].includes(String(keys[0])) && keys.length === 1
+		|| keys.length === 2 && keys[0] === "details" && keys[1] === "bodyText"
+		|| keys[0] === "message" && keys.length === 2 && ["content", "output", "summary"].includes(String(keys[1]))
+		|| keys[0] === "message" && keys[1] === "content" && keys.length === 4 && keys[3] === "text"
+		|| keys[0] === "content" && keys.length === 3 && keys[2] === "text";
+}
+function allowed(value: any, field: string): boolean {
+	const keys = JSON.parse(field) as (string | number)[];
+	if (value.type === "custom_message") {
+		if (value.display === false) return false;
+		if (keys[0] === "details") return value.customType === "subagent-human-message";
+		if (value.customType === "subagent-human-message" && typeof value.details?.bodyText === "string") return false;
+		return keys[0] === "content" && (keys.length === 1 || value.content?.[Number(keys[1])]?.type === "text");
+	}
+	if (["compaction", "branch_summary"].includes(value.type)) return keys.length === 1 && keys[0] === "summary";
+	if (value.type !== "message" || keys[0] !== "message") return false;
+	const message = value.message;
+	if (!message) return false;
+	if (["compactionSummary", "branchSummary"].includes(message.role)) return keys[1] === "summary";
+	if (message.role === "bashExecution") return keys[1] === "output";
+	if (!["user", "assistant", "toolResult", "custom"].includes(message.role) || message.role === "custom" && message.display === false) return false;
+	return keys[1] === "content" && (keys.length === 2 || message.content?.[Number(keys[2])]?.type === "text");
+}
+export function compactEntry(value: any): Record<string, any> {
+	let remaining = 4096;
+	const clean = (input: any, key?: string): any => {
+		if (typeof input === "string") { const text = safeText(input, ["id", "parentId", "type", "role", "toolCallId", "name", "toolName", "timestamp"].includes(key ?? "") ? 512 : Math.min(512, remaining)); remaining -= Math.min(remaining, text.length); return text; }
+		if (Array.isArray(input)) return input.slice(0, 32).map((item) => clean(item));
+		if (input && typeof input === "object") return Object.fromEntries(Object.entries(input).map(([name, item]) => [name, clean(item, name)]));
+		return input;
+	};
+	return clean(value);
+}
+/** Exact matches never use previews. Oversized selected text has no fingerprint. */
+class AnswerTexts {
+	private fields = new Map<number, string>();
+	private length = 0;
+	private unavailable = false;
+	write(keys: readonly (string | number)[], text: string): void {
+		if (this.unavailable || keys.length !== 4 || keys[0] !== "message" || keys[1] !== "content" || !["text", "thinking"].includes(String(keys[3]))) return;
+		this.length += text.length;
+		if (this.length > 16 * 1024 * 1024 || Number(keys[2]) >= 4096) { this.unavailable = true; this.fields.clear(); return; }
+		const index = Number(keys[2]);
+		this.fields.set(index, (this.fields.get(index) ?? "") + text);
+	}
+	fingerprints(value: any, id: string): Array<[string, string]> {
+		if (this.unavailable || value.type !== "message" || value.message?.role !== "assistant") return [];
+		const content = Array.isArray(value.message.content) ? value.message.content : [];
+		const first = content.findIndex((part: any, index: number) => ["text", "thinking"].includes(part.type) && Boolean(this.fields.get(index)));
+		if (first < 0 || !content.some((part: any, index: number) => part.type === "text" && Boolean(this.fields.get(index)))) return [];
+		const item = `${id}:${first}`, parts = content.map((part: any, index: number) => ["text", "thinking"].includes(part.type) ? stripTerminalSequences(this.fields.get(index) ?? "") : part.type === "image" ? `[Image: ${part.mimeType ?? "image"}]` : "").filter(Boolean);
+		const texts = [parts.join("\n"), ...content.flatMap((part: any, index: number) => part.type === "text" ? [stripTerminalSequences(this.fields.get(index) ?? "")] : [])];
+		return [...new Set(texts.map((text) => stripAcceptanceReport(text).trim()).filter(Boolean))].map((text) => [exactTextDigest(text), item]);
+	}
+}
+function visibleId(value: any, id: string): string | null {
+	if (["custom_message", "compaction", "branch_summary"].includes(value.type)) return id;
+	if (value.type !== "message") return null;
+	if (["user", "toolResult", "bashExecution"].includes(value.message?.role)) return id;
+	if (value.message?.role !== "assistant") return null;
+	const content = Array.isArray(value.message.content) ? value.message.content : [];
+	const last = content.findLastIndex((part: any) => part.type === "toolCall" || part.type === "text" && part.text || part.type === "thinking" && part.thinking);
+	return value.message.errorMessage ? `${id}:error` : last < 0 ? null : `${id}:${last}`;
+}
+interface TextToken { text: string; start: number; end: number }
+interface TextBuffer { word: string; wordStart: number; received: number; tokens: TextToken[]; length: number; terminal: TerminalText }
+/** Streaming ECMA-48 removal, retaining original UTF-16 positions without buffering control payloads. */
+class TerminalText {
+	private state: "text" | "escape" | "intermediate" | "csi" | "string" | "stringEscape" = "text";
+	write(text: string, visible: (text: string, start: number) => void): void {
+		let position = 0;
+		while (position < text.length) {
+			if (this.state === "text") {
+				const next = text.slice(position).search(/[\x1b\x90\x98\x9b\x9d-\x9f]/);
+				if (next < 0) { visible(text.slice(position), position); return; }
+				visible(text.slice(position, position + next), position); position += next;
+				const control = text[position++];
+				this.state = control === "\x1b" ? "escape" : control === "\x9b" ? "csi" : "string";
+				continue;
+			}
+			const char = text[position++];
+			if (this.state === "escape") this.state = char === "[" ? "csi" : "]PX^_".includes(char) ? "string" : char >= " " && char <= "/" ? "intermediate" : "text";
+			else if (this.state === "intermediate") { if (char >= "0" && char <= "~") this.state = "text"; }
+			else if (this.state === "csi") { if (char >= "@" && char <= "~") this.state = "text"; }
+			else if (char === "\x07" || char === "\x9c" || this.state === "stringEscape" && char === "\\") this.state = "text";
+			else this.state = char === "\x1b" ? "stringEscape" : "string";
+		}
+	}
+}
+/** Token windows overlap 12 tokens, including phrases across arbitrarily long whitespace. */
+class TextChunks {
+	private fields = new Map<string, TextBuffer>();
+	private staged: Array<{ field: string; start: number; end: number; text: string }> = [];
+	private emit(field: string, buffer: TextBuffer, final = false): void {
+		if (!buffer.tokens.length) return;
+		this.staged.push({ field, start: buffer.tokens[0].start, end: buffer.tokens.at(-1)!.end, text: buffer.tokens.map((token) => token.text).join(" ") });
+		buffer.tokens = final ? [] : buffer.tokens.slice(-12);
+		buffer.length = buffer.tokens.reduce((length, token) => length + token.text.length + 1, 0);
+	}
+	private token(field: string, buffer: TextBuffer, end: number): void {
+		if (!buffer.word) return;
+		buffer.tokens.push({ text: buffer.word, start: buffer.wordStart, end });
+		buffer.length += buffer.word.length + 1; buffer.word = "";
+		if (buffer.tokens.length >= 256 || buffer.length >= 8192 && buffer.tokens.length > 12) this.emit(field, buffer);
+	}
+	write(keys: readonly (string | number)[], text: string): void {
+		if (!candidate(keys)) return;
+		const field = JSON.stringify(keys);
+		let buffer = this.fields.get(field);
+		if (!buffer) {
+			if (this.fields.size >= 4096) throw new HistoryIndexError("RECORD_COMPLEXITY", "Record exceeds the 4096 visible text-field indexing budget.");
+			this.fields.set(field, buffer = { word: "", wordStart: 0, received: 0, tokens: [], length: 0, terminal: new TerminalText() });
+		}
+		buffer.terminal.write(text, (visible, offset) => {
+			for (const part of visible.matchAll(/[\p{L}\p{N}\p{M}]+|[^\p{L}\p{N}\p{M}]+/gu)) {
+				const start = buffer.received + offset + part.index!;
+				if (/^[\p{L}\p{N}\p{M}]/u.test(part[0])) {
+					if (!buffer.word) buffer.wordStart = start;
+					buffer.word += part[0];
+					if (buffer.word.length > 65_536) throw new HistoryIndexError("RECORD_COMPLEXITY", "Record contains a lexical token exceeding the 64 KiB indexing budget.");
+				} else this.token(field, buffer, start);
+			}
+		});
+		buffer.received += text.length;
+	}
+	flush(store: HistoryStore, finish = false): void {
+		if (finish) {
+			for (const [field, buffer] of this.fields) { this.token(field, buffer, buffer.received); this.emit(field, buffer, true); }
+			this.fields.clear();
+		}
+		if (this.staged.length) {
+			const insert = store.db.prepare("INSERT INTO pending_text VALUES (?,?,?,?)");
+			for (const item of this.staged) insert.run(item.field, item.start, item.end, item.text);
+			this.staged = [];
+		}
+	}
+}
+
+/** One physical source, incremental byte framing and short row/cursor publication transactions. */
+export class SourceIngest {
+	private fd: number;
+	private initial: fs.BigIntStats;
+	private source: SourceRow;
+	private recordDigest: Hash = createHash("sha256");
+	/** prefix_digest is "start:sha256" of the last indexed line; earlier releases hashed the whole prefix. */
+	private legacy = false;
+	private expected: string | null = null;
+	private verified: Hash = createHash("sha256");
+	private line: Hash = createHash("sha256");
+	private lineStart = 0;
+	private verifyAt = 0;
+	private position: number;
+	private frames: JournalFrames;
+	private text = new TextChunks();
+	private answerTexts = new AnswerTexts();
+	private buffer = Buffer.alloc(0);
+	private record?: { value: any; start: number; end: number; digest: string };
+	private stagedEntry?: number;
+	private textCursor = 0;
+	private malformed = false;
+	private complexity = false;
+	private phase: "verify" | "read" | "text" | "done";
+	private closed = false;
+	private headerSeen: boolean;
+	private store: HistoryStore;
+	readonly id: string;
+	constructor(store: HistoryStore, source: SourceRow, force = false) {
+		this.store = store; this.source = source; this.id = source.id;
+		store.beginIngest();
+		let opened: number | undefined;
+		try {
+			this.source = source = store.source(source.id)!;
+			if (!source) throw new HistoryIndexError("SOURCE_CHANGED", "Source was removed before ingestion.");
+			store.operations.sourceChecks++; store.operations.sourceOpens++;
+			this.fd = opened = fs.openSync(source.path, "r"); this.initial = fs.fstatSync(this.fd, { bigint: true });
+			if (!this.initial.isFile() || this.initial.size > BigInt(Number.MAX_SAFE_INTEGER)) throw new Error("Journal is not a supported regular file.");
+			if (source.identity && (source.identity !== identity(this.initial) || Number(this.initial.size) < source.cursor)) { store.resetSource(source.id); this.source = source = store.source(source.id)!; }
+			this.position = source.cursor;
+			this.headerSeen = source.cursor > 0;
+			store.clearUnpublished(source.id);
+			store.run("DELETE FROM pending_text");
+			// Appends re-hash only the last indexed line, so live sources cost their new bytes, not their size.
+			const anchor = /^(\d+):([0-9a-f]{64})$/.exec(source.prefix_digest ?? "");
+			this.legacy = !anchor; this.expected = anchor ? anchor[2] : source.prefix_digest;
+			this.verifyAt = this.lineStart = anchor ? Number(anchor[1]) : 0;
+			this.phase = !force && source.stamp === stamp(this.initial) ? "done" : source.cursor ? "verify" : "read";
+			if (this.phase !== "done") store.run("UPDATE sources SET state='indexing',error=NULL WHERE id=?", source.id);
+			this.frames = new JournalFrames(previewProjection, (record) => {
+				this.record = { ...record, digest: this.recordDigest.copy().digest("hex") };
+			}, "inspect", source.cursor, (_start, _end, error) => { this.malformed = true; this.complexity = error instanceof RangeError || error instanceof HistoryIndexError && error.code === "RECORD_COMPLEXITY"; }, (keys, chunk) => { this.text.write(keys, chunk); this.answerTexts.write(keys, chunk); }, undefined, true, previewLimits);
+		} catch (error) {
+			if (opened !== undefined) fs.closeSync(opened);
+			store.endIngest();
+			throw error;
+		}
+	}
+	private published(end: number, work?: () => void): void {
+		this.store.transaction(() => {
+			const current = this.store.source(this.id)!;
+			if (!current || current.generation !== this.source.generation || current.cursor !== this.source.cursor) throw new HistoryIndexError("SOURCE_CHANGED", "Another writer advanced this source; reconcile before replaying.");
+			work?.();
+			this.store.run("UPDATE sources SET cursor=?,prefix_digest=?,identity=? WHERE id=?", end, `${this.source.cursor}:${this.recordDigest.copy().digest("hex")}`, identity(this.initial), this.id);
+			this.store.bump();
+		});
+		this.source.cursor = end;
+		this.recordDigest = createHash("sha256"); this.text = new TextChunks(); this.answerTexts = new AnswerTexts();
+		this.store.run("DELETE FROM pending_text"); this.record = undefined; this.malformed = this.complexity = false;
+	}
+	private finishRecord(end: number): void {
+		this.text.flush(this.store, true);
+		if (this.malformed) {
+			if (!this.headerSeen) throw new Error("Malformed native session header.");
+			this.published(end, () => this.store.run("UPDATE sources SET malformed=malformed+1,complexity=complexity+? WHERE id=?", this.complexity ? 1 : 0, this.id)); return;
+		}
+		const record = this.record;
+		if (!record) { this.published(end); return; }
+		const value = record.value;
+		if (!this.headerSeen) {
+			if (value.type !== "session") throw new Error("Linked source does not have a native session header.");
+			this.published(end, () => this.store.run("UPDATE sources SET session_id=?,header_digest=?,format_version=? WHERE id=?", typeof value.id === "string" ? value.id : null, record.digest, typeof value.version === "number" ? value.version : null, this.id));
+			this.source.format_version = typeof value.version === "number" ? value.version : null;
+			this.headerSeen = true; return;
+		}
+		if (value.type === "session") throw new Error("Unexpected session header inside a native source.");
+		const nativeId = typeof value.id === "string" ? value.id : null;
+		const id = nativeId ?? `legacy-${record.start}`;
+		if (this.store.get("SELECT rowid FROM entries WHERE source_id=? AND generation=? AND id=?", this.id, this.source.generation, id)) {
+			this.published(end, () => this.store.run("UPDATE sources SET malformed=malformed+1 WHERE id=?", this.id)); return;
+		}
+		const timestamp = typeof value.timestamp === "number" ? value.timestamp : Date.parse(value.timestamp);
+		const parent = this.source.format_version === 1 ? this.store.get("SELECT id FROM entries WHERE source_id=? AND generation=? AND published=1 ORDER BY start DESC LIMIT 1", this.id, this.source.generation)?.id ?? null : typeof value.parentId === "string" ? value.parentId : null;
+		const preview = compactEntry(value); preview.id = id; preview.parentId = parent;
+		const model = value.type === "model_change" && typeof value.modelId === "string" ? `${value.provider}/${value.modelId}` : value.type === "message" && value.message?.role === "assistant" && typeof value.message.model === "string" ? `${value.message.provider}/${value.message.model}` : null;
+		this.stagedEntry = Number(this.store.run("INSERT INTO entries(source_id,generation,id,native_id,parent_id,start,end,digest,timestamp,type,preview,role,assistant_text,visible_id,human_id,configuration_model,configuration_thinking) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", this.id, this.source.generation, id, nativeId, parent, record.start, end, record.digest, Number.isFinite(timestamp) ? timestamp : null, typeof value.type === "string" ? value.type : "unknown", JSON.stringify(preview), typeof value.message?.role === "string" ? value.message.role : null,
+			value.message?.role === "assistant" && Array.isArray(value.message.content) && value.message.content.some((part: any) => part.type === "text" && part.text) ? 1 : 0,
+			visibleId(value, id), value.type === "custom_message" && value.customType === "subagent-human-message" && typeof value.details?.message?.id === "string" ? value.details.message.id : null, model, value.type === "thinking_level_change" && typeof value.thinkingLevel === "string" ? value.thinkingLevel : null).lastInsertRowid);
+		this.textCursor = 0; this.phase = "text";
+	}
+	private indexText(): void {
+		const chunks = this.store.all("SELECT rowid,* FROM pending_text WHERE rowid>? ORDER BY rowid LIMIT 32", this.textCursor);
+		if (chunks.length) {
+			this.store.transaction(() => {
+				for (const chunk of chunks) {
+					if (!allowed(this.record!.value, chunk.field)) continue;
+					const text = safeText(chunk.text, 1_048_576);
+					const id = this.store.run("INSERT INTO documents(entry_rowid,field,text_start,text_end,preview) VALUES (?,?,?,?,?)", this.stagedEntry!, chunk.field, chunk.position, chunk.end_position, safeText(text, 512)).lastInsertRowid;
+					this.store.run("INSERT INTO corpus(rowid,text) VALUES (?,?)", id, text);
+				}
+			});
+			this.textCursor = chunks.at(-1).rowid; return;
+		}
+		const value = this.record!.value, rowid = this.stagedEntry!;
+		const answers = this.answerTexts.fingerprints(value, typeof value.id === "string" ? value.id : `legacy-${this.record!.start}`);
+		this.published(this.record!.end, () => {
+			if (this.store.run("UPDATE entries SET published=1 WHERE rowid=? AND source_id=? AND generation=? AND published=0", rowid, this.id, this.source.generation).changes !== 1) throw new HistoryIndexError("SOURCE_CHANGED", "Staged entry disappeared before publication; reconcile before advancing the source.");
+			for (const [digest, item] of answers) this.store.run("INSERT OR IGNORE INTO answers VALUES (?,?,?)", rowid, digest, item);
+			for (const part of Array.isArray(value.message?.content) ? value.message.content : []) if (part.type === "toolCall" && typeof part.id === "string") this.store.run("INSERT INTO tools VALUES (?,?,?,?,?)", rowid, this.id, this.source.generation, part.id, "call");
+			if (value.message?.role === "toolResult" && typeof value.message.toolCallId === "string") this.store.run("INSERT INTO tools VALUES (?,?,?,?,?)", rowid, this.id, this.source.generation, value.message.toolCallId, "result");
+		});
+		this.stagedEntry = undefined; this.phase = "read";
+	}
+	/** Each pump reads <=64 KiB or indexes <=32 chunks, then returns to the IPC event loop. */
+	step(): boolean {
+		if (this.phase === "done") { this.close(); return true; }
+		if (this.phase === "text") { this.indexText(); return false; }
+		if (this.phase === "verify") {
+			const bytes = Buffer.allocUnsafe(Math.min(64 * 1024, this.source.cursor - this.verifyAt));
+			const count = fs.readSync(this.fd, bytes, 0, bytes.length, this.verifyAt);
+			if (count !== bytes.length) throw new Error("Journal truncated during prefix verification.");
+			this.store.operations.sourceBytesRead += count;
+			this.verified.update(bytes);
+			if (this.legacy) {
+				// A whole-prefix check also finds the last indexed line, which anchors every later check.
+				const before = this.source.cursor - 2 - this.verifyAt, lf = before < 0 ? -1 : bytes.lastIndexOf(10, before);
+				if (lf < 0) this.line.update(bytes);
+				else { this.lineStart = this.verifyAt + lf + 1; this.line = createHash("sha256").update(bytes.subarray(lf + 1)); }
+			}
+			this.verifyAt += count;
+			if (this.verifyAt === this.source.cursor) {
+				if (this.verified.digest("hex") !== this.expected) {
+					this.store.resetSource(this.id); throw new HistoryIndexError("SOURCE_CHANGED", "Journal prefix changed; a new physical generation was scheduled.");
+				}
+				if (this.legacy) this.store.run("UPDATE sources SET prefix_digest=? WHERE id=?", `${this.lineStart}:${this.line.digest("hex")}`, this.id);
+				this.phase = "read";
+			}
+			return false;
+		}
+		if (!this.buffer.length) {
+			const remaining = Number(this.initial.size) - this.position;
+			if (remaining <= 0) return this.finish();
+			const bytes = Buffer.allocUnsafe(Math.min(64 * 1024, remaining));
+			const count = fs.readSync(this.fd, bytes, 0, bytes.length, this.position);
+			if (count !== bytes.length) throw new Error("Journal truncated during ingestion.");
+			this.store.operations.sourceBytesRead += count;
+			this.position += count; this.buffer = bytes;
+		}
+		const newline = this.buffer.indexOf(10);
+		const bytes = newline < 0 ? this.buffer : this.buffer.subarray(0, newline + 1);
+		this.recordDigest.update(bytes); this.frames.write(bytes); this.text.flush(this.store);
+		this.buffer = this.buffer.subarray(bytes.length);
+		if (newline >= 0) this.finishRecord(this.frames.offset);
+		return false;
+	}
+	private finish(): boolean {
+		this.frames.finish();
+		const now = fs.statSync(this.source.path, { bigint: true });
+		if (identity(now) !== identity(this.initial) || now.size < this.initial.size || now.size === this.initial.size && stamp(now) !== stamp(this.initial)) {
+			this.store.resetSource(this.id); throw new HistoryIndexError("SOURCE_CHANGED", "Journal changed during ingestion; retrying its new generation.");
+		}
+		const { malformed, complexity } = this.store.source(this.id)!;
+		this.store.run("UPDATE sources SET identity=?,stamp=?,checked_at=?,state=?,error=? WHERE id=?", identity(this.initial), stamp(this.initial), Date.now(), malformed ? "degraded" : this.source.cursor < Number(this.initial.size) ? "partial" : "current", malformed ? `${malformed} malformed, duplicate or over-budget LF-published records were skipped.${complexity ? ` ${complexity} exceeded bounded history structure/text budgets.` : ""}` : null, this.id);
+		this.phase = "done"; this.close(); return true;
+	}
+	close(): void { if (!this.closed) { fs.closeSync(this.fd); this.store.endIngest(); this.closed = true; } }
+}