@shanepadgett/tau-agent 0.16.0 → 0.17.0

package/extensions/explore/read-cache.ts

@@ -1,12 +1,16 @@
-import { resolve } from "node:path";
 import type { ExtensionContext } from "@earendil-works/pi-coding-agent";
+import { resolve } from "node:path";
+import { replayContextPruningState } from "../../shared/context-pruning-state.ts";
+import { applyCompleteFileDiff, COMPLETE_FILE_SCOPE, decodeCompleteFileSource } from "./full-file-knowledge.ts";
 
 export type ReadCacheMode = "baseline" | "recovery" | "unchanged" | "diff";
+export type ReadCachePresentation = "plain" | "line-numbered";
 
 export interface ReadCacheMetaV1 {
 	v: 1;
 	pathKey: string;
 	scopeKey: string;
+	presentation: ReadCachePresentation;
 	servedHash: string;
 	baseHash?: string;
 	mode: ReadCacheMode;
@@ -16,70 +20,68 @@ export interface ReadCacheMetaV1 {
 	summary: string;
 }
 
-interface ScopeTrust {
+export interface ReadCacheScopeTrust {
 	hash: string;
-	sequence: number;
+	text?: string;
+	rowIds: readonly string[];
+}
+
+export interface AcceptedReadCacheRow {
+	rowId: string;
+	pathKey: string;
+	scopeKey: string;
+	meta: ReadCacheMetaV1;
+	dependencyRowIds: readonly string[];
 }
 
-interface ReplayState {
-	trust: Map<string, Map<string, ScopeTrust>>;
-	unlockedPaths: Set<string>;
-	sequence: number;
+export interface CompleteFileDependencyChain {
+	pathKey: string;
+	servedHash: string;
+	rowIds: readonly string[];
+	sourceText: string;
 }
 
-interface OverlayState {
-	trust: Map<string, Map<string, ScopeTrust>>;
-	sequence: number;
+export interface ReadCacheReplayResult {
+	scopeTrust: ReadonlyMap<string, ReadonlyMap<string, ReadCacheScopeTrust>>;
+	acceptedRows: readonly AcceptedReadCacheRow[];
+	completeFileChains: ReadonlyMap<string, CompleteFileDependencyChain>;
+	failedPatchRecoveryPaths: ReadonlySet<string>;
 }
 
 export interface ReadCacheDecision {
 	baseHash?: string;
+	baselineText?: string;
 	recovery: boolean;
 }
 
 export interface ReadCacheStore {
 	decision(ctx: ExtensionContext, pathKey: string, scopeKey: string): ReadCacheDecision;
-	record(ctx: ExtensionContext, meta: ReadCacheMetaV1): void;
-	clear(): void;
 }
 
 export function createReadCacheStore(): ReadCacheStore {
-	const overlays = new Map<string, OverlayState>();
-
 	return {
 		decision(ctx, pathKey, scopeKey) {
-			const state = replay(ctx);
-			const overlay = overlays.get(overlayKey(ctx));
-			if (overlay) mergeTrust(state, overlay.trust);
+			const manager = ctx.sessionManager;
+			if (!manager) {
+				return { baseHash: undefined, baselineText: undefined, recovery: false };
+			}
+			const pruning = replayContextPruningState(manager.getBranch());
+			const ignoredRowIds = new Set([...pruning.prunedToolCallIds, ...pruning.prunedAutoreadRowIds]);
+			const state = replayReadCache(manager.buildContextEntries(), ctx.cwd, ignoredRowIds);
+			const trust = state.scopeTrust.get(pathKey)?.get(scopeKey);
 			return {
-				baseHash: state.trust.get(pathKey)?.get(scopeKey)?.hash,
-				recovery: state.unlockedPaths.has(pathKey),
+				baseHash: trust?.hash,
+				baselineText: trust?.text,
+				recovery: state.failedPatchRecoveryPaths.has(pathKey),
 			};
 		},
-		record(ctx, meta) {
-			const key = overlayKey(ctx);
-			let overlay = overlays.get(key);
-			if (!overlay) {
-				overlay = { trust: new Map(), sequence: 1_000_000_000 };
-				overlays.set(key, overlay);
-			}
-			overlay.sequence += 1;
-			setTrust(overlay.trust, meta.pathKey, meta.scopeKey, meta.servedHash, overlay.sequence);
-			while (overlays.size > 32) {
-				const oldest = overlays.keys().next().value as string | undefined;
-				if (oldest === undefined) break;
-				overlays.delete(oldest);
-			}
-		},
-		clear() {
-			overlays.clear();
-		},
 	};
 }
 
 function parseReadCacheMeta(value: unknown): ReadCacheMetaV1 | undefined {
 	if (!isRecord(value) || value.v !== 1) return undefined;
 	if (typeof value.pathKey !== "string" || typeof value.scopeKey !== "string") return undefined;
+	if (value.presentation !== "plain" && value.presentation !== "line-numbered") return undefined;
 	if (typeof value.servedHash !== "string" || value.servedHash.length === 0) return undefined;
 	if (!isReadCacheMode(value.mode)) return undefined;
 	if (!isNonNegativeFinite(value.baselineTokens) || !isNonNegativeFinite(value.returnedTokens)) return undefined;
@@ -90,6 +92,7 @@ function parseReadCacheMeta(value: unknown): ReadCacheMetaV1 | undefined {
 		v: 1,
 		pathKey: value.pathKey,
 		scopeKey: value.scopeKey,
+		presentation: value.presentation,
 		servedHash: value.servedHash,
 		baseHash: value.baseHash,
 		mode: value.mode,
@@ -110,41 +113,120 @@ export function readMetaFromMessage(message: unknown): ReadCacheMetaV1 | undefin
 	return parseReadCacheMeta(message.details.readCache);
 }
 
-function replay(ctx: ExtensionContext): ReplayState {
-	const state: ReplayState = { trust: new Map(), unlockedPaths: new Set(), sequence: 0 };
-	const manager = ctx.sessionManager;
-	if (!manager) return state;
-	const branch = manager.getBranch();
-	let start = 0;
-	for (let index = 0; index < branch.length; index += 1) {
-		const entry = branch[index];
-		if (isRecord(entry) && entry.type === "compaction") start = index + 1;
-	}
-
-	for (const entry of branch.slice(start)) {
-		if (!isRecord(entry) || entry.type !== "message" || !("message" in entry)) continue;
-		const message = entry.message;
-		const meta = readMetaFromMessage(message);
-		if (meta) {
-			state.sequence += 1;
-			applyReadMeta(state, meta);
+export function replayReadCache(
+	entries: readonly unknown[],
+	cwd: string,
+	ignoredRowIds: ReadonlySet<string> = new Set(),
+): ReadCacheReplayResult {
+	const trust = new Map<string, Map<string, ReadCacheScopeTrust>>();
+	const acceptedRows: AcceptedReadCacheRow[] = [];
+	const failedPatchRecoveryPaths = new Set<string>();
+	for (const entry of entries) {
+		if (!isRecord(entry)) continue;
+		let message: unknown;
+		if (entry.type === "message" && "message" in entry) message = entry.message;
+		else if (entry.type === "custom_message") message = entry;
+		else continue;
+		const parsedMeta = readMetaFromMessage(message);
+		if (parsedMeta) {
+			const rowId = readRowId(message);
+			if (rowId && ignoredRowIds.has(rowId)) continue;
+			const pathKey = resolve(cwd, parsedMeta.pathKey);
+			const meta = pathKey === parsedMeta.pathKey ? parsedMeta : { ...parsedMeta, pathKey };
+			if (rowId && applyReadMeta(trust, failedPatchRecoveryPaths, message, meta, rowId)) {
+				const acceptedTrust = trust.get(meta.pathKey)?.get(meta.scopeKey);
+				if (acceptedTrust) {
+					acceptedRows.push({
+						rowId,
+						pathKey: meta.pathKey,
+						scopeKey: meta.scopeKey,
+						meta,
+						dependencyRowIds: [...acceptedTrust.rowIds],
+					});
+				}
+			}
 			continue;
 		}
-		for (const path of failedPatchPaths(message, ctx.cwd)) state.unlockedPaths.add(path);
+		for (const path of failedPatchPaths(message, cwd)) failedPatchRecoveryPaths.add(path);
+	}
+	const completeFileChains = new Map<string, CompleteFileDependencyChain>();
+	for (const [pathKey, scopes] of trust) {
+		const complete = scopes.get(COMPLETE_FILE_SCOPE);
+		if (!complete || complete.text === undefined) continue;
+		completeFileChains.set(pathKey, {
+			pathKey,
+			servedHash: complete.hash,
+			rowIds: [...complete.rowIds],
+			sourceText: complete.text,
+		});
 	}
-	return state;
+	return { scopeTrust: trust, acceptedRows, completeFileChains, failedPatchRecoveryPaths };
 }
 
-function applyReadMeta(state: ReplayState, meta: ReadCacheMetaV1): void {
-	const existing = state.trust.get(meta.pathKey)?.get(meta.scopeKey);
+function applyReadMeta(
+	trust: Map<string, Map<string, ReadCacheScopeTrust>>,
+	failedPatchRecoveryPaths: Set<string>,
+	message: unknown,
+	meta: ReadCacheMetaV1,
+	rowId: string,
+): boolean {
+	const existing = trust.get(meta.pathKey)?.get(meta.scopeKey);
 	if (meta.mode === "baseline" || meta.mode === "recovery") {
-		setTrust(state.trust, meta.pathKey, meta.scopeKey, meta.servedHash, state.sequence);
-		state.unlockedPaths.delete(meta.pathKey);
-		return;
+		let text: string | undefined;
+		if (meta.scopeKey === COMPLETE_FILE_SCOPE) {
+			if (!isRecord(message)) return false;
+			let pathHeader: string | undefined;
+			if (message.customType === "tau.autoread") {
+				if (!isRecord(message.details)) return false;
+				if (typeof message.details.path !== "string" || typeof message.details.cwd !== "string") return false;
+				if (resolve(message.details.cwd, message.details.path) !== meta.pathKey) return false;
+				pathHeader = message.details.path;
+			}
+			const decoded = decodeCompleteFileSource({
+				content: message.content,
+				autoread: message.customType === "tau.autoread",
+				...(pathHeader === undefined ? {} : { pathHeader }),
+				presentation: meta.presentation,
+				servedHash: meta.servedHash,
+			});
+			if (!decoded.valid || decoded.text === undefined) return false;
+			text = decoded.text;
+		}
+		setTrust(trust, meta.pathKey, meta.scopeKey, { hash: meta.servedHash, text, rowIds: [rowId] });
+		failedPatchRecoveryPaths.delete(meta.pathKey);
+		return true;
+	}
+	if (!meta.baseHash || existing?.hash !== meta.baseHash) return false;
+	if (meta.mode === "unchanged" && meta.servedHash !== meta.baseHash) return false;
+	let text = meta.mode === "unchanged" ? existing.text : undefined;
+	if (meta.scopeKey === COMPLETE_FILE_SCOPE && meta.mode === "diff") {
+		if (existing.text === undefined) return false;
+		const decoded = applyCompleteFileDiff({
+			content: isRecord(message) ? message.content : undefined,
+			baseText: existing.text,
+			baseHash: meta.baseHash,
+			servedHash: meta.servedHash,
+		});
+		if (!decoded.valid || decoded.text === undefined) return false;
+		text = decoded.text;
+	}
+	setTrust(trust, meta.pathKey, meta.scopeKey, {
+		hash: meta.servedHash,
+		text,
+		rowIds: [...existing.rowIds, rowId],
+	});
+	return true;
+}
+
+function readRowId(message: unknown): string | undefined {
+	if (!isRecord(message)) return undefined;
+	if (message.role === "toolResult" && message.toolName === "read") {
+		return typeof message.toolCallId === "string" && message.toolCallId.length > 0 ? message.toolCallId : undefined;
 	}
-	if (!meta.baseHash || existing?.hash !== meta.baseHash) return;
-	if (meta.mode === "unchanged" && meta.servedHash !== meta.baseHash) return;
-	setTrust(state.trust, meta.pathKey, meta.scopeKey, meta.servedHash, state.sequence);
+	if (message.customType !== "tau.autoread" || !isRecord(message.details)) return undefined;
+	return typeof message.details.rowId === "string" && message.details.rowId.length > 0
+		? message.details.rowId
+		: undefined;
 }
 
 function failedPatchPaths(message: unknown, cwd: string): string[] {
@@ -165,35 +247,17 @@ function failedPatchPaths(message: unknown, cwd: string): string[] {
 }
 
 function setTrust(
-	trust: Map<string, Map<string, ScopeTrust>>,
+	trust: Map<string, Map<string, ReadCacheScopeTrust>>,
 	pathKey: string,
 	scopeKey: string,
-	hash: string,
-	sequence: number,
+	value: ReadCacheScopeTrust,
 ): void {
 	let scopes = trust.get(pathKey);
 	if (!scopes) {
 		scopes = new Map();
 		trust.set(pathKey, scopes);
 	}
-	scopes.set(scopeKey, { hash, sequence });
-}
-
-function mergeTrust(state: ReplayState, overlay: Map<string, Map<string, ScopeTrust>>): void {
-	for (const [pathKey, scopes] of overlay) {
-		for (const [scopeKey, candidate] of scopes) {
-			const current = state.trust.get(pathKey)?.get(scopeKey);
-			if (!current || candidate.sequence > current.sequence) {
-				setTrust(state.trust, pathKey, scopeKey, candidate.hash, candidate.sequence);
-			}
-		}
-	}
-}
-
-function overlayKey(ctx: ExtensionContext): string {
-	const manager = ctx.sessionManager;
-	if (!manager) return `test:${ctx.cwd}`;
-	return `${manager.getSessionId()}:${manager.getLeafId() ?? "root"}`;
+	scopes.set(scopeKey, value);
 }
 
 function isReadCacheMode(value: unknown): value is ReadCacheMode {

package/extensions/explore/read-snapshots.ts

@@ -1,4 +1,5 @@
-const MAX_SNAPSHOT_BYTES = 1024 * 1024;
+import { MAX_COMPLETE_FILE_SNAPSHOT_BYTES } from "./full-file-knowledge.ts";
+
 const MAX_TOTAL_BYTES = 16 * 1024 * 1024;
 
 interface Snapshot {
@@ -8,13 +9,16 @@ interface Snapshot {
 
 export interface ReadSnapshotStore {
 	get(hash: string): string | undefined;
-	set(hash: string, text: string, bytes: number): void;
+	epoch(): number;
+	isCurrent(epoch: number): boolean;
+	set(hash: string, text: string, bytes: number, epoch: number): boolean;
 	clear(): void;
 }
 
 export function createReadSnapshotStore(): ReadSnapshotStore {
 	const snapshots = new Map<string, Snapshot>();
 	let totalBytes = 0;
+	let epoch = 0;
 
 	return {
 		get(hash) {
@@ -24,8 +28,15 @@ export function createReadSnapshotStore(): ReadSnapshotStore {
 			snapshots.set(hash, snapshot);
 			return snapshot.text;
 		},
-		set(hash, text, bytes) {
-			if (bytes > MAX_SNAPSHOT_BYTES || snapshots.has(hash)) return;
+		epoch() {
+			return epoch;
+		},
+		isCurrent(candidate) {
+			return candidate === epoch;
+		},
+		set(hash, text, bytes, candidateEpoch) {
+			if (candidateEpoch !== epoch || bytes > MAX_COMPLETE_FILE_SNAPSHOT_BYTES) return false;
+			if (snapshots.has(hash)) return true;
 			snapshots.set(hash, { text, bytes });
 			totalBytes += bytes;
 			while (totalBytes > MAX_TOTAL_BYTES) {
@@ -35,10 +46,12 @@ export function createReadSnapshotStore(): ReadSnapshotStore {
 				snapshots.delete(oldestHash);
 				totalBytes -= oldest?.bytes ?? 0;
 			}
+			return true;
 		},
 		clear() {
 			snapshots.clear();
 			totalBytes = 0;
+			epoch += 1;
 		},
 	};
 }

package/extensions/explore/read.ts

@@ -2,7 +2,6 @@ import {
 	createReadToolDefinition,
 	DEFAULT_MAX_BYTES,
 	formatSize,
-	generateUnifiedPatch,
 	truncateHead,
 	type ReadToolDetails,
 	type ToolDefinition,
@@ -13,9 +12,20 @@ import { readFile } from "node:fs/promises";
 import { isAbsolute, resolve } from "node:path";
 import { Type, type Static } from "typebox";
 import { formatToolRowTitle, type ToolRowStateStore } from "../../shared/tool-row-state.js";
+import {
+	createCompleteFileMeta,
+	estimateTokens,
+	readScopeKey,
+	selectCompleteFileResponse,
+} from "./full-file-knowledge.ts";
 import { normalizeCountLimit } from "./limits.ts";
 import { stripLeadingAt } from "./path-display.ts";
-import { createReadCacheStore, type ReadCacheMetaV1, type ReadCacheStore } from "./read-cache.ts";
+import {
+	createReadCacheStore,
+	type ReadCacheMetaV1,
+	type ReadCachePresentation,
+	type ReadCacheStore,
+} from "./read-cache.ts";
 import { createReadSnapshotStore, type ReadSnapshotStore } from "./read-snapshots.ts";
 
 const readSchema = Type.Object({
@@ -43,6 +53,7 @@ interface BaselineTextResult {
 	endLine: number;
 	completeFile: boolean;
 	scopeKey: string;
+	presentation: ReadCachePresentation;
 	summary: string;
 	cacheable: boolean;
 }
@@ -86,10 +97,6 @@ function renderCallSummary(args: ReadToolInput | undefined): string {
 	return `${path}:${start}${end === "" ? "" : `-${end}`}`;
 }
 
-function estimateTokens(text: string): number {
-	return Math.ceil(text.length / 4);
-}
-
 function baselineText(text: string, params: ReadToolInput): BaselineTextResult {
 	const allLines = text.split("\n");
 	const startIndex = params.offset ? Math.max(0, params.offset - 1) : 0;
@@ -134,7 +141,8 @@ function baselineText(text: string, params: ReadToolInput): BaselineTextResult {
 
 	const endLine = outputLines === 0 ? startLine : startLine + outputLines - 1;
 	const completeFile = startIndex === 0 && selectedEnd === allLines.length && !truncation.truncated;
-	const scopeKey = `${completeFile ? "full" : `r:${startLine}:${endLine}`}:n${params.lineNumbers ? 1 : 0}`;
+	const presentation: ReadCachePresentation = params.lineNumbers ? "line-numbered" : "plain";
+	const scopeKey = readScopeKey(completeFile, startLine, endLine, presentation);
 	const summary = completeFile ? `${allLines.length} lines` : `${outputLines} lines`;
 	return {
 		text: outputText,
@@ -144,6 +152,7 @@ function baselineText(text: string, params: ReadToolInput): BaselineTextResult {
 		endLine,
 		completeFile,
 		scopeKey,
+		presentation,
 		summary,
 		cacheable,
 	};
@@ -165,10 +174,24 @@ function createMeta(
 	baseHash?: string,
 	summary = baseline.summary,
 ): ReadCacheMetaV1 {
+	if (baseline.completeFile) {
+		return createCompleteFileMeta({
+			pathKey,
+			presentation: baseline.presentation,
+			servedHash: hash,
+			mode,
+			sourceText: baseline.text,
+			returnedText,
+			totalLines: baseline.totalLines,
+			summary,
+			baseHash,
+		});
+	}
 	return {
 		v: 1,
 		pathKey,
 		scopeKey: baseline.scopeKey,
+		presentation: baseline.presentation,
 		servedHash: hash,
 		baseHash,
 		mode,
@@ -179,16 +202,6 @@ function createMeta(
 	};
 }
 
-function countDiffLines(patch: string): { added: number; removed: number } {
-	let added = 0;
-	let removed = 0;
-	for (const line of patch.split("\n")) {
-		if (line.startsWith("+") && !line.startsWith("+++")) added += 1;
-		else if (line.startsWith("-") && !line.startsWith("---")) removed += 1;
-	}
-	return { added, removed };
-}
-
 export function createExploreReadTool(
 	rowState: ToolRowStateStore,
 	cache: ReadCacheStore = createReadCacheStore(),
@@ -197,6 +210,8 @@ export function createExploreReadTool(
 	const baseDefinition = readDefinitionForCwd(process.cwd());
 	return {
 		...baseDefinition,
+		description:
+			"Read a relevant file as a whole by omitting offset, limit, and lineNumbers. Later whole-file reads avoid repeating unchanged content and return useful diffs or current source after changes. Use ranges only for targeted peripheral inspection or to continue after truncation.",
 		parameters: readSchema,
 		async execute(
 			toolCallId: Parameters<ReadExecute>[0],
@@ -208,6 +223,7 @@ export function createExploreReadTool(
 			const definition = readDefinitionForCwd(ctx.cwd);
 			const normalized = normalizeReadParams(params);
 			const path = isAbsolute(normalized.path) ? resolve(normalized.path) : resolve(ctx.cwd, normalized.path);
+			const snapshotEpoch = snapshots.epoch();
 			const buffer = await readFile(path);
 			if (isSupportedImage(buffer)) {
 				return definition.execute(
@@ -222,7 +238,7 @@ export function createExploreReadTool(
 			if (signal?.aborted) throw new Error("Operation aborted");
 			let text: string;
 			try {
-				text = new TextDecoder("utf-8", { fatal: true }).decode(buffer);
+				text = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(buffer);
 			} catch {
 				return definition.execute(toolCallId, normalized, signal, onUpdate, ctx);
 			}
@@ -232,36 +248,39 @@ export function createExploreReadTool(
 				return { content: [{ type: "text", text: baseline.text }], details: baseline.details };
 			}
 			const hash = createHash("sha256").update(buffer).digest("hex");
-			const decision = cache.decision(ctx, path, baseline.scopeKey);
+			const staleEpoch = !snapshots.isCurrent(snapshotEpoch);
+			const decision = staleEpoch
+				? { baseHash: undefined, baselineText: undefined, recovery: false }
+				: cache.decision(ctx, path, baseline.scopeKey);
 			let output = baseline.text;
 			let mode: ReadCacheMetaV1["mode"] = decision.recovery ? "recovery" : "baseline";
 			let summary = baseline.summary;
 
-			if (!decision.recovery && decision.baseHash === hash) {
-				output = baseline.completeFile
-					? `unchanged, ${baseline.totalLines} lines`
-					: `unchanged, lines ${baseline.startLine}-${baseline.endLine} of ${baseline.totalLines}`;
+			if (baseline.completeFile) {
+				const selected = selectCompleteFileResponse({
+					displayPath: normalized.path,
+					currentText: text,
+					currentHash: hash,
+					fullText: baseline.text,
+					totalLines: baseline.totalLines,
+					recovery: decision.recovery,
+					baseHash: decision.baseHash,
+					baselineText:
+						decision.baselineText ??
+						(decision.baseHash === undefined ? undefined : snapshots.get(decision.baseHash)),
+				});
+				output = selected.text;
+				mode = selected.mode;
+				summary = selected.summary;
+			} else if (!decision.recovery && decision.baseHash === hash) {
+				output = `unchanged, lines ${baseline.startLine}-${baseline.endLine} of ${baseline.totalLines}`;
 				mode = "unchanged";
 				summary = output;
-			} else if (!decision.recovery && decision.baseHash && baseline.completeFile && !normalized.lineNumbers) {
-				const baseText = snapshots.get(decision.baseHash);
-				if (baseText !== undefined) {
-					const patch = generateUnifiedPatch(normalized.path, baseText, text, 3);
-					const counts = countDiffLines(patch);
-					const candidate = `[read: ${counts.added} lines added, ${counts.removed} removed of ${baseline.totalLines}]\n${patch}`;
-					const candidateTruncation = truncateHead(candidate);
-					if (!candidateTruncation.truncated && estimateTokens(candidate) < estimateTokens(baseline.text)) {
-						output = candidate;
-						mode = "diff";
-						summary = `+${counts.added} -${counts.removed}`;
-					}
-				}
 			}
 
 			if (signal?.aborted) throw new Error("Operation aborted");
-			snapshots.set(hash, text, buffer.byteLength);
+			snapshots.set(hash, text, buffer.byteLength, snapshotEpoch);
 			const meta = createMeta(baseline, path, hash, mode, output, decision.baseHash, summary);
-			cache.record(ctx, meta);
 			return withMeta(baseline, meta, output);
 		},
 		renderCall(