@namzu/sdk 62.7.0 → 62.9.0

package/src/store/session-log/verified-prefix.ts

@@ -0,0 +1,243 @@
+import * as crypto from 'node:crypto'
+import type { SessionId } from '../../types/ids/index.js'
+import type { RecordPointer } from '../../types/session/records.js'
+import { LineSplitter, type LogBytes, type SessionLogEntry } from './chain.js'
+import { SessionTurnState } from './fold.js'
+
+/**
+ * What this process has already verified of a session log, so that checking
+ * it again costs the bytes and not the parsing.
+ *
+ * A ledger remembers, for every record this process verified IN FULL (schema,
+ * parse, hash chain, seq, gen, links: everything `SessionLogChain.accept`
+ * checks), the byte length of its line and the SHA-256 of those bytes, plus
+ * the head pointer, the `gen`, the session id and the turn state folded to
+ * the head. A later claim of the same log reads every byte again and hashes
+ * each line; a line whose length and digest equal the remembered ones is
+ * byte-identical to a line that already passed every rule, so it passes them
+ * again without being parsed. Only what lies after the remembered head is
+ * verified in full. Nothing about the file's identity is trusted for
+ * soundness: the content is compared. Any difference (a changed byte, a
+ * shorter file, a replaced file, a line that no longer splits the same way)
+ * makes the caller discard the ledger and verify from byte 0 as before.
+ *
+ * A ledger is fed only from entries that came out of a full verification and
+ * that continue it (`seq` = count + 1 and `prev` equal to the head), so what
+ * it holds is a prefix of one valid chain and nothing else. It costs 36 bytes
+ * per record.
+ */
+
+const DIGEST_BYTES = 32
+/** Logs kept; the least recently used is dropped. */
+const LEDGERS_KEPT = 16
+/** A log with more records than this is not remembered (about 72 MB of digests). */
+const MAX_RECORDS = 1 << 21
+
+/**
+ * The digest of one line as lowercase hex. A string, not a Buffer: a Buffer per
+ * line (82,000 of them in a long log) is native memory the collector has to
+ * chase, and measured about twice the CPU. The one-shot form where Node has it.
+ */
+const hexDigestOf: (line: Uint8Array) => string =
+	typeof crypto.hash === 'function'
+		? (line) => crypto.hash('sha256', line, 'hex')
+		: (line) => crypto.createHash('sha256').update(line).digest('hex')
+
+/** Whether `hex` (64 lowercase hex characters) spells the 32 bytes at `at` in `bytes`. */
+function hexEqualsBytes(hex: string, bytes: Uint8Array, at: number): boolean {
+	if (hex.length !== DIGEST_BYTES * 2) return false
+	for (let i = 0; i < DIGEST_BYTES; i++) {
+		const high = hex.charCodeAt(i * 2)
+		const low = hex.charCodeAt(i * 2 + 1)
+		const value =
+			((high <= 0x39 ? high - 0x30 : high - 0x57) << 4) | (low <= 0x39 ? low - 0x30 : low - 0x57)
+		if (value !== bytes[at + i]) return false
+	}
+	return true
+}
+
+export class VerifiedPrefix {
+	#count = 0
+	#lengths = new Uint32Array(1024)
+	#digests = Buffer.allocUnsafe(1024 * DIGEST_BYTES)
+	#head: RecordPointer | null = null
+	#gen = 0
+	#sessionId: SessionId | undefined
+	#end = 0
+	#dead = false
+	readonly #turns = new SessionTurnState()
+
+	/** Records remembered. */
+	get count(): number {
+		return this.#count
+	}
+	get head(): RecordPointer | null {
+		return this.#head
+	}
+	get gen(): number {
+		return this.#gen
+	}
+	get sessionId(): SessionId | undefined {
+		return this.#sessionId
+	}
+	/** Byte offset just after the last remembered record. */
+	get end(): number {
+		return this.#end
+	}
+	/** The turn state at the head, as its own copy. */
+	turns(): SessionTurnState {
+		return this.#turns.clone()
+	}
+	/** Lengths and digests as they stand now; entries below `count` never change. */
+	get lengths(): Uint32Array {
+		return this.#lengths
+	}
+	get digests(): Buffer {
+		return this.#digests
+	}
+
+	/** Whether the remembered record `seq` is exactly `pointer` (length and digest). */
+	holds(pointer: RecordPointer): boolean {
+		const at = pointer.seq - 1
+		if (at < 0 || at >= this.#count) return false
+		if (this.#lengths[at] !== pointer.length) return false
+		return (
+			this.#digests.toString('hex', at * DIGEST_BYTES, (at + 1) * DIGEST_BYTES) === pointer.sha256
+		)
+	}
+
+	/**
+	 * Remember a record that was verified in full. Ignored unless it continues
+	 * what is held: the next seq, naming the head by its pointer.
+	 */
+	append(entry: SessionLogEntry): void {
+		if (this.#dead) return
+		const { record, pointer } = entry
+		if (pointer.seq !== this.#count + 1 || pointer.offset !== this.#end) return
+		const prev = record.prev
+		if (this.#head === null ? prev !== null : !samePointer(prev, this.#head)) return
+		if (this.#sessionId !== undefined && record.sessionId !== this.#sessionId) return
+		if (this.#count >= MAX_RECORDS) {
+			this.#dead = true
+			return
+		}
+		if (this.#count === this.#lengths.length) {
+			const lengths = new Uint32Array(this.#lengths.length * 2)
+			lengths.set(this.#lengths)
+			this.#lengths = lengths
+			const digests = Buffer.allocUnsafe(this.#digests.length * 2)
+			this.#digests.copy(digests)
+			this.#digests = digests
+		}
+		this.#lengths[this.#count] = pointer.length
+		this.#digests.write(pointer.sha256, this.#count * DIGEST_BYTES, DIGEST_BYTES, 'hex')
+		this.#count += 1
+		this.#head = pointer
+		this.#gen = record.gen
+		this.#end = pointer.offset + pointer.length
+		this.#sessionId ??= record.sessionId
+		this.#turns.apply(record)
+	}
+}
+
+function samePointer(a: RecordPointer | null | undefined, b: RecordPointer): boolean {
+	return (
+		a !== null &&
+		a !== undefined &&
+		a.seq === b.seq &&
+		a.offset === b.offset &&
+		a.length === b.length &&
+		a.sha256 === b.sha256
+	)
+}
+
+const ledgers = new Map<string, VerifiedPrefix>()
+
+/** The ledger of a log, if this process has one; marks it recently used. */
+export function verifiedPrefixOf(key: string): VerifiedPrefix | undefined {
+	const found = ledgers.get(key)
+	if (found !== undefined) {
+		ledgers.delete(key)
+		ledgers.set(key, found)
+	}
+	return found
+}
+
+/** Discard what is remembered of a log and start an empty ledger for it. */
+export function startVerifiedPrefix(key: string): VerifiedPrefix {
+	ledgers.delete(key)
+	if (ledgers.size >= LEDGERS_KEPT) {
+		const oldest = ledgers.keys().next().value
+		if (oldest !== undefined) ledgers.delete(oldest)
+	}
+	const fresh = new VerifiedPrefix()
+	ledgers.set(key, fresh)
+	return fresh
+}
+
+/** Forget every remembered log: what a process that has not read them yet looks like. For tests. */
+export function clearVerifiedPrefixes(): void {
+	ledgers.clear()
+}
+
+/** How many logs are remembered. For tests. */
+export function verifiedPrefixCount(): number {
+	return ledgers.size
+}
+
+/**
+ * Read the first `count` lines of `bytes` and compare each one's length and
+ * SHA-256 with the ledger's. Resolves `true` only when every one is equal.
+ * Parses nothing and keeps nothing. Any difference, a log that ends sooner,
+ * or a fragment no record could be, resolves `false`: the caller then verifies
+ * in full, which says what is wrong.
+ */
+export async function rehashPrefix(
+	bytes: Pick<LogBytes, 'size'> & {
+		stream(offset: number): AsyncIterable<Uint8Array>
+		scan?(offset: number, visit: (chunk: Uint8Array) => boolean): Promise<void>
+	},
+	ledger: VerifiedPrefix,
+	count: number,
+): Promise<boolean> {
+	if (count === 0) return true
+	if (count > ledger.count) return false
+	const lengths = ledger.lengths
+	const digests = ledger.digests
+	if ((await bytes.size()) < endOfLine(lengths, count)) return false
+	const splitter = new LineSplitter(0)
+	let index = 0
+	let matched = false
+	// True once the answer is known. A chunk is read from a buffer the caller of `visit`
+	// may reuse, so nothing here keeps a slice of it: a line is hashed where it stands.
+	const visit = (chunk: Uint8Array): boolean => {
+		for (const { line } of splitter.push(chunk)) {
+			if (line.byteLength !== lengths[index]) return true
+			if (!hexEqualsBytes(hexDigestOf(line), digests, index * DIGEST_BYTES)) return true
+			index += 1
+			if (index === count) {
+				matched = true
+				return true
+			}
+		}
+		return false
+	}
+	try {
+		if (bytes.scan !== undefined) {
+			await bytes.scan(0, visit)
+		} else {
+			for await (const chunk of bytes.stream(0)) if (visit(chunk)) break
+		}
+	} catch (error) {
+		// A fragment longer than any record can be: not the log that was verified.
+		if ((error as { name?: string }).name === 'SessionLogIntegrityError') return false
+		throw error
+	}
+	return matched
+}
+
+function endOfLine(lengths: Uint32Array, count: number): number {
+	let total = 0
+	for (let i = 0; i < count; i++) total += lengths[i] as number
+	return total
+}

package/src/tools/admit-facts.ts

@@ -189,6 +189,7 @@ export function admitToolFacts(value: unknown): ToolFacts | undefined {
 				kind: 'failure',
 				reason,
 				subject: text(value.subject),
+				note: text(value.note),
 			})
 		}
 		default:

package/src/tools/builtins/bash.ts

@@ -520,6 +520,11 @@ export function execHostShell(
 
 export { dangerousCommandReason }
 
+/** A reason as a sentence a person reads under a row: first letter up. */
+function sentence(reason: string): string {
+	return reason ? `${reason.charAt(0).toUpperCase()}${reason.slice(1)}` : reason
+}
+
 /** The dangerous-pattern check `bash` applies, shared so the `monitor` tool refuses the same lines. */
 export function isDangerousCommand(command: string): boolean {
 	return DANGEROUS_PATTERNS.some((pattern) => pattern.test(command))
@@ -596,7 +601,12 @@ const BashToolBase = defineTool({
 				success: false,
 				output: '',
 				error: `Dangerous command blocked: "${input.command}". Why: ${dangerousCommandReason(input.command)}. Nothing ran. Use a narrower command that avoids it, or ask the user to run this one.`,
-				facts: { kind: 'failure', reason: 'blocked', subject: 'bash' },
+				facts: {
+					kind: 'failure',
+					reason: 'blocked',
+					subject: 'bash',
+					note: sentence(dangerousCommandReason(input.command)),
+				},
 			}
 		}
 

package/src/tools/builtins/bounded-file-read.ts

@@ -1,4 +1,4 @@
-import { constants } from 'node:fs'
+import { constants, closeSync, fstatSync, openSync, readSync } from 'node:fs'
 import { open } from 'node:fs/promises'
 
 const CHUNK_BYTES = 64 * 1024
@@ -40,10 +40,13 @@ export async function readBoundedFile(
 	maximum: number,
 	signal?: AbortSignal,
 	selectLimit?: (prefix: Buffer) => number,
+	extraFlags = 0,
 ): Promise<Buffer> {
 	// O_NONBLOCK prevents a substituted FIFO from blocking open before fstat.
+	// `extraFlags` adds to it, for a caller that also needs the last component
+	// refused when it is a link.
 	signal?.throwIfAborted()
-	const file = await open(path, constants.O_RDONLY | constants.O_NONBLOCK)
+	const file = await open(path, constants.O_RDONLY | constants.O_NONBLOCK | extraFlags)
 	try {
 		const info = await file.stat()
 		if (!info.isFile()) throw new Error('Read requires a regular file.')
@@ -78,3 +81,57 @@ function inputTooLarge(limit: number): Error {
 		`File exceeds the bounded read limit (${limit} bytes). Extract a smaller file with a shell command.`,
 	)
 }
+
+/** True where the platform can refuse a final-component link at open time. */
+export const CAN_REFUSE_LINKS = typeof constants.O_NOFOLLOW === 'number' && constants.O_NOFOLLOW > 0
+
+/** The flag a caller passes to {@link readBoundedFile} to refuse a final-component link. */
+export const REFUSE_LINK_FLAG: number = CAN_REFUSE_LINKS ? constants.O_NOFOLLOW : 0
+
+/**
+ * The same bounded read as {@link readBoundedFile} through one synchronous
+ * handle: open, fstat, read, close. A link in the last path component is
+ * refused at open (`O_NOFOLLOW`), so a link swapped in after a walk listed the
+ * file cannot be followed. Callers yield between files; one file is at most
+ * `maximum` bytes; a file whose size is over it answers `'too-large'`. With a
+ * `scratch` buffer, a file that fits one chunk comes back as a view of it, valid
+ * until the next call. Requires {@link CAN_REFUSE_LINKS}.
+ *
+ * With `syncMaximum`, a file whose size is over it (and within `maximum`) is not
+ * read here: the answer is `'large'`, so the caller reads it through
+ * {@link readBoundedFile}, which does not hold the event loop for the whole file.
+ */
+export function readBoundedFileSync(
+	path: string,
+	maximum: number,
+	scratch?: Buffer,
+	syncMaximum = maximum,
+): Buffer | 'too-large' | 'large' {
+	const fd = openSync(path, constants.O_RDONLY | constants.O_NONBLOCK | constants.O_NOFOLLOW)
+	try {
+		const info = fstatSync(fd)
+		if (!info.isFile()) throw new Error('Read requires a regular file.')
+		if (info.size > maximum) return 'too-large'
+		if (info.size > syncMaximum) return 'large'
+		const chunks: Buffer[] = []
+		let offset = 0
+		for (;;) {
+			const length = Math.min(CHUNK_BYTES, maximum + 1 - offset)
+			const buffer =
+				offset === 0 && scratch && scratch.length >= length ? scratch : Buffer.allocUnsafe(length)
+			let used = 0
+			while (used < length) {
+				const bytesRead = readSync(fd, buffer, used, length - used, offset + used)
+				if (bytesRead === 0) break
+				used += bytesRead
+			}
+			offset += used
+			if (offset > maximum) throw inputTooLarge(maximum)
+			chunks.push(buffer.subarray(0, used))
+			if (used < length)
+				return chunks.length === 1 ? (chunks[0] as Buffer) : Buffer.concat(chunks, offset)
+		}
+	} finally {
+		closeSync(fd)
+	}
+}

package/src/tools/builtins/grep.ts

@@ -1,5 +1,5 @@
 import { stat } from 'node:fs/promises'
-import { isAbsolute, join, relative, sep } from 'node:path'
+import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path'
 import { z } from 'zod'
 import { walkFilesLocally } from '../../sandbox/file-walk.js'
 import type { SandboxWalkFilesOptions, SandboxWalkSummary } from '../../types/sandbox/index.js'
@@ -7,6 +7,7 @@ import type { ToolContext } from '../../types/tool/index.js'
 import { subscribeToAbort } from '../../utils/abort.js'
 import { defineTool } from '../defineTool.js'
 import {
+	canonicalize,
 	resolveWithin,
 	resolveWithinAny,
 	resolveWithinAnyReal,
@@ -14,7 +15,13 @@ import {
 	toolRoots,
 } from '../paths.js'
 import { relativePosix, resolveWithinPosix } from '../posix-path.js'
-import { readBoundedChunks, readBoundedFile } from './bounded-file-read.js'
+import {
+	CAN_REFUSE_LINKS,
+	REFUSE_LINK_FLAG,
+	readBoundedChunks,
+	readBoundedFile,
+	readBoundedFileSync,
+} from './bounded-file-read.js'
 import {
 	MAX_FILES_SEARCHED,
 	SEARCH_BUDGET_MS,
@@ -24,6 +31,7 @@ import {
 	isReadDenied,
 	openSearchBudget,
 	plural,
+	readAhead,
 	stoppedNotice,
 } from './search-support.js'
 
@@ -49,6 +57,125 @@ async function readWithSignal(read: () => Promise<Buffer>, signal?: AbortSignal)
 	}
 }
 
+/** Longest the local read loop holds the event loop before it yields. */
+const READ_SLICE_MS = 8
+
+/**
+ * Largest file the local reader reads synchronously, by the size on the open
+ * handle. One synchronous read of a file is one uninterrupted block, which on a
+ * slow mount can last tens of milliseconds; a larger file is read through the
+ * asynchronous bounded read instead, which lets the event loop run between
+ * chunks.
+ */
+export const LOCAL_SYNC_READ_MAX_BYTES = 64 * 1024
+
+/**
+ * Reads files of a local search root. Each directory is resolved to its real
+ * path once and checked against the real root once; a file is then one open
+ * (links refused at the last component), fstat, read and close, all
+ * synchronous, with a yield to the event loop every few milliseconds. That is
+ * the same containment decision as resolving every file, because the walk
+ * lists only regular files in real directories and the open refuses a file
+ * that has become a link. Not used for a sandbox, nor where the platform cannot
+ * refuse links.
+ */
+function createLocalReader(root: string, signal: AbortSignal | undefined) {
+	const scratch = Buffer.allocUnsafe(64 * 1024)
+	const rootRealPromise = canonicalize(resolve(root))
+	const directories = new Map<string, Promise<string>>()
+	let sliceStart = performance.now()
+	const realDirectory = (directory: string): Promise<string> => {
+		let known = directories.get(directory)
+		if (!known) {
+			known = canonicalize(directory)
+			directories.set(directory, known)
+		}
+		return known
+	}
+	let rootReal: string | undefined
+	return {
+		async read(
+			filePath: string,
+		): Promise<{ buffer: Buffer | 'too-large' | 'gone'; contained: string }> {
+			if (performance.now() - sliceStart > READ_SLICE_MS) {
+				await new Promise<void>((done) => setImmediate(done))
+				sliceStart = performance.now()
+			}
+			signal?.throwIfAborted()
+			const directory = dirname(filePath)
+			const real = await realDirectory(directory)
+			rootReal ??= await rootRealPromise
+			const contained =
+				real === directory ? filePath : join(real, filePath.slice(directory.length + 1))
+			if (!(contained.startsWith(rootReal + sep) || contained === rootReal)) {
+				const rel = relative(rootReal, contained)
+				if (rel.startsWith('..') || isAbsolute(rel)) {
+					throw new Error('Path escapes the working directory through a link.')
+				}
+			}
+			signal?.throwIfAborted()
+			try {
+				const read = readBoundedFileSync(
+					contained,
+					MAX_FILE_SIZE,
+					scratch,
+					LOCAL_SYNC_READ_MAX_BYTES,
+				)
+				if (read !== 'large') return { buffer: read, contained }
+				// The handle that measured the file is closed; the asynchronous read opens it
+				// again with the same flags (links refused) and repeats every check.
+				const large = await readBoundedFile(
+					contained,
+					MAX_FILE_SIZE,
+					signal,
+					undefined,
+					REFUSE_LINK_FLAG,
+				)
+				sliceStart = performance.now()
+				return { buffer: large, contained }
+			} catch (error) {
+				// Gone since the walk listed it: absent, as a walk that ran later would find it.
+				const code = (error as NodeJS.ErrnoException).code
+				if (code === 'ENOENT' || code === 'ENOTDIR') return { buffer: 'gone', contained }
+				throw error
+			}
+		},
+	}
+}
+
+/** A walked path that is already inside `root` in normal form, or null when it needs the full check. */
+const PLAIN_TAIL = /(^|\/)\.{1,2}(\/|$)|\/\/|\/$/
+
+/**
+ * The fast form of what `resolveWithin` and the display path compute for the
+ * walk's own entries on a POSIX host: a path the walk built under `root` is
+ * already normal, so it is returned as it is and its display name is a
+ * prefix plus the tail. Anything else, and every other platform, takes the
+ * full computation.
+ */
+function createWalkedPaths(root: string, lexicalRoot: string, workingDirectory: string) {
+	if (sep !== '/') return undefined
+	const prefix = root === '/' ? '/' : `${root}/`
+	const rootRel = relative(workingDirectory, lexicalRoot)
+	const display = rootRel.startsWith('..') || isAbsolute(rootRel) ? undefined : rootRel
+	return {
+		within(path: string): string {
+			if (path.startsWith(prefix) && path.length > prefix.length) {
+				if (!PLAIN_TAIL.test(path.slice(prefix.length))) return path
+			}
+			return resolveWithin(root, path)
+		},
+		displayName(path: string): string | undefined {
+			if (display === undefined || !path.startsWith(prefix) || path.length === prefix.length) {
+				return undefined
+			}
+			const tail = path.slice(prefix.length)
+			if (PLAIN_TAIL.test(tail)) return undefined
+			return display === '' ? tail : `${display}/${tail}`
+		},
+	}
+}
+
 function relativeInclude(include: string, root: string, sandboxed: boolean): string {
 	let pattern = sandboxed ? include : include.split(sep).join('/')
 	if (pattern.split('/').includes('..'))
@@ -201,11 +328,7 @@ const MAX_FILE_SIZE = 5 * 1024 * 1024 // 5 MB — skip binaries/large files
 const BINARY_CHECK_BYTES = 512
 
 function isBinaryContent(buffer: Buffer): boolean {
-	const check = buffer.subarray(0, BINARY_CHECK_BYTES)
-	for (const byte of check) {
-		if (byte === 0) return true
-	}
-	return false
+	return buffer.subarray(0, BINARY_CHECK_BYTES).includes(0)
 }
 
 export const GrepTool = defineTool({
@@ -291,7 +414,11 @@ export const GrepTool = defineTool({
 				walkEnd = value
 			},
 		}
-		const entries = walkSandbox ? walkSandbox(root, options) : walkFilesLocally(root, options)
+		// A local walk is read ahead so its directory reads overlap with matching; a
+		// sandbox's walk is left as the adapter produces it.
+		const entries = walkSandbox
+			? walkSandbox(root, options)
+			: readAhead(walkFilesLocally(root, { ...options, withoutSizes: CAN_REFUSE_LINKS }))
 
 		// `execute` is reachable without the schema's defaults.
 		const mode = input.output_mode ?? 'files_with_matches'
@@ -321,12 +448,16 @@ export const GrepTool = defineTool({
 		const filesToCollect =
 			headLimit > 0 ? offset + Math.max(headLimit * 4, 1000) : Number.POSITIVE_INFINITY
 		let enough = false
+		const walked = sandbox
+			? undefined
+			: createWalkedPaths(root, lexicalRoot, context.workingDirectory)
+		const localReader = !sandbox && CAN_REFUSE_LINKS ? createLocalReader(root, signal) : undefined
 		try {
 			for await (const entry of entries) {
 				signal?.throwIfAborted()
 				const filePath = sandbox
 					? resolveWithinPosix(root, entry.path)
-					: resolveWithin(root, entry.path)
+					: (walked?.within(entry.path) ?? resolveWithin(root, entry.path))
 				if (filesSearched >= MAX_FILES_SEARCHED) {
 					fileLimit = true
 					break
@@ -339,21 +470,34 @@ export const GrepTool = defineTool({
 				let content: string
 				let contained: string | undefined
 				try {
-					const buffer = await readWithSignal(
-						() =>
-							sandbox
-								? readBoundedChunks(
-										(offset, length) =>
-											sandbox.readFile(filePath, { offset, length, signal: signal }),
-										MAX_FILE_SIZE,
-										signal,
-									)
-								: resolveWithinReal(root, filePath).then((path) => {
-										contained = path
-										return readBoundedFile(path, MAX_FILE_SIZE, signal)
-									}),
-						signal,
-					)
+					let buffer: Buffer
+					if (localReader) {
+						const read = await localReader.read(filePath)
+						contained = read.contained
+						// Over the cap by the size on the open handle: skipped, not unreadable.
+						if (read.buffer === 'too-large') continue
+						// Gone since the walk listed it: as if the walk had not listed it.
+						if (read.buffer === 'gone') {
+							filesSearched--
+							continue
+						}
+						buffer = read.buffer
+					} else
+						buffer = await readWithSignal(
+							() =>
+								sandbox
+									? readBoundedChunks(
+											(offset, length) =>
+												sandbox.readFile(filePath, { offset, length, signal: signal }),
+											MAX_FILE_SIZE,
+											signal,
+										)
+									: resolveWithinReal(root, filePath).then((path) => {
+											contained = path
+											return readBoundedFile(path, MAX_FILE_SIZE, signal)
+										}),
+							signal,
+						)
 					signal?.throwIfAborted()
 					if (buffer.length > MAX_FILE_SIZE || isBinaryContent(buffer)) continue
 					content = buffer.toString('utf-8')
@@ -367,7 +511,8 @@ export const GrepTool = defineTool({
 				let fileMatches = 0
 				const relPath = sandbox
 					? relativePosix(sandbox.rootDir, filePath)
-					: hostDisplayPath(context.workingDirectory, join(lexicalRoot, relative(root, filePath)))
+					: (walked?.displayName(filePath) ??
+						hostDisplayPath(context.workingDirectory, join(lexicalRoot, relative(root, filePath))))
 
 				if (mode === 'content' && input.only_matching) {
 					for (const fragment of matchingFragments(content, regex, multiline)) {
@@ -580,7 +725,8 @@ function lineHits(
 ): [number, number][] {
 	const hits: [number, number][] = []
 	for (let i = 0; i < lines.length; i++) {
-		context.abortSignal?.throwIfAborted()
+		// A cancellation is seen within a few hundred lines; per line it cost more than the match.
+		if ((i & 255) === 0) context.abortSignal?.throwIfAborted()
 		regex.lastIndex = 0
 		if (regex.test(lines[i] ?? '')) hits.push([i, i])
 	}

package/src/tools/builtins/search-support.ts

@@ -105,3 +105,65 @@ export function isReadDenied(
 	if (relative === '..' || relative.startsWith('../') || posix.isAbsolute(relative)) return true
 	return context.readDenied(joinPath(context.workingDirectory, relative))
 }
+
+/**
+ * Pulls from `source` ahead of the consumer, up to `ahead` items, so a walk's
+ * directory reads overlap with the matching of files already listed. Order is
+ * unchanged. An error from the source is thrown to the consumer after the
+ * items listed before it; when the consumer stops, the source is closed (its
+ * own cleanup runs) before this returns.
+ */
+export async function* readAhead<T>(source: AsyncIterable<T>, ahead = 256): AsyncGenerator<T> {
+	const iterator = source[Symbol.asyncIterator]()
+	const buffer: T[] = []
+	let finished = false
+	let failure: { error: unknown } | undefined
+	let stop = false
+	let wakeConsumer: (() => void) | undefined
+	let wakeProducer: (() => void) | undefined
+	const notifyConsumer = () => {
+		const wake = wakeConsumer
+		wakeConsumer = undefined
+		wake?.()
+	}
+	const producer = (async () => {
+		try {
+			while (!stop) {
+				if (buffer.length >= ahead) {
+					await new Promise<void>((resolve) => {
+						wakeProducer = resolve
+					})
+					continue
+				}
+				const next = await iterator.next()
+				if (next.done) break
+				buffer.push(next.value)
+				notifyConsumer()
+			}
+		} catch (error) {
+			failure = { error }
+		}
+		finished = true
+		notifyConsumer()
+	})()
+	try {
+		for (;;) {
+			if (buffer.length > 0) {
+				const item = buffer.shift() as T
+				wakeProducer?.()
+				yield item
+				continue
+			}
+			if (failure) throw failure.error
+			if (finished) return
+			await new Promise<void>((resolve) => {
+				wakeConsumer = resolve
+			})
+		}
+	} finally {
+		stop = true
+		wakeProducer?.()
+		await producer
+		await iterator.return?.()
+	}
+}