@namzu/sdk 62.7.0 → 62.8.0

package/src/store/session-log/incremental.ts

@@ -0,0 +1,147 @@
+import type { RecordPointer } from '../../types/session/records.js'
+import { type SessionLogEntry, SessionLogIntegrityError } from './chain.js'
+import type { SessionLog } from './core.js'
+
+/**
+ * Reductions over a whole session log, advanced rather than recomputed.
+ *
+ * A session log only grows, and every turn asks it the same questions (which
+ * checkpoints were committed, which turn ids exist, what the folded context
+ * is). Answering each from byte 0 parses and verifies every record the
+ * session ever wrote, so a turn cost more the longer the session. A reduction
+ * held here remembers the last record it folded and, on the next ask, walks
+ * only what was appended after it. `SessionLog.read({ after })` checks that
+ * record against the log byte for byte before continuing, so a log that was
+ * replaced, truncated or rewritten at that record is noticed and the
+ * reduction starts again from byte 0 with the original behaviour (strict reads
+ * refuse a broken chain, tolerant reads stop at the break).
+ *
+ * What a resumed read does not do is re-verify the records before the anchor:
+ * they were verified when they were first folded, by this process. A byte
+ * flipped in them afterwards, with every later record left as it was, is not
+ * noticed by this path; a fresh process (a resume) always starts from byte 0.
+ *
+ * The cache is keyed by session id and the reduction's own name and mode,
+ * holds a few sessions, and is anchored by content, so two logs that share a
+ * session id never share a result.
+ *
+ * Cost that remains: the state of the "every message ever recorded" reduction
+ * holds each message the log ever recorded, superseded and compacted ones
+ * included, so it grows with the log and not with the live context, and it
+ * lives for the life of the process (for the sessions kept). Reading it
+ * returns a copy of all of them, which is O(history) per turn. Both are small
+ * next to the whole-log walks this file removes, and are left as they are.
+ * More than SESSIONS_KEPT sessions in use at once evict one another, and each
+ * eviction restarts from byte 0, the cost before this file existed.
+ */
+
+const SESSIONS_KEPT = 4
+
+interface Kept {
+	state: unknown
+	/** The last record folded; the next read resumes after it. */
+	anchor: RecordPointer | undefined
+	/** Calls on one reduction take turns: the state is advanced in place. */
+	turn: Promise<unknown>
+}
+
+const sessions = new Map<string, Map<string, Kept>>()
+
+function keptFor(sessionId: string, key: string): Kept {
+	let reductions = sessions.get(sessionId)
+	if (reductions === undefined) {
+		reductions = new Map()
+		if (sessions.size >= SESSIONS_KEPT) {
+			const oldest = sessions.keys().next().value
+			if (oldest !== undefined) sessions.delete(oldest)
+		}
+	} else {
+		sessions.delete(sessionId)
+	}
+	sessions.set(sessionId, reductions)
+	let kept = reductions.get(key)
+	if (kept === undefined) {
+		kept = { state: undefined, anchor: undefined, turn: Promise.resolve() }
+		reductions.set(key, kept)
+	}
+	return kept
+}
+
+export interface IncrementalFold<S, R> {
+	/** The reduction's name; one cache slot per name and read mode. */
+	readonly name: string
+	readonly mode: 'strict' | 'tolerant'
+	/** A fresh, empty state. */
+	create(): S
+	/** Fold one record into the state, in place. */
+	apply(state: S, entry: SessionLogEntry): void
+	/**
+	 * The answer, from a state that is current for as long as this call runs.
+	 * It must not keep or hand out the state itself, which later calls advance.
+	 */
+	result(state: S): Promise<R> | R
+}
+
+/**
+ * Fold `log` into `fold.create()` through `fold.apply`, resuming from the
+ * record the previous call for this session stopped at when the log still
+ * holds it, and answer with `fold.result`.
+ */
+export async function foldSessionLog<S, R>(
+	log: SessionLog,
+	fold: IncrementalFold<S, R>,
+): Promise<R> {
+	const kept = keptFor(log.sessionId, `${fold.name}:${fold.mode}`)
+	const run = async (): Promise<R> => {
+		let state = kept.state as S | undefined
+		let anchor = kept.anchor
+		for (let attempt = 0; attempt < 2; attempt++) {
+			const resuming = state !== undefined && anchor !== undefined
+			if (!resuming) {
+				state = fold.create()
+				anchor = undefined
+			}
+			let moved = false
+			try {
+				const walk = log.read({ mode: fold.mode, ...(anchor ? { after: anchor } : {}) })
+				let summary: Awaited<ReturnType<typeof walk.next>>
+				for (;;) {
+					summary = await walk.next()
+					if (summary.done) break
+					moved = true
+					fold.apply(state as S, summary.value)
+				}
+				const ended = summary.value
+				if (resuming && !moved && ended.break?.reason === 'anchor-mismatch') {
+					kept.state = undefined
+					kept.anchor = undefined
+					state = undefined
+					anchor = undefined
+					continue
+				}
+				kept.state = state
+				kept.anchor = ended.head?.pointer ?? anchor
+				return await fold.result(state as S)
+			} catch (error) {
+				// The state may be half-advanced: never reuse it.
+				kept.state = undefined
+				kept.anchor = undefined
+				if (
+					resuming &&
+					!moved &&
+					error instanceof SessionLogIntegrityError &&
+					error.reason === 'anchor-mismatch'
+				) {
+					state = undefined
+					anchor = undefined
+					continue
+				}
+				throw error
+			}
+		}
+		throw new Error('unreachable')
+	}
+	const result = kept.turn.then(run, run)
+	kept.turn = result.catch(() => undefined)
+	return result
+}

package/src/tools/admit-facts.ts

@@ -189,6 +189,7 @@ export function admitToolFacts(value: unknown): ToolFacts | undefined {
 				kind: 'failure',
 				reason,
 				subject: text(value.subject),
+				note: text(value.note),
 			})
 		}
 		default:

package/src/tools/builtins/bash.ts

@@ -520,6 +520,11 @@ export function execHostShell(
 
 export { dangerousCommandReason }
 
+/** A reason as a sentence a person reads under a row: first letter up. */
+function sentence(reason: string): string {
+	return reason ? `${reason.charAt(0).toUpperCase()}${reason.slice(1)}` : reason
+}
+
 /** The dangerous-pattern check `bash` applies, shared so the `monitor` tool refuses the same lines. */
 export function isDangerousCommand(command: string): boolean {
 	return DANGEROUS_PATTERNS.some((pattern) => pattern.test(command))
@@ -596,7 +601,12 @@ const BashToolBase = defineTool({
 				success: false,
 				output: '',
 				error: `Dangerous command blocked: "${input.command}". Why: ${dangerousCommandReason(input.command)}. Nothing ran. Use a narrower command that avoids it, or ask the user to run this one.`,
-				facts: { kind: 'failure', reason: 'blocked', subject: 'bash' },
+				facts: {
+					kind: 'failure',
+					reason: 'blocked',
+					subject: 'bash',
+					note: sentence(dangerousCommandReason(input.command)),
+				},
 			}
 		}
 

package/src/tools/builtins/bounded-file-read.ts

@@ -1,4 +1,4 @@
-import { constants } from 'node:fs'
+import { constants, closeSync, fstatSync, openSync, readSync } from 'node:fs'
 import { open } from 'node:fs/promises'
 
 const CHUNK_BYTES = 64 * 1024
@@ -40,10 +40,13 @@ export async function readBoundedFile(
 	maximum: number,
 	signal?: AbortSignal,
 	selectLimit?: (prefix: Buffer) => number,
+	extraFlags = 0,
 ): Promise<Buffer> {
 	// O_NONBLOCK prevents a substituted FIFO from blocking open before fstat.
+	// `extraFlags` adds to it, for a caller that also needs the last component
+	// refused when it is a link.
 	signal?.throwIfAborted()
-	const file = await open(path, constants.O_RDONLY | constants.O_NONBLOCK)
+	const file = await open(path, constants.O_RDONLY | constants.O_NONBLOCK | extraFlags)
 	try {
 		const info = await file.stat()
 		if (!info.isFile()) throw new Error('Read requires a regular file.')
@@ -78,3 +81,57 @@ function inputTooLarge(limit: number): Error {
 		`File exceeds the bounded read limit (${limit} bytes). Extract a smaller file with a shell command.`,
 	)
 }
+
+/** True where the platform can refuse a final-component link at open time. */
+export const CAN_REFUSE_LINKS = typeof constants.O_NOFOLLOW === 'number' && constants.O_NOFOLLOW > 0
+
+/** The flag a caller passes to {@link readBoundedFile} to refuse a final-component link. */
+export const REFUSE_LINK_FLAG: number = CAN_REFUSE_LINKS ? constants.O_NOFOLLOW : 0
+
+/**
+ * The same bounded read as {@link readBoundedFile} through one synchronous
+ * handle: open, fstat, read, close. A link in the last path component is
+ * refused at open (`O_NOFOLLOW`), so a link swapped in after a walk listed the
+ * file cannot be followed. Callers yield between files; one file is at most
+ * `maximum` bytes; a file whose size is over it answers `'too-large'`. With a
+ * `scratch` buffer, a file that fits one chunk comes back as a view of it, valid
+ * until the next call. Requires {@link CAN_REFUSE_LINKS}.
+ *
+ * With `syncMaximum`, a file whose size is over it (and within `maximum`) is not
+ * read here: the answer is `'large'`, so the caller reads it through
+ * {@link readBoundedFile}, which does not hold the event loop for the whole file.
+ */
+export function readBoundedFileSync(
+	path: string,
+	maximum: number,
+	scratch?: Buffer,
+	syncMaximum = maximum,
+): Buffer | 'too-large' | 'large' {
+	const fd = openSync(path, constants.O_RDONLY | constants.O_NONBLOCK | constants.O_NOFOLLOW)
+	try {
+		const info = fstatSync(fd)
+		if (!info.isFile()) throw new Error('Read requires a regular file.')
+		if (info.size > maximum) return 'too-large'
+		if (info.size > syncMaximum) return 'large'
+		const chunks: Buffer[] = []
+		let offset = 0
+		for (;;) {
+			const length = Math.min(CHUNK_BYTES, maximum + 1 - offset)
+			const buffer =
+				offset === 0 && scratch && scratch.length >= length ? scratch : Buffer.allocUnsafe(length)
+			let used = 0
+			while (used < length) {
+				const bytesRead = readSync(fd, buffer, used, length - used, offset + used)
+				if (bytesRead === 0) break
+				used += bytesRead
+			}
+			offset += used
+			if (offset > maximum) throw inputTooLarge(maximum)
+			chunks.push(buffer.subarray(0, used))
+			if (used < length)
+				return chunks.length === 1 ? (chunks[0] as Buffer) : Buffer.concat(chunks, offset)
+		}
+	} finally {
+		closeSync(fd)
+	}
+}

package/src/tools/builtins/grep.ts

@@ -1,5 +1,5 @@
 import { stat } from 'node:fs/promises'
-import { isAbsolute, join, relative, sep } from 'node:path'
+import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path'
 import { z } from 'zod'
 import { walkFilesLocally } from '../../sandbox/file-walk.js'
 import type { SandboxWalkFilesOptions, SandboxWalkSummary } from '../../types/sandbox/index.js'
@@ -7,6 +7,7 @@ import type { ToolContext } from '../../types/tool/index.js'
 import { subscribeToAbort } from '../../utils/abort.js'
 import { defineTool } from '../defineTool.js'
 import {
+	canonicalize,
 	resolveWithin,
 	resolveWithinAny,
 	resolveWithinAnyReal,
@@ -14,7 +15,13 @@ import {
 	toolRoots,
 } from '../paths.js'
 import { relativePosix, resolveWithinPosix } from '../posix-path.js'
-import { readBoundedChunks, readBoundedFile } from './bounded-file-read.js'
+import {
+	CAN_REFUSE_LINKS,
+	REFUSE_LINK_FLAG,
+	readBoundedChunks,
+	readBoundedFile,
+	readBoundedFileSync,
+} from './bounded-file-read.js'
 import {
 	MAX_FILES_SEARCHED,
 	SEARCH_BUDGET_MS,
@@ -24,6 +31,7 @@ import {
 	isReadDenied,
 	openSearchBudget,
 	plural,
+	readAhead,
 	stoppedNotice,
 } from './search-support.js'
 
@@ -49,6 +57,125 @@ async function readWithSignal(read: () => Promise<Buffer>, signal?: AbortSignal)
 	}
 }
 
+/** Longest the local read loop holds the event loop before it yields. */
+const READ_SLICE_MS = 8
+
+/**
+ * Largest file the local reader reads synchronously, by the size on the open
+ * handle. One synchronous read of a file is one uninterrupted block, which on a
+ * slow mount can last tens of milliseconds; a larger file is read through the
+ * asynchronous bounded read instead, which lets the event loop run between
+ * chunks.
+ */
+export const LOCAL_SYNC_READ_MAX_BYTES = 64 * 1024
+
+/**
+ * Reads files of a local search root. Each directory is resolved to its real
+ * path once and checked against the real root once; a file is then one open
+ * (links refused at the last component), fstat, read and close, all
+ * synchronous, with a yield to the event loop every few milliseconds. That is
+ * the same containment decision as resolving every file, because the walk
+ * lists only regular files in real directories and the open refuses a file
+ * that has become a link. Not used for a sandbox, nor where the platform cannot
+ * refuse links.
+ */
+function createLocalReader(root: string, signal: AbortSignal | undefined) {
+	const scratch = Buffer.allocUnsafe(64 * 1024)
+	const rootRealPromise = canonicalize(resolve(root))
+	const directories = new Map<string, Promise<string>>()
+	let sliceStart = performance.now()
+	const realDirectory = (directory: string): Promise<string> => {
+		let known = directories.get(directory)
+		if (!known) {
+			known = canonicalize(directory)
+			directories.set(directory, known)
+		}
+		return known
+	}
+	let rootReal: string | undefined
+	return {
+		async read(
+			filePath: string,
+		): Promise<{ buffer: Buffer | 'too-large' | 'gone'; contained: string }> {
+			if (performance.now() - sliceStart > READ_SLICE_MS) {
+				await new Promise<void>((done) => setImmediate(done))
+				sliceStart = performance.now()
+			}
+			signal?.throwIfAborted()
+			const directory = dirname(filePath)
+			const real = await realDirectory(directory)
+			rootReal ??= await rootRealPromise
+			const contained =
+				real === directory ? filePath : join(real, filePath.slice(directory.length + 1))
+			if (!(contained.startsWith(rootReal + sep) || contained === rootReal)) {
+				const rel = relative(rootReal, contained)
+				if (rel.startsWith('..') || isAbsolute(rel)) {
+					throw new Error('Path escapes the working directory through a link.')
+				}
+			}
+			signal?.throwIfAborted()
+			try {
+				const read = readBoundedFileSync(
+					contained,
+					MAX_FILE_SIZE,
+					scratch,
+					LOCAL_SYNC_READ_MAX_BYTES,
+				)
+				if (read !== 'large') return { buffer: read, contained }
+				// The handle that measured the file is closed; the asynchronous read opens it
+				// again with the same flags (links refused) and repeats every check.
+				const large = await readBoundedFile(
+					contained,
+					MAX_FILE_SIZE,
+					signal,
+					undefined,
+					REFUSE_LINK_FLAG,
+				)
+				sliceStart = performance.now()
+				return { buffer: large, contained }
+			} catch (error) {
+				// Gone since the walk listed it: absent, as a walk that ran later would find it.
+				const code = (error as NodeJS.ErrnoException).code
+				if (code === 'ENOENT' || code === 'ENOTDIR') return { buffer: 'gone', contained }
+				throw error
+			}
+		},
+	}
+}
+
+/** A walked path that is already inside `root` in normal form, or null when it needs the full check. */
+const PLAIN_TAIL = /(^|\/)\.{1,2}(\/|$)|\/\/|\/$/
+
+/**
+ * The fast form of what `resolveWithin` and the display path compute for the
+ * walk's own entries on a POSIX host: a path the walk built under `root` is
+ * already normal, so it is returned as it is and its display name is a
+ * prefix plus the tail. Anything else, and every other platform, takes the
+ * full computation.
+ */
+function createWalkedPaths(root: string, lexicalRoot: string, workingDirectory: string) {
+	if (sep !== '/') return undefined
+	const prefix = root === '/' ? '/' : `${root}/`
+	const rootRel = relative(workingDirectory, lexicalRoot)
+	const display = rootRel.startsWith('..') || isAbsolute(rootRel) ? undefined : rootRel
+	return {
+		within(path: string): string {
+			if (path.startsWith(prefix) && path.length > prefix.length) {
+				if (!PLAIN_TAIL.test(path.slice(prefix.length))) return path
+			}
+			return resolveWithin(root, path)
+		},
+		displayName(path: string): string | undefined {
+			if (display === undefined || !path.startsWith(prefix) || path.length === prefix.length) {
+				return undefined
+			}
+			const tail = path.slice(prefix.length)
+			if (PLAIN_TAIL.test(tail)) return undefined
+			return display === '' ? tail : `${display}/${tail}`
+		},
+	}
+}
+
 function relativeInclude(include: string, root: string, sandboxed: boolean): string {
 	let pattern = sandboxed ? include : include.split(sep).join('/')
 	if (pattern.split('/').includes('..'))
@@ -201,11 +328,7 @@ const MAX_FILE_SIZE = 5 * 1024 * 1024 // 5 MB — skip binaries/large files
 const BINARY_CHECK_BYTES = 512
 
 function isBinaryContent(buffer: Buffer): boolean {
-	const check = buffer.subarray(0, BINARY_CHECK_BYTES)
-	for (const byte of check) {
-		if (byte === 0) return true
-	}
-	return false
+	return buffer.subarray(0, BINARY_CHECK_BYTES).includes(0)
 }
 
 export const GrepTool = defineTool({
@@ -291,7 +414,11 @@ export const GrepTool = defineTool({
 				walkEnd = value
 			},
 		}
-		const entries = walkSandbox ? walkSandbox(root, options) : walkFilesLocally(root, options)
+		// A local walk is read ahead so its directory reads overlap with matching; a
+		// sandbox's walk is left as the adapter produces it.
+		const entries = walkSandbox
+			? walkSandbox(root, options)
+			: readAhead(walkFilesLocally(root, { ...options, withoutSizes: CAN_REFUSE_LINKS }))
 
 		// `execute` is reachable without the schema's defaults.
 		const mode = input.output_mode ?? 'files_with_matches'
@@ -321,12 +448,16 @@ export const GrepTool = defineTool({
 		const filesToCollect =
 			headLimit > 0 ? offset + Math.max(headLimit * 4, 1000) : Number.POSITIVE_INFINITY
 		let enough = false
+		const walked = sandbox
+			? undefined
+			: createWalkedPaths(root, lexicalRoot, context.workingDirectory)
+		const localReader = !sandbox && CAN_REFUSE_LINKS ? createLocalReader(root, signal) : undefined
 		try {
 			for await (const entry of entries) {
 				signal?.throwIfAborted()
 				const filePath = sandbox
 					? resolveWithinPosix(root, entry.path)
-					: resolveWithin(root, entry.path)
+					: (walked?.within(entry.path) ?? resolveWithin(root, entry.path))
 				if (filesSearched >= MAX_FILES_SEARCHED) {
 					fileLimit = true
 					break
@@ -339,21 +470,34 @@ export const GrepTool = defineTool({
 				let content: string
 				let contained: string | undefined
 				try {
-					const buffer = await readWithSignal(
-						() =>
-							sandbox
-								? readBoundedChunks(
-										(offset, length) =>
-											sandbox.readFile(filePath, { offset, length, signal: signal }),
-										MAX_FILE_SIZE,
-										signal,
-									)
-								: resolveWithinReal(root, filePath).then((path) => {
-										contained = path
-										return readBoundedFile(path, MAX_FILE_SIZE, signal)
-									}),
-						signal,
-					)
+					let buffer: Buffer
+					if (localReader) {
+						const read = await localReader.read(filePath)
+						contained = read.contained
+						// Over the cap by the size on the open handle: skipped, not unreadable.
+						if (read.buffer === 'too-large') continue
+						// Gone since the walk listed it: as if the walk had not listed it.
+						if (read.buffer === 'gone') {
+							filesSearched--
+							continue
+						}
+						buffer = read.buffer
+					} else
+						buffer = await readWithSignal(
+							() =>
+								sandbox
+									? readBoundedChunks(
+											(offset, length) =>
+												sandbox.readFile(filePath, { offset, length, signal: signal }),
+											MAX_FILE_SIZE,
+											signal,
+										)
+									: resolveWithinReal(root, filePath).then((path) => {
+											contained = path
+											return readBoundedFile(path, MAX_FILE_SIZE, signal)
+										}),
+							signal,
+						)
 					signal?.throwIfAborted()
 					if (buffer.length > MAX_FILE_SIZE || isBinaryContent(buffer)) continue
 					content = buffer.toString('utf-8')
@@ -367,7 +511,8 @@ export const GrepTool = defineTool({
 				let fileMatches = 0
 				const relPath = sandbox
 					? relativePosix(sandbox.rootDir, filePath)
-					: hostDisplayPath(context.workingDirectory, join(lexicalRoot, relative(root, filePath)))
+					: (walked?.displayName(filePath) ??
+						hostDisplayPath(context.workingDirectory, join(lexicalRoot, relative(root, filePath))))
 
 				if (mode === 'content' && input.only_matching) {
 					for (const fragment of matchingFragments(content, regex, multiline)) {
@@ -580,7 +725,8 @@ function lineHits(
 ): [number, number][] {
 	const hits: [number, number][] = []
 	for (let i = 0; i < lines.length; i++) {
-		context.abortSignal?.throwIfAborted()
+		// A cancellation is seen within a few hundred lines; per line it cost more than the match.
+		if ((i & 255) === 0) context.abortSignal?.throwIfAborted()
 		regex.lastIndex = 0
 		if (regex.test(lines[i] ?? '')) hits.push([i, i])
 	}

package/src/tools/builtins/search-support.ts

@@ -105,3 +105,65 @@ export function isReadDenied(
 	if (relative === '..' || relative.startsWith('../') || posix.isAbsolute(relative)) return true
 	return context.readDenied(joinPath(context.workingDirectory, relative))
 }
+
+/**
+ * Pulls from `source` ahead of the consumer, up to `ahead` items, so a walk's
+ * directory reads overlap with the matching of files already listed. Order is
+ * unchanged. An error from the source is thrown to the consumer after the
+ * items listed before it; when the consumer stops, the source is closed (its
+ * own cleanup runs) before this returns.
+ */
+export async function* readAhead<T>(source: AsyncIterable<T>, ahead = 256): AsyncGenerator<T> {
+	const iterator = source[Symbol.asyncIterator]()
+	const buffer: T[] = []
+	let finished = false
+	let failure: { error: unknown } | undefined
+	let stop = false
+	let wakeConsumer: (() => void) | undefined
+	let wakeProducer: (() => void) | undefined
+	const notifyConsumer = () => {
+		const wake = wakeConsumer
+		wakeConsumer = undefined
+		wake?.()
+	}
+	const producer = (async () => {
+		try {
+			while (!stop) {
+				if (buffer.length >= ahead) {
+					await new Promise<void>((resolve) => {
+						wakeProducer = resolve
+					})
+					continue
+				}
+				const next = await iterator.next()
+				if (next.done) break
+				buffer.push(next.value)
+				notifyConsumer()
+			}
+		} catch (error) {
+			failure = { error }
+		}
+		finished = true
+		notifyConsumer()
+	})()
+	try {
+		for (;;) {
+			if (buffer.length > 0) {
+				const item = buffer.shift() as T
+				wakeProducer?.()
+				yield item
+				continue
+			}
+			if (failure) throw failure.error
+			if (finished) return
+			await new Promise<void>((resolve) => {
+				wakeConsumer = resolve
+			})
+		}
+	} finally {
+		stop = true
+		wakeProducer?.()
+		await producer
+		await iterator.return?.()
+	}
+}

package/src/types/tool/presentation.ts

@@ -220,6 +220,12 @@ export type ToolFacts =
 			readonly reason: ToolFailureReason
 			/** What it was about: a path, or the tool's name for `unknown_tool`. */
 			readonly subject?: string
+			/**
+			 * Why a refused call did not run, as one plain line the kernel wrote
+			 * (a policy rule, the command guard). Absent when no reason was
+			 * recorded; a host shows nothing for it rather than a guess.
+			 */
+			readonly note?: string
 	  }
 
 /** The same shapes, for what a call produced. */