@namzu/sdk 62.7.0 → 62.9.0

package/src/runtime/query/executor.ts

@@ -705,7 +705,7 @@ export type PriorToolResults = ReadonlyMap<string, PriorToolResult>
  * for administrator rights is named as that; everything else is a plain block.
  * Read from the call's own argument, never from the refusal's wording.
  */
-function refusalFacts(toolName: string, input: unknown): ToolFacts {
+function refusalFacts(toolName: string, input: unknown, reason?: string): ToolFacts {
 	const command =
 		input && typeof input === 'object' ? (input as { command?: unknown }).command : undefined
 	const elevated = typeof command === 'string' && /(^|[\s;&|(])(sudo|doas)\b/.test(command)
@@ -713,9 +713,31 @@ function refusalFacts(toolName: string, input: unknown): ToolFacts {
 		kind: 'failure',
 		reason: elevated ? 'needs_administrator' : 'blocked',
 		subject: toolName,
+		...(refusalNote(reason) ? { note: refusalNote(reason) } : {}),
 	}
 }
 
+const REFUSAL_NOTE_MAX = 200
+
+/**
+ * The refusal's own reason as one line for the person's view: the first
+ * non-empty line, the leading "Blocked:" label dropped, first letter up, bounded. The policy
+ * wrote it; nothing here reads the model's text.
+ */
+function refusalNote(reason: string | undefined): string | undefined {
+	const line = reason
+		?.split(/\r?\n/)
+		.map((part) => part.trim())
+		.find(Boolean)
+	if (!line) return undefined
+	const plain = line.replace(/^Blocked(?: by [^:]{1,60})?:\s*/i, '').trim()
+	if (!plain) return undefined
+	const sentence = `${plain.charAt(0).toUpperCase()}${plain.slice(1)}`
+	return sentence.length > REFUSAL_NOTE_MAX
+		? `${sentence.slice(0, REFUSAL_NOTE_MAX - 1)}…`
+		: sentence
+}
+
 export function deniedToolOutput(toolName: string, reason: string): string {
 	return `Error: Tool "${toolName}" was not executed. ${reason}`
 }
@@ -3633,7 +3655,7 @@ export class ToolExecutor {
 			...(declined ? { presentation: this.declinedPresentation(toolName, input, declined) } : {}),
 			// A refusal by policy says what kind it was; the person's own No is
 			// already carried by the presentation above.
-			...(declined ? {} : { facts: refusalFacts(toolName, input) }),
+			...(declined ? {} : { facts: refusalFacts(toolName, input, reason) }),
 			isError: true,
 		})
 

package/src/runtime/query/index.ts

@@ -2248,7 +2248,10 @@ export async function* query(params: QueryParams): AsyncGenerator<SessionEvent,
 				}
 				const pushRestored = (message: Message, ids: ReadonlyMap<Message, MessageId>): void => {
 					const messageId = ids.get(message)
-					ctx.recorder.pushMessage(message, messageId ? { messageId } : {})
+					ctx.recorder.pushMessage(message, {
+						...(messageId ? { messageId } : {}),
+						restored: true,
+					})
 				}
 
 				if (params.resumeFromCheckpoint) {

package/src/runtime/query/iteration/phases/compaction.ts

@@ -7,6 +7,7 @@ import { replaceableSummaries, summarizeWithModel } from '../../../../compaction
 import {
 	type CompactionPlan,
 	DEFAULT_SOFT_TARGET,
+	canShedSalienceWorkingSet,
 	planCompaction,
 	planSalienceWorkingSet,
 } from '../../../../compaction/plan.js'
@@ -787,6 +788,23 @@ async function runCompactionCheckInner(
 	const startAt = salience ? Math.min(config.softTarget ?? DEFAULT_SOFT_TARGET, trigger) : trigger
 	if (!options?.force && usage < startAt) return
 
+	// A pass that can shed nothing and has not reached the summary trigger
+	// does nothing but announce itself: two hooks, a started event and a log
+	// line around a plan with no action. From the soft target until the
+	// trigger it ran on every iteration of every turn once every narration
+	// was cut and every large result cleared. Not asked of a forced pass, of
+	// a turn with a reducer (it owns the reduction), or at the trigger, where
+	// the summary path is what the pass is for.
+	if (
+		salience &&
+		!options?.force &&
+		!ctx.contextReducer &&
+		usage < trigger &&
+		!canShedSalienceWorkingSet(ctx.recorder.messages, config)
+	) {
+		return
+	}
+
 	// Three failed passes in a row: stop proactive compaction for the rest of
 	// the turn. A pass the provider forced is never gated.
 	const breaker = compactionBreaker(ctx)

package/src/runtime/query/prepare-turn.ts

@@ -8,7 +8,7 @@ import {
 import { assertSessionLogAttribution } from '../../manager/session/attribution.js'
 import {
 	type RecordedMessage,
-	readEverRecordedMessages,
+	readEverRecordedMessageIndex,
 	readFoldedHistory,
 } from '../../manager/session/turn-recorder.js'
 import { resolveModelPricing } from '../../pricing/index.js'
@@ -23,6 +23,7 @@ import { withProviderRetry } from '../../provider/retry.js'
 import { withTokenBudget } from '../../provider/token-budget.js'
 import { resolveAttachments } from '../../store/attachment/index.js'
 import type { SessionTokenBudget } from '../../store/budget/index.js'
+import { foldSessionLog } from '../../store/session-log/incremental.js'
 import type { SessionLog } from '../../store/session-log/index.js'
 import { NAMZU } from '../../telemetry/attributes.js'
 import type { SerializedSpanContext } from '../../telemetry/attributes.js'
@@ -1163,7 +1164,8 @@ async function reconcileCallerMessages(
 	continuationMode: boolean,
 ): Promise<Message[]> {
 	const hasId = messages.some((message) => message.id !== undefined)
-	const recordedById = hasId ? await readEverRecordedMessages(log) : undefined
+	const everRecorded = hasId ? await readEverRecordedMessageIndex(log) : undefined
+	const recordedById = everRecorded?.latest
 	// Ids a caller message already claimed by direct lookup above: a fold
 	// entry that gave one of THESE out must not also be offered to the no-id
 	// alignment below, or a message that already reconciled by id could be
@@ -1181,7 +1183,13 @@ async function reconcileCallerMessages(
 		if (id !== undefined) {
 			const recorded = recordedById?.get(id)
 			if (recorded === undefined) throw staleCachedHistoryError('foreign', message)
-			if (!sameMessageContent(recorded, message)) throw staleCachedHistoryError('edited', message)
+			if (!sameMessageContent(recorded, message)) {
+				// A compaction pass edits a message under its own id; a host that
+				// cached the history before the pass still holds the first text.
+				// Answered from the reduction already read: no read of the log.
+				if (!(await everRecorded?.heldBeforeAnEdit(id, message)))
+					throw staleCachedHistoryError('edited', message)
+			}
 			claimedIds.add(id)
 			continue // already durable: the fold supplies it
 		}
@@ -1232,14 +1240,21 @@ async function assertTurnIdIsNew(
 	storage: SessionStorage,
 ): Promise<void> {
 	if (params.turnId === undefined) return
-	for await (const { record } of storage.log.read({ mode: 'tolerant' })) {
-		if (record.type === 'turn_started' && record.turnId === turnId) {
-			throw new NamzuError({
-				code: 'invalid_config',
-				message: `Turn ${turnId} already exists in session ${params.sessionId}. A new turn needs a new id; to continue that turn, pass resumeFromCheckpoint.`,
-				details: { sessionId: params.sessionId, turnId },
-			})
-		}
+	const exists = await foldSessionLog(storage.log, {
+		name: 'turn-ids',
+		mode: 'tolerant',
+		create: () => new Set<string>(),
+		apply: (ids, { record }) => {
+			if (record.type === 'turn_started') ids.add(record.turnId)
+		},
+		result: (ids) => ids.has(turnId),
+	})
+	if (exists) {
+		throw new NamzuError({
+			code: 'invalid_config',
+			message: `Turn ${turnId} already exists in session ${params.sessionId}. A new turn needs a new id; to continue that turn, pass resumeFromCheckpoint.`,
+			details: { sessionId: params.sessionId, turnId },
+		})
 	}
 }
 

package/src/runtime/query/session-storage.ts

@@ -14,6 +14,7 @@ import {
 	InMemorySessionCheckpointStore,
 	type SessionCheckpointStore,
 } from '../../store/checkpoint/index.js'
+import { foldSessionLog } from '../../store/session-log/incremental.js'
 import {
 	DiskSessionLog,
 	type InMemoryLogMedium,
@@ -175,7 +176,10 @@ export async function locateSession(
 	const parent = (await findSessionLocator(paths, parentSessionId)) ?? {
 		sessionId: parentSessionId,
 	}
-	return { sessionId, ancestors: [...(parent.ancestors ?? []), parent.sessionId] }
+	return {
+		sessionId,
+		ancestors: [...(parent.ancestors ?? []), parent.sessionId],
+	}
 }
 
 /** Deeper than any delegation tree a project allows; a guard against a cyclic tree on disk. */
@@ -234,7 +238,10 @@ async function listDirectory(
 /** `SessionPaths` for the working directory's project under `resolveNamzuHome()`. */
 export async function defaultSessionPaths(workingDirectory?: string): Promise<SessionPaths> {
 	const home = resolveNamzuHome()
-	const project = await ensureProject({ home, cwd: workingDirectory ?? process.cwd() })
+	const project = await ensureProject({
+		home,
+		cwd: workingDirectory ?? process.cwd(),
+	})
 	return new SessionPaths({ home, slug: project.slug })
 }
 
@@ -242,7 +249,9 @@ function heldState(log: InMemorySessionLog): HeldState {
 	const current = held.get(log.medium)
 	if (current) return current
 	const next: HeldState = {
-		checkpoints: new InMemorySessionCheckpointStore({ log: checkpointLogView(log) }),
+		checkpoints: new InMemorySessionCheckpointStore({
+			log: checkpointLogView(log),
+		}),
 		tokenBudgets: new InMemorySessionTokenBudgetStore(),
 	}
 	held.set(log.medium, next)
@@ -303,5 +312,35 @@ export function checkpointLogView(log: SessionLog): CheckpointLogView {
 			}
 			return open.values()
 		},
+		async pruneFacts(scope, checkpointIds) {
+			own(scope)
+			// Every committed checkpoint and every open decision, advanced from the
+			// last record the previous prune folded rather than read from byte 0:
+			// a prune runs every iteration of every turn.
+			return foldSessionLog(log, {
+				name: 'prune-facts',
+				mode: 'tolerant',
+				create: () => ({
+					written: new Map<CheckpointId, string>(),
+					open: new Map<string, CheckpointId>(),
+				}),
+				apply: ({ written, open }, { record }) => {
+					if (record.type === 'checkpoint_written')
+						written.set(record.checkpointId, record.docSha256)
+					if (record.type === 'decision_requested') open.set(record.decisionId, record.checkpointId)
+					if (record.type === 'decision_resolved' || record.type === 'decision_expired') {
+						open.delete(record.decisionId)
+					}
+				},
+				result: ({ written, open }) => {
+					const wanted = new Map<CheckpointId, string>()
+					for (const id of checkpointIds) {
+						const sha256 = written.get(id)
+						if (sha256 !== undefined) wanted.set(id, sha256)
+					}
+					return { written: wanted, open: [...open.values()] }
+				},
+			})
+		},
 	}
 }

package/src/sandbox/file-walk.ts

@@ -15,6 +15,9 @@ import { VCS_NAMES, createIgnoreStack } from './ignore-matcher.js'
 /** Entries a walk examines before it stops with what it has: a safety valve, not a budget. */
 export const DEFAULT_MAX_VISITED_ENTRIES = 500_000
 
+/** Longest the walk holds the event loop between yields. */
+const WALK_SLICE_MS = 8
+
 const GLOBSTAR = Symbol('globstar')
 type Segment = string | { source: string; flags: string } | typeof GLOBSTAR
 
@@ -185,6 +188,7 @@ async function* executeWalk(
 	plan: WalkPlan,
 	signal?: AbortSignal,
 	onSummary?: (summary: SandboxWalkSummary) => void,
+	withoutSizes = false,
 ): AsyncGenerator<SandboxFileEntry> {
 	const fs = await import('node:fs/promises')
 	const path = await import('node:path')
@@ -196,6 +200,7 @@ async function* executeWalk(
 		/** Whether leaving this directory also drops its ignore layer. */
 		layered: boolean
 	}[] = []
+	let sliceStart = performance.now()
 	let visited = 0
 	let emitted = 0
 	let ignoredDirectories = 0
@@ -327,8 +332,22 @@ async function* executeWalk(
 		while (open.length > 0) {
 			check()
 			const frame = open[open.length - 1] as (typeof open)[number]
-			const entry = await wait(frame.dir.read())
-			check()
+			let entry: import('node:fs').Dirent | null
+			if (withoutSizes) {
+				// A local search consumer reads a directory's buffered entries
+				// synchronously: one promise per entry costs more than the listing.
+				// The loop yields on a time slice, and a blocked read cannot be
+				// raced, which only the default path promises.
+				if (performance.now() - sliceStart > WALK_SLICE_MS) {
+					await new Promise<void>((done) => setImmediate(done))
+					sliceStart = performance.now()
+				}
+				check()
+				entry = frame.dir.readSync()
+			} else {
+				entry = await wait(frame.dir.read())
+				check()
+			}
 			if (!entry) {
 				open.pop()
 				if (frame.layered) ignore?.leave()
@@ -351,6 +370,13 @@ async function* executeWalk(
 			}
 			if (!entry.isFile() || !matchers.some((matcher) => matcher.test(relative))) continue
 			if (ignore?.isIgnored(absolute, false)) continue
+			if (withoutSizes) {
+				// The caller opens the file anyway and reads its size from the handle.
+				emitted += 1
+				yield { path: absolute, size: 0 }
+				if (emitted >= plan.maxEntries) return
+				continue
+			}
 			let info: import('node:fs').Stats
 			try {
 				info = await wait(fs.lstat(absolute))
@@ -384,10 +410,21 @@ function globalIgnoreFiles(): string[] {
 /** Internal host path; the caller first resolves its authorized filesystem root. */
 export async function* walkFilesLocally(
 	rootPath: string,
-	options: SandboxWalkFilesOptions,
+	options: SandboxWalkFilesOptions & {
+		/**
+		 * Skip the per-file size read: entries carry `size: 0`. For a caller that
+		 * opens every file itself and takes the size from the handle.
+		 */
+		readonly withoutSizes?: boolean
+	},
 ): AsyncGenerator<SandboxFileEntry> {
 	options.signal?.throwIfAborted()
-	yield* executeWalk(planWalk(rootPath, options), options.signal, options.onSummary)
+	yield* executeWalk(
+		planWalk(rootPath, options),
+		options.signal,
+		options.onSummary,
+		options.withoutSizes === true,
+	)
 }
 
 /** The existing sandbox execution seam; no separate worker protocol is needed. */

package/src/store/checkpoint/contract.ts

@@ -62,6 +62,21 @@ export interface CheckpointLogView {
 	 * `decision_resolved` or `decision_expired` yet. `prune` never deletes one.
 	 */
 	openDecisionCheckpoints(scope: CheckpointScope): Promise<Iterable<CheckpointId>>
+	/**
+	 * Both answers `prune` needs in one read of the log: for each of
+	 * `checkpointIds`, what `writtenDocSha256` would return (absent from
+	 * `written` when there is no record), and what `openDecisionCheckpoints`
+	 * would return. Optional: a view without it is asked the two questions
+	 * separately, one walk of the log each, which costs a walk per stored
+	 * checkpoint on every prune.
+	 */
+	pruneFacts?(
+		scope: CheckpointScope,
+		checkpointIds: readonly CheckpointId[],
+	): Promise<{
+		readonly written: ReadonlyMap<CheckpointId, string>
+		readonly open: Iterable<CheckpointId>
+	}>
 }
 
 /** Why a restore refused a checkpoint. */

package/src/store/checkpoint/disk.ts

@@ -1,4 +1,4 @@
-import { link, mkdir, open, readFile, readdir, unlink } from 'node:fs/promises'
+import { link, mkdir, open, readFile, readdir, stat, unlink } from 'node:fs/promises'
 import type { FileHandle } from 'node:fs/promises'
 import { dirname } from 'node:path'
 import type { SessionLocator, SessionPaths } from '../../session/paths.js'
@@ -38,6 +38,17 @@ export interface DiskSessionCheckpointStoreOptions {
 
 const DOCUMENT = /^(.+)\.json$/
 
+/**
+ * Which turn a checkpoint document on disk belongs to, as last read, with the
+ * size and mtime it had then. A session keeps its checkpoints for every turn
+ * in one directory, so listing one turn's used to read and parse every
+ * document of the session, and the directory only grows. A document is not
+ * read again while a `stat` says it is the one that was; one that belonged to
+ * another turn is skipped on that evidence alone.
+ */
+const documentTurns = new Map<string, { turnId: string; size: number; mtimeMs: number }>()
+const DOCUMENT_TURNS_KEPT = 20_000
+
 /**
  * {@link SessionCheckpointStore} at `<session-dir>/checkpoints/<id>.json`.
  *
@@ -145,10 +156,31 @@ export class DiskSessionCheckpointStore implements SessionCheckpointStore {
 			// Sidecars (`<id>.json.<pid>.<n>.<hex>.tmp`) and strangers are not documents.
 			const id = DOCUMENT.exec(name)?.[1]
 			if (id === undefined || !isEntityId(id, 'checkpoint')) continue
-			const text = await readIfPresent(this.#paths.checkpointFile(locator, id))
+			const path = this.#paths.checkpointFile(locator, id)
+			const seen = await stat(path).catch((error: NodeJS.ErrnoException) => {
+				if (error.code === 'ENOENT') return null
+				throw error
+			})
 			// Deleted between the listing and the read: a concurrent prune, not damage.
+			if (seen === null) continue
+			const known = documentTurns.get(path)
+			if (
+				known !== undefined &&
+				known.turnId !== checked.turnId &&
+				known.size === seen.size &&
+				known.mtimeMs === seen.mtimeMs
+			) {
+				continue
+			}
+			const text = await readIfPresent(path)
 			if (text === null) continue
 			const checkpoint = parseStoredCheckpoint(text, checked, id)
+			if (documentTurns.size >= DOCUMENT_TURNS_KEPT) documentTurns.clear()
+			documentTurns.set(path, {
+				turnId: checkpoint.turnId,
+				size: seen.size,
+				mtimeMs: seen.mtimeMs,
+			})
 			if (checkpoint.turnId === checked.turnId) checkpoints.push(checkpoint)
 		}
 		return checkpoints.sort(compareCheckpoints)

package/src/store/checkpoint/prune.ts

@@ -79,6 +79,19 @@ export async function selectStoredCheckpointsToPrune(
 	if (!Number.isSafeInteger(keepLast) || keepLast < 0) {
 		throw new RangeError(`keepLast must be a nonnegative integer, got ${String(keepLast)}`)
 	}
+	if (log.pruneFacts) {
+		// One read of the log answers both questions for every stored
+		// checkpoint, instead of one full walk each.
+		const facts = await log.pruneFacts(
+			scope,
+			stored.map((checkpoint) => checkpoint.checkpointId),
+		)
+		return selectSessionCheckpointsToPrune(
+			stored.filter((checkpoint) => facts.written.has(checkpoint.checkpointId)),
+			keepLast,
+			new Set(facts.open),
+		)
+	}
 	const committed: Checkpoint[] = []
 	for (const checkpoint of stored) {
 		if ((await log.writtenDocSha256(scope, checkpoint.checkpointId)) !== null) {

package/src/store/evidence/disk.ts

@@ -157,22 +157,64 @@ export function createSessionEvidenceSource(
 	})
 }
 
+/** What a scan of a session log's complete records, from byte 0 up to `consumed`, found. */
+interface SessionScan {
+	readonly identity: string
+	/** Offset just past the last complete record scanned. */
+	readonly consumed: number
+	readonly first: Buffer
+	readonly active: string | undefined
+	readonly turnClosed: boolean
+}
+
+/**
+ * Scans already done, by log file and scope, so that an access reads only the
+ * records appended since the last one instead of parsing the whole log again.
+ * A session log only grows, and every search page makes several accesses.
+ */
+const SCAN_CACHE_LIMIT = 64
+const scanCache = new Map<string, SessionScan>()
+
+function remember(key: string, scan: SessionScan): void {
+	scanCache.delete(key)
+	scanCache.set(key, scan)
+	if (scanCache.size > SCAN_CACHE_LIMIT) {
+		const oldest = scanCache.keys().next().value
+		if (oldest !== undefined) scanCache.delete(oldest)
+	}
+}
+
 /**
  * The first record of a session log (its `session_started`) and whether the
  * scope is closed: no turn of the session active, or — for a scope narrowed
  * to one turn — that turn settled.
+ *
+ * The scan resumes from the one a previous access left in `scanCache` when it
+ * is of the same file (same identity, first record unchanged on disk) and
+ * ended at or before `size`; otherwise it starts again from byte 0.
  */
 async function sessionState(
 	handle: FileHandle,
 	size: number,
 	scope: z.infer<typeof scopeSchema>,
+	cacheKey: string,
+	identity: string,
 ): Promise<{ first: Buffer; closed: boolean }> {
 	const unmetered: EvidenceBudget = { bytes: 0, limit: Number.MAX_SAFE_INTEGER }
 	let first: Buffer | undefined
 	let active: string | undefined
 	let turnClosed = false
-	let buffered = Buffer.alloc(0)
 	let offset = 0
+	const cached = scanCache.get(cacheKey)
+	if (cached && cached.identity === identity && cached.consumed <= size) {
+		const head = await readBytes(handle, 0, cached.first.length, unmetered)
+		if (head.equals(cached.first)) {
+			;({ first, active, turnClosed } = cached)
+			offset = cached.consumed
+		}
+	}
+	let consumed = offset
+	let buffered = Buffer.alloc(0)
 	while (offset < size || buffered.length > 0) {
 		let newline = buffered.indexOf(10)
 		if (newline < 0) {
@@ -186,6 +228,7 @@ async function sessionState(
 		const raw = buffered.subarray(0, newline + 1)
 		buffered = buffered.subarray(newline + 1)
 		newline = -1
+		consumed += raw.length
 		const record = JSON.parse(decode(raw)) as Record<string, unknown>
 		if (!first) {
 			if (
@@ -206,6 +249,12 @@ async function sessionState(
 		}
 	}
 	if (!first) throw new Error('The session log is empty; evidence is incomplete.')
+	// A source anchored earlier in the log scans less than another access has;
+	// keep the longer scan for the next one.
+	const kept = scanCache.get(cacheKey)
+	if (!kept || kept.identity !== identity || kept.consumed <= consumed) {
+		remember(cacheKey, { identity, consumed, first, active, turnClosed })
+	}
 	return { first, closed: scope.turnId !== undefined ? turnClosed : active === undefined }
 }
 
@@ -258,7 +307,13 @@ function createSource(
 						? await completePrefix(handle, before.size, budget)
 						: before.size
 			}
-			const state = await sessionState(handle, size, scope)
+			const state = await sessionState(
+				handle,
+				size,
+				scope,
+				`${logPath}\0${scopeKey}`,
+				fileIdentity(before),
+			)
 			if (!state.closed && consistency === 'closed') {
 				throw new Error('Evidence scope has an active turn; it is not closed.')
 			}