@namzu/sdk 38.1.0 → 39.0.0

package/src/runtime/query/tool-output-budget.ts

@@ -1,6 +1,7 @@
 import { createHash } from 'node:crypto'
 import { mkdirSync, writeFileSync } from 'node:fs'
 import { join } from 'node:path'
+import { digest, spillManifest } from '../../store/evidence/format.js'
 
 /**
  * Model-visible size cap for a single tool result.
@@ -104,6 +105,8 @@ export interface ToolOutputBudgetResult {
 	readonly truncated: boolean
 	/** Where the full output was written, when it was. */
 	readonly spillPath?: string
+	/** Digest of the bounded chunk manifest, recorded at retention time. */
+	readonly spillIntegrity?: string
 }
 
 /**
@@ -155,6 +158,10 @@ export interface ApplyToolOutputBudgetOptions {
 	readonly toolUseId: string
 	readonly output: string
 	readonly maxChars: number
+	/** Optional condensed presentation; used only after authenticated retention of output. */
+	readonly preview?: string
+	/** Smaller preview only after the full output and its integrity manifest are saved. */
+	readonly retainedPreviewChars?: number
 	/** An omission notice that shares the text budget, never extends it. */
 	readonly notice?: string
 	/**
@@ -168,12 +175,10 @@ export interface ApplyToolOutputBudgetOptions {
 /**
  * Bound a tool result to the model-visible budget.
  *
- * Spilling beats truncating on every axis that matters: nothing is lost,
- * tokens are paid only if the agent decides the rest is worth re-reading,
- * and retrieval uses `read`/`grep` — tools it already has — rather than a
- * new affordance. A hosted agent runtime does the same thing above 100k
- * characters. Middle-elision is the fallback for a run with no directory
- * to write to.
+ * Retention keeps the original while bounding its model-visible preview.
+ * The host owns the recovery route and its permissions; a spill path is not
+ * proof that workspace read/grep tools can access it. Middle-elision is the
+ * fallback for a run with no directory to write to.
  *
  * The preview keeps head AND tail because the two ends carry different
  * information: the head has the schema/opening of a document, the tail has
@@ -194,26 +199,65 @@ export function applyToolOutputBudget(opts: ApplyToolOutputBudgetOptions): ToolO
 			: Math.max(0, limit - notice.length - (notice && output ? 2 : 0))
 	const withNotice = (text: string) => [text, notice].filter(Boolean).join('\n\n')
 
-	if (textBudget === undefined || originalLength <= textBudget) {
+	const preview =
+		opts.preview !== undefined && opts.preview.length < originalLength ? opts.preview : undefined
+	if (preview === undefined && (textBudget === undefined || originalLength <= textBudget)) {
 		return { output: withNotice(output), originalLength, truncated: false }
 	}
 
-	const spillPath = opts.spillDir
+	const retained = opts.spillDir
 		? spill(opts.spillDir, opts.toolUseId, output, opts.onError)
 		: undefined
 
+	const spillPath = retained?.path
 	const recovery = spillPath
 		? [
 				`${SPILL_MARKER} ${spillPath}`,
-				'Read a specific window with `read` (offset/limit) or search it with `grep`. Do NOT read it whole — that is what exceeded the budget.',
+				'This path identifies retained output, not the original input. Use the host-authorized retained-output recovery tools when available; the path does not grant filesystem access. A fresh observation of the original input cannot recover its earlier contents. If recovery is unavailable, report the missing detail; do not replay the original action.',
 			].join('\n')
 		: 'The full output was not retained. Use a saved artifact or a read-only observation; do not repeat a state-changing action to recover its output.'
+	const requestedPreview = opts.retainedPreviewChars
+	// Separate the spill threshold from the cost of carrying its preview on every
+	// later request. Failed retention must not silently discard additional text.
+	// A small configured preview still needs room for the durable recovery path.
+	const previewLimit =
+		retained?.integrity &&
+		requestedPreview !== undefined &&
+		Number.isFinite(requestedPreview) &&
+		requestedPreview > 0
+			? Math.min(limit as number, Math.floor(requestedPreview))
+			: (limit as number)
+	const previewTextBudget = Math.max(0, previewLimit - notice.length - (notice && output ? 2 : 0))
+	const minimumRecoveryChars = `\n[... omitted ...]\n${SPILL_MARKER} ${spillPath}\n`.length
+	const effectiveTextBudget =
+		previewTextBudget > minimumRecoveryChars ? previewTextBudget : textBudget
+	// Condensation is a display choice, not permission to discard evidence. The
+	// original can be under the ordinary cap while still losing distinct rows.
+	// Failed retention falls back to the original, bounded by the usual cap.
+	if (preview !== undefined && retained?.integrity) {
+		const condensed = [preview, recovery].filter(Boolean).join('\n\n')
+		if (effectiveTextBudget === undefined || condensed.length <= effectiveTextBudget) {
+			return {
+				output: withNotice(condensed),
+				originalLength,
+				truncated: true,
+				spillPath,
+				spillIntegrity: retained.integrity,
+			}
+		}
+	}
+	if (textBudget === undefined || originalLength <= textBudget) {
+		return { output: withNotice(output), originalLength, truncated: false }
+	}
 
 	return {
-		output: withNotice(boundedPreview(output, textBudget, opts.toolName, recovery)),
+		output: withNotice(
+			boundedPreview(output, effectiveTextBudget ?? textBudget, opts.toolName, recovery),
+		),
 		originalLength,
 		truncated: true,
 		...(spillPath ? { spillPath } : {}),
+		...(retained?.integrity ? { spillIntegrity: retained.integrity } : {}),
 	}
 }
 
@@ -222,7 +266,7 @@ function spill(
 	toolUseId: string,
 	content: string,
 	onError?: (message: string) => void,
-): string | undefined {
+): { path: string; integrity?: string } | undefined {
 	try {
 		// `0o700` on the directory and `0o600` on the file: a spilled output is
 		// routinely the largest and most sensitive thing a run produces — whole
@@ -250,8 +294,16 @@ function spill(
 		// so randomising the filename would add nothing this does not already
 		// have. Do not "improve" it back to a random name and a plain `w` —
 		// that trades a guarantee for a guess.
-		writeFileSync(path, content, { encoding: 'utf-8', flag: 'wx', mode: 0o600 })
-		return path
+		const bytes = Buffer.from(content, 'utf8')
+		writeFileSync(path, bytes, { flag: 'wx', mode: 0o600 })
+		try {
+			const manifest = spillManifest(bytes)
+			writeFileSync(`${path}.manifest.json`, manifest, { flag: 'wx', mode: 0o600 })
+			return { path, integrity: digest(manifest) }
+		} catch {
+			onError?.('The full output was retained, but its integrity manifest could not be written.')
+			return { path }
+		}
 	} catch (err) {
 		// A spill failure must never fail the tool call — the model still
 		// gets the preview, just without a path to recover the rest.

package/src/runtime/query/tooling.ts

@@ -47,8 +47,10 @@ export interface ToolingBootstrapConfig {
 	maxToolCalls?: number
 	readToolCallBudgetEvents?: () => Promise<readonly RunEvent[]>
 	maxToolOutputChars?: number
+	retainedToolPreviewChars?: number
 	maxToolContentBytes?: number
-	toolOutputDir?: string
+	captureRunEvidence?: import('../../types/tool/index.js').ToolContext['captureRunEvidence']
+	toolOutputDir?: string | (() => string | undefined)
 	repairToolCall?: RepairToolCall
 	/** Operator authorization shared with the direct-call review path. */
 	authorizationGate?: AuthorizationGate
@@ -79,6 +81,7 @@ export class ToolingBootstrap {
 				abortSignal: config.abortSignal,
 				allowedTools: config.allowedTools,
 				invocationState: config.invocationState,
+				captureRunEvidence: config.captureRunEvidence,
 				pluginManager: config.pluginManager,
 				...(config.backgroundJobs ? { backgroundJobs: config.backgroundJobs } : {}),
 				...(config.backgroundJobOwner ? { backgroundJobOwner: config.backgroundJobOwner } : {}),
@@ -98,6 +101,9 @@ export class ToolingBootstrap {
 				...(config.maxToolOutputChars !== undefined
 					? { maxToolOutputChars: config.maxToolOutputChars }
 					: {}),
+				...(config.retainedToolPreviewChars !== undefined
+					? { retainedToolPreviewChars: config.retainedToolPreviewChars }
+					: {}),
 				...(config.maxToolContentBytes !== undefined
 					? { maxToolContentBytes: config.maxToolContentBytes }
 					: {}),

package/src/scheduler/completion-inbox.ts

@@ -77,6 +77,7 @@ export class CompletionInbox {
 	 * takes one, and a host that owns a gateway naturally reuses it.
 	 */
 	private readonly ours = new Set<TaskId>()
+	private readonly recentOwned: TaskId[] = []
 	/**
 	 * Announcements that arrived before anyone said whose task it was.
 	 *
@@ -192,6 +193,8 @@ export class CompletionInbox {
 	launched(taskId: TaskId): void {
 		if (this.ours.has(taskId)) return
 		this.ours.add(taskId)
+		this.recentOwned.push(taskId)
+		if (this.recentOwned.length > 16) this.recentOwned.shift()
 
 		if (this.claimed.has(taskId) || this.unheard.has(taskId)) return
 
@@ -278,6 +281,28 @@ export class CompletionInbox {
 		return this.unheard.size > 0
 	}
 
+	/** Bounded, non-consuming scheduler observations. Delivery never establishes user-facing completion. */
+	describeOwnedWork(): string | undefined {
+		if (this.ours.size === 0) return
+		const ids = this.recentOwned
+		const tasks = ids.map((taskId) => {
+			let handle = this.unheard.get(taskId)
+			try {
+				handle = this.gateway?.getTask(taskId) ?? handle
+			} catch {
+				// A broken observation must not block execution or invent a task state.
+			}
+			return {
+				taskId,
+				state: handle?.state ?? 'unknown',
+				...(handle?.result?.status ? { runStatus: handle.result.status } : {}),
+				...(handle?.result?.stopReason ? { stopReason: handle.result.stopReason } : {}),
+				resultDelivery: this.claimed.has(taskId) ? 'delivered-to-history' : 'not-delivered',
+			}
+		})
+		return `Owned delegated work (current scheduler observations). Result delivery is not proof that you answered the original request. Answer the latest operator message and complete the requested synthesis of available results unless the operator cancelled or changed that request. Reuse delivered results when sufficient; do not relaunch completed work. If a delivered result is no longer visible, retrieve it by task ID. A terminal state with a non-end_turn stop reason is not successful task completion. Do not repeat a synthesis already given.\n${JSON.stringify({ tasks, omitted: this.ours.size - ids.length })}`
+	}
+
 	/**
 	 * Tasks this run launched that are still running.
 	 *
@@ -366,6 +391,7 @@ export class CompletionInbox {
 		this.unheard.clear()
 		this.outstanding.clear()
 		this.ours.clear()
+		this.recentOwned.length = 0
 		this.claimed.clear()
 		this.unowned.clear()
 		// Release anyone still waiting. A closed inbox would otherwise hold

package/src/store/evidence/compaction-archive.ts

@@ -0,0 +1,139 @@
+import { randomUUID } from 'node:crypto'
+import { lstat, mkdir, writeFile } from 'node:fs/promises'
+import { join } from 'node:path'
+import { z } from 'zod'
+import type { RunEvent } from '../../types/run/events.js'
+import type { CompactedToolMetadata } from './compaction-provenance.js'
+import { compactedTexts } from './compaction-text.js'
+import { digest, spillManifest } from './format.js'
+import { RECORD_BYTES, decode, noLinks, openEvidence, stamp } from './io.js'
+
+const integer = z.number().int().nonnegative().safe()
+const hash = z.string().regex(/^[a-f0-9]{64}$/)
+const role = z.enum(['system', 'user', 'assistant', 'tool'])
+export const compactionArchiveSchema = z.object({
+	type: z.literal('compaction_archive'),
+	runId: z.string().uuid(),
+	seq: integer.positive().optional(),
+	iteration: integer,
+	reason: z.enum(['threshold', 'overflow', 'manual']),
+	archive: z.object({
+		version: z.literal(1),
+		id: z.string().uuid(),
+		bytes: integer,
+		sha256: hash,
+		messageCount: integer,
+		parts: z
+			.array(
+				z.object({
+					role,
+					summary: z.literal(true).optional(),
+					manifest: hash,
+					toolName: z.string().min(1).max(1024).optional(),
+					isError: z.boolean().optional(),
+				}),
+			)
+			.max(8192),
+	}),
+})
+
+/** Internal storage encoding only; RunStore readers restore compaction_shed. */
+export function compactionArchiveDirectory(runDir: string, id: string): string {
+	return join(runDir, 'compaction-output', z.string().uuid().parse(id))
+}
+
+export function compactionPartPath(runDir: string, id: string, part: number): string {
+	return join(compactionArchiveDirectory(runDir, id), `${integer.parse(part)}.txt`)
+}
+
+/** Keep JSONL index records bounded while preserving whole original messages. */
+export async function retainCompactionRecord(
+	event: RunEvent,
+	runDir: string,
+): Promise<RunEvent | z.infer<typeof compactionArchiveSchema>> {
+	if (event.type !== 'compaction_shed') return event
+	const { messages: removed, type: _type, ...envelope } = event
+	const messageCount = removed.length
+	const original = Buffer.from(JSON.stringify(event.messages), 'utf8')
+	if (original.length <= 3 * 1024 * 1024) return event
+	const texts = compactedTexts(event.messages)
+	if (texts.length > 8192) throw new Error('Compaction archive exceeds 8192 textual parts.')
+	// The same 4096-chunk ceiling the reader enforces for retained tool text.
+	if (texts.some((part) => Buffer.byteLength(part.text, 'utf8') > 256 * 1024 * 1024))
+		throw new Error('Compaction archive text exceeds 256 MiB per part.')
+	const id = randomUUID()
+	const dir = compactionArchiveDirectory(runDir, id)
+	await noLinks(runDir)
+	await mkdir(join(runDir, 'compaction-output'), { recursive: true, mode: 0o700 })
+	await noLinks(join(runDir, 'compaction-output'))
+	await mkdir(dir, { mode: 0o700 })
+	await writeFile(join(dir, 'messages.json'), original, { flag: 'wx', mode: 0o600 })
+	const parts: ({
+		role: z.infer<typeof role>
+		manifest: string
+		summary?: true
+	} & CompactedToolMetadata)[] = []
+	for (const [part, value] of texts.entries()) {
+		const bytes = Buffer.from(value.text, 'utf8')
+		const manifest = spillManifest(bytes)
+		if (Buffer.byteLength(manifest, 'utf8') > RECORD_BYTES)
+			throw new Error('Compaction text manifest exceeds its 4 MiB read limit.')
+		const path = compactionPartPath(runDir, id, part)
+		await writeFile(path, bytes, { flag: 'wx', mode: 0o600 })
+		await writeFile(`${path}.manifest.json`, manifest, { flag: 'wx', mode: 0o600 })
+		parts.push({
+			role: value.role,
+			summary: value.summary,
+			manifest: digest(manifest),
+			toolName: value.toolName,
+			isError: value.isError,
+		})
+	}
+	return {
+		...envelope,
+		...compactionArchiveSchema.parse({
+			type: 'compaction_archive',
+			runId: event.runId,
+			seq: event.seq,
+			iteration: event.iteration,
+			reason: event.reason,
+			archive: {
+				version: 1,
+				id,
+				bytes: original.length,
+				sha256: digest(original),
+				messageCount,
+				parts,
+			},
+		}),
+	}
+}
+
+/** Full event readback is explicit and may load large bodies; search never calls this. */
+export async function restoreCompactionRecord(
+	record: Record<string, unknown>,
+	runDir: string,
+): Promise<Record<string, unknown>> {
+	if (record.type !== 'compaction_archive') return record
+	const saved = compactionArchiveSchema.parse(record)
+	const path = join(compactionArchiveDirectory(runDir, saved.archive.id), 'messages.json')
+	const file = await openEvidence(path)
+	try {
+		const before = await file.stat()
+		if (before.size !== saved.archive.bytes) throw new Error('Compaction archive size changed.')
+		const bytes = await file.readFile()
+		if (
+			stamp(before) !== stamp(await file.stat()) ||
+			stamp(before) !== stamp(await lstat(path)) ||
+			digest(bytes) !== saved.archive.sha256
+		)
+			throw new Error('Compaction archive bytes changed.')
+		const messages: unknown = JSON.parse(decode(bytes))
+		if (!Array.isArray(messages) || messages.length !== saved.archive.messageCount)
+			throw new Error('Compaction archive message count changed.')
+		const { archive: _archive, ...rest } = record
+		return { ...rest, type: 'compaction_shed', messages }
+	} finally {
+		await file.close()
+	}
+}

package/src/store/evidence/compaction-provenance.ts

@@ -0,0 +1,52 @@
+export interface CompactedToolMetadata {
+	toolName?: string
+	isError?: boolean
+}
+
+function record(value: unknown): Record<string, unknown> | undefined {
+	return value !== null && typeof value === 'object' && !Array.isArray(value)
+		? (value as Record<string, unknown>)
+		: undefined
+}
+
+/** A copy keeps its own source/time; this only identifies an unambiguous local tool pair. */
+export function compactedToolMetadata(messages: readonly unknown[]): CompactedToolMetadata[] {
+	const calls = new Map<string, { index: number; name?: string; count: number }>()
+	const results = new Map<string, number>()
+	for (const [index, value] of messages.entries()) {
+		const message = record(value)
+		if (message?.role === 'tool' && typeof message.toolCallId === 'string')
+			results.set(message.toolCallId, (results.get(message.toolCallId) ?? 0) + 1)
+		if (message?.role !== 'assistant' || !Array.isArray(message.toolCalls)) continue
+		for (const value of message.toolCalls) {
+			const call = record(value)
+			if (typeof call?.id !== 'string' || !call.id || call.id.length > 1024) continue
+			const name = call.type === 'function' ? record(call.function)?.name : undefined
+			calls.set(call.id, {
+				index,
+				name: typeof name === 'string' && name.length > 0 && name.length <= 1024 ? name : undefined,
+				count: (calls.get(call.id)?.count ?? 0) + 1,
+			})
+		}
+	}
+	let ownerIndex = -1
+	return messages.map((value, index) => {
+		const message = record(value)
+		if (message?.role !== 'tool') {
+			ownerIndex = index
+			return {}
+		}
+		const id = typeof message.toolCallId === 'string' ? message.toolCallId : undefined
+		const call = id === undefined ? undefined : calls.get(id)
+		return {
+			...(id !== undefined &&
+			call?.count === 1 &&
+			call.index === ownerIndex &&
+			results.get(id) === 1 &&
+			call.name
+				? { toolName: call.name }
+				: {}),
+			...(typeof message.isError === 'boolean' ? { isError: message.isError } : {}),
+		}
+	})
+}

package/src/store/evidence/compaction-text.ts

@@ -0,0 +1,61 @@
+import type { MessageRole } from '../../types/message/index.js'
+import { type CompactedToolMetadata, compactedToolMetadata } from './compaction-provenance.js'
+
+interface CompactedText extends CompactedToolMetadata {
+	role: MessageRole
+	text: string
+	summary?: true
+}
+
+function record(value: unknown): Record<string, unknown> | undefined {
+	return value !== null && typeof value === 'object' && !Array.isArray(value)
+		? (value as Record<string, unknown>)
+		: undefined
+}
+
+/**
+ * Exact text only: do not stringify binary blocks or invent separators.
+ * Plain-string parts keep their historical ordinals. Block text is appended
+ * after that prefix, in message/block order, so durable seq/part addresses
+ * from before block indexing still resolve to the same original text.
+ */
+export function compactedTexts(messages: unknown): CompactedText[] {
+	if (!Array.isArray(messages)) throw new Error('Invalid shed messages.')
+	const strings: CompactedText[] = []
+	const blocks: CompactedText[] = []
+	const metadata = compactedToolMetadata(messages)
+	for (const [index, value] of messages.entries()) {
+		const message = record(value)
+		if (
+			!message ||
+			typeof message.role !== 'string' ||
+			!['system', 'user', 'assistant', 'tool'].includes(message.role)
+		)
+			throw new Error('Invalid shed message.')
+		const role = message.role as MessageRole
+		if (typeof message.content === 'string') {
+			strings.push({
+				role,
+				text: message.content,
+				...metadata[index],
+				...(role === 'system' && record(message.source)?.type === 'compaction-summary'
+					? { summary: true as const }
+					: {}),
+			})
+		} else if (role === 'tool' && Array.isArray(message.content)) {
+			for (const value of message.content) {
+				const block = record(value)
+				if (block?.type === 'text' && typeof block.text === 'string')
+					blocks.push({ role, text: block.text, ...metadata[index] })
+				else if (
+					!block ||
+					(block.type !== 'image' && block.type !== 'document') ||
+					typeof block.data !== 'string' ||
+					typeof block.mediaType !== 'string'
+				)
+					throw new Error('Invalid shed tool content block.')
+			}
+		}
+	}
+	return strings.concat(blocks)
+}