@namzu/sdk 26.0.0 → 27.0.0

package/src/connector/mcp/prompt-adapter.ts

@@ -34,10 +34,16 @@ import type { MCPClient } from './client.js'
 /**
  * Marks where a remote party's words begin and end.
  *
- * A prompt is composed by a SERVER. Untrusted content arriving through a
- * tool result is the standard prompt-injection surface, and the mitigation
- * that survives contact is saying plainly whose words these are — an
- * unlabelled block reads exactly like the agent's own instructions.
+ * A prompt is composed by a SERVER. Untrusted content arriving this way is
+ * the standard prompt-injection surface, and an unlabelled block reads
+ * exactly like the agent's own instructions — so this says whose words
+ * they are.
+ *
+ * Marking, not stopping. See `tools/untrusted-envelope.ts` for the
+ * measurement: delimiting reports near-zero attack success on a static
+ * benchmark and above 95% once the attacker adapts (arXiv:2510.09023).
+ * This paragraph used to call it "the mitigation that survives contact",
+ * which was the same overstatement in a second file.
  */
 export function renderPromptMessages(
 	serverName: string,

package/src/connector/mcp/server-stdio.ts

@@ -0,0 +1,137 @@
+import type { Readable, Writable } from 'node:stream'
+
+import type { MCPJsonRpcMessage, MCPTransport } from '../../types/connector/mcp.js'
+import { type Logger, getRootLogger } from '../../utils/logger.js'
+
+/**
+ * The server half of stdio, so `MCPServer` has something to run on.
+ *
+ * `MCPServer` is a complete implementation — `initialize`, `tools/list`,
+ * `tools/call`, resource and prompt providers — and nothing anywhere
+ * constructed one, because every transport in this directory is the
+ * CLIENT side. `StdioTransport` spawns a child and talks to its streams;
+ * this reads the streams THIS process was given. Same interface, opposite
+ * end of the pipe.
+ *
+ * Stdio is the first transport rather than an afterthought: a client
+ * spawns the server as a child process, so there is no port, no bind
+ * address and no inbound authentication question to get wrong. The
+ * boundary is the process, and the process was started by the client.
+ *
+ * **stdout belongs to the protocol.** A single stray `console.log`
+ * anywhere in the process corrupts the message stream, and the symptom is
+ * a client that reports malformed JSON rather than anything naming the
+ * culprit. This repository's logger already writes to stderr, which is
+ * what makes this transport safe to add; keep it that way.
+ */
+export class ServerStdioTransport implements MCPTransport {
+	private readonly input: Readable
+	private readonly output: Writable
+	private buffer = ''
+	private connected = false
+	private readonly log: Logger
+
+	private messageHandler?: (message: MCPJsonRpcMessage) => void
+	private closeHandler?: () => void
+	private errorHandler?: (error: Error) => void
+
+	private readonly onData = (chunk: Buffer | string): void => this.consume(chunk)
+	private readonly onEnd = (): void => {
+		this.connected = false
+		this.closeHandler?.()
+	}
+	private readonly onStreamError = (err: Error): void => this.errorHandler?.(err)
+
+	/**
+	 * Streams are injected rather than read from `process` directly so a
+	 * test can drive this without owning the process's own stdio — a test
+	 * that replaced `process.stdin` would break every other test sharing
+	 * the runner.
+	 */
+	constructor(streams?: { input?: Readable; output?: Writable }) {
+		this.input = streams?.input ?? process.stdin
+		this.output = streams?.output ?? process.stdout
+		this.log = getRootLogger().child({ component: 'ServerStdioTransport' })
+	}
+
+	async connect(): Promise<void> {
+		if (this.connected) return
+		this.input.setEncoding?.('utf8')
+		this.input.on('data', this.onData)
+		this.input.on('end', this.onEnd)
+		this.input.on('error', this.onStreamError)
+		this.connected = true
+		this.log.info('MCP stdio server listening on this process')
+	}
+
+	async close(): Promise<void> {
+		if (!this.connected) return
+		this.input.off('data', this.onData)
+		this.input.off('end', this.onEnd)
+		this.input.off('error', this.onStreamError)
+		this.connected = false
+		this.closeHandler?.()
+	}
+
+	async send(message: MCPJsonRpcMessage): Promise<void> {
+		// Newline-delimited JSON, one message per line. A message carrying a
+		// literal newline would split into two unparseable halves, so it is
+		// stripped by `JSON.stringify` escaping rather than by trusting the
+		// payload.
+		this.output.write(`${JSON.stringify(message)}\n`)
+	}
+
+	onMessage(handler: (message: MCPJsonRpcMessage) => void): void {
+		this.messageHandler = handler
+	}
+
+	onClose(handler: () => void): void {
+		this.closeHandler = handler
+	}
+
+	onError(handler: (error: Error) => void): void {
+		this.errorHandler = handler
+	}
+
+	isConnected(): boolean {
+		return this.connected
+	}
+
+	/**
+	 * A chunk is not a message. Reads arrive at whatever size the pipe
+	 * hands over, so one read can carry half a message, several messages,
+	 * or the tail of one and the head of the next. Buffering until a
+	 * newline is the whole protocol framing.
+	 */
+	private consume(chunk: Buffer | string): void {
+		this.buffer += typeof chunk === 'string' ? chunk : chunk.toString('utf8')
+
+		let newline = this.buffer.indexOf('\n')
+		while (newline !== -1) {
+			const line = this.buffer.slice(0, newline).trim()
+			this.buffer = this.buffer.slice(newline + 1)
+			if (line) this.deliver(line)
+			newline = this.buffer.indexOf('\n')
+		}
+	}
+
+	private deliver(line: string): void {
+		let message: MCPJsonRpcMessage
+		try {
+			message = JSON.parse(line) as MCPJsonRpcMessage
+		} catch (err) {
+			// Reported, not thrown, and not fatal. A client that sends one bad
+			// line has not ended the session, and killing the transport over it
+			// would take down every other conversation on this pipe. The
+			// alternative — silence — is worse: a request that vanishes looks
+			// to the client exactly like a server that hung.
+			this.errorHandler?.(
+				new Error(
+					`MCP stdio server could not parse a message: ${err instanceof Error ? err.message : String(err)}`,
+				),
+			)
+			return
+		}
+		this.messageHandler?.(message)
+	}
+}

package/src/eval/experiment.ts

@@ -7,6 +7,8 @@ import type {
 	Score,
 	Scorer,
 } from './types.js'
+import type { ScoreUncertainty } from './uncertainty.js'
+import { describeUncertainty, uncertaintyOf } from './uncertainty.js'
 
 export interface ExperimentConfig<TInput = unknown> {
 	name: string
@@ -151,6 +153,10 @@ export async function runExperiment<TInput>(
 		name: config.name,
 		cases: settled,
 		mean,
+		// Over the same cases the mean is over. Computing spread across a
+		// different denominator than the average it qualifies would produce
+		// an interval that does not belong to the number beside it.
+		uncertainty: uncertaintyOf(scored.map((r) => r.mean)),
 		passed: settled.filter((r) => r.status === 'passed').length,
 		failed: settled.filter((r) => r.status === 'failed').length,
 		inconclusive: settled.filter((r) => r.status === 'inconclusive').length,
@@ -278,9 +284,33 @@ function meanByScorer(results: readonly CaseResult[]): Record<string, number> {
  * Failures print their scorer reasons, because a CI log that says
  * "0.62" is a log that sends someone back to reproduce it by hand.
  */
+/**
+ * Uncertainty for a report that did not carry its own.
+ *
+ * A suite file is loaded at runtime and may be plain JavaScript, so a
+ * report can reach here hand-built — the type cannot stop it. Deriving
+ * from the cases it does carry is better than either alternative:
+ * printing the mean alone leaves the reader where they started, and
+ * refusing to format would turn a missing convenience into a broken
+ * command.
+ *
+ * Uses the same exclusion `runExperiment` uses, so a derived interval and
+ * a carried one are the same number rather than two conventions.
+ */
+function derivedUncertainty(report: ExperimentReport): ScoreUncertainty {
+	return uncertaintyOf(report.cases.filter((c) => c.status !== 'inconclusive').map((c) => c.mean))
+}
+
 export function formatReport(report: ExperimentReport): string {
 	const lines: string[] = [
 		`${report.name}: ${report.passed}/${report.cases.length} passed (mean ${report.mean.toFixed(2)}) in ${report.durationMs}ms`,
+		// On its own line and always printed, including when the interval is
+		// undefined. A mean printed alone is the thing that has been
+		// over-read: two runs three points apart look like a difference, and
+		// at the n a hand-built suite has they are usually the same run
+		// twice. Computing the interval and not showing it would leave the
+		// reader exactly where they started.
+		`  ${describeUncertainty(report.mean, report.uncertainty ?? derivedUncertainty(report))}`,
 		'',
 	]
 

package/src/eval/index.ts

@@ -10,6 +10,8 @@ export {
 	stepBudgetScorer,
 	trajectoryScorer,
 } from './scorers.js'
+export { describeUncertainty, uncertaintyOf } from './uncertainty.js'
+export type { ScoreUncertainty } from './uncertainty.js'
 export type {
 	CaseResult,
 	CaseStatus,

package/src/eval/types.ts

@@ -1,4 +1,5 @@
 import type { StepResult } from '../types/run/step.js'
+import type { ScoreUncertainty } from './uncertainty.js'
 
 /**
  * One case in a dataset: an input, and what a good run looks like.
@@ -143,6 +144,22 @@ export interface ExperimentReport {
 	cases: readonly CaseResult[]
 	/** Mean score across the cases that produced one. */
 	mean: number
+	/**
+	 * How much of {@link mean} is signal.
+	 *
+	 * A mean on its own has been read as a result, and at the n a
+	 * hand-built suite has it usually is not one: two runs three points
+	 * apart are normally the same run twice.
+	 *
+	 * Optional, and deliberately so after trying it the other way. A suite
+	 * file is loaded at runtime and may be plain JavaScript, so a required
+	 * field is not enforced at the boundary that matters — it buys type
+	 * safety for one kind of consumer and a crash for the other. Producers
+	 * that go through `runExperiment` always set it; `formatReport` derives
+	 * it from {@link cases} when a hand-built report does not, so no report
+	 * is printed without an interval either way.
+	 */
+	uncertainty?: ScoreUncertainty
 	passed: number
 	failed: number
 	/** Cases where no scorer could produce a judgement. */

package/src/eval/uncertainty.ts

@@ -0,0 +1,124 @@
+/**
+ * How much of a score is signal.
+ *
+ * A suite reported a mean and nothing else, so two runs differing by three
+ * points read as a difference. At the n a hand-built suite has, that is
+ * usually noise, and there was no number on the page that would have said
+ * so.
+ *
+ * Evan Miller, "Adding Error Bars to Evals" (arXiv:2411.00640), is the
+ * reference. Two of its results shape what is and is not computed here.
+ */
+
+/**
+ * Two-sided 95% critical values of Student's t, by degrees of freedom.
+ *
+ * The normal approximation (1.96) is what most harnesses use and it is
+ * wrong in the direction that matters: at n=5 the true multiplier is 2.78,
+ * so a normal interval is nearly 30% too narrow exactly where a suite is
+ * small enough for that to mislead. Eval suites are small; this table is
+ * the difference between an interval that covers and one that flatters.
+ */
+const T_95: readonly number[] = [
+	12.706, 4.303, 3.182, 2.776, 2.571, 2.447, 2.365, 2.306, 2.262, 2.228, 2.201, 2.179, 2.16, 2.145,
+	2.131, 2.12, 2.11, 2.101, 2.093, 2.086, 2.08, 2.074, 2.069, 2.064, 2.06, 2.056, 2.052, 2.048,
+	2.045, 2.042,
+]
+
+function critical95(df: number): number {
+	if (df < 1) return Number.NaN
+	// Beyond 30 the t value is within ~1% of the normal, and pretending
+	// otherwise would imply a precision the rest of this does not have.
+	return T_95[df - 1] ?? 1.96
+}
+
+export interface ScoreUncertainty {
+	/** Cases that produced a score. Not the number of cases run. */
+	readonly n: number
+	/** Sample standard deviation, Bessel-corrected. */
+	readonly stdDev: number
+	/** Standard error of the mean. */
+	readonly stdError: number
+	/** Half-width of the 95% interval: the mean plus or minus this. */
+	readonly margin95: number
+	/**
+	 * The 95% interval, clamped to the score range.
+	 *
+	 * Clamped because a mean of 0.95 with a wide interval otherwise reports
+	 * an upper bound above 1, which is not a possible score and makes a
+	 * reader distrust the whole figure. The clamp is cosmetic and the
+	 * margin above is not — read that one for the width.
+	 */
+	readonly ci95: readonly [low: number, high: number]
+	/**
+	 * True when there is not enough data for an interval at all.
+	 *
+	 * One case has no spread to measure. Reporting `±0` there would be the
+	 * most confident-looking output the suite can produce, from the least
+	 * evidence it can have.
+	 */
+	readonly undefinedInterval: boolean
+}
+
+/**
+ * Spread of a set of scores, with the interval a reader should apply.
+ *
+ * **Assumes the cases are independent, and they may not be.** Miller's
+ * clustered standard errors run up to 3x the naive figure when cases come
+ * in related groups — several cases derived from one scenario, or one
+ * document, or one seed. This harness has no grouping key on a case, so
+ * there is nothing here to cluster on and this returns the naive figure.
+ * Where a suite does build several cases from one source, treat the
+ * interval below as a floor rather than as the answer.
+ *
+ * Stated rather than silently assumed because a too-narrow interval is
+ * worse than none: it turns "we cannot tell" into a number that looks
+ * like we can.
+ */
+export function uncertaintyOf(scores: readonly number[]): ScoreUncertainty {
+	const n = scores.length
+	if (n < 2) {
+		return {
+			n,
+			stdDev: 0,
+			stdError: 0,
+			margin95: Number.NaN,
+			ci95: [Number.NaN, Number.NaN],
+			undefinedInterval: true,
+		}
+	}
+
+	const mean = scores.reduce((a, b) => a + b, 0) / n
+	// Bessel-corrected: dividing by n estimates the spread of THESE cases,
+	// and the question is about the suite they were drawn from.
+	const variance = scores.reduce((acc, s) => acc + (s - mean) ** 2, 0) / (n - 1)
+	const stdDev = Math.sqrt(variance)
+	const stdError = stdDev / Math.sqrt(n)
+	const margin95 = critical95(n - 1) * stdError
+
+	return {
+		n,
+		stdDev,
+		stdError,
+		margin95,
+		ci95: [Math.max(0, mean - margin95), Math.min(1, mean + margin95)],
+		undefinedInterval: false,
+	}
+}
+
+/**
+ * One line a reader can act on, for a surface that prints a score.
+ *
+ * Names the interval rather than only the mean, because the mean alone is
+ * the thing that has been over-read. An interval spanning most of the
+ * scale says the suite cannot currently tell two runs apart, and that is
+ * the most useful sentence such a suite can produce.
+ */
+export function describeUncertainty(mean: number, u: ScoreUncertainty): string {
+	if (u.undefinedInterval) {
+		return u.n === 0
+			? 'no scored cases, so no score'
+			: `${mean.toFixed(3)} from a single case — no interval, and one case cannot show spread`
+	}
+	return `${mean.toFixed(3)} ±${u.margin95.toFixed(3)} (95% CI ${u.ci95[0].toFixed(3)}–${u.ci95[1].toFixed(3)}, n=${u.n}); assumes cases are independent`
+}

package/src/public-runtime.ts

@@ -63,6 +63,7 @@ export {
 export { normaliseModelId, resolveModelPricing, VENDOR_RATES } from './pricing/index.js'
 export { toErrorMessage } from './utils/error.js'
 export { configureLogger, getRootLogger, Logger } from './utils/logger.js'
+export { isTrustedReadOnly } from './tools/trusted-read-only.js'
 export { buildToolResultHashes, hashToolResult } from './utils/hash.js'
 export {
 	compressShellOutput,

package/src/registry/tool/execute.ts

@@ -3,6 +3,7 @@ import { assertStrictSchema } from '../../provider/strict-schema.js'
 import { GENAI, NAMZU, toolSpanName } from '../../telemetry/attributes.js'
 import { recordToolCall } from '../../telemetry/metrics.js'
 import { getTracer } from '../../telemetry/runtime-accessors.js'
+import { isTrustedReadOnly } from '../../tools/trusted-read-only.js'
 import type {
 	LLMToolSchema,
 	ToolAvailability,
@@ -487,7 +488,7 @@ Executable tool names, descriptions, and JSON input schemas are attached through
 
 				const mode = context.permissionContext?.mode ?? 'auto'
 				if (mode === 'plan') {
-					const isReadOnly = tool.isReadOnly ? tool.isReadOnly(rawInput) : false
+					const isReadOnly = isTrustedReadOnly(tool, rawInput)
 					if (!isReadOnly) {
 						const msg = `plan mode: non-read-only tool "${toolName}" blocked`
 						span.setAttributes({

package/src/session/workspace/git-worktree.ts

@@ -89,7 +89,26 @@ export class GitWorktreeDriver implements WorkspaceBackendDriver {
 		try {
 			await this.exec('git', argv)
 		} catch (cause) {
-			throw new WorkspaceBackendError({ op: 'create', kind: this.kind, cause })
+			// A non-zero exit here does not mean the worktree was not created.
+			// `git worktree add` runs the repository's post-checkout hook AFTER
+			// the checkout has completed, so a hook that fails — or that a
+			// timeout kills — reports failure over a worktree that is finished
+			// and usable. Treating the status as the answer throws away a good
+			// checkout AND leaks it: the path stays registered, and the next
+			// attempt fails differently, with "already exists".
+			//
+			// So the exit code is a hint and the repository is the evidence.
+			// The bar is deliberately high: registered under this exact path
+			// AND carrying the branch this call asked for. A registered path
+			// alone can be a half-finished checkout, or somebody else's.
+			if (!(await this.createdDespite(worktreePath, branch))) {
+				throw new WorkspaceBackendError({ op: 'create', kind: this.kind, cause })
+			}
+			this.log.warn('git-worktree add reported failure but the worktree is present', {
+				branch,
+				worktreePath,
+				cause: cause instanceof Error ? cause.message : String(cause),
+			})
 		}
 
 		const meta: GitWorktreeBackendMeta = {
@@ -142,6 +161,41 @@ export class GitWorktreeDriver implements WorkspaceBackendDriver {
 		}
 	}
 
+	/**
+	 * Did the worktree arrive despite the command reporting failure?
+	 *
+	 * Answers only for the branch this call created. A path registered
+	 * without that branch is not this call's worktree — it is a leftover
+	 * from a killed attempt, or a checkout somebody else owns, and the two
+	 * are indistinguishable from here. Claiming either would mean handing
+	 * a caller a workspace whose contents nobody vouched for, so both are
+	 * left to surface as the failure they are.
+	 *
+	 * Any error while checking is itself a "no". This runs on a path that
+	 * has already gone wrong once, and guessing optimistically there is how
+	 * a recovery turns a bad situation into a wrong one.
+	 */
+	private async createdDespite(worktreePath: string, branch: string): Promise<boolean> {
+		try {
+			const { stdout } = await this.exec('git', [
+				'-C',
+				this.repoRoot,
+				'worktree',
+				'list',
+				'--porcelain',
+			])
+			const entry = parseWorktreeList(stdout, worktreePath)
+			// `--porcelain` writes the branch as a full ref (`refs/heads/x`),
+			// and `branch` here is the short name this call passed to `-b`.
+			// Comparing them directly is a check that can never pass, which
+			// would make this whole recovery path silently dead — the exact
+			// shape it exists to catch.
+			return entry?.branch === `refs/heads/${branch}`
+		} catch {
+			return false
+		}
+	}
+
 	async inspect(ref: WorkspaceRef): Promise<WorkspaceInspection> {
 		let listStdout: string
 		try {

package/src/tools/builtins/bash.ts

@@ -60,6 +60,27 @@ const inputSchema = z.object({
 
 type BashInput = z.infer<typeof inputSchema>
 
+/**
+ * The last line worth showing from one chunk of streamed output.
+ *
+ * A progress line is a status, not a log: the host renders one line and
+ * replaces it as the next arrives, so sending a whole chunk sends a wall
+ * of text into a slot that shows one line of it. A chunk usually ends
+ * mid-line and usually ends with a newline, so the last NON-EMPTY line is
+ * the most recent complete thing the command actually said.
+ *
+ * Progress is capped rather than truncated with an ellipsis: this is
+ * glanced at, and a marker in a line nobody reads to the end is noise.
+ */
+function lastNonEmptyLine(chunk: string): string | undefined {
+	const lines = chunk.split('\n')
+	for (let i = lines.length - 1; i >= 0; i--) {
+		const line = lines[i]?.trim()
+		if (line) return line.length > 160 ? line.slice(0, 160) : line
+	}
+	return undefined
+}
+
 function isDangerousCommand(command: string): boolean {
 	return DANGEROUS_PATTERNS.some((pattern) => pattern.test(command))
 }
@@ -114,6 +135,19 @@ export const BashTool = defineTool({
 				// Same reason as the host path below: a Stop must reach the
 				// process, not just the promise waiting on it.
 				signal: context.abortSignal,
+				// The worker has always streamed its output; nothing asked for
+				// it, so a command that ran for minutes said nothing until it
+				// exited. `report` is ephemeral by design — it answers "is it
+				// still working?" for a live view and is excluded from the
+				// durable transcript — so this is a progress signal, not a
+				// second copy of the output. `result.stdout` remains the
+				// answer the model is given.
+				onOutput: context.report
+					? ({ data }) => {
+							const line = lastNonEmptyLine(data)
+							if (line) context.report?.(line)
+						}
+					: undefined,
 			})
 
 			if (result.timedOut) {

package/src/tools/coordinator/agent.ts

@@ -7,6 +7,7 @@ import { defineTool } from '../defineTool.js'
 import { wrapUntrusted } from '../untrusted-envelope.js'
 import { failureLabel, taskSucceeded } from './outcome.js'
 
+import { DELEGATION_TIMEOUT_MS } from './index.js'
 import type { TaskLaunchedCallback } from './index.js'
 
 /**
@@ -107,6 +108,17 @@ export function buildAgentTool(opts: AgentToolOptions): ToolDefinition {
 		readOnly: false,
 		destructive: false,
 		concurrencySafe: true,
+		// Declaring nothing here does not mean "no deadline"; it means the
+		// executor's 120-second default, which is a bound for a tool call and
+		// absurd for a whole agent run. `create_task` in the sibling module
+		// carries the same reasoning and the same hour, and the measurement
+		// behind that number is in its docblock: three delegated children took
+		// 4m21s, 5m58s and 8m04s, and all three parents gave up at 120s.
+		//
+		// This surface did not get that fix when its twin did, and the file's
+		// own note above records the pair doing exactly this before. The two
+		// tools are twins; a bound applied to one of them is not applied.
+		timeoutMs: DELEGATION_TIMEOUT_MS,
 		...(opts.terminal !== undefined ? { terminal: opts.terminal } : {}),
 		async execute({ description, prompt, subagent_type }, context) {
 			// With a single registered subagent the type is optional — default to