@namzu/sdk 41.0.0 → 42.0.1

package/src/runtime/query/finalize-run.ts

@@ -0,0 +1,192 @@
+import type { Span } from '@opentelemetry/api'
+import { consolidationEntry } from '../../compaction/consolidation.js'
+import type { WorkingStateManager } from '../../compaction/manager.js'
+import { NAMZU } from '../../constants/telemetry/index.js'
+import type { RunEvent, StepResult } from '../../types/run/index.js'
+import { toErrorMessage } from '../../utils/error.js'
+import type { RunContext } from './context.js'
+import type { EventTranslator } from './events.js'
+import { runOutputGuardrails } from './guardrails.js'
+import type { QueryParams } from './index.js'
+import { applyLifecycleHookResults } from './plugin-hooks.js'
+import type { ResultAssembler } from './result.js'
+
+/**
+ * What a run does on its way out, once the loop has stopped.
+ *
+ * The loop returns for any of a dozen reasons — an answer, a budget, a stop
+ * condition, a cancelled signal — and this is the one place all of them pass
+ * through: the `run_end`/`subagent_stop` hooks, the step record, the output
+ * guardrails, consolidation into a memory store, and the terminal events.
+ *
+ * It is a generator because it emits, and it is reached with `yield*` so
+ * every one of those emits suspends the caller at exactly the point it did
+ * when the code lived inline.
+ *
+ * Two positions here are load-bearing, and they are stated where they happen.
+ * `markCancelled` runs before the assembler, because `completeRun` marks a
+ * `running` run `completed` — the reverse order would overwrite the
+ * cancellation the abort signal had already declared. And
+ * `memory_consolidated` precedes `run_completed`, so a host folding the
+ * stream in order has the memory before the run that produced it.
+ *
+ * `setSteps` keeps its position too, but on the move's terms rather than on
+ * its own. This file used to claim the assembler needed it first "or the
+ * returned `Run` loses the final turn's steps" — that is not true, and a
+ * mutation proves it: `completeRun` reads `result`, `stopReason` and the
+ * budget and never `steps`, the returned `Run` is built by `finalize()`
+ * (which runs after the whole `try`/`catch`/`finally`), and moving
+ * `setSteps` below the assembler leaves the suite green, including the test
+ * that asserts `run.steps` on a returned run. What holds `setSteps` where it
+ * is, is the byte-identity of this move: every position was preserved, not
+ * just the consequential ones. Its read is still deferred to the same
+ * moment — after the `run_end` hooks, immediately before the record is
+ * written — which is why the caller passes `takeSteps` rather than an array.
+ */
+export interface RunFinalization {
+	readonly ctx: RunContext
+	readonly params: QueryParams
+	readonly eventTranslator: EventTranslator
+	/**
+	 * The steps the loop recorded, read HERE rather than handed over as an
+	 * array: it is read where it always was, after the `run_end` hooks and
+	 * immediately before the record is written.
+	 */
+	readonly takeSteps: () => readonly StepResult[]
+	readonly workingStateManager: WorkingStateManager | undefined
+	readonly resultAssembler: ResultAssembler
+	readonly rootSpan: Span
+}
+
+export async function* finalizeRun(finalization: RunFinalization): AsyncGenerator<RunEvent, void> {
+	const {
+		ctx,
+		params,
+		eventTranslator,
+		takeSteps,
+		workingStateManager,
+		resultAssembler,
+		rootSpan,
+	} = finalization
+
+	if (params.pluginManager) {
+		const hookResults = await params.pluginManager.executeHooks(
+			'run_end',
+			{ runId: ctx.runId, signal: ctx.abortController.signal },
+			eventTranslator.emitEvent,
+		)
+		applyLifecycleHookResults('run_end', hookResults)
+		yield* eventTranslator.drainPending()
+		// A delegated run says so once more, by name, so a hook that
+		// only cares when a subagent finishes need not read parent ids
+		// off every run_end.
+		if (params.parentRunId !== undefined) {
+			const stopResults = await params.pluginManager.executeHooks(
+				'subagent_stop',
+				{
+					runId: ctx.runId,
+					parentRunId: params.parentRunId,
+					signal: ctx.abortController.signal,
+				},
+				eventTranslator.emitEvent,
+			)
+			applyLifecycleHookResults('subagent_stop', stopResults)
+			yield* eventTranslator.drainPending()
+		}
+	}
+
+	// Hand the step record to the run before it settles, so the
+	// returned `Run` carries it.
+	ctx.runMgr.setSteps(takeSteps())
+
+	// Gates the FINAL result, not the stream — `text_delta` already
+	// reached the host as the model produced it. A rewrite is
+	// therefore a correction, and the event says so; buffering every
+	// token to gate the stream itself would trade the streaming UX
+	// for the guarantee, which is the host's call, not the SDK's.
+	if (params.outputGuardrails && params.outputGuardrails.length > 0) {
+		// Read what the run produced WITHOUT settling it. This used to
+		// call `markCompleted()` just to materialize the text, which
+		// force-marked a cancelled or paused run `completed` merely
+		// because a guardrail was configured — the presence of a
+		// safety check silently rewrote the run's own outcome.
+		const produced = ctx.runMgr.materializeResult()
+		const outputVerdict = await runOutputGuardrails(
+			params.outputGuardrails,
+			{ runId: ctx.runId, output: produced, messages: ctx.runMgr.messages },
+			ctx.log,
+		)
+
+		if (outputVerdict.blocked || outputVerdict.rewritten !== undefined) {
+			ctx.runMgr.clearStructuredOutput()
+			if (
+				params.structuredOutput &&
+				outputVerdict.rewritten !== undefined &&
+				ctx.runMgr.stopReason === 'end_turn'
+			)
+				ctx.runMgr.setStopReason('output_guardrail')
+		}
+
+		if (outputVerdict.blocked) {
+			await eventTranslator.emitEvent({
+				type: 'guardrail_triggered',
+				runId: ctx.runId,
+				stage: 'output',
+				action: 'block',
+				...(outputVerdict.name ? { guardrail: outputVerdict.name } : {}),
+				...(outputVerdict.reason ? { reason: outputVerdict.reason } : {}),
+			})
+			yield* eventTranslator.drainPending()
+			// Same reasoning as the input-guardrail branch above.
+			await ctx.runMgr.recordAudit({
+				what: { action: 'guardrail:output', resource: outputVerdict.name },
+				outcome: 'refused',
+				reason: outputVerdict.reason ?? 'blocked by an output guardrail',
+				...(params.persona?.identity.role ? { persona: params.persona.identity.role } : {}),
+			})
+			ctx.runMgr.setStopReason('output_guardrail')
+			ctx.runMgr.setLastError(outputVerdict.reason ?? 'blocked by an output guardrail')
+			ctx.runMgr.setResult('')
+		} else if (outputVerdict.rewritten !== undefined) {
+			await eventTranslator.emitEvent({
+				type: 'guardrail_triggered',
+				runId: ctx.runId,
+				stage: 'output',
+				action: 'rewrite',
+				...(outputVerdict.name ? { guardrail: outputVerdict.name } : {}),
+				...(outputVerdict.reason ? { reason: outputVerdict.reason } : {}),
+			})
+			yield* eventTranslator.drainPending()
+			ctx.runMgr.setResult(outputVerdict.rewritten)
+		}
+	}
+
+	if (params.consolidateInto && workingStateManager) {
+		const entry = consolidationEntry(workingStateManager.getState(), {
+			runId: ctx.runId,
+			at: Date.now(),
+		})
+		if (entry) {
+			try {
+				const { entry: saved } = await params.consolidateInto.create(entry)
+				await eventTranslator.emitEvent({
+					type: 'memory_consolidated',
+					runId: ctx.runId,
+					memoryId: saved.id,
+					title: entry.title,
+					decisions: workingStateManager.getState().decisions.length,
+					discoveries: workingStateManager.getState().discoveries.length,
+					failures: workingStateManager.getState().failures.length,
+				})
+				yield* eventTranslator.drainPending()
+			} catch (error) {
+				ctx.log.warn('consolidation into the memory store failed', {
+					[NAMZU.RUN_ID]: ctx.runId,
+					'namzu.memory.error': toErrorMessage(error),
+				})
+			}
+		}
+	}
+	if (ctx.abortController.signal.aborted) ctx.runMgr.markCancelled()
+	yield* resultAssembler.completeRun(rootSpan)
+}

package/src/runtime/query/guardrail-presets.ts

@@ -1,9 +1,12 @@
+import { PLUGIN_NAMESPACE_SEPARATOR } from '../../constants/plugin/index.js'
 import { OUTPUT_SECRET_PATTERNS } from '../../constants/secret-patterns.js'
+import { untrustedEnvelopeBody } from '../../tools/untrusted-envelope.js'
 import type {
 	GuardrailVerdict,
 	NamedGuardrail,
 	OutputGuardrail,
 	ToolResultGuardrail,
+	ToolResultGuardrailSpec,
 	ToolResultVerdict,
 } from '../../types/guardrail/index.js'
 
@@ -169,3 +172,356 @@ export function toolResultInjectionGuardrail(): NamedGuardrail<ToolResultGuardra
 		},
 	}
 }
+
+/**
+ * Requests shorter than this are not compared against the result.
+ *
+ * A one-word echo cannot be told from a one-word answer, and the shorter the
+ * value the likelier it is to recur in a legitimate result by coincidence:
+ * `ls` of a directory holding one entry called `src`, called with
+ * `{ path: "src" }`, returns `src`. The value of catching a five-character
+ * restatement is nil — nothing rides on it — so the comparison starts where
+ * a restatement actually means something.
+ *
+ * Counted in CODE POINTS, not UTF-16 units. Eight emoji are eight characters
+ * to every reader of this file and sixteen to `String.length`, and a floor
+ * that lets them through while the docblock says it excludes short values is
+ * a floor that is not doing what it says.
+ */
+const MIN_RESTATEMENT_LENGTH = 16
+
+/** Whether `text` is at least `minimum` code points long, without counting all of them. */
+function isAtLeastCodePoints(text: string, minimum: number): boolean {
+	let count = 0
+	for (const _character of text) {
+		count += 1
+		if (count >= minimum) return true
+	}
+	return false
+}
+
+export interface ToolResultCorrespondenceOptions {
+	/**
+	 * Which results this screen judges.
+	 *
+	 * `'framed'` — the default — is the results that carry the untrusted
+	 * envelope: content this process did not author, marked as such by
+	 * `wrapUntrusted`. For a connected server's tool that is every result the
+	 * adapter returns (see `frameServerResult`), and it is also a host tool
+	 * that framed its own result the same way — which is the point, and what
+	 * an earlier `provenance`-based predicate got wrong: the CLI's own remote
+	 * Exa search frames its answer and was left unscreened by accident of
+	 * registration, while a fetch was screened by the same accident.
+	 *
+	 * `'all'` adds this process's own unframed tools. That is the host's call
+	 * and not a safe default: a host that asks for it owns the consequences
+	 * and names its own exceptions in {@link passthroughTools}.
+	 */
+	readonly scope?: 'framed' | 'all'
+
+	/**
+	 * Tools whose answer IS the request, and must not be refused for saying
+	 * so: a validator returning what it validated, a normaliser returning the
+	 * normalised form, a search that repeats its query when it found nothing,
+	 * a fetch whose page body is its own URL.
+	 *
+	 * This is the escape hatch the check needs rather than a tuning knob. The
+	 * screen is right about a tool that was asked a question and handed the
+	 * question back; it is wrong about a tool whose purpose is to hand
+	 * something back, and nothing on the context distinguishes the two. The
+	 * false positive is the way this control dies — the host switches the
+	 * screen off and it then protects nothing — so the exemption has to be
+	 * cheap enough to be the first thing reached for.
+	 *
+	 * A tool answers to several names and each REGISTRATION SHAPE yields its
+	 * own set; {@link passthroughToolNames} is the one implementation and
+	 * spells them out, rather than this option and the check each carrying
+	 * half of the rule.
+	 */
+	readonly passthroughTools?: readonly string[]
+}
+
+interface RequestText {
+	/** Where in the request the text sits, e.g. `city` or `filters.city`. */
+	readonly path: string
+	readonly text: string
+}
+
+function normalizeText(text: string): string {
+	return text.trim().replace(/\s+/g, ' ')
+}
+
+/**
+ * Every string the call was made with, and where it sat.
+ *
+ * Strings only. A number is a plausible answer — a count of 10, a port of
+ * 8080 — and comparing one would fire on `{ maxResults: 10 }` answered with
+ * `10`, which is a legitimate result from a legitimate tool.
+ */
+function collectRequestText(value: unknown, path: string, into: RequestText[]): void {
+	if (typeof value === 'string') {
+		const text = normalizeText(value)
+		if (isAtLeastCodePoints(text, MIN_RESTATEMENT_LENGTH)) into.push({ path, text })
+		return
+	}
+	if (Array.isArray(value)) {
+		for (const [index, item] of value.entries()) {
+			collectRequestText(item, `${path}[${index}]`, into)
+		}
+		return
+	}
+	if (value === null || typeof value !== 'object') return
+	for (const [key, nested] of Object.entries(value)) {
+		collectRequestText(nested, path === '' ? key : `${path}.${key}`, into)
+	}
+}
+
+/** How the refusal names the argument that came back, for a caller that cannot see the path syntax. */
+function describeSubject(path: string): string {
+	if (path === '') return 'what this call sent'
+	const item = /^\[(\d+)\]$/.exec(path)
+	if (item) return `item ${item[1]} of the request`
+	return `the "${path}" argument this call sent`
+}
+
+/**
+ * Every name a tool answers to, so `passthroughTools` does not require the
+ * host to write the connector's own internal spelling.
+ *
+ * Each registration shape yields its own set, and the sets are NOT
+ * interchangeable — an exemption written for one shape does not exempt
+ * another:
+ *
+ *  - A server connected directly registers `mcp_<server>_<tool>` (the
+ *    adapter concatenates `mcp_${serverName}_${toolName}` verbatim). For
+ *    `mcp_weather-co_lookup` with `server: 'weather-co'`, the accepted names
+ *    are `mcp_weather-co_lookup`, `lookup`, and `weather-co:lookup`.
+ *  - A server a plugin contributes registers
+ *    `<plugin>__mcp__<server>__<tool>`. For
+ *    `myplugin__mcp__weather__lookup` the accepted names are
+ *    `myplugin__mcp__weather__lookup`, the bare tail after the last `__`
+ *    (`lookup`), and `weather:lookup` — NOT `mcp_weather_lookup`, which is
+ *    the direct shape's spelling of a name this tool does not have.
+ *  - A tool of this process's own has no server and no namespace, so the
+ *    registered name is the whole set.
+ *
+ * The bare name is what the server's manifest calls the tool — the name the
+ * operator has in front of them when they read its manifest — and
+ * `server:tool` is what a human writes. Both are derived from the registered
+ * name and the tool's provenance, so a caller that has only one of the two
+ * gets the registered name and nothing else.
+ */
+export function passthroughToolNames(toolName: string, server?: string): readonly string[] {
+	const names = new Set<string>([toolName])
+
+	const separator = toolName.lastIndexOf(PLUGIN_NAMESPACE_SEPARATOR)
+	const pluginBare =
+		separator < 0 ? undefined : toolName.slice(separator + PLUGIN_NAMESPACE_SEPARATOR.length)
+	if (pluginBare !== undefined) names.add(pluginBare)
+
+	const directPrefix = server === undefined ? undefined : `mcp_${server}_`
+	const connectedBare =
+		directPrefix !== undefined && toolName.startsWith(directPrefix)
+			? toolName.slice(directPrefix.length)
+			: undefined
+	if (connectedBare !== undefined) names.add(connectedBare)
+
+	const bare = connectedBare ?? pluginBare
+	if (bare !== undefined && server !== undefined) names.add(`${server}:${bare}`)
+
+	return [...names]
+}
+
+/**
+ * Refuse a result that is the request rather than an answer to it.
+ *
+ * #399 recorded that a screen at this boundary cannot know what a tool SHOULD
+ * have answered, and the issue that asked for this screen named three
+ * mismatches — a weather lookup for one city answering about another, a read
+ * of one path returning another's contents, a search returning instructions
+ * rather than results. **This screen decides none of the three.** The first
+ * two need a declaration of which argument names the subject, and the
+ * framework cannot infer one: for a file read the answer is the file's
+ * contents, which do not name the path, so the natural rule — an answer
+ * mentions its subject — would refuse every ordinary read. The third is what
+ * {@link toolResultInjectionGuardrail} already screens for at this same
+ * boundary, and re-implementing it here would give the two the same blind spot
+ * rather than the different one this adds.
+ *
+ * What is left is the one correspondence failure that needs no domain
+ * knowledge and cannot be an answer: **the result restates the request**. A
+ * tool handed a question and returning that question has answered nothing,
+ * whatever it is a tool for. The comparison is exact equality after
+ * whitespace normalisation against each string the call carried, so a result
+ * that says anything extra — the answer plus anything at all — passes.
+ *
+ * ## Why it judges framed results by default
+ *
+ * Because the envelope is the only marker that means *this process did not
+ * write this*, and the judgment it supports is the one this screen can make.
+ * Content arrives framed — by `wrapUntrusted`, through `frameServerResult`
+ * for a connector's tool result — when it came from outside: a connected
+ * server answered, or a host tool decided its own answer was not this
+ * process's to vouch for. A result that IS the request is a signal there.
+ *
+ * An earlier version judged results whose tool definition carried
+ * `provenance`, which is set by the MCP adapter and by nothing else. That
+ * predicate and this one agree on every connector result the comparison can
+ * act on, and they differ exactly where registration is an accident of the
+ * code rather than a fact about the content: the CLI's own remote Exa search
+ * frames its answer with this envelope and registers as a host tool, so a
+ * search that restated its query was out of scope while a fetch that did was
+ * in it. `scope` is a statement about what the screen is willing to judge, so
+ * it is expressed in terms of the content rather than of who mounted the
+ * tool.
+ *
+ * What the default leaves alone is this process's own UNFRAMED tools.
+ * `web_fetch` returns the page body, and a page whose body IS the URL it was
+ * fetched from — a redirect stub, a "you are here" placeholder, an echo
+ * endpoint — is a true result from a working tool that frames nothing.
+ * `test/…/a-result-that-does-not-answer-its-request` runs the shipped tool
+ * set through this screen with real calls and asserts none is refused, which
+ * is where that claim is checked rather than argued.
+ *
+ * `scope: 'all'` adds those too. That is the host's call, and it is the scope
+ * that costs something: under it, a fetch-like tool whose answer is its own
+ * argument is refused until the host names it in
+ * {@link ToolResultCorrespondenceOptions.passthroughTools}.
+ *
+ * The two other ways out were both worse. An exemption list of builtin NAMES
+ * inside this preset is a list that drifts — the next fetch-like tool has to
+ * remember to add itself, and a host tool that happens to be called
+ * `web_fetch` would be exempted by accident — and a subject declaration on
+ * every tool would be a new field on `ToolDefinition` that exists for this one
+ * screen.
+ *
+ * ## Three things it deliberately does not touch
+ *
+ *  - **An empty result for a non-empty request.** Real, and not a signal:
+ *    a search that matched nothing and a file with nothing in it both return
+ *    nothing, and refusing either tells the model to stop looking when there
+ *    was nothing to find.
+ *  - **A shape contradicting `ToolDefinition.outputSchema`.** The schema is
+ *    not on {@link ToolResultGuardrailContext} and is documented as shown to
+ *    the model, never validated. Carrying it and enforcing it are changes to
+ *    that contract, not this screen's to make.
+ *  - **A failed call.** On `success: false` the text is a diagnostic, and
+ *    `screenToolResult` replaces the output when it refuses — so refusing
+ *    here would trade an echo nobody needs to catch for the error message
+ *    the model needs to read.
+ *
+ * The comparison runs on the frame as well as on the text: a connector's
+ * result reaches a screen already wrapped by `wrapUntrusted`, so comparing
+ * the raw output would never match the one case this is most worth having —
+ * a connected server returning the request. Both the whole output and, when
+ * it is one wrapped block, the body inside it are compared.
+ *
+ * Detection is partial and this says so rather than implying coverage: an
+ * answer about the wrong subject, an answer that is plausible prose, and a
+ * restatement shorter than {@link MIN_RESTATEMENT_LENGTH} all pass.
+ */
+export function toolResultCorrespondenceGuardrail(
+	options: ToolResultCorrespondenceOptions = {},
+): NamedGuardrail<ToolResultGuardrail> {
+	const passthrough = new Set(options.passthroughTools ?? [])
+	const scope = options.scope ?? 'framed'
+
+	return {
+		name: 'tool-result-correspondence',
+		check: ({ toolName, input, output, success, provenance }): ToolResultVerdict => {
+			if (!success) return { action: 'pass' }
+			// `output` is typed as a string and a tool that ignores its own
+			// contract can hand back something else. Nothing here can compare
+			// a number against a request, and a screen that throws fails
+			// CLOSED — so the one thing this must not do is refuse a result
+			// because it could not read it.
+			if (typeof output !== 'string') return { action: 'pass' }
+
+			// The scope, and the same reader the comparison below uses rather
+			// than a second "looks framed" test that could disagree with it.
+			//
+			// This is a superset of the `provenance`-based predicate it
+			// replaced, for every result the comparison can act on. The only
+			// tool definition carrying `provenance` is the MCP adapter's, and
+			// `frameServerResult` frames every non-empty text it returns — so
+			// a connected result that reaches the comparison at all carries a
+			// legible frame. The one case the new predicate drops is a
+			// connected result with an EMPTY output, which the comparison
+			// skipped anyway (`normalizeText('')` is `''`, and an empty string
+			// is never a request of the minimum length).
+			const body = untrustedEnvelopeBody(output)
+			if (scope !== 'all' && body === undefined) return { action: 'pass' }
+
+			if (
+				passthrough.size > 0 &&
+				passthroughToolNames(toolName, provenance?.server).some((name) => passthrough.has(name))
+			) {
+				return { action: 'pass' }
+			}
+
+			const request: RequestText[] = []
+			collectRequestText(input, '', request)
+			if (request.length === 0) return { action: 'pass' }
+
+			// The whole output, and the body when the output is one wrapped
+			// block. The two are never the same string — the frame carries the
+			// tags — so this is two comparisons, not the same one twice, and
+			// the refusal says which one matched.
+			for (const [text, insideFrame] of [
+				[normalizeText(output), false],
+				[normalizeText(body ?? ''), true],
+			] as const) {
+				if (text === '') continue
+				const restated = request.find((entry) => entry.text === text)
+				if (!restated) continue
+				// The reason names the tool, and says what the comparison
+				// actually did. Both were wrong before: a host reading the
+				// transcript is the person who can exempt this tool, and
+				// "verbatim" was a false claim about a comparison that ran
+				// after whitespace normalisation.
+				const matched = insideFrame
+					? `the text inside the untrusted frame from "${toolName}"`
+					: `the result from "${toolName}"`
+				return {
+					action: 'refuse',
+					reason: `${matched} is ${describeSubject(restated.path)}, whitespace-normalised, so it is the request rather than an answer to it`,
+				}
+			}
+
+			return { action: 'pass' }
+		},
+	}
+}
+
+/**
+ * The screens a run installs on itself when its host configured none.
+ *
+ * A DEFAULT, and the reason it is here rather than on
+ * {@link ToolRegistryConfig.resultGuardrails}: a host assembles a registry
+ * and hands it to `runAgent`, so a registry-constructor default would be the
+ * host's to write and this repository's default would reach nobody. The run
+ * is the thing that has to carry it, and a run that wants none says so with
+ * an empty array — which is the escape hatch, and it exists precisely because
+ * the screen below can refuse a result.
+ *
+ * One screen, scoped to results framed as untrusted. See the preset for why
+ * the scope is what it is: an unframed host tool whose answer IS the request —
+ * `web_fetch` returning a page whose body is its own URL — is an ordinary
+ * result, and a default that refuses those is a default that breaks a working
+ * tool.
+ *
+ * **A default that refuses a legitimate result is a default that gets turned
+ * off, so the exemption has to be reachable from wherever the screen is
+ * installed.** `toolResultCorrespondenceGuardrail({ passthroughTools })` is
+ * how a host writing SDK code says so; a run's `toolResultGuardrails` array
+ * is where it substitutes its own configured instance. An application that
+ * ships the default without also shipping a way to name an exception has
+ * shipped a switch with one position.
+ *
+ * Frozen, because it is handed to callers who may want to extend it:
+ * `[...DEFAULT_TOOL_RESULT_GUARDRAILS, myScreen()]`. A caller that could
+ * mutate it would be mutating the default for every other run in the process.
+ */
+export const DEFAULT_TOOL_RESULT_GUARDRAILS: readonly ToolResultGuardrailSpec[] = Object.freeze([
+	toolResultCorrespondenceGuardrail(),
+])