@namzu/sdk 0.5.0 → 1.0.0

package/src/runtime/query/iteration/index.ts

@@ -24,6 +24,7 @@ import { toErrorMessage } from '../../../utils/error.js'
 import { generateMessageId } from '../../../utils/id.js'
 import type { Logger } from '../../../utils/logger.js'
 import type { CheckpointManager } from '../checkpoint.js'
+import { AUTO_CONTINUATION_USER_MESSAGE } from '../continuation.js'
 import type { EmitEvent } from '../events.js'
 import type { ToolExecutor } from '../executor.js'
 import type { GuardCoordinator } from '../guard.js'
@@ -58,6 +59,35 @@ export interface IterationConfig {
 	pluginManager?: import('../../../plugin/lifecycle.js').PluginLifecycleManager
 }
 
+/**
+ * Escape the five XML metacharacters so an interpolated value cannot
+ * break out of a tag. Used for the simple identifier fields in the
+ * `<task-notification>` envelope (taskId, agentId, status) — values
+ * here are controlled enums / opaque ids in practice, but escaping
+ * keeps the envelope robust against any future producer that lets a
+ * `<` or `&` leak in.
+ */
+function xmlEscape(value: string): string {
+	return value
+		.replace(/&/g, '&amp;')
+		.replace(/</g, '&lt;')
+		.replace(/>/g, '&gt;')
+		.replace(/"/g, '&quot;')
+		.replace(/'/g, '&apos;')
+}
+
+/**
+ * Wrap free-form worker output in a CDATA section. CDATA preserves
+ * the raw text — code, markdown angle brackets, ampersands — so the
+ * supervisor sees what the worker actually produced instead of an
+ * escape-encoded approximation. The only termination CDATA forbids
+ * is the literal `]]>` sequence; we split-and-rejoin around it to
+ * keep the section well-formed regardless of payload.
+ */
+function cdataWrap(value: string): string {
+	return `<![CDATA[${value.replace(/]]>/g, ']]]]><![CDATA[>')}]]>`
+}
+
 /**
  * Map a provider's coarse `finishReason` plus the orchestrator's
  * `forceFinalize` flag onto the per-message {@link MessageStopReason}
@@ -104,8 +134,11 @@ interface StreamingTurnResult {
  *   finally-style fall-through path with `stopReason: 'refusal'`.
  * - `tool_input_delta` with no `toolUseId` registered yet: we drop
  *   the fragment and log a warning (proxies seen to misorder events).
- * - `chunk.error`: we surface as a thrown error after emitting the
- *   message_completed terminator so consumer cards still close.
+ * - `chunk.error`: when no tool input is recoverable, we surface as
+ *   a thrown error after emitting the message_completed terminator so
+ *   consumer cards still close. If a tool-use block was already open,
+ *   we instead synthesize a tool call with runtime truncation metadata
+ *   so the executor can return a model-readable retry hint.
  */
 async function* streamProviderTurn(
 	provider: LLMProvider,
@@ -134,7 +167,26 @@ async function* streamProviderTurn(
 	}
 	const toolBuckets = new Map<
 		number,
-		{ id: string; name: string; argsBuf: string; started: boolean; completed: boolean }
+		{
+			id: string
+			name: string
+			argsBuf: string
+			started: boolean
+			completed: boolean
+			/**
+			 * Parsed input. `null` while the bucket is still streaming.
+			 * The synthesized
+			 * `ChatCompletionResponse.toolCalls[].function.arguments` is
+			 * derived from this — never from the raw buffer — so the
+			 * downstream executor (`runtime/query/executor.ts`) never has
+			 * to re-parse a truncated string. A truncated tool call is
+			 * surfaced as `arguments: "{}"` plus `metadata.inputTruncated`
+			 * so tool args remain clean while the executor can still
+			 * return a specific retry hint.
+			 */
+			parsed: unknown | null
+			inputTruncated: boolean
+		}
 	>()
 	let streamError: string | undefined
 
@@ -169,6 +221,8 @@ async function* streamProviderTurn(
 						argsBuf: '',
 						started: false,
 						completed: false,
+						parsed: null,
+						inputTruncated: false,
 					}
 					toolBuckets.set(tc.index, bucket)
 				}
@@ -218,17 +272,20 @@ async function* streamProviderTurn(
 					try {
 						parsed = bucket.argsBuf ? JSON.parse(bucket.argsBuf) : {}
 					} catch (err) {
+						bucket.inputTruncated = true
 						log.warn('tool input JSON parse failed at content_block_stop', {
 							runId,
 							toolUseId: endId,
 							error: err instanceof Error ? err.message : String(err),
 						})
 					}
+					bucket.parsed = parsed
 					await emitEvent({
 						type: 'tool_input_completed',
 						runId,
 						toolUseId: endId as ToolUseId,
 						input: parsed,
+						...(bucket.inputTruncated ? { inputTruncated: true } : {}),
 					})
 					yield* drainPending()
 				}
@@ -242,29 +299,108 @@ async function* streamProviderTurn(
 	}
 
 	// Flush any tool buckets the provider failed to close (no toolCallEnd
-	// arrived — defensive against providers that don't yet emit it).
+	// arrived — defensive against providers that don't yet emit it, and
+	// the load-bearing path when the provider stream ends with
+	// `stop_reason: "max_tokens"` mid-`input_json_delta`. In that case
+	// Anthropic's SSE never sends `content_block_stop` for the open
+	// tool_use block: the upstream model ran out of completion tokens
+	// before it could close the JSON literal, so the buffered
+	// `argsBuf` ends with something like `"content":"…some prefix` —
+	// not parseable.
+	//
+	// Two cases coalesce here:
+	//   1. The buffer parses cleanly (the provider just forgot to emit
+	//      `content_block_stop` but the args are intact) — keep parsed.
+	//   2. The buffer is truncated mid-literal — `parsed = {}` is the
+	//      safe fallback so the executor's `JSON.parse(arguments)`
+	//      succeeds and downstream consumers don't crash. The PRICE
+	//      we used to pay was the model getting back a generic
+	//      "<field> is required" Zod error and not realising its
+	//      previous tool call was truncated server-side, so it would
+	//      retry with the SAME long input and hit the same cutoff in
+	//      a loop. Detect the truncation case and mark the tool call
+	//      with runtime metadata; the executor surfaces a specific
+	//      "your tool call was cut off by max_tokens — retry with
+	//      shorter input or split into smaller calls" message that the
+	//      model can act on.
 	for (const bucket of toolBuckets.values()) {
 		if (bucket.started && !bucket.completed) {
 			bucket.completed = true
 			let parsed: unknown = {}
-			try {
-				parsed = bucket.argsBuf ? JSON.parse(bucket.argsBuf) : {}
-			} catch {
-				// leave parsed = {}
+			let truncated = false
+			if (bucket.argsBuf) {
+				try {
+					parsed = JSON.parse(bucket.argsBuf)
+				} catch {
+					// argsBuf had content but didn't parse — almost
+					// certainly the max_tokens-mid-literal cutoff. Mark
+					// the bucket so the executor can return a model-
+					// readable hint instead of a generic Zod error.
+					truncated = true
+					parsed = {}
+				}
+			}
+			bucket.parsed = parsed
+			bucket.inputTruncated = truncated
+			if (truncated) {
+				log.warn('tool input truncated by upstream cutoff (no toolCallEnd, argsBuf unparsable)', {
+					runId,
+					toolUseId: bucket.id,
+					toolName: bucket.name,
+					bufferLength: bucket.argsBuf.length,
+				})
 			}
 			await emitEvent({
 				type: 'tool_input_completed',
 				runId,
 				toolUseId: bucket.id as ToolUseId,
 				input: parsed,
+				...(truncated ? { inputTruncated: true } : {}),
 			})
 			yield* drainPending()
 		}
 	}
 
+	// `arguments` MUST be valid JSON for the executor's `JSON.parse`
+	// (`runtime/query/executor.ts:executeSingle`) to succeed. We
+	// always serialise from the bucket's `parsed` object (filled by
+	// either the `toolCallEnd` branch above or the post-stream flush
+	// loop) instead of re-emitting `argsBuf`. When the provider
+	// stream truncated mid-input, `metadata.inputTruncated` carries that
+	// state; the executor parses cleanly and returns a specific
+	// model-readable retry hint instead of the generic "Invalid JSON in
+	// tool arguments" intercept.
+	const toolCalls = [...toolBuckets.entries()]
+		.sort(([a], [b]) => a - b)
+		.map(([, b]) => ({
+			id: b.id,
+			type: 'function' as const,
+			function: {
+				name: b.name,
+				arguments: JSON.stringify(b.parsed ?? {}),
+			},
+			...(b.inputTruncated ? { metadata: { inputTruncated: true } } : {}),
+		}))
+
+	const recoveredToolInputFromStreamError =
+		streamError !== undefined && toolCalls.some((tc) => tc.id && tc.function.name)
+	const effectiveFinishReason: ChatCompletionResponse['finishReason'] =
+		recoveredToolInputFromStreamError ? 'tool_calls' : finishReason
+
+	if (recoveredToolInputFromStreamError) {
+		log.warn('provider stream failed after tool input; surfacing tool call to executor', {
+			runId,
+			iteration,
+			error: streamError,
+			toolCallCount: toolCalls.length,
+		})
+	}
+
 	const stopReason: MessageStopReason = streamError
-		? 'refusal'
-		: synthesizeMessageStopReason(finishReason, forceFinalize)
+		? recoveredToolInputFromStreamError
+			? 'tool_use'
+			: 'refusal'
+		: synthesizeMessageStopReason(effectiveFinishReason, forceFinalize)
 
 	await emitEvent({
 		type: 'message_completed',
@@ -277,18 +413,10 @@ async function* streamProviderTurn(
 	})
 	yield* drainPending()
 
-	if (streamError) {
+	if (streamError && !recoveredToolInputFromStreamError) {
 		throw new Error(`Provider stream error: ${streamError}`)
 	}
 
-	const toolCalls = [...toolBuckets.entries()]
-		.sort(([a], [b]) => a - b)
-		.map(([, b]) => ({
-			id: b.id,
-			type: 'function' as const,
-			function: { name: b.name, arguments: b.argsBuf },
-		}))
-
 	const response: ChatCompletionResponse = {
 		id: id || messageId,
 		model: model || params.model,
@@ -297,7 +425,7 @@ async function* streamProviderTurn(
 			content: textBuf.length > 0 ? textBuf : null,
 			toolCalls: toolCalls.length > 0 ? toolCalls : undefined,
 		},
-		finishReason,
+		finishReason: effectiveFinishReason,
 		usage,
 	}
 	return { response, messageId }
@@ -354,17 +482,19 @@ export class IterationOrchestrator {
 		const { model } = runConfig
 		const tracer = getTracer()
 
+		// Worker-completion delivery used to fan out through a global
+		// onTaskCompleted listener that pushed handles onto
+		// `pendingNotifications`; the iteration loop then drained
+		// them as <task-notification> envelopes. Both `create_task`
+		// and the `Agent` tool are now blocking and return their
+		// worker output as the dispatching tool_use's canonical
+		// tool_result, so the listener path would only DUPLICATE
+		// every completion (once as tool_result, once as injected
+		// envelope user-message). Leaving the binding out closes
+		// the duplicate notification surface entirely; the dormant
+		// drain stays as a no-op until a follow-up tears it out.
 		let unsubscribeTaskListener: (() => void) | undefined
-		if (this.ctx.taskGateway) {
-			unsubscribeTaskListener = this.ctx.taskGateway.onTaskCompleted((handle) => {
-				this.ctx.pendingNotifications.push(handle)
-				this.ctx.log.debug('Task completion queued for notification', {
-					taskId: handle.taskId,
-					agentId: handle.agentId,
-					state: handle.state,
-				})
-			})
-		}
+		void unsubscribeTaskListener
 
 		try {
 			const planSignal = yield* runPlanGate(this.ctx)
@@ -588,6 +718,43 @@ export class IterationOrchestrator {
 						const hasContent =
 							response.message.content !== null && response.message.content.length > 0
 
+						// Auto-continuation on `stop_reason: max_tokens`. The
+						// model hit its per-call output cap mid-text (NOT
+						// mid-tool-use — that path is handled separately
+						// below via `inputTruncated`). Push a synthetic
+						// "continue" user message and let the loop fire
+						// another turn. The provider receives the partial
+						// assistant content + the continue prompt and
+						// resumes from where it left off, mirroring the
+						// Claude.ai "Continue" affordance.
+						//
+						// Guards:
+						//   - `hasContent` so we don't loop forever on an
+						//     empty cutoff (Anthropic occasionally emits
+						//     `stop_reason: max_tokens` with no content
+						//     when an injected pre-fill blocks the model).
+						//   - `!forceFinalize` so the forced-finalize path
+						//     never auto-continues — that path is invoked
+						//     specifically to extract a closing summary.
+						//   - max_iterations bounds the loop in any case.
+						if (!forceFinalize && response.finishReason === 'length' && hasContent) {
+							this.ctx.log.info('LLM hit max_tokens mid-text — auto-continuing', {
+								runId: runMgr.id,
+								iteration: iterationNum,
+								completionTokens: response.usage.completionTokens,
+							})
+							runMgr.pushMessage(createUserMessage(AUTO_CONTINUATION_USER_MESSAGE))
+							await this.ctx.emitEvent({
+								type: 'iteration_completed',
+								runId: runMgr.id,
+								iteration: iterationNum,
+								hasToolCalls: false,
+							})
+							yield* this.ctx.drainPending()
+							iterSpan.end()
+							continue
+						}
+
 						if (!hasContent && !forceFinalize) {
 							this.ctx.log.warn('Empty completion detected — requesting final summary', {
 								iteration: iterationNum,
@@ -686,47 +853,75 @@ export class IterationOrchestrator {
 		await this.injectOneTaskNotification()
 	}
 
+	/**
+	 * Canonical async completion delivery (ses_009-task-notification-envelope).
+	 *
+	 * Drains every pending task completion in one pass and emits each as
+	 * a plain USER text message wrapped in the `<task-notification>`
+	 * envelope the supervisor prompt expects.
+	 *
+	 * Why not a `tool_result` block bound to the dispatching tool_use_id:
+	 * `create_task` is documented as NON-BLOCKING and returns
+	 * "Task launched: …" immediately. That immediate return is already
+	 * recorded as the canonical tool_result for that tool_use, so a
+	 * second tool_result for the SAME tool_use_id — emitted later, after
+	 * intervening assistant turns — is rejected by Anthropic with
+	 * `messages.<n>.content.0: unexpected tool_use_id found in
+	 * tool_result blocks` because the immediately-prior assistant
+	 * message no longer carries the matching tool_use. Wrapping as a
+	 * user text envelope sidesteps the pairing rule entirely.
+	 *
+	 * Coalescing N drops into one drain replaces the previous
+	 * one-at-a-time pattern which forced a separate orchestrator
+	 * iteration per completed task on wide fan-outs.
+	 */
 	private async injectOneTaskNotification(): Promise<void> {
-		const handle = this.ctx.pendingNotifications.shift()
-		if (!handle) return
-		const meta = this.ctx.launchedTasks.get(handle.taskId)
-		const resultText =
-			handle.result?.result ??
-			handle.result?.lastError ??
-			`Task finished with state: ${handle.state}`
-
-		if (meta?.planTaskId && this.ctx.taskStore) {
-			const success = handle.state === 'completed'
-			await this.ctx.taskStore.update(meta.planTaskId as `task_${string}`, {
-				status: 'completed',
-				description: success ? undefined : `Failed: ${resultText.substring(0, 200)}`,
+		if (this.ctx.pendingNotifications.length === 0) return
+		const handles = this.ctx.pendingNotifications.splice(0)
+
+		for (const handle of handles) {
+			const meta = this.ctx.launchedTasks.get(handle.taskId)
+			const resultText =
+				handle.result?.result ??
+				handle.result?.lastError ??
+				`Task finished with state: ${handle.state}`
+
+			if (meta?.planTaskId && this.ctx.taskStore) {
+				const success = handle.state === 'completed'
+				await this.ctx.taskStore.update(meta.planTaskId as `task_${string}`, {
+					status: 'completed',
+					description: success ? undefined : `Failed: ${resultText.substring(0, 200)}`,
+				})
+			}
+
+			this.ctx.launchedTasks.delete(handle.taskId)
+
+			// `remaining-tasks` = inflight workers still pending after this
+			// one drains. `launchedTasks` is the single source of truth:
+			// it holds every dispatched worker that has NOT yet been
+			// drained + delete()'d. The drain batch entries are still
+			// inside launchedTasks until each iteration's delete() above
+			// removes them, so reading the size right after that delete
+			// gives the honest count. Adding `handles.length - 1 - i`
+			// here used to double-count this same queue.
+			const remainingTasks = this.ctx.launchedTasks.size
+			const envelope =
+				`<task-notification>\n<task-id>${xmlEscape(handle.taskId)}</task-id>\n` +
+				`<agent-id>${xmlEscape(handle.agentId)}</agent-id>\n` +
+				`<status>${xmlEscape(handle.state)}</status>\n` +
+				`<result>${cdataWrap(resultText)}</result>\n` +
+				`<remaining-tasks>${remainingTasks}</remaining-tasks>\n</task-notification>`
+
+			this.ctx.runMgr.pushMessage(createUserMessage(envelope))
+
+			this.ctx.log.info('Task notification injected', {
+				taskId: handle.taskId,
+				agentId: handle.agentId,
+				state: handle.state,
+				planTaskId: meta?.planTaskId,
+				remainingNotifications: remainingTasks,
 			})
 		}
-
-		this.ctx.launchedTasks.delete(handle.taskId)
-		const remainingTasks = this.ctx.launchedTasks.size
-
-		const notification = [
-			'<task-notification>',
-			`  <task-id>${handle.taskId}</task-id>`,
-			`  <agent-id>${handle.agentId}</agent-id>`,
-			`  <status>${handle.state}</status>`,
-			`  <description>${meta?.description ?? 'agent task'}</description>`,
-			`  <result>${resultText}</result>`,
-			`  <remaining-tasks>${remainingTasks}</remaining-tasks>`,
-			'</task-notification>',
-		].join('\n')
-
-		this.ctx.runMgr.pushMessage(createUserMessage(notification))
-
-		this.ctx.log.info('Task notification injected', {
-			taskId: handle.taskId,
-			agentId: handle.agentId,
-			state: handle.state,
-			planTaskId: meta?.planTaskId,
-			remainingTasks,
-			remainingNotifications: this.ctx.pendingNotifications.length,
-		})
 	}
 
 	private async requestFinalResponse(model: string, reason: StopReason): Promise<void> {

package/src/runtime/query/iteration/phases/context.ts

@@ -22,6 +22,16 @@ export interface LaunchedTaskMeta {
 	readonly agentId: string
 	readonly description: string
 	readonly planTaskId?: string
+	/**
+	 * The `tool_use_id` of the assistant `create_task` block that
+	 * spawned this background task. Required to emit the canonical
+	 * `tool_result` content block when the task completes — without
+	 * it we'd fall back to the legacy synthetic-user-message inject
+	 * (see ses_009-task-notification-envelope). Optional because
+	 * older call paths that don't thread `ToolContext.toolUseId`
+	 * still publish the meta without it.
+	 */
+	readonly originalToolUseId?: string
 }
 
 export interface IterationContext {

package/src/runtime/query/prompt.ts

@@ -1,5 +1,5 @@
 import { FILESYSTEM_TOOLS } from '../../constants/tools/index.js'
-import { assembleSystemPrompt } from '../../persona/assembler.js'
+import { assembleSystemPrompt, renderSkillsSection } from '../../persona/assembler.js'
 import type { AgentRuntimeContext } from '../../types/agent/base.js'
 import type { AgentContextLevel } from '../../types/agent/factory.js'
 import type { AgentPersona } from '../../types/persona/index.js'
@@ -83,6 +83,14 @@ export class PromptBuilder {
 			parts.push(this.config.systemPrompt)
 		} else if (this.config.persona) {
 			parts.push(assembleSystemPrompt(this.config.persona, this.config.skills))
+		} else {
+			const skillSection = renderSkillsSection(this.config.skills)
+			if (skillSection) parts.push(skillSection)
+		}
+
+		if (this.config.systemPrompt) {
+			const skillSection = renderSkillsSection(this.config.skills)
+			if (skillSection) parts.push(skillSection)
 		}
 
 		if (contextLevel !== 'minimal') {
@@ -133,6 +141,14 @@ export class PromptBuilder {
 			if (this.config.persona.sessionContext) {
 				dynamicParts.push(`## Session Context\n${this.config.persona.sessionContext.trim()}`)
 			}
+		} else {
+			const skillSection = renderSkillsSection(this.config.skills)
+			if (skillSection) staticParts.push(skillSection)
+		}
+
+		if (this.config.systemPrompt) {
+			const skillSection = renderSkillsSection(this.config.skills)
+			if (skillSection) staticParts.push(skillSection)
 		}
 
 		if (contextLevel !== 'minimal') {

package/src/runtime/query/tooling.ts

@@ -17,6 +17,7 @@ export interface ToolingBootstrapConfig {
 	permissionMode: PermissionMode
 	env: Record<string, string>
 	abortSignal: AbortSignal
+	allowedTools?: readonly string[]
 	invocationState?: InvocationState
 	pluginManager?: PluginLifecycleManager
 }
@@ -36,6 +37,7 @@ export class ToolingBootstrap {
 				permissionMode: config.permissionMode,
 				env: config.env,
 				abortSignal: config.abortSignal,
+				allowedTools: config.allowedTools,
 				invocationState: config.invocationState,
 				pluginManager: config.pluginManager,
 			},

package/src/sandbox/provider/local.ts

@@ -4,8 +4,10 @@ import {
 	readFile as fsReadFile,
 	writeFile as fsWriteFile,
 	mkdir,
+	readdir,
 	rename,
 	rm,
+	stat,
 } from 'node:fs/promises'
 import { tmpdir } from 'node:os'
 import { dirname, isAbsolute, join, relative, resolve } from 'node:path'
@@ -24,6 +26,7 @@ import type {
 	SandboxEnvironment,
 	SandboxExecOptions,
 	SandboxExecResult,
+	SandboxFileEntry,
 	SandboxProvider,
 	SandboxStatus,
 } from '../../types/sandbox/index.js'
@@ -302,6 +305,36 @@ class LocalSandbox implements Sandbox {
 		return fsReadFile(resolved)
 	}
 
+	async listFiles(rootPath: string): Promise<readonly SandboxFileEntry[]> {
+		if (this._status === 'destroyed') {
+			throw new Error(`Sandbox ${this.id} is destroyed`)
+		}
+
+		const resolved = assertInsideSandbox(this.rootDir, rootPath)
+		const root = await stat(resolved).catch(() => null)
+		if (!root || !root.isDirectory()) return []
+
+		const entries: SandboxFileEntry[] = []
+		const stack: string[] = [resolved]
+		while (stack.length > 0) {
+			const dir = stack.pop()
+			if (!dir) break
+			const dirents = await readdir(dir, { withFileTypes: true }).catch(() => [])
+			for (const ent of dirents) {
+				const full = join(dir, ent.name)
+				if (ent.isDirectory()) {
+					stack.push(full)
+					continue
+				}
+				if (!ent.isFile()) continue
+				const info = await stat(full).catch(() => null)
+				if (!info) continue
+				entries.push({ path: full, size: info.size })
+			}
+		}
+		return entries
+	}
+
 	async destroy(): Promise<void> {
 		if (this._status === 'destroyed') {
 			return