@namzu/sdk 2.0.0 → 3.0.0

package/src/runtime/query/__tests__/durable-park-and-trace.test.ts

@@ -0,0 +1,245 @@
+import { mkdtemp, rm } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { afterEach, beforeEach, describe, expect, it } from 'vitest'
+
+import { DiskCheckpointStore } from '../../../store/run/checkpoint-disk.js'
+import { serializeSpan } from '../../../telemetry/attributes.js'
+import type { HITLDecisionRequest, IterationCheckpoint } from '../../../types/hitl/index.js'
+import type { CheckpointId, RunId } from '../../../types/ids/index.js'
+import type { CheckpointRunScope } from '../../../types/run/checkpoint-store.js'
+import { deriveRunStatus } from '../../../types/run/derive-status.js'
+import {
+	CheckpointManager,
+	findPendingCheckpoint,
+	isExpiredPark,
+	listExpiredParks,
+} from '../checkpoint.js'
+
+/**
+ * Two durability holes that only show up across a process boundary.
+ *
+ * A park had no deadline. Every timer in the SDK is an in-process
+ * `setTimeout` and the park-record delay is deliberately `unref`'d, so
+ * nothing in memory can outlive a redeploy: a run parks for approval, the
+ * worker is replaced, nobody answers, and the checkpoint stays outstanding
+ * forever — every approval-queue reader keeps serving it. The run timeout
+ * cannot cover it either, because it is only checked between iterations
+ * and a park suspends mid-iteration.
+ *
+ * And a checkpoint recorded no trace, so a run that crashed at iteration
+ * 12 and resumed produced two traces with different ids and no link. The
+ * run id correlates them well enough to find both by query and not well
+ * enough to see one waterfall — and for a replay fork, which mints a new
+ * run id, not even that.
+ */
+
+const RID = 'run_1' as RunId
+const SCOPE: CheckpointRunScope = {
+	runId: RID,
+	tenantId: 'ten_1' as never,
+	projectId: 'prj_1' as never,
+	sessionId: 'ses_1' as never,
+}
+
+const request = (): HITLDecisionRequest => ({
+	type: 'tool_review',
+	runId: RID,
+	checkpointId: 'cp_x' as CheckpointId,
+	toolCalls: [{ id: 't1', name: 'deploy', input: {}, isDestructive: true }],
+})
+
+function runMgr(): never {
+	return {
+		id: RID,
+		messages: [],
+		currentIteration: 3,
+		tokenUsage: { promptTokens: 10, completionTokens: 5, totalTokens: 15 },
+		costInfo: { totalCost: 0.01 },
+		getSession: () => ({ startedAt: Date.now() - 1_000 }),
+	} as never
+}
+
+describe('a park that nobody answers', () => {
+	let dir: string
+	let store: DiskCheckpointStore
+	let manager: CheckpointManager
+
+	beforeEach(async () => {
+		dir = await mkdtemp(join(tmpdir(), 'namzu-park-'))
+		store = new DiskCheckpointStore({ baseDir: dir })
+		manager = new CheckpointManager(store, SCOPE)
+	})
+
+	afterEach(async () => {
+		await rm(dir, { recursive: true, force: true })
+	})
+
+	const park = async (ttlMs?: number): Promise<IterationCheckpoint> => {
+		const checkpoint = await manager.create(runMgr(), 1)
+		return manager.park(checkpoint, request(), ttlMs === undefined ? undefined : { ttlMs })
+	}
+
+	it('records an ABSOLUTE deadline, so it survives the process that set it', async () => {
+		const parked = await park(60_000)
+		// A duration plus an in-process timer cannot survive a redeploy,
+		// which is the whole failure being fixed.
+		expect(parked.pending?.deadlineAt).toBeGreaterThan(Date.now())
+		expect(parked.pending?.deadlineAt).toBe((parked.pending?.parkedAt ?? 0) + 60_000)
+	})
+
+	it('stops being served once the deadline passes', async () => {
+		await park(1)
+		await new Promise((resolve) => setTimeout(resolve, 5))
+
+		// Serving it re-presents an approval whose window has closed, and
+		// every queue reader would keep re-presenting it forever.
+		expect(await findPendingCheckpoint(store, SCOPE)).toBeNull()
+	})
+
+	it('is still served before the deadline', async () => {
+		const parked = await park(60_000)
+		expect((await findPendingCheckpoint(store, SCOPE))?.id).toBe(parked.id)
+	})
+
+	it('never expires when no deadline was set', async () => {
+		// The default has to stay today's behaviour: a host that never asked
+		// for a TTL must not start losing approvals.
+		const parked = await park(undefined)
+		expect(parked.pending?.deadlineAt).toBeUndefined()
+		expect((await findPendingCheckpoint(store, SCOPE, { now: Date.now() + 10 ** 9 }))?.id).toBe(
+			parked.id,
+		)
+	})
+
+	it('is enumerable so a host can sweep it', async () => {
+		await park(1)
+		await new Promise((resolve) => setTimeout(resolve, 5))
+
+		// The out-of-process timer stays a host concern, but a host cannot
+		// sweep what it cannot enumerate.
+		expect(await listExpiredParks(store, SCOPE)).toHaveLength(1)
+	})
+
+	it('records the expiry rather than deleting the evidence', async () => {
+		const parked = await park(1)
+		await new Promise((resolve) => setTimeout(resolve, 5))
+
+		const expired = await manager.expire(parked.id)
+		expect(expired?.pending?.resolvedAt).toBeGreaterThan(0)
+		// A checkpoint showing what was asked and that nobody answered is
+		// the evidence an approval gate is worth having.
+		expect(expired?.pending?.request).toEqual(parked.pending?.request)
+		expect(expired?.pending?.decision).toMatchObject({ action: 'pause' })
+	})
+
+	it('does not re-expire a park that was already answered', async () => {
+		const parked = await park(1)
+		await manager.unpark(parked.id, { action: 'approve_tools' })
+		expect(await manager.expire(parked.id)).toBeNull()
+	})
+
+	it('applies the manager default when no per-park ttl is given', async () => {
+		manager.setParkTtl(30_000)
+		const parked = await park(undefined)
+		expect(parked.pending?.deadlineAt).toBe((parked.pending?.parkedAt ?? 0) + 30_000)
+	})
+})
+
+describe('deciding whether a park has expired', () => {
+	it('is false without a deadline, at any time', () => {
+		expect(isExpiredPark({ request: request(), parkedAt: 0 }, 10 ** 12)).toBe(false)
+	})
+
+	it('is true at the deadline, not only past it', () => {
+		expect(isExpiredPark({ request: request(), parkedAt: 0, deadlineAt: 100 }, 100)).toBe(true)
+	})
+})
+
+describe('projecting a run onto the session-layer status', () => {
+	const park = { request: request(), parkedAt: 0, deadlineAt: 100 }
+
+	it('produces the variant that never had a producer', () => {
+		// `awaiting_hitl_resolution` has documented a "persisted wait after
+		// a HITL timeout" since it was declared, for a timeout nothing could
+		// raise.
+		expect(deriveRunStatus({ status: 'running', park, now: 200 })).toBe('awaiting_hitl_resolution')
+	})
+
+	it('reports a live park as awaiting, not expired', () => {
+		expect(deriveRunStatus({ status: 'running', park, now: 50 })).toBe('awaiting_hitl')
+	})
+
+	it('lets a terminal run beat a stale park record', () => {
+		// A run that finished is not waiting for anyone, whatever the park
+		// record still says.
+		expect(deriveRunStatus({ status: 'completed', park, now: 200 })).toBe('succeeded')
+		expect(deriveRunStatus({ status: 'failed', park, now: 200 })).toBe('failed')
+	})
+
+	it('ignores a park that was already answered', () => {
+		expect(
+			deriveRunStatus({ status: 'running', park: { ...park, resolvedAt: 60 }, now: 200 }),
+		).toBe('running')
+	})
+
+	it('maps an unparked run without inventing a wait', () => {
+		expect(deriveRunStatus({ status: 'running' })).toBe('running')
+		expect(deriveRunStatus({ status: 'idle' })).toBe('queued')
+		expect(deriveRunStatus({ status: 'cancelled' })).toBe('cancelled')
+	})
+})
+
+describe('the trace a checkpoint was taken inside', () => {
+	let dir: string
+	let manager: CheckpointManager
+
+	beforeEach(async () => {
+		dir = await mkdtemp(join(tmpdir(), 'namzu-trace-'))
+		manager = new CheckpointManager(new DiskCheckpointStore({ baseDir: dir }), SCOPE)
+	})
+
+	afterEach(async () => {
+		await rm(dir, { recursive: true, force: true })
+	})
+
+	const fakeSpan = (traceId: string, spanId: string) =>
+		({ spanContext: () => ({ traceId, spanId, traceFlags: 1 }) }) as never
+
+	it('is recorded, so a resume can join it', async () => {
+		manager.setTraceSource(() => serializeSpan(fakeSpan('a'.repeat(32), 'b'.repeat(16))))
+		const checkpoint = await manager.create(runMgr(), 1)
+
+		expect(checkpoint.traceContext).toMatchObject({
+			traceId: 'a'.repeat(32),
+			spanId: 'b'.repeat(16),
+		})
+	})
+
+	it('is readable back without loading the whole restore path', async () => {
+		manager.setTraceSource(() => serializeSpan(fakeSpan('c'.repeat(32), 'd'.repeat(16))))
+		const checkpoint = await manager.create(runMgr(), 1)
+
+		// Read before the root span is minted, because a parent can only be
+		// set at creation.
+		expect((await manager.readTraceContext(checkpoint.id))?.traceId).toBe('c'.repeat(32))
+	})
+
+	it('never fails a resume over telemetry', async () => {
+		// A missing checkpoint here must not throw: the restore path
+		// immediately after reports it far better than a tracing helper can.
+		expect(await manager.readTraceContext('cp_missing' as CheckpointId)).toBeUndefined()
+	})
+
+	it('refuses an all-zero trace id rather than emitting an orphan', async () => {
+		// An exporter given a zero id drops the span silently, which turns
+		// "linked to the crash" into "no resume trace at all" — worse than
+		// the disconnected traces this replaces.
+		expect(serializeSpan(fakeSpan('0'.repeat(32), 'e'.repeat(16)))).toBeUndefined()
+	})
+
+	it('is absent when no telemetry is registered', async () => {
+		const checkpoint = await manager.create(runMgr(), 1)
+		expect(checkpoint.traceContext).toBeUndefined()
+	})
+})

package/src/runtime/query/__tests__/durable-question-park.test.ts

@@ -0,0 +1,305 @@
+import { describe, expect, it, vi } from 'vitest'
+
+import { buildCoordinatorTools } from '../../../tools/coordinator/index.js'
+import type {
+	CheckpointId,
+	HITLResumeDecision,
+	IterationCheckpoint,
+} from '../../../types/hitl/index.js'
+import type { RunId } from '../../../types/ids/index.js'
+import { createAssistantMessage, createUserMessage } from '../../../types/message/index.js'
+import type { Message } from '../../../types/message/index.js'
+import type { Logger } from '../../../utils/logger.js'
+import { PendingAnswers, QuestionParkBinding } from '../question-park.js'
+import { planPendingResume } from '../resume-pending.js'
+
+/**
+ * `ask_user_question` parked through the raw handler under a synthetic
+ * `cp_question_<toolUseId>` id that was never written. The checkpoint
+ * therefore did not exist: nothing on disk said a human owed this run an
+ * answer, and a remote host could not even OBSERVE the question except
+ * through the in-process callback.
+ *
+ * Kill the process while somebody is looking at the card and the answer
+ * could never be applied — the restore path stripped the whole assistant
+ * turn, discarding work sibling tools in the same batch had finished and
+ * re-billing the turn.
+ *
+ * The re-entry half is the one that was deferred: on resume the batch is
+ * re-executed, which is HOW the asking tool is re-entered, and the
+ * recorded answer is handed to it instead of a second question.
+ */
+
+const RID = 'run_1' as RunId
+
+function makeLogger(): Logger {
+	const self = { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() } as unknown as Logger
+	;(self as { child: (ctx: unknown) => Logger }).child = vi.fn(() => self)
+	return self
+}
+
+const QUESTION_INPUT = {
+	question: 'Which environment?',
+	options: [
+		{ label: 'Staging', description: 'safe' },
+		{ label: 'Production', description: 'not safe' },
+	],
+	multiSelect: false,
+	allowFreeText: true,
+}
+
+function askTool(opts: {
+	resumeHandler: (request: unknown) => Promise<HITLResumeDecision>
+	questionParks?: QuestionParkBinding
+	pendingAnswers?: PendingAnswers
+}) {
+	const tools = buildCoordinatorTools({
+		gateway: {} as never,
+		workingDirectory: '/tmp',
+		allowedAgentIds: [],
+		runId: RID,
+		resumeHandler: opts.resumeHandler as never,
+		...(opts.questionParks ? { questionParks: opts.questionParks } : {}),
+		...(opts.pendingAnswers ? { pendingAnswers: opts.pendingAnswers } : {}),
+	})
+	const tool = tools.find((t) => t.name === 'ask_user_question')
+	if (!tool) throw new Error('ask_user_question was not built')
+	return tool
+}
+
+const answer = (questionId: string): HITLResumeDecision => ({
+	action: 'answer_question',
+	selectedOptionIds: ['opt_1'],
+	questionId,
+})
+
+describe('recording the park', () => {
+	it('writes a real checkpoint and hands the tool its id', async () => {
+		const recorded: string[] = []
+		const parks = new QuestionParkBinding()
+		parks.bind({
+			record: async (q) => {
+				recorded.push(q.questionId)
+				return 'cp_real' as CheckpointId
+			},
+			resolve: async () => {},
+		})
+
+		let seenCheckpointId: string | undefined
+		const tool = askTool({
+			questionParks: parks,
+			resumeHandler: async (request) => {
+				seenCheckpointId = (request as { checkpointId: string }).checkpointId
+				return answer('t1')
+			},
+		})
+
+		await tool.execute(QUESTION_INPUT as never, { toolUseId: 't1' } as never)
+
+		expect(recorded).toEqual(['t1'])
+		// The synthetic id was never written anywhere, so a host given it
+		// could not read the checkpoint back.
+		expect(seenCheckpointId).toBe('cp_real')
+	})
+
+	it('clears the park once the answer arrives', async () => {
+		const resolved: string[] = []
+		const parks = new QuestionParkBinding()
+		parks.bind({
+			record: async () => 'cp_real' as CheckpointId,
+			resolve: async (id) => {
+				resolved.push(id)
+			},
+		})
+
+		const tool = askTool({ questionParks: parks, resumeHandler: async () => answer('t1') })
+		await tool.execute(QUESTION_INPUT as never, { toolUseId: 't1' } as never)
+
+		// An approval queue that keeps serving an answered question is the
+		// same defect the tool-review park already fixed.
+		expect(resolved).toEqual(['cp_real'])
+	})
+
+	it('still asks when the park cannot be recorded', async () => {
+		// An unrecorded park is a lost cross-process handoff, not a reason
+		// to fail the tool — the in-process await is still perfectly valid.
+		const parks = new QuestionParkBinding()
+		parks.bind({ record: async () => null, resolve: async () => {} })
+
+		const tool = askTool({ questionParks: parks, resumeHandler: async () => answer('t1') })
+		const result = await tool.execute(QUESTION_INPUT as never, { toolUseId: 't1' } as never)
+
+		expect(result.success).toBe(true)
+	})
+
+	it('is inert when nothing has bound it', async () => {
+		// The tool outlives the run that binds it, so an unbound channel is
+		// the normal state outside a run — and must behave exactly as it
+		// did before any of this existed.
+		const parks = new QuestionParkBinding()
+		expect(await parks.record({ questionId: 't1' } as never)).toBeNull()
+		await expect(parks.resolve('cp_x' as CheckpointId, answer('t1'))).resolves.toBeUndefined()
+	})
+
+	it('stops writing into a run that has settled', async () => {
+		const parks = new QuestionParkBinding()
+		parks.bind({ record: async () => 'cp_real' as CheckpointId, resolve: async () => {} })
+		parks.unbind()
+
+		expect(await parks.record({ questionId: 't1' } as never)).toBeNull()
+	})
+})
+
+describe('re-entering the tool with the answer', () => {
+	it('returns the carried answer without asking again', async () => {
+		const handler = vi.fn(async () => answer('t1'))
+		const pending = new PendingAnswers()
+		pending.set('t1', answer('t1'))
+
+		const tool = askTool({ pendingAnswers: pending, resumeHandler: handler })
+		const result = await tool.execute(QUESTION_INPUT as never, { toolUseId: 't1' } as never)
+
+		// Asking again would put a question the user already answered back
+		// in front of them — and headless, would auto-answer with the
+		// no-consent sentinel and throw the real answer away.
+		expect(handler).not.toHaveBeenCalled()
+		expect(result.success).toBe(true)
+		expect(String(result.output)).toContain('Staging')
+	})
+
+	it('does not record a park for a question it already has an answer to', async () => {
+		const recorded: string[] = []
+		const parks = new QuestionParkBinding()
+		parks.bind({
+			record: async (q) => {
+				recorded.push(q.questionId)
+				return 'cp_real' as CheckpointId
+			},
+			resolve: async () => {},
+		})
+		const pending = new PendingAnswers()
+		pending.set('t1', answer('t1'))
+
+		const tool = askTool({
+			questionParks: parks,
+			pendingAnswers: pending,
+			resumeHandler: async () => answer('t1'),
+		})
+		await tool.execute(QUESTION_INPUT as never, { toolUseId: 't1' } as never)
+
+		// Parking an answered question leaves an outstanding record for a
+		// decision that has been made.
+		expect(recorded).toEqual([])
+	})
+
+	it('consumes an answer once', async () => {
+		// A tool that asks the same question twice in one resumed run is
+		// asking something genuinely new the second time; answering it from
+		// a stale record would fabricate consent.
+		const pending = new PendingAnswers()
+		pending.set('t1', answer('t1'))
+
+		expect(pending.take('t1')).toBeDefined()
+		expect(pending.take('t1')).toBeUndefined()
+	})
+
+	it('falls through to a real ask for a question it has no answer to', async () => {
+		const handler = vi.fn(async () => answer('t2'))
+		const pending = new PendingAnswers()
+		pending.set('t1', answer('t1'))
+
+		const tool = askTool({ pendingAnswers: pending, resumeHandler: handler })
+		await tool.execute(QUESTION_INPUT as never, { toolUseId: 't2' } as never)
+
+		expect(handler).toHaveBeenCalledTimes(1)
+	})
+
+	it('carries only an answer that names its question', async () => {
+		// The misdirection guard: a decision with no questionId cannot be
+		// matched to a call, so it must not be delivered to whichever tool
+		// happens to run next.
+		const unaddressed = PendingAnswers.from({
+			action: 'answer_question',
+			selectedOptionIds: ['opt_1'],
+		})
+		expect(unaddressed.size).toBe(0)
+
+		const approval = PendingAnswers.from({ action: 'approve_tools' })
+		expect(approval.size).toBe(0)
+	})
+})
+
+describe('planning the resume', () => {
+	const parkedTurn = (): Message[] => [
+		createUserMessage('deploy it'),
+		{
+			...createAssistantMessage(''),
+			toolCalls: [
+				{
+					id: 't1',
+					type: 'function' as const,
+					function: { name: 'ask_user_question', arguments: '{}' },
+				},
+				{ id: 't2', type: 'function' as const, function: { name: 'read', arguments: '{}' } },
+			],
+		} as Message,
+	]
+
+	const checkpoint = (questionId: string, messages = parkedTurn()): IterationCheckpoint =>
+		({
+			id: 'cp_1' as CheckpointId,
+			messages,
+			pending: {
+				parkedAt: 0,
+				request: {
+					type: 'user_question',
+					runId: RID,
+					checkpointId: 'cp_1' as CheckpointId,
+					question: {
+						questionId,
+						question: 'Which?',
+						options: [],
+						multiSelect: false,
+						allowFreeText: true,
+					},
+				},
+			},
+		}) as unknown as IterationCheckpoint
+
+	it('takes over a question park instead of declining it', async () => {
+		// This used to return null with "out of scope", so the restore path
+		// stripped the turn and the answer was lost.
+		const plan = planPendingResume(checkpoint('t1'), answer('t1'), makeLogger())
+
+		expect(plan).not.toBeNull()
+		// The whole batch is re-executed — that is how the asking tool gets
+		// re-entered at all.
+		expect(plan?.response.message.toolCalls).toHaveLength(2)
+	})
+
+	it('carries the answer for the call that asked', async () => {
+		const plan = planPendingResume(checkpoint('t1'), answer('t1'), makeLogger())
+		expect(plan?.answers?.take('t1')).toBeDefined()
+	})
+
+	it('denies nothing — a question is not an approval gate', async () => {
+		const plan = planPendingResume(checkpoint('t1'), answer('t1'), makeLogger())
+		expect(plan?.denials.size).toBe(0)
+	})
+
+	it('refuses an answer to a question not in this turn', async () => {
+		// A stale client answering an earlier question would otherwise have
+		// its answer delivered to whatever tool now holds that slot.
+		const plan = planPendingResume(checkpoint('t9'), answer('t9'), makeLogger())
+		expect(plan).toBeNull()
+	})
+
+	it('refuses a checkpoint whose turn is already answered', async () => {
+		const answered: Message[] = [
+			...parkedTurn(),
+			{ role: 'tool', content: 'staging', toolCallId: 't1' } as Message,
+			{ role: 'tool', content: 'read', toolCallId: 't2' } as Message,
+		]
+		expect(planPendingResume(checkpoint('t1', answered), answer('t1'), makeLogger())).toBeNull()
+	})
+})