@namzu/sdk 58.1.1 → 60.0.0

package/src/execution/workflow/resume.ts

@@ -0,0 +1,313 @@
+import { createHash, createHmac, timingSafeEqual } from 'node:crypto'
+
+import type {
+	ChainedWorkflowJournalRecord,
+	CheckedWorkflowJournal,
+	WorkflowFinishStatus,
+	WorkflowJournalRecord,
+	WorkflowResumeState,
+} from './types.js'
+
+/** The key version. A change to how keys are made bumps it, so old keys simply never match. */
+const KEY_VERSION = 'k1'
+
+function sha256Hex(text: string): string {
+	return createHash('sha256').update(text).digest('hex')
+}
+
+/** JSON with every object's keys sorted and `__proto__` dropped, so equal values give equal text. */
+function canonicalJson(value: unknown): string {
+	if (Array.isArray(value)) return `[${value.map((item) => canonicalJson(item)).join(',')}]`
+	if (typeof value === 'object' && value !== null) {
+		const record = value as Record<string, unknown>
+		const keys = Object.keys(record)
+			.filter((key) => key !== '__proto__' && record[key] !== undefined)
+			.sort()
+		return `{${keys.map((key) => `${JSON.stringify(key)}:${canonicalJson(record[key])}`).join(',')}}`
+	}
+	return JSON.stringify(value) ?? 'null'
+}
+
+/** What of an `agent()` call decides its result. `label`, `phase` and progress callbacks do not. */
+export interface WorkflowCallIdentity {
+	readonly schema?: unknown
+	readonly model?: string
+	readonly effort?: string
+	readonly agentType?: string
+}
+
+/**
+ * The key a call is saved and looked up under: the prompt, the options that
+ * change what the agent does (schema, model, effort, agent type) and how many
+ * identical calls came before it in the same pass. Each part is length-prefixed
+ * so one cannot be read as part of another.
+ */
+export function workflowCallKey(
+	prompt: string,
+	call: WorkflowCallIdentity,
+	ordinal: number,
+): string {
+	return `${KEY_VERSION}:${sha256Hex(`${workflowCallContent(prompt, call)}${ordinal}`)}`
+}
+
+/**
+ * What makes two calls the same call: the prompt and the options that change
+ * the agent, in a form where key order and `undefined` do not matter. The
+ * runtime counts repeats of this text to give identical calls distinct slots,
+ * and the key hashes the same text, so the two can never disagree.
+ */
+export function workflowCallContent(prompt: string, call: WorkflowCallIdentity): string {
+	const options = canonicalJson({
+		schema: call.schema,
+		model: call.model,
+		effort: call.effort,
+		agentType: call.agentType,
+	})
+	return `${prompt.length}:${prompt}${options.length}:${options}`
+}
+
+/** The hash the first line of a run's journal chains from. It binds the file to its run id. */
+export function workflowJournalSeed(runId: string): string {
+	return sha256Hex(`namzu-workflow-journal\0${runId}`)
+}
+
+/** The hash a line contributes to the next one: its exact bytes, newline included. */
+export function workflowJournalLineHash(line: string): string {
+	return sha256Hex(`${line}\n`)
+}
+
+/** The line `record` is written as. With a key, a `mac` over the rest of the line is added last. */
+export function formatWorkflowJournalLine(
+	record: WorkflowJournalRecord,
+	seq: number,
+	prev: string,
+	key?: string | Uint8Array,
+): string {
+	const body = { seq, prev, ...record }
+	if (key === undefined) return JSON.stringify(body)
+	const mac = createHmac('sha256', key).update(JSON.stringify(body)).digest('hex')
+	return JSON.stringify({ ...body, mac })
+}
+
+function macMatches(record: Record<string, unknown>, key: string | Uint8Array): boolean {
+	const { mac, ...rest } = record
+	if (typeof mac !== 'string') return false
+	const expected = createHmac('sha256', key).update(JSON.stringify(rest)).digest()
+	const actual = Buffer.from(mac, 'hex')
+	return actual.length === expected.length && timingSafeEqual(actual, expected)
+}
+
+function parseLine(line: string): Record<string, unknown> | undefined {
+	try {
+		const value: unknown = JSON.parse(line)
+		return typeof value === 'object' && value !== null && !Array.isArray(value)
+			? (value as Record<string, unknown>)
+			: undefined
+	} catch {
+		return undefined
+	}
+}
+
+/**
+ * Reads a journal's text and keeps only the records that can be trusted.
+ *
+ * Every line carries `seq` (counting trusted lines from 1) and `prev` (the hash
+ * of the line before; the first chains from a seed bound to the run id). A
+ * line that does not chain, a line that does not parse, and anything after
+ * one is dropped until a `run_started` of a resumed pass picks the chain up
+ * again at the last trusted line. A last line without its newline that is not
+ * whole JSON is a torn write and is ignored; one that is whole JSON is read
+ * like any other line. The file is never rewritten.
+ *
+ * The chain alone cannot notice an edit to the final line (nothing chains from
+ * it yet). With `key`, every line must also carry a `mac` made with that
+ * secret, which closes that gap against anyone who does not hold the key.
+ */
+export function readWorkflowJournalChecked(
+	text: string,
+	runId: string,
+	options: { readonly key?: string | Uint8Array } = {},
+): CheckedWorkflowJournal {
+	const endsWithNewline = text === '' || text.endsWith('\n')
+	const pieces = text.split('\n')
+	const fragment = endsWithNewline ? undefined : pieces.pop()
+	if (endsWithNewline) pieces.pop()
+	// A last line without its newline that is whole JSON was only missing the newline: it is
+	// judged like any other line (the writer adds the newline before the next record, so the
+	// hash it contributes is the one of the line with its newline). Anything else is a torn write.
+	const wholeFragment =
+		fragment !== undefined && fragment !== '' && parseLine(fragment) !== undefined
+	if (wholeFragment) pieces.push(fragment as string)
+	const tornTail = fragment !== undefined && fragment !== '' && !wholeFragment
+
+	const parsed = pieces.map((line) => parseLine(line))
+	const firstRecord = parsed.find((record) => record !== undefined)
+	if (firstRecord !== undefined && typeof firstRecord.seq !== 'number') {
+		return {
+			records: [],
+			legacy: true,
+			tornTail,
+			lastSeq: 0,
+			lastHash: workflowJournalSeed(runId),
+			endsWithNewline,
+			untrusted: parsed.filter((record) => record !== undefined) as WorkflowJournalRecord[],
+		}
+	}
+
+	const records: ChainedWorkflowJournalRecord[] = []
+	let lastSeq = 0
+	let lastHash = workflowJournalSeed(runId)
+	let gapStart: number | undefined
+	let gapLines = 0
+	let gapParsed = false
+	let brokenAt: number | undefined
+	const closeGap = (): void => {
+		if (gapStart !== undefined && (gapLines > 1 || gapParsed)) brokenAt ??= gapStart
+		gapStart = undefined
+		gapLines = 0
+		gapParsed = false
+	}
+
+	for (let i = 0; i < pieces.length; i++) {
+		const line = pieces[i] as string
+		const record = parsed[i]
+		const chains =
+			record !== undefined &&
+			record.seq === lastSeq + 1 &&
+			record.prev === lastHash &&
+			(options.key === undefined || macMatches(record, options.key))
+		const first = records.length === 0
+		const acceptable =
+			chains &&
+			(first
+				? record.type === 'run_started' && record.runId === runId
+				: gapStart === undefined || record.type === 'run_started')
+		if (acceptable) {
+			closeGap()
+			records.push(record as unknown as ChainedWorkflowJournalRecord)
+			lastSeq = record.seq as number
+			lastHash = workflowJournalLineHash(line)
+			continue
+		}
+		if (gapStart === undefined) gapStart = i + 1
+		gapLines++
+		if (record !== undefined) gapParsed = true
+	}
+	// A gap that runs to the end is a change, except a single line that does not parse
+	// (what a torn write leaves once the next pass closed it with a newline).
+	closeGap()
+
+	return {
+		records,
+		legacy: false,
+		tornTail,
+		...(brokenAt !== undefined ? { brokenAt } : {}),
+		lastSeq,
+		lastHash,
+		endsWithNewline,
+		untrusted: [],
+	}
+}
+
+function hasResult(record: { readonly result?: unknown }): boolean {
+	return Object.hasOwn(record, 'result') && record.result !== undefined && record.result !== null
+}
+
+/**
+ * Folds a checked journal into what a resumed pass needs.
+ *
+ * The last saved result for a key wins. An agent that was asked for again
+ * after a result was saved gives that result up until it finishes again. A
+ * failed or skipped outcome removes the result and marks the key failed; an
+ * outcome the run recorded as interrupted (stopped, not failed) leaves the key
+ * as seen but unfinished. Lines marked `cached` repeat an earlier result and
+ * are skipped, and so is any agent record from a pass older than the latest
+ * `run_started`, which a slow agent of an earlier pass can still write.
+ */
+export function loadWorkflowResume(checked: CheckedWorkflowJournal): WorkflowResumeState {
+	const results = new Map<
+		string,
+		{ value: unknown; pass: number; index: number; childSessionId?: string }
+	>()
+	const seen = new Set<string>()
+	const failed = new Set<string>()
+	const demoted = new Set<string>()
+	const problems: string[] = []
+	const keysByIndex = new Map<string, string>()
+	let currentPass = 0
+	let scriptSha256: string | undefined
+	let previousFinish: WorkflowFinishStatus | 'unfinished' = 'unfinished'
+
+	for (const record of checked.records) {
+		if (record.type === 'run_started') {
+			currentPass = record.resume?.pass ?? 1
+			scriptSha256 = record.scriptSha256
+			previousFinish = 'unfinished'
+			keysByIndex.clear()
+			continue
+		}
+		if (record.type === 'run_finished') {
+			previousFinish = record.status
+			continue
+		}
+		if (record.type !== 'agent_request' && record.type !== 'agent_outcome') continue
+		const pass = record.pass ?? 1
+		if (pass < currentPass || record.cached === true) continue
+		if (record.type === 'agent_request') {
+			if (record.key === undefined) continue
+			keysByIndex.set(`${pass}:${record.index}`, record.key)
+			seen.add(record.key)
+			if (results.delete(record.key)) demoted.add(record.key)
+			continue
+		}
+		const key = record.key ?? keysByIndex.get(`${pass}:${record.index}`)
+		if (key === undefined) continue
+		seen.add(key)
+		if (record.state === 'done' && hasResult(record)) {
+			results.set(key, {
+				value: record.result,
+				pass,
+				index: record.index,
+				...(record.childSessionId !== undefined ? { childSessionId: record.childSessionId } : {}),
+			})
+			failed.delete(key)
+			demoted.delete(key)
+		} else if (record.state === 'done' && record.unsaved === true) {
+			results.delete(key)
+			demoted.add(key)
+		} else if (record.interrupted === true) {
+			// Stopped, not failed: the agent runs again and the cache stays open.
+			results.delete(key)
+		} else {
+			results.delete(key)
+			demoted.delete(key)
+			failed.add(key)
+		}
+	}
+
+	if (checked.legacy) {
+		problems.push(
+			'This run was saved before resuming was possible, so its agents cannot be reused.',
+		)
+	}
+	if (checked.brokenAt !== undefined) {
+		problems.push(
+			`The journal was changed or cut short at line ${checked.brokenAt}. Results after that point are not trusted and those agents will run again.`,
+		)
+	}
+	const started = checked.records.some((record) => record.type === 'run_started')
+	if (!checked.legacy && !started) problems.push('The journal has no usable record of the run.')
+
+	return {
+		pass: currentPass + 1,
+		results,
+		seen,
+		failed,
+		demoted,
+		...(scriptSha256 !== undefined ? { scriptSha256 } : {}),
+		previousFinish,
+		problems,
+		resumable: !checked.legacy && started,
+		...(checked.brokenAt !== undefined ? { brokenAt: checked.brokenAt } : {}),
+	}
+}

package/src/execution/workflow/runtime.ts

@@ -9,6 +9,7 @@ import { preview, sha256 } from './journal.js'
 import { splitWorkflowScript } from './meta.js'
 import { type PhaseAgentView, PhaseRegistry, summarizeWorkflowPhases } from './phases.js'
 import { buildPrelude, preludeLineCount } from './prelude.js'
+import { workflowCallContent, workflowCallKey } from './resume.js'
 import { parseJsonText, validateAgainstSchema } from './schema.js'
 import type {
 	WorkflowAgentEvent,
@@ -83,7 +84,9 @@ export function resolveWorkflowLimits(limits: WorkflowLimits = {}): WorkflowReso
 			16 * 1024 * 1024,
 		),
 		...(limits.tokenBudget !== undefined
-			? { tokenBudget: integer('tokenBudget', limits.tokenBudget, 1, Number.MAX_SAFE_INTEGER) }
+			? {
+					tokenBudget: integer('tokenBudget', limits.tokenBudget, 1, Number.MAX_SAFE_INTEGER),
+				}
 			: {}),
 		maxDurationMs: integer(
 			'maxDurationMs',
@@ -198,6 +201,14 @@ interface AgentRecord {
 	error?: string
 	skipped: boolean
 	userSkipped: boolean
+	key?: string
+	/** Set when the result came from an earlier pass instead of a run. */
+	cached?: {
+		readonly pass: number
+		readonly index: number
+		readonly childSessionId?: string
+	}
+	childSessionId?: string
 }
 
 interface Waiter {
@@ -283,6 +294,10 @@ export class WorkflowRuntime {
 		const runId =
 			options.runId ?? `w${[...randomBytes(8)].map((b) => (b % 36).toString(36)).join('')}`
 		const startedAt = now()
+		const pass = options.resume?.pass ?? 1
+		const resume = options.resume
+		let latch = false
+		const seenContent = new Map<string, number>()
 		const phases = new PhaseRegistry()
 		const agents = new Map<number, AgentRecord>()
 		const inFlight = new Set<Promise<unknown>>()
@@ -335,7 +350,13 @@ export class WorkflowRuntime {
 					declared,
 					at,
 				})
-				journal({ type: 'phase', at, index: phase.index, title: phase.title, declared })
+				journal({
+					type: 'phase',
+					at,
+					index: phase.index,
+					title: phase.title,
+					declared,
+				})
 			}
 			return phase
 		}
@@ -361,6 +382,7 @@ export class WorkflowRuntime {
 			...(agent.resultPreview !== undefined ? { resultPreview: agent.resultPreview } : {}),
 			...(agent.error !== undefined ? { error: agent.error } : {}),
 			...(agent.skipped ? { skipped: true } : {}),
+			...(agent.cached !== undefined ? { cached: true } : {}),
 		})
 
 		// FIFO admission: an agent starts when fewer than maxConcurrency run.
@@ -411,7 +433,13 @@ export class WorkflowRuntime {
 
 		const settle = (
 			agent: AgentRecord,
-			outcome: { state: 'done' | 'error'; skipped?: boolean; value?: unknown; error?: string },
+			outcome: {
+				state: 'done' | 'error'
+				skipped?: boolean
+				value?: unknown
+				error?: string
+				interrupted?: boolean
+			},
 		): HostCallResult => {
 			const at = now()
 			agent.state = outcome.state
@@ -434,11 +462,25 @@ export class WorkflowRuntime {
 				value = null
 			}
 			emit(agentEvent(agent))
+			const finalState = agent.state === 'error' ? 'error' : agent.skipped ? 'skipped' : 'done'
+			let saved: { result: unknown } | { unsaved: true } | undefined
+			if (finalState === 'done' && value !== null && agent.cached === undefined) {
+				try {
+					saved = { result: JSON.parse(JSON.stringify(value)) as unknown }
+				} catch {
+					saved = { unsaved: true }
+					journal({
+						type: 'log',
+						at,
+						message: "An agent's result could not be saved, so a resume will run it again.",
+					})
+				}
+			}
 			journal({
 				type: 'agent_outcome',
 				at,
 				index: agent.index,
-				state: agent.state === 'error' ? 'error' : agent.skipped ? 'skipped' : 'done',
+				state: finalState,
 				tokens: agent.tokens,
 				toolCalls: agent.toolCalls,
 				queuedAt: agent.queuedAt,
@@ -446,6 +488,22 @@ export class WorkflowRuntime {
 				finishedAt: at,
 				...(agent.resultPreview !== undefined ? { resultPreview: agent.resultPreview } : {}),
 				...(agent.error !== undefined ? { error: agent.error } : {}),
+				...(agent.key !== undefined ? { key: agent.key } : {}),
+				pass,
+				...(agent.cached !== undefined
+					? {
+							cached: true as const,
+							resultFrom: {
+								pass: agent.cached.pass,
+								index: agent.cached.index,
+							},
+						}
+					: {}),
+				...(saved ?? {}),
+				...(outcome.interrupted === true && finalState === 'error'
+					? { interrupted: true as const }
+					: {}),
+				...(agent.childSessionId !== undefined ? { childSessionId: agent.childSessionId } : {}),
 			})
 			return { ok: true, value: { v: value, t: totalTokens() } }
 		}
@@ -457,6 +515,7 @@ export class WorkflowRuntime {
 		): HostCallResult => {
 			agent.tokens = count(outcome.tokens)
 			agent.toolCalls = count(outcome.toolCalls)
+			if (outcome.sessionId !== undefined) agent.childSessionId = outcome.sessionId
 			if (agent.userSkipped || outcome.state === 'skipped') {
 				return settle(agent, { state: 'done', skipped: true })
 			}
@@ -464,10 +523,14 @@ export class WorkflowRuntime {
 				return settle(agent, {
 					state: 'error',
 					error: 'The workflow was stopped before this agent finished.',
+					interrupted: true,
 				})
 			}
 			if (outcome.state === 'error') {
-				return settle(agent, { state: 'error', error: outcome.error ?? 'The agent failed.' })
+				return settle(agent, {
+					state: 'error',
+					error: outcome.error ?? 'The agent failed.',
+				})
 			}
 			if (call.schema === undefined) {
 				return settle(agent, { state: 'done', value: outcome.text ?? '' })
@@ -503,6 +566,18 @@ export class WorkflowRuntime {
 				return { ok: false, error: reason }
 			}
 			const index = ++requested
+			const contentId = workflowCallContent(call.prompt, call)
+			const ordinal = seenContent.get(contentId) ?? 0
+			seenContent.set(contentId, ordinal + 1)
+			const key = workflowCallKey(call.prompt, call, ordinal)
+			const hit = latch ? undefined : resume?.results.get(key)
+			if (
+				resume !== undefined &&
+				hit === undefined &&
+				!(resume.seen.has(key) && !resume.failed.has(key) && !resume.demoted.has(key))
+			) {
+				latch = true
+			}
 			const phaseTitle = call.phase ?? call.cursor
 			const phase = phaseTitle?.trim() ? announce(phaseTitle, false) : undefined
 			const queuedAt = now()
@@ -521,13 +596,26 @@ export class WorkflowRuntime {
 				toolCalls: 0,
 				skipped: false,
 				userSkipped: false,
+				key,
 			}
 			agents.set(index, agent)
+			if (hit !== undefined) {
+				agent.cached = {
+					pass: hit.pass,
+					index: hit.index,
+					...(hit.childSessionId !== undefined ? { childSessionId: hit.childSessionId } : {}),
+				}
+				if (hit.childSessionId !== undefined) agent.childSessionId = hit.childSessionId
+				agent.startedAt = queuedAt
+			}
 			emit(agentEvent(agent))
 			journal({
 				type: 'agent_request',
 				at: queuedAt,
 				index,
+				key,
+				pass,
+				...(hit !== undefined ? { cached: true as const } : {}),
 				label: agent.label,
 				...(phase ? { phaseIndex: phase.index, phaseTitle: phase.title } : {}),
 				promptPreview: preview(call.prompt),
@@ -540,6 +628,12 @@ export class WorkflowRuntime {
 				},
 			})
 
+			if (hit !== undefined) {
+				return settle(agent, {
+					state: 'done',
+					value: structuredClone(hit.value),
+				})
+			}
 			const admitted = await acquire(agent)
 			if (!admitted) {
 				return agent.userSkipped
@@ -547,6 +641,7 @@ export class WorkflowRuntime {
 					: settle(agent, {
 							state: 'error',
 							error: 'The workflow was stopped before this agent started.',
+							interrupted: true,
 						})
 			}
 			try {
@@ -554,6 +649,7 @@ export class WorkflowRuntime {
 					return settle(agent, {
 						state: 'error',
 						error: `Not started: the token budget for this workflow (${limits.tokenBudget} tokens) is spent.`,
+						interrupted: true,
 					})
 				}
 				if (controller.signal.aborted || agent.userSkipped) {
@@ -562,6 +658,7 @@ export class WorkflowRuntime {
 						: settle(agent, {
 								state: 'error',
 								error: 'The workflow was stopped before this agent started.',
+								interrupted: true,
 							})
 				}
 				agent.startedAt = now()
@@ -630,7 +727,7 @@ export class WorkflowRuntime {
 		// --- start ---
 		if (options.journal?.writeScript) {
 			try {
-				await options.journal.writeScript(options.script, sha256(options.script))
+				await options.journal.writeScript(options.script, sha256(options.script), pass)
 			} catch (error) {
 				journalError = error instanceof Error ? error.message : String(error)
 			}
@@ -644,6 +741,17 @@ export class WorkflowRuntime {
 			meta,
 			args,
 			limits,
+			...(resume !== undefined
+				? {
+						resume: {
+							pass,
+							previousFinish: resume.previousFinish,
+							scriptSha256Changed:
+								resume.scriptSha256 !== undefined && resume.scriptSha256 !== sha256(options.script),
+						},
+					}
+				: {}),
+			...(options.scriptPath !== undefined ? { scriptPath: options.scriptPath } : {}),
 		})
 		emit({
 			type: 'workflow_started',
@@ -665,7 +773,11 @@ export class WorkflowRuntime {
 				maxPendingHostCalls: limits.maxAgents + 100,
 			})
 
-		const preludeConfig = { args, tokenLimit: limits.tokenBudget ?? null }
+		const preludeConfig = {
+			args,
+			tokenLimit: limits.tokenBudget ?? null,
+			deterministic: options.deterministic === true,
+		}
 		const runResult = await codeRuntime.run({
 			source: `${buildPrelude(preludeConfig)}\n${body}`,
 			allowedCalls: [WORKFLOW_HOST_CALL],
@@ -676,7 +788,10 @@ export class WorkflowRuntime {
 			onOutput,
 			onHostCall: (request) => {
 				if (request.name !== WORKFLOW_HOST_CALL) {
-					return Promise.resolve({ ok: false, error: `Unknown host call "${request.name}".` })
+					return Promise.resolve({
+						ok: false,
+						error: `Unknown host call "${request.name}".`,
+					})
 				}
 				let call: AgentCall
 				try {
@@ -738,7 +853,7 @@ export class WorkflowRuntime {
 			const progress =
 				begun.length === 0
 					? 'No agent had started.'
-					: `Before it failed: ${begun.length} agent${begun.length === 1 ? '' : 's'} started, ${finished} finished, ${failedAgents} failed. A rerun starts the whole script again, so finished agents run again.`
+					: `Before it failed: ${begun.length} agent${begun.length === 1 ? '' : 's'} started, ${finished} finished, ${failedAgents} failed. ${options.canResume === true ? 'Resume the run to continue: agents whose prompt and options are unchanged return their saved results.' : 'A rerun starts the whole script again, so finished agents run again.'}`
 			error = `${describeScriptFailure(outcome, body, preludeLineCount(preludeConfig))}\n${progress}${scriptPhase !== undefined ? ` Last phase: "${scriptPhase}".` : ''}`
 		}
 
@@ -751,6 +866,9 @@ export class WorkflowRuntime {
 			tokens: all.reduce((sum, a) => sum + a.tokens, 0),
 			toolCalls: all.reduce((sum, a) => sum + a.toolCalls, 0),
 			elapsedMs: Math.max(0, now() - startedAt),
+			...(all.some((a) => a.cached !== undefined)
+				? { cachedAgents: all.filter((a) => a.cached !== undefined).length }
+				: {}),
 		}
 		const at = now()
 		emit({