@namzu/sdk 49.0.0 → 50.0.0

package/src/runtime/jobs/registry.ts

@@ -1,9 +1,14 @@
-import { spawn } from 'node:child_process'
+import type { spawn } from 'node:child_process'
 
 import { SANDBOX_KILL_GRACE_MS } from '../../constants/sandbox/index.js'
 import { killTree } from '../../process/kill-tree.js'
-import { hostShellSpawn } from '../../tools/command-shell.js'
+import { describeSpawnFailure, spawnHostShell } from '../../tools/command-shell.js'
 import { scrubInheritedEnv } from '../../tools/env-scrub.js'
+import { createHostOutputDecoder } from '../../tools/host-output.js'
+import type {
+	BackgroundJobOutputWaitOptions,
+	BackgroundJobOutputWaitResult,
+} from '../../types/job/index.js'
 import { subscribeToAbort } from '../../utils/abort.js'
 
 /**
@@ -40,6 +45,17 @@ export interface BackgroundJob {
 	readonly exitedAt?: number
 	readonly exitCode?: number
 	readonly signal?: string
+	/**
+	 * Why the command never started, in words for the model: set when the
+	 * process could not be spawned at all (missing shell, bad working
+	 * directory). Such a job has no `exitCode`; it is not a command that
+	 * ran and returned a status.
+	 */
+	readonly error?: string
+	/** Stopping the owned tree failed; the job remains owned and counted as live. */
+	readonly recoveryRequired?: boolean
+	/** Safe registry diagnostic for an unconfirmed stop; excludes transport secrets. */
+	readonly stopError?: string
 }
 
 export interface BackgroundJobOutput {
@@ -57,6 +73,8 @@ export interface BackgroundJobOutput {
 	readonly droppedBytes: number
 	readonly status: BackgroundJobStatus
 	readonly exitCode?: number
+	/** Set when the command could not be started; see {@link BackgroundJob.error}. */
+	readonly error?: string
 }
 
 /** A job's process, when something other than the registry starts it. */
@@ -64,6 +82,8 @@ export interface JobProcess {
 	readonly child: ReturnType<typeof spawn>
 	/** How to stop it, when the registry's process-group kill would not reach everything. */
 	kill?(signal: NodeJS.Signals): void
+	/** Confirm the owned tree stopped; reject instead of claiming an unconfirmed exit. */
+	terminate?(signal?: NodeJS.Signals): Promise<void>
 }
 
 export interface StartJobParams {
@@ -90,10 +110,47 @@ export interface BackgroundJobRegistryConfig {
 	readonly maxJobsPerOwner?: number
 	/** Retained output per job. Oldest bytes go first, and are counted. */
 	readonly maxOutputBytesPerJob?: number
+	/** Concurrent output observers per owner; default 32. Refused rather than queued. */
+	readonly maxOutputWaitersPerOwner?: number
+	/** Concurrent output observers across the registry; default 256. */
+	readonly maxOutputWaiters?: number
 }
 
 const DEFAULT_MAX_JOBS_PER_OWNER = 8
 const DEFAULT_MAX_OUTPUT_BYTES = 1024 * 1024
+const MAX_CHANNEL_CHUNKS = 4096
+const MAX_WAIT_RESULT_BYTES = 32 * 1024
+const MAX_OUTPUT_WAIT_MS = 60 * 60 * 1000
+
+/** Starting another observer would exceed its owner's or registry's bound. */
+export class BackgroundJobOutputWaitLimitError extends Error {
+	constructor(readonly details: { owner: string; limit: number; scope: 'owner' | 'registry' }) {
+		super(
+			`Background output wait limit ${details.limit} reached for ${details.scope}; finish or cancel an existing wait first.`,
+		)
+		this.name = 'BackgroundJobOutputWaitLimitError'
+	}
+}
+
+interface OutputChunk {
+	readonly stream: 'stdout' | 'stderr'
+	readonly offset: number
+	readonly bytes: Buffer
+}
+
+/** Start at a complete UTF-8 code point when a byte cap or cursor bisects one. */
+function utf8Start(bytes: Buffer, start: number): number {
+	let at = start
+	while (at < bytes.length && ((bytes[at] ?? 0) & 0xc0) === 0x80) at++
+	return at
+}
+
+function outputTail(text: string, maxBytes: number): { text: string; omitted: number } {
+	const bytes = Buffer.from(text)
+	if (bytes.length <= maxBytes) return { text, omitted: 0 }
+	const start = utf8Start(bytes, bytes.length - maxBytes)
+	return { text: bytes.subarray(start).toString('utf8'), omitted: start }
+}
 
 /** A start that would exceed a declared bound. */
 export class BackgroundJobLimitError extends Error {
@@ -124,11 +181,21 @@ interface JobEntry {
 	child: ReturnType<typeof spawn>
 	/** The spawner's own kill, when it gave one. */
 	killProcess?: (signal: NodeJS.Signals) => void
+	terminateProcess?: (signal?: NodeJS.Signals) => Promise<void>
+	terminationAttempt?: Promise<void>
+	killRequested?: boolean
+	closedResult?: { code: number | null; signal: NodeJS.Signals | null }
+	finalize?: (code: number | null, signal: NodeJS.Signals | null) => void
 	/** Retained tail. */
 	buffer: string
 	/** Bytes produced in total, including the ones the cap dropped. */
 	produced: number
+	/** Channel identity is retained separately, under the same byte cap and a chunk cap. */
+	outputChunks: OutputChunk[]
+	outputObservers: Set<(chunk?: OutputChunk) => void>
 	exit: Promise<void>
+	/** Settles once the process exists or failed to; `error` says which. */
+	started: Promise<{ readonly error?: string }>
 }
 
 /** Whether any process of the group led by `pid` is still alive. */
@@ -148,6 +215,8 @@ export class BackgroundJobRegistry {
 	private readonly jobs = new Map<string, JobEntry>()
 	private counter = 0
 	private readonly exitListeners = new Set<(job: BackgroundJob) => void>()
+	private outputWaiters = 0
+	private readonly outputWaitersByOwner = new Map<string, number>()
 
 	/**
 	 * Be told when a job ends, whoever owns it. A job outlives the call that
@@ -172,7 +241,43 @@ export class BackgroundJobRegistry {
 		}
 	}
 
-	constructor(private readonly config: BackgroundJobRegistryConfig = {}) {}
+	/** A failed remote stop never releases ownership or announces a terminal job. */
+	private confirmTermination(entry: JobEntry): Promise<void> {
+		if (entry.terminationAttempt) return entry.terminationAttempt
+		const terminate = entry.terminateProcess
+		if (!terminate) return Promise.reject(new Error('The process has no confirmed termination API'))
+		const attempt = Promise.resolve()
+			.then(() => terminate('SIGTERM'))
+			.then(() => {
+				const { recoveryRequired: _recovery, stopError: _error, ...record } = entry.record
+				entry.record = record
+				if (entry.closedResult) entry.finalize?.(entry.closedResult.code, entry.closedResult.signal)
+			})
+			.catch((error: unknown) => {
+				entry.record = {
+					...entry.record,
+					recoveryRequired: true,
+					stopError:
+						'The process provider could not confirm the owned job stopped; retry termination or recover its environment.',
+				}
+				for (const observer of entry.outputObservers) observer()
+				throw error
+			})
+			.finally(() => {
+				entry.terminationAttempt = undefined
+			})
+		entry.terminationAttempt = attempt
+		return attempt
+	}
+
+	constructor(private readonly config: BackgroundJobRegistryConfig = {}) {
+		for (const key of ['maxOutputWaiters', 'maxOutputWaitersPerOwner'] as const) {
+			const value = config[key]
+			if (value !== undefined && (!Number.isSafeInteger(value) || value < 1)) {
+				throw new RangeError(`${key} must be a positive safe integer.`)
+			}
+		}
+	}
 
 	private get maxJobs(): number {
 		return this.config.maxJobsPerOwner ?? DEFAULT_MAX_JOBS_PER_OWNER
@@ -206,33 +311,42 @@ export class BackgroundJobRegistry {
 		// started it, which makes the leak longer-lived, not smaller.
 		const inherited = scrubInheritedEnv()
 
-		// The same shell as a foreground `bash` call, so the permission
-		// rules' reading of the line holds for the job too.
-		const shell = hostShellSpawn(params.command, { ...inherited.env, ...params.env })
+		// The same spawn as a foreground `bash` call, so the permission
+		// rules' reading of the line holds for the job too, and so the two
+		// cannot disagree about how a shell is started on a given platform.
 		const started = params.spawn
 			? params.spawn()
 			: {
-					child: spawn(
-						shell.file ?? '/bin/sh',
-						shell.file === undefined ? ['-c', params.command] : [...shell.args],
-						{
-							cwd: params.workingDirectory,
-							env: shell.env,
-							// Leader of its own process group, which is what `killTree` needs
-							// to reach the command and everything it forks rather than only the
-							// wrapping shell. See `process/kill-tree.ts`.
-							detached: process.platform !== 'win32',
-							stdio: ['ignore', 'pipe', 'pipe'],
-						},
-					),
+					child: spawnHostShell(params.command, {
+						cwd: params.workingDirectory,
+						env: { ...inherited.env, ...params.env },
+						// `detached` makes the child the leader of its own process group
+						// off Windows, which is what `killTree` needs to reach the command
+						// and everything it forks rather than only the wrapping shell. See
+						// `process/kill-tree.ts`.
+						stdio: ['ignore', 'pipe', 'pipe'],
+					}),
 				}
 		const child = started.child
 		// Started, not adopted: this process stays the parent for the job's
 		// whole life. `unref` would let Node exit with the job still running,
 		// which is the orphan this registry exists to prevent.
-		child.stdout?.setEncoding('utf8')
-		child.stderr?.setEncoding('utf8')
+		// Raw bytes, decoded here: a Windows console writes its OEM code page.
+		// A process a sandbox started runs a Linux guest, so it is UTF-8.
+		const decoders = {
+			stdout: createHostOutputDecoder(params.spawn ? { platform: 'linux' } : {}),
+			stderr: createHostOutputDecoder(params.spawn ? { platform: 'linux' } : {}),
+		}
+		let startError: string | undefined
+		let settleStarted!: () => void
+		const startedPromise = new Promise<{ readonly error?: string }>((resolve) => {
+			settleStarted = () => resolve(startError === undefined ? {} : { error: startError })
+		})
+		// A process the host's own spawner started is the host's to vouch for:
+		// it may be remote, and need not emit `spawn` at all.
+		if (params.spawn || child.pid !== undefined) settleStarted()
 
+		let finalizeEntry!: (code: number | null, signal: NodeJS.Signals | null) => void
 		const entry: JobEntry = {
 			record: {
 				id,
@@ -243,35 +357,58 @@ export class BackgroundJobRegistry {
 			},
 			child,
 			...(started.kill ? { killProcess: started.kill.bind(started) } : {}),
+			...(started.terminate ? { terminateProcess: started.terminate.bind(started) } : {}),
 			buffer: '',
 			produced: 0,
+			outputChunks: [],
+			outputObservers: new Set(),
+			started: startedPromise,
 			exit: new Promise<void>((resolve) => {
 				let spawned = child.pid !== undefined
 				let failedToSpawn = false
+				let finalized = false
 				const finalize = (code: number | null, signal: NodeJS.Signals | null): void => {
+					if (finalized) return
+					finalized = true
 					entry.record = {
 						...entry.record,
 						// A job killed by this registry says `killed`, not
 						// `exited` with a signal a reader has to interpret. The
 						// two are different answers to "why did my job stop".
-						status: entry.record.status === 'killed' ? 'killed' : 'exited',
+						status: entry.killRequested || entry.record.status === 'killed' ? 'killed' : 'exited',
 						exitedAt: Date.now(),
 						...(code === null ? {} : { exitCode: code }),
 						...(signal ? { signal } : {}),
 					}
 					resolve()
+					for (const observer of entry.outputObservers) observer()
 					this.announceExit(entry.record)
 				}
+				finalizeEntry = finalize
 				child.once('spawn', () => {
 					spawned = true
+					settleStarted()
 				})
-				child.on('error', () => {
+				child.on('error', (error) => {
 					// A failed spawn emits both `error` and `close`. Waiting for
 					// `close` also covers errors on an already-running process
 					// (such as a failed kill) without announcing a false exit.
-					if (!spawned) failedToSpawn = true
+					if (spawned) return
+					failedToSpawn = true
+					startError ??= describeSpawnFailure(error, params.workingDirectory)
+					settleStarted()
 				})
 				child.once('close', (code, signal) => {
+					for (const stream of ['stdout', 'stderr'] as const) {
+						const rest = decoders[stream].end()
+						if (rest) append(rest, stream)
+					}
+					if (failedToSpawn && startError !== undefined) {
+						// In the output too, so a caller that only reads output (or
+						// redirects it, as a model does) still learns why it is empty.
+						append(`${startError}\n`, 'stderr')
+						entry.record = { ...entry.record, error: startError }
+					}
 					// The job is the process GROUP, not the shell. A command that
 					// backgrounds its real work (`server &`) exits the shell at
 					// once and leaves the server as the group's survivor; calling
@@ -282,6 +419,14 @@ export class BackgroundJobRegistry {
 					// alive, and ends — with the shell's exit code — when the
 					// group is empty.
 					const exitCode = failedToSpawn ? null : code
+					if (entry.terminateProcess) {
+						entry.closedResult = { code: exitCode, signal }
+						void this.confirmTermination(entry).catch(() => {
+							// The retained running record exposes recoveryRequired. A
+							// later kill retries instead of forgetting the guest tree.
+						})
+						return
+					}
 					if (entry.record.status !== 'killed' && groupAlive(child.pid)) {
 						const poll = setInterval(() => {
 							if (groupAlive(child.pid)) return
@@ -295,30 +440,58 @@ export class BackgroundJobRegistry {
 				})
 			}),
 		}
+		entry.finalize = finalizeEntry
 
-		const append = (text: string): void => {
-			entry.produced += Buffer.byteLength(text)
+		const append = (text: string, stream: 'stdout' | 'stderr'): void => {
+			const chunk: OutputChunk = { stream, offset: entry.produced, bytes: Buffer.from(text) }
+			if (chunk.bytes.length === 0) return
+			entry.produced += chunk.bytes.length
 			entry.buffer += text
-			const over = Buffer.byteLength(entry.buffer) - this.maxBytes
-			if (over > 0) {
-				// Oldest first. A cap that dropped the NEWEST bytes would hide
-				// exactly the part a poller is waiting for.
-				//
-				// Nothing accumulates a drop counter here, deliberately. What a
-				// reader needs is how much IT missed, which depends on where its
-				// own offset was — a running total of everything ever dropped
-				// answers a different question, and `read` derives the right one
-				// from `produced` and the retained length.
-				entry.buffer = Buffer.from(entry.buffer).subarray(over).toString('utf8')
+			entry.buffer = outputTail(entry.buffer, this.maxBytes).text
+			entry.outputChunks.push(chunk)
+			const retainedStart = entry.produced - Buffer.byteLength(entry.buffer)
+			while (
+				entry.outputChunks.length > MAX_CHANNEL_CHUNKS ||
+				(entry.outputChunks[0] &&
+					entry.outputChunks[0].offset + entry.outputChunks[0].bytes.length <= retainedStart)
+			) {
+				entry.outputChunks.shift()
+			}
+			const first = entry.outputChunks[0]
+			if (first && first.offset < retainedStart) {
+				const skip = utf8Start(first.bytes, retainedStart - first.offset)
+				entry.outputChunks[0] = {
+					...first,
+					offset: first.offset + skip,
+					bytes: Buffer.from(first.bytes.subarray(skip)),
+				}
 			}
+			// Deliver the actual chunk even when its beginning has already fallen
+			// outside retention: an active observer must not miss a fast producer.
+			for (const observer of entry.outputObservers) observer(chunk)
+		}
+		const onData = (stream: 'stdout' | 'stderr') => (data: Buffer | string) => {
+			const text = typeof data === 'string' ? data : decoders[stream].write(data)
+			if (text) append(text, stream)
 		}
-		child.stdout?.on('data', (text: string) => append(text))
-		child.stderr?.on('data', (text: string) => append(text))
+		child.stdout?.on('data', onData('stdout'))
+		child.stderr?.on('data', onData('stderr'))
 
 		this.jobs.set(id, entry)
 		return entry.record
 	}
 
+	/**
+	 * Resolves once the job's process exists, or once it could not be
+	 * created (`error` then says why). Lets a caller report a start that
+	 * failed in the same reply that would otherwise say "started".
+	 */
+	awaitStarted(id: string): Promise<{ readonly error?: string }> {
+		const entry = this.jobs.get(id)
+		if (!entry) throw new UnknownBackgroundJobError({ id })
+		return entry.started
+	}
+
 	/** The record, or throw for an id this registry does not know. */
 	get(id: string): BackgroundJob {
 		const entry = this.jobs.get(id)
@@ -369,6 +542,171 @@ export class BackgroundJobRegistry {
 		})
 	}
 
+	/**
+	 * Observe a literal in one pipe, including retained output after the cursor.
+	 * A marker is evidence of output, not exit intent or service health. Every
+	 * observer is independent and bounded; cancellation never stops the job.
+	 */
+	waitForOutput(
+		id: string,
+		opts: BackgroundJobOutputWaitOptions,
+	): Promise<BackgroundJobOutputWaitResult> {
+		const entry = this.jobs.get(id)
+		if (!entry) throw new UnknownBackgroundJobError({ id })
+		// Capture the admitted condition and clocks once; a caller retaining its
+		// mutable options object cannot alter a bound after validation.
+		const {
+			literal: literalText,
+			stream: requestedStream,
+			fromOffset,
+			timeoutMs,
+			idleTimeoutMs,
+			signal,
+		} = opts
+		if (
+			typeof literalText !== 'string' ||
+			literalText.length === 0 ||
+			Buffer.byteLength(literalText) > 4096 ||
+			Buffer.from(literalText).toString('utf8') !== literalText
+		) {
+			throw new RangeError('Output literal must be non-empty valid UTF-8, at most 4096 bytes.')
+		}
+		const stream = requestedStream ?? 'either'
+		if (!['stdout', 'stderr', 'either'].includes(stream))
+			throw new RangeError('Unknown output stream.')
+		for (const [name, value] of [
+			['timeoutMs', timeoutMs],
+			['idleTimeoutMs', idleTimeoutMs],
+		] as const) {
+			if (
+				(name === 'timeoutMs' || value !== undefined) &&
+				(value === undefined ||
+					!Number.isSafeInteger(value) ||
+					value < 1 ||
+					value > MAX_OUTPUT_WAIT_MS)
+			) {
+				throw new RangeError(`${name} must be an integer from 1 through ${MAX_OUTPUT_WAIT_MS}.`)
+			}
+		}
+		const from = fromOffset ?? 0
+		if (!Number.isSafeInteger(from) || from < 0 || from > entry.produced) {
+			throw new RangeError('fromOffset must be an observed, nonnegative output byte cursor.')
+		}
+		const literal = Buffer.from(literalText)
+		const suffix = { stdout: Buffer.alloc(0), stderr: Buffer.alloc(0) }
+		const match = (chunk: OutputChunk): 'stdout' | 'stderr' | undefined => {
+			if (
+				(stream !== 'either' && stream !== chunk.stream) ||
+				chunk.offset + chunk.bytes.length <= from
+			)
+				return
+			const skip = utf8Start(chunk.bytes, Math.max(0, from - chunk.offset))
+			const bytes = Buffer.concat([suffix[chunk.stream], chunk.bytes.subarray(skip)])
+			const found = bytes.indexOf(literal) !== -1
+			// Only the bytes that can prefix a future match survive each chunk.
+			// Copy so a tiny suffix never retains a large producer buffer.
+			suffix[chunk.stream] = Buffer.from(
+				bytes.subarray(Math.max(0, bytes.length - literal.length + 1)),
+			)
+			return found ? chunk.stream : undefined
+		}
+		let output = ''
+		let cursor = from
+		let droppedBytes = 0
+		let omittedOutputBytes = 0
+		const read = (): void => {
+			const observed = this.read(id, { fromOffset: cursor })
+			cursor = observed.nextOffset
+			droppedBytes += observed.droppedBytes
+			const tail = outputTail(output + observed.chunk, MAX_WAIT_RESULT_BYTES)
+			output = tail.text
+			omittedOutputBytes += tail.omitted
+		}
+		read()
+		const retainedStart = entry.produced - Buffer.byteLength(entry.buffer)
+		const searchableStart = entry.outputChunks[0]?.offset ?? entry.produced
+		const unsearchedBytes = Math.max(0, searchableStart - Math.max(from, retainedStart))
+		const progress = () => ({
+			status: entry.record.status,
+			...(entry.record.exitCode === undefined ? {} : { exitCode: entry.record.exitCode }),
+			...(entry.record.error === undefined ? {} : { error: entry.record.error }),
+			output,
+			nextOffset: cursor,
+			droppedBytes,
+			unsearchedBytes,
+			omittedOutputBytes,
+		})
+		if (signal?.aborted) return Promise.resolve({ ...progress(), kind: 'aborted' })
+		// Check synchronously before subscribing. No process event can intervene
+		// in this call stack, so even a marker printed before activation is seen.
+		for (const chunk of entry.outputChunks) {
+			const matchedStream = match(chunk)
+			if (matchedStream) return Promise.resolve({ ...progress(), kind: 'matched', matchedStream })
+		}
+		if (entry.record.status !== 'running') {
+			return Promise.resolve({
+				...progress(),
+				kind: entry.record.status === 'killed' ? 'stopped' : 'exited',
+			})
+		}
+		const owner = entry.record.owner
+		const ownerCount = this.outputWaitersByOwner.get(owner) ?? 0
+		const ownerLimit = this.config.maxOutputWaitersPerOwner ?? 32
+		const globalLimit = this.config.maxOutputWaiters ?? 256
+		if (ownerCount >= ownerLimit)
+			throw new BackgroundJobOutputWaitLimitError({ owner, limit: ownerLimit, scope: 'owner' })
+		if (this.outputWaiters >= globalLimit)
+			throw new BackgroundJobOutputWaitLimitError({ owner, limit: globalLimit, scope: 'registry' })
+		this.outputWaiters++
+		this.outputWaitersByOwner.set(owner, ownerCount + 1)
+		return new Promise((resolve) => {
+			const startedAt = Date.now()
+			let settled = false
+			let disposeAbort = () => {}
+			let idleTimer: ReturnType<typeof setTimeout> | undefined
+			const finish = (result: BackgroundJobOutputWaitResult): void => {
+				if (settled) return
+				settled = true
+				entry.outputObservers.delete(observe)
+				clearTimeout(wallTimer)
+				clearTimeout(idleTimer)
+				disposeAbort()
+				this.outputWaiters--
+				const remaining = (this.outputWaitersByOwner.get(owner) ?? 1) - 1
+				if (remaining === 0) this.outputWaitersByOwner.delete(owner)
+				else this.outputWaitersByOwner.set(owner, remaining)
+				resolve(result)
+			}
+			const timeout = (cause: 'wall' | 'idle'): void => {
+				read()
+				finish({ ...progress(), kind: 'timeout', cause, elapsedMs: Date.now() - startedAt })
+			}
+			const resetIdle = (): void => {
+				if (idleTimeoutMs === undefined) return
+				clearTimeout(idleTimer)
+				idleTimer = setTimeout(() => timeout('idle'), idleTimeoutMs)
+				idleTimer.unref?.()
+			}
+			const observe = (chunk?: OutputChunk): void => {
+				read()
+				const matchedStream = chunk ? match(chunk) : undefined
+				if (matchedStream) finish({ ...progress(), kind: 'matched', matchedStream })
+				else if (entry.record.status !== 'running') {
+					finish({ ...progress(), kind: entry.record.status === 'killed' ? 'stopped' : 'exited' })
+				} else if (chunk) resetIdle()
+			}
+			entry.outputObservers.add(observe)
+			const wallTimer = setTimeout(() => timeout('wall'), timeoutMs)
+			wallTimer.unref?.()
+			resetIdle()
+			if (signal)
+				disposeAbort = subscribeToAbort(signal, () => {
+					read()
+					finish({ ...progress(), kind: 'aborted' })
+				})
+		})
+	}
+
 	/**
 	 * Output since `fromOffset`, with what the cap dropped stated.
 	 *
@@ -386,14 +724,17 @@ export class BackgroundJobRegistry {
 		// the gap is reported rather than closed over.
 		const effective = Math.max(from, bufferStart)
 		const skip = Math.max(0, effective - bufferStart)
-		const chunk = Buffer.from(entry.buffer).subarray(skip).toString('utf8')
+		const bytes = Buffer.from(entry.buffer)
+		const aligned = utf8Start(bytes, skip)
+		const chunk = bytes.subarray(aligned).toString('utf8')
 
 		return {
 			chunk,
 			nextOffset: entry.produced,
-			droppedBytes: Math.max(0, effective - from),
+			droppedBytes: Math.max(0, effective - from) + Math.max(0, aligned - skip),
 			status: entry.record.status,
 			...(entry.record.exitCode === undefined ? {} : { exitCode: entry.record.exitCode }),
+			...(entry.record.error === undefined ? {} : { error: entry.record.error }),
 		}
 	}
 
@@ -402,11 +743,18 @@ export class BackgroundJobRegistry {
 		const entry = this.jobs.get(id)
 		if (!entry) throw new UnknownBackgroundJobError({ id })
 		if (entry.record.status !== 'running') return entry.record
+		if (entry.terminateProcess) {
+			entry.killRequested = true
+			await this.confirmTermination(entry)
+			await entry.exit
+			return entry.record
+		}
 
 		// Marked before the signal, so the `close` handler that follows can
 		// tell a kill from an ordinary exit. Set it after and the race decides
 		// which of two different answers a reader gets.
 		entry.record = { ...entry.record, status: 'killed' }
+		for (const observer of entry.outputObservers) observer()
 		const signal = (sig: NodeJS.Signals) =>
 			entry.killProcess ? entry.killProcess(sig) : killTree(entry.child, sig)
 		signal('SIGTERM')
@@ -509,6 +857,10 @@ export function bindOwner(
 			})
 		},
 		get: (id: string) => mine(id),
+		awaitStarted: (id: string) => {
+			mine(id)
+			return registry.awaitStarted(id)
+		},
 		read: (id: string, opts?: { fromOffset?: number }) => {
 			mine(id)
 			return registry.read(id, opts ?? {})
@@ -517,6 +869,10 @@ export function bindOwner(
 			mine(id)
 			return registry.waitForExit(id, opts ?? {})
 		},
+		waitForOutput: (id: string, opts: BackgroundJobOutputWaitOptions) => {
+			mine(id)
+			return registry.waitForOutput(id, opts)
+		},
 		...(defaults.onAwaited
 			? {
 					markAwaited: (id: string) => {

package/src/runtime/query/abandon-turn.ts

@@ -20,6 +20,18 @@ export interface SessionLocatorOptions {
 	readonly lease?: SessionLease
 }
 
+/** Options for {@link abandonTurn}. */
+export interface AbandonTurnOptions extends SessionLocatorOptions {
+	/**
+	 * The caller gives the turn's messages back to the person, for instance by
+	 * putting the prompt back in a message box to send again. The abandoned
+	 * turn then contributes no message to the next turn's context, so the
+	 * provider does not see that prompt twice. The records stay in the log.
+	 * Absent (the default): the turn's messages stay in the context.
+	 */
+	readonly withdrawMessages?: boolean
+}
+
 /** Lease time-to-live for a lease a session-level operation claims for one append. */
 const OPERATION_LEASE_TTL_MS = 30_000
 
@@ -103,8 +115,12 @@ export async function abandonTurn(
 	sessionId: SessionId,
 	turnId: TurnId,
 	reason: string,
-	options: SessionLocatorOptions = {},
+	options: AbandonTurnOptions = {},
 ): Promise<void> {
 	const log = await locateSessionLog(sessionId, options)
-	await withSessionLease(log, options.lease, (lease) => log.abandonTurn(lease, turnId, reason))
+	await withSessionLease(log, options.lease, (lease) =>
+		log.abandonTurn(lease, turnId, reason, {
+			...(options.withdrawMessages ? { withdrawMessages: true } : {}),
+		}),
+	)
 }

package/src/runtime/query/checkpoint.ts

@@ -268,6 +268,8 @@ export class CheckpointManager {
 	private restoredReviewAttempts = 0
 	private nativeStructuredAttemptsSource?: () => number
 	private restoredNativeAttempts = 0
+	private toolStructuredAttemptsSource?: () => number
+	private restoredToolAttempts = 0
 	private lastCreatedId?: CheckpointId
 	private traceSource?: () => SerializedSpanContext | undefined
 	private parkTtlMs?: number
@@ -314,6 +316,13 @@ export class CheckpointManager {
 		return this.restoredNativeAttempts
 	}
 
+	setToolStructuredAttemptsSource(source: () => number): void {
+		this.toolStructuredAttemptsSource = source
+	}
+	get restoredToolStructuredAttempts(): number {
+		return this.restoredToolAttempts
+	}
+
 	setStructuredReviewAttemptsSource(source: () => number): void {
 		this.structuredReviewAttemptsSource = source
 	}
@@ -398,6 +407,7 @@ export class CheckpointManager {
 				answerAttempts: this.answerReviewAttemptsSource?.() ?? this.restoredAnswerAttempts,
 				nativeStructuredAttempts:
 					this.nativeStructuredAttemptsSource?.() ?? this.restoredNativeAttempts,
+				toolStructuredAttempts: this.toolStructuredAttemptsSource?.() ?? this.restoredToolAttempts,
 			},
 			...(latestUserMessageId ? { latestUserMessageId } : {}),
 			...withDefined('workingState', this.workingStateSource?.()),
@@ -597,6 +607,7 @@ export class CheckpointManager {
 		this.turnCreatedAt ??= document.turnCreatedAt
 		this.restoredAnswerAttempts = document.review.answerAttempts
 		this.restoredNativeAttempts = document.review.nativeStructuredAttempts
+		this.restoredToolAttempts = document.review.toolStructuredAttempts ?? 0
 		this.restoredReviewAttempts = document.review.structuredAttempts
 		this.restoredUserMessage = restored.latestUserMessage
 		this.restoredUserMessageId = restored.latestUserMessage