@namzu/sdk 59.0.0 → 60.1.0

package/src/store/memory/operation-lock.ts

@@ -1,10 +1,13 @@
 import { randomUUID } from 'node:crypto'
-import { type FileHandle, lstat, open, readFile, unlink } from 'node:fs/promises'
+import type { Stats } from 'node:fs'
+import { lstat, open, readFile, unlink } from 'node:fs/promises'
+import { hostname as osHostname } from 'node:os'
 import { setTimeout as delay } from 'node:timers/promises'
 
 import { NamzuError } from '../../types/errors/index.js'
+import type { Logger } from '../../utils/logger.js'
 
-/** Wait bound, not a lease: a slow or crashed owner is never silently replaced. */
+/** Wait bound, not a lease: a slow owner that is still running is never replaced. */
 export const DEFAULT_MEMORY_LOCK_TIMEOUT_MS = 10_000
 
 export function validateMemoryLockTimeout(timeoutMs: number): void {
@@ -17,83 +20,311 @@ export function validateMemoryLockTimeout(timeoutMs: number): void {
 	}
 }
 
-function lockError(path: string, reason: string, timeoutMs: number): NamzuError {
+/**
+ * An empty or unparsable lock file is only believed abandoned once it is older
+ * than this. A live acquirer writes its record immediately after creating the
+ * file, so a gap of seconds means the writer died in between (disk full, kill).
+ */
+export const MEMORY_LOCK_RECORD_GRACE_MS = 5_000
+
+export interface MemoryOperationLockOptions {
+	readonly logger?: Logger
+	/** Clock compared against file mtimes. */
+	readonly now?: () => number
+	/** True when a process with this pid exists on this machine. */
+	readonly isPidAlive?: (pid: number) => boolean
+	readonly hostname?: string
+}
+
+interface OwnerRecord {
+	readonly pid: number
+	readonly token: string
+	readonly acquiredAt: number
+	readonly hostname?: string
+}
+
+interface Inspection {
+	readonly stats: Stats
+	readonly content: string
+	readonly pid: number | undefined
+	/** Set only when the owner is provably gone. */
+	readonly staleReason: string | undefined
+}
+
+interface Resolved {
+	readonly logger: Logger | undefined
+	readonly now: () => number
+	readonly isPidAlive: (pid: number) => boolean
+	readonly hostname: string
+}
+
+/**
+ * Residual risk, deliberately accepted: a pid that is alive is treated as the
+ * owner even when it is an unrelated process that reused the number after the
+ * real owner died. That cannot be disproved from here, so such a lock stays and
+ * the caller gets the plain "another process is using it" error. It is not
+ * bounded by `acquiredAt` age: a lock's age is not a lease.
+ */
+function defaultIsPidAlive(pid: number): boolean {
+	try {
+		process.kill(pid, 0)
+		return true
+	} catch (error) {
+		return (error as NodeJS.ErrnoException).code !== 'ESRCH'
+	}
+}
+
+function lockError(path: string, message: string, timeoutMs: number): NamzuError {
 	return new NamzuError({
 		code: 'storage_error',
-		message: `Memory operation lock ${path}: ${reason}. Stop all processes using this memory store and inspect the lock owner before removing a stale lock; it is never broken automatically.`,
+		message,
 		details: { lockPath: path, timeoutMs },
 		retryable: false,
 	})
 }
 
+function busyError(path: string, pid: number | undefined, timeoutMs: number): NamzuError {
+	const who =
+		pid === undefined ? 'Another Namzu process is' : `Another Namzu process (pid ${pid}) is`
+	return lockError(
+		path,
+		`${who} using the stored memory. Close it, or wait for it to finish.`,
+		timeoutMs,
+	)
+}
+
+function parseOwner(content: string): OwnerRecord | undefined {
+	try {
+		const value: unknown = JSON.parse(content)
+		if (!value || typeof value !== 'object') return undefined
+		const pid = (value as { pid?: unknown }).pid
+		if (typeof pid !== 'number' || !Number.isSafeInteger(pid) || pid <= 0) return undefined
+		const host = (value as { hostname?: unknown }).hostname
+		return {
+			pid,
+			token: String((value as { token?: unknown }).token ?? ''),
+			acquiredAt: Number((value as { acquiredAt?: unknown }).acquiredAt ?? 0),
+			...(typeof host === 'string' && host.length > 0 ? { hostname: host } : {}),
+		}
+	} catch {
+		return undefined
+	}
+}
+
+/** Reads the file at `path`; undefined when it is gone or was swapped while reading. */
+async function inspect(path: string, env: Resolved): Promise<Inspection | undefined> {
+	const before = await lstat(path).catch((error: NodeJS.ErrnoException) => {
+		if (error.code === 'ENOENT') return undefined
+		throw error
+	})
+	if (!before) return undefined
+	if (!before.isFile() || before.isSymbolicLink())
+		return { stats: before, content: '', pid: undefined, staleReason: undefined }
+	const content = await readFile(path, 'utf8').catch((error: NodeJS.ErrnoException) => {
+		if (error.code === 'ENOENT') return undefined
+		throw error
+	})
+	if (content === undefined) return undefined
+	const after = await lstat(path).catch(() => undefined)
+	if (!after || after.dev !== before.dev || after.ino !== before.ino) return undefined
+
+	const owner = parseOwner(content)
+	if (owner) {
+		// A record from another host on a shared filesystem cannot be checked from
+		// here; it is live. A record without a host predates the field: local.
+		if (owner.hostname !== undefined && owner.hostname !== env.hostname) {
+			return { stats: before, content, pid: owner.pid, staleReason: undefined }
+		}
+		return {
+			stats: before,
+			content,
+			pid: owner.pid,
+			staleReason: env.isPidAlive(owner.pid) ? undefined : `owner process ${owner.pid} is gone`,
+		}
+	}
+	const age = env.now() - before.mtimeMs
+	return {
+		stats: before,
+		content,
+		pid: undefined,
+		staleReason:
+			age > MEMORY_LOCK_RECORD_GRACE_MS
+				? `lock holds no readable owner record and is ${Math.round(age / 1000)}s old`
+				: undefined,
+	}
+}
+
+/**
+ * Creates `path` exclusively and writes this process's owner record. Returns
+ * undefined when the path is occupied. A failed write (a full disk) leaves no
+ * file behind.
+ */
+async function createOwned(
+	path: string,
+	env: Resolved,
+): Promise<{ identity: Stats; owner: string } | undefined> {
+	let handle: Awaited<ReturnType<typeof open>>
+	try {
+		handle = await open(path, 'wx', 0o600)
+	} catch (error) {
+		if ((error as NodeJS.ErrnoException).code === 'EEXIST') return undefined
+		throw error
+	}
+	const owner = `${JSON.stringify({
+		pid: process.pid,
+		token: randomUUID(),
+		acquiredAt: Date.now(),
+		hostname: env.hostname,
+	})}\n`
+	let identity: Stats | undefined
+	try {
+		identity = await handle.stat()
+		await handle.writeFile(owner, 'utf8')
+		await handle.sync()
+	} catch (error) {
+		await handle.close().catch(() => undefined)
+		// We created this path exclusively a moment ago; drop it so no empty or
+		// partial record is left to block the next start.
+		const current = await lstat(path).catch(() => undefined)
+		if (!identity || (current?.dev === identity.dev && current.ino === identity.ino)) {
+			await unlink(path).catch(() => undefined)
+		}
+		throw error
+	}
+	await handle.close()
+	return { identity, owner }
+}
+
+async function releaseOwned(
+	path: string,
+	identity: Stats,
+	owner: string,
+	timeoutMs: number,
+): Promise<void> {
+	const current = await lstat(path)
+	if (
+		!current.isFile() ||
+		current.isSymbolicLink() ||
+		current.dev !== identity.dev ||
+		current.ino !== identity.ino ||
+		(await readFile(path, 'utf8')) !== owner
+	) {
+		throw lockError(
+			path,
+			`Memory operation lock ${path}: the lock owner changed before release.`,
+			timeoutMs,
+		)
+	}
+	await unlink(path)
+}
+
+function sameFile(a: Stats, b: Stats): boolean {
+	return a.dev === b.dev && a.ino === b.ino
+}
+
+/** Unlinks `path` only if it is still exactly the stale file that was inspected. */
+async function unlinkIfStillStale(path: string, seen: Inspection, env: Resolved): Promise<boolean> {
+	const again = await inspect(path, env)
+	if (!again || !sameFile(again.stats, seen.stats) || again.content !== seen.content) return false
+	if (again.staleReason === undefined) return false
+	// Narrow the window further: the inode must still be the inspected one now.
+	const last = await lstat(path).catch(() => undefined)
+	if (!last || !sameFile(last, seen.stats)) return false
+	await unlink(path).catch((error: NodeJS.ErrnoException) => {
+		if (error.code !== 'ENOENT') throw error
+	})
+	return true
+}
+
+/**
+ * Removes a lock whose owner is provably gone. Two processes that see the same
+ * stale lock are serialised through `<lock>.break`; the lock is re-inspected
+ * under it and only the inspected inode is unlinked. Returns whether the lock
+ * was removed.
+ */
+async function breakStaleLock(
+	path: string,
+	seen: Inspection,
+	timeoutMs: number,
+	env: Resolved,
+): Promise<boolean> {
+	const breakPath = `${path}.break`
+	let held: { identity: Stats; owner: string } | undefined
+	for (let attempt = 0; attempt < 2 && !held; attempt++) {
+		held = await createOwned(breakPath, env)
+		if (held) break
+		const breaker = await inspect(breakPath, env)
+		if (!breaker) continue
+		// A dead breaker's file follows the same rule, once.
+		if (breaker.staleReason === undefined || !(await unlinkIfStillStale(breakPath, breaker, env))) {
+			return false
+		}
+	}
+	if (!held) return false
+	try {
+		const removed = await unlinkIfStillStale(path, seen, env)
+		if (removed) {
+			env.logger?.info('Broke an abandoned memory operation lock', {
+				'namzu.store.lock_path': path,
+				'namzu.store.lock_reason': seen.staleReason,
+			})
+		}
+		return removed
+	} finally {
+		await releaseOwned(breakPath, held.identity, held.owner, timeoutMs).catch(() => undefined)
+	}
+}
+
 /**
  * Local-filesystem exclusion for cooperating processes. Atomic rename of the
  * index alone cannot serialize the index/content read-modify-write transaction.
- * An abandoned lock fails closed; guessing a lease expired would permit two
- * live writers to overwrite each other again.
+ * A lock whose owner is provably gone (dead pid on this host, or an owner-less
+ * file older than the grace period) is broken; a live or unverifiable owner is
+ * waited for until the timeout, never replaced on a guessed lease.
  */
 export async function acquireMemoryOperationLock(
 	path: string,
 	timeoutMs: number,
+	options: MemoryOperationLockOptions = {},
 ): Promise<() => Promise<void>> {
+	const env: Resolved = {
+		logger: options.logger,
+		now: options.now ?? Date.now,
+		isPidAlive: options.isPidAlive ?? defaultIsPidAlive,
+		hostname: options.hostname ?? osHostname(),
+	}
 	const deadline = performance.now() + timeoutMs
 	let attempted = false
+	let lastPid: number | undefined
+	// The wait bound limits waiting for a live owner, not the attempt that
+	// follows a successful break: the lock is free then, and breaking it can
+	// itself outlast a short bound on a slow disk.
+	let justBroke = false
 	for (;;) {
-		if (attempted && performance.now() >= deadline) {
-			throw lockError(path, `acquisition timed out after ${timeoutMs} ms`, timeoutMs)
+		if (attempted && !justBroke && performance.now() >= deadline) {
+			throw busyError(path, lastPid, timeoutMs)
 		}
+		justBroke = false
 		attempted = true
-		let handle: FileHandle
-		try {
-			handle = await open(path, 'wx', 0o600)
-		} catch (error) {
-			if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error
-			const existing = await lstat(path).catch((statError: NodeJS.ErrnoException) => {
-				if (statError.code === 'ENOENT') return undefined
-				throw statError
-			})
-			if (!existing) continue
-			if (!existing.isFile() || existing.isSymbolicLink()) {
-				throw lockError(path, 'the occupied lock is not a regular file', timeoutMs)
-			}
-			if (performance.now() >= deadline) {
-				throw lockError(path, `acquisition timed out after ${timeoutMs} ms`, timeoutMs)
-			}
-			await delay(Math.min(25, Math.max(1, deadline - performance.now())))
-			continue
+		const created = await createOwned(path, env)
+		if (created) {
+			return () => releaseOwned(path, created.identity, created.owner, timeoutMs)
 		}
-
-		const owner = `${JSON.stringify({ pid: process.pid, token: randomUUID(), acquiredAt: Date.now() })}\n`
-		const identity = await handle.stat().catch(async (error: unknown) => {
-			await handle.close()
-			// Exclusive create happened, but ownership could not be established.
-			// Leave the path for explicit recovery rather than unlink an unknown inode.
-			throw error
-		})
-		try {
-			await handle.writeFile(owner, 'utf8')
-			await handle.sync()
-		} catch (error) {
-			await handle.close()
-			const current = await lstat(path).catch(() => undefined)
-			if (current?.dev === identity.dev && current.ino === identity.ino) {
-				await unlink(path).catch(() => undefined)
-			}
-			throw error
+		const seen = await inspect(path, env)
+		if (!seen) continue
+		if (!seen.stats.isFile() || seen.stats.isSymbolicLink()) {
+			throw lockError(
+				path,
+				`The stored memory lock ${path} is not a regular file, so Namzu will not use it. Move it aside to continue.`,
+				timeoutMs,
+			)
 		}
-		await handle.close()
-		return async () => {
-			const current = await lstat(path)
-			if (
-				!current.isFile() ||
-				current.isSymbolicLink() ||
-				current.dev !== identity.dev ||
-				current.ino !== identity.ino ||
-				(await readFile(path, 'utf8')) !== owner
-			) {
-				throw lockError(path, 'the lock owner changed before release', timeoutMs)
-			}
-			await unlink(path)
+		lastPid = seen.pid
+		if (seen.staleReason !== undefined && (await breakStaleLock(path, seen, timeoutMs, env))) {
+			justBroke = true
+			continue
 		}
+		if (performance.now() >= deadline) throw busyError(path, lastPid, timeoutMs)
+		await delay(Math.min(25, Math.max(1, deadline - performance.now())))
 	}
 }

package/src/store/session-index/sqlite.ts

@@ -1,3 +1,4 @@
+import { existsSync, statSync } from 'node:fs'
 import { mkdir, readdir, rename, rm, stat } from 'node:fs/promises'
 import { createRequire } from 'node:module'
 import { basename, dirname, join } from 'node:path'
@@ -305,32 +306,135 @@ async function sweepAbandonedRebuilds(path: string): Promise<void> {
 	}
 }
 
+/** `device:inode` of the file at `path`, or undefined when there is none. */
+function fileIdentity(path: string): string | undefined {
+	try {
+		const { dev, ino } = statSync(path)
+		return `${dev}:${ino}`
+	} catch {
+		return undefined
+	}
+}
+
 function userVersion(db: DatabaseSync): number {
 	return Number(db.prepare('PRAGMA user_version').get()?.user_version)
 }
 
+/** SQLite primary result codes for "the image is malformed" and "the file is not a database". */
+const SQLITE_CORRUPT = 11
+const SQLITE_NOTADB = 26
+
+/** Whether `error` says the file's content is not a usable SQLite database. */
+function isNotADatabase(error: unknown): boolean {
+	const code = (error as { errcode?: unknown } | null)?.errcode
+	return typeof code === 'number' && [SQLITE_CORRUPT, SQLITE_NOTADB].includes(code & 0xff)
+}
+
+const SQLITE_BUSY = 5
+const SQLITE_READONLY = 8
+/** `SQLITE_READONLY_DBMOVED`: the file this connection opened was renamed or unlinked. */
+const SQLITE_READONLY_DBMOVED = 1032
+
+function errcode(error: unknown): number | undefined {
+	const code = (error as { errcode?: unknown } | null)?.errcode
+	return typeof code === 'number' ? code : undefined
+}
+
+/** Whether `error` says another process holds the index locked past the busy timeout. */
+function isBusy(error: unknown): boolean {
+	const code = errcode(error)
+	return code !== undefined && (code & 0xff) === SQLITE_BUSY
+}
+
+/** Whether `error` says the file under this connection was replaced by another process. */
+function isMoved(error: unknown): boolean {
+	const code = errcode(error)
+	return code === SQLITE_READONLY_DBMOVED && (code & 0xff) === SQLITE_READONLY
+}
+
+/** A plain-language error for an index another process keeps locked. */
+function busyError(path: string, cause: unknown): SessionIndexError {
+	return new SessionIndexError(
+		`The session index at ${path} is in use by another process. Try again in a moment.`,
+		{ cause },
+	)
+}
+
 /**
- * The version of the index file at `path`: -1 when there is none, 0 when it
- * cannot be read as a SQLite database, otherwise its `user_version`.
+ * The version of the index file at `path`: -1 when there is none, 0 when its
+ * content is not a SQLite database, otherwise its `user_version`.
+ *
+ * The file is opened for writing, never read-only. A writer that died inside
+ * a transaction leaves a hot journal beside the file, and only a connection
+ * that may write can roll it back; a read-only one fails on it, which used to
+ * read as "not a database" and sent the caller to replace a healthy index,
+ * leaving the dead writer's journal beside the new file for the next
+ * connection to play back into it. Only a SQLite "corrupt" or "not a
+ * database" answer means 0; being busy, read-only or unable to read is thrown,
+ * never a reason to rebuild.
  */
 function indexVersion(path: string): number {
+	if (!existsSync(path)) return -1
 	const { DatabaseSync } = requireSqlite()
-	let db: DatabaseSync
-	try {
-		db = new DatabaseSync(path, { readOnly: true })
-	} catch {
-		return -1
-	}
+	const db = new DatabaseSync(path)
 	try {
 		db.exec('PRAGMA busy_timeout = 10000')
 		return userVersion(db)
-	} catch {
-		return 0
+	} catch (error) {
+		if (isNotADatabase(error)) return 0
+		throw isBusy(error) ? busyError(path, error) : error
 	} finally {
 		db.close()
 	}
 }
 
+/**
+ * Put the finished `temporary` file in place of the index at `path`.
+ *
+ * Takes the old index's exclusive lock first. That waits out a peer that is
+ * mid-write and rolls back a hot journal left by one that died, so no journal
+ * for the old file survives to be played into the new one. A leftover journal
+ * of a file that is not a database is removed too: it can describe nothing
+ * but that file.
+ */
+async function replaceIndexFile(temporary: string, path: string, force: boolean): Promise<boolean> {
+	let old: DatabaseSync | undefined
+	if (existsSync(path)) {
+		const { DatabaseSync } = requireSqlite()
+		old = new DatabaseSync(path)
+		try {
+			old.exec('PRAGMA busy_timeout = 10000')
+			old.exec('BEGIN EXCLUSIVE')
+			// A peer may have finished its own rebuild while this one ran. With the
+			// lock held, nobody can change the file between this check and the
+			// rename, so a current index is left alone rather than replaced under
+			// the connections that already adopted it.
+			if (!force && userVersion(old) === SESSION_INDEX_VERSION) {
+				old.close()
+				old = undefined
+				return false
+			}
+		} catch (error) {
+			old?.close()
+			old = undefined
+			if (!isNotADatabase(error)) throw isBusy(error) ? busyError(path, error) : error
+		}
+	}
+	try {
+		// Windows will not rename over a file that is open, so the lock is let go
+		// first there; elsewhere it is held until the new file is in place.
+		if (process.platform === 'win32') {
+			old?.close()
+			old = undefined
+		}
+		await rm(`${path}-journal`, { force: true })
+		await rename(temporary, path)
+		return true
+	} finally {
+		old?.close()
+	}
+}
+
 export interface SqliteSessionIndexOptions {
 	/** The index file, normally `$NAMZU_HOME/index.sqlite`. */
 	readonly path: string
@@ -371,15 +475,21 @@ export function rebuildSqliteSessionIndex(
  */
 export class SqliteSessionIndex extends SessionIndexBase {
 	readonly backend = 'sqlite' as const
-	readonly #db: DatabaseSync
+	#db: DatabaseSync
+	/** Where `#db` was opened, or undefined for a temporary build file that is never watched. */
+	readonly #path: string | undefined
+	/** Identity of the file `#db` has open, to notice another process replacing it. */
+	#identity: string | undefined
 	readonly #sink: IndexSink
 	readonly #statements = new Map<string, StatementSync>()
 	#open = true
 
 	/** Wraps an open database whose schema is current. Use {@link SqliteSessionIndex.open}. */
-	private constructor(db: DatabaseSync) {
+	private constructor(db: DatabaseSync, path?: string) {
 		super()
 		this.#db = db
+		this.#path = path
+		this.#identity = path === undefined ? undefined : fileIdentity(path)
 		const run = (sql: string, ...values: SQLInputValue[]) => {
 			this.#statement(sql).run(...values)
 		}
@@ -583,8 +693,7 @@ export class SqliteSessionIndex extends SessionIndexBase {
 			if (!options.force && (await pathExists(options.path))) {
 				if (indexVersion(options.path) === SESSION_INDEX_VERSION) return false
 			}
-			await rename(temporary, options.path)
-			return true
+			return await replaceIndexFile(temporary, options.path, options.force === true)
 		} finally {
 			await rm(temporary, { force: true })
 			await rm(`${temporary}-journal`, { force: true })
@@ -615,13 +724,55 @@ export class SqliteSessionIndex extends SessionIndexBase {
 			db.close()
 			throw error
 		}
-		const index = new SqliteSessionIndex(db)
+		const index = new SqliteSessionIndex(db, options.path)
 		await index.sync(options.home)
 		return index
 	}
 
+	/**
+	 * Another process rebuilt the index and renamed a new file over it: this
+	 * connection still has the old, unlinked file, whose reads go stale and
+	 * whose writes fail with SQLITE_READONLY_DBMOVED. Reopen the file now at
+	 * the path. A no-op for a temporary build file, and when the path holds
+	 * nothing to adopt.
+	 */
+	#reopenIfReplaced(force = false): void {
+		const path = this.#path
+		if (path === undefined || !this.#open) return
+		const identity = fileIdentity(path)
+		if (identity === undefined || (!force && identity === this.#identity)) return
+		const { DatabaseSync } = requireSqlite()
+		const db = new DatabaseSync(path)
+		try {
+			configure(db)
+			if (userVersion(db) !== SESSION_INDEX_VERSION) {
+				throw new SessionIndexError(`The index at ${path} changed version while in use.`)
+			}
+		} catch (error) {
+			db.close()
+			throw error
+		}
+		const previous = this.#db
+		this.#statements.clear()
+		this.#db = db
+		this.#identity = fileIdentity(path)
+		try {
+			previous.close()
+		} catch {
+			// The old file is already unlinked; nothing is lost if it will not close.
+		}
+	}
+
 	protected transaction<T>(work: (sink: IndexSink) => T): T {
-		this.#db.exec('BEGIN IMMEDIATE')
+		this.#reopenIfReplaced()
+		try {
+			this.#db.exec('BEGIN IMMEDIATE')
+		} catch (error) {
+			// Nothing has run yet, so the work can start again on the new file.
+			if (!isMoved(error)) throw error
+			this.#reopenIfReplaced(true)
+			this.#db.exec('BEGIN IMMEDIATE')
+		}
 		try {
 			const result = work(this.#sink)
 			this.#db.exec('COMMIT')
@@ -643,10 +794,12 @@ export class SqliteSessionIndex extends SessionIndexBase {
 	}
 
 	#all(sql: string, ...values: SQLInputValue[]): Row[] {
+		this.#reopenIfReplaced()
 		return this.#statement(sql).all(...values) as Row[]
 	}
 
 	protected readSession(sessionId: SessionId): SessionRow | undefined {
+		this.#reopenIfReplaced()
 		const row = this.#statement('SELECT * FROM sessions WHERE id = ?').get(sessionId) as
 			| Row
 			| undefined
@@ -707,6 +860,7 @@ export class SqliteSessionIndex extends SessionIndexBase {
 		kind: ExternalRefKind,
 		externalId: string,
 	): Promise<ExternalRefTarget | undefined> {
+		this.#reopenIfReplaced()
 		const row = this.#db
 			.prepare(
 				`SELECT * FROM external_refs WHERE protocol = ? AND kind = ? AND external_id = ?
@@ -749,6 +903,7 @@ export class SqliteSessionIndex extends SessionIndexBase {
 			values.push(options.sessionId)
 		}
 		const clause = where.length === 0 ? '' : `WHERE ${where.join(' AND ')}`
+		this.#reopenIfReplaced()
 		const statement = this.#db.prepare(
 			`SELECT text, session_id, turn_id, seq, part, source, tool_name, is_error FROM evidence_fts
 			${clause} ORDER BY session_id, seq, part`,

package/src/tools/builtins/bash.ts

@@ -751,12 +751,14 @@ const BashToolBase = defineTool({
 				clipped.length > 0
 					? `[${clipped.join(' and ')} was truncated by the sandbox output cap. The omitted output is unavailable in this result. Use a saved artifact or a read-only observation; do not repeat a state-changing action to recover its output.]`
 					: '',
+				result.attribution ? `[${result.attribution.message}]` : '',
 			]
 				.filter(Boolean)
 				.join('\n\n')
 
 			const judged = result.timedOut ? undefined : judgeExit(input.command, result.exitCode)
-			const answered = judged !== undefined && !judged.isError
+			const answered =
+				judged !== undefined && !judged.isError && result.attribution?.kind !== 'runner-failure'
 			return {
 				success: answered,
 				output: output || '(no output)',
@@ -767,12 +769,17 @@ const BashToolBase = defineTool({
 					timedOut: result.timedOut,
 					stdoutTruncated: result.stdoutTruncated ?? false,
 					stderrTruncated: result.stderrTruncated ?? false,
+					...(result.attribution ? { sandboxAttribution: result.attribution.kind } : {}),
 				},
 				error: result.timedOut
 					? `Command timed out after ${input.timeout}ms. Any captured output before the deadline is above.`
-					: !answered
-						? `Command exited with code ${result.exitCode}`
-						: undefined,
+					: result.attribution?.kind === 'runner-failure'
+						? 'The sandbox launcher failed before the command ran (runner failure, not a command failure).'
+						: !answered
+							? result.attribution?.kind === 'denial'
+								? `Command exited with code ${result.exitCode}; looks like a sandbox denial`
+								: `Command exited with code ${result.exitCode}`
+							: undefined,
 				facts: commandFacts({
 					command: input.command,
 					exitCode: result.exitCode,