@edgehero/pi-dispatch 1.10.3 → 2.1.0

package/src/run-container.mjs

@@ -1,5 +1,8 @@
 import { spawn } from "node:child_process";
-import { buildDockerRunArgs, CONTAINER_SESSION_FILE } from "./docker-run.mjs";
+import { readFileSync, rmSync } from "node:fs";
+import { DOCKER_NEVER_STARTED_EXITS } from "./backends.mjs";
+import { CONTAINER_HOME } from "./container-spec.mjs";
+import { buildDockerRunArgs, CONTAINER_SESSION_FILE, insideDir } from "./docker-run.mjs";
 import { createJobNetwork, networkNameFor, removeJobNetwork } from "./egress.mjs";
 import { buildContainerEnv } from "./env-allowlist.mjs";
 import { resolveJobImage } from "./image-preflight.mjs";
@@ -7,7 +10,7 @@ import { InfraRetry } from "./processor.mjs";
 
 /**
  * The real `runContainer` the processor injects. Launches one job container and returns
- * `{ code, aborted, turns, tokens, session, usage, context }`, where `aborted` records whether the WORKER initiated the stop (docker stop on
+ * `{ code, aborted, turns, tokens, session, usage, context, exitReason }`, where `aborted` records whether the WORKER initiated the stop (docker stop on
  * the 30-min timeout or graceful shutdown), which the processor classifies as POLICY (no retry) per
  * INT-RUNNER-EXIT-CODE-PROTOCOL. The numeric `code` alone cannot say this: a worker SIGKILL and a
  * kernel OOM both surface as 137, so the abort FLAG -- not the code -- is the discriminator.
@@ -30,7 +33,7 @@ export function makeRunContainer({
 	image, // the DEPLOYMENT default (PI_JOB_IMAGE); a trigger's own run.image overrides it per job
 	hostEnv = process.env,
 	onOutput = (c) => process.stdout.write(c),
-	openJobLog = () => ({ write() {}, close: async () => ({ turns: null, tokens: null, session: null, usage: null, context: null }) }),
+	openJobLog = () => ({ write() {}, close: async () => ({ turns: null, tokens: null, session: null, usage: null, context: null, exitReason: null }) }),
 	spawnFn = spawn,
 	globalPiDir = null, // REQ-GLOBAL-PI-OVERLAY: operator's global pi overlay dir, mounted :ro; null = off
 	allowGlobalExtensions = true, // REQ-GLOBAL-PI-OVERLAY: the staged overlay's extensions load unless PI_GLOBAL_ALLOW_EXTENSIONS=0
@@ -43,14 +46,42 @@ export function makeRunContainer({
 	forgeHosts = {}, // per-forge self-hosted instance URLs, so a forge CLI in the container talks to the right one
 	egress = false, // REQ-EGRESS-ALLOWLIST: put this job on its own --internal network behind the allowlist proxy
 	egressProxy, // the proxy component attached to that network; undefined = egress.mjs's default name
+	// Issue #345: the exits this runtime spells "never started", after which the cidfile says whether a container was
+	// created anyway; and the host fs the cidfile is read and removed through. Seams for the tests.
+	neverStartedExits = DOCKER_NEVER_STARTED_EXITS,
+	fs = { readFileSync, rmSync },
+	detachedCheck = {}, // the clock and bounds `stopDetached` takes, a seam so a test does not wait out a real 10 s
+	// Issue #354: the runtime's CLI and its argv builder, a PAIR. Every spawn below names `bin` -- the run, the job
+	// network's create and teardown, the detached check -- so a podman venue cannot run its container under one binary
+	// and look for it (or remove its network) under another. The defaults are the local venue's, byte-identical.
+	bin = "docker",
+	buildArgs = buildDockerRunArgs,
+	// Issue #452, gate round 4: `(job) => runtime | undefined`, the runtime THIS job was ADMITTED on (recorded per job),
+	// handed to the teardown's detach gate so the teardown never reads the daemon again; `undefined` falls back to a read.
+	// `log` names a teardown the gate refused, which leaves the job's network for the boot reaper.
+	teardownRuntime = null,
+	log = () => {},
 }) {
 	// async so a synchronous throw (e.g. buildContainerEnv on an unconfigured provider) surfaces as
 	// a rejection, uniformly awaitable by the processor and by tests.
-	return async function runContainer({ job, token, prepared, secrets = {}, name, signal }) {
-		if (signal?.aborted) return { code: 137, aborted: true, turns: null, tokens: null, session: null, usage: null, context: null }; // killed before it could start
+	// `relabel` (issue #355) is the processor's, off the same job-user answer as `user`: true where the daemon confines
+	// containers with SELinux, so the worker's own per-job mounts carry `:Z`. Defaults off, so a caller that predates it
+	// builds exactly the argv it always did.
+	return async function runContainer({ job, token, prepared, secrets = {}, name, signal, user = null, home = null, relabel = false }) {
+		if (signal?.aborted) return { code: 137, aborted: true, turns: null, tokens: null, session: null, usage: null, context: null, exitReason: null }; // killed before it could start
+		// Issue #341. `user` and `home` travel as a PAIR: a uid with no passwd entry in the image gets `HOME=/` from
+		// Docker and `HOME=/workspace` from Podman (measured), so a `--user` without this HOME is refused here rather
+		// than started. The builder does not insist, because `doctor --live`'s probes run `--user` with no environment.
+		if (user !== null && home !== CONTAINER_HOME) {
+			throw new Error(`runContainer: a job user (${user}) must be paired with HOME=${CONTAINER_HOME}`);
+		}
 
 		// Closed env allowlist: only the provider key + the declared PI_* vars. Throws (config) if
-		// the provider is unconfigured -- the processor turns that into a pre-spend refusal.
+		// the provider is unconfigured, which the processor turns into a policy refusal that refunds the
+		// reserve. That sentence was aspirational until issue #310: the processor did not read the
+		// `piDispatchConfig` tag at all, so the throw fell through to a bare rethrow with the budget kept.
+		// The common case no longer reaches here either, because the processor probes the same resolution
+		// among its free gates, ahead of the mint and the clone.
 		const env = buildContainerEnv({
 			provider: job.provider,
 			model: job.model,
@@ -86,18 +117,33 @@ export function makeRunContainer({
 			// before any spend (command-unregistered). Same guard shape as `flow` directly above, and
 			// mutually exclusive with it by parse -- a job carries one or the other, never both.
 			command: typeof job.command === "string" && job.command.trim() !== "" ? job.command : undefined,
+			// Issue #291: the trigger's tool denylist, off `job` like command/flow. The loader guarantees a
+			// non-empty validated array; the guard is the same defensive shape `flow` above wears, so a
+			// hand-built job with junk in the field emits no variable rather than an empty one.
+			excludeTools: Array.isArray(job.excludeTools) && job.excludeTools.length > 0 ? job.excludeTools : undefined,
 			authFromPi, // source the provider key from pi's auth.json when the env has none
 			// REQ-TRIGGER-SECRETS: this trigger's resolved secrets, fetched by the processor BEFORE anything
 			// spent. Off the call bag rather than off `job` or the closure: it is neither a per-job fact the
 			// record may carry nor a deployment setting, it is a live credential, and `token` is its precedent.
 			secrets,
+			home: user !== null ? home : null, // issue #341: HOME only beside --user, assigned after the forward loops
 		});
 
 		// `-net` on this container's own name (egress.mjs). null when no policy is armed, and docker-run's
 		// guard then omits the flag entirely, so the argv is byte-identical to one built before this feature.
 		const network = egress ? networkNameFor(name) : null;
 
-		const args = buildDockerRunArgs({
+		// Issue #345: BESIDE the job directory, never inside the `/job:ro` mount. Removed first because the docker CLI refuses
+		// to start with an existing cidfile; every attempt's job dir is a fresh mkdtemp, so a leftover is not expected, and
+		// this keeps one from turning into a start failure if a caller ever reuses a directory.
+		const cidFile = `${prepared.jobDir}.cid`;
+		try {
+			fs.rmSync(cidFile, { force: true });
+		} catch {
+			// an unremovable stale file makes the run itself fail to start, which is reported as that
+		}
+
+		const args = buildArgs({
 			// Same split as packagePaths above: the per-job value off `job`, the deployment value off the closure,
 			// so a trigger can name its own toolchain (INT-TRIGGERS-FILE-CONTRACT). Resolved through the SAME
 			// function the pre-spend preflight uses (image-preflight.mjs), so the tag that was checked is the tag
@@ -113,6 +159,15 @@ export function makeRunContainer({
 			globalPiDir, // undefined/null -> docker-run's guard skips the /opt/pi-global mount
 			name,
 			network, // REQ-EGRESS-ALLOWLIST: null when no policy is armed, and the flag is then absent
+			user, // issue #341: the worker's own "<uid>:<gid>" on a daemon that enforces bind-mount ownership, else null
+			cidFile, // issue #345: where the CLI writes this attempt's container ID, read below when the run exits "never started"
+			// Issue #355. `=== true`, so only the processor's boolean re-owns anything. The workspace is relabelled only when the
+			// worker made it: a forge job's is its own clone under the job dir, a local job's IS the operator's folder, which a
+			// private label would take from every other container and from the operator's own labelling.
+			relabel: relabel === true,
+			// Both facts, the kind the preparers branch on AND containment in this job's own directory, so a kind added later
+			// that works on a folder in place fails closed rather than relabelling it.
+			workspaceOwned: job?.kind !== "local" && insideDir(prepared.jobDir, prepared.workspace),
 		});
 
 		// REQ-EGRESS-ALLOWLIST. This job's own --internal network, created here rather than at boot because
@@ -123,7 +178,7 @@ export function makeRunContainer({
 		//
 		// A failure to build it is INFRA, not policy: nothing has been spent, a retry may well succeed, and
 		// `container-never-started` is literally true, so the reservation is given back (processor.mjs).
-		if (network && !(await createJobNetwork(spawnFn, { network, proxy: egressProxy }))) {
+		if (network && !(await createJobNetwork(spawnFn, { network, proxy: egressProxy, bin }))) {
 			throw new InfraRetry("container-never-started", { reason: "container-never-started" });
 		}
 
@@ -132,7 +187,7 @@ export function makeRunContainer({
 		const sink = openJobLog(name);
 
 		const run = new Promise((resolve, reject) => {
-			const child = spawnFn("docker", args, { stdio: ["ignore", "pipe", "pipe"] });
+			const child = spawnFn(bin, args, { stdio: ["ignore", "pipe", "pipe"] });
 			// A throwing sink.write is swallowed so a misbehaving sink cannot break the tee or hang the run.
 			const tee = (chunk) => {
 				onOutput(chunk);
@@ -151,25 +206,31 @@ export function makeRunContainer({
 			});
 			child.on("close", async (code) => {
 				const aborted = signal?.aborted === true; // capture BEFORE the await
-				// A rejecting sink.close is swallowed so a misbehaving sink cannot hang the run; turns/tokens/session/usage/context fall back to null.
+				// A rejecting sink.close is swallowed so a misbehaving sink cannot hang the run; turns/tokens/session/usage/context/exitReason fall back to null.
 				let turns = null;
 				let tokens = null;
 				let session = null;
 				let usage = null;
 				let context = null;
+				// Issue #437: the runner's exit-2 reason, already filtered by parseExitReason to the closed
+				// RUNNER_POLICY_REASONS set and to a line that itself said code 2. The processor still decides
+				// the retry class from `code` alone; this only picks the label inside exit 2.
+				let exitReason = null;
 				try {
 					// `context = null` is a DEFAULT rather than a plain destructure: an injected sink that
 					// predates the field returns no such key, and `undefined` would then reach the record's
 					// shape where every other absence is spelled `null`.
-					({ turns, tokens, session, usage, context = null } = await sink.close());
+					// `exitReason` defaults the same way, for the same reason.
+					({ turns, tokens, session, usage, context = null, exitReason = null } = await sink.close());
 				} catch {
 					turns = null;
 					tokens = null;
 					session = null;
 					usage = null;
 					context = null;
+					exitReason = null;
 				}
-				resolve(aborted ? { code: code ?? 137, aborted: true, turns, tokens, session, usage, context } : { code: code ?? 1, aborted: false, turns, tokens, session, usage, context });
+				resolve(aborted ? { code: code ?? 137, aborted: true, turns, tokens, session, usage, context, exitReason } : { code: code ?? 1, aborted: false, turns, tokens, session, usage, context, exitReason });
 			});
 		});
 
@@ -177,9 +238,115 @@ export function makeRunContainer({
 		// container has already exited, its code is the job's answer, and a teardown fault must not rewrite
 		// that answer. What a failure leaves behind is a memberless network, which the boot reaper sweeps.
 		try {
-			return await run;
+			const result = await run;
+			// Issue #345: an exit that says "never started" is checked against the cidfile BEFORE the network goes, so a
+			// container found running is stopped while its network still exists. Only when the worker did not abort it.
+			if (!result.aborted && (neverStartedExits ?? []).includes(result.code) && (await stopDetached({ spawnFn, cidFile, fs, bin, ...detachedCheck }))) {
+				return { ...result, detached: true };
+			}
+			return result;
 		} finally {
-			if (network) await removeJobNetwork(spawnFn, { network, proxy: egressProxy });
+			if (network) {
+				await removeJobNetwork(spawnFn, {
+					network,
+					proxy: egressProxy,
+					bin,
+					...(typeof teardownRuntime === "function" ? { readRuntime: async () => teardownRuntime(job) } : {}),
+					onRefused: (reason) => log("job_network_not_removed", { network, reason }),
+				});
+			}
+			try {
+				fs.rmSync(cidFile, { force: true });
+			} catch {
+				// best effort: a leftover file is removed before this job's next attempt starts
+			}
 		}
 	};
 }
+
+/** The detached check's whole bound, retries included, so a daemon that stopped answering cannot hold the slot for long. */
+export const DETACHED_CHECK_TIMEOUT_MS = 10_000;
+/** The least time any single step of it is given, so the last one tried at the deadline is not killed before it is sent. */
+export const DETACHED_MIN_STEP_MS = 5_000;
+
+/**
+ * Whether THIS attempt created a container that is still there after `docker run` exited "never started" (issue #345), and
+ * if so stop and remove it by ID. Measured on rootful Podman 5.8.2: killing the API service mid-job made the docker CLI
+ * exit 125 while the container kept running, outside every abort and every refund.
+ *
+ * The cidfile is the evidence it was THIS attempt's: the docker CLI writes the ID right after a successful create and
+ * removes the file when nothing was created, so a plain name conflict (another attempt's live container) leaves no ID and
+ * is never touched. On Podman the service also deletes the file when it removes the container (measured), which reads
+ * as nothing to check, the same as a container already gone. With an ID, `ps -a --no-trunc --filter id=` answers
+ * (`--no-trunc`, because `{{.ID}}` prints 12 characters and the cidfile holds 64):
+ *   - nothing at all: nothing runs on, so never started, as before;
+ *   - this ID listed `created`: created and never started, so it is removed and still counts as never started;
+ *   - this ID in any other state, any other output (a daemon that ignored `--no-trunc`), or `ps` failing or timing out:
+ *     this attempt's container may be running, so it is stopped and removed best effort, and the run is DETACHED (it
+ *     did start, so its slot is not refunded).
+ *
+ * A `ps`, `stop` or `rm` that fails is ASKED AGAIN until `timeoutMs` has passed since the check began: measured with
+ * Podman's service SIGKILLed, the CLI exits 125 within tens of milliseconds and a single `ps` meets a refused connection,
+ * so a restarting service would never be asked to stop the container. Each try is bounded by what is left of the deadline,
+ * but never below `minStepMs`, so the step that runs at the deadline (a `stop` that waited out its grace period, then the
+ * `rm -f`) still has time to reach the daemon. Worst case: the deadline plus two of those.
+ */
+export async function stopDetached({ spawnFn, cidFile, fs, timeoutMs = DETACHED_CHECK_TIMEOUT_MS, minStepMs = DETACHED_MIN_STEP_MS, now = () => Date.now(), delay = (ms) => new Promise((resolve) => setTimeout(resolve, ms)), retryMs = 500, run = dockerStep, bin = "docker" }) {
+	let id;
+	try {
+		id = String(fs.readFileSync(cidFile, "utf8")).trim();
+	} catch {
+		return false;
+	}
+	if (!/^[0-9a-f]{64}$/.test(id)) return false;
+	const deadline = now() + timeoutMs;
+	// One step, asked again while it fails and the check's own deadline has not passed; each try is bounded by what is left.
+	const step = async (args) => {
+		for (;;) {
+			const result = await run(spawnFn, args, Math.max(minStepMs, deadline - now()), bin);
+			if (result.code === 0 || now() + retryMs >= deadline) return result;
+			await delay(retryMs);
+		}
+	};
+	const listed = await step(["ps", "-a", "--no-trunc", "--filter", `id=${id}`, "--format", "{{.ID}} {{.State}}"]);
+	const lines = listed.stdout.split(/\r?\n/).map((l) => l.trim()).filter(Boolean);
+	if (listed.code === 0 && lines.length === 0) return false;
+	const line = lines.find((l) => l.startsWith(`${id} `));
+	if (listed.code === 0 && lines.length === 1 && line && /^created$/i.test(line.slice(id.length + 1).trim())) {
+		await step(["rm", "-f", id]);
+		return false;
+	}
+	await step(["stop", id]);
+	await step(["rm", "-f", id]);
+	return true;
+}
+
+/** One bounded CLI step under `bin`: `{ code, stdout }`, `code: null` when it could not run or overran. Never throws. */
+function dockerStep(spawnFn, args, timeoutMs, bin = "docker") {
+	return new Promise((resolve) => {
+		let child;
+		try {
+			child = spawnFn(bin, args, { stdio: ["ignore", "pipe", "ignore"] });
+		} catch {
+			resolve({ code: null, stdout: "" });
+			return;
+		}
+		let stdout = "";
+		let done = false;
+		const finish = (code) => {
+			if (done) return;
+			done = true;
+			clearTimeout(timer);
+			resolve({ code, stdout });
+		};
+		const timer = setTimeout(() => {
+			try {
+				child.kill("SIGKILL");
+			} catch {}
+			finish(null);
+		}, timeoutMs);
+		child.stdout?.on("data", (d) => (stdout += d));
+		child.on("error", () => finish(null));
+		child.on("close", (code) => finish(code));
+	});
+}

package/src/run-history.mjs

@@ -1,5 +1,7 @@
 import * as nodeFs from "node:fs";
+import { scrubCredentials } from "./redact.mjs";
 import { basename, join } from "node:path";
+import { resolveBackendName } from "./backend-registry.mjs";
 import { isForgeKind, targetSeparator } from "./forges.mjs";
 
 /**
@@ -15,7 +17,7 @@ import { isForgeKind, targetSeparator } from "./forges.mjs";
  * with a fake and no disk. `makeLogSink` streams a job's raw output to a per-job `.log` and recovers the
  * turn count from a bounded tail; `makeRecordWriter` serialises a finished run to a JSON sidecar;
  * `makeFindPreviousRun` reads a scheduler's most recent prior sidecar back; `makeLogReaper` sweeps aged
- * `.log`/`.json` files at boot.
+ * `.log`/`.json` files at boot and on the retention timer (issue #292).
  */
 
 /**
@@ -59,7 +61,7 @@ export function sanitizeJobId(id) {
  * END (a mid-write death) has no complete object and is skipped too. A fragment is never MISREAD as a
  * value: a broken one does not parse and an anchorless one is not repaired.
  *
- * NEVER throws, like the five scanners that call it.
+ * NEVER throws, like the six scanners that call it.
  */
 function parseTailLine(line) {
 	try {
@@ -82,13 +84,18 @@ function parseTailLine(line) {
  * Recover the agent's turn count from buffered container stdout, or `null` if it is not reported.
  *
  * The stream interleaves docker/agent noise and other JSON events (`pi_auto_retry`) with the runner's
- * own lines. Only the success exit line carries `turns` (`image/runner/run-job.mjs:263`); the
- * catch-path exit line (`:277`) omits it. Scan from the end and return the turns of the last `exit`
- * event that reports an integer count, repairing a glued line on the way (`parseTailLine`).
+ * own lines. Only the decided-outcome exit line carries `turns` (`image/runner/run-job.mjs:431`); the
+ * catch-path exit line (`:446`) omits it. The decided line's `retryTurns` (issue #449, pi's own
+ * auto-retry turns, which the turn budget does not count) is not recovered here: it is diagnostic, read
+ * from the container log, and the record's `turns` stays the budgeted count. Scan from the end and
+ * return the turns of the last `exit` event that reports an integer count, repairing a glued line on
+ * the way (`parseTailLine`).
  *
  * This is read-only telemetry: it MUST NEVER throw and MUST NOT feed exit-code or retry
  * classification -- that is the container exit code's job (INT-RUNNER-EXIT-CODE-PROTOCOL). Every parse
- * is guarded; a truncated or non-JSON line is skipped.
+ * is guarded; a truncated or non-JSON line is skipped. The one exit-line field that reaches the
+ * terminal outcome at all is `parseExitReason`'s, and it cannot move a job between retry classes: the
+ * container exit code alone decides the class, and that parsed reason only picks a label INSIDE exit 2.
  */
 export function parseExitTurns(text) {
 	if (typeof text !== "string") return null;
@@ -103,6 +110,43 @@ export function parseExitTurns(text) {
 	return null;
 }
 
+/**
+ * The runner's exit-2 reasons the worker names in the record and the status comment (issue #437). CLOSED
+ * and EXPORTED because three lookups key off it and each would fail quietly on a member it lacks: the
+ * processor comments `TERMINAL_COMMENTS[reason]` (a missing row posts `undefined`), start.mjs builds
+ * `HOOK_POLICY_REASONS` by spreading this set (so a member cannot be left unpaged), and parseExitReason
+ * below admits nothing else. A test requires a TERMINAL_COMMENTS row per member. The runner writes the
+ * literal in `image/runner/src/outcome.mjs`, which the shipped worker cannot import, so a test reads that
+ * source and requires every member here to appear there verbatim.
+ */
+export const RUNNER_POLICY_REASONS = new Set(["provider-auth-refused"]);
+
+/**
+ * The reason off the LAST runner exit line, or `null`: a member of `RUNNER_POLICY_REASONS` and only when
+ * that same line says `code: 2`. Scanned from the end exactly as `parseExitTurns` is, repairing a glued
+ * line through `parseTailLine`, and NEVER throws.
+ *
+ * Why this may reach the outcome when its five siblings may not: it cannot change the retry class. The
+ * processor consults it only inside its container-exit-2 branch, so a container that exits 1 while
+ * printing `{"event":"exit","code":2,"reason":"provider-auth-refused"}` is still InfraRetry, and the
+ * worst a forged line can do on a real exit 2 is swap one not-retried label for another from a closed
+ * set. The `code === 2` check on the line itself is what keeps a stale or mismatched line (a runner
+ * that said 1 in its own words) from naming the outcome. Only the LAST exit event counts, because an
+ * earlier one in the tail is not the one the runner exited on. A fixed enum, so the PII-free record stays so.
+ */
+export function parseExitReason(text) {
+	if (typeof text !== "string") return null;
+	const lines = text.split("\n");
+	for (let i = lines.length - 1; i >= 0; i--) {
+		const line = lines[i].trim();
+		if (line === "") continue;
+		const parsed = parseTailLine(line);
+		if (parsed?.event !== "exit") continue;
+		return parsed?.code === 2 && RUNNER_POLICY_REASONS.has(parsed?.reason) ? parsed.reason : null;
+	}
+	return null;
+}
+
 /**
  * Recover the agent's token usage from buffered container stdout, or `null` if it is not reported.
  *
@@ -126,7 +170,16 @@ export function parseExitTurns(text) {
  * beside the store because this module is where the container's copy is admitted, and an enum that lives
  * anywhere but the admission point is a comment, not a check.
  */
-const SESSION_REASONS = new Set([
+/**
+ * EXPORTED so the enum has ONE home. It is written out in five places -- here, the record shape and the
+ * producer rows in `INT-RUN-HISTORY-FILE-CONTRACT`, and the two tables in `docs/sessions.md` -- and until
+ * issue #375's review rounds nothing compared them: an invented token added here survived the whole suite,
+ * and a token removed was caught only because a test restated the list by hand. `session-reasons.test.mjs`
+ * derives the record-shape enum and both operator tables from this set. THE PRODUCER ROWS ARE NOT DERIVED
+ * and nothing checks them, which is said here rather than left to be assumed: they say which PATH can emit
+ * each token, and this set knows only the vocabulary.
+ */
+export const SESSION_REASONS = new Set([
 	"resumed",
 	"absent",
 	"expired",
@@ -136,7 +189,11 @@ const SESSION_REASONS = new Set([
 	"too-large",
 	"unparseable",
 	"not-a-regular-file",
+	"key-not-a-directory",
+	"transcript-diverted",
+	"venue-changed",
 	"pi-version-changed",
+	"transcript-replaced",
 	"locked",
 	"promote-failed",
 	"disabled",
@@ -374,11 +431,11 @@ function rebuildUsage(u) {
  * path embeds the operator's OS account name.
  *
  * `reason` is a fixed enum passthrough (worker-abort | over-budget | unprotected-branch |
- * runner-policy | job-image-missing | egress-proxy-missing | ...), never free-form or payload text. `exitCode`, `turns`, and `budgetReserved`
+ * runner-policy | provider-auth-refused | job-image-missing | egress-proxy-missing | ...), never free-form or payload text. `exitCode`, `turns`, and `budgetReserved`
  * default to `null` when the outcome does not carry them, so the record shape is stable whether or not
  * the source reports those fields.
  */
-export function buildRecord({ job, result, error, startedAt, endedAt, host = null }) {
+export function buildRecord({ job, result, error, startedAt, endedAt, host = null, defaultBackend = null }) {
 	const data = job.data ?? {};
 	const kind = data.kind ?? job.name;
 	const source = result ?? error ?? {};
@@ -408,7 +465,12 @@ export function buildRecord({ job, result, error, startedAt, endedAt, host = nul
 		provider: source.provider ?? null,
 		model: source.model ?? null,
 		budgetReserved: source.budgetReserved ?? null,
-		attempt: job.attemptsMade ?? 0,
+		// The ATTEMPT NUMBER, 1-based: the first run of a job is 1, its retry 2 (ledger item, issue #464 round). Every
+		// record is written while the job is still processing, where BullMQ's `attemptsMade` counts the attempts FINISHED
+		// before this one (it increments in moveToFinished/moveToFailed, measured against bullmq 5.80.4), so it read 0 on
+		// a first attempt and 1 on the last of two, and the panel's "attempt 1" named the retry. +1 is the number an
+		// operator reads, and what the worker's own `job_failed` log line already shows (logged after the increment).
+		attempt: (Number.isInteger(job.attemptsMade) && job.attemptsMade >= 0 ? job.attemptsMade : 0) + 1,
 		// Chain telemetry (INT-RUN-HISTORY-FILE-CONTRACT): additive and nullable, explicit literals, no spread.
 		// parentJobId/chainDepth come from a chained child's own job.data; chainRefused counts a PARENT's
 		// /outbox requests that were refused. A chain refusal is pre-enqueue of the child, so the `reason` enum
@@ -463,6 +525,26 @@ export function buildRecord({ job, result, error, startedAt, endedAt, host = nul
 		// Passed in rather than read from a module-level value, so `buildRecord` stays pure and every
 		// existing caller keeps getting `null` without knowing this field exists.
 		host,
+		// Which VENUE it resolved to (issue #277). Additive, an explicit literal, TAIL position after `host`,
+		// on host's own argument: field order is the serialisation order, so the tail leaves twenty-five
+		// existing positions untouched. Unconditional, on the same `tokens`/`usage`/`session` precedent.
+		//
+		// THE RESOLVED NAME, NEVER THE RAW FIELD. `data.backend` is ABSENT for every trigger that names no
+		// venue, which is nearly all of them, and an absent key would mean "whatever the default was that
+		// day" -- exactly what a record exists not to mean. `resolveBackendName` is the registry's own
+		// expression, so what this records is what dispatch chose rather than a second opinion of it.
+		//
+		// ON A JOB REFUSED BEFORE ANY CONTAINER, it is still the venue the job RESOLVED to, not a claim that a
+		// container ran there; `outcome` and `reason` say whether one did, the way `provider`/`model` above
+		// still attribute a catch-path death. A `backend-unblessed` refusal therefore records the unblessed
+		// name -- the default would be false (the registry never falls back) and a null would discard the
+		// one fact that refusal is about.
+		//
+		// ADMISSIBLE for the reason `processor.mjs` gives where it logs the same name: a backend name is
+		// operator-authored config checked against a charset at load, never payload. `null` only where no
+		// default was passed (a dependency-injection seam; `recordRun` in start.mjs always passes one) or
+		// where the job data carries an explicit null, which no producer writes and the blessed gate refuses.
+		backend: resolveBackendName(data, defaultBackend),
 	};
 }
 
@@ -544,7 +626,11 @@ export function makeLogSink({ logsDir, enabled, fs = nodeFs, log = () => {} }) {
 		}
 
 		async function close({ timeoutMs = 2000 } = {}) {
-			// Capture turns/tokens/session/usage/context from the tail first, so they survive even if the flush errors or times out.
+			// Capture turns/tokens/session/usage/context/exitReason from the tail first, so they survive even if the flush errors or times out.
+			// The tail accumulates whether or not `enabled` is set, which is what keeps exitReason in the record
+			// when raw logs are off: a label that vanished with log capture would make the record depend on an
+			// opt-in PII switch.
+			const exitReason = parseExitReason(tail);
 			const turns = parseExitTurns(tail);
 			const tokens = parseExitTokens(tail);
 			const session = parseExitSession(tail);
@@ -573,7 +659,7 @@ export function makeLogSink({ logsDir, enabled, fs = nodeFs, log = () => {} }) {
 			} catch (err) {
 				log("log_sink_error", { jobId, reason: err?.message });
 			}
-			return { turns, tokens, session, usage, context };
+			return { turns, tokens, session, usage, context, exitReason };
 		}
 
 		return { write, close };
@@ -664,8 +750,11 @@ export function makeFindPreviousRun({ logsDir, fs = nodeFs }) {
 }
 
 /**
- * The durable log reaper: a boot-time sweep that deletes `.log` and `.json` history files older than
- * the retention window, keeping the logs directory bounded across restarts.
+ * The durable log reaper: an age sweep that deletes `.log` and `.json` history files older than the
+ * retention window, keeping the logs directory bounded. Runs at boot AND on the retention timer since
+ * issue #292 (`PI_SWEEP_INTERVAL_HOURS`), because a worker that never restarts never re-swept. It holds
+ * no state between calls, so it is safe to re-run; deletes are per FILE, so no single call is unbounded
+ * the way the sandbox reaper's tree deletes are.
  *
  * Fault isolation is the contract, mirroring `makeReaper` in `start.mjs`: `reapLogs` NEVER throws under
  * any input. A missing logs directory on first boot (`readdirSync` ENOENT), an unreadable entry, or an
@@ -693,11 +782,11 @@ export function makeLogReaper({ logsDir, retentionDays, fs = nodeFs, log = () =>
 						log("reaped_log", { file: name });
 					}
 				} catch (err) {
-					log("log_reaper_skipped", { file: name, reason: err?.message });
+					log("log_reaper_skipped", { file: name, reason: scrubCredentials(err?.message) });
 				}
 			}
 		} catch (err) {
-			log("log_reaper_skipped", { reason: err?.message });
+			log("log_reaper_skipped", { reason: scrubCredentials(err?.message) });
 		}
 	};
 }