@edgehero/pi-dispatch 2.0.0 → 3.0.0

package/src/run-history.mjs

@@ -1,8 +1,11 @@
+import { createHmac, timingSafeEqual } from "node:crypto";
 import * as nodeFs from "node:fs";
 import { scrubCredentials } from "./redact.mjs";
 import { basename, join } from "node:path";
 import { resolveBackendName } from "./backend-registry.mjs";
 import { isForgeKind, targetSeparator } from "./forges.mjs";
+import { MODEL_REF_PATTERN as USAGE_ID_PATTERN } from "./model-ref.mjs";
+import { isProjectId } from "./project-id.mjs";
 
 /**
  * Durable per-run history.
@@ -118,8 +121,15 @@ export function parseExitTurns(text) {
  * below admits nothing else. A test requires a TERMINAL_COMMENTS row per member. The runner writes the
  * literal in `image/runner/src/outcome.mjs`, which the shipped worker cannot import, so a test reads that
  * source and requires every member here to appear there verbatim.
+ *
+ * The last four are the policy stops of issues #501 (a per-job dollar cap) and #502 (an allowed-model list):
+ * `cost-cap` and `model-not-allowed` when the runner's pre-call guard stopped a call, and
+ * `cost-cap-unenforceable` and `model-policy-unenforceable` when the runner refused, before any call, a policy
+ * it could not enforce before a call. The two refusals are live from the first image that knows the variables.
+ * `cost-cap` is live from the image that declares `costCap`, whose runner carries the cost guard, and
+ * `model-not-allowed` from the image that declares `modelPolicy`, whose runner carries the model guard.
  */
-export const RUNNER_POLICY_REASONS = new Set(["provider-auth-refused"]);
+export const RUNNER_POLICY_REASONS = new Set(["provider-auth-refused", "cost-cap", "model-not-allowed", "cost-cap-unenforceable", "model-policy-unenforceable"]);
 
 /**
  * The reason off the LAST runner exit line, or `null`: a member of `RUNNER_POLICY_REASONS` and only when
@@ -147,6 +157,59 @@ export function parseExitReason(text) {
 	return null;
 }
 
+/**
+ * The cost guard's refusal rules a `cost-cap` exit line may name as its `why` (issue #507): the runner's
+ * `COST_REFUSALS` (`image/runner/src/outcome.mjs`), restated because the shipped worker cannot import the runner, and
+ * pinned to it by a test that reads that source. CLOSED, because the value reaches the PII-free record.
+ */
+export const COST_CAP_WHYS = Object.freeze(["unboundable", "external", "over-cap"]);
+
+/**
+ * The `why` off the LAST runner exit line, or `null`: a member of `COST_CAP_WHYS`, and only when that same line says
+ * `code: 2` and `reason: "cost-cap"`. Scanned from the end exactly as `parseExitReason` is, and NEVER throws.
+ *
+ * The line is container-written, so nothing it carries is trusted as text: a value outside the closed set is null,
+ * never a string copied through. It feeds no classification. The processor reads it only in its exit-2 branch, and
+ * only beside a `cost-cap` reason, where it becomes the record's `why`: the worst a forged line can do is name the
+ * wrong rule of the three.
+ */
+export function parseExitWhy(text) {
+	if (typeof text !== "string") return null;
+	const lines = text.split("\n");
+	for (let i = lines.length - 1; i >= 0; i--) {
+		const line = lines[i].trim();
+		if (line === "") continue;
+		const parsed = parseTailLine(line);
+		if (parsed?.event !== "exit") continue;
+		return parsed?.code === 2 && parsed?.reason === "cost-cap" && COST_CAP_WHYS.includes(parsed?.why) ? parsed.why : null;
+	}
+	return null;
+}
+
+/**
+ * The LAST runner exit line's own `code`, an integer, or `null` (issue #501, PR #542's review round 3). Scanned from
+ * the end exactly as its siblings are, repairing a glued line through `parseTailLine`, and NEVER throws.
+ *
+ * Read for ONE purpose: the dollar settlement trusts an exit line's cost only when the line's `code` equals the
+ * container's real exit code (processor.mjs). The runner writes the code it exits with on both exit lines
+ * (`image/runner/run-job.mjs`: `...capExitMessage(outcome)` then `return outcome.code` on the decided path, and
+ * `code: capped.code` on the catch path; pinned by a test), so a genuine last line always matches, while a line a
+ * job's own tool forged before a `docker stop` (exit 137, no genuine line after it) does not. It never feeds the
+ * retry class (INT-RUNNER-EXIT-CODE-PROTOCOL).
+ */
+export function parseExitCode(text) {
+	if (typeof text !== "string") return null;
+	const lines = text.split("\n");
+	for (let i = lines.length - 1; i >= 0; i--) {
+		const line = lines[i].trim();
+		if (line === "") continue;
+		const parsed = parseTailLine(line);
+		if (parsed?.event !== "exit") continue;
+		return Number.isSafeInteger(parsed?.code) ? parsed.code : null;
+	}
+	return null;
+}
+
 /**
  * Recover the agent's token usage from buffered container stdout, or `null` if it is not reported.
  *
@@ -186,6 +249,7 @@ export const SESSION_REASONS = new Set([
 	"conversation-too-old",
 	"resume-chain-too-long",
 	"context-too-full",
+	"compaction-summary-empty",
 	"too-large",
 	"unparseable",
 	"not-a-regular-file",
@@ -274,8 +338,31 @@ export function parseExitContext(text) {
  * (`image/runner/run-job.mjs` -> `pickTotals`, which sends the first four and `metered: false`). Order
  * matters because it is what makes a conformant runner's object round-trip byte-identically through the
  * rebuild below, so the record's bytes do not move for anyone running a real image.
+ *
+ * After `unpriced` come the three child keys of issue #500 (`DES-USAGE-METER-VIA-API-PROVIDER-REGISTRY`), which a
+ * metered runner from part E on writes on every line, zeros with no children: `childTotal` (the children's billed
+ * tokens, the fourth part of root + other + loose + child = total), `childProcesses` (child ledger files plus pi
+ * processes found with none) and `unmeteredChildren` (pi children whose spend the job could not count). The last is a
+ * floor counter (`FLOOR_COUNTERS`), so it is on this list for the same reason as the policy counters below: a dropped
+ * key would read as an honest zero. A worker before issue #500 part F drops all three, and the `total` it keeps
+ * already includes the children.
+ *
+ * Then `costUnreported` (issue #571), which the metered runner writes on every line, capped or not, after the child
+ * keys: calls on a priced model whose answer carried broken usage (pi records a missing usage block as zeros), so
+ * `cost` is short. A floor counter, on this list for the same reason as `unmeteredChildren`.
+ *
+ * The last seven are the policy counters of issues #501 and #502, in the order the runner emits them after
+ * it: `costCapMicros` (the job's cap), `costRefused` (calls the cost guard stopped),
+ * `boundExceeded` (calls that cost more than their bound), `longContext` (calls priced past a long-context
+ * threshold the catalog does not tier), `costUnjudged` (compat entries found displaced under the cap, so calls
+ * may have run unjudged and unmetered), `costUnanswered` (failed calls that never started, charged their metered
+ * cost though a provider may have billed one, and since issue #571 calls whose dispatch threw before any answer),
+ * `modelRefused` (calls the model guard stopped). The cost guard
+ * writes the first six whenever a cost cap is set, and the model guard `modelRefused` whenever a list is. All seven are on
+ * this closed list because a key missing from it is DROPPED, and the dollar settlement reads them to decide
+ * whether a metered cost is complete: a dropped counter would read as an honest zero.
  */
-const TOKEN_KEYS = ["input", "output", "total", "cost", "metered", "rootTotal", "otherTotal", "looseTotal", "sessions", "calls", "unresolved", "unpriced"];
+export const TOKEN_KEYS = Object.freeze(["input", "output", "total", "cost", "metered", "rootTotal", "otherTotal", "looseTotal", "sessions", "calls", "unresolved", "unpriced", "childTotal", "childProcesses", "unmeteredChildren", "costUnreported", "costCapMicros", "costRefused", "boundExceeded", "longContext", "costUnjudged", "costUnanswered", "modelRefused"]);
 
 /**
  * Rebuild the billed totals from a closed key list rather than passing the container's object through.
@@ -289,7 +376,7 @@ const TOKEN_KEYS = ["input", "output", "total", "cost", "metered", "rootTotal",
  * did not, and the asymmetry was an oversight rather than a decision.
  *
  * A key the runner omitted stays OMITTED rather than becoming null: the fallback shape legitimately
- * carries only five of the twelve, and a null there would read as "measured zero" for a number nobody
+ * carries only five of the twenty-three, and a null there would read as "measured zero" for a number nobody
  * measured. `typeof === "number"` rather than `Number.isFinite`, deliberately, so this narrows WHICH
  * KEYS survive and never which objects are admitted -- the admission gate above is unchanged.
  */
@@ -320,10 +407,6 @@ export function parseExitTokens(text) {
 	return null;
 }
 
-/** The id allowlist for a ledger row's provider/model, applied AFTER lowercasing. The first-char class
- *  has no dot, colon or slash, so `.hidden`, `../etc` and `:` shapes fail at character one. */
-const USAGE_ID_PATTERN = /^[a-z0-9][a-z0-9._:/-]{0,63}$/;
-
 /** The ten per-row counters, in the row's serialisation order. Absent is an honest zero; anything
  *  present must be a finite non-negative number or the whole block is refused. */
 const USAGE_ROW_NUMERIC_KEYS = ["calls", "input", "output", "cacheRead", "cacheWrite", "cacheWrite1h", "reasoning", "total", "cost", "unpriced"];
@@ -376,7 +459,10 @@ function rebuildUsage(u) {
 		if (typeof u.piAi !== "string" || !/^\d+\.\d+\.\d+$/.test(u.piAi)) return null;
 		piAi = u.piAi;
 	}
-	let truncated = 0;
+	// ABSENT stays null, never 0 (PR #549's review): a runner that did not report how many rows it folded did not
+	// measure it, and the per-model dollar settlement floors on anything but a present 0. Readers that count folded
+	// ledgers treat null as none (`?? 0`).
+	let truncated = null;
 	if (u.truncated !== undefined) {
 		if (!Number.isInteger(u.truncated) || u.truncated < 0) return null;
 		truncated = u.truncated;
@@ -392,6 +478,7 @@ function rebuildUsage(u) {
 		// itself never has to admit uppercase.
 		const provider = row.provider.toLowerCase();
 		const model = row.model.toLowerCase();
+		// The id allowlist (model-ref.mjs, widened for issues #501/#502 so every builtin catalog id passes).
 		if (!USAGE_ID_PATTERN.test(provider) || !USAGE_ID_PATTERN.test(model)) return null;
 		const nums = {};
 		for (const key of USAGE_ROW_NUMERIC_KEYS) {
@@ -430,12 +517,12 @@ function rebuildUsage(u) {
  * path stay out of the record -- for local jobs only the folder's `basename` is kept, because the full
  * path embeds the operator's OS account name.
  *
- * `reason` is a fixed enum passthrough (worker-abort | over-budget | unprotected-branch |
+ * `reason` is a fixed enum passthrough (worker-abort | over-budget | dollar-cap | allocation-cap | envelope-mismatch | portfolio-no-envelope | portfolio-snapshot-oversize | unprotected-branch |
  * runner-policy | provider-auth-refused | job-image-missing | egress-proxy-missing | ...), never free-form or payload text. `exitCode`, `turns`, and `budgetReserved`
  * default to `null` when the outcome does not carry them, so the record shape is stable whether or not
  * the source reports those fields.
  */
-export function buildRecord({ job, result, error, startedAt, endedAt, host = null, defaultBackend = null }) {
+export function buildRecord({ job, result, error, startedAt, endedAt, host = null, defaultBackend = null, project = null }) {
 	const data = job.data ?? {};
 	const kind = data.kind ?? job.name;
 	const source = result ?? error ?? {};
@@ -545,9 +632,85 @@ export function buildRecord({ job, result, error, startedAt, endedAt, host = nul
 		// default was passed (a dependency-injection seam; `recordRun` in start.mjs always passes one) or
 		// where the job data carries an explicit null, which no producer writes and the blessed gate refuses.
 		backend: resolveBackendName(data, defaultBackend),
+		// The dollar reservation's outcome (issue #501, INT-RUN-HISTORY-FILE-CONTRACT). Additive, nullable, an explicit
+		// literal REBUILT here (never the source's object), TAIL position after `backend` on the same contract: field order
+		// is the serialisation order. `{ reservedMicros, settledMicros, basis, modelBasis }`: two integers of micro-dollars,
+		// one fixed token (`metered` | `floor` | `refunded` | `unreserved`), and `modelBasis`, how the per-model windows
+		// settled (`metered` | `floor` | `refunded`, null when the job held none). Null when no dollar window applied to the job (no dollar setting, or a
+		// refusal before the reservation step), which is every record of a deployment that sets none.
+		dollars: dollarsOf(source.dollars),
+		// The refusal's detail (PR #558, the end-of-round check of #501 and #502): which rule of the reason refused the job, the same fixed token
+		// the worker's log line carries, e.g. `overlay-link` under `model-unknown`, so the drill-in can tell the
+		// deployment's file from the job's model. Additive, nullable, an explicit literal, TAIL position after `dollars`
+		// on the same contract. A token of the fixed charset (`WHY_RE`) or null, never a free string, so the record stays
+		// PII-free by construction; null for every reason that carries no detail.
+		why: typeof source.why === "string" && WHY_RE.test(source.why) ? source.why : null,
+		// The job's project (issue #499, INT-PROJECTS-FILE-CONTRACT). Additive, nullable, an explicit literal, TAIL position
+		// after `why` on the same contract. The project's ID only, never its `name`: the id is operator-authored and
+		// charset-checked (`PROJECT_ID_RE`), never payload, the argument `host` and `backend` make, so the record stays
+		// PII-free by construction; the name is free text and has no path here. Passed in, resolved at the pickup gate
+		// (start.mjs `recordRun`), so `buildRecord` stays pure. Anything that is not a well-formed id records null.
+		project: isProjectId(project) ? project : null,
+		// The priorities plan a completed portfolio job wrote (issue #505, INT-RUN-HISTORY-FILE-CONTRACT). Additive, nullable,
+		// an explicit literal REBUILT here, TAIL position after `project` on the same contract. `{ outcome, reason, planId,
+		// clamped }`: a fixed outcome, a fixed reason or null, a 16-hex content hash or null, a boolean. Null for every
+		// run that left no `/outbox/priorities.json` and was no confirmed portfolio job, which is every run of a deployment
+		// with no portfolio trigger; a confirmed one that wrote none says `plan-absent` (issue #507). Never
+		// the plan's weights or its reasons: those are in the allocation audit file, and a reason is agent text.
+		plan: planOf(source.plan),
 	};
 }
 
+/** What became of a collected plan. */
+export const PLAN_RECORD_OUTCOMES = Object.freeze(["applied", "duplicate", "refused"]);
+/**
+ * Every reason a record's `plan` may carry: the collector's own rungs (outbox-plan.mjs `PLAN_COLLECT_REASONS`), the
+ * plan's judgement (`plan-invalid`) and the apply ladder (priorities.mjs `PLAN_LADDER`, with `envelope-mismatch`).
+ * Restated here so this module imports neither, and pinned to them by a test.
+ */
+export const PLAN_RECORD_REASONS = Object.freeze([
+	"plan-absent",
+	"plan-not-portfolio",
+	"plan-oversize",
+	"plan-not-regular-file",
+	"plan-unreadable",
+	"plan-parse-error",
+	"plan-collect-error",
+	"plan-invalid",
+	"delegation-off",
+	"writer-not-allowed",
+	"envelope-mismatch",
+	"plan-duplicate",
+	"plan-stale",
+	"plan-too-soon",
+	"plan-incomplete",
+	"plan-busy",
+]);
+const PLAN_ID_RE = /^[0-9a-f]{16}$/;
+
+/** The record's `plan`, rebuilt from named fields, or null when the source carries none or a malformed one. */
+function planOf(p) {
+	if (p === null || typeof p !== "object" || !PLAN_RECORD_OUTCOMES.includes(p.outcome)) return null;
+	const reason = PLAN_RECORD_REASONS.includes(p.reason) ? p.reason : null;
+	// An applied plan has no reason, and a duplicate or a refusal always has one; a source that breaks that is not trusted.
+	if ((p.outcome === "applied") !== (reason === null)) return null;
+	return { outcome: p.outcome, reason, planId: typeof p.planId === "string" && PLAN_ID_RE.test(p.planId) ? p.planId : null, clamped: p.clamped === true };
+}
+
+/** The charset a record's `why` must match: a lowercase token, the shape of every `why` the processor returns. */
+const WHY_RE = /^[a-z][a-z0-9-]{0,63}$/;
+
+const DOLLAR_BASES = new Set(["metered", "floor", "refunded", "unreserved"]);
+// How the job's MODEL windows settled (issue #502 part 6); anything else, null included, records null.
+const MODEL_BASES = new Set(["metered", "floor", "refunded"]);
+
+/** The record's `dollars`, rebuilt from named fields only, or null when the source carries none or a malformed one. */
+function dollarsOf(d) {
+	if (d === null || typeof d !== "object") return null;
+	if (!Number.isSafeInteger(d.reservedMicros) || d.reservedMicros < 0 || !Number.isSafeInteger(d.settledMicros) || d.settledMicros < 0 || !DOLLAR_BASES.has(d.basis)) return null;
+	return { reservedMicros: d.reservedMicros, settledMicros: d.settledMicros, basis: d.basis, modelBasis: MODEL_BASES.has(d.modelBasis) ? d.modelBasis : null };
+}
+
 /**
  * A stable, non-PII target label. Forge jobs read `repo<sep>number`; local jobs read `local:<basename>` --
  * basename only, so the full folder path (which on Windows carries the OS account name) never lands in
@@ -577,6 +740,49 @@ export function targetFor(kind, data) {
 /** Retain only the last ~8KB of container output for turn recovery, so per-job memory stays flat. */
 const TAIL_CAP_BYTES = 8 * 1024;
 
+/** A signed exit line's tail: `,"auth":"<64 hex>"}`, the MAC as the LAST key (image/runner/src/exit-line.mjs). */
+const SIGNED_TAIL = /,"auth":"([0-9a-f]{64})"\}$/;
+
+/**
+ * The runner exit lines in `text` that carry a valid MAC under `key`, each with its `auth` key removed, joined by
+ * newlines in their original order; "" when there is none (issue #545, INT-RUNNER-EXIT-CODE-PROTOCOL).
+ *
+ * The worker hands each job a fresh random key on the container's stdin, and the runner signs its exit line with
+ * HMAC-SHA256 over the line's unsigned bytes. A job's own tool can write any bytes it likes to the container's stdout,
+ * but not that MAC: the key left the pipe before any tool existed and the runner's memory is closed to the job's uid
+ * (the image's exec-only node). So when a key was issued, only these lines are the runner's, and every `parseExit*`
+ * scanner runs over this text instead of the raw tail: a forged line, before or after the genuine one, is not in it.
+ *
+ * Each candidate starts at a `{"event":"exit"` anchor, the `parseTailLine` repair's reasoning: those raw bytes cannot
+ * occur inside a runner line, so a line glued to stray bytes before it is still found, and an anchor inside the stray
+ * bytes simply fails the MAC. The comparison is `timingSafeEqual` on equal-length digests. Never throws.
+ */
+export function authenticExitLines(text, key) {
+	if (typeof text !== "string" || typeof key !== "string" || key === "") return "";
+	const verified = [];
+	for (const raw of text.split("\n")) {
+		const line = raw.trim();
+		const match = SIGNED_TAIL.exec(line);
+		if (match === null) continue;
+		const signed = line.slice(0, match.index);
+		let from = signed.indexOf('{"event":"exit"');
+		while (from !== -1) {
+			const body = `${signed.slice(from)}}`;
+			try {
+				const expected = createHmac("sha256", key).update(body, "utf8").digest();
+				if (timingSafeEqual(expected, Buffer.from(match[1], "hex"))) {
+					verified.push(body);
+					break;
+				}
+			} catch {
+				// a digest that will not compare is not a match
+			}
+			from = signed.indexOf('{"event":"exit"', from + 1);
+		}
+	}
+	return verified.join("\n");
+}
+
 /**
  * The durable log sink: the I/O layer that streams a job's raw container output to a per-job `.log`
  * file and recovers the turn count from a bounded tail of that same output.
@@ -604,7 +810,10 @@ export function makeLogSink({ logsDir, enabled, fs = nodeFs, log = () => {} }) {
 		log("logs_dir_error", { reason: err?.message });
 	}
 
-	return function openJobLog(jobId) {
+	// `exitKey` (issue #545): the key this job's runner signs its exit line with, or null when none was issued (an image
+	// that does not declare `exitAuth`, or a caller that predates the field). With a key, the exit-line fields are read
+	// from the AUTHENTICATED lines only, and none at all reads exactly like a container that wrote no exit line.
+	return function openJobLog(jobId, { exitKey = null } = {}) {
 		let tail = "";
 		let stream = null;
 
@@ -630,12 +839,20 @@ export function makeLogSink({ logsDir, enabled, fs = nodeFs, log = () => {} }) {
 			// The tail accumulates whether or not `enabled` is set, which is what keeps exitReason in the record
 			// when raw logs are off: a label that vanished with log capture would make the record depend on an
 			// opt-in PII switch.
-			const exitReason = parseExitReason(tail);
-			const turns = parseExitTurns(tail);
-			const tokens = parseExitTokens(tail);
-			const session = parseExitSession(tail);
-			const usage = parseExitUsage(tail);
-			const context = parseExitContext(tail);
+			// With a key, the scanners see only the lines the runner signed (`authenticExitLines`), so a line a job's tool
+			// wrote, before or after the genuine one, is never the one they read. `exitAuth` says which case this was:
+			// absent (no key issued: today's whole-tail read), "verified" or "unverified" (a key, and no line carried it).
+			const keyed = typeof exitKey === "string" && exitKey !== "";
+			const exitText = keyed ? authenticExitLines(tail, exitKey) : tail;
+			const exitAuth = exitText === "" ? "unverified" : "verified";
+			const exitReason = parseExitReason(exitText);
+			const exitLineCode = parseExitCode(exitText);
+			const exitWhy = parseExitWhy(exitText);
+			const turns = parseExitTurns(exitText);
+			const tokens = parseExitTokens(exitText);
+			const session = parseExitSession(exitText);
+			const usage = parseExitUsage(exitText);
+			const context = parseExitContext(exitText);
 			try {
 				if (stream !== null) {
 					const s = stream;
@@ -659,7 +876,9 @@ export function makeLogSink({ logsDir, enabled, fs = nodeFs, log = () => {} }) {
 			} catch (err) {
 				log("log_sink_error", { jobId, reason: err?.message });
 			}
-			return { turns, tokens, session, usage, context, exitReason };
+			// `exitAuth` only when a key was issued, so a keyless close returns exactly the object it always did. `exitWhy`
+			// (issue #507) only when the line named one, for the same reason.
+			return { turns, tokens, session, usage, context, exitReason, exitLineCode, ...(exitWhy !== null ? { exitWhy } : {}), ...(keyed ? { exitAuth } : {}) };
 		}
 
 		return { write, close };
@@ -716,6 +935,11 @@ export function makeRecordWriter({ logsDir, fs = nodeFs, log = () => {} }) {
  * (schedulers "a" vs "a_1": `repeat_a_1_100.json` has a non-digit tail after "repeat_a_", so it never
  * matches scheduler "a"). The max millis strictly below `beforeMillis` is the previous fire.
  *
+ * A fire by hand (`pi-dispatch run --trigger <id>`, issue #505) counts as a run of that trigger: its id is
+ * `manual:<schedulerId>:<millis>`, the millis floored to the minute it was queued, stored as `manual_<id>_<millis>.json`.
+ * It ran the trigger's own data, so the next tick's `previousRunAt` names it rather than an older scheduled fire. A hand
+ * fire and a tick in the same minute share a millis; then the one that ended later is the previous run.
+ *
  * Returns `record.endedAt ?? record.startedAt ?? null` as an ISO string. `endedAt` first: BullMQ never
  * overlaps two fires of one scheduler, so the prior run's end is the honest high-water mark; `startedAt`
  * covers a crashed run's partial record. ANY failure -- missing dir, no prior run, unreadable file, bad
@@ -729,26 +953,157 @@ export function makeFindPreviousRun({ logsDir, fs = nodeFs }) {
 	return function findPreviousRun({ schedulerId, beforeMillis } = {}) {
 		try {
 			if (typeof beforeMillis !== "number" || !Number.isFinite(beforeMillis)) return null;
-			const prefix = sanitizeJobId(`repeat:${schedulerId}:`);
-			const escaped = prefix.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
-			const pattern = new RegExp(`^${escaped}(\\d+)\\.json$`);
+			const escape = (text) => text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+			const pattern = new RegExp(`^(?:${escape(sanitizeJobId(`repeat:${schedulerId}:`))}|${escape(sanitizeJobId(`manual:${schedulerId}:`))})(\\d+)\\.json$`);
 			let best = null;
 			for (const name of fs.readdirSync(logsDir)) {
 				const m = pattern.exec(name);
 				if (m === null) continue;
 				const millis = Number(m[1]);
 				if (!Number.isFinite(millis) || millis >= beforeMillis) continue;
-				if (best === null || millis > best.millis) best = { millis, name };
+				if (best === null || millis > best.millis) best = { millis, names: [name] };
+				else if (millis === best.millis) best.names.push(name);
 			}
 			if (best === null) return null;
-			const record = JSON.parse(fs.readFileSync(join(logsDir, best.name), "utf8"));
-			return record.endedAt ?? record.startedAt ?? null;
+			let at = null;
+			for (const name of best.names) {
+				const record = JSON.parse(fs.readFileSync(join(logsDir, name), "utf8"));
+				const end = record.endedAt ?? record.startedAt ?? null;
+				if (typeof end === "string" && (at === null || end > at)) at = end;
+			}
+			return at;
 		} catch {
 			return null; // NEVER throws: a history fault must not fail the prepare that asked
 		}
 	};
 }
 
+/**
+ * What `makeReadRecord` returns for a record file that exists but cannot be read as a record (unreadable, bad JSON,
+ * not an object). Distinct from `null` (no file), so the lost-lock check can say which it met.
+ */
+export const UNREADABLE_RECORD = Object.freeze({ unreadable: true });
+
+/**
+ * Read one job's run record back, by the name `makeRecordWriter` gave it: `<logsDir>/<sanitizeJobId(id)>.json`
+ * (INT-RUN-HISTORY-FILE-CONTRACT, unchanged). Returns the parsed object, `null` when there is no file, or
+ * `UNREADABLE_RECORD` when there is one that is not a record.
+ *
+ * Why the worker reads its own record: the queue can lose a job's lock after the processor finished and wrote
+ * this file, when Valkey was unreachable for longer than BullMQ's lock renewal window. BullMQ then refuses the
+ * completion ("Missing lock") and its stall check takes the job back, so the queue alone says the job failed or
+ * must run again. The record is the store that already knows it finished (DES-TERMINAL-COMMENTS-AND-FAILURE-HOOK).
+ *
+ * The id inside is NOT checked here: `sanitizeJobId` maps `a:b` and `a_b` to one name, and `recordVerdict` refuses
+ * the other job's record as `id-mismatch`, which is the one place that can also say so. NEVER throws.
+ */
+export function makeReadRecord({ logsDir, fs = nodeFs }) {
+	return function readRecord(jobId) {
+		if (jobId === null || jobId === undefined || jobId === "") return null;
+		let text;
+		try {
+			text = fs.readFileSync(join(logsDir, `${sanitizeJobId(jobId)}.json`), "utf8");
+		} catch (err) {
+			return err?.code === "ENOENT" || err?.code === "ENOTDIR" ? null : UNREADABLE_RECORD;
+		}
+		try {
+			const record = JSON.parse(text);
+			return record !== null && typeof record === "object" && !Array.isArray(record) ? record : UNREADABLE_RECORD;
+		} catch {
+			return UNREADABLE_RECORD;
+		}
+	};
+}
+
+/**
+ * How far a record's `startedAt` may sit BEFORE the job's creation and still be this job's record: 5 minutes.
+ *
+ * The two instants come from two clocks. `job.timestamp` is stamped by whoever ENQUEUED the job (the receiver, the
+ * CLI, another worker) and `startedAt` by the worker that ran it, so a producer whose clock runs ahead of the
+ * worker's would make every genuine record look older than its job and silently turn this check off. The
+ * tolerance absorbs ordinary skew between hosts; what it must still refuse is a record of an OLDER job under a
+ * reused id, and such a record is older by at least that job's whole run, its removal from the queue and a new
+ * enqueue, which in practice is far more than 5 minutes. A skew larger than this is an operator problem the
+ * `job_lost_lock_record_rejected` line (`older-than-job`) makes visible.
+ */
+export const RECORD_CLOCK_SKEW_MS = 5 * 60 * 1000;
+
+/**
+ * Does this record say that THIS attempt of THIS job finished without failing? Returns `null` when it does,
+ * `"absent"` when there is no record, and otherwise the reason it does not, from a fixed set:
+ * `unreadable`, `id-mismatch`, `other-attempt`, `failed`, `older-than-job`. Pure; never throws.
+ *
+ * `attempt` is the 1-based attempt number the record would carry, which is BullMQ's `attemptsMade + 1` while the
+ * job is processing (see `buildRecord`) and `attemptsMade` after BullMQ failed it (its moveToFailed adds one
+ * before the failed event). A record of an earlier attempt does not answer for a later one.
+ *
+ * `since` is the job's creation time in millis (BullMQ's `job.timestamp`). A job id can be used again once the
+ * old job is gone (a fixed `--trigger` id, a redelivered webhook), while its record lives for the retention
+ * window; a record that started before this job existed (less `RECORD_CLOCK_SKEW_MS`) belongs to the old one. A
+ * missing or unparseable `startedAt` is not trusted for the same reason.
+ *
+ * `failed` (an outcome of `failed`, or none) is refused on purpose: a record written by the catch path says the run
+ * threw, and the queue's verdict (failed, or run again) is then the right one.
+ */
+export function recordVerdict(record, { jobId, attempt, since } = {}) {
+	try {
+		if (record === null || record === undefined) return "absent";
+		if (record === UNREADABLE_RECORD || typeof record !== "object" || Array.isArray(record)) return "unreadable";
+		if (record.jobId !== jobId) return "id-mismatch";
+		if (!Number.isInteger(attempt) || record.attempt !== attempt) return "other-attempt";
+		if (typeof record.outcome !== "string" || record.outcome === "failed") return "failed";
+		if (Number.isFinite(since)) {
+			const started = Date.parse(record.startedAt ?? "");
+			if (!Number.isFinite(started) || started < since - RECORD_CLOCK_SKEW_MS) return "older-than-job";
+		}
+		return null;
+	} catch {
+		return "unreadable";
+	}
+}
+
+/** `recordVerdict` as a yes or no. */
+export const recordSettlesAttempt = (record, opts) => recordVerdict(record, opts) === null;
+
+/**
+ * The lookup the worker asks when the queue lost a job's lock: the local record first, then the fleet's copy.
+ *
+ * On a fleet without a shared `PI_LOGS_DIR`, the host that meets the stalled job is not always the host that ran
+ * it, so the local file can be absent or hold an older attempt. The run mirror (`run-mirror.mjs`) holds the same
+ * record's bytes, so it is asked when the local file does not settle the attempt. `readMirrored` is `null` on a
+ * deployment that declared no worker name, which has no mirror and is one host.
+ *
+ * `onReject(reason, source)` is told about every record it FOUND and refused (`source` is `local` or `mirror`,
+ * `reason` one of `recordVerdict`'s tokens, never `absent`), so the caller can log why the failure path stood.
+ *
+ * Resolves to the record or `null`, and NEVER rejects: a mirror fault is no record, which fails toward today's
+ * behaviour (a failure comment, or a run).
+ */
+export function makeSettledRecord({ readRecord, readMirrored = null }) {
+	return async function settledRecord(jobId, { attempt, since, onReject = () => {} } = {}) {
+		const judge = (record, source) => {
+			const why = recordVerdict(record, { jobId, attempt, since });
+			if (why !== null && why !== "absent") {
+				try {
+					onReject(why, source);
+				} catch {
+					// a reporting fault must not change the verdict
+				}
+			}
+			return why === null;
+		};
+		try {
+			const local = readRecord(jobId);
+			if (judge(local, "local")) return local;
+			if (typeof readMirrored !== "function") return null;
+			const mirrored = await readMirrored(jobId);
+			return judge(mirrored, "mirror") ? mirrored : null;
+		} catch {
+			return null;
+		}
+	};
+}
+
 /**
  * The durable log reaper: an age sweep that deletes `.log` and `.json` history files older than the
  * retention window, keeping the logs directory bounded. Runs at boot AND on the retention timer since

package/src/run-mirror.mjs

@@ -1,3 +1,5 @@
+import { UNREADABLE_RECORD } from "./run-history.mjs";
+
 /**
  * A fleet-visible copy of the run history (issue #57, Gap 3).
  *
@@ -186,6 +188,34 @@ export async function readMirroredRuns(redis, { limit = 50, sinceMs = 0, now = (
 	return { runs, degraded: runs.length >= limit ? "truncated" : "ok" };
 }
 
+/**
+ * One run's mirrored record, by its sanitized id: the parsed object, `null`, or `UNREADABLE_RECORD`. Never throws,
+ * never rejects.
+ *
+ * The by-id read the lost-lock check needs (`makeSettledRecord` in `run-history.mjs`): one bounded `GET`, not the
+ * index scan `readMirroredRuns` does for the panel. Unreachable and absent are `null` (nothing found); a value that
+ * is there but is not a record (unparseable, empty, not an object) is `UNREADABLE_RECORD`, exactly as the local
+ * reader says it, so the check can log `unreadable` from `mirror`. Neither is ever accepted: the caller keeps
+ * today's path.
+ */
+export async function readMirroredRecord(redis, sanitizedJobId, { timeoutMs = OP_TIMEOUT_MS } = {}) {
+	if (!redis || !sanitizedJobId) return null;
+	try {
+		const raw = await bounded(redis.get(runRecordKey(sanitizedJobId)), timeoutMs);
+		if (raw === null || raw === undefined) return null;
+		if (typeof raw !== "string" || raw === "") return UNREADABLE_RECORD;
+		let rec;
+		try {
+			rec = JSON.parse(raw);
+		} catch {
+			return UNREADABLE_RECORD;
+		}
+		return rec !== null && typeof rec === "object" && !Array.isArray(rec) ? rec : UNREADABLE_RECORD;
+	} catch {
+		return null; // unreachable or timed out: nothing was read
+	}
+}
+
 /**
  * One list from two sources.
  *