@edgehero/pi-dispatch 1.7.0 → 1.9.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@edgehero/pi-dispatch",
-  "version": "1.7.0",
+  "version": "1.9.0",
   "type": "module",
   "description": "Self-hosted job harness for the pi coding agent: a BullMQ worker that drains the queue, mints scoped forge tokens, and runs one container per job — plus the pi-dispatch CLI (init, up, doctor, service).",
   "keywords": [
@@ -48,6 +48,7 @@
     "./open-browser": "./src/open-browser.mjs",
     "./git-dirty": "./src/git-dirty.mjs",
     "./queue": "./src/queue.mjs",
+    "./capabilities": "./src/capabilities.mjs",
     "./connection": "./src/connection.mjs",
     "./job-id": "./src/job-id.mjs",
     "./forges": "./src/forges.mjs",
@@ -66,6 +67,7 @@
     "./get-token": "./src/get-token.mjs",
     "./runtime-settings": "./src/runtime-settings.mjs",
     "./run-history": "./src/run-history.mjs",
+    "./run-mirror": "./src/run-mirror.mjs",
     "./sandbox": "./src/sandbox.mjs",
     "./sandbox-store": "./src/sandbox-store.mjs",
     "./subscriptions": "./src/subscriptions.mjs",

package/src/capabilities.mjs

@@ -0,0 +1,179 @@
+/**
+ * What a host can serve, and which queue a job that needs one of those things belongs on (issue #57,
+ * `OQ-032`).
+ *
+ * Two trigger fields bind a job to a MACHINE rather than to a repository: `run.secretsProfile` names a
+ * resolver the operator declared in that host's `PI_SECRET_PROFILES`, and a `run.waitFor` condition names a
+ * check script from its `PI_WAIT_PROFILES`. Both are refused pre-spend when the host popping the job has
+ * not declared them, and both refusals are RETURNED rather than thrown, so they are never retried. The
+ * same trigger therefore succeeds or fails depending on which worker happened to take the delivery, and it
+ * reads like a configuration error rather than a placement one.
+ *
+ * #57's Gap 2 exempted forge jobs on the grounds that their workspace is a fresh clone that any host can
+ * build. Issues #225 and #230 retracted that without saying so: a clone is portable, a resolver on one
+ * machine's disk is not.
+ *
+ * WHY THIS IS DECIDED AT ENQUEUE. The obvious alternative is to let any host take the job and defer it if
+ * it cannot serve it. That does not work here, and the reason is upstream rather than ours: BullMQ promotes
+ * a delayed job on EACH WORKER'S OWN CLOCK (`scripts.js` passes the client clock as the cut-off), so the
+ * host whose clock runs fastest wins every attempt, deterministically. If the host that cannot serve the
+ * job is the fast one, the job never reaches the one that can. Jitter changes when the attempt happens, not
+ * who wins it.
+ *
+ * THE ONE CAPABILITY DELIBERATELY NOT ROUTED is `run.resume`. A session key is `sha256(kind, repo, ref)`
+ * and `session-store.mjs` records that it is "not random... anyone who knows the repository and the branch
+ * can compute it", so publishing keys to route on them would disclose which repositories and branches a
+ * deployment works on, recoverable by guessing a repo name. That is more disclosing than everything else in
+ * the registry combined. A resume that lands on the wrong host cold-starts and says so in the record.
+ */
+
+/** Class letters. Open enum: `g:` is reserved for forge credentials, the highest-value follow-on. */
+export const CAP_SECRET = "s";
+export const CAP_WAIT = "w";
+
+/**
+ * The name charset, duplicated from `secret-profiles.mjs` and `wait-for.mjs` deliberately: this module
+ * imports nothing, and the two it copies already copy it from `triggers.mjs` for the same reason. What
+ * matters here is what the set EXCLUDES -- a comma, so the token list joins unambiguously, and a colon, so
+ * `<class>:<name>` decomposes at the first one.
+ */
+const PROFILE_NAME = /^[A-Za-z0-9._-]+$/;
+
+/**
+ * How fresh a host's registry row must be before a job is ROUTED to it.
+ *
+ * Not the 90s TTL, and the difference is the point. The TTL is a crash backstop: it answers "has this host
+ * definitely gone", and it is deliberately six missed beats so a blip cannot evict a working host from the
+ * panel. A routing decision needs the opposite polarity -- evidence of LIFE, not absence of expiry --
+ * because a job routed onto a dead host's queue sits there until that host comes back, and nothing else
+ * will take it. Three beats is late enough to ride out a slow beat and early enough that a stopped host
+ * stops attracting work long before its row expires.
+ */
+export const ROUTE_FRESH_MS = 45_000;
+
+/** One token, or null when the name is not one this deployment would accept. */
+function token(cls, name) {
+	return typeof name === "string" && PROFILE_NAME.test(name) ? `${cls}:${name}` : null;
+}
+
+/**
+ * What THIS host can serve, from its own config, as a sorted token list.
+ *
+ * Sorted so the published string is stable: an unstable one would make the row differ every beat and any
+ * future fingerprint over it useless.
+ */
+export function capabilityTokens({ secretProfiles, waitProfiles } = {}) {
+	const out = new Set();
+	for (const name of Object.keys(secretProfiles ?? {})) {
+		const t = token(CAP_SECRET, name);
+		if (t) out.add(t);
+	}
+	for (const name of Object.keys(waitProfiles ?? {})) {
+		const t = token(CAP_WAIT, name);
+		if (t) out.add(t);
+	}
+	return [...out].sort();
+}
+
+/** The registry value. Comma-joined, which the charset makes unambiguous. */
+export function serializeCaps(tokens) {
+	return (tokens ?? []).join(",");
+}
+
+/**
+ * A peer's tokens, from its registry row. Peer-written, so every element is re-validated here rather than
+ * trusted: a row is written by another process and this one decides where money-spending work goes.
+ */
+export function parseCaps(raw) {
+	const out = new Set();
+	for (const part of String(raw ?? "").split(",")) {
+		const at = part.indexOf(":");
+		if (at <= 0) continue;
+		const cls = part.slice(0, at);
+		const name = part.slice(at + 1);
+		if ((cls === CAP_SECRET || cls === CAP_WAIT) && PROFILE_NAME.test(name)) out.add(part);
+	}
+	return out;
+}
+
+/**
+ * What a JOB needs, from the same two fields the worker's own refusals read.
+ *
+ * `waitProfileNames`-shaped inline rather than imported, because this module is loaded by the RECEIVER and
+ * `wait-for.mjs` carries the whole wait engine. The extraction is three lines and the charset check below
+ * is the same one that module makes.
+ */
+export function jobNeeds(job) {
+	const needs = new Set();
+	const secret = token(CAP_SECRET, job?.secretsProfile);
+	if (secret) needs.add(secret);
+	if (Array.isArray(job?.waitFor)) {
+		for (const condition of job.waitFor) {
+			const wait = token(CAP_WAIT, condition?.profile);
+			if (wait) needs.add(wait);
+		}
+	}
+	return [...needs].sort();
+}
+
+/**
+ * Which queue this job belongs on: a host queue name, or `null` for the shared queue.
+ *
+ * FOUR ABSTENTIONS, and each one lands on today's behaviour rather than on something new. That is what
+ * makes this safe to put in front of every forge delivery: the rule can only ever move a job that would
+ * otherwise have had a coin flip decide whether it ran.
+ *
+ *   1. The job needs nothing host-specific. The overwhelming majority of deliveries.
+ *   2. EVERY live host can serve it. The shared queue is then strictly better than picking one, because it
+ *      load-balances, and it is what the docs tell operators to aim for by declaring the same profiles
+ *      everywhere. A deployment that follows that advice is byte-identical to before.
+ *   3. NO host can serve it. Routing cannot help, and the shared queue produces the existing pre-spend
+ *      refusal (`secret-profile-unknown` / `wait-profile-unknown`), which is the honest answer and already
+ *      names what to fix. Inventing a new terminal state here would be worse than the one that exists.
+ *   4. No CAPABLE host has a queue of its own, or the registry could not be read. Nothing to route to.
+ *
+ * Otherwise the job goes to a capable host, chosen by hashing its jobId: deterministic, so a redelivery of
+ * the same job lands the same way and dedup still works, and spread, so a delivery fanned out into replicas
+ * does not pile every replica onto one machine.
+ */
+export function routeForgeJob({ hosts, needs, jobId, now = () => Date.now(), freshMs = ROUTE_FRESH_MS } = {}) {
+	if (!Array.isArray(needs) || needs.length === 0) return null; // (1)
+	if (!Array.isArray(hosts) || hosts.length === 0) return null; // (4) unreadable registry, or nobody home
+
+	const live = hosts.filter((h) => {
+		// `staleMs` is derived by the reader; a row without one is a row we cannot date, and an undatable
+		// row is not evidence of life.
+		const stale = Number(h?.staleMs);
+		return Number.isFinite(stale) && stale <= freshMs;
+	});
+	if (live.length === 0) return null;
+
+	const serves = (h) => {
+		const caps = parseCaps(h?.caps);
+		return needs.every((n) => caps.has(n));
+	};
+	const capable = live.filter(serves);
+	if (capable.length === 0) return null; // (3)
+	if (capable.length === live.length) return null; // (2)
+
+	// Only a host that DECLARED a name drains a queue of its own; an undeclared one reads the shared queue
+	// only, so routing to it would be routing into a queue nothing drains.
+	const routable = capable.filter((h) => h?.routes === true || h?.routes === "true").map((h) => h?.name).filter((n) => typeof n === "string" && n !== "");
+	if (routable.length === 0) return null; // (4)
+
+	routable.sort();
+	return routable[hashIndex(String(jobId ?? ""), routable.length)];
+}
+
+/**
+ * A stable index from a string. FNV-1a, inline: this module imports nothing, and a cryptographic hash would
+ * be a strange dependency for choosing between two machines.
+ */
+function hashIndex(text, modulo) {
+	let h = 0x811c9dc5;
+	for (let i = 0; i < text.length; i++) {
+		h ^= text.charCodeAt(i);
+		h = Math.imul(h, 0x01000193) >>> 0;
+	}
+	return h % modulo;
+}

package/src/cli.mjs

@@ -43,7 +43,11 @@ Config comes from the environment (see .env.example); flags override it per run.
 Prefer being walked through all of this? The operator panel's /dispatch setup does every step
 with a consent per action:  pi install npm:@edgehero/pi-dispatch-admin`;
 
-export async function main(argv = process.argv.slice(2), env = process.env) {
+	// Where this command's output goes. Defaults to the real stdout, so the CLI is byte-identical; a test
+	// injects a collector instead of reassigning `process.stdout.write`. That matters because `node --test`
+	// runs each file in a child process that serialises its own results over that same stdout, so a test
+	// holding a replacement across an `await` swallows the runner's result frames (issue #266).
+export async function main(argv = process.argv.slice(2), env = process.env, { write = (chunk) => process.stdout.write(chunk) } = {}) {
 	const cmd = argv[0];
 
 	if (cmd === "init") {
@@ -69,7 +73,7 @@ export async function main(argv = process.argv.slice(2), env = process.env) {
 			const { runGithubAppSetup } = await import("./github-app-setup.mjs");
 			return runGithubAppSetup(argv.slice(2), { env });
 		}
-		process.stdout.write(`pi-dispatch setup <target> — guided credential setup\n\n  targets: github\n\n  pi-dispatch setup github (--webhook-url <URL> | --no-webhook) [--org <org>] [--name <appName>]\n      mint GitHub App credentials via the App Manifest flow — one browser click returns the app id,\n      private key, and webhook secret; every write is shown first and individually consented\n`);
+		write(`pi-dispatch setup <target> — guided credential setup\n\n  targets: github\n\n  pi-dispatch setup github (--webhook-url <URL> | --no-webhook) [--org <org>] [--name <appName>]\n      mint GitHub App credentials via the App Manifest flow — one browser click returns the app id,\n      private key, and webhook secret; every write is shown first and individually consented\n`);
 		return argv[1] ? 1 : 0;
 	}
 
@@ -144,7 +148,7 @@ export async function main(argv = process.argv.slice(2), env = process.env) {
 				// throws inside buildDockerRunArgs after a budget slot is reserved.
 				image: values.image || undefined,
 			});
-			process.stdout.write(`queued ${jobId} — folder ${folder}\nrun \`pi-dispatch worker\` to process it.\n`);
+			write(`queued ${jobId} — folder ${folder}\nrun \`pi-dispatch worker\` to process it.\n`);
 		} catch (error) {
 			return fail(`could not reach Valkey at ${config.valkeyUrl} — is it running? (docker compose up)\n  ${error.message}`);
 		} finally {
@@ -208,7 +212,7 @@ export async function main(argv = process.argv.slice(2), env = process.env) {
 					// as "nothing happened" and walks away from a fleet with one host still spending.
 					return fail(`could not ${cmd} the whole deployment at ${url}\n  ${done.length > 0 ? `${cmd}d: ${done.join(", ")}` : "nothing changed"}\n  failed at: ${names[done.length]}\n  ${error.message}`);
 				}
-				process.stdout.write(cmd === "pause" ? `paused — worker will stop taking new jobs (jobs still enqueue)${span}\n` : `resumed${span}\n`);
+				write(cmd === "pause" ? `paused — worker will stop taking new jobs (jobs still enqueue)${span}\n` : `resumed${span}\n`);
 			} else {
 				// "paused" is included in the counts because jobs enqueued while paused land in the
 				// `paused` list, not `wait` -- omitting it would report backlog 0 in the exact state
@@ -226,7 +230,7 @@ export async function main(argv = process.argv.slice(2), env = process.env) {
 				const pausedState = states.every(Boolean);
 				const pausedPartial = !pausedState && states.some(Boolean);
 				const out = { pausedState, ...(pausedPartial ? { pausedPartial, pausedQueues: names.filter((_, i) => states[i]) } : {}), ...counts, ...(blind ? { fleet: blind } : {}) };
-				process.stdout.write(`${JSON.stringify(out)}\n`);
+				write(`${JSON.stringify(out)}\n`);
 			}
 		} catch (error) {
 			return fail(`could not reach Valkey at ${url} — is it running? (docker compose up)\n  ${error.message}`);
@@ -236,7 +240,7 @@ export async function main(argv = process.argv.slice(2), env = process.env) {
 		return 0;
 	}
 
-	process.stdout.write(`${USAGE}\n`);
+	write(`${USAGE}\n`);
 	return cmd ? 1 : 0;
 }
 

package/src/docker-run.mjs

@@ -50,7 +50,20 @@ export const ISOLATION_FLAGS = [
 ];
 
 /**
- * Build the full `docker run` argv (excluding the leading "docker").
+ * WHAT the box is, with no Docker vocabulary in it.
+ *
+ * Split from the argv builder below so the description of a container exists as a VALUE before it becomes
+ * one runtime's flags. `buildDockerRunArgs` is unchanged in name, signature and output -- it is now
+ * `dockerArgsFromSpec(containerSpec(opts))` -- so every caller and every assertion is untouched, and the
+ * only thing that is new is that the middle of that sentence can be read on its own.
+ *
+ * Mounts are structured (`{host, container, readOnly}`) rather than pre-flattened `host:container:ro`
+ * strings, because the flattening IS the Docker part: a runtime that does not bind-mount has to be able to
+ * see which host path becomes which container path, and what may be written.
+ *
+ * `dockerExtra` is named for what it is. It carries raw Docker flags (`-i -t --entrypoint bash`, a
+ * Linux-only `--user`), so it is the one field a non-Docker consumer must refuse rather than translate.
+ * Calling it `extraFlags` at the boundary would have hidden that.
  *
  * @param image      pinned job image tag/digest
  * @param env        the closed env map from buildContainerEnv -- passed as explicit -e NAME=VALUE
@@ -66,7 +79,7 @@ export const ISOLATION_FLAGS = [
  *                   docker default bridge, which is what every job did before that requirement existed
  * @param extraFlags escape hatch for a Linux-only --user uid:gid on a bind-mounted local folder
  */
-export function buildDockerRunArgs({
+export function containerSpec({
 	image,
 	env,
 	jobDir,
@@ -84,6 +97,56 @@ export function buildDockerRunArgs({
 	if (!name) throw new Error("docker run: container name is required");
 	if (!workspace) throw new Error("docker run: workspace mount is required");
 
+	const mounts = [];
+	// The WHOLE /job dir is read-only (INT-CONTAINER-JOB-INPUTS): it holds prompt.md and pi/, and
+	// the agent cannot rewrite any of it. /workspace is the only writable mount.
+	if (jobDir) mounts.push({ host: jobDir, container: "/job", readOnly: true });
+	mounts.push({ host: workspace, container: "/workspace", readOnly: false });
+	// Local jobs get a writable /outbox host bind, the same host-bind mechanism as /workspace
+	// (DES-WORKER-ON-HOST). github jobs pass no outboxDir, so the request channel does not exist for
+	// them -- an untrusted issue author cannot chain (INT-OUTBOX-CONTRACT).
+	if (outboxDir) mounts.push({ host: outboxDir, container: "/outbox", readOnly: false });
+
+	// This job's OWN copy of its session transcript (REQ-RESUMABLE-SESSION, INT-SESSION-STORE-CONTRACT).
+	// Writable, because pi appends to it as the agent works -- and per-job, exactly like jobDir, which is
+	// the whole reason CONST-ISOLATION-CONTAINER-PER-JOB's "none host-wide" clause still reads true. The
+	// shared store under PI_SESSIONS_DIR is NEVER bind-mounted: one job here would otherwise be able to
+	// read and rewrite every other branch's and every other repository's transcripts, which is not a
+	// weakening of that constraint but its inversion. Absent unless the trigger armed run.resume AND a key
+	// resolved, so an unarmed job's argv is byte-identical to one built before this feature existed.
+	if (sessionDir) mounts.push({ host: sessionDir, container: CONTAINER_SESSION_DIR, readOnly: false });
+
+	// The operator's global pi overlay (REQ-GLOBAL-PI-OVERLAY): custom models, global skills, a global
+	// persona, layered UNDER each repo's own .pi/. Read-only -- it is operator-authored deploy-time config,
+	// the same trust class as the baked floor, but the agent still must not rewrite it. Both job kinds.
+	if (globalPiDir) mounts.push({ host: globalPiDir, container: CONTAINER_GLOBAL_PI_DIR, readOnly: true });
+
+	return {
+		image,
+		name,
+		memory,
+		cpus,
+		network,
+		// UNCONDITIONALLY true, and there is deliberately no parameter that can unset it. The boundary is
+		// not a thing a caller opts into -- CONST-ISOLATION-CONTAINER-PER-JOB is why every other flag here
+		// exists -- so the spec is simply unable to describe an unisolated container, and the builder below
+		// refuses one it is handed. A field that could be false would be a way to ask for less.
+		isolated: true,
+		mounts,
+		env,
+		dockerExtra: extraFlags,
+	};
+}
+
+/**
+ * HOW Docker spells it. The only consumer of a spec today.
+ */
+export function dockerArgsFromSpec(spec) {
+	// The builder CANNOT DECLINE the boundary. `containerSpec` cannot produce anything but `true`, so this
+	// only ever fires on a hand-built spec -- and a hand-built spec that forgot the field is exactly the
+	// case that must fail loudly rather than quietly emit a container with no isolation flags at all.
+	if (spec?.isolated !== true) throw new Error("docker run: refusing to build an argv for a spec that is not isolated");
+
 	// `--network` sits HERE, beside --memory and --cpus, and deliberately NOT inside ISOLATION_FLAGS.
 	// That array is the LITERAL, value-free, unconditional set, and two separate places assert every member
 	// of it reaches the sandbox argv *against the imported array, not a copy* (CONST-ISOLATION-CONTAINER-PER-JOB
@@ -94,41 +157,34 @@ export function buildDockerRunArgs({
 	//
 	// null => the flag is ABSENT, so a job argv without an egress policy is byte-identical to one built
 	// before this feature existed. Same shape as the sessionDir/outboxDir/globalPiDir mounts below.
-	const args = ["run", `--name=${name}`, ...ISOLATION_FLAGS, `--memory=${memory}`, `--cpus=${cpus}`];
-	if (network) args.push(`--network=${network}`);
-	args.push(...extraFlags);
+	const args = ["run", `--name=${spec.name}`, ...ISOLATION_FLAGS, `--memory=${spec.memory}`, `--cpus=${spec.cpus}`];
+	if (spec.network) args.push(`--network=${spec.network}`);
+	args.push(...(spec.dockerExtra ?? []));
 
 	// Explicit env allowlist. Each entry is `-e NAME=VALUE`, built from the closed map -- so a
 	// stray host variable cannot ride along (no bare `-e NAME` inheriting from the host, no
 	// --env-file). Undefined values are skipped, never passed as an empty string.
-	for (const [k, v] of Object.entries(env ?? {})) {
+	for (const [k, v] of Object.entries(spec.env ?? {})) {
 		if (v === undefined || v === null) continue;
 		args.push("-e", `${k}=${v}`);
 	}
 
-	// The WHOLE /job dir is read-only (INT-CONTAINER-JOB-INPUTS): it holds prompt.md and pi/, and
-	// the agent cannot rewrite any of it. /workspace is the only writable mount.
-	if (jobDir) args.push("-v", `${jobDir}:/job:ro`);
-	args.push("-v", `${workspace}:/workspace`);
-	// Local jobs get a writable /outbox host bind, the same host-bind mechanism as /workspace
-	// (DES-WORKER-ON-HOST). github jobs pass no outboxDir, so the request channel does not exist for
-	// them -- an untrusted issue author cannot chain (INT-OUTBOX-CONTRACT).
-	if (outboxDir) args.push("-v", `${outboxDir}:/outbox`);
+	// `-v` and its value stay TWO argv elements rather than one `--volume=` token. Not cosmetic: the mount
+	// assertions across this suite extract mounts by adjacency (`args[i - 1] === "-v"`), so collapsing the
+	// pair would make those filters return nothing and turn several exact-array checks vacuously green.
+	for (const m of spec.mounts ?? []) args.push("-v", `${m.host}:${m.container}${m.readOnly ? ":ro" : ""}`);
 
-	// This job's OWN copy of its session transcript (REQ-RESUMABLE-SESSION, INT-SESSION-STORE-CONTRACT).
-	// Writable, because pi appends to it as the agent works -- and per-job, exactly like jobDir, which is
-	// the whole reason CONST-ISOLATION-CONTAINER-PER-JOB's "none host-wide" clause still reads true. The
-	// shared store under PI_SESSIONS_DIR is NEVER bind-mounted: one job here would otherwise be able to
-	// read and rewrite every other branch's and every other repository's transcripts, which is not a
-	// weakening of that constraint but its inversion. Absent unless the trigger armed run.resume AND a key
-	// resolved, so an unarmed job's argv is byte-identical to one built before this feature existed.
-	if (sessionDir) args.push("-v", `${sessionDir}:${CONTAINER_SESSION_DIR}`);
-
-	// The operator's global pi overlay (REQ-GLOBAL-PI-OVERLAY): custom models, global skills, a global
-	// persona, layered UNDER each repo's own .pi/. Read-only -- it is operator-authored deploy-time config,
-	// the same trust class as the baked floor, but the agent still must not rewrite it. Both job kinds.
-	if (globalPiDir) args.push("-v", `${globalPiDir}:${CONTAINER_GLOBAL_PI_DIR}:ro`);
-
-	args.push(image);
+	args.push(spec.image);
 	return args;
 }
+
+/**
+ * Build the full `docker run` argv (excluding the leading "docker").
+ *
+ * The public entry point, unchanged: same name, same parameters, same argv byte for byte. Kept as the
+ * name rather than replaced by `dockerArgsFromSpec` because `CONST-EGRESS-POLICY-IN-THE-ARGV` cites this
+ * symbol in its Code evidence, and because a rename would churn every call site and assertion for nothing.
+ */
+export function buildDockerRunArgs(opts) {
+	return dockerArgsFromSpec(containerSpec(opts));
+}

package/src/run-mirror.mjs

@@ -0,0 +1,221 @@
+/**
+ * A fleet-visible copy of the run history (issue #57, Gap 3).
+ *
+ * Every worker writes its records to its own `PI_LOGS_DIR`, so on more than one machine each host's panel
+ * lists only the runs on its own disk. The operator sees a third of their deployment and has no way to
+ * know it.
+ *
+ * SHARED STORAGE IS THE OTHER ANSWER AND IT IS NOT SECOND-BEST. On a shared `PI_LOGS_DIR` the local read
+ * IS the merged read, with no machinery at all, and this module is redundant. It ships because that trade
+ * runs both ways and an operator must be allowed to decline it: sharing the directory also shares the
+ * PII-bearing raw `.log`, and a mount outage becomes a LOST RECORD where a Valkey outage costs only a
+ * fleet view. Both shapes work; `docs/multi-host.md` says which is which.
+ *
+ * THE FILE IS THE RECORD AND THIS IS A VIEW. Three things follow, and each is load-bearing:
+ *
+ *   - The file is written FIRST, always. A crash between the two leaves a fleet-visible run whose durable
+ *     source does not exist, which inverts the one claim this design rests on.
+ *   - The mirror's TTL is never longer than the file retention window, so it can never show a run whose
+ *     file has already been reaped. A view that outlives its source is a second source of truth, which is
+ *     exactly what `DES-RUN-HISTORY-FLAT-FILES-NO-DB` refuses.
+ *   - Nothing derived is stored. The bytes are the sidecar's own bytes, so there is nothing to be stale
+ *     RELATIVE TO: a retry overwrites the same key exactly as it overwrites the same file, and cost
+ *     classification is still computed at fold time from `subscriptions.json` rather than frozen here.
+ *
+ * WHY THE WHOLE RECORD RATHER THAN A PROJECTION. The record is PII-free BY CONSTRUCTION -- it holds no
+ * attacker-chosen string, which `INT-RUN-HISTORY-FILE-CONTRACT` states and `buildRecord` enforces field by
+ * field. Copying it whole inherits that property; a projection would re-derive it at a second serialiser,
+ * where the next person to add a field has to remember this file exists. It is also what the readers need:
+ * the cost fold, the graph and the insights view read eleven fields between them.
+ *
+ * NOT MIRRORED: the raw `.log`. It is the one artifact here that holds issue text, comment text and tool
+ * output, and mirroring it would move that off the machine the operator chose to keep it on. A foreign
+ * run's record names its host, so the panel can say where the bytes are rather than pretending there are
+ * none.
+ */
+
+/** The index: sanitized jobId -> the run's end (or start) in millis. */
+export const RUNS_INDEX = "runs:index";
+
+/** One run's own bytes. */
+export const runRecordKey = (sanitizedJobId) => `runs:rec:${sanitizedJobId}`;
+
+/**
+ * The deepest any reader asks. `SCAN_WINDOW_MAX_DAYS` in the admin is 92, so a longer window would hold
+ * bytes nothing can request.
+ */
+export const MIRROR_MAX_DAYS = 92;
+
+/**
+ * A hard ceiling on index members, independent of the time window.
+ *
+ * The window alone does not bound memory: a deployment running thousands of jobs a day would hold a
+ * quarter of a million members for ninety-two days. This caps what the fleet view can cost at roughly the
+ * depth a panel can display, and the file on disk remains the complete history either way.
+ */
+export const RUNS_INDEX_MAX = 5_000;
+
+const DAY_MS = 24 * 60 * 60 * 1000;
+const OP_TIMEOUT_MS = 2_000;
+
+/**
+ * How long a mirrored record lives.
+ *
+ * Never longer than the operator's own retention, and never longer than what any reader asks for.
+ * `retentionDays: 0` means keep the files forever, which is the one case where the mirror is the shorter
+ * of the two, so it clamps to the reader's ceiling rather than to infinity.
+ */
+export function mirrorWindowMs(retentionDays) {
+	const days = Number(retentionDays) > 0 ? Math.min(Number(retentionDays), MIRROR_MAX_DAYS) : MIRROR_MAX_DAYS;
+	return days * DAY_MS;
+}
+
+/**
+ * BullMQ's connections carry `maxRetriesPerRequest: null`, so a command against an unreachable server
+ * QUEUES FOREVER rather than rejecting. Every await here is bounded for that reason; an unbounded one
+ * would not fail the mirror, it would hang the job that was writing to it.
+ */
+function bounded(promise, ms) {
+	let timer;
+	return Promise.race([
+		promise,
+		new Promise((_, reject) => {
+			timer = setTimeout(() => reject(new Error("mirror timeout")), ms);
+		}),
+	]).finally(() => clearTimeout(timer));
+}
+
+/**
+ * The writer. Returns `{ mirror, close }`; `mirror` never throws and never rejects.
+ *
+ * A history blip must not fail a job that has already run and already been recorded to disk. Every failure
+ * here costs a row in a fleet view and nothing else, which is why the whole body is wrapped and the result
+ * is a boolean nobody is obliged to read.
+ */
+export function makeRunMirror({ redis, retentionDays, now = () => Date.now(), log = () => {}, timeoutMs = OP_TIMEOUT_MS, indexMax = RUNS_INDEX_MAX } = {}) {
+	const windowMs = mirrorWindowMs(retentionDays);
+	let warned = false;
+
+	return {
+		async mirror(record, sanitizedJobId) {
+			if (!redis || !record || !sanitizedJobId) return false;
+			try {
+				const at = Date.parse(record.endedAt ?? record.startedAt ?? "");
+				const score = Number.isFinite(at) ? at : now();
+				const body = JSON.stringify(record);
+				await bounded(redis.set(runRecordKey(sanitizedJobId), body, "PX", windowMs), timeoutMs);
+				await bounded(redis.zadd(RUNS_INDEX, score, sanitizedJobId), timeoutMs);
+				// Trimmed by the WRITER, twice: by age, and by count. Two `ZREMRANGE`s against a run that
+				// took minutes is free, and it means no reader has to pay for a backlog it did not create.
+				await bounded(redis.zremrangebyscore(RUNS_INDEX, "-inf", `(${now() - windowMs}`), timeoutMs);
+				await bounded(redis.zremrangebyrank(RUNS_INDEX, 0, -indexMax - 1), timeoutMs);
+				// ROLLING expiry, deliberately unlike `budget.mjs`'s set-once rule and deliberately like
+				// `pi-dispatch:sched-stalls:<schedulerId>`. A budget window must not be pushed forward by traffic or a busy
+				// day never resets; an ACTIVITY index should roll with traffic, because that is what it
+				// describes. A fleet that stops running jobs loses its index one window later, which is
+				// correct: there is nothing left to show.
+				await bounded(redis.pexpire(RUNS_INDEX, windowMs), timeoutMs);
+				warned = false;
+				return true;
+			} catch (err) {
+				// Once per transition, not once per job: a Valkey outage during a busy hour must not turn one
+				// fault into a thousand log lines (`notePackageKey`'s precedent).
+				if (!warned) {
+					warned = true;
+					log("run_mirror_failed", { jobId: sanitizedJobId, reason: err?.message });
+				}
+				return false;
+			}
+		},
+	};
+}
+
+/**
+ * The reader. Returns `{ runs, degraded }` and never throws.
+ *
+ * `degraded` is a DISCRIMINATED channel rather than a silence, and two of its values must not collapse:
+ * `"off"` means the index is absent, which is what a single-host deployment and a fleet of workers still
+ * below the version floor both look like, while `"unreachable"` means we could not tell. A new panel
+ * meeting old workers has to read "off", not "error".
+ *
+ * Two round trips regardless of how many runs come back: one `ZREVRANGEBYSCORE` for the ids, one `MGET`
+ * for the bodies. Never one read per run.
+ */
+export async function readMirroredRuns(redis, { limit = 50, sinceMs = 0, now = () => Date.now(), timeoutMs = OP_TIMEOUT_MS } = {}) {
+	if (!redis) return { runs: [], degraded: "off" };
+	let ids;
+	try {
+		ids = await bounded(redis.zrevrangebyscore(RUNS_INDEX, "+inf", `(${sinceMs}`, "LIMIT", 0, Math.max(1, limit)), timeoutMs);
+	} catch (err) {
+		return { runs: [], degraded: `unreachable (${err?.message ?? "?"})` };
+	}
+	if (!Array.isArray(ids) || ids.length === 0) return { runs: [], degraded: "off" };
+
+	let bodies;
+	try {
+		bodies = await bounded(redis.mget(...ids.map(runRecordKey)), timeoutMs);
+	} catch (err) {
+		return { runs: [], degraded: `unreachable (${err?.message ?? "?"})` };
+	}
+
+	const runs = [];
+	const stale = [];
+	for (let i = 0; i < ids.length; i++) {
+		const raw = bodies?.[i];
+		if (typeof raw !== "string" || raw === "") {
+			// An id whose body has expired: the per-key TTL fired and the index member outlived it. The
+			// READER prunes it, which is what `wait:held` does for the same shape and for the same reason --
+			// a writer that crashed cannot clean up after itself, and a reader is already here.
+			stale.push(ids[i]);
+			continue;
+		}
+		try {
+			const rec = JSON.parse(raw);
+			if (rec && typeof rec === "object") runs.push(rec);
+		} catch {
+			stale.push(ids[i]); // unparseable is indistinguishable from gone, and equally not showable
+		}
+	}
+	if (stale.length > 0) {
+		try {
+			await bounded(redis.zrem(RUNS_INDEX, ...stale), timeoutMs);
+		} catch {
+			// best-effort: a straggler in the index costs one skipped row next time, never a wrong one
+		}
+	}
+	return { runs, degraded: runs.length >= limit ? "truncated" : "ok" };
+}
+
+/**
+ * One list from two sources.
+ *
+ * DEDUP BY LATER `endedAt`, LOCAL BREAKS A TIE. Not decoration: a retry can land on a different host, so
+ * host A may hold attempt 0 (failed) while host B mirrored attempt 1 (completed). "Local wins" alone would
+ * show the stale one. Local breaking an exact tie keeps a single-host deployment reading its own files.
+ *
+ * CUT AFTER THE SORT, never before. Slicing first is the defect `held.test.mjs` already exists to prevent:
+ * it makes the result depend on which source happened to be longer.
+ */
+export function mergeRuns(local, mirrored, { limit = 50 } = {}) {
+	const by = new Map();
+	const at = (r) => {
+		const t = Date.parse(r?.endedAt ?? r?.startedAt ?? "");
+		return Number.isFinite(t) ? t : -Infinity;
+	};
+	// Mirrored first, so a local record with an equal timestamp overwrites it on the second pass.
+	for (const r of Array.isArray(mirrored) ? mirrored : []) if (r?.jobId) by.set(r.jobId, r);
+	for (const r of Array.isArray(local) ? local : []) {
+		if (!r?.jobId) continue;
+		const seen = by.get(r.jobId);
+		if (!seen || at(r) >= at(seen)) by.set(r.jobId, r);
+	}
+	const out = [...by.values()].sort((a, b) => at(b) - at(a));
+	return out.slice(0, Math.max(0, limit));
+}
+
+/** The distinct hosts a merged list came from, computed from the RECORDS rather than from the mirror. */
+export function hostsIn(runs) {
+	const names = new Set();
+	for (const r of runs ?? []) if (typeof r?.host === "string" && r.host !== "") names.add(r.host);
+	return [...names].sort();
+}

package/src/scheduler-stall-guard.mjs

@@ -1,5 +1,5 @@
 /**
- * Per-scheduler stall accounting -- the money backstop for cron (constitution.md:203-216).
+ * Per-scheduler stall accounting -- the money backstop for cron (CONST-RETRY-INFRA-ONLY).
  *
  * BullMQ's `maxStalledCount` does not cover scheduler jobs: `moveStalledJobsToWait` derives
  * `isRepeatableJob` from the job's `rjk` field and skips the stall-fail for a live scheduler, so a
@@ -10,16 +10,41 @@
  * Injected `redis` (ioredis-compatible), `removeJobScheduler`, and `log` keep the logic testable with
  * no queue, no bullmq import, and no real Valkey.
  *
- * Custom: per-scheduler stall accounting; BullMQ's maxStalledCount does not cover scheduler jobs -- constitution.md:203-216 carve-out ("BullMQ will never do this for us")
+ * Custom: per-scheduler stall accounting; BullMQ's maxStalledCount does not cover scheduler jobs -- CONST-RETRY-INFRA-ONLY carve-out ("BullMQ will never do this for us")
  */
 
-// The Redis hash of per-scheduler stall counts (field = schedulerId, value = count). Exported so the admin
-// panel can read it (HGETALL) for the cron drill-in without re-deriving the key string.
+// The prefix every stall counter lives under, so `KEYS pi-dispatch:sched-stalls*` still shows an operator
+// the whole feature -- the affordance `wait:`, `slot:` and `budget:` all assume. Exported alongside the
+// builder so the admin panel and the integration teardown compose keys through one definition and cannot
+// drift from the writer.
 export const STALL_KEY = "pi-dispatch:sched-stalls";
 
-// A rolling window: the EXPIRE is re-set on every stall, so a scheduler that stops stalling for a full
-// day drops back to zero. This prevents unrelated transient stalls weeks apart from accumulating into a
-// false teardown -- only sustained stalling inside one window trips the threshold.
+/**
+ * One scheduler's counter. The id is VALIDATED upstream rather than hashed here: it is operator-declared in
+ * `triggers.json`, `triggers.mjs` already refuses a `:` in it precisely to protect this parse, and the value
+ * of a readable keyspace is that `GET pi-dispatch:sched-stalls:nightly` answers the question directly.
+ */
+export const stallKey = (schedulerId) => `${STALL_KEY}:${schedulerId}`;
+
+// ONE KEY PER SCHEDULER, so the window is per scheduler.
+//
+// This was one HASH with a field per scheduler and a single `EXPIRE` on the whole key, which meant any
+// scheduler's stall pushed the TTL forward for EVERY scheduler's count. The window never reset on a
+// deployment where anything stalled regularly, so the guard silently degraded from "sustained stalling
+// inside one window" to "cumulative stalling ever": three stalls ninety days apart tore a scheduler down
+// if a neighbour was stalling twice a day, and did not if the deployment was quiet. Same scheduler, same
+// stalls, opposite outcome, decided by an unrelated trigger (issue #267).
+//
+// Per-field TTLs would have fixed it in place and are not available: `HEXPIRE` does not exist on the pinned
+// `valkey/valkey:8` (verified, `ERR unknown command`, recorded under DES-HOST-REGISTRY). A key per entity is
+// the only shape that gets per-entity expiry.
+//
+// The EXPIRE still ROLLS on every stall, deliberately, and that is not `budget.mjs`'s set-once rule being
+// broken. A budget window is a CALENDAR window and must not be pushed forward by traffic or a busy day
+// never resets. This is a STREAK detector -- "is this scheduler wedged right now" -- and quiet for a day
+// genuinely should forget. `poll:<repo>:close-gate:<deliveryId>` is the in-repo precedent, a bounded
+// consecutive-failure counter given its own key and TTL for exactly this reason: it must decay with the
+// thing it measures rather than with a larger family.
 const STALL_WINDOW_SECONDS = 24 * 60 * 60;
 
 /**
@@ -45,8 +70,9 @@ export function makeStallGuard({ redis, threshold, removeJobScheduler, log }) {
 				return;
 			}
 
-			const count = Number(await redis.hincrby(STALL_KEY, schedulerId, 1));
-			await redis.expire(STALL_KEY, STALL_WINDOW_SECONDS);
+			const key = stallKey(schedulerId);
+			const count = Number(await redis.incr(key));
+			await redis.expire(key, STALL_WINDOW_SECONDS);
 
 			if (count > threshold) {
 				try {
@@ -56,7 +82,7 @@ export function makeStallGuard({ redis, threshold, removeJobScheduler, log }) {
 					// state, not an error -- swallow it so hdel and the teardown alert still run.
 					log("scheduler_teardown_remove_failed", { schedulerId, error: error?.message });
 				}
-				await redis.hdel(STALL_KEY, schedulerId);
+				await redis.del(key);
 				// The loud log is the "alert" half of the constitution's "removeJobScheduler -- or alert".
 				log("scheduler_torn_down", { schedulerId, stalls: count });
 			}

package/src/start.mjs

@@ -15,6 +15,7 @@ import { makeAzureAuth } from "./azure-auth.mjs";
 import { makeAzureHost } from "./azure-host.mjs";
 import { makeEgressPreflight } from "./egress.mjs";
 import { checkSlotKey, makeFleetLease, makeScopeClaimSweeper, scopeSlotKey } from "./fleet-lease.mjs";
+import { capabilityTokens, serializeCaps } from "./capabilities.mjs";
 import { cronFingerprint } from "./fingerprint.mjs";
 import { makeHostRegistry } from "./host-registry.mjs";
 import { makeImagePreflight } from "./image-preflight.mjs";
@@ -33,7 +34,8 @@ import { makeWaitState } from "./wait-state.mjs";
 import { hostQueueName, makeQueue } from "./queue.mjs";
 import { makeRunContainer } from "./run-container.mjs";
 import { makeSecretsResolver } from "./secrets.mjs";
-import { buildRecord, makeFindPreviousRun, makeLogReaper, makeLogSink, makeRecordWriter } from "./run-history.mjs";
+import { buildRecord, makeFindPreviousRun, makeLogReaper, makeLogSink, makeRecordWriter, sanitizeJobId } from "./run-history.mjs";
+import { makeRunMirror } from "./run-mirror.mjs";
 import { effectiveSettings, readOverlay } from "./runtime-settings.mjs";
 import { authoredCron, loadSchedules, servedSchedules } from "./schedules.mjs";
 import { makeStallGuard } from "./scheduler-stall-guard.mjs";
@@ -231,12 +233,22 @@ export function makeReaper({ log }) {
 export async function startWorker(
 	env = process.env,
 	{
+		// WHERE THE BOOT LOG BYTES GO. Defaults to the real stdout, so production is byte-identical; a test
+		// passes a collector instead of reassigning `process.stdout.write`.
+		//
+		// That distinction is not stylistic. `node --test` runs each file in a CHILD PROCESS that serialises
+		// its own results over `process.stdout`, so a test that replaces the global and holds the replacement
+		// across an `await` swallows the runner's result frames for whatever completes in that window. Three
+		// tests in `start-wiring.test.mjs` were reported as not existing at all -- no name, no count, exit 0 --
+		// because this function's log line went through the same channel the runner needed (issue #266).
+		write = (chunk) => process.stdout.write(chunk),
 		makeAuth = makeGitHubAuth,
 		makeHost = makeGitHubHost,
 		createWorkerFn = createWorker,
 		makeReaper: makeReaperFn = makeReaper,
 		makeLogSink: makeLogSinkFn = makeLogSink,
 		makeRecordWriter: makeRecordWriterFn = makeRecordWriter,
+		makeRunMirror: makeRunMirrorFn = makeRunMirror,
 		makeLogReaper: makeLogReaperFn = makeLogReaper,
 		makeSandboxReaper: makeSandboxReaperFn = makeSandboxReaper,
 		makeRunContainer: makeRunContainerFn = makeRunContainer,
@@ -260,7 +272,7 @@ export async function startWorker(
 	// other module takes `log` injected, and two tests pin the KEY SET of the fields object handed to an
 	// injected log (`run_record_failed`, `wait_check`); a `host` added at any call site would break them,
 	// while one added inside this closure cannot reach them.
-	const log = (event, fields = {}) => process.stdout.write(`${JSON.stringify({ event, ...fields, host: config.workerName })}\n`);
+	const log = (event, fields = {}) => write(`${JSON.stringify({ event, ...fields, host: config.workerName })}\n`);
 
 	// DES-CRON-VIA-BULLMQ-SCHEDULER: load and validate the triggers file with the operator present and
 	// before any Valkey contact, so a misconfigured schedule refuses startup loudly (configError) rather
@@ -446,10 +458,22 @@ export async function startWorker(
 	// that can neither disarm nor pre-spend-check.
 	const onceTriggersFile = env.PI_TRIGGERS_FILE ?? join(process.cwd(), "triggers.json");
 	const disarmOnce = makeDisarmOnce({ triggersPath: onceTriggersFile, log });
+	// The fleet-visible copy of the run history (issue #57, Gap 3). Armed only on a deployment that declared
+	// a worker name: an unnamed one is a single host, its own files ARE the whole history, and a mirror
+	// would be bytes nothing reads. That is also what keeps a single-host deployment byte-identical, since
+	// no job then issues a single extra Valkey command.
+	const runMirror = config.workerNameDeclared ? makeRunMirrorFn({ redis, retentionDays: config.logRetentionDays, log }) : null;
 	const recordRun = ({ job, result, error, startedAt, endedAt }) => {
 		// The `host` is stamped HERE rather than inside the processor, which is what keeps every one of its
 		// four `recordRun` call sites byte-unchanged and `buildRecord` a pure function of its arguments.
-		writeRecord(buildRecord({ job, result, error, startedAt, endedAt, host: config.workerName }));
+		const record = buildRecord({ job, result, error, startedAt, endedAt, host: config.workerName });
+		writeRecord(record);
+		// STRICTLY AFTER the file, and deliberately not awaited. After, because a crash between the two must
+		// leave a record with no fleet row rather than a fleet row with no record -- the mirror is a VIEW,
+		// and a view that can outlive its source is a second source of truth. Not awaited, because this is
+		// the job's own completion path: a slow Valkey may cost a row in a panel and must never hold up a
+		// job that has already finished and already been written to disk. `mirror` never rejects.
+		void runMirror?.mirror(record, sanitizeJobId(record.jobId));
 		// Strictly AFTER the durable record: "fired" means "produced a run record", and the crash
 		// direction this ordering buys is the chosen one -- an armed one-shot with a record, never a
 		// disarm before writeRecord RETURNED. Returned, not succeeded: the record writer swallows fs
@@ -585,6 +609,13 @@ export async function startWorker(
 		// Whether this host DRAINS a queue of its own. Every worker publishes a row; only a host that declared
 		// a name has somewhere for routed work to go, and a reader must not invent a queue for one that has not.
 		routes: config.workerNameDeclared,
+		// What this host can serve that another might not (issue #57, `OQ-032`): the secret and wait profiles
+		// it has declared. NAMES only, never the resolver paths behind them -- a path is PII on Windows and
+		// operator topology everywhere, and the receiver only needs to know WHICH host, not what it runs.
+		//
+		// Recomputed per beat rather than frozen at boot, for the reason the digest is: a host that gains a
+		// profile on restart must start attracting that work within one beat, and one that loses it must stop.
+		caps: () => serializeCaps(capabilityTokens(config)),
 		// The host's IANA zone, because a cron PATTERN carries none: `triggers.json` has no `tz` field and
 		// BullMQ hands the pattern to cron-parser with no zone, so it resolves in each worker's LOCAL time.
 		// On one host that is exactly what an operator means; on two in different zones the same pattern is
@@ -820,7 +851,7 @@ export async function startWorker(
 	// CONST-RETRY-INFRA-ONLY money backstop: BullMQ's maxStalledCount does not bound scheduler jobs, so a
 	// wedged scheduled run is re-paid on every stall. The guard counts stalls per scheduler and tears the
 	// scheduler down past the threshold. Keyed on "stalled", not "failed" -- only a stall is the unbounded re-run.
-	const guard = makeStallGuard({
+	const onStalled = makeStallGuard({
 		redis,
 		threshold: config.schedulerStallMax,
 		// The queue the schedulers were actually INSTALLED on. Torn down from `runtimeQueue` on a named
@@ -828,7 +859,10 @@ export async function startWorker(
 		removeJobScheduler: (id) => cronQueue.removeJobScheduler(id),
 		log,
 	});
-	for (const w of allWorkers) w.on("stalled", (jobId) => void guard.onStalled(jobId));
+	// `makeStallGuard` returns the LISTENER, not an object holding one -- hence the name of the local. It was
+	// called as `guard.onStalled(jobId)` here for the whole life of the feature, which is `undefined(jobId)`,
+	// so every stall threw a TypeError and the money backstop never counted one (issue #267).
+	for (const w of allWorkers) w.on("stalled", (jobId) => void onStalled(jobId));
 
 	// DES-CRON-VIA-BULLMQ-SCHEDULER: install the schedule set (and prune orphans) before announcing the
 	// worker is up, so schedules_installed always precedes worker_started. An empty set skips the reconcile