@edgehero/pi-dispatch 1.8.0 → 1.9.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@edgehero/pi-dispatch",
-  "version": "1.8.0",
+  "version": "1.9.0",
   "type": "module",
   "description": "Self-hosted job harness for the pi coding agent: a BullMQ worker that drains the queue, mints scoped forge tokens, and runs one container per job — plus the pi-dispatch CLI (init, up, doctor, service).",
   "keywords": [

package/src/cli.mjs

@@ -43,7 +43,11 @@ Config comes from the environment (see .env.example); flags override it per run.
 Prefer being walked through all of this? The operator panel's /dispatch setup does every step
 with a consent per action:  pi install npm:@edgehero/pi-dispatch-admin`;
 
-export async function main(argv = process.argv.slice(2), env = process.env) {
+	// Where this command's output goes. Defaults to the real stdout, so the CLI is byte-identical; a test
+	// injects a collector instead of reassigning `process.stdout.write`. That matters because `node --test`
+	// runs each file in a child process that serialises its own results over that same stdout, so a test
+	// holding a replacement across an `await` swallows the runner's result frames (issue #266).
+export async function main(argv = process.argv.slice(2), env = process.env, { write = (chunk) => process.stdout.write(chunk) } = {}) {
 	const cmd = argv[0];
 
 	if (cmd === "init") {
@@ -69,7 +73,7 @@ export async function main(argv = process.argv.slice(2), env = process.env) {
 			const { runGithubAppSetup } = await import("./github-app-setup.mjs");
 			return runGithubAppSetup(argv.slice(2), { env });
 		}
-		process.stdout.write(`pi-dispatch setup <target> — guided credential setup\n\n  targets: github\n\n  pi-dispatch setup github (--webhook-url <URL> | --no-webhook) [--org <org>] [--name <appName>]\n      mint GitHub App credentials via the App Manifest flow — one browser click returns the app id,\n      private key, and webhook secret; every write is shown first and individually consented\n`);
+		write(`pi-dispatch setup <target> — guided credential setup\n\n  targets: github\n\n  pi-dispatch setup github (--webhook-url <URL> | --no-webhook) [--org <org>] [--name <appName>]\n      mint GitHub App credentials via the App Manifest flow — one browser click returns the app id,\n      private key, and webhook secret; every write is shown first and individually consented\n`);
 		return argv[1] ? 1 : 0;
 	}
 
@@ -144,7 +148,7 @@ export async function main(argv = process.argv.slice(2), env = process.env) {
 				// throws inside buildDockerRunArgs after a budget slot is reserved.
 				image: values.image || undefined,
 			});
-			process.stdout.write(`queued ${jobId} — folder ${folder}\nrun \`pi-dispatch worker\` to process it.\n`);
+			write(`queued ${jobId} — folder ${folder}\nrun \`pi-dispatch worker\` to process it.\n`);
 		} catch (error) {
 			return fail(`could not reach Valkey at ${config.valkeyUrl} — is it running? (docker compose up)\n  ${error.message}`);
 		} finally {
@@ -208,7 +212,7 @@ export async function main(argv = process.argv.slice(2), env = process.env) {
 					// as "nothing happened" and walks away from a fleet with one host still spending.
 					return fail(`could not ${cmd} the whole deployment at ${url}\n  ${done.length > 0 ? `${cmd}d: ${done.join(", ")}` : "nothing changed"}\n  failed at: ${names[done.length]}\n  ${error.message}`);
 				}
-				process.stdout.write(cmd === "pause" ? `paused — worker will stop taking new jobs (jobs still enqueue)${span}\n` : `resumed${span}\n`);
+				write(cmd === "pause" ? `paused — worker will stop taking new jobs (jobs still enqueue)${span}\n` : `resumed${span}\n`);
 			} else {
 				// "paused" is included in the counts because jobs enqueued while paused land in the
 				// `paused` list, not `wait` -- omitting it would report backlog 0 in the exact state
@@ -226,7 +230,7 @@ export async function main(argv = process.argv.slice(2), env = process.env) {
 				const pausedState = states.every(Boolean);
 				const pausedPartial = !pausedState && states.some(Boolean);
 				const out = { pausedState, ...(pausedPartial ? { pausedPartial, pausedQueues: names.filter((_, i) => states[i]) } : {}), ...counts, ...(blind ? { fleet: blind } : {}) };
-				process.stdout.write(`${JSON.stringify(out)}\n`);
+				write(`${JSON.stringify(out)}\n`);
 			}
 		} catch (error) {
 			return fail(`could not reach Valkey at ${url} — is it running? (docker compose up)\n  ${error.message}`);
@@ -236,7 +240,7 @@ export async function main(argv = process.argv.slice(2), env = process.env) {
 		return 0;
 	}
 
-	process.stdout.write(`${USAGE}\n`);
+	write(`${USAGE}\n`);
 	return cmd ? 1 : 0;
 }
 

package/src/docker-run.mjs

@@ -50,7 +50,20 @@ export const ISOLATION_FLAGS = [
 ];
 
 /**
- * Build the full `docker run` argv (excluding the leading "docker").
+ * WHAT the box is, with no Docker vocabulary in it.
+ *
+ * Split from the argv builder below so the description of a container exists as a VALUE before it becomes
+ * one runtime's flags. `buildDockerRunArgs` is unchanged in name, signature and output -- it is now
+ * `dockerArgsFromSpec(containerSpec(opts))` -- so every caller and every assertion is untouched, and the
+ * only thing that is new is that the middle of that sentence can be read on its own.
+ *
+ * Mounts are structured (`{host, container, readOnly}`) rather than pre-flattened `host:container:ro`
+ * strings, because the flattening IS the Docker part: a runtime that does not bind-mount has to be able to
+ * see which host path becomes which container path, and what may be written.
+ *
+ * `dockerExtra` is named for what it is. It carries raw Docker flags (`-i -t --entrypoint bash`, a
+ * Linux-only `--user`), so it is the one field a non-Docker consumer must refuse rather than translate.
+ * Calling it `extraFlags` at the boundary would have hidden that.
  *
  * @param image      pinned job image tag/digest
  * @param env        the closed env map from buildContainerEnv -- passed as explicit -e NAME=VALUE
@@ -66,7 +79,7 @@ export const ISOLATION_FLAGS = [
  *                   docker default bridge, which is what every job did before that requirement existed
  * @param extraFlags escape hatch for a Linux-only --user uid:gid on a bind-mounted local folder
  */
-export function buildDockerRunArgs({
+export function containerSpec({
 	image,
 	env,
 	jobDir,
@@ -84,6 +97,56 @@ export function buildDockerRunArgs({
 	if (!name) throw new Error("docker run: container name is required");
 	if (!workspace) throw new Error("docker run: workspace mount is required");
 
+	const mounts = [];
+	// The WHOLE /job dir is read-only (INT-CONTAINER-JOB-INPUTS): it holds prompt.md and pi/, and
+	// the agent cannot rewrite any of it. /workspace is the only writable mount.
+	if (jobDir) mounts.push({ host: jobDir, container: "/job", readOnly: true });
+	mounts.push({ host: workspace, container: "/workspace", readOnly: false });
+	// Local jobs get a writable /outbox host bind, the same host-bind mechanism as /workspace
+	// (DES-WORKER-ON-HOST). github jobs pass no outboxDir, so the request channel does not exist for
+	// them -- an untrusted issue author cannot chain (INT-OUTBOX-CONTRACT).
+	if (outboxDir) mounts.push({ host: outboxDir, container: "/outbox", readOnly: false });
+
+	// This job's OWN copy of its session transcript (REQ-RESUMABLE-SESSION, INT-SESSION-STORE-CONTRACT).
+	// Writable, because pi appends to it as the agent works -- and per-job, exactly like jobDir, which is
+	// the whole reason CONST-ISOLATION-CONTAINER-PER-JOB's "none host-wide" clause still reads true. The
+	// shared store under PI_SESSIONS_DIR is NEVER bind-mounted: one job here would otherwise be able to
+	// read and rewrite every other branch's and every other repository's transcripts, which is not a
+	// weakening of that constraint but its inversion. Absent unless the trigger armed run.resume AND a key
+	// resolved, so an unarmed job's argv is byte-identical to one built before this feature existed.
+	if (sessionDir) mounts.push({ host: sessionDir, container: CONTAINER_SESSION_DIR, readOnly: false });
+
+	// The operator's global pi overlay (REQ-GLOBAL-PI-OVERLAY): custom models, global skills, a global
+	// persona, layered UNDER each repo's own .pi/. Read-only -- it is operator-authored deploy-time config,
+	// the same trust class as the baked floor, but the agent still must not rewrite it. Both job kinds.
+	if (globalPiDir) mounts.push({ host: globalPiDir, container: CONTAINER_GLOBAL_PI_DIR, readOnly: true });
+
+	return {
+		image,
+		name,
+		memory,
+		cpus,
+		network,
+		// UNCONDITIONALLY true, and there is deliberately no parameter that can unset it. The boundary is
+		// not a thing a caller opts into -- CONST-ISOLATION-CONTAINER-PER-JOB is why every other flag here
+		// exists -- so the spec is simply unable to describe an unisolated container, and the builder below
+		// refuses one it is handed. A field that could be false would be a way to ask for less.
+		isolated: true,
+		mounts,
+		env,
+		dockerExtra: extraFlags,
+	};
+}
+
+/**
+ * HOW Docker spells it. The only consumer of a spec today.
+ */
+export function dockerArgsFromSpec(spec) {
+	// The builder CANNOT DECLINE the boundary. `containerSpec` cannot produce anything but `true`, so this
+	// only ever fires on a hand-built spec -- and a hand-built spec that forgot the field is exactly the
+	// case that must fail loudly rather than quietly emit a container with no isolation flags at all.
+	if (spec?.isolated !== true) throw new Error("docker run: refusing to build an argv for a spec that is not isolated");
+
 	// `--network` sits HERE, beside --memory and --cpus, and deliberately NOT inside ISOLATION_FLAGS.
 	// That array is the LITERAL, value-free, unconditional set, and two separate places assert every member
 	// of it reaches the sandbox argv *against the imported array, not a copy* (CONST-ISOLATION-CONTAINER-PER-JOB
@@ -94,41 +157,34 @@ export function buildDockerRunArgs({
 	//
 	// null => the flag is ABSENT, so a job argv without an egress policy is byte-identical to one built
 	// before this feature existed. Same shape as the sessionDir/outboxDir/globalPiDir mounts below.
-	const args = ["run", `--name=${name}`, ...ISOLATION_FLAGS, `--memory=${memory}`, `--cpus=${cpus}`];
-	if (network) args.push(`--network=${network}`);
-	args.push(...extraFlags);
+	const args = ["run", `--name=${spec.name}`, ...ISOLATION_FLAGS, `--memory=${spec.memory}`, `--cpus=${spec.cpus}`];
+	if (spec.network) args.push(`--network=${spec.network}`);
+	args.push(...(spec.dockerExtra ?? []));
 
 	// Explicit env allowlist. Each entry is `-e NAME=VALUE`, built from the closed map -- so a
 	// stray host variable cannot ride along (no bare `-e NAME` inheriting from the host, no
 	// --env-file). Undefined values are skipped, never passed as an empty string.
-	for (const [k, v] of Object.entries(env ?? {})) {
+	for (const [k, v] of Object.entries(spec.env ?? {})) {
 		if (v === undefined || v === null) continue;
 		args.push("-e", `${k}=${v}`);
 	}
 
-	// The WHOLE /job dir is read-only (INT-CONTAINER-JOB-INPUTS): it holds prompt.md and pi/, and
-	// the agent cannot rewrite any of it. /workspace is the only writable mount.
-	if (jobDir) args.push("-v", `${jobDir}:/job:ro`);
-	args.push("-v", `${workspace}:/workspace`);
-	// Local jobs get a writable /outbox host bind, the same host-bind mechanism as /workspace
-	// (DES-WORKER-ON-HOST). github jobs pass no outboxDir, so the request channel does not exist for
-	// them -- an untrusted issue author cannot chain (INT-OUTBOX-CONTRACT).
-	if (outboxDir) args.push("-v", `${outboxDir}:/outbox`);
+	// `-v` and its value stay TWO argv elements rather than one `--volume=` token. Not cosmetic: the mount
+	// assertions across this suite extract mounts by adjacency (`args[i - 1] === "-v"`), so collapsing the
+	// pair would make those filters return nothing and turn several exact-array checks vacuously green.
+	for (const m of spec.mounts ?? []) args.push("-v", `${m.host}:${m.container}${m.readOnly ? ":ro" : ""}`);
 
-	// This job's OWN copy of its session transcript (REQ-RESUMABLE-SESSION, INT-SESSION-STORE-CONTRACT).
-	// Writable, because pi appends to it as the agent works -- and per-job, exactly like jobDir, which is
-	// the whole reason CONST-ISOLATION-CONTAINER-PER-JOB's "none host-wide" clause still reads true. The
-	// shared store under PI_SESSIONS_DIR is NEVER bind-mounted: one job here would otherwise be able to
-	// read and rewrite every other branch's and every other repository's transcripts, which is not a
-	// weakening of that constraint but its inversion. Absent unless the trigger armed run.resume AND a key
-	// resolved, so an unarmed job's argv is byte-identical to one built before this feature existed.
-	if (sessionDir) args.push("-v", `${sessionDir}:${CONTAINER_SESSION_DIR}`);
-
-	// The operator's global pi overlay (REQ-GLOBAL-PI-OVERLAY): custom models, global skills, a global
-	// persona, layered UNDER each repo's own .pi/. Read-only -- it is operator-authored deploy-time config,
-	// the same trust class as the baked floor, but the agent still must not rewrite it. Both job kinds.
-	if (globalPiDir) args.push("-v", `${globalPiDir}:${CONTAINER_GLOBAL_PI_DIR}:ro`);
-
-	args.push(image);
+	args.push(spec.image);
 	return args;
 }
+
+/**
+ * Build the full `docker run` argv (excluding the leading "docker").
+ *
+ * The public entry point, unchanged: same name, same parameters, same argv byte for byte. Kept as the
+ * name rather than replaced by `dockerArgsFromSpec` because `CONST-EGRESS-POLICY-IN-THE-ARGV` cites this
+ * symbol in its Code evidence, and because a rename would churn every call site and assertion for nothing.
+ */
+export function buildDockerRunArgs(opts) {
+	return dockerArgsFromSpec(containerSpec(opts));
+}

package/src/run-mirror.mjs

@@ -110,7 +110,7 @@ export function makeRunMirror({ redis, retentionDays, now = () => Date.now(), lo
 				await bounded(redis.zremrangebyscore(RUNS_INDEX, "-inf", `(${now() - windowMs}`), timeoutMs);
 				await bounded(redis.zremrangebyrank(RUNS_INDEX, 0, -indexMax - 1), timeoutMs);
 				// ROLLING expiry, deliberately unlike `budget.mjs`'s set-once rule and deliberately like
-				// `pi-dispatch:sched-stalls`. A budget window must not be pushed forward by traffic or a busy
+				// `pi-dispatch:sched-stalls:<schedulerId>`. A budget window must not be pushed forward by traffic or a busy
 				// day never resets; an ACTIVITY index should roll with traffic, because that is what it
 				// describes. A fleet that stops running jobs loses its index one window later, which is
 				// correct: there is nothing left to show.

package/src/scheduler-stall-guard.mjs

@@ -1,5 +1,5 @@
 /**
- * Per-scheduler stall accounting -- the money backstop for cron (constitution.md:203-216).
+ * Per-scheduler stall accounting -- the money backstop for cron (CONST-RETRY-INFRA-ONLY).
  *
  * BullMQ's `maxStalledCount` does not cover scheduler jobs: `moveStalledJobsToWait` derives
  * `isRepeatableJob` from the job's `rjk` field and skips the stall-fail for a live scheduler, so a
@@ -10,16 +10,41 @@
  * Injected `redis` (ioredis-compatible), `removeJobScheduler`, and `log` keep the logic testable with
  * no queue, no bullmq import, and no real Valkey.
  *
- * Custom: per-scheduler stall accounting; BullMQ's maxStalledCount does not cover scheduler jobs -- constitution.md:203-216 carve-out ("BullMQ will never do this for us")
+ * Custom: per-scheduler stall accounting; BullMQ's maxStalledCount does not cover scheduler jobs -- CONST-RETRY-INFRA-ONLY carve-out ("BullMQ will never do this for us")
  */
 
-// The Redis hash of per-scheduler stall counts (field = schedulerId, value = count). Exported so the admin
-// panel can read it (HGETALL) for the cron drill-in without re-deriving the key string.
+// The prefix every stall counter lives under, so `KEYS pi-dispatch:sched-stalls*` still shows an operator
+// the whole feature -- the affordance `wait:`, `slot:` and `budget:` all assume. Exported alongside the
+// builder so the admin panel and the integration teardown compose keys through one definition and cannot
+// drift from the writer.
 export const STALL_KEY = "pi-dispatch:sched-stalls";
 
-// A rolling window: the EXPIRE is re-set on every stall, so a scheduler that stops stalling for a full
-// day drops back to zero. This prevents unrelated transient stalls weeks apart from accumulating into a
-// false teardown -- only sustained stalling inside one window trips the threshold.
+/**
+ * One scheduler's counter. The id is VALIDATED upstream rather than hashed here: it is operator-declared in
+ * `triggers.json`, `triggers.mjs` already refuses a `:` in it precisely to protect this parse, and the value
+ * of a readable keyspace is that `GET pi-dispatch:sched-stalls:nightly` answers the question directly.
+ */
+export const stallKey = (schedulerId) => `${STALL_KEY}:${schedulerId}`;
+
+// ONE KEY PER SCHEDULER, so the window is per scheduler.
+//
+// This was one HASH with a field per scheduler and a single `EXPIRE` on the whole key, which meant any
+// scheduler's stall pushed the TTL forward for EVERY scheduler's count. The window never reset on a
+// deployment where anything stalled regularly, so the guard silently degraded from "sustained stalling
+// inside one window" to "cumulative stalling ever": three stalls ninety days apart tore a scheduler down
+// if a neighbour was stalling twice a day, and did not if the deployment was quiet. Same scheduler, same
+// stalls, opposite outcome, decided by an unrelated trigger (issue #267).
+//
+// Per-field TTLs would have fixed it in place and are not available: `HEXPIRE` does not exist on the pinned
+// `valkey/valkey:8` (verified, `ERR unknown command`, recorded under DES-HOST-REGISTRY). A key per entity is
+// the only shape that gets per-entity expiry.
+//
+// The EXPIRE still ROLLS on every stall, deliberately, and that is not `budget.mjs`'s set-once rule being
+// broken. A budget window is a CALENDAR window and must not be pushed forward by traffic or a busy day
+// never resets. This is a STREAK detector -- "is this scheduler wedged right now" -- and quiet for a day
+// genuinely should forget. `poll:<repo>:close-gate:<deliveryId>` is the in-repo precedent, a bounded
+// consecutive-failure counter given its own key and TTL for exactly this reason: it must decay with the
+// thing it measures rather than with a larger family.
 const STALL_WINDOW_SECONDS = 24 * 60 * 60;
 
 /**
@@ -45,8 +70,9 @@ export function makeStallGuard({ redis, threshold, removeJobScheduler, log }) {
 				return;
 			}
 
-			const count = Number(await redis.hincrby(STALL_KEY, schedulerId, 1));
-			await redis.expire(STALL_KEY, STALL_WINDOW_SECONDS);
+			const key = stallKey(schedulerId);
+			const count = Number(await redis.incr(key));
+			await redis.expire(key, STALL_WINDOW_SECONDS);
 
 			if (count > threshold) {
 				try {
@@ -56,7 +82,7 @@ export function makeStallGuard({ redis, threshold, removeJobScheduler, log }) {
 					// state, not an error -- swallow it so hdel and the teardown alert still run.
 					log("scheduler_teardown_remove_failed", { schedulerId, error: error?.message });
 				}
-				await redis.hdel(STALL_KEY, schedulerId);
+				await redis.del(key);
 				// The loud log is the "alert" half of the constitution's "removeJobScheduler -- or alert".
 				log("scheduler_torn_down", { schedulerId, stalls: count });
 			}

package/src/start.mjs

@@ -233,6 +233,15 @@ export function makeReaper({ log }) {
 export async function startWorker(
 	env = process.env,
 	{
+		// WHERE THE BOOT LOG BYTES GO. Defaults to the real stdout, so production is byte-identical; a test
+		// passes a collector instead of reassigning `process.stdout.write`.
+		//
+		// That distinction is not stylistic. `node --test` runs each file in a CHILD PROCESS that serialises
+		// its own results over `process.stdout`, so a test that replaces the global and holds the replacement
+		// across an `await` swallows the runner's result frames for whatever completes in that window. Three
+		// tests in `start-wiring.test.mjs` were reported as not existing at all -- no name, no count, exit 0 --
+		// because this function's log line went through the same channel the runner needed (issue #266).
+		write = (chunk) => process.stdout.write(chunk),
 		makeAuth = makeGitHubAuth,
 		makeHost = makeGitHubHost,
 		createWorkerFn = createWorker,
@@ -263,7 +272,7 @@ export async function startWorker(
 	// other module takes `log` injected, and two tests pin the KEY SET of the fields object handed to an
 	// injected log (`run_record_failed`, `wait_check`); a `host` added at any call site would break them,
 	// while one added inside this closure cannot reach them.
-	const log = (event, fields = {}) => process.stdout.write(`${JSON.stringify({ event, ...fields, host: config.workerName })}\n`);
+	const log = (event, fields = {}) => write(`${JSON.stringify({ event, ...fields, host: config.workerName })}\n`);
 
 	// DES-CRON-VIA-BULLMQ-SCHEDULER: load and validate the triggers file with the operator present and
 	// before any Valkey contact, so a misconfigured schedule refuses startup loudly (configError) rather
@@ -842,7 +851,7 @@ export async function startWorker(
 	// CONST-RETRY-INFRA-ONLY money backstop: BullMQ's maxStalledCount does not bound scheduler jobs, so a
 	// wedged scheduled run is re-paid on every stall. The guard counts stalls per scheduler and tears the
 	// scheduler down past the threshold. Keyed on "stalled", not "failed" -- only a stall is the unbounded re-run.
-	const guard = makeStallGuard({
+	const onStalled = makeStallGuard({
 		redis,
 		threshold: config.schedulerStallMax,
 		// The queue the schedulers were actually INSTALLED on. Torn down from `runtimeQueue` on a named
@@ -850,7 +859,10 @@ export async function startWorker(
 		removeJobScheduler: (id) => cronQueue.removeJobScheduler(id),
 		log,
 	});
-	for (const w of allWorkers) w.on("stalled", (jobId) => void guard.onStalled(jobId));
+	// `makeStallGuard` returns the LISTENER, not an object holding one -- hence the name of the local. It was
+	// called as `guard.onStalled(jobId)` here for the whole life of the feature, which is `undefined(jobId)`,
+	// so every stall threw a TypeError and the money backstop never counted one (issue #267).
+	for (const w of allWorkers) w.on("stalled", (jobId) => void onStalled(jobId));
 
 	// DES-CRON-VIA-BULLMQ-SCHEDULER: install the schedule set (and prune orphans) before announcing the
 	// worker is up, so schedules_installed always precedes worker_started. An empty set skips the reconcile