@edgehero/pi-dispatch 1.8.0 → 1.10.0

package/src/scheduler-stall-guard.mjs

@@ -1,5 +1,5 @@
 /**
- * Per-scheduler stall accounting -- the money backstop for cron (constitution.md:203-216).
+ * Per-scheduler stall accounting -- the money backstop for cron (CONST-RETRY-INFRA-ONLY).
  *
  * BullMQ's `maxStalledCount` does not cover scheduler jobs: `moveStalledJobsToWait` derives
  * `isRepeatableJob` from the job's `rjk` field and skips the stall-fail for a live scheduler, so a
@@ -10,16 +10,41 @@
  * Injected `redis` (ioredis-compatible), `removeJobScheduler`, and `log` keep the logic testable with
  * no queue, no bullmq import, and no real Valkey.
  *
- * Custom: per-scheduler stall accounting; BullMQ's maxStalledCount does not cover scheduler jobs -- constitution.md:203-216 carve-out ("BullMQ will never do this for us")
+ * Custom: per-scheduler stall accounting; BullMQ's maxStalledCount does not cover scheduler jobs -- CONST-RETRY-INFRA-ONLY carve-out ("BullMQ will never do this for us")
  */
 
-// The Redis hash of per-scheduler stall counts (field = schedulerId, value = count). Exported so the admin
-// panel can read it (HGETALL) for the cron drill-in without re-deriving the key string.
+// The prefix every stall counter lives under, so `KEYS pi-dispatch:sched-stalls*` still shows an operator
+// the whole feature -- the affordance `wait:`, `slot:` and `budget:` all assume. Exported alongside the
+// builder so the admin panel and the integration teardown compose keys through one definition and cannot
+// drift from the writer.
 export const STALL_KEY = "pi-dispatch:sched-stalls";
 
-// A rolling window: the EXPIRE is re-set on every stall, so a scheduler that stops stalling for a full
-// day drops back to zero. This prevents unrelated transient stalls weeks apart from accumulating into a
-// false teardown -- only sustained stalling inside one window trips the threshold.
+/**
+ * One scheduler's counter. The id is VALIDATED upstream rather than hashed here: it is operator-declared in
+ * `triggers.json`, `triggers.mjs` already refuses a `:` in it precisely to protect this parse, and the value
+ * of a readable keyspace is that `GET pi-dispatch:sched-stalls:nightly` answers the question directly.
+ */
+export const stallKey = (schedulerId) => `${STALL_KEY}:${schedulerId}`;
+
+// ONE KEY PER SCHEDULER, so the window is per scheduler.
+//
+// This was one HASH with a field per scheduler and a single `EXPIRE` on the whole key, which meant any
+// scheduler's stall pushed the TTL forward for EVERY scheduler's count. The window never reset on a
+// deployment where anything stalled regularly, so the guard silently degraded from "sustained stalling
+// inside one window" to "cumulative stalling ever": three stalls ninety days apart tore a scheduler down
+// if a neighbour was stalling twice a day, and did not if the deployment was quiet. Same scheduler, same
+// stalls, opposite outcome, decided by an unrelated trigger (issue #267).
+//
+// Per-field TTLs would have fixed it in place and are not available: `HEXPIRE` does not exist on the pinned
+// `valkey/valkey:8` (verified, `ERR unknown command`, recorded under DES-HOST-REGISTRY). A key per entity is
+// the only shape that gets per-entity expiry.
+//
+// The EXPIRE still ROLLS on every stall, deliberately, and that is not `budget.mjs`'s set-once rule being
+// broken. A budget window is a CALENDAR window and must not be pushed forward by traffic or a busy day
+// never resets. This is a STREAK detector -- "is this scheduler wedged right now" -- and quiet for a day
+// genuinely should forget. `poll:<repo>:close-gate:<deliveryId>` is the in-repo precedent, a bounded
+// consecutive-failure counter given its own key and TTL for exactly this reason: it must decay with the
+// thing it measures rather than with a larger family.
 const STALL_WINDOW_SECONDS = 24 * 60 * 60;
 
 /**
@@ -45,8 +70,9 @@ export function makeStallGuard({ redis, threshold, removeJobScheduler, log }) {
 				return;
 			}
 
-			const count = Number(await redis.hincrby(STALL_KEY, schedulerId, 1));
-			await redis.expire(STALL_KEY, STALL_WINDOW_SECONDS);
+			const key = stallKey(schedulerId);
+			const count = Number(await redis.incr(key));
+			await redis.expire(key, STALL_WINDOW_SECONDS);
 
 			if (count > threshold) {
 				try {
@@ -56,7 +82,7 @@ export function makeStallGuard({ redis, threshold, removeJobScheduler, log }) {
 					// state, not an error -- swallow it so hdel and the teardown alert still run.
 					log("scheduler_teardown_remove_failed", { schedulerId, error: error?.message });
 				}
-				await redis.hdel(STALL_KEY, schedulerId);
+				await redis.del(key);
 				// The loud log is the "alert" half of the constitution's "removeJobScheduler -- or alert".
 				log("scheduler_torn_down", { schedulerId, stalls: count });
 			}

package/src/schedules.mjs

@@ -137,7 +137,7 @@ function normalizeCronSchedule({ on, run }, path, existsSync, fleet) {
 	// key. A command trigger carries no flow/task at all (the validator enforces the XOR), so those two
 	// keys hold undefined here and drop at JSON serialization -- the command schedule's data is exactly
 	// kind/folder/command plus the shared fields.
-	const data = { kind: "local", folder: run.folder, flow: run.flow, task: run.task, ...(run.command !== undefined && { command: run.command }), provider: run.provider, model: run.model, maxTurns: run.maxTurns, github: run.github, packages: run.packages, image: run.image, ...(run.skillsDir !== undefined && { skillsDir: run.skillsDir }), ...(run.secrets !== undefined && { secrets: run.secrets }), ...(run.secretsProfile !== undefined && { secretsProfile: run.secretsProfile }), resume: run.resume, trigger: { id: on.id, pattern: on.pattern } };
+	const data = { kind: "local", folder: run.folder, flow: run.flow, task: run.task, ...(run.command !== undefined && { command: run.command }), provider: run.provider, model: run.model, maxTurns: run.maxTurns, github: run.github, packages: run.packages, image: run.image, ...(run.backend !== undefined && { backend: run.backend }), ...(run.skillsDir !== undefined && { skillsDir: run.skillsDir }), ...(run.secrets !== undefined && { secrets: run.secrets }), ...(run.secretsProfile !== undefined && { secretsProfile: run.secretsProfile }), resume: run.resume, trigger: { id: on.id, pattern: on.pattern } };
 	// Retention only; the deterministic repeat:<id>:<millis> jobId supplies dedup, so no jobId here, and
 	// scheduler jobs are not retried (DES-CRON-VIA-BULLMQ-SCHEDULER) so no attempts/backoff.
 	const opts = { removeOnComplete: { age: 24 * 3600 }, removeOnFail: { age: 7 * 24 * 3600 } };

package/src/start.mjs

@@ -1,7 +1,5 @@
-import { execFile } from "node:child_process";
 import { readFileSync, watch } from "node:fs";
 import { dirname, basename, join } from "node:path";
-import { promisify } from "node:util";
 import { configError, loadConfig } from "./config.mjs";
 import { makeRedisClient, parseConnection } from "./connection.mjs";
 import { reconcileGated, reloadSchedules } from "./cron.mjs";
@@ -32,6 +30,10 @@ import { loadScopedLimits, scopeKeyPrefix } from "./scoped-limits.mjs";
 import { makeWaitChecker } from "./wait-check.mjs";
 import { makeWaitState } from "./wait-state.mjs";
 import { hostQueueName, makeQueue } from "./queue.mjs";
+import { makeLocalBackend, makeReaper, makeStopContainer } from "./backend-local.mjs";
+import { makeBackendRegistry, reapAll } from "./backend-registry.mjs";
+import { DEFAULT_BACKEND } from "./backends.mjs";
+
 import { makeRunContainer } from "./run-container.mjs";
 import { makeSecretsResolver } from "./secrets.mjs";
 import { buildRecord, makeFindPreviousRun, makeLogReaper, makeLogSink, makeRecordWriter, sanitizeJobId } from "./run-history.mjs";
@@ -40,7 +42,6 @@ import { effectiveSettings, readOverlay } from "./runtime-settings.mjs";
 import { authoredCron, loadSchedules, servedSchedules } from "./schedules.mjs";
 import { makeStallGuard } from "./scheduler-stall-guard.mjs";
 
-const exec = promisify(execFile);
 
 /** How long boot will wait for `docker image inspect` before shipping without a digest. */
 const BOOT_IMAGE_TIMEOUT_MS = 5_000;
@@ -172,49 +173,6 @@ function watchScopedLimitsFile(config, ref, log) {
 	}
 }
 
-export function makeReaper({ log }) {
-	return async function reap() {
-		try {
-			const { stdout } = await exec("docker", ["ps", "--filter", "name=pi-job-", "--format", "{{.Names}}"]);
-			const names = stdout
-				.split("\n")
-				.map((n) => n.trim())
-				.filter(Boolean);
-			for (const name of names) {
-				await exec("docker", ["rm", "-f", name]);
-				log("reaped_container", { name });
-			}
-			// REQ-EGRESS-ALLOWLIST: the per-job networks those containers were on. Swept AFTER the containers,
-			// because a network with a member still attached cannot be removed -- and swept by the SAME
-			// `pi-job-` filter, so the namespace rule that keeps an operator's live sandbox safe from the
-			// container reaper keeps their sandbox NETWORK safe too, with no second rule to remember.
-			//
-			// A crashed worker is the case this exists for: `runContainer`'s own finally removes the network
-			// on every ordinary path, so anything still here outlived a process that did not get to run it.
-			// A network still in use by something else fails to remove and is skipped, which is correct: this
-			// is a best-effort sweep and never a reason not to boot.
-			const { stdout: nets } = await exec("docker", ["network", "ls", "--filter", "name=pi-job-", "--format", "{{.Name}}"]);
-			for (const net of nets.split("\n").map((n) => n.trim()).filter(Boolean)) {
-				try {
-					await exec("docker", ["network", "rm", net]);
-					log("reaped_network", { network: net });
-				} catch {} // still in use, or already gone -- either way not this boot's problem
-			}
-			// Whether the enumeration HAPPENED, which the scope-claim sweep depends on: it may only delete a
-			// claim naming this host once this host has actually established that it holds no containers.
-			return { reaped: true };
-		} catch (err) {
-			// The `docker ps` is inside this try, so this path CANNOT establish that this host holds no
-			// containers -- whether it failed before listing anything or after reaping some and then losing
-			// the daemon. Either way the claim "I hold nothing" is unproven, and sweeping on it would free
-			// slots for containers that may STILL BE RUNNING, letting another host start more alongside
-			// them: a money overrun rather than a tidy-up. Conservative in the only safe direction.
-			log("reaper_skipped", { reason: err?.message });
-			return { reaped: false };
-		}
-	};
-}
-
 /**
  * The runnable worker. Reads config, connects to Valkey, wires every REAL dependency the processor
  * needs, and starts draining the queue. `createWorker` already installs the timeout, the
@@ -233,10 +191,23 @@ export function makeReaper({ log }) {
 export async function startWorker(
 	env = process.env,
 	{
+		// WHERE THE BOOT LOG BYTES GO. Defaults to the real stdout, so production is byte-identical; a test
+		// passes a collector instead of reassigning `process.stdout.write`.
+		//
+		// That distinction is not stylistic. `node --test` runs each file in a CHILD PROCESS that serialises
+		// its own results over `process.stdout`, so a test that replaces the global and holds the replacement
+		// across an `await` swallows the runner's result frames for whatever completes in that window. Three
+		// tests in `start-wiring.test.mjs` were reported as not existing at all -- no name, no count, exit 0 --
+		// because this function's log line went through the same channel the runner needed (issue #266).
+		write = (chunk) => process.stdout.write(chunk),
 		makeAuth = makeGitHubAuth,
 		makeHost = makeGitHubHost,
 		createWorkerFn = createWorker,
 		makeReaper: makeReaperFn = makeReaper,
+		makeBackendRegistry: makeBackendRegistryFn = makeBackendRegistry,
+		// Additional backend bundles, in registration order after `local`. The deployment still decides which
+		// are BLESSED (PI_BACKENDS) and which is default; this only says which exist.
+		extraBackends = [],
 		makeLogSink: makeLogSinkFn = makeLogSink,
 		makeRecordWriter: makeRecordWriterFn = makeRecordWriter,
 		makeRunMirror: makeRunMirrorFn = makeRunMirror,
@@ -263,7 +234,7 @@ export async function startWorker(
 	// other module takes `log` injected, and two tests pin the KEY SET of the fields object handed to an
 	// injected log (`run_record_failed`, `wait_check`); a `host` added at any call site would break them,
 	// while one added inside this closure cannot reach them.
-	const log = (event, fields = {}) => process.stdout.write(`${JSON.stringify({ event, ...fields, host: config.workerName })}\n`);
+	const log = (event, fields = {}) => write(`${JSON.stringify({ event, ...fields, host: config.workerName })}\n`);
 
 	// DES-CRON-VIA-BULLMQ-SCHEDULER: load and validate the triggers file with the operator present and
 	// before any Valkey contact, so a misconfigured schedule refuses startup loudly (configError) rather
@@ -344,9 +315,24 @@ export async function startWorker(
 	// Clear strays left by a previous crash before the worker starts draining. Best-effort: the reaper
 	// swallows its own docker errors; this guard keeps any reaper failure from blocking boot.
 	// Whether the container reaper actually ENUMERATED, which the scope-claim sweep below depends on.
+	// #227. ONE map, read twice: the boot sweep below combines every entry, and each backend's bundle takes
+	// its own reaper from it by name. Held here rather than derived from the bundles because the bundles
+	// cannot exist yet -- they need the log sink, the package resolver and the image preflight, all built
+	// further down -- and reaching for one here is a temporal dead zone the boot try/catch would swallow.
+	let backendReaps = {};
+
 	let reaped = false;
 	try {
-		reaped = (await makeReaperFn({ log })())?.reaped === true;
+		// CONSTRUCTED INSIDE THE GUARD, not above it. The comment on this try says it "keeps any reaper
+		// failure from blocking boot", and a factory that throws is a reaper failure -- an earlier draft
+		// hoisted the construction out and quietly made that sentence false.
+		// DERIVED from the bundles, not a hand-kept parallel map. An earlier draft had a literal here and the
+		// registry cross-checking it, which made adding a venue three edits that nothing forced to agree --
+		// and a forgotten one is INVISIBLE, because `reapAll` is conservative over the reapers it is handed
+		// rather than over the venues that exist. A bundle already carries its own `reap`, so taking it from
+		// there is one place. `local`'s is built here because its bundle cannot exist yet.
+		backendReaps = { [DEFAULT_BACKEND]: makeReaperFn({ log }), ...Object.fromEntries(extraBackends.map((b) => [b?.name, b?.reap])) };
+		reaped = (await reapAll(Object.values(backendReaps), { log }))?.reaped === true;
 	} catch (err) {
 		log("reaper_skipped", { reason: err?.message });
 	}
@@ -620,8 +606,85 @@ export async function startWorker(
 	});
 
 
+	// #227: WHERE this job's container runs. The three functions that decide whether a container may start and
+	// then start it -- two pre-spend gates and the launcher -- bundled into one value with a completeness
+	// check, so the set has a name instead of being three unrelated `deps` keys. Byte-identical to passing them individually -- the same three functions reach the same three keys,
+	// built from the same config, behind the same injectable factories -- and this is the seam a second
+	// backend is selected at once there is one to select.
+	//
+	// Assigned key by key below rather than spread, because the bundle also carries `name` and `declares`,
+	// and `deps` is the processor's namespace: a spread would put a backend's name into it under a key the
+	// processor is free to mean something else by.
+	const localBackend = makeLocalBackend({
+		// #227. The two the earlier slices deferred, now real seams. `reap` keeps its tri-state: the boot
+		// sweep below only sweeps this host's scope claims once the reaper has PROVEN this host holds no
+		// job containers, and an unproven answer must never free a slot.
+		stopContainer: makeStopContainer(),
+		reap: backendReaps[DEFAULT_BACKEND],
+		// One deployment default, two consumers, adjacent by construction: the preflight that refuses a missing
+		// image BEFORE the budget slot, and the factory that puts it in the argv. Both resolve a trigger's own
+		// `run.image` through the same resolveJobImage, so the image that was checked is the image that runs.
+		// Nothing is memoised (see its construction above): `docker image inspect` costs ~tens of ms against a
+		// container run of minutes, and a cache would be wrong in both directions -- an operator who builds the
+		// image mid-day would stay refused, one who removes it would stay admitted. Contrast the staged-package
+		// manifest, correctly read once at boot because it is deploy-time state under a :ro mount.
+		imagePreflight,
+		// REQ-EGRESS-ALLOWLIST, and built here for the same reason the image preflight is: one deployment
+		// value, one place, so the gate that checks the proxy and the runner that attaches to its network
+		// cannot disagree about which proxy is meant. Nothing is memoised here either -- an operator who
+		// starts the proxy mid-day must not stay refused, and one who stops it must not stay admitted.
+		// Unarmed it spawns nothing at all, so a deployment without a policy pays for none of this.
+		egressPreflight: makeEgressPreflightFn({ proxy: config.egressProxy, armed: config.egress }),
+		runContainer: makeRunContainerFn({
+			image: config.jobImage,
+			hostEnv: env,
+			egress: config.egress, // REQ-EGRESS-ALLOWLIST: the per-job network and the proxy variables
+			egressProxy: config.egressProxy,
+			openJobLog,
+			globalPiDir: config.globalPiDir, // REQ-GLOBAL-PI-OVERLAY: :ro overlay mount when configured
+			allowGlobalExtensions: config.allowGlobalExtensions,
+			// REQ-GLOBAL-PI-OVERLAY: staged package paths; every job receives them unless its trigger set
+			// packages:false. A RESOLVER, not the array: the factory is still constructed exactly once, only
+			// the value it reads became a call, so a re-stage takes effect on the next job without a restart.
+			packagePaths: getPackagePaths,
+			forwardEnv: config.forwardEnv,
+			authFromPi: config.authFromPi, // source the provider key from ~/.pi/agent/auth.json when env has none
+			// Self-hosted instance URLs, keyed by forge. A MAP rather than one scalar per forge: the table says
+			// which variable each lands in, so a forge with no self-hosted concept simply has no entry, and
+			// adding one does not widen this signature again.
+			forgeHosts: { gitlab: config.gitlab?.apiUrl ?? null, forgejo: config.forgejo?.apiUrl ?? null, azure: config.azure?.orgUrl ?? null },
+		}),
+	});
+
+	// #227. WHICH backend runs which job, and the one place that decides. One bundle today, so every
+	// resolution returns it -- but the mechanism is real, so `run.backend` stops being a validated label and
+	// the abort path can reach a venue it did not build. `config.backends[0]` is the deployment's default,
+	// and the registry refuses a default it does not hold rather than discovering it at the first pickup.
+	const backends = makeBackendRegistryFn({
+		// #227. A SEAM, not a literal. `docs/backends.md` tells an adapter author to register their bundle
+		// here, and until this was injectable that instruction described code nobody could run: the array
+		// was hard-coded, so a venue could pass the conformance suite, get a table entry and be blessed in
+		// PI_BACKENDS, and then be refused at boot as blessed-but-unbuilt with nowhere to put it. It is also
+		// what lets a wiring test prove `startWorker` actually CONNECTS the registry to the processor --
+		// six mutations reverting that connection survived the whole suite, which is the same shape as the
+		// bug that shipped: invisible while there is one venue.
+		bundles: [localBackend, ...extraBackends],
+		defaultName: config.defaultBackend,
+		// Cross-checked at boot rather than discovered at the first pickup: a name PI_BACKENDS blesses but
+		// nothing builds passes both the loader and the pre-spend gate, and a venue with no boot reaper is
+		// swept by nothing while still reporting the host as proven clean.
+		blessed: config.backends,
+		reaps: backendReaps,
+	});
+
 	const worker = createWorkerFn({
 		connection: parseConnection(config.valkeyUrl),
+		// #227. The abort path's stop, resolved per job rather than hard-wired to docker. A container NAME is
+		// not enough to find the runtime holding it once there is more than one venue.
+		stopContainer: backends.stopContainer,
+		// The NAME the abort stops, built by the venue that will build the container rather than by the local
+		// adapter reached for directly -- the name and the stop have to come from the same venue.
+		containerName: backends.containerName,
 		hostQueue,
 		// Names the BullMQ Worker, which makes `getWorkers()` rows tell hosts apart -- bullmq appends
 		// `:w:<name>` to the client name and `moveToActive` stamps `processedBy` onto each active job's
@@ -709,20 +772,8 @@ export async function startWorker(
 				const state = await held.getState();
 				return state === "delayed" || state === "waiting" || state === "active" || state === "prioritized" || state === "waiting-children";
 			},
-			// One deployment default, two consumers, adjacent by construction: the preflight that refuses a missing
-			// image BEFORE the budget slot, and the factory that puts it in the argv. Both resolve a trigger's own
-			// `run.image` through the same resolveJobImage, so the image that was checked is the image that runs.
-			// Nothing is memoised: `docker image inspect` costs ~tens of ms against a container run of minutes, and a
-			// cache would be wrong in both directions -- an operator who builds the image mid-day would stay refused,
-			// one who removes it would stay admitted. Contrast the staged-package manifest, correctly read once at
-			// boot because it is deploy-time state under a :ro mount; the host's image set is not.
-			imagePreflight,
-			// REQ-EGRESS-ALLOWLIST, and built here for the same reason the image preflight is: one deployment
-			// value, one place, so the gate that checks the proxy and the runner that attaches to its network
-			// cannot disagree about which proxy is meant. Nothing is memoised here either -- an operator who
-			// starts the proxy mid-day must not stay refused, and one who stops it must not stay admitted.
-			// Unarmed it spawns nothing at all, so a deployment without a policy pays for none of this.
-			egressPreflight: makeEgressPreflightFn({ proxy: config.egressProxy, armed: config.egress }),
+			imagePreflight: backends.imagePreflight,
+			egressPreflight: backends.egressPreflight,
 			// Completed-only, so a policy or infra exit leaves the canonical transcript byte-identical and a
 			// retry starts from what the first attempt did (CONST-RETRY-INFRA-ONLY).
 			promoteSession: sessionStore.promoteSession,
@@ -739,25 +790,21 @@ export async function startWorker(
 			// the panel should not have to restart the worker to use it. A deployment that declares nothing
 			// spawns nothing at all: the gate only calls this when a trigger is armed.
 			resolveSecrets: makeSecretsResolverFn({ envProfiles: config.secretProfiles, roots: config.secretResolverRoots, timeoutMs: config.secretResolveTimeoutMs, forwardEnv: config.forwardEnv, log }),
-			runContainer: makeRunContainerFn({
-				image: config.jobImage,
-				hostEnv: env,
-				egress: config.egress, // REQ-EGRESS-ALLOWLIST: the per-job network and the proxy variables
-				egressProxy: config.egressProxy,
-				openJobLog,
-				globalPiDir: config.globalPiDir, // REQ-GLOBAL-PI-OVERLAY: :ro overlay mount when configured
-				allowGlobalExtensions: config.allowGlobalExtensions,
-				// REQ-GLOBAL-PI-OVERLAY: staged package paths; every job receives them unless its trigger set
-				// packages:false. A RESOLVER, not the array: the factory is still constructed exactly once, only
-				// the value it reads became a call, so a re-stage takes effect on the next job without a restart.
-				packagePaths: getPackagePaths,
-				forwardEnv: config.forwardEnv,
-				authFromPi: config.authFromPi, // source the provider key from ~/.pi/agent/auth.json when env has none
-				// Self-hosted instance URLs, keyed by forge. A MAP rather than one scalar per forge: the table says
-				// which variable each lands in, so a forge with no self-hosted concept simply has no entry, and
-				// adding one does not widen this signature again.
-				forgeHosts: { gitlab: config.gitlab?.apiUrl ?? null, forgejo: config.forgejo?.apiUrl ?? null, azure: config.azure?.orgUrl ?? null },
-			}),
+			// #227. What PI_BACKENDS blessed, so a trigger naming an unblessed venue refuses pre-spend. The
+			// panel's picker is bounded by the same list, and this is the half that binds: the overlay is
+			// not the reviewed artifact (DES-PER-TRIGGER-SECRET-PROFILE).
+			blessedBackends: config.backends,
+			runContainer: backends.runContainer,
+			// #227. Resolved through the registry like every other per-job backend fact, so the integers the
+			// processor treats as "never started" are the ones the venue that ran the job actually uses.
+			// NOT `?? []`. An empty list means "this venue never reports a never-started exit", which sends a
+			// 125 to the unknown-exit branch -- no `reason`, so the budget slot is KEPT and BullMQ retries,
+			// burning a second one per never-started job. That is the bug the explicit case group was added
+			// to fix. A bundle that omits the field is a wiring defect, so it fails loudly here rather than
+			// silently in the money direction; `backends.mjs` requires it of every bundle.
+			// Off the REGISTRY's surface, like every other per-job backend fact. Rebuilding it at the call site
+			// is how a fact ends up dispatched on one path and hardcoded on another.
+			neverStartedExits: backends.neverStartedExits,
 			prepareWorkspace: makePrepareWorkspace({
 				jobsDir: config.jobsDir,
 				forgeFor,
@@ -842,7 +889,7 @@ export async function startWorker(
 	// CONST-RETRY-INFRA-ONLY money backstop: BullMQ's maxStalledCount does not bound scheduler jobs, so a
 	// wedged scheduled run is re-paid on every stall. The guard counts stalls per scheduler and tears the
 	// scheduler down past the threshold. Keyed on "stalled", not "failed" -- only a stall is the unbounded re-run.
-	const guard = makeStallGuard({
+	const onStalled = makeStallGuard({
 		redis,
 		threshold: config.schedulerStallMax,
 		// The queue the schedulers were actually INSTALLED on. Torn down from `runtimeQueue` on a named
@@ -850,7 +897,10 @@ export async function startWorker(
 		removeJobScheduler: (id) => cronQueue.removeJobScheduler(id),
 		log,
 	});
-	for (const w of allWorkers) w.on("stalled", (jobId) => void guard.onStalled(jobId));
+	// `makeStallGuard` returns the LISTENER, not an object holding one -- hence the name of the local. It was
+	// called as `guard.onStalled(jobId)` here for the whole life of the feature, which is `undefined(jobId)`,
+	// so every stall threw a TypeError and the money backstop never counted one (issue #267).
+	for (const w of allWorkers) w.on("stalled", (jobId) => void onStalled(jobId));
 
 	// DES-CRON-VIA-BULLMQ-SCHEDULER: install the schedule set (and prune orphans) before announcing the
 	// worker is up, so schedules_installed always precedes worker_started. An empty set skips the reconcile

package/src/triggers.mjs

@@ -23,6 +23,7 @@
  * Custom: triggers validated inline per config.mjs/schedules.mjs precedent; zod not in deps
  */
 
+import { BACKEND_NAMES, backendFor } from "./backends.mjs";
 import { EGRESS_ENV_VARS, WORKER_ONLY_SECRET_VARS, configError } from "./config.mjs";
 // SKILL_NAME_RE is the single-sourced skill charset (flow-gate exports it for exactly this reason:
 // materialize.mjs and the admin already import it, and a keep-in-sync copy would drift where a
@@ -350,6 +351,7 @@ function normalizeCron(on, run, index, path, state) {
 	validateDisarmed(on, `cron trigger "${id}"`, path, { onType: "cron" });
 	const secrets = validateSecrets(run, `cron trigger "${id}"`, path);
 	const secretsProfile = validateSecretsProfile(run, `cron trigger "${id}"`, path);
+	const backend = validateBackend(on, run, `cron trigger "${id}"`, path, { localWorkspace: true });
 	// Called for its refusal only: the returned `run` below deliberately grows no `waitFor` key, exactly as
 	// it grows no `replicas` one, because a cron entry can never carry either (issue #230).
 	validateWaitFor(on, run, `cron trigger "${id}"`, path, { onType: "cron" });
@@ -361,7 +363,7 @@ function normalizeCron(on, run, index, path, state) {
 	// freeze today's default into every stored repeatable.
 	return {
 		on: { type: "cron", id, pattern },
-		run: { kind: "local", folder: run.folder, flow: run.flow, task: run.task, provider: run.provider, model: run.model, maxTurns: run.maxTurns, github: run.github, packages, image, resume, ...(command !== undefined && { command }), ...(skillsDir !== undefined && { skillsDir }), ...(secrets !== undefined && { secrets }), ...(secretsProfile !== undefined && { secretsProfile }) },
+		run: { kind: "local", folder: run.folder, flow: run.flow, task: run.task, provider: run.provider, model: run.model, maxTurns: run.maxTurns, github: run.github, packages, image, resume, ...(command !== undefined && { command }), ...(skillsDir !== undefined && { skillsDir }), ...(secrets !== undefined && { secrets }), ...(secretsProfile !== undefined && { secretsProfile }), ...(backend !== undefined && { backend }) },
 	};
 }
 
@@ -992,6 +994,115 @@ function validateSecretsProfile(run, at, path) {
 	return profile;
 }
 
+/**
+ * Would running a folder-bound job in this venue be impossible? Exported ONLY so it can be driven.
+ *
+ * The refusal it answers cannot be reached through `parseTriggers` today: the table holds one entry and it
+ * is local, so no trigger can name a remote venue and the branch below is unreachable. A mutation pass duly
+ * deleted that branch and the whole suite stayed green. A rule nothing can exercise is a rule that will be
+ * wrong on the day it finally matters, which is the day a second backend lands -- so the decision lives
+ * here, as one line over an explicit entry, and a test drives it against a synthetic remote one.
+ *
+ * `!== false` and not `=== true`: `remote` is a plain field on the table entry rather than a member of the
+ * closed PROPERTIES list, so an entry that simply omits it would otherwise read as local and be admitted
+ * onto a folder-bound trigger. That is the fail-open direction `backends.mjs`'s own rule forbids ("a backend
+ * that omits one is not admitted for it"), and it would defeat with an absent key a refusal this file calls
+ * physics. A test pins that every entry declares it, so both halves have to fail together.
+ */
+export function refusesLocalWorkspace(entry, localWorkspace) {
+	return Boolean(localWorkspace) && entry?.remote !== false;
+}
+
+/**
+ * `run.backend` -- WHICH venue this trigger's container is built in (issue #227).
+ *
+ * A NAME, never a configuration. `validateSecretsProfile` above is the template end to end, and the reason
+ * is the same: a trigger SELECTS among what the deployment already blessed, and never configures a posture.
+ * `run.network` was rejected outright for that reason, and this field must not become a way back to it --
+ * naming `remote-vendor` cannot turn egress off, because what egress a venue can provide is the backend
+ * table's answer and the deployment's floor bounds which venues are reachable at all.
+ *
+ * WHAT IS CHECKED HERE AND WHAT IS NOT. This file answers what the FILE knows: that the name is a name, and
+ * that the venue could ever run this kind of job. Whether the DEPLOYMENT blessed it is `PI_BACKENDS`, which
+ * lives in the environment, and a loader that read it would refuse a reviewed file on a per-host setting --
+ * so the processor refuses that pre-spend instead. Same split `secretsProfile` already draws between the
+ * charset check here and `secret-profile-unknown` there.
+ *
+ * A NEAR-MISS SPELLING IS REFUSED, which puts this in `waitFor`'s class rather than `run.imgae`'s. A
+ * misspelled image gives you the default image and a job that ran; a misspelled `backend` gives you the
+ * DEFAULT VENUE and a job that ran -- byte-identical in the record, the panel and the log to one that
+ * correctly named a venue, while the operator reads the file as though it chose. That is the destructive
+ * absence `validateWaitFor` refuses near-misses for, one field over.
+ */
+function validateBackend(on, run, at, path, { localWorkspace }) {
+	// The near-miss sweep, exactly `validateWaitFor`'s: on `run` the exact spelling is the field, on `on`
+	// every spelling is wrong including the correct one, because a venue is a property of the run.
+	for (const [label, source, exactIsLegal] of [
+		["run", run, true],
+		["on", on, false],
+	]) {
+		for (const key of Object.keys(source ?? {})) {
+			if (exactIsLegal && key === "backend") continue;
+			// The PLURAL is included, and it is the likeliest miss of all: the deployment-side variable is
+			// `PI_BACKENDS`, so `run.backends` is what an operator writes from memory. Nothing else in this
+			// file needs that, which is why the comparison is against a small set rather than `waitFor`'s
+			// single normalized string.
+			// A HOMOGLYPH is caught too. `replace(/[^a-z0-9]/gi, "")` DELETES a non-ASCII character rather
+			// than failing on it, so a Cyrillic "a" in `b<U+0430>ckend` normalizes to `bckend` and an
+			// Armenian "n" in `backe<U+0578>d` to `backed` -- both miss an equality test and are dropped in
+			// exactly the silence this sweep exists to prevent. Such a key arrives by paste from a rendered
+			// doc or a chat client, the same route homoglyphs take into package names.
+			//
+			// So a key carrying non-ASCII is compared as a SUBSEQUENCE: whatever ASCII letters survive must
+			// still be obtainable from `backend`/`backends` in order. That catches any number of substituted
+			// glyphs without guessing which ones, and a genuinely unrelated non-ASCII key (`fl<U+00F6>w`)
+			// keeps falling through to this file's documented tolerance of unknown `run` keys.
+			const normalized = key.replace(/[^a-z0-9]/gi, "").toLowerCase();
+			const targets = ["backend", "backends"];
+			const isSubsequence = (needle, hay) => {
+				let i = 0;
+				for (const ch of hay) if (i < needle.length && needle[i] === ch) i++;
+				return i === needle.length;
+			};
+			// The floor is 4 rather than 5 so a key with three substituted glyphs is still caught. Only a
+			// non-ASCII key reaches this branch, so the false-positive surface is a non-ASCII key whose
+			// surviving letters happen to be an in-order subset of "backend", which no field here is.
+			const suspicious = /^[\x20-\x7E]*$/.test(key)
+				? targets.includes(normalized)
+				: normalized.length >= 4 && targets.some((t) => isSubsequence(normalized, t));
+			if (!suspicious) continue;
+			throw configError(`${at}: ${label}.${key} is not a field -- did you mean run.backend? A venue the loader drops runs the job somewhere else while the file reads as though it chose, so a near miss is refused rather than dropped: ${path}`);
+		}
+	}
+
+	const backend = run.backend;
+	if (backend === undefined) return undefined;
+
+	if (!isNonEmptyString(backend)) {
+		throw configError(`${at}: run.backend must be a non-empty string naming one of the backends this deployment can run jobs in (got ${JSON.stringify(backend)}): ${path}`);
+	}
+	if (!ID_CHARSET.test(backend)) {
+		throw configError(`${at}: run.backend ${JSON.stringify(backend)} may use letters, digits, dot, dash and underscore only -- PI_BACKENDS is a comma-separated list, so a name carrying that separator could not be blessed: ${path}`);
+	}
+	const entry = backendFor(backend);
+	if (!entry) {
+		throw configError(`${at}: run.backend ${JSON.stringify(backend)} is not a backend this build knows (known: ${BACKEND_NAMES.join(", ")}) -- whether your deployment BLESSES it is PI_BACKENDS, checked before the job spends: ${path}`);
+	}
+
+	// A LOCAL job cannot run in a remote venue, and this is physics rather than policy. `DES-WORKER-ON-HOST`
+	// finding (2): "the operator's own folder must be bind-mounted as /workspace, edited in place. There is
+	// no volume to hide behind." A cron trigger normalizes to a local run for the same reason. Refused at
+	// LOAD and permanently, because no amount of deployment configuration can make a folder on this machine
+	// appear inside someone else's daemon -- so this is not a bound an operator could widen, it is one the
+	// world imposes, and discovering it per job would burn a pickup for an answer the file already had.
+	// The decision itself is `refusesLocalWorkspace` above, so it can be driven; see its comment.
+	if (refusesLocalWorkspace(entry, localWorkspace)) {
+		throw configError(`${at}: run.backend ${JSON.stringify(backend)} is remote, and this trigger runs against a folder on THIS machine that has to be bind-mounted and edited in place -- there is no volume to hide behind (DES-WORKER-ON-HOST): ${path}`);
+	}
+
+	return backend;
+}
+
 /**
  * `run.waitFor` -- the conditions that must all clear before this trigger's job starts (issue #230).
  *
@@ -1147,10 +1258,11 @@ function normalizeLabel(on, run, index, path) {
 	const replicas = validateReplicas(run, at, path);
 	const secrets = validateSecrets(run, at, path);
 	const secretsProfile = validateSecretsProfile(run, at, path);
+	const backend = validateBackend(on, run, at, path, { localWorkspace: false });
 	const waitFor = validateWaitFor(on, run, at, path, { onType: on.type });
 	return {
 		on: { type: "label", any: predicate.any, all: predicate.all, none: predicate.none },
-		run: { kind: run.kind, flow: run.flow, packages, image, resume, replicas, ...(command !== undefined && { command }), ...(skillsDir !== undefined && { skillsDir }), ...(instructions !== undefined && { instructions }), ...(repository !== undefined && { repository }), ...(secrets !== undefined && { secrets }), ...(secretsProfile !== undefined && { secretsProfile }), ...(waitFor !== undefined && { waitFor }) },
+		run: { kind: run.kind, flow: run.flow, packages, image, resume, replicas, ...(command !== undefined && { command }), ...(skillsDir !== undefined && { skillsDir }), ...(instructions !== undefined && { instructions }), ...(repository !== undefined && { repository }), ...(secrets !== undefined && { secrets }), ...(secretsProfile !== undefined && { secretsProfile }), ...(waitFor !== undefined && { waitFor }), ...(backend !== undefined && { backend }) },
 	};
 }
 
@@ -1186,10 +1298,11 @@ function normalizeComment(on, run, index, path, state) {
 	const replicas = validateReplicas(run, at, path);
 	const secrets = validateSecrets(run, at, path);
 	const secretsProfile = validateSecretsProfile(run, at, path);
+	const backend = validateBackend(on, run, at, path, { localWorkspace: false });
 	const waitFor = validateWaitFor(on, run, at, path, { onType: on.type });
 	return {
 		on: { type: "comment", phrase: on.phrase },
-		run: { kind: run.kind, flow: run.flow, packages, image, resume, replicas, ...(command !== undefined && { command }), ...(skillsDir !== undefined && { skillsDir }), ...(instructions !== undefined && { instructions }), ...(repository !== undefined && { repository }), ...(secrets !== undefined && { secrets }), ...(secretsProfile !== undefined && { secretsProfile }), ...(waitFor !== undefined && { waitFor }) },
+		run: { kind: run.kind, flow: run.flow, packages, image, resume, replicas, ...(command !== undefined && { command }), ...(skillsDir !== undefined && { skillsDir }), ...(instructions !== undefined && { instructions }), ...(repository !== undefined && { repository }), ...(secrets !== undefined && { secrets }), ...(secretsProfile !== undefined && { secretsProfile }), ...(waitFor !== undefined && { waitFor }), ...(backend !== undefined && { backend }) },
 	};
 }
 
@@ -1252,6 +1365,7 @@ function normalizeIssue(on, run, index, path) {
 	const replicas = validateReplicas(run, at, path);
 	const secrets = validateSecrets(run, at, path);
 	const secretsProfile = validateSecretsProfile(run, at, path);
+	const backend = validateBackend(on, run, at, path, { localWorkspace: false });
 	const waitFor = validateWaitFor(on, run, at, path, { onType: on.type });
 	return {
 		on: {
@@ -1262,7 +1376,7 @@ function normalizeIssue(on, run, index, path) {
 			...(number !== undefined && { number }),
 			...(once !== undefined && { once }),
 		},
-		run: { kind: run.kind, flow: run.flow, packages, image, resume, replicas, ...(command !== undefined && { command }), ...(skillsDir !== undefined && { skillsDir }), ...(instructions !== undefined && { instructions }), ...(secrets !== undefined && { secrets }), ...(secretsProfile !== undefined && { secretsProfile }), ...(waitFor !== undefined && { waitFor }) },
+		run: { kind: run.kind, flow: run.flow, packages, image, resume, replicas, ...(command !== undefined && { command }), ...(skillsDir !== undefined && { skillsDir }), ...(instructions !== undefined && { instructions }), ...(secrets !== undefined && { secrets }), ...(secretsProfile !== undefined && { secretsProfile }), ...(waitFor !== undefined && { waitFor }), ...(backend !== undefined && { backend }) },
 	};
 }
 
@@ -1336,6 +1450,7 @@ function normalizePullRequest(on, run, index, path) {
 	const replicas = validateReplicas(run, at, path);
 	const secrets = validateSecrets(run, at, path);
 	const secretsProfile = validateSecretsProfile(run, at, path);
+	const backend = validateBackend(on, run, at, path, { localWorkspace: false });
 	const waitFor = validateWaitFor(on, run, at, path, { onType: on.type });
 	return {
 		on: {
@@ -1351,7 +1466,7 @@ function normalizePullRequest(on, run, index, path) {
 			...(number !== undefined && { number }),
 			...(once !== undefined && { once }),
 		},
-		run: { kind: run.kind, flow: run.flow, packages, image, resume, replicas, ...(command !== undefined && { command }), ...(skillsDir !== undefined && { skillsDir }), ...(instructions !== undefined && { instructions }), ...(secrets !== undefined && { secrets }), ...(secretsProfile !== undefined && { secretsProfile }), ...(waitFor !== undefined && { waitFor }) },
+		run: { kind: run.kind, flow: run.flow, packages, image, resume, replicas, ...(command !== undefined && { command }), ...(skillsDir !== undefined && { skillsDir }), ...(instructions !== undefined && { instructions }), ...(secrets !== undefined && { secrets }), ...(secretsProfile !== undefined && { secretsProfile }), ...(waitFor !== undefined && { waitFor }), ...(backend !== undefined && { backend }) },
 	};
 }