@edgehero/pi-dispatch 2.1.0 → 3.1.0

package/src/index.mjs

@@ -8,7 +8,15 @@ import { InfraRetry, NETNS_KEEPER_CRASH_LOOP, NETNS_KEEPER_NOT_HOLDING, TERMINAL
 import { NETNS_KEEPER_YOUNG_HOLD_MAX_MS, netnsKeeperCrashLoopSentence, netnsKeeperLoopAgainSentence } from "./netns-keeper.mjs";
 import { PODMAN_RESTART_HOLD_EXPIRED, PODMAN_RESTART_HOLD_MAX_MS, PODMAN_RESTART_HOLD_RECHECK_MS } from "./runtime-observations.mjs";
 import { targetFor } from "./run-history.mjs";
-import { budgetCapsFor, canonicalScope, concurrencyFor, makeInFlight, scopeKeyPrefix } from "./scoped-limits.mjs";
+import { isPerMachineHost } from "./backends.mjs";
+import { hash16 } from "./fleet-lease.mjs";
+import { endpointsForModel } from "./model-endpoints.mjs";
+import { splitModelEntry } from "./model-ref.mjs";
+import { effectiveCostCapMicros } from "./money.mjs";
+import { dollarWindowCaps } from "./dollar-budget.mjs";
+import { concurrencyFor, dollarCapsFor, makeInFlight, modelDollarRows, projectDollarCapsFor, projectRowFor, rowScopeFor, scopedLedgers } from "./scoped-limits.mjs";
+import { memberScopeOf, projectOf } from "./projects.mjs";
+import { governedDollars } from "./allocation.mjs";
 import { WAIT_AFTER_MAX_DEFAULT_MS, WAIT_INTERVAL_FLOOR_MS, afterMs, unreadableConditions, waitArmed, waitBackoffMs, waitLabel, waitProfileNames } from "./wait-for.mjs";
 import { makeWaitState } from "./wait-state.mjs";
 
@@ -18,6 +26,17 @@ export const QUEUE = "pi-jobs";
 /** The key the host-wide in-flight count lives under. One machine, one counter, whatever the queue. */
 export const HOST_SLOT_KEY = "host";
 export const JOB_TIMEOUT_MS = 30 * 60 * 1000; // REQ-JOB-TIMEOUT-30M
+
+/**
+ * The failed reason BullMQ gives a job its stall check failed (`maxStalledCount: 0` below): the literal in the pinned
+ * bullmq's `moveStalledJobsToWait`, stored as the job's deferred failure and thrown as an UnrecoverableError at the
+ * next pickup. Matched EXACTLY by start.mjs's failed listener, and pinned against the installed bullmq source by a
+ * test, so a bullmq bump that rewords it fails that test rather than silently turning the lost-lock check off.
+ */
+export const STALLED_FAILED_REASON = "job stalled more than allowable limit";
+
+/** How many (job, stall count, attempt, source) keys the processor remembers having logged a refused record for. */
+export const REJECTED_SEEN_MAX = 1000;
 // The scope-busy re-check (issue #242): a held scope has no natural "until" (the holder may run to
 // JOB_TIMEOUT_MS), so a deferred job re-tests on a fixed cadence. 5s keeps the worst case trivial
 // (<=360 wakes across a 30-minute hold, each ~1ms of synchronous predicate briefly occupying a slot)
@@ -26,6 +45,13 @@ export const JOB_TIMEOUT_MS = 30 * 60 * 1000; // REQ-JOB-TIMEOUT-30M
 // docker daemon bounds any herd by its own concurrency, and a contended wake just re-defers.
 export const SCOPE_BUSY_RECHECK_MS = 5_000;
 
+// The endpoint-busy re-check (issue #503): a job whose model server has every slot held re-tests on a fixed
+// cadence, for the scope re-check's reason (a held slot has no natural "until"). Its own value rather than a
+// borrow of 5s, because nothing records WHY a job sits in the delayed set and the wake instant is the only
+// evidence an operator has: 7s is distinct from the scope re-check, the wait throttle floor and the supersede
+// re-ask, and a test keeps all four apart. Short, because a local model run is often short too.
+export const ENDPOINT_BUSY_RECHECK_MS = 7_000;
+
 // How long a job waits before re-asking whether a target's holder is still alive (issue #230). Reached only
 // when the liveness probe could not answer, which is a redis or queue fault rather than a normal state, so
 // this is a short retry rather than a cadence: the job is deciding nothing and holding nothing while it
@@ -111,9 +137,93 @@ function boundAfterAbort(run, signal, job, log, graceMs = ABORT_GRACE_MS) {
  * under `job.data > overlay > env` precedence and re-binds the worker slot count via `applyConcurrency`.
  * The overlay changes which values the spend caps take, never when they are checked -- reserveBudget still
  * runs inside runJob against the freshly passed caps (CONST-BUDGET-BEFORE-TOKENS).
+ *
+ * The read happens once per pickup, right after the scope gate and before the model endpoint gate (issue #503),
+ * because that gate needs the effective provider and model, which the overlay can supply.
+ */
+/**
+ * A job's endpoint set (issues #503, #502): the endpoints its main model AND every model on its effective allowed
+ * list are served by, so a job that may switch to a listed model on another local server holds that server's slot
+ * too. The caller deduplicates by endpoint id. With no list it is the main model's alone, and the residual stays
+ * as #503 named it: an unrestricted job's mid-run switch to an undeclared model takes no slot. The keyless verdict
+ * is a different question ("every model of the provider", `keylessVerdict`) and does not read this set.
+ *
+ * `models` here is the parsed overlay models.json (the endpoint module's name for it); the list is `job.models`.
+ */
+export function mainModelEndpoints({ models, job, endpoints }) {
+	const set = endpointsForModel({ models, provider: job.provider, modelId: job.model, endpoints });
+	for (const entry of Array.isArray(job.models) ? job.models : []) {
+		const ref = splitModelEntry(entry);
+		if (ref !== null) set.push(...endpointsForModel({ models, provider: ref.provider, modelId: ref.model, endpoints }));
+	}
+	return set;
+}
+
+/**
+ * The job runJob is handed: `job.data > overlay > env` precedence, field by field (INT-CONFIG-OVERLAY-CONTRACT).
+ * ONE function for the two readers (the endpoint gate at pickup and the runJob call), so the gate can never lease
+ * for a model other than the one the container is started with.
  */
-export function makeProcessor({ cancelJob, stopContainer, containerName = (job) => jobContainerName(job.id), redis, getSettings, applyConcurrency = () => {}, pauseUntil = () => null, scopedLimits = () => [], inFlight = makeInFlight(), hostBound = null, checkLease = null, scopeLease = null, deps, recordRun = () => {}, timeoutMs = JOB_TIMEOUT_MS, cancelPollMs = 2_000, cancelStopBoundMs = CANCEL_STOP_BOUND_MS, hostName = "", now = () => Date.now(), waitState = makeWaitState({ redis, now }), afterMaxMs = () => WAIT_AFTER_MAX_DEFAULT_MS, checkSlots = makeInFlight(), checkSlotCount = () => 1, checkTimeoutMs = () => 10_000, concurrencyNow = () => 3, intervalMs = () => WAIT_INTERVAL_FLOOR_MS * 2, maxWaitMs = () => 24 * 3600 * 1000, maxChecks = () => 96, maxFaults = () => 5, random = Math.random }) {
+export function effectiveJobOf(data, settings, allowedModels = null, log = () => {}) {
+	// Issue #502: the allowed-model list is the trigger's, else the deployment's PI_ALLOWED_MODELS, else none. The
+	// env list arrives as its own argument and never through `settings`, which the overlay (and so `dispatch_set`)
+	// writes. Absent stays absent, so an unrestricted job's effective job has no `models` key at all.
+	const models = data.models ?? allowedModels ?? null;
+	return {
+		...data,
+		provider: data.provider ?? settings.provider,
+		model: data.model ?? settings.model,
+		maxTurns: data.maxTurns ?? settings.maxTurns,
+		maxTokens: data.maxTokens ?? settings.maxTokens, // optional per-job token budget (issue #25); null => runner meter only
+		...(models !== null ? { models } : {}),
+		// Issue #501: the per-job dollar cap in integer micro-dollars, or null for none. NOT `??` like the fields
+		// above: a trigger's `run.maxCostUsd` may only NARROW, so this is the smaller of the trigger's and the
+		// deployment's, each counted only when set (`effectiveCostCapMicros` says why a trigger-only cap applies
+		// and what a malformed job value reads as). buildContainerEnv sends it as PI_MAX_COST_MICROS, and the
+		// image preflight requires `costCap` of any job that carries one.
+		// A malformed queued value is logged by KEY only (`job_cost_cap_malformed`), never by value.
+		maxCostMicros: effectiveCostCapMicros(data.maxCostUsd, settings.maxCostUsd, (key) => log("job_cost_cap_malformed", { key })),
+	};
+}
+
+export function makeProcessor({ cancelJob, stopContainer, containerName = (job) => jobContainerName(job.id), redis, getSettings, applyConcurrency = () => {}, pauseUntil = () => null, scopedLimits = () => [], projects = () => [], allocation = null, inFlight = makeInFlight(), hostBound = null, checkLease = null, scopeLease = null, endpointSlots = makeInFlight(), endpointLease = null, modelEndpoints = null, overlayModels = () => null, endpointSetFor = mainModelEndpoints, deps, recordRun = () => {}, settledRecord = null, timeoutMs = JOB_TIMEOUT_MS, cancelPollMs = 2_000, cancelStopBoundMs = CANCEL_STOP_BOUND_MS, hostName = "", now = () => Date.now(), waitState = makeWaitState({ redis, now }), afterMaxMs = () => WAIT_AFTER_MAX_DEFAULT_MS, checkSlots = makeInFlight(), checkSlotCount = () => 1, checkTimeoutMs = () => 10_000, concurrencyNow = () => 3, intervalMs = () => WAIT_INTERVAL_FLOOR_MS * 2, maxWaitMs = () => 24 * 3600 * 1000, maxChecks = () => 96, maxFaults = () => 5, random = Math.random }) {
+	// The lost-lock gate's rejection lines, said ONCE per job id, stall count and attempt. The gate runs before every
+	// deferral (pause window, wait, scope or endpoint busy), and BullMQ never resets a job's stall count, so a stalled
+	// scheduled job whose record is refused meets the gate again on every deferred pickup: a 30-minute scope-busy hold
+	// is about 360 of them. The verdict cannot change between them (same record, same attempt), so one line says it.
+	// Bounded, oldest out first, so a long-lived worker holds at most REJECTED_SEEN_MAX keys.
+	const rejectedSeen = new Set();
+	const firstRejection = (key) => {
+		if (rejectedSeen.has(key)) return false;
+		rejectedSeen.add(key);
+		if (rejectedSeen.size > REJECTED_SEEN_MAX) rejectedSeen.delete(rejectedSeen.values().next().value);
+		return true;
+	};
 	return async function processor(job, token, signal) {
+		// THE LOST-LOCK GATE, first because it is free and because it can only ever stop a run (CONST-RETRY-INFRA-ONLY).
+		// BullMQ hands a job to the processor again after its stall check took it back. That happens to a job whose
+		// processor FINISHED when Valkey was unreachable for longer than the lock renewal window: the record was
+		// written, the completion was refused ("Missing lock"), and the job stayed active without a lock. A plain job
+		// then carries a deferred failure and never reaches here (start.mjs's failed listener handles it), but a
+		// scheduled job is moved back to wait and would run again, PAID, with its second record overwriting the
+		// first. So a job that has stalled (`stalledCounter > 0`) and whose record says this same attempt finished
+		// without failing ends as that record says, without a container. `budgetReserved` is the record's own: the
+		// completed listener then pages exactly when it would have for the first finish, which never reached it.
+		// No record (a worker that died mid-run, a lookup fault) keeps today's path: the job runs, and the stall
+		// guard bounds how often.
+		if (Number(job.stalledCounter) > 0 && typeof settledRecord === "function") {
+			const attempt = (Number.isInteger(job.attemptsMade) && job.attemptsMade >= 0 ? job.attemptsMade : 0) + 1;
+			// A record found and refused is said, with its fixed reason, because the job then RUNS again (paid).
+			const onReject = (reason, source) => {
+				if (firstRejection(`${job.id}\u0000${job.stalledCounter}\u0000${attempt}\u0000${source}`)) deps?.log?.("job_lost_lock_record_rejected", { jobId: job.id, reason, source });
+			};
+			const record = await settledRecord(job.id, { attempt, since: job.timestamp, onReject });
+			if (record) {
+				deps?.log?.("job_lost_lock_after_completion", { jobId: job.id, outcome: record.outcome, ...(record.reason ? { reason: record.reason } : {}) });
+				return { outcome: record.outcome, reason: record.reason ?? null, exitCode: record.exitCode ?? null, turns: record.turns ?? null, tokens: record.tokens ?? null, budgetReserved: record.budgetReserved ?? null };
+			}
+		}
+
 		// Scoped pause windows (REQ-SCOPED-PAUSE-WINDOWS): if this job's folder/repo is inside an active pause
 		// window, DEFER it to the window end via BullMQ's delayed set -- the job keeps its identity/dedup and
 		// auto-resumes when re-picked. This is FIRST, before the kill timer, the settings read, and the budget
@@ -448,14 +558,14 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 		}
 
 		// Per-scope concurrency and the one-job-per-folder mutex (issue #242,
-		// INT-SCOPED-LIMITS-FILE-CONTRACT). LAST of the three gates, after the pause gate (a paused job must
+		// INT-SCOPED-LIMITS-FILE-CONTRACT). After the pause gate (a paused job must
 		// not burn re-check wakes) and after the wait gate (a job holding until tomorrow must not sit on a
 		// folder while it does), and STRICTLY above the `try` below, like the pause gate and for the same two
 		// reasons: a DelayedError thrown inside the try would be converted to UnrecoverableError by the
 		// catch, and a moveToDelayed rejection here must escape RAW into BullMQ's normal failed-attempt
 		// handling exactly as the pause gate's does (inside the try it would become a permanent failure
 		// plus a failure record for what was a transient blip). The limits snapshot is read ONCE here and
-		// shared with `scopedCaps` below, so the gate and the money ledger cannot disagree mid-job.
+		// shared with `scopedLedgers` below, so the gate and the money ledger cannot disagree mid-job.
 		// tryAcquire is a synchronous check-and-increment -- no await between read and take, so Node's
 		// single thread makes it atomic at any concurrency -- and the local-folder limit is a structural 1
 		// (concurrencyFor) with no file and no off-switch: the scheduler mints a cron trigger's next
@@ -483,26 +593,59 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 		}
 
 		const limits = scopedLimits();
-		const scope = canonicalScope(job.data);
-		let held = false;
-		let scopeSlot = null;
+		// The job's project (issue #499, INT-PROJECTS-FILE-CONTRACT), resolved ONCE here from one read of the projects ref,
+		// beside the limits snapshot and for its reason: the gate, the ledger and the record agree for this attempt,
+		// whatever an operator does to projects.json mid-run. A retry or a deferral is a new pickup and resolves again.
+		// Every record below this line carries it (through `recordAfterGate`); a record written before this gate carries
+		// none and is resolved from the live ref (start.mjs). An id or null, never a name.
+		const pickupProjects = projects();
+		const project = projectOf(job.data, pickupProjects);
+		// THE ONE RECORDER BELOW THE GATE, bound once, so the pickup project is a property of the path and not of each call
+		// site: every record from here on goes through it, and none can drop the field and fall back to the live ref in
+		// start.mjs, which would disagree with the pickup value exactly when projects.json was edited mid-run. A bolt in
+		// project-pickup.test.mjs refuses a bare `recordRun(` call below this line.
+		const recordAfterGate = (args) => recordRun({ ...args, project });
+		// The MATCHED ROW's scope keys both the in-process slot and the fleet lease (issue #498), the same string
+		// `budgetCapsFor` hashes below and the boot sweeper hashes from the file: a qualified `github:acme/web` row holds
+		// GitHub jobs only, a bare `acme/web` row holds every forge's under the key it always had. With no row it is the
+		// job's canonical scope, so the folder mutex is keyed exactly as before.
+		const scope = rowScopeFor(job.data, limits);
+		// THE SCOPE HOLDS, in acquire order (issue #499 part B): the repo or folder slot, then the project slot. Each is
+		// `{ key, fleet }`: the in-process slot under `key` (the row scope) and its fleet claim, or null. ONE drain gives
+		// every hold back, last first, at every exit (a deferral, the setup guard, the finally), the endpoint holds' shape:
+		// a release site that lists slots by hand is the one that forgets the slot added after it was written.
+		const scopeHolds = [];
+		// Drains the holds, so a second call releases nothing: the in-process map's release is not idempotent. The
+		// in-process half goes back synchronously, so a caller that cannot await (the setup guard) still frees every local
+		// slot before it rethrows; the fleet half is release-if-mine and awaited where it can be.
+		const releaseScopeHolds = () => {
+			const taken = scopeHolds.splice(0).reverse();
+			for (const hold of taken) inFlight.release(hold.key);
+			return Promise.all(taken.map((hold) => hold.fleet?.release?.()));
+		};
+		// Give every hold back (scope and host), then defer. Every scope-gate deferral goes through here.
+		const deferScope = async (fields) => {
+			await releaseScopeHolds();
+			// The host slot goes back before we defer: `makeInFlight().release` is not idempotent, so a slot
+			// held across a deferral would be a slot this machine never gets back.
+			if (hostHeld) {
+				hostBound.slots.release(HOST_SLOT_KEY);
+				hostHeld = false;
+			}
+			deps?.log?.(fields.event, { jobId: job.id, kind: job.data?.kind === "local" ? "local" : "forge", delayMs: SCOPE_BUSY_RECHECK_MS, ...fields.extra });
+			await job.moveToDelayed(nowMs + SCOPE_BUSY_RECHECK_MS, token);
+			throw new DelayedError();
+		};
 		if (scope) {
 			const ceiling = concurrencyFor(job.data, limits);
 			if (!inFlight.tryAcquire(scope, ceiling)) {
 				// Optional-chained: makeProcessor gives `deps` no default and bare wirings pass deps: {}.
 				// The scope itself stays out of the log line (no-pii-in-logs -- a local scope is a full
 				// host path); the delayed count and the job id are what an operator needs to see it.
-				deps?.log?.("scope_busy_deferred", { jobId: job.id, kind: job.data?.kind === "local" ? "local" : "forge", delayMs: SCOPE_BUSY_RECHECK_MS });
-				// The host slot goes back before we defer: `makeInFlight().release` is not idempotent, so a slot
-				// held across a deferral would be a slot this machine never gets back.
-				if (hostHeld) {
-					hostBound.slots.release(HOST_SLOT_KEY);
-					hostHeld = false;
-				}
-				await job.moveToDelayed(nowMs + SCOPE_BUSY_RECHECK_MS, token);
-				throw new DelayedError();
+				await deferScope({ event: "scope_busy_deferred" });
 			}
-			held = true;
+			const repoHold = { key: scope, fleet: null };
+			scopeHolds.push(repoHold);
 
 			// THE FLEET-WIDE HALF of a scoped ceiling (issue #57). A `scoped-limits.json` row's day/week/month
 			// caps are already atomic INCRs on shared keys; its `concurrent` was a per-process Map, so it
@@ -519,20 +662,152 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 			// And an unlimited forge scope never claims either: `concurrencyFor` returns Infinity with no
 			// matching row, so a deployment with no scoped-limits file issues no command at all.
 			if (scopeLease && job.data?.kind !== "local" && Number.isFinite(ceiling)) {
-				scopeSlot = await scopeLease.acquire(job.id, { slots: ceiling, keyArgs: [scopeKeyPrefix(scope).slice("budget:s:".length)] });
-				if (!scopeSlot) {
-					// The local slot goes back BEFORE we defer: `makeInFlight().release` is not idempotent, so a
-					// slot held across a deferral is a slot this host never gets back.
-					inFlight.release(scope);
-					held = false;
+				repoHold.fleet = await scopeLease.acquire(job.id, { slots: ceiling, keyArgs: [hash16(scope)] });
+				if (!repoHold.fleet) await deferScope({ event: "scope_busy_deferred", extra: { where: "fleet" } });
+			}
+		}
+
+		// THE PROJECT SLOT (issue #499 part B): a project row's `concurrent` bounds every member of the project together.
+		// Taken AFTER the repo slot, in the same order for every job, so two jobs cannot each hold one and wait on the
+		// other; given back with it, last first, by the one drain above. Keyed by the project ROW's scope
+		// (`project:<id>`): the in-process slot under that string, the fleet lease under `slot:s:<hash16(project:<id>)>`,
+		// which is exactly the key the boot sweeper hashes from the row. A LOCAL member takes the fleet half too, where
+		// its own folder slot does not: a folder path carries no identity across hosts, but a project id does (every
+		// host carries the same projects.json, INT-PROJECTS-FILE-CONTRACT). A deferral, never a refusal, like the repo's.
+		const projectRow = projectRowFor(limits, project);
+		if (projectRow && Number.isSafeInteger(projectRow.concurrent)) {
+			if (!inFlight.tryAcquire(projectRow.scope, projectRow.concurrent)) {
+				await deferScope({ event: "scope_busy_deferred", extra: { ledger: "project" } });
+			}
+			const projectHold = { key: projectRow.scope, fleet: null };
+			scopeHolds.push(projectHold);
+			if (scopeLease) {
+				projectHold.fleet = await scopeLease.acquire(job.id, { slots: projectRow.concurrent, keyArgs: [hash16(projectRow.scope)] });
+				if (!projectHold.fleet) await deferScope({ event: "scope_busy_deferred", extra: { ledger: "project", where: "fleet" } });
+			}
+		}
+
+		// THE ONE SETTINGS READ (issue #503), hoisted here from the top of the main `try` below because the
+		// endpoint gate after it needs the effective provider and model, which the overlay can supply. Still ONE
+		// call per pickup: two reads could straddle an overlay edit, and the gate would then lease for one model
+		// while the container ran another. A throw is CAPTURED, never raised here: it is re-raised at the old spot
+		// inside the main `try`, so its record, its retry decision and the releases are exactly what they were.
+		let settings = null;
+		let settingsError = null;
+		let settingsThrew = false;
+		try {
+			settings = await getSettings();
+		} catch (error) {
+			settingsThrew = true;
+			settingsError = error;
+		}
+
+		// THE MODEL ENDPOINT GATE (issue #503, INT-MODEL-ENDPOINTS-FILE-CONTRACT). A declared local model server
+		// has a fixed number of parallel slots, and nothing else bounds how many jobs pile onto it: three per host
+		// on several hosts, each metering $0. So a job whose main model, or any model on its effective allowed-model
+		// list (issue #502: `run.models`, else PI_ALLOWED_MODELS), is served by a declared endpoint takes one of each
+		// such endpoint's slots here and holds it until its container is gone (`endpointSetFor`). An UNRESTRICTED job's
+		// set is its main model's alone, so its mid-run switch to another declared model is not counted, a named residual.
+		//
+		// LOCAL JOBS TAKE IT TOO, where they skip the fleet scope lease: a folder path carries no identity across
+		// hosts, but an endpoint is one physical server whoever calls it.
+		//
+		// Two halves, like the scope's: the in-process bound (`endpointSlots`, shared by both Workers like the host
+		// slot) and the fleet lease, which exists only with a declared worker name. A Valkey fault fails the fleet
+		// half OPEN, and the in-process bound underneath is then the whole bound for this host, said in the log.
+		//
+		// In ID ORDER, so two jobs on two shared endpoints cannot each hold one and wait on the other: every job
+		// asks in the same order, which is what rules the cycle out. On any miss EVERYTHING taken so far goes back
+		// (endpoint holds, the scope's fleet claim, its in-process slot, the host slot) before the deferral: an
+		// in-process release is not idempotent, so a slot kept across a deferral is one this host never gets back,
+		// and a deferred job holding slots would starve the jobs that could run.
+		//
+		// A deferral, never a refusal: a full server is transient state (CONST-RETRY-INFRA-ONLY), and it is free,
+		// decided before any token, clone or reservation (CONST-BUDGET-BEFORE-TOKENS). Skipped when the settings
+		// are unreadable or invalid: that job is refused or retried below without starting anything.
+		//
+		// One snapshot per pickup (the endpoints, the overlay models, the derived set), handed to runJob as
+		// `modelEndpoints` so a later gate reads the same declaration this one leased against. With no endpoints
+		// declared the overlay is not even read, and the job touches nothing new: no command, no key, no field.
+		const endpointHolds = [];
+		let endpointSnapshot = null;
+		// Drains the holds, so a second call releases nothing: the in-process map's release is not idempotent.
+		// The in-process half goes back synchronously, so a caller that cannot await (the setup guard) still frees
+		// every local slot before it rethrows; the fleet half is release-if-mine and awaited where it can be.
+		const releaseEndpointHolds = () => {
+			const taken = endpointHolds.splice(0).reverse();
+			for (const hold of taken) endpointSlots.release(hold.id);
+			return Promise.all(taken.map((hold) => hold.fleet?.release?.()));
+		};
+		if (modelEndpoints && !settingsThrew && !settings?.invalid) {
+			let endpoints = [];
+			let models = null;
+			let set = [];
+			// A failure to READ the overlay (any errno `readOverlayModels` rethrows), carried in the snapshot with its code,
+			// so the credential gate gives no keyless verdict on it. The model gate runs first and reads the file too: it
+			// refuses every job on a permanent errno (EACCES, EISDIR...) or a models.json that is a link, and retries a
+			// transient errno (issue #552), so the credential gate sees this only when the read failed here and not there.
+			// Absent, or not valid JSON, is determinate.
+			let modelsUnreadable = null;
+			try {
+				endpoints = modelEndpoints() ?? [];
+				if (endpoints.length > 0) {
+					try {
+						models = overlayModels();
+					} catch (err) {
+						// FAIL OPEN, and say so: an overlay models.json that does not parse names no endpoint for any model, and
+						// refusing here would refuse every job on a hosted model too. The job runs without an endpoint slot.
+						// A FIXED reason, never the error's message: a JSON.parse message quotes the file's text around the fault,
+						// and models.json holds keys (PR #518's gate measured one in this line). The settings reader's posture.
+						// An fs error is named by its code: `readOverlayModels` returns null for absence and rethrows every other
+						// errno as-is (PR #520 round 2), carried for the gate, which retries rather than calling the provider keyless.
+						// A configError is determinate: the file was read and is not JSON, or not an object (`[]`), and is named as
+						// such, never as "unreadable", which is the fs-error wording.
+						const reason = typeof err?.code === "string" ? err.code : err?.overlayLink === true ? "overlay models.json is a link" : err?.overlayNotAFile === true ? "overlay models.json is not a regular file" : /not valid JSON/.test(String(err?.message)) ? "overlay models.json is not valid JSON" : err?.piDispatchConfig === true ? "overlay models.json is not a valid models.json" : "overlay models.json is unreadable";
+						deps?.log?.("endpoint_models_unreadable", { jobId: job.id, reason });
+						if (typeof err?.code === "string") modelsUnreadable = { code: err.code };
+					}
+					// ONE hold per endpoint id: a set naming an endpoint twice would take its slot and then wait on itself,
+					// forever on `slots: 1`. Deduplicated before the sort, so the order rule sees each endpoint once.
+					const byId = new Map();
+					for (const e of endpointSetFor({ models, job: effectiveJobOf(job.data, settings, deps?.allowedModels ?? null), endpoints })) if (!byId.has(e.id)) byId.set(e.id, e);
+					set = [...byId.values()].sort((a, b) => (a.id < b.id ? -1 : a.id > b.id ? 1 : 0));
+				}
+			} catch (err) {
+				// The derivation reads data only, so this is a defect, not a state: the job runs unbounded and the log says so.
+				deps?.log?.("endpoint_gate_unavailable", { jobId: job.id, reason: scrubCredentials(err?.message) });
+				set = [];
+			}
+			endpointSnapshot = { endpoints, models, set, ...(modelsUnreadable ? { modelsUnreadable } : {}) };
+			for (const endpoint of set) {
+				let where = null;
+				let fleet = null;
+				if (!endpointSlots.tryAcquire(endpoint.id, endpoint.slots)) {
+					where = "host";
+				} else if (endpointLease && !isPerMachineHost(endpoint.host)) {
+					// A host alias NAME is a DIFFERENT server on every machine (PR #518's gate: two Macs each declaring
+					// host.docker.internal shared one fleet bound), so every other host, any address included, takes the
+					// fleet half; the in-process bound above is exact for a server only this host reaches.
+					fleet = await endpointLease.acquire(job.id, { slots: endpoint.slots, keyArgs: [hash16(endpoint.id)] });
+					if (!fleet) {
+						endpointSlots.release(endpoint.id);
+						where = "fleet";
+					} else if (fleet.degraded) {
+						deps?.log?.("endpoint_lease_degraded", { jobId: job.id, endpoint: endpoint.id });
+					}
+				}
+				if (where !== null) {
+					await releaseEndpointHolds();
+					await releaseScopeHolds();
 					if (hostHeld) {
 						hostBound.slots.release(HOST_SLOT_KEY);
 						hostHeld = false;
 					}
-					deps?.log?.("scope_busy_deferred", { jobId: job.id, kind: job.data?.kind === "local" ? "local" : "forge", delayMs: SCOPE_BUSY_RECHECK_MS, where: "fleet" });
-					await job.moveToDelayed(nowMs + SCOPE_BUSY_RECHECK_MS, token);
+					deps?.log?.("endpoint_busy_deferred", { jobId: job.id, endpoint: endpoint.id, where, delayMs: ENDPOINT_BUSY_RECHECK_MS });
+					await job.moveToDelayed(nowMs + ENDPOINT_BUSY_RECHECK_MS, token);
 					throw new DelayedError();
 				}
+				endpointHolds.push({ id: endpoint.id, fleet });
 			}
 		}
 
@@ -691,25 +966,22 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 				cancelPoll.unref?.();
 			}
 		} catch (error) {
-			// Release and CLEAR the flag: this throw never reaches the main finally below, but a shared
-			// scope must never be releasable twice -- a double release frees another holder's slot.
-			if (held) {
-				inFlight.release(scope);
-				held = false;
-			}
+			// Release and DRAIN: this throw never reaches the main finally below, but a shared scope must never be
+			// releasable twice -- a double release frees another holder's slot. Last taken, first given back.
+			void releaseEndpointHolds();
+			void releaseScopeHolds();
 			if (hostHeld) {
 				hostBound.slots.release(HOST_SLOT_KEY);
 				hostHeld = false;
 			}
-			void scopeSlot?.release?.();
-			scopeSlot = null;
 			clearTimeout(timer);
 			clearInterval(cancelPoll);
 			throw error;
 		}
 
 		try {
-			const settings = await getSettings();
+			// The read itself happened once, above the endpoint gate; its throw lands HERE, where it always did.
+			if (settingsThrew) throw settingsError;
 			if (settings.invalid) {
 				// A present-but-invalid overlay is a POLICY refusal, RETURNED (never thrown) so BullMQ marks the
 				// job completed and does not retry a file that can never parse (CONST-RETRY-INFRA-ONLY). Resolved
@@ -719,7 +991,7 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 				// refusals are the others): each is decided before or during the settings read, so no honest effective
 				// value exists yet -- buildRecord defaults both null.
 				const result = { outcome: "policy", reason: "settings-overlay-invalid", exitCode: null, turns: null, tokens: null, budgetReserved: false };
-				recordRun({ job, result, startedAt, endedAt: new Date().toISOString() });
+				recordAfterGate({ job, result, startedAt, endedAt: new Date().toISOString() });
 				return result;
 			}
 
@@ -729,17 +1001,38 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 
 			// Fill the effective job settings under `job.data > overlay > env` precedence: an explicit per-job
 			// field wins; an omitted one takes the overlay value, else env, resolved at this job's start
-			// (INT-CONFIG-OVERLAY-CONTRACT). Receiver GitHub jobs carry no provider/model/maxTurns, so this fill
-			// supplies the provider the container env allowlist requires -- absent it, the allowlist refuses a job
-			// only after its budget slot is reserved. The `caps`/`softHoldPct` passed to runJob change which
+			// (INT-CONFIG-OVERLAY-CONTRACT). A forge job carries provider/model/maxTurns only when its trigger
+			// named them (#502), so for most jobs this fill supplies the provider the container env allowlist
+			// requires -- absent it, the allowlist refuses a job only after its budget slot is reserved. The `caps`/`softHoldPct` passed to runJob change which
 			// values reserveBudget checks, never when it runs.
-			const effectiveJob = {
-				...job.data,
-				provider: job.data.provider ?? settings.provider,
-				model: job.data.model ?? settings.model,
-				maxTurns: job.data.maxTurns ?? settings.maxTurns,
-				maxTokens: job.data.maxTokens ?? settings.maxTokens, // optional per-job token budget (issue #25); null => runner meter only
-			};
+			// The one call that logs a malformed queued cap (#501): the endpoint gate above computes the same job and
+			// stays quiet, so it is reported once per pickup.
+			const effectiveJob = effectiveJobOf(job.data, settings, deps?.allowedModels ?? null, (event, fields) => deps?.log?.(event, { jobId: job.id, ...fields }));
+
+			// THE ENVELOPE (issue #504 part B, DES-DELEGATED-ALLOCATION-INSIDE-ENVELOPE). With one loaded, every job is governed:
+			// its project's, or `_other`'s, share of the applied split narrows the dollar ledgers below, read ONCE here
+			// beside the limits snapshot and the pickup project, so the gate and the reservation judge one split. The read
+			// also brings the state in line (the neutral seed, expiry, a re-base) and says when this host's envelope is not
+			// the applied split's, which the processor refuses as `envelope-mismatch` before anything is spent. A Valkey
+			// fault throws here, inside the try, and is retried like any reserve fault: nothing has started.
+			const operatorDollars = { dollarCaps: dollarWindowCaps(settings), scopedDollars: dollarCapsFor(job.data, limits), projectDollars: projectDollarCapsFor(limits, project) };
+			let dollarInputs = { ...operatorDollars };
+			const governing = allocation?.current?.() ?? null;
+			try {
+				if (governing?.envelope) {
+					const { state, mismatch } = await allocation.reconcile({ envelope: governing.envelope, digest: governing.digest, now: new Date(nowMs) });
+					dollarInputs = mismatch ? { ...operatorDollars, envelopeMismatch: true } : governedDollars({ envelope: governing.envelope, state, member: project === null ? null : { id: project, member: memberScopeOf(job.data) }, operator: operatorDollars });
+				} else if (typeof allocation?.fleetGoverned === "function" && (await allocation.fleetGoverned())) {
+					// A host with no envelope in a fleet with an applied split: ungoverned, so refused like a differing envelope.
+					dollarInputs = { ...operatorDollars, envelopeMismatch: "no-envelope" };
+				}
+			} catch (error) {
+				// INFRASTRUCTURE, never a verdict (CONST-RETRY-INFRA-ONLY): Valkey did not answer, replied with an error, or the
+				// audit file could not be written. Nothing has been reserved or started, so the job is retried rather than
+				// dropped as the UnrecoverableError a plain throw would become below.
+				deps?.log?.("allocation_read_failed", { jobId: job.id, code: typeof error?.code === "string" ? error.code : "error" });
+				throw new InfraRetry("the allocation state could not be read", { cause: error, reason: "container-never-started", provider: effectiveJob.provider ?? null, model: effectiveJob.model ?? null, budgetReserved: false });
+			}
 
 			const result = await runJob(effectiveJob, {
 				redis,
@@ -750,11 +1043,45 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 				// The daily TOKEN cap (issue #25), same overlay > env resolution. Check-AFTER, so it gates the
 				// NEXT job on prior recorded spend; null => the daily token counter is disabled.
 				tokenCap: settings.dailyTokenCap,
-				// This job's scoped budget windows (issue #242), from the SAME limits snapshot the pickup
-				// gate above read -- one read per pickup, so gate and ledger agree for this job's whole
-				// life. Null when no row carries a money window for this scope.
-				scopedCaps: budgetCapsFor(job.data, limits),
+				// This job's scoped job-count ledgers in reserve order (issues #242 and #499 part B): its repo or folder
+				// row's, then its project row's, from the SAME limits snapshot and the SAME pickup project the gate above
+				// read -- one read per pickup, so gate, ledger and record agree for this attempt. Empty when no row carries
+				// a job-count window for this job.
+				scopedLedgers: scopedLedgers(job.data, limits, project),
+				// Issue #501: the deployment's dollar windows in micro-dollars, resolved this job-start under overlay > env
+				// like the caps above, or null when none is set (then nothing is reserved and no dollar key is written).
+				dollarCaps: dollarInputs.dollarCaps,
+				// Issues #501 part 5 and #502 part 6: this job's repo or folder dollar windows and the model dollar windows it
+				// reserves in, from the SAME limits snapshot. The model rows follow the job's EFFECTIVE list (the trigger's,
+				// else PI_ALLOWED_MODELS); a job with none reserves in every model row (`modelDollarRows` says why).
+				scopedDollars: dollarInputs.scopedDollars,
+				// Issue #499 part B: the project row's dollar windows, keyed by the project row's scope, for the pickup project.
+				projectDollars: dollarInputs.projectDollars,
+				// Issue #504 part B: the project was decided on the folder AS NAMED; prepare mounts the
+				// folder it RESOLVES. The processor asks which project the resolved folder belongs to, from the same projects
+				// snapshot, and refuses before any reserve when it is another project's.
+				pickupProject: project,
+				folderProject: (folder) => projectOf({ kind: "local", folder }, pickupProjects),
+				// Issue #504 part B: `_other`'s ledger, the deployment cap's source and the envelope verdict, under an envelope;
+				// absent without one, so the processor's defaults keep such a deployment byte-identical.
+				...(dollarInputs.otherDollars ? { otherDollars: dollarInputs.otherDollars } : {}),
+				...(dollarInputs.dollarCapSource ? { dollarCapSource: dollarInputs.dollarCapSource } : {}),
+				...(dollarInputs.envelopeMismatch ? { envelopeMismatch: dollarInputs.envelopeMismatch } : {}),
+				// Issue #505: this host's envelope delegation block, for the `portfolio-no-envelope` gate; absent without an
+				// envelope, where the processor's default (null, no envelope) is the truth.
+				...(governing?.envelope ? { envelopeDelegation: governing.envelope.delegation ?? null } : {}),
+				modelDollars: modelDollarRows(limits, effectiveJob.models ?? null),
+				// The endpoint gate's snapshot (issue #503): the declared endpoints, the overlay models and this job's
+				// derived set, read once at pickup. Absent on a wiring with no endpoint seam, so a bare processor's
+				// runJob context is unchanged.
+				...(endpointSnapshot ? { modelEndpoints: endpointSnapshot } : {}),
 				...deps,
+				// Issue #502: the skew check must see `models` as the job ARRIVED, not as `effectiveJobOf` filled it from
+				// PI_ALLOWED_MODELS, or a trigger list a stale receiver dropped would read as present and the job would run on
+				// the deployment's list. Only when the wiring supplies the check, so a bare processor keeps the default.
+				// The same for `maxCostUsd` (#501): `effectiveJobOf` keeps the arrived value under its own key and writes the
+				// resolved cap as `maxCostMicros`, but it is read off `job.data` here too, so no later fill can hide a drop.
+				...(deps?.checkWaitSkew ? { checkWaitSkew: (j, ...rest) => deps.checkWaitSkew({ ...j, models: job.data?.models, maxCostUsd: job.data?.maxCostUsd }, ...rest) } : {}),
 				// #227. BOUNDED AFTER THE ABORT, and this is what makes `abortable` an honest declaration.
 				//
 				// `makeRunContainer`'s promise settles ONLY on the docker child's `close` or `error`. Nothing
@@ -811,6 +1138,9 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 				// mirroring the name/signal injection above. Omitted when unwired so a bare processor falls back
 				// to runJob's no-op default (a chain fault can never flip a completed outcome either way).
 				...(deps.collectChain ? { collectChain: (ctx) => deps.collectChain({ ...ctx, job }) } : {}),
+				// The plan collector (issue #505) the same way and for the same reason: the writer it records is the REAL job's
+				// `.id`, and the authority it checks is `.data` as queued. The pickup's `portfolio` decision rides in `ctx`.
+				...(deps.collectPlan ? { collectPlan: (ctx) => deps.collectPlan({ ...ctx, job }) } : {}),
 				// prepareWorkspace needs the REAL BullMQ job's `.id` to derive a cron job's scheduled-for
 				// instant from the deterministic repeat:<id>:<millis> jobId (DES-CRON-VIA-BULLMQ-SCHEDULER)
 				// for the local /job/event.json. runJob's own `job` is the effectiveJob -- a spread of
@@ -846,7 +1176,7 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 				deps.log?.("cancel_acked_before_result", { jobId: job.id, was: `${result?.outcome}/${result?.reason ?? ""}` });
 				outcome = { ...result, outcome: "policy", reason: "operator-cancel" };
 			}
-			recordRun({ job, result: outcome, startedAt, endedAt: new Date().toISOString() });
+			recordAfterGate({ job, result: outcome, startedAt, endedAt: new Date().toISOString() });
 			return outcome;
 		} catch (error) {
 			// Issue #448 (gate round 2 of PR #473): a local job held until rootful Podman's service restarts goes back to the
@@ -896,10 +1226,10 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 			const endCancelled = async ({ retryable }) => {
 				const spent = error?.budgetReserved === true;
 				const beforeStart = retryable && !spent;
-				const result = { outcome: "policy", reason: "operator-cancel", exitCode: spent ? (error.exitCode ?? null) : null, turns: spent ? (error.turns ?? null) : null, tokens: spent ? (error.tokens ?? null) : null, ...(spent && error.usage ? { usage: error.usage } : {}), provider: error?.provider ?? null, model: error?.model ?? null, session: error?.session ?? null, budgetReserved: retryable ? spent : (error?.budgetReserved ?? null) };
+				const result = { outcome: "policy", reason: "operator-cancel", exitCode: spent ? (error.exitCode ?? null) : null, turns: spent ? (error.turns ?? null) : null, tokens: spent ? (error.tokens ?? null) : null, ...(spent && error.usage ? { usage: error.usage } : {}), provider: error?.provider ?? null, model: error?.model ?? null, session: error?.session ?? null, budgetReserved: retryable ? spent : (error?.budgetReserved ?? null), ...(error?.dollars ? { dollars: error.dollars } : {}) };
 				deps?.log?.("job_cancelled_instead_of_retry", { jobId: job.id, spent, retryable, ...(retryable ? {} : { failure: scrubCredentials(String(error?.message ?? error)).slice(0, 300) }) });
 				if (deps?.comment) await Promise.resolve(deps.comment(job.data, beforeStart ? CANCELLED_BEFORE_START_COMMENT : TERMINAL_COMMENTS["operator-cancel"])).catch(() => {});
-				recordRun({ job, result, startedAt, endedAt: new Date().toISOString() });
+				recordAfterGate({ job, result, startedAt, endedAt: new Date().toISOString() });
 				return result;
 			};
 			// STOP THE POLL BEFORE ASKING (gate round 2 of PR #479). The poll ran until the finally below, so a request that
@@ -932,7 +1262,7 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 					throw new DelayedError();
 				}
 				const expired = Object.assign(new UnrecoverableError(`held ${Math.round((heldAt - since) / 60_000)} min for rootful Podman's service to restart, and it did not: ${error.message}`), { reason: PODMAN_RESTART_HOLD_EXPIRED, provider: error.provider ?? null, model: error.model ?? null, budgetReserved: false });
-				recordRun({ job, error: expired, startedAt, endedAt: new Date().toISOString() });
+				recordAfterGate({ job, error: expired, startedAt, endedAt: new Date().toISOString() });
 				throw expired;
 			}
 			// Issue #476: a job whose egress preflight found the rootless network keeper running on its own bridge but younger
@@ -957,14 +1287,14 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 				const was = keeperHold.startedMs ?? young.startedMs;
 				await markKeeperLoop(job, was === young.startedMs ? [was] : [was, young.startedMs]);
 				const loop = new InfraRetry(netnsKeeperCrashLoopSentence({ was, now: young.startedMs, heldMs, remedy: error.keeperRemedy }), { reason: NETNS_KEEPER_CRASH_LOOP, provider: error.provider ?? null, model: error.model ?? null, budgetReserved: false });
-				recordRun({ job, error: loop, startedAt, endedAt: new Date().toISOString() });
+				recordAfterGate({ job, error: loop, startedAt, endedAt: new Date().toISOString() });
 				throw loop;
 			}
 			if (error?.reason === NETNS_KEEPER_NOT_HOLDING && keeperHold.startedMs !== null) {
 				// The keeper this job was waiting on is no longer running on its bridge: it died young, the loop's other face.
 				await markKeeperLoop(job, [keeperHold.startedMs]);
 				const loop = new InfraRetry(netnsKeeperCrashLoopSentence({ was: keeperHold.startedMs, now: null, problem: error.keeperProblem ?? null, heldMs: keeperHold.at - keeperHold.since, remedy: error.keeperRemedy }), { reason: NETNS_KEEPER_CRASH_LOOP, provider: error.provider ?? null, model: error.model ?? null, budgetReserved: false });
-				recordRun({ job, error: loop, startedAt, endedAt: new Date().toISOString() });
+				recordAfterGate({ job, error: loop, startedAt, endedAt: new Date().toISOString() });
 				throw loop;
 			}
 			// A LATER ATTEMPT OF A JOB THAT SAW THE LOOP (gate of PR #479). The loop's retry comes after the queue's 60 s
@@ -975,10 +1305,10 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 			const loopSeen = job.data?.netnsKeeperLoopSeen;
 			if (error?.reason === NETNS_KEEPER_NOT_HOLDING && Array.isArray(loopSeen) && loopSeen.length > 0) {
 				const loop = new InfraRetry(netnsKeeperLoopAgainSentence({ seen: loopSeen, problem: error.keeperProblem ?? null, remedy: error.keeperRemedy }), { reason: NETNS_KEEPER_CRASH_LOOP, provider: error.provider ?? null, model: error.model ?? null, budgetReserved: false });
-				recordRun({ job, error: loop, startedAt, endedAt: new Date().toISOString() });
+				recordAfterGate({ job, error: loop, startedAt, endedAt: new Date().toISOString() });
 				throw loop;
 			}
-			recordRun({ job, error, startedAt, endedAt: new Date().toISOString() });
+			recordAfterGate({ job, error, startedAt, endedAt: new Date().toISOString() });
 			if (error instanceof InfraRetry) throw error; // retryable: BullMQ retries per attempts
 			// A non-retryable, non-infra error (our bug) must not retry forever. UnrecoverableError
 			// records it as failed-and-distinct in the queue's failed set without a retry.
@@ -986,14 +1316,18 @@ export function makeProcessor({ cancelJob, stopContainer, containerName = (job)
 		} finally {
 			// Release FIRST and never throw (release clamps at zero by construction): a throw here would
 			// mask the job's real error, and a missed release wedges the scope until a worker restart.
-			if (held) inFlight.release(scope);
-			if (hostHeld) hostBound.slots.release(HOST_SLOT_KEY);
+			// The in-process halves go back synchronously inside each drain, before its first await.
 			// AWAITED, not fire-and-forget. Two reasons, and the second is the one that bites: an unawaited
 			// DEL is dropped by `shutdown`'s `process.exit(0)`, stranding the claim for its whole TTL on a
 			// restart -- and the next same-scope job would otherwise race the release, be denied, and sit out a
 			// full re-check interval while the slot it wanted went free behind it. The finally is already inside
-			// an async function, and `release` never throws.
-			await scopeSlot?.release?.();
+			// an async function, and `release` never throws. Last taken, first given back: the endpoint holds
+			// (issue #503), then the scope holds (project, then repo), then the host slot.
+			const endpointsReleased = releaseEndpointHolds();
+			const scopesReleased = releaseScopeHolds();
+			if (hostHeld) hostBound.slots.release(HOST_SLOT_KEY);
+			await endpointsReleased;
+			await scopesReleased;
 			clearTimeout(timer);
 			clearInterval(cancelPoll);
 			signal.removeEventListener("abort", onAbort);
@@ -1035,7 +1369,7 @@ export function keeperHoldState(job, at) {
 	return { at, since, startedMs };
 }
 
-export function createWorker({ connection, name, stopContainer, containerName, hostQueue = null, checkLease = null, scopeLease = null, checkTimeoutMs, concurrency, getSettings, redis, deps, recordRun, limiter, pauseUntil, scopedLimits, inFlight = makeInFlight(), waitState, afterMaxMs, checkSlots = makeInFlight(), checkSlotCount, concurrencyNow, intervalMs, maxWaitMs, maxChecks, maxFaults, hostSlots = makeInFlight(), extraClosers = [] }) {
+export function createWorker({ connection, name, stopContainer, containerName, hostQueue = null, checkLease = null, scopeLease = null, checkTimeoutMs, concurrency, getSettings, redis, deps, recordRun, settledRecord = null, limiter, pauseUntil, scopedLimits, projects, allocation = null, inFlight = makeInFlight(), waitState, afterMaxMs, checkSlots = makeInFlight(), checkSlotCount, concurrencyNow, intervalMs, maxWaitMs, maxChecks, maxFaults, hostSlots = makeInFlight(), endpointSlots = makeInFlight(), endpointLease = null, modelEndpoints = null, overlayModels, extraClosers = [] }) {
 	// One Worker per queue name (issue #57). A host-affine job -- one whose folder, secret resolver or wait
 	// check lives on THIS machine -- is enqueued to `pi-jobs@<name>` rather than filtered for at pickup,
 	// because BullMQ has no selective pop and the put-it-back alternative does not work: promotion out of
@@ -1091,9 +1425,17 @@ export function createWorker({ connection, name, stopContainer, containerName, h
 			// folder mutex, the per-scope ceiling and the wait-check lease all bound the HOST, so two
 			// independent maps would double every one of them exactly as two Workers double concurrency.
 			scopedLimits,
+			projects,
+			allocation,
 			inFlight,
 			hostBound,
 			scopeLease,
+			// Issue #503: the model endpoint bound. `endpointSlots` is SHARED across both Workers for the host
+			// slot's reason: it bounds this host's calls on one server, and two maps would double it.
+			endpointSlots,
+			endpointLease,
+			modelEndpoints,
+			overlayModels,
 			// Issue #230. Undefined pass-throughs take makeProcessor's own defaults (a wait state over the same
 			// redis client, and the shared 30-day `after` ceiling), so a bare wiring behaves like a wired one.
 			waitState,
@@ -1112,6 +1454,9 @@ export function createWorker({ connection, name, stopContainer, containerName, h
 			maxFaults,
 			deps,
 			recordRun,
+			// The run-record lookup a stalled job is checked against before it can run again (see the processor's
+			// first gate). `null` in a bare wiring, which keeps today's behaviour: the job runs.
+			settledRecord,
 		});
 
 		// Issue #464: only a connection `parseConnection` built, which judges and pins the Valkey it dials.