@edgehero/pi-dispatch 4.3.0 → 5.0.0

package/src/mcp-preflight.mjs

@@ -35,7 +35,7 @@
  */
 import { allowlistEntryFor, egressRefusalFor, packageRunnerHosts } from "./egress-allowlist.mjs";
 import { FORGE_MCP_KINDS, FORGE_MCP_SERVER_NAME, forgeMcpEgressUrl, forgeMcpServer, gitlabUrlProblem, jobMcpDocument, MCP_SERVER_NAME_RE, readOverlayMcp } from "./mcp-config.mjs";
-import { MCP_SERVERS_MAX, mcpServerToolPattern } from "./triggers.mjs";
+import { MCP_SERVERS_MAX, RESERVED_ENV_NAMES, mcpServerToolPattern } from "./triggers.mjs";
 
 /**
  * Errnos a read of the overlay may pass on a second try. Restated from model-catalog.mjs `OVERLAY_TRANSIENT_READ_CODES`
@@ -78,6 +78,23 @@ export function jobProviders(job, { defaultProvider = null, allowedModels = null
 	return [...new Set([job?.provider ?? defaultProvider, ...listed].filter((p) => typeof p === "string" && p !== ""))];
 }
 
+/**
+ * The names a job's own container holds for itself, which no server may be handed: what `makeMcpCheck`'s `reserved` answers,
+ * one function for the worker's pickup and the console's preview. `readModels` reads the overlay models.json. A read that may
+ * pass on a retry is thrown to the check, which retries the job; any other failure adds no overlay names, and the model gates,
+ * which run first, refuse every job such a file breaks.
+ */
+export function reservedJobNames(job, { readModels, defaultProvider = null, allowedModels = null, everyProvider = [], candidatesOf = () => [] }) {
+	let overlayModels = null;
+	try {
+		overlayModels = readModels();
+	} catch (error) {
+		if (MCP_TRANSIENT_READ_CODES.has(error?.code)) throw error;
+	}
+	const providers = jobProviders(job, { defaultProvider, allowedModels, everyProvider });
+	return new Set([...RESERVED_ENV_NAMES, ...providerKeyNames({ providers, candidatesOf, overlayModels })]);
+}
+
 /**
  * The check, bound to this deployment. `reserved(job)` is the set of variable names the job's own container holds for
  * itself; `hostEnv` is the worker's environment, which decides whether a PI_FORWARD_ENV name reaches the job at all;
@@ -221,9 +238,10 @@ function exposuresOf(server) {
  * `autoEnableCodemode` is not false) and tool_search when some server has a deferred one; the resource tools take their
  * server's own exposure, and a `hidden` server's resources are never offered. Judged only for what the trigger's
  * exclusions take away: a server the overlay leaves no way in is the overlay's, not this trigger's to be refused for.
- * Returns `{ name, why }` for the first server the exclusions cut off, or null.
+ * Returns a `{ name, why }` for each server the exclusions cut off, in the list's order. The console says each one it did
+ * not already cut (`trigger-environment-plans.mjs`); a pickup refuses on the first, `mcpRouteProblem`.
  */
-export function mcpRouteProblem(servers, names, excluded, { autoEnableCodemode = true } = {}) {
+export function mcpRouteCuts(servers, names, excluded, { autoEnableCodemode = true } = {}) {
 	const routes = (removed) => {
 		const all = names.map((name) => exposuresOf(servers[name]));
 		const has = (kind) => all.some(({ own, tools }) => own === kind || tools.includes(kind));
@@ -242,8 +260,12 @@ export function mcpRouteProblem(servers, names, excluded, { autoEnableCodemode =
 	};
 	const before = routes(new Set());
 	const after = routes(excluded);
-	const i = names.findIndex((_, k) => before[k] && !after[k]);
-	return i < 0 ? null : { name: names[i], why: "run.excludeTools removes every tool that reaches it (its own, the discovery tools that reach them, and read_mcp_resource)" };
+	return names.filter((_, k) => before[k] && !after[k]).map((name) => ({ name, why: "run.excludeTools removes every tool that reaches it (its own, the discovery tools that reach them, and read_mcp_resource)" }));
+}
+
+/** The first server `mcpRouteCuts` finds, or null: what a pickup refuses on. */
+export function mcpRouteProblem(servers, names, excluded, options) {
+	return mcpRouteCuts(servers, names, excluded, options)[0] ?? null;
 }
 
 /**
@@ -260,3 +282,12 @@ export const MCP_REFUSAL_COMMENT = Object.freeze({
 	"mcp-egress-unreachable": "the egress policy does not let a job reach that server",
 	"mcp-package-runner": "the egress policy does not let a job reach the registry that server is fetched from",
 });
+
+/**
+ * The whole public comment of an MCP refusal before starting: the server's NAME when the check named one (operator
+ * configuration, the same class as the image ref on the image refusals) and the reason's clause above. One function
+ * for the processor, which posts it, and the reason catalog (issue #710), which reads the nameless form as its sentence.
+ */
+export function mcpRefusalComment(refused, name) {
+	return `Refused: this trigger names MCP server${name ? ` "${name}"` : "s"} in \`run.mcp\`, and ${MCP_REFUSAL_COMMENT[refused] ?? "it cannot be used"}. The job would have run without the servers its flow was written against; the worker's log says why (\`refused_mcp\`). Not run.`;
+}

package/src/model-attempts.mjs

@@ -0,0 +1,84 @@
+/**
+ * A job's model attempt key (`INT-MODEL-COOLDOWN-KEY`): `dispatch:model-attempt:<sha256(queueName:jobId)>`, the model the
+ * job's last attempt ran on and whether its retry may switch away from it. The processor writes it before each attempt
+ * (model-failover.mjs `writeModelAttempt`); this module names it, ends it and keeps it. Light on purpose, `node:crypto`
+ * and the retention leaf only, because `cancel-state.mjs`, which the admin imports, deletes it too, and the admin's retry
+ * keeps it.
+ *
+ * ITS LIFE FOLLOWS THE JOB'S (issue #812), the rule the approval requests keep (`run:approval:request:*`):
+ *   - while a retry may still come it has no expiry. A retry can wait for any time (a paused queue, a backoff, a hold),
+ *     so an expiry set when the key is written could take the evidence away from a retry that still comes;
+ *   - once the queue says the job finished (the worker's `completed` event, or `failed` with `finishedOn` set, start.mjs)
+ *     it gets `MODEL_ATTEMPT_FINISHED_TTL_MS`, a day past a forge job's retention, so it outlives the finished job's own
+ *     hash, from which a console retry can still bring the job back. Never when a run record is written: the record is
+ *     written before the queue acknowledges the job, and a job whose acknowledgement is lost can still run again;
+ *   - when a cancel or the host-lost sweep removes the job, it is deleted;
+ *   - when the console retries a failed job, the expiry is taken off again (`keepModelAttempt`), as the job is back
+ *     where a retry may come for any time.
+ * A retry's write is a plain `SET`, which clears the expiry again.
+ */
+import { createHash } from "node:crypto";
+import { MODEL_ATTEMPT_FINISHED_TTL_MS } from "./retention.mjs";
+
+/** The key of the attempt kept under `id` (`modelAttemptIdOf`). */
+export const modelAttemptKey = (id) => `dispatch:model-attempt:${createHash("sha256").update(id).digest("hex")}`;
+
+/** The id a job's attempt is kept under: its queue's name and its job id, as the processor names it. */
+export const modelAttemptIdOf = (queueName, jobId) => `${queueName}:${jobId}`;
+
+/**
+ * Give a finished job's attempt its expiry. `PEXPIRE` sets nothing on a key that is absent, so a job that never chose a
+ * model (no `fallbackModels`) creates nothing here. Rejects as the client does; the caller logs it.
+ */
+export async function finishModelAttempt(redis, id) {
+	await redis.pexpire(modelAttemptKey(id), MODEL_ATTEMPT_FINISHED_TTL_MS);
+}
+
+/**
+ * Send one `command` on the key `keyOf()` names, bounded by `timeoutMs` and best-effort: it never throws (a client
+ * without the command, a key that cannot be named, a throw, a rejection and a reply that never comes all leave the key as
+ * it was), and the bound's timer is cleared when the command answers first, so a CLI that sets its exit code rather than
+ * exiting is not held open by it. The one body behind every bounded write below and cancel-state.mjs's defer record.
+ */
+async function bestEffort(redis, command, keyOf, timeoutMs) {
+	if (typeof redis?.[command] !== "function") return;
+	let timer;
+	try {
+		await Promise.race([
+			Promise.resolve().then(() => redis[command](keyOf())),
+			new Promise((resolve) => {
+				timer = setTimeout(resolve, timeoutMs);
+			}),
+		]);
+	} catch {
+		// Left as it was: see above.
+	} finally {
+		clearTimeout(timer);
+	}
+}
+
+/** Delete `key`, bounded and best-effort (`bestEffort`): cancel-state.mjs deletes a removed job's defer record with it. */
+export function dropKey(redis, key, timeoutMs = 2_000) {
+	return bestEffort(redis, "del", () => key, timeoutMs);
+}
+
+/**
+ * Delete a removed job's attempt, bounded and best-effort (`bestEffort`). A delete that fails leaves the key as the job
+ * left it, without expiry.
+ */
+export function dropModelAttempt(redis, id, timeoutMs = 2_000) {
+	return bestEffort(redis, "del", () => modelAttemptKey(id), timeoutMs);
+}
+
+/**
+ * Take a retried job's attempt's expiry off again, bounded and best-effort (`bestEffort`): the console's retry of a
+ * failed job (admin job-actions.mjs) calls it once BullMQ has put the job back. The failure gave the key
+ * `MODEL_ATTEMPT_FINISHED_TTL_MS`, and only the processor's next selection writes it again without one; the retried job
+ * can wait longer than what is left of it before that write (a paused queue or pause window, a busy scope, a hold for a
+ * newer release), and would then find no evidence and be refused `model-failover-no-candidate`. `PERSIST` on an absent
+ * key creates nothing, so a job without fallbacks, or one whose key already expired, is left as it was; so is a key a
+ * `PERSIST` that fails could not reach, which keeps its expiry.
+ */
+export function keepModelAttempt(redis, id, timeoutMs = 2_000) {
+	return bestEffort(redis, "persist", () => modelAttemptKey(id), timeoutMs);
+}

package/src/model-catalog.mjs

@@ -297,3 +297,86 @@ export function declaredFallbacks(provider, id, overlay = null) {
 	}
 	return found ?? [];
 }
+
+/**
+ * The runner's own input overhead (`BOUND_OVERHEAD_TOKENS`, image/runner/src/usage-meter.mjs), which its bound adds to
+ * every call. Copied, because the worker package does not ship the runner; `fleet-dollars.test.mjs` holds the two equal.
+ */
+export const FIRST_CALL_OVERHEAD_TOKENS = 8192;
+/**
+ * The least a job's FIRST call carries before its task, in bytes: at the pin, pi's default system prompt, which names pi's install
+ * path, and the schemas of its four default tools serialise to about 5 KB, and a short task's first request is 10,667 bytes. 4,096
+ * is below both, so the bound below stays a lower bound for a job that adds nothing. `worker/test/first-call-bytes.pinned.test.mjs`
+ * computes that against the pinned pi, and holds the order.
+ */
+export const FIRST_CALL_MIN_CONTEXT_BYTES = 4096;
+/**
+ * 10,667 bytes, a short task's first request. A cap at the floor above admits a request nobody sends: the same default prompt
+ * and tools alone come to about 5 KB, and a one-line task adds little. So this, not the floor, is the least a person can be told to set. A repository's context files, which go in the system prompt before any task
+ * does, and a longer task, need more still.
+ */
+export const FIRST_CALL_SHORT_TASK_BYTES = 10_667;
+
+/**
+ * A LOWER BOUND, in integer micro-dollars, on what the runner's cost guard (`callCostBound`) reserves for a job's
+ * first call on `model`, or null when this cannot say. The guard refuses a call when its bound would pass the cap, so
+ * a per-job cap below this number can never admit a call on the model: with the main model, the job makes no call at
+ * all (the default model's first call has been bounded at $1.03 under a $1 cap).
+ *
+ * The guard's bound is (request bytes + 8,192) x the dearest input rate + the output limit x the dearest output rate,
+ * times a service-tier multiplier, over the model's table, its tiers and its fallbacks. This takes the parts of that
+ * which cannot be smaller: the model's own table (not its tiers or fallbacks, which only raise the bound), the output
+ * limit `model.maxTokens` (pi sends none of its own, so the guard uses the model's), an input of the overhead plus
+ * `FIRST_CALL_MIN_CONTEXT_BYTES`, and the runner's own service-tier multiplier (`serviceTierMultiplier`). The input
+ * rate is the guard's: the highest of input, cache read and cache write (not the 1h write, which needs long
+ * retention). Null for a table or limit it cannot read.
+ */
+export function firstCallFloorMicros(model) {
+	return firstCallBoundMicros(model, FIRST_CALL_MIN_CONTEXT_BYTES);
+}
+
+/**
+ * The bound `firstCallFloorMicros` describes, for a request of `contextBytes` serialised, or null when this cannot say. At
+ * `FIRST_CALL_SHORT_TASK_BYTES` it is what a short task's first call needs from the guard. Held equal to the runner's own bound
+ * at that size, for every builtin chat model, by `fleet-dollars.test.mjs`.
+ */
+export function firstCallBoundMicros(model, contextBytes) {
+	const cost = model?.cost;
+	const usable = (v) => typeof v === "number" && Number.isFinite(v) && v >= 0;
+	if (!cost || !["input", "output", "cacheRead", "cacheWrite"].every((field) => usable(cost[field]))) return null;
+	const maxTokens = model.maxTokens;
+	if (!(typeof maxTokens === "number" && Number.isFinite(maxTokens) && maxTokens > 0)) return null;
+	const inputRate = Math.max(cost.input, cost.cacheRead, cost.cacheWrite);
+	return Math.ceil(((FIRST_CALL_OVERHEAD_TOKENS + contextBytes) * inputRate + maxTokens * cost.output) * serviceTierMultiplier(model));
+}
+
+/**
+ * The runner's service-tier multiplier (`callCostBound` step 7, image/runner/src/usage-meter.mjs), copied like the
+ * overhead above: pi multiplies a settled cost by the tier the RESPONSE reports on these two apis, which can be the
+ * account's default rather than the request's, so the runner always bounds at the dearest tier. Leaving it out
+ * (PR #551's review) let doctor stay silent on a cap the runner refuses every call under: openai/gpt-5.5 under $5.
+ * `fleet-dollars.test.mjs` derives the runner's multiplier from its own bound for every priced api and holds this
+ * equal to it.
+ */
+export function serviceTierMultiplier(model) {
+	if (model?.api !== "openai-responses" && model?.api !== "openai-codex-responses") return 1;
+	return model.id === "gpt-5.5" ? 2.5 : 2;
+}
+
+/**
+ * The model object the floor above is computed from: the builtin catalog's (`builtinModel`), or a
+ * CUSTOM model the overlay `models.json` defines (its own `cost` and `maxTokens`). Null, so the model is not judged,
+ * when the overlay redefines or overrides a builtin model: pi composes the two, and this does not copy that rule.
+ */
+export function boundModelOf({ provider, id }, { builtinModel, overlay = null }) {
+	const providers = overlay?.providers;
+	const entry = providers !== null && typeof providers === "object" && !Array.isArray(providers) && Object.hasOwn(providers, provider) ? providers[provider] : null;
+	const builtin = builtinModel(provider, id);
+	if (entry !== null && typeof entry === "object") {
+		const overrides = entry.modelOverrides;
+		if (overrides !== null && typeof overrides === "object" && Object.hasOwn(overrides, id)) return null;
+		const definition = Array.isArray(entry.models) ? entry.models.find((m) => m !== null && typeof m === "object" && m.id === id) : undefined;
+		if (definition !== undefined) return builtin ? null : definition;
+	}
+	return builtin;
+}

package/src/model-cooldowns.mjs

@@ -0,0 +1,22 @@
+/** Read-only configured cooldown observations, shared by failover and console. No keyspace enumeration. */
+import { createHash } from "node:crypto";
+import { providerFailureOf } from "./model-failover-protocol.mjs";
+
+export const MODEL_COOLDOWN_MAX_MS = 6 * 60 * 60 * 1000;
+export const MODEL_RECHECK_MS = 60_000;
+const digest = (value) => createHash("sha256").update(value).digest("hex");
+export const modelCooldownKey = (ref) => `dispatch:model-cooldown:${digest(ref)}`;
+
+export async function readModelCooldown(redis, ref, { now = () => Date.now() } = {}) {
+	const raw = await redis.get(modelCooldownKey(ref));
+	if (raw === null || raw === undefined) return null;
+	let row;
+	try { row = JSON.parse(raw); } catch { throw new Error("invalid model cooldown"); }
+	if (!providerFailureOf(row) || !Number.isSafeInteger(row.until) || row.until < 0 || row.until > 253_402_300_799_999 || row.kind === "context-overflow") throw new Error("invalid model cooldown");
+	return row.until > now() ? { ref, kind: row.kind, until: row.until } : null;
+}
+
+export async function readModelCooldowns(redis, refs, options = {}) {
+	const rows = await Promise.all([...new Set(refs)].map((ref) => readModelCooldown(redis, ref, options)));
+	return rows.filter(Boolean);
+}

package/src/model-failover-protocol.mjs

@@ -0,0 +1,11 @@
+export const PROVIDER_FAILURE_KINDS = Object.freeze(["rate-limited", "quota-exhausted", "overloaded", "context-overflow"]);
+export const MODEL_WHY = Object.freeze(["primary", "fallback-cooldown", "fallback-window", "fallback-unavailable", "fallback-context"]);
+
+export function providerFailureOf(value) {
+	if (!value || typeof value !== "object" || Array.isArray(value) || !PROVIDER_FAILURE_KINDS.includes(value.kind)) return null;
+	if (value.resetAt !== undefined && (!Number.isSafeInteger(value.resetAt) || value.resetAt < 0 || value.resetAt > 253_402_300_799_999)) return null;
+	if (value.subscriptionCode !== undefined && !["usage_limit_reached", "usage_not_included", "rate_limit_exceeded", "quota_exhausted"].includes(value.subscriptionCode)) return null;
+	if (value.subscriptionCode !== undefined && !["openai", "anthropic"].includes(value.subscriptionProvider)) return null;
+	if (value.subscriptionProvider !== undefined && !["openai", "anthropic"].includes(value.subscriptionProvider)) return null;
+	return { kind: value.kind, ...(value.resetAt !== undefined ? { resetAt: value.resetAt } : {}), ...(value.subscriptionCode ? { subscriptionCode: value.subscriptionCode } : {}), ...(value.subscriptionProvider ? { subscriptionProvider: value.subscriptionProvider } : {}) };
+}

package/src/model-failover.mjs

@@ -0,0 +1,134 @@
+import { resolveBackendName } from "./backend-registry.mjs";
+import { MODEL_WHY, providerFailureOf } from "./model-failover-protocol.mjs";
+import { failoverListUsable, modelListProblem, modelOnList, splitModelEntry } from "./model-ref.mjs";
+import { modelDollarRows } from "./scoped-limits.mjs";
+import { dayKey, weekKey, monthKey } from "./budget.mjs";
+import { builtinModel, checkModelsKnown } from "./model-catalog.mjs";
+import { egressRefusalFor } from "./egress-allowlist.mjs";
+import { LOCAL_FOLDER_HOLDS_SUBSCRIPTION_AUTH } from "./private-store-boundary.mjs";
+
+export { MODEL_WHY, PROVIDER_FAILURE_KINDS, providerFailureOf } from "./model-failover-protocol.mjs";
+
+export function failoverEgressRequired(job, { egress = false, defaultBackend = "local" } = {}) {
+	return egress || ["openshell", "openshell-bind"].includes(resolveBackendName(job, defaultBackend));
+}
+
+import { MODEL_COOLDOWN_MAX_MS, MODEL_RECHECK_MS, modelCooldownKey, readModelCooldown, readModelCooldowns } from "./model-cooldowns.mjs";
+import { FAILOVER_PRICED_APIS, failoverModelView, failoverPriceEligible } from "./model-status.mjs";
+export { MODEL_COOLDOWN_MAX_MS, MODEL_RECHECK_MS, modelCooldownKey, readModelCooldown, readModelCooldowns };
+export { FAILOVER_PRICED_APIS, failoverModelView };
+// The key and its life live in model-attempts.mjs (issue #812); re-exported, so an import of it from here keeps working.
+import { modelAttemptKey } from "./model-attempts.mjs";
+export { modelAttemptKey };
+
+export async function markModelCooldown(redis, ref, failure, { now = () => Date.now() } = {}) {
+	const signal = providerFailureOf(failure);
+	if (!signal || signal.kind === "context-overflow") return null;
+	const at = now();
+	const delay = signal.resetAt === undefined ? (signal.kind === "quota-exhausted" ? MODEL_COOLDOWN_MAX_MS : MODEL_RECHECK_MS) : Math.max(1000, signal.resetAt - at);
+	const ttl = Math.min(MODEL_COOLDOWN_MAX_MS, delay);
+	const row = { kind: signal.kind, until: at + ttl };
+	// Concurrent observations must never shorten an already published provider reset. One atomic script also
+	// prevents two workers from both reading an old reset and then overwriting the longer one in reverse order.
+	await redis.eval(`local old = redis.call('GET', KEYS[1]); if old then local ok, row = pcall(cjson.decode, old); if ok and type(row['until']) == 'number' and row['until'] > tonumber(ARGV[1]) then return 0 end end; redis.call('SET', KEYS[1], ARGV[2], 'PX', ARGV[3]); return 1`, 1, modelCooldownKey(ref), row.until, JSON.stringify(row), ttl);
+	return { ref, ...row };
+}
+
+export async function readModelAttempt(redis, id) {
+	const raw = await redis.get(modelAttemptKey(id));
+	if (raw === null || raw === undefined) return null;
+	let row;
+	try { row = JSON.parse(raw); } catch { throw new Error("invalid model attempt"); }
+	if (!splitModelEntry(row?.ref) || typeof row.safe !== "boolean" || (row.safe && !providerFailureOf(row.failure))) throw new Error("invalid model attempt");
+	return { ref: row.ref, safe: row.safe && providerFailureOf(row.failure) !== null, failure: providerFailureOf(row.failure), ...(MODEL_WHY.includes(row.modelWhy) ? { modelWhy: row.modelWhy } : {}) };
+}
+
+export async function writeModelAttempt(redis, id, { ref, safe = false, failure = null, modelWhy = null }) {
+	if (!splitModelEntry(ref)) throw new Error("invalid model attempt ref");
+	await redis.set(modelAttemptKey(id), JSON.stringify({ ref, safe: safe === true, failure: providerFailureOf(failure), ...(MODEL_WHY.includes(modelWhy) ? { modelWhy } : {}) }));
+}
+
+export async function checkFailoverCandidate(job, { models = null, endpoints = [], credential, egress = false, allowlist = [], lookup = builtinModel } = {}) {
+	// The second ref makes the existing declared-provider-fallback check enforce this attempt's singleton too.
+	const known = checkModelsKnown([{ provider: job.provider, id: job.model, main: true }, { provider: job.provider, id: job.model }], { readOverlay: () => models });
+	if (known.unavailable) throw new Error("model catalog unavailable");
+	if (known.unknown) return "model-unknown";
+	if (known.fallbackUnlisted) return "model-not-allowed";
+	const view = failoverModelView(`${job.provider}/${job.model}`, { models, lookup });
+	if (!failoverPriceEligible(view)) return "model-unpriced";
+	if (credential) {
+		const checked = await credential(job, { modelEndpoints: { models, endpoints } });
+		if (checked.unavailable) throw new Error("provider credential unavailable");
+		if (!checked.ok) return "provider-unconfigured";
+	}
+	if (egress && egressRefusalFor(view.baseUrl, { allowlist, endpoints }) !== null) return "model-host-unlisted";
+	return null;
+}
+
+async function modelWindowHasRoom(redis, ref, job, limits, at) {
+	if (job.maxCostMicros === 0) return true;
+	for (const row of modelDollarRows(limits, [ref])) {
+		// reserveDollars remains the atomic authority. This read chooses a candidate without acquiring money;
+		// its later reservation may lose a race, which must refuse rather than run with an uncovered window.
+		if (!Number.isSafeInteger(job.maxCostMicros)) return false;
+		for (const [window, key] of [["day", dayKey(new Date(at), row.keyPrefix)], ["week", weekKey(new Date(at), row.keyPrefix)], ["month", monthKey(new Date(at), row.keyPrefix)]]) {
+			const cap = row.caps?.[window];
+			if (cap === null || cap === undefined) continue;
+			const raw = await redis.get(key);
+			const spent = raw === null || raw === undefined ? 0 : Number(raw);
+			if (!Number.isSafeInteger(spent) || spent < 0) throw new Error("invalid model window");
+			if (spent + job.maxCostMicros > cap) return false;
+		}
+	}
+	return true;
+}
+
+/** Only opt-in jobs enter here. Every candidate has a singleton executable policy beneath the reviewed ceiling. */
+export async function selectFailoverJob(job, { redis, id, retry = false, recoveryModel = null, retryNeverStarted = false, models = null, limits = [], now = () => Date.now(), checkCandidate, subscriptionCheck = null, reservationFor = (candidate) => candidate.maxCostMicros, acquireEndpoints = async () => true, modelView = (ref) => failoverModelView(ref, { models }), log = () => {} } = {}) {
+	if (!failoverListUsable(job)) return { refused: "model-failover-no-candidate" };
+	const primary = `${job.provider}/${job.model}`;
+	const previous = retry ? await readModelAttempt(redis, id) : null;
+	if (retry && !previous && !retryNeverStarted && !recoveryModel) return { refused: "model-failover-no-candidate" };
+	const locked = recoveryModel ?? (retry && previous?.safe !== true ? previous?.ref ?? primary : null);
+	const needsLarger = !locked && previous?.failure?.kind === "context-overflow";
+	const largerThan = needsLarger ? modelView(previous.ref)?.contextWindow : null;
+	let why = previous?.failure?.kind === "context-overflow" ? "fallback-context" : "fallback-unavailable";
+	let wait = null;
+	const primaryAuth = subscriptionCheck ? await subscriptionCheck({ ...job, models: [primary] }) : null;
+	// Changing models cannot make a local mount stop exposing the host's refresh store.
+	if (primaryAuth?.reason === LOCAL_FOLDER_HOLDS_SUBSCRIPTION_AUTH) return { refused: LOCAL_FOLDER_HOLDS_SUBSCRIPTION_AUTH };
+	let subscriptionPrimary = primaryAuth?.managed === true || Array.isArray(primaryAuth?.managed) && primaryAuth.managed.length > 0;
+	for (const ref of [...new Set(locked ? [locked] : [primary, ...job.fallbackModels])]) {
+		const split = splitModelEntry(ref);
+		if (!split || !modelOnList(job.models, split.provider, split.model)) { log("model_candidate_skipped", { model: split ? ref : null, reason: "model-not-allowed" }); continue; }
+		const candidate = { ...job, provider: split.provider, model: split.model, models: [ref] };
+		if (subscriptionCheck) {
+			const subscription = await subscriptionCheck(candidate);
+			if (subscription.unavailable) throw new Error("subscription state unavailable");
+			if (ref === primary && (subscription.managed === true || Array.isArray(subscription.managed) && subscription.managed.length > 0)) subscriptionPrimary = true;
+			if (subscriptionPrimary && subscription.managed !== true && !(Array.isArray(subscription.managed) && subscription.managed.length > 0)) {
+				log("model_candidate_skipped", { model: ref, reason: "auth-mixed-billing" });
+				continue;
+			}
+			if (!subscription.ok) {
+				if (subscription.wait) wait ??= subscription.wait;
+				log("model_candidate_skipped", { model: ref, reason: subscription.wait ?? subscription.reason ?? "auth-unavailable" });
+				continue;
+			}
+		}
+		const bad = await checkCandidate(candidate);
+		if (bad) { log("model_candidate_skipped", { model: ref, reason: bad }); continue; }
+		if (needsLarger && (!Number.isFinite(largerThan) || !(modelView(ref)?.contextWindow > largerThan))) { log("model_candidate_skipped", { model: ref, reason: "context-too-small" }); continue; }
+		if (!locked) {
+			const cooldown = await readModelCooldown(redis, ref, { now });
+			const failedHere = previous?.safe === true && previous.ref === ref && previous.failure?.kind !== "context-overflow";
+			if (cooldown || failedHere) { if (ref === primary) why = "fallback-cooldown"; if (!failedHere) wait ??= "model-cooldown"; log("model_candidate_skipped", { model: ref, reason: failedHere ? "previous-attempt" : "cooldown" }); continue; }
+		}
+		const reservationJob = { ...candidate, maxCostMicros: reservationFor(candidate) };
+		if (modelDollarRows(limits, [ref]).length > 0 && !Number.isSafeInteger(reservationJob.maxCostMicros)) { log("model_candidate_skipped", { model: ref, reason: "model-cost-cap-missing" }); continue; }
+		if (!(await modelWindowHasRoom(redis, ref, reservationJob, limits, now()))) { if (ref === primary) why = "fallback-window"; wait ??= "model-window"; log("model_candidate_skipped", { model: ref, reason: "model-window" }); continue; }
+		if (!(await acquireEndpoints(candidate))) { wait ??= "endpoint"; log("model_candidate_skipped", { model: ref, reason: "endpoint-busy" }); continue; }
+		return { job: { ...candidate, modelWhy: ref === primary ? "primary" : locked ? previous?.modelWhy ?? "fallback-unavailable" : why }, ref };
+	}
+	return wait ? { wait, until: now() + MODEL_RECHECK_MS } : { refused: "model-failover-no-candidate" };
+}

package/src/model-observation.mjs

@@ -0,0 +1,50 @@
+/** Model policy observations use pickup's resolver and never acquire a slot, reserve or start work. */
+import { realpathSync } from "node:fs";
+import { isAbsolute } from "node:path";
+import { projectOf } from "./projects.mjs";
+import { splitModelEntry } from "./model-ref.mjs";
+import { resolveModelRoles } from "./model-roles.mjs";
+
+/** The same role resolver as pickup, expanded over every project a broad forge trigger can reach. */
+export function resolveConfiguredModelRuns({ runs = [], deployment, envList = null, modelRoles = {}, projects = [], resolveFolder = realpathSync, conservativeLocal = false }) {
+	const resolved = [];
+	const checks = [];
+	for (const run of runs) {
+		if (Object.keys(modelRoles).length === 0 && !run.model?.startsWith("role:")) { resolved.push(run); continue; }
+		const list = run.models ?? envList;
+		const job = { ...run, provider: run.provider ?? deployment.provider, model: run.model ?? deployment.model, ...(list ? { models: list } : {}) };
+		// Forge events without a fully qualified repository can reach any matching project. Azure
+		// run.repository alone is only a repository name, not enough to infer a project-qualified scope.
+		let projectUnknown = false;
+		if (job.kind === "local" && typeof job.folder === "string") {
+			try { job.folder = resolveFolder(job.folder); } catch { projectUnknown = conservativeLocal; }
+		}
+		const fixed = run.kind === "local" || typeof run.repo === "string";
+		const possible = projectUnknown ? [null, ...projects.filter(project => project.members?.some(member => typeof member === "string" && isAbsolute(member))).map(project => project.id)] : fixed ? [projectOf(job, projects)] : [null, ...projects.filter((project) => project.members?.some((member) => typeof member === "string" && member.startsWith(`${run.kind}:`))).map((project) => project.id)];
+		for (const project of new Set(possible)) {
+			const label = project === null ? run.label : `${run.label} (project ${project})`;
+			try {
+				const selected = resolveModelRoles(job, modelRoles, project, { authoredProvider: Object.hasOwn(run, "authoredProvider") ? run.authoredProvider : run.provider });
+				resolved.push({ ...selected, label, project, ...(projectUnknown ? { projectUnknown: true } : {}), roleScope: selected.modelRole && Object.hasOwn(modelRoles?.[project] ?? {}, selected.modelRole) ? project : "default" });
+			} catch (error) {
+				checks.push({ ok: false, warn: true, sourceIndex: run.sourceIndex ?? null, project, ...(projectUnknown ? { projectUnknown: true } : {}), reason: error.modelSelectionReason ?? "model-roles-invalid", label: `${label}: reviewed model roles refuse this job (${error.modelSelectionReason ?? "model-roles-invalid"})`, fix: "review triggers.json modelRoles and this trigger's provider/model/allowed list; project roles override defaults" });
+			}
+		}
+	}
+	return { runs: resolved, checks };
+}
+
+/** The configured failover models, never arbitrary keyspace members. Non-failover jobs add no read. */
+export function cooldownModelRefs(runs, deployment, { modelRoles = {}, projects = [], envList = null } = {}) {
+	const refs = new Set();
+	for (const run of resolveConfiguredModelRuns({ runs, deployment, envList, modelRoles, projects }).runs) {
+		if (!Array.isArray(run.fallbackModels) || run.fallbackModels.length === 0) continue;
+		for (const ref of [`${run.provider ?? deployment.provider}/${run.model ?? deployment.model}`, ...run.fallbackModels]) {
+			if (splitModelEntry(ref)) refs.add(ref);
+		}
+	}
+	return [...refs];
+}
+
+export const RADIUS_COST_CAP_LABEL = "Radius (pi-messages) has no verified local cost bound, including zero-rate routes and custom gateways, so the runner refuses every call under a per-job cost cap as unboundable";
+export const RADIUS_COST_CAP_FIX = "use a provider with a locally bounded API for capped jobs; raising the cap or changing catalog rates cannot bound Radius: docs/costs.md";

package/src/model-proposal-protocol.mjs

@@ -0,0 +1,21 @@
+/** Closed proposal metadata. A generic successful doctor check never gains publishable prose. */
+import { splitModelEntry, providerIdProblem } from "./model-ref.mjs";
+import { MODEL_ROLE_RE } from "./model-roles.mjs";
+import { isProjectId } from "./project-id.mjs";
+export const MODEL_PROPOSAL_KINDS = Object.freeze(["retirement", "newer-sibling", "cheaper-route", "catalog-checked", "unavailable"]);
+export function modelProposalOf(value) {
+	if (!value || typeof value !== "object" || Array.isArray(value) || !MODEL_PROPOSAL_KINDS.includes(value.kind)) return null;
+	const out = { kind: value.kind };
+	for (const key of ["model", "candidate"]) if (value[key] !== undefined) {
+		if (!splitModelEntry(value[key])) return null;
+		out[key] = value[key];
+	}
+	if (value.scope !== undefined) { if (!isProjectId(value.scope)) return null; out.scope = value.scope; }
+	if (value.role !== undefined) { if (!MODEL_ROLE_RE.test(value.role) || ["inherit", "constructor", "prototype"].includes(value.role)) return null; out.role = value.role; }
+	if (value.provider !== undefined) { if (providerIdProblem(value.provider) !== null) return null; out.provider = value.provider; }
+	if (["retirement", "newer-sibling"].includes(out.kind) && (!out.model || !out.scope || !out.role)) return null;
+	if (["newer-sibling", "cheaper-route"].includes(out.kind) && !out.candidate) return null;
+	if (out.kind === "cheaper-route" && !out.model) return null;
+	if (out.kind === "catalog-checked" && !out.provider) return null;
+	return out;
+}

package/src/model-proposals.mjs

@@ -0,0 +1,75 @@
+/** Reviewed suggestions only (#717). Data enters here; no routing or settings writes leave it. */
+
+const REF = /^(?=.{1,256}$)[~@]?[A-Za-z0-9][A-Za-z0-9._:/@_-]*$/;
+export function catalogDiff(before, after, providers) {
+	const index = (rows) => {
+		const out = new Map();
+		for (const row of rows) {
+			if (!row || typeof row.provider !== "string" || typeof row.id !== "string" || !REF.test(row.provider) || !REF.test(row.id) || !providers.includes(row.provider) || row.type !== "chat") continue;
+			const key = `${row.provider}/${row.id}`;
+			if (out.has(key)) throw new Error("ambiguous model catalog");
+			out.set(key, row);
+		}
+		return out;
+	};
+	const old = index(before), next = index(after);
+	const changed = [];
+	for (const key of [...new Set([...old.keys(), ...next.keys()])].sort()) {
+		const a = old.get(key), b = next.get(key);
+		const canonical = (value) => Array.isArray(value) ? value.map(canonical) : value && typeof value === "object" ? Object.fromEntries(Object.keys(value).sort().map((key) => [key, canonical(value[key])])) : value;
+		const rates = (row) => JSON.stringify(canonical(row?.cost ?? null));
+		if (!a || !b || rates(a) !== rates(b)) changed.push({ ref: key, kind: !a ? "added" : !b ? "removed" : "repriced", before: a ?? null, after: b ?? null });
+	}
+	return changed;
+}
+
+/** Same worst-call bound as enforcement, for an explicitly disclosed empty-text fixture, not a job forecast. */
+export function proposalCallBound(model, outputTokens = 4096, callCostBound) {
+	const micros = callCostBound("streamSimple", model, { messages: [] }, { maxTokens: outputTokens }, {});
+	return Number.isFinite(micros) ? micros / 1e6 : null;
+}
+
+export function renderCatalogProposal(changes, { providers, capsUsd, outputTokens = 4096 }, callCostBound) {
+	const lines = ["## Model catalog proposals", "", `Public report scope: ${providers.join(", ")}. Reference caps: ${capsUsd.map((n) => `$${n}`).join(", ")}; these are public report inputs, not deployment caps.`, "", `First-call bounds use the runner's guard with an empty-text context and maxTokens=${outputTokens}; actual job prompts, images, thinking and options can increase the bound. Models remain unchanged until a reviewed edit.`, ""];
+	if (!changes.length) lines.push("No added, removed or repriced chat models in this scope.");
+	for (const change of changes) {
+		const bound = change.after ? proposalCallBound(change.after, outputTokens, callCostBound) : null;
+		const price = bound === null ? "not boundable" : `$${bound.toFixed(6)}`;
+		const beforeBound = change.before ? proposalCallBound(change.before, outputTokens, callCostBound) : null;
+		const previous = change.before ? `; previous reference bound: ${beforeBound === null ? "not boundable" : `$${beforeBound.toFixed(6)}`}` : "";
+		lines.push(`- ${change.kind}: \`${change.ref}\`; reference first-call bound: ${change.after ? price : "removed"}${previous}${bound === null ? "" : `; ${capsUsd.map((cap) => `${bound <= cap ? "fits" : "exceeds"} $${cap}`).join(", ")}`}.`);
+	}
+	return lines.join("\n");
+}
+
+/** Provider fields verified against their Models API references, 2026-10-29. No name-based sibling guesses. */
+export function lifecycleModels(provider, response) {
+	if (!Array.isArray(response?.data) || response.data.length > 1000) throw new Error("invalid lifecycle catalog");
+	return response.data.map((row) => {
+		if (!row || typeof row.id !== "string" || !REF.test(row.id)) throw new Error("invalid lifecycle model");
+		const date = provider === "anthropic" ? row.retires_at : row.shutdown_date;
+		if (date != null && (typeof date !== "string" || !Number.isFinite(Date.parse(date)))) throw new Error("invalid retirement date");
+		if (provider === "anthropic" && !["active", "deprecated", "retired"].includes(row.lifecycle)) throw new Error("invalid lifecycle stage");
+		return { id: row.id, retiresAt: date ?? null, deprecated: provider === "anthropic" ? row.lifecycle !== "active" : date != null, retired: provider === "anthropic" && row.lifecycle === "retired", line: typeof row.line === "string" ? row.line : null, createdAt: row.created_at ?? null };
+	});
+}
+
+export function roleProposalChecks({ roles, catalogs = {}, replacements = {}, now, noticeDays = 90 }) {
+	const checks = [];
+	for (const [scope, mapping] of Object.entries(roles)) for (const [role, value] of Object.entries(mapping)) {
+		const ref = typeof value === "string" ? value : value?.model;
+		if (typeof ref !== "string") continue;
+		const slash = ref.indexOf("/"), provider = ref.slice(0, slash), id = ref.slice(slash + 1);
+		const rows = catalogs[provider] ?? [];
+		const model = rows.find((m) => m.id === id);
+		if (!model) continue;
+		const retirement = model.retiresAt === null ? null : Date.parse(model.retiresAt);
+		if (model.deprecated || retirement !== null && retirement - now <= noticeDays * 86400000) checks.push({ proposal: { kind: "retirement", scope, role, model: ref, ...(replacements[ref] ? { candidate: replacements[ref] } : {}) }, ok: false, warn: true, label: `Model role ${scope}/${role}: ${ref} is ${model.retired ? "retired" : "deprecated"}${model.retiresAt ? `; retirement ${model.retiresAt}` : ""}`, fix: replacements[ref] ? `review provider replacement ${replacements[ref]} with a capped canary, then edit modelRoles` : "review the provider's deprecation notice and run a capped canary before editing modelRoles" });
+		const siblings = model.line ? rows.filter((m) => m.line === model.line && !m.deprecated && Number.isFinite(Date.parse(m.createdAt)) && Date.parse(m.createdAt) > Date.parse(model.createdAt)) : [];
+		if (siblings.length) {
+			const sibling = siblings.sort((a, b) => Date.parse(b.createdAt) - Date.parse(a.createdAt))[0].id;
+			checks.push({ proposal: { kind: "newer-sibling", scope, role, model: ref, candidate: `${provider}/${sibling}` }, ok: true, label: `Model role ${scope}/${role}: newer provider-declared sibling ${sibling}; review with a capped canary` });
+		}
+	}
+	return checks;
+}