@desplega.ai/agent-swarm 1.158.0 → 1.160.0

package/src/commands/runner.ts

@@ -36,7 +36,9 @@ import {
 import type { SteerDeliveryResult } from "../providers/types.ts";
 import { initTelemetry, telemetry } from "../telemetry.ts";
 import {
+  type ModelTierOverrides,
   type ProviderName,
+  parseWorkerModelTierOverrides,
   type ReasoningEffort,
   type RepoGuidelines,
   resolveTaskModelSelection,
@@ -44,6 +46,7 @@ import {
 } from "../types.ts";
 import { getApiKey } from "../utils/api-key.ts";
 import { computeBudgetBackoffMs } from "../utils/budget-backoff.ts";
+import { isCodexAuthFailureReason } from "../utils/codex-auth-failure.ts";
 import { getMcpBaseUrl } from "../utils/constants.ts";
 import { getContextWindowSize } from "../utils/context-window.ts";
 import {
@@ -51,16 +54,16 @@ import {
   ModelWindowExhaustedError,
   resolveCredentialPools,
 } from "../utils/credentials.ts";
-import {
-  type RateLimitWindowTelemetry,
-  resolveCodexCreditsExhaustedCooldownMs,
-} from "../utils/error-tracker.ts";
+import { resolveCodexCreditsExhaustedCooldownMs } from "../utils/error-tracker.ts";
+import { probeHarnessCliVersion, reportHarnessModelOutcome } from "../utils/harness-cli-version.ts";
 import { resolveHarnessProvider } from "../utils/harness-provider.ts";
 import { prettyPrintLine, prettyPrintStderr } from "../utils/pretty-print.ts";
 import { terminateRegisteredProcessGroups } from "../utils/process-group.ts";
+import { refreshRuntimeModelCatalog } from "../utils/runtime-model-catalog.ts";
 import { resolveScriptsOnlyMode } from "../utils/scripts-only-mode.ts";
 import { scrubSecrets } from "../utils/secret-scrubber.ts";
 import { refreshSkillsIfChanged } from "../utils/skills-refresh.ts";
+import { guardSpawnModel } from "../utils/spawn-model-guard.ts";
 import { isSteeringEnabled } from "../utils/steering-enabled.ts";
 import { interpolate } from "../utils/template.ts";
 import { detectVcsProvider } from "../vcs/index.ts";
@@ -71,6 +74,7 @@ import {
   buildResumeContextPreamble,
   prependContextPreamble,
 } from "./context-preamble.ts";
+import { reportCredentialOutcomeThenFinish } from "./credential-outcome-report.ts";
 import { type CredentialRefreshState, refreshCredentialStatus } from "./credential-refresh.ts";
 import {
   awaitCredentials,
@@ -95,7 +99,6 @@ import {
   reportLatestModel,
   sendCredStatusReport,
 } from "./provider-credentials.ts";
-import { buildFinalRateLimitWindows, classifyRateLimitOutcome } from "./rate-limit-outcome.ts";
 import {
   type ResumeSessionCandidate,
   type ResumeSessionResolution,
@@ -105,6 +108,7 @@ import {
 import "./templates.ts";
 
 export { buildAttachmentsSection } from "./attachments-section.ts";
+export { reportKeyRateLimitWindows } from "./credential-outcome-report.ts";
 
 /** Throttle interval for progress updates (3 seconds). */
 const PROGRESS_THROTTLE_MS = 3000;
@@ -773,6 +777,16 @@ export interface ResolvedEnvResult {
 // Preserve the deployment value before config reloads mutate process.env.
 const deploymentMemoryRaters = process.env.MEMORY_RATERS;
 
+/**
+ * Container value (undefined when unset) of each RELOADABLE_ENV_KEYS entry a
+ * swarm_config row has overwritten in process.env, recorded before the first
+ * overwrite. A reload starts from process.env, so without this a deleted row
+ * would leave its value in place until the container restarts. Keys no row
+ * ever overwrote are absent and keep the container value. MEMORY_RATERS keeps
+ * its own deployment snapshot above.
+ */
+const rowOverriddenBootEnv = new Map<string, string | undefined>();
+
 export async function fetchResolvedEnv(
   apiUrl: string,
   apiKey: string,
@@ -823,6 +837,11 @@ export async function fetchResolvedEnv(
         // Only reset after a successful fetch so outages retain the current value.
         env.MEMORY_RATERS =
           baseEnv === process.env ? deploymentMemoryRaters : baseEnv.MEMORY_RATERS;
+        // Same for every other key a row overwrote: start from the container
+        // value, so a row that is gone no longer applies.
+        if (baseEnv === process.env) {
+          for (const [key, bootValue] of rowOverriddenBootEnv) env[key] = bootValue;
+        }
 
         if (data.configs?.length) {
           scriptsOnlyConfigValue = data.configs.find(
@@ -1045,7 +1064,7 @@ export async function provisionAgentFsAfterRegistration(opts: {
  *
  * - STEERING_ENABLED — read per-poll by `isSteeringEnabled()` and by the
  *   system-prompt builder; flipping it mid-run just gates a feature.
- * - ANONYMIZED_TELEMETRY — read per-event by `telemetry.isEnabled()`.
+ * - ANONYMIZED_TELEMETRY — read per-event by `isTelemetryEnabled()`.
  * - MEMORY_RATERS — read per hook/prompt invocation.
  * - TEMPLATE_REGISTRY_URL — read per registry fetch.
  * - SLACK_DISABLE — read by the prompt builder to gate the Slack tool section.
@@ -1069,6 +1088,9 @@ export const RELOADABLE_ENV_KEYS: ReadonlySet<string> = new Set([
   "TEMPLATE_REGISTRY_URL",
   "SLACK_DISABLE",
   "SWARM_ORG_NAME",
+  // pi reads these from process.env for its traits and its session.
+  "PI_TOOL_DEFERRAL",
+  "PI_CODEMODE",
 ]);
 
 /**
@@ -1105,11 +1127,20 @@ export function applyResolvedEnvToProcessEnv(
   const changed: string[] = [];
   for (const key of RELOADABLE_ENV_KEYS) {
     const next = freshEnv[key];
-    if (key === "MEMORY_RATERS" && next === undefined && process.env[key] !== undefined) {
+    if (
+      next === undefined &&
+      process.env[key] !== undefined &&
+      (key === "MEMORY_RATERS" || rowOverriddenBootEnv.has(key))
+    ) {
+      // A deleted row for a key the container never set. Keys no row
+      // overwrote stay as the container set them.
       delete process.env[key];
       changed.push(key);
     } else if (next !== undefined && next !== process.env[key]) {
       const previous = process.env[key];
+      if (key !== "MEMORY_RATERS" && !rowOverriddenBootEnv.has(key)) {
+        rowOverriddenBootEnv.set(key, previous);
+      }
       process.env[key] = next;
       changed.push(key);
       // Make a reload that blanks a previously-set value loud — silently
@@ -1220,6 +1251,8 @@ const SWARM_TOOL_LABELS: Record<string, string | null> = {
   "set-config": "⚙️ Setting config",
   "list-config": "⚙️ Listing config",
   "delete-config": "⚙️ Deleting config",
+  "model-catalog-refresh": "🧠 Refreshing model catalog",
+  "model-catalog-overlay-upsert": "🧠 Updating model catalog overlay",
   // Schedules
   "create-schedule": "📅 Creating schedule",
   "list-schedules": "📅 Listing schedules",
@@ -1773,14 +1806,19 @@ export async function resolveCodexOAuthCredentialInfo(
       const slots = await loadAllCodexOAuthSlots(apiUrl, apiKey);
       if (slots.length > 0) {
         let availableIndices: number[] | undefined;
+        let authFailureFence: number | undefined;
         try {
           const resp = await fetch(
             `${apiUrl}/api/keys/available?keyType=CODEX_OAUTH&totalKeys=${slots.length}`,
             { headers: { Authorization: `Bearer ${apiKey}` } },
           );
           if (resp.ok) {
-            const data = (await resp.json()) as { availableIndices: number[] };
+            const data = (await resp.json()) as {
+              availableIndices: number[];
+              authFailureFence?: number;
+            };
             availableIndices = data.availableIndices;
+            authFailureFence = data.authFailureFence;
             if (availableIndices.length < slots.length) {
               console.log(
                 `[credentials] CODEX_OAUTH: ${availableIndices.length}/${slots.length} slots available (${slots.length - availableIndices.length} rate-limited)`,
@@ -1821,7 +1859,10 @@ export async function resolveCodexOAuthCredentialInfo(
             },
             last_refresh: new Date(slotEntry.creds.expires).toISOString(),
           };
-          const sel = authJsonToCredentialSelection(authJson, selectedSlot, slots.length);
+          const sel = {
+            ...authJsonToCredentialSelection(authJson, selectedSlot, slots.length),
+            authFailureFence,
+          };
           console.log(
             `[credentials] Selected CODEX_OAUTH slot ${selectedSlot + 1}/${slots.length} [...${sel.keySuffix}]`,
           );
@@ -1864,92 +1905,54 @@ export async function resolveCodexOAuthCredentialInfo(
   }
 }
 
-/** Report a rate-limited key to the API (fire-and-forget) */
-async function reportKeyRateLimit(
+/** Upper bound on each awaited credential-outcome report, so a slow API never stalls completions. */
+const KEY_OUTCOME_REPORT_TIMEOUT_MS = 5000;
+
+/** Report a Codex pool auth failure; the API benches after 2 in a row. Awaited, never throws. */
+async function reportKeyAuthFailure(
   apiUrl: string,
   apiKey: string,
   keyType: string,
   keySuffix: string,
   keyIndex: number,
-  rateLimitedUntil: string,
+  taskId: string,
+  timeoutMs = KEY_OUTCOME_REPORT_TIMEOUT_MS,
 ): Promise<void> {
   try {
-    await fetch(`${apiUrl}/api/keys/report-rate-limit`, {
+    const resp = await fetch(`${apiUrl}/api/keys/report-auth-failure`, {
       method: "POST",
       headers: {
         "Content-Type": "application/json",
         Authorization: `Bearer ${apiKey}`,
       },
-      body: JSON.stringify({
-        keyType,
-        keySuffix,
-        keyIndex,
-        rateLimitedUntil,
-      }),
+      body: JSON.stringify({ keyType, keySuffix, keyIndex, taskId }),
+      signal: AbortSignal.timeout(timeoutMs),
     });
+    if (!resp.ok) throw new Error(`HTTP ${resp.status}`);
+    const data = (await resp.json()) as {
+      consecutiveAuthFailures: number;
+      benched: boolean;
+      rateLimitedUntil: string | null;
+    };
     console.log(
-      `[credentials] Reported key ...${keySuffix} as rate-limited until ${rateLimitedUntil}`,
-    );
-  } catch {
-    // Non-blocking
-  }
-}
-
-/**
- * Reports rate-limit window telemetry for a key. Returns the underlying
- * fetch promise (does not swallow errors) so a caller that needs the post to
- * complete before the task finishes (a model-scoped block) can await it and
- * decide how to handle a failure; a caller that wants the legacy
- * fire-and-forget behavior appends `.catch(() => {})`.
- *
- * Throws on a non-2xx response so a failed persistence surfaces to the
- * caller instead of logging success while only the in-process guard took
- * effect — otherwise other workers redraw the same exhausted key at once.
- *
- * `logKeySuffix` defaults to true for the legacy full-telemetry call site;
- * the model-scoped call site passes false since it already logs the model
- * family and key index itself (see the `[credential] model window ...`
- * log above the call).
- */
-export async function reportKeyRateLimitWindows(
-  apiUrl: string,
-  apiKey: string,
-  keyType: string,
-  keySuffix: string,
-  keyIndex: number,
-  windows: RateLimitWindowTelemetry,
-  logKeySuffix = true,
-): Promise<void> {
-  if (Object.keys(windows).length === 0) return;
-  const response = await fetch(`${apiUrl}/api/keys/report-rate-limit-windows`, {
-    method: "POST",
-    headers: {
-      "Content-Type": "application/json",
-      Authorization: `Bearer ${apiKey}`,
-    },
-    body: JSON.stringify({
-      keyType,
-      keySuffix,
-      keyIndex,
-      windows,
-    }),
-  });
-  if (!response.ok) {
-    throw new Error(
-      `Failed to report rate-limit windows for key #${keyIndex}: HTTP ${response.status}`,
+      `[credentials] Auth failure on ...${keySuffix}: ${data.consecutiveAuthFailures} in a row${
+        data.benched ? `; benched until ${data.rateLimitedUntil}` : ""
+      }`,
     );
-  }
-  if (logKeySuffix) {
-    console.log(`[credentials] Reported rate-limit windows for key ...${keySuffix}`);
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    console.warn(scrubSecrets(`[credentials] Failed to report auth failure: ${message}`));
   }
 }
 
-/** Clear a stale rate-limit record after a successful task (fire-and-forget) */
+/** Clear a stale rate-limit record after a successful task. Bounded, never throws. */
 async function reportKeyClearRateLimit(
   apiUrl: string,
   apiKey: string,
   keyType: string,
   keySuffix: string,
+  authFence: number | undefined,
+  timeoutMs = KEY_OUTCOME_REPORT_TIMEOUT_MS,
 ): Promise<void> {
   try {
     const resp = await fetch(`${apiUrl}/api/keys/clear-rate-limit`, {
@@ -1958,7 +1961,11 @@ async function reportKeyClearRateLimit(
         "Content-Type": "application/json",
         Authorization: `Bearer ${apiKey}`,
       },
-      body: JSON.stringify({ keyType, keySuffix }),
+      // A task that exited 0 proves the login works, so it may lift an auth bench.
+      // `authFence` (read before the task started) stops a late (timed-out) report,
+      // from this or any other worker, from clearing failures recorded after it.
+      body: JSON.stringify({ keyType, keySuffix, clearAuthBench: true, authFence }),
+      signal: AbortSignal.timeout(timeoutMs),
     });
     if (resp.ok) {
       const data = (await resp.json()) as { cleared?: boolean };
@@ -1973,6 +1980,49 @@ async function reportKeyClearRateLimit(
   }
 }
 
+/**
+ * Report a finished task's credential outcome: a success resets the auth-failure
+ * count (and lifts an auth bench); a counted Codex auth failure adds to it.
+ * Awaited by the caller before it processes the next completion, so the API sees
+ * success and failure reports in completion order. A success carries the
+ * `authFence` the task read before it started: when its report times out and
+ * lands late, the API keeps every auth failure recorded after that fence.
+ * Without a fence, a success clears only an ordinary rate limit. Bounded, never throws.
+ */
+export async function reportKeyCompletionOutcome(opts: {
+  apiUrl: string;
+  apiKey: string;
+  credential: { keyType: string; keySuffix: string; keyIndex: number; authFence?: number };
+  taskId: string;
+  exitCode: number;
+  failureReason: string | undefined;
+  timeoutMs?: number;
+}): Promise<void> {
+  const { apiUrl, apiKey, credential, taskId, exitCode, failureReason, timeoutMs } = opts;
+  if (exitCode === 0) {
+    await reportKeyClearRateLimit(
+      apiUrl,
+      apiKey,
+      credential.keyType,
+      credential.keySuffix,
+      credential.authFence,
+      timeoutMs,
+    );
+    return;
+  }
+  if (credential.keyType === "CODEX_OAUTH" && isCodexAuthFailureReason(failureReason)) {
+    await reportKeyAuthFailure(
+      apiUrl,
+      apiKey,
+      credential.keyType,
+      credential.keySuffix,
+      credential.keyIndex,
+      taskId,
+      timeoutMs,
+    );
+  }
+}
+
 /**
  * Supersede a task via the API (for graceful shutdown / context-limit /
  * operator-triggered). Returns `{ ok: true, resumeTaskId }` on success.
@@ -2342,7 +2392,7 @@ export interface RunnerOptions {
 }
 
 /** Running task state for parallel execution */
-interface RunningTask {
+export interface RunningTask {
   taskId: string;
   session: ProviderSession;
   logFile: string;
@@ -2361,6 +2411,8 @@ interface RunningTask {
     keyType: string;
     keySuffix: string;
     keyIndex: number;
+    /** `authFailureFence` read before the task started (Codex pool only). */
+    authFence?: number;
   };
   /**
    * Harness provider this session was actually spawned/resumed on, snapshotted
@@ -2392,7 +2444,7 @@ interface RunningTask {
 }
 
 /** Runner state for tracking concurrent tasks */
-interface RunnerState {
+export interface RunnerState {
   activeTasks: Map<string, RunningTask>;
   maxConcurrent: number;
   startedAt: number;
@@ -2909,6 +2961,16 @@ interface PollOptions {
   pollInterval: number;
   pollTimeout: number;
   since?: string; // Optional: for filtering finished tasks
+  /** Live harness provider; keys the MODEL_TIER_* overrides sent with the poll. */
+  harnessProvider?: ProviderName;
+}
+
+/**
+ * The worker's own MODEL_TIER_* overrides, parsed from its process env (not
+ * the swarm_config-merged env: global config is already visible server-side).
+ */
+function workerModelTierOverrides(provider: ProviderName): ModelTierOverrides {
+  return parseWorkerModelTierOverrides(process.env, provider);
 }
 
 type RequesterProfile = NonNullable<Trigger["requestedBy"]>;
@@ -2940,6 +3002,86 @@ export async function buildRequesterProfilePrompt(
   return result.skipped ? "" : result.text.trim();
 }
 
+/** A non-2xx answer from `POST /api/agents`; carries the status for retry triage. */
+export class AgentRegistrationHttpError extends Error {
+  readonly status: number;
+  readonly body: string;
+  constructor(status: number, body: string) {
+    super(`Failed to register agent: ${status} ${body}`);
+    this.name = "AgentRegistrationHttpError";
+    this.status = status;
+    this.body = body;
+  }
+}
+
+/**
+ * Whether a boot registration failure is worth retrying. A freshly deployed
+ * API can answer `500 {"error":"database is locked"}` for minutes while boot
+ * work holds the SQLite write lock, and is unreachable while it restarts;
+ * both clear on their own. A 4xx (bad key, bad payload) never will.
+ */
+export function isRetryableRegistrationError(err: unknown): boolean {
+  if (err instanceof AgentRegistrationHttpError) {
+    return (
+      err.status >= 500 ||
+      err.status === 408 ||
+      err.status === 429 ||
+      /database is locked/i.test(err.body)
+    );
+  }
+  // Anything else escaped fetch itself: connection refused, reset, DNS, timeout.
+  return true;
+}
+
+export const REGISTRATION_RETRY_BUDGET_MS = 5 * 60_000;
+const REGISTRATION_RETRY_BASE_DELAY_MS = 2_000;
+const REGISTRATION_RETRY_MAX_DELAY_MS = 30_000;
+
+/**
+ * Boot registration with bounded exponential backoff (2s doubling to 30s,
+ * with jitter) inside a total wall-clock budget. Throws the last error on a
+ * non-retryable failure or once the next wait would overrun the budget, so
+ * the caller still exits and a restart policy can take over.
+ */
+export async function registerAgentWithRetry(
+  register: () => Promise<{ serverCapabilities?: string[] }>,
+  opts: {
+    label: string;
+    budgetMs?: number;
+    baseDelayMs?: number;
+    maxDelayMs?: number;
+    sleep?: (ms: number) => Promise<void>;
+    now?: () => number;
+  },
+): Promise<{ serverCapabilities?: string[] }> {
+  const budgetMs = opts.budgetMs ?? REGISTRATION_RETRY_BUDGET_MS;
+  const baseDelayMs = opts.baseDelayMs ?? REGISTRATION_RETRY_BASE_DELAY_MS;
+  const maxDelayMs = opts.maxDelayMs ?? REGISTRATION_RETRY_MAX_DELAY_MS;
+  const sleep = opts.sleep ?? ((ms: number) => Bun.sleep(ms));
+  const now = opts.now ?? Date.now;
+  const startedAt = now();
+  for (let attempt = 1; ; attempt++) {
+    try {
+      return await register();
+    } catch (err) {
+      if (!isRetryableRegistrationError(err)) throw err;
+      const backoff = Math.min(maxDelayMs, baseDelayMs * 2 ** (attempt - 1));
+      const delay = Math.round(backoff * (0.8 + Math.random() * 0.4));
+      const elapsed = now() - startedAt;
+      if (elapsed + delay > budgetMs) {
+        console.error(
+          `[${opts.label}] Registration still failing after ${attempt} attempts in ${Math.round(elapsed / 1000)}s; giving up`,
+        );
+        throw err;
+      }
+      console.warn(
+        `[${opts.label}] Registration attempt ${attempt} failed (${err}); retrying in ${Math.round(delay / 1000)}s`,
+      );
+      await sleep(delay);
+    }
+  }
+}
+
 /** Register agent via HTTP API. Exported so tests can exercise the real boot ordering. */
 export async function registerAgent(opts: {
   apiUrl: string;
@@ -2990,12 +3132,14 @@ export async function registerAgent(opts: {
       provider,
       harness_provider: harnessProvider,
       runtimeInstanceId: opts.runtimeInstanceId,
+      modelTierOverrides: workerModelTierOverrides(harnessProvider),
+      harnessCliVersion: (await probeHarnessCliVersion(harnessProvider)) ?? undefined,
     }),
   });
 
   if (!response.ok) {
     const error = await response.text();
-    throw new Error(`Failed to register agent: ${response.status} ${error}`);
+    throw new AgentRegistrationHttpError(response.status, error);
   }
 
   // The register response carries the SERVER's enabled capability flags (which
@@ -3035,6 +3179,14 @@ async function pollForTrigger(opts: PollOptions): Promise<Trigger | null> {
 
 async function pollForTriggerOnce(opts: PollOptions): Promise<Trigger | null> {
   const startTime = Date.now();
+  // Keep the process-wide model catalog fresh (TTL-cached, never throws) so
+  // context windows, pricing and reasoning levels cover models added to the
+  // API's catalog after this worker booted.
+  void refreshRuntimeModelCatalog({
+    apiUrl: opts.apiUrl,
+    apiKey: opts.apiKey,
+    agentId: opts.agentId,
+  });
   const headers: Record<string, string> = {
     "X-Agent-ID": opts.agentId,
   };
@@ -3045,6 +3197,11 @@ async function pollForTriggerOnce(opts: PollOptions): Promise<Trigger | null> {
     headers.Authorization = `Bearer ${opts.apiKey}`;
   }
   injectTraceContext(headers);
+  if (opts.harnessProvider) {
+    headers["X-Model-Tier-Overrides"] = encodeURIComponent(
+      JSON.stringify(workerModelTierOverrides(opts.harnessProvider)),
+    );
+  }
 
   while (Date.now() - startTime < opts.pollTimeout) {
     try {
@@ -3527,6 +3684,9 @@ async function spawnProviderProcess(
     taskId?: string;
     model?: string;
     modelTier?: string;
+    /** Server claim-time resolution (task.resolvedModel); wins over the local one. */
+    resolvedModel?: string;
+    modelSource?: string;
     effort?: ReasoningEffort;
     resumeSessionId?: string;
     harnessProvider: ProviderName;
@@ -3562,7 +3722,7 @@ async function spawnProviderProcess(
       opts.apiKey,
       opts.agentId,
       process.env,
-      opts.model,
+      opts.resolvedModel || opts.model,
       {
         repoId: sessionRepo?.id,
         provider: adapter.name as ProviderName,
@@ -3573,8 +3733,7 @@ async function spawnProviderProcess(
     ));
   } catch (err) {
     if (err instanceof ModelWindowExhaustedError && realTaskId) {
-      const modelLabel = err.model.charAt(0).toUpperCase() + err.model.slice(1);
-      const reason = `No ${err.keyType} key has ${modelLabel} capacity until ${err.earliestResetAt ?? "unknown"}. Re-dispatch with another model or modelTier.`;
+      const reason = err.message;
       console.warn(`[${opts.role}] ${reason}`);
       await ensureTaskFinished(
         { apiUrl: opts.apiUrl, apiKey: opts.apiKey, agentId: opts.agentId },
@@ -3619,8 +3778,44 @@ async function spawnProviderProcess(
     harnessProvider: opts.harnessProvider,
     env: freshEnv,
   });
-  const taskModel = taskModelSelection.model || "";
-  const model = taskModel || configModel || "";
+  // The server resolves the model at claim time (worker-env > tier-config >
+  // tier-default, `latest:` aliases, guardrails) and records it on the task.
+  // The local resolution stays as a check: a difference usually means the
+  // server saw a stale MODEL_TIER_* override or a swarm_config tier value.
+  if (
+    opts.resolvedModel &&
+    taskModelSelection.model &&
+    taskModelSelection.model !== opts.resolvedModel
+  ) {
+    console.log(
+      `[${opts.role}] model resolution mismatch for task ${opts.taskId ?? "?"}: server=${opts.resolvedModel} (${opts.modelSource ?? "?"}) local=${taskModelSelection.model}; using server value`,
+    );
+  }
+  const taskModel = opts.resolvedModel || taskModelSelection.model || "";
+  // Never start a CLI with a model from another harness family (runbooks/model-tiers.md).
+  const spawnModel = guardSpawnModel({
+    taskModel,
+    configModel,
+    harness: opts.harnessProvider,
+    role: opts.role,
+  });
+  if (spawnModel.kind === "mismatch") {
+    console.warn(`[${opts.role}] ${spawnModel.reason}`);
+    if (realTaskId) {
+      await ensureTaskFinished(
+        { apiUrl: opts.apiUrl, apiKey: opts.apiKey, agentId: opts.agentId },
+        opts.role,
+        realTaskId,
+        1,
+        spawnModel.reason,
+        undefined,
+        opts.harnessProvider,
+      );
+    }
+    throw new Error(spawnModel.reason);
+  }
+  if (spawnModel.warning) console.warn(spawnModel.warning);
+  const model = spawnModel.model;
 
   // Resolve Codex OAuth pool slot BEFORE building ProviderSessionConfig so we
   // can pass codexSlot through and the adapter writes token refreshes back to
@@ -4389,6 +4584,7 @@ async function spawnProviderProcess(
         keyType: primarySelection.keyType,
         keySuffix: primarySelection.keySuffix,
         keyIndex: primarySelection.index,
+        authFence: primarySelection.authFailureFence,
       }
     : undefined;
 
@@ -4428,8 +4624,8 @@ async function spawnProviderProcess(
   return runningTask;
 }
 
-/** Check for completed processes and remove them from active tasks */
-async function checkCompletedProcesses(
+/** Check for completed processes and remove them from active tasks. Exported for tests. */
+export async function checkCompletedProcesses(
   state: RunnerState,
   role: string,
   apiConfig?: ApiConfig,
@@ -4517,77 +4713,26 @@ async function checkCompletedProcesses(
         console.log(`[${role}] Detected error for task ${taskId.slice(0, 8)}: ${failureReason}`);
       }
 
-      // If rate-limited and we know which key was used, report it.
-      // Codex adapter prefixes failure reasons with `[rate-limit]` /
-      // `[usage-limit]` (see codex-adapter.formatTerminalError); Claude
-      // surfaces "rate limit" / "hit your limit" via SessionErrorTracker.
-      //
-      // classifyRateLimitOutcome tests model-scoped windows (Fable/Opus/
-      // Sonnet weekly limits) before the legacy key-wide gate, so a
-      // model-scoped rejection blocks only that model family on this key —
-      // never the whole key — via report-rate-limit-windows instead of
-      // report-rate-limit. A key-wide rejection seen in the same session is
-      // still reported alongside it (windows are independent). The session's
-      // window telemetry and the classified model rejection go out as ONE
-      // payload, so an older `allowed` snapshot never overwrites the terminal
-      // rejection. The post is awaited so it completes before the task
-      // finishes.
+      // Record whether this worker's CLI accepts the model (harness_model_support).
+      void reportHarnessModelOutcome({
+        apiUrl: apiConfig.apiUrl,
+        apiKey: apiConfig.apiKey,
+        agentId: apiConfig.agentId,
+        harness: harnessProvider,
+        model,
+        exitCode: result.exitCode,
+        failureReason,
+      });
+
       if (credentialInfo) {
-        const outcome = classifyRateLimitOutcome(
-          result,
+        await reportKeyCompletionOutcome({
+          apiUrl: apiConfig.apiUrl,
+          apiKey: apiConfig.apiKey,
+          credential: credentialInfo,
+          taskId,
+          exitCode: result.exitCode,
           failureReason,
-          Date.now(),
-          state.codexCreditsExhaustedCooldownMs,
-        );
-        const keyRateLimitedUntil =
-          outcome.kind === "key"
-            ? outcome.rateLimitedUntil
-            : outcome.kind === "model"
-              ? outcome.keyRateLimitedUntil
-              : undefined;
-        if (keyRateLimitedUntil) {
-          console.log(`[credentials] Rate limit reset: ${keyRateLimitedUntil}`);
-          reportKeyRateLimit(
-            apiConfig.apiUrl,
-            apiConfig.apiKey,
-            credentialInfo.keyType,
-            credentialInfo.keySuffix,
-            credentialInfo.keyIndex,
-            keyRateLimitedUntil,
-          ).catch(() => {});
-        }
-        if (outcome.kind === "model") {
-          const resetsAtIso = new Date(outcome.resetsAtSec * 1000).toISOString();
-          const blockKey = `${credentialInfo.keyType}:${credentialInfo.keyIndex}:${outcome.window}`;
-          state.modelWindowBlocks.set(blockKey, outcome.resetsAtSec * 1000);
-          console.log(
-            `[credential] ${outcome.model} weekly window exhausted (${outcome.window}) on key #${credentialInfo.keyIndex} until ${resetsAtIso}`,
-          );
-        }
-
-        const finalWindows = buildFinalRateLimitWindows(
-          result.rateLimitWindows,
-          outcome,
-          new Date().toISOString(),
-        );
-        if (finalWindows) {
-          const report = reportKeyRateLimitWindows(
-            apiConfig.apiUrl,
-            apiConfig.apiKey,
-            credentialInfo.keyType,
-            credentialInfo.keySuffix,
-            credentialInfo.keyIndex,
-            finalWindows,
-            outcome.kind !== "model",
-          ).catch((err) => {
-            console.warn(
-              `[credential] Failed to report rate-limit windows: ${err instanceof Error ? err.message : String(err)}`,
-            );
-          });
-          // A model rejection gates admission on other workers: land it before
-          // the task finishes. Plain telemetry stays fire-and-forget.
-          if (outcome.kind === "model") await report;
-        }
+        });
       }
       let bridgeDiagnostics: Awaited<ReturnType<typeof getBridgeFailureDiagnostics>> | undefined;
       if (result.exitCode !== 0 && harnessProvider === "claude" && workingDir) {
@@ -4603,19 +4748,34 @@ async function checkCompletedProcesses(
         bridgeDiagnostics?.paneTail != null
           ? `Claude bridge final tmux pane tail (${bridgeDiagnostics.artifactPath}):\n${bridgeDiagnostics.paneTail}`
           : undefined;
-      await ensureTaskFinished(
-        apiConfig,
-        role,
-        taskId,
-        result.exitCode,
-        failureReason,
-        // Runner-buffered last assistant text is a harness-agnostic fallback
-        // for adapters that never populate `ProviderResult.output` (Codex
-        // today, any future adapter). Empty buffer -> `undefined`, byte
-        // identical to pre-fix behavior.
-        resolveProviderOutput(result, assistantText),
-        harnessProvider,
-        bridgeFailureDiagnostics,
+      // Reports that gate admission on other workers (a seat mismatch, a
+      // model window rejection) land before the task finishes.
+      await reportCredentialOutcomeThenFinish(
+        {
+          apiUrl: apiConfig.apiUrl,
+          apiKey: apiConfig.apiKey,
+          credentialInfo,
+          result,
+          failureReason,
+          model,
+          codexCreditsExhaustedCooldownMs: state.codexCreditsExhaustedCooldownMs,
+          modelWindowBlocks: state.modelWindowBlocks,
+        },
+        () =>
+          ensureTaskFinished(
+            apiConfig,
+            role,
+            taskId,
+            result.exitCode,
+            failureReason,
+            // Runner-buffered last assistant text is a harness-agnostic fallback
+            // for adapters that never populate `ProviderResult.output` (Codex
+            // today, any future adapter). Empty buffer -> `undefined`, byte
+            // identical to pre-fix behavior.
+            resolveProviderOutput(result, assistantText),
+            harnessProvider,
+            bridgeFailureDiagnostics,
+          ),
       );
 
       telemetry.taskEvent("session_completed", {
@@ -4646,15 +4806,6 @@ async function checkCompletedProcesses(
       }
       state.tasksProcessed += 1;
 
-      if (result.exitCode === 0 && credentialInfo) {
-        reportKeyClearRateLimit(
-          apiConfig.apiUrl,
-          apiConfig.apiKey,
-          credentialInfo.keyType,
-          credentialInfo.keySuffix,
-        ).catch(() => {});
-      }
-
       ensure({
         id: "worker_process_finished",
         flow: "task",
@@ -5271,18 +5422,22 @@ export async function runAgent(config: RunnerConfig, opts: RunnerOptions) {
     }
   };
   try {
-    const reg = await registerAgent({
-      apiUrl,
-      apiKey,
-      agentId,
-      name: agentName,
-      role,
-      isLead,
-      capabilities,
-      maxTasks: maxConcurrent,
-      harnessProvider: bootProvider,
-      runtimeInstanceId,
-    });
+    const reg = await registerAgentWithRetry(
+      () =>
+        registerAgent({
+          apiUrl,
+          apiKey,
+          agentId,
+          name: agentName,
+          role,
+          isLead,
+          capabilities,
+          maxTasks: maxConcurrent,
+          harnessProvider: bootProvider,
+          runtimeInstanceId,
+        }),
+      { label: role },
+    );
     lastServerCapsRefreshAt = Date.now();
     // Rebuilds the prompt immediately: the initial build above ran before
     // registration (serverCapabilities unknown), and the later identity
@@ -5911,6 +6066,8 @@ export async function runAgent(config: RunnerConfig, opts: RunnerOptions) {
               taskId: task.id,
               model: (task as { model?: string }).model,
               modelTier: (task as { modelTier?: string }).modelTier,
+              resolvedModel: (task as { resolvedModel?: string }).resolvedModel,
+              modelSource: (task as { modelSource?: string }).modelSource,
               effort: (task as { effort?: ReasoningEffort }).effort,
               harnessProvider: state.harnessProvider,
               cwd: resumeCwd,
@@ -6127,6 +6284,7 @@ export async function runAgent(config: RunnerConfig, opts: RunnerOptions) {
         pollInterval: PollIntervalMs,
         runtimeInstanceId,
         pollTimeout: effectiveTimeout,
+        harnessProvider: state.harnessProvider,
       });
 
       if (trigger) {
@@ -6407,6 +6565,16 @@ export async function runAgent(config: RunnerConfig, opts: RunnerOptions) {
           .join("\n\n");
         const taskSystemPrompt = taskPromptParts + cwdWarning;
 
+        // The server refused the task's explicit model for this worker's CLI
+        // version (harness_model_support = unsupported): fail fast, no spawn.
+        const modelUnsupported = (trigger.task as { modelUnsupported?: string } | undefined)
+          ?.modelUnsupported;
+        if (trigger.taskId && modelUnsupported) {
+          console.log(`[${role}] ${modelUnsupported}`);
+          await ensureTaskFinished(apiConfig, role, trigger.taskId, 1, modelUnsupported);
+          continue;
+        }
+
         iteration++;
         const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
         const taskIdSlice = trigger.taskId?.slice(0, 8) || "notask";
@@ -6472,6 +6640,9 @@ export async function runAgent(config: RunnerConfig, opts: RunnerOptions) {
               taskId: trigger.taskId,
               model: taskModel,
               modelTier: taskModelTier,
+              resolvedModel: (trigger.task as { resolvedModel?: string } | undefined)
+                ?.resolvedModel,
+              modelSource: (trigger.task as { modelSource?: string } | undefined)?.modelSource,
               effort: taskEffort,
               harnessProvider: state.harnessProvider,
               cwd: effectiveCwd,