@staix/agent-hub 0.12.19 → 0.12.21

package/src/hub/tool-refusal.ts

@@ -0,0 +1,7 @@
+/** A hub guard proved this request was refused before any task or board effect (screen telemetry is allowed). Never wrap arbitrary failures in this type. */
+export class PreEffectToolRefusal extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "PreEffectToolRefusal";
+  }
+}

package/src/hub/ui.ts

@@ -1,3 +1,4 @@
+import { taskProgress } from "../ui/task-progress.ts";
 import { createHash, randomBytes } from "node:crypto";
 import { readFileSync } from "node:fs";
 
@@ -20,7 +21,9 @@ export function startDashboard(options: DashboardOptions) {
   const now = options.now ?? Date.now;
   const tickets = new Map<string, number>();
   const sessions = new Map<string, number>();
-  const html = readFileSync(new URL("../ui/index.html", import.meta.url), "utf8");
+  // Inject the same pure public-board model used by the console before computing CSP hashes.
+  const html = readFileSync(new URL("../ui/index.html", import.meta.url), "utf8")
+    .replace("/* TASK_PROGRESS_MODEL */", () => `const taskProgress = ${taskProgress.toString()};`);
   const hashes = (tag: string) => [...html.matchAll(new RegExp(`<${tag}>([\\s\\S]*?)</${tag}>`, "g"))]
     .map((m) => `'sha256-${createHash("sha256").update(m[1]!).digest("base64")}'`).join(" ");
   const headers = {

package/src/local/deny.ts

@@ -4,7 +4,8 @@ import { basename, sep } from "node:path";
  * The one denylist. tools.ts checks paths against it, sandbox.ts turns it into seatbelt rules, capture.ts keeps
  * matching calls out of memory. Two lists would drift, and the gap is where a secret leaks.
  */
-export const DENY_SEGMENTS = [".maru/secrets", ".agenthub/state"];
+// `.agenthub/archive` holds the state directories `ahub reset --all` moved there: hub.db with task text, PII included.
+export const DENY_SEGMENTS = [".maru/secrets", ".agenthub/state", ".agenthub/archive"];
 /** Regex sources, matched against a file's basename. */
 export const DENY_NAMES = ["^\\.env", "\\.pem$", "\\.key$", "^id_rsa", "^id_ed25519", "^credentials(\\.|$)", "^auth\\.json$", "^\\.netrc$", "^\\.npmrc$", "^\\.pypirc$"];
 

package/src/local/tools.ts

@@ -1,7 +1,8 @@
 import { lstatSync, mkdirSync, readFileSync, statSync, writeFileSync } from "node:fs";
 import { dirname, join, relative, resolve } from "node:path";
 import { realPath } from "../hub/project.ts";
-import { hasSegment, isDenied } from "./deny.ts";
+import { isHubWritePath } from "../hub/permission-mode.ts";
+import { isDenied } from "./deny.ts";
 import { OUTPUT_CAP, sandboxedExec } from "./sandbox.ts";
 
 export { isDenied };
@@ -17,14 +18,11 @@ export interface ToolContext {
   /** Turn cancellation signal, including the execution-budget wall cap. */
   signal?: AbortSignal;
   /** Ask the console. Resolves false on deny or timeout. */
-  permit: (title: string) => Promise<boolean>;
+  permit: (title: string, tool?: string, signal?: AbortSignal, path?: string) => Promise<boolean>;
   /** Publish a message to other peers mid-turn. Returns a one-line receipt. */
   send: (text: string, to?: string[]) => string;
 }
 
-/** Not secret, but writing them changes what runs outside the worker's control. */
-const WRITE_DENY_SEGMENTS = [".git", ".agenthub"];
-
 const lexists = (p: string) => {
   try {
     lstatSync(p);
@@ -52,7 +50,9 @@ export function guardPath(ctx: Pick<ToolContext, "cwd" | "deny">, path: string,
   const rel = relative(root, real);
   if (rel.startsWith("..") || resolve(root, rel) !== real) throw new Error(`${path} is outside the project directory`);
   if (isDenied(rel, ctx.deny)) throw new Error(`${path} is on the secrets denylist`);
-  if (mode === "write" && WRITE_DENY_SEGMENTS.some((s) => hasSegment(rel, s))) throw new Error(`${path} is not writable by the local worker`);
+  // Not secret, but writing .git or .agenthub changes what runs outside the worker's control. Names are compared
+  // folded: a new `.GIT/config` is `.git/config` to a case-insensitive disk, and git then runs what it names.
+  if (mode === "write" && isHubWritePath(rel)) throw new Error(`${path} is not writable by the local worker`);
   return real;
 }
 
@@ -129,23 +129,27 @@ export async function runTool(name: string, rawArgs: string, ctx: ToolContext):
       }
       case "write": {
         let file = guardPath(ctx, String(a.path), "write");
+        const approvedTarget = file;
         const content = String(a.content ?? "");
-        if (!(await ctx.permit(`write ${a.path} (${content.length} chars):\n${preview(content)}`))) return "error: the user did not approve this write";
+        if (!(await ctx.permit(`write ${a.path} (${content.length} chars):\n${preview(content)}`, "write", ctx.signal, file))) return "error: the user did not approve this write";
         if (ctx.signal?.aborted) return "error: turn cancelled before write";
         file = guardPath(ctx, String(a.path), "write");
+        if (file !== approvedTarget) return "error: path target changed during approval; inspect before retrying";
         mkdirSync(dirname(file), { recursive: true });
         writeFileSync(file, content);
         return `wrote ${a.path}`;
       }
       case "edit": {
         let file = guardPath(ctx, String(a.path), "write");
+        const approvedTarget = file;
         let text = readFileSync(file, "utf8");
         let count = text.split(String(a.old)).length - 1;
         if (!a.old || count !== 1) return `error: \`old\` must match exactly once, it matched ${count} times`;
-        if (!(await ctx.permit(`edit ${a.path}:\n- ${preview(String(a.old), 600)}\n+ ${preview(String(a.new ?? ""), 600)}`))) return "error: the user did not approve this edit";
+        if (!(await ctx.permit(`edit ${a.path}:\n- ${preview(String(a.old), 600)}\n+ ${preview(String(a.new ?? ""), 600)}`, "edit", ctx.signal, file))) return "error: the user did not approve this edit";
         if (ctx.signal?.aborted) return "error: turn cancelled before edit";
         // Approval can outlive another peer's edit or a path change. Apply only the approved fragment to current bytes.
         file = guardPath(ctx, String(a.path), "write");
+        if (file !== approvedTarget) return "error: path target changed during approval; inspect before retrying";
         text = readFileSync(file, "utf8");
         count = text.split(String(a.old)).length - 1;
         if (count !== 1) return `error: file changed during approval; \`old\` must match exactly once, it matched ${count} times`;
@@ -157,7 +161,7 @@ export async function runTool(name: string, rawArgs: string, ctx: ToolContext):
         if (!command.trim()) return "error: empty command";
         // The approver sees the whole command, not a prefix: what is hidden cannot be approved.
         if (command.length > 4000) return "error: command longer than 4000 characters; put it in a script file with write, then run that";
-        if (!(await ctx.permit(`bash: ${command}`))) return "error: the user did not approve this command";
+        if (!(await ctx.permit(`bash: ${command}`, "bash", ctx.signal))) return "error: the user did not approve this command";
         const res = await sandboxedExec(["/bin/bash", "-c", command], { cwd: ctx.cwd, profile: ctx.sandboxProfile, timeoutMs: (Number(a.timeout_s) || 120) * 1000, ...(ctx.sandboxEnv ? { env: ctx.sandboxEnv } : {}), ...(ctx.signal ? { signal: ctx.signal } : {}) });
         return `${res.output}\n(exit ${res.code})`;
       }
@@ -167,7 +171,7 @@ export async function runTool(name: string, rawArgs: string, ctx: ToolContext):
         if (!GIT_READ.has(sub) && !GIT_WRITE.has(sub)) return `error: git ${sub} is not available to the local worker`;
         const problem = gitArgsProblem(args, ctx.deny);
         if (problem) return `error: git ${sub}: ${problem}`;
-        if (GIT_WRITE.has(sub) && !(await ctx.permit(`git ${args.join(" ")}`))) return "error: the user did not approve this git command";
+        if (GIT_WRITE.has(sub) && !(await ctx.permit(`git ${args.join(" ")}`, "git", ctx.signal))) return "error: the user did not approve this git command";
         // Sandboxed like bash: flags such as --output or an editor cannot write outside the project or reach the network.
         const res = await sandboxedExec(["git", "--no-pager", ...args], { cwd: ctx.cwd, profile: ctx.sandboxProfile, ...(ctx.sandboxEnv ? { env: ctx.sandboxEnv } : {}), ...(ctx.signal ? { signal: ctx.signal } : {}) });
         return `${res.output}\n(exit ${res.code})`;

package/src/models/mlx.ts

@@ -48,7 +48,8 @@ export interface MlxHandle {
   readonly status: () => MlxStatus;
   readonly url: string;
   readonly model: string;
-  readonly acquire: (signal?: AbortSignal) => Promise<() => void>;
+  /** `waitMs` bounds the wait for a free generation slot (default 120 s); past it, `MlxBusyError`. */
+  readonly acquire: (signal?: AbortSignal, waitMs?: number) => Promise<() => void>;
   readonly close: () => Promise<void>;
 }
 
@@ -177,6 +178,8 @@ interface GenerationOwner {
 }
 
 class GenerationSlotBusy extends Error {}
+/** Every generation slot stayed taken for the whole wait. */
+export class MlxBusyError extends Error {}
 
 function generationDatabase(runtimeDir: string): Database {
   mkdirSync(runtimeDir, { recursive: true, mode: 0o700 });
@@ -239,10 +242,10 @@ function claimGenerationSlot(db: Database, slot: number, readInfo: (pid: number)
   };
 }
 
-export async function acquireGeneration(runtimeDir: string, maxConcurrency: number, signal?: AbortSignal, readInfo: (pid: number) => ProcessSignature | undefined = processInfo): Promise<() => void> {
+export async function acquireGeneration(runtimeDir: string, maxConcurrency: number, signal?: AbortSignal, readInfo: (pid: number) => ProcessSignature | undefined = processInfo, waitMs = 120_000): Promise<() => void> {
   if (!Number.isInteger(maxConcurrency) || maxConcurrency < 1) throw new Error("MLX maxConcurrency must be a positive integer");
   if (signal?.aborted) throw new Error("MLX generation acquisition cancelled");
-  const deadline = Date.now() + 120_000;
+  const deadline = Date.now() + waitMs;
   const db = generationDatabase(runtimeDir);
   try {
     for (;;) {
@@ -251,7 +254,7 @@ export async function acquireGeneration(runtimeDir: string, maxConcurrency: numb
         if (release) return release;
       }
       if (signal?.aborted) throw new Error("MLX generation acquisition cancelled");
-      if (Date.now() >= deadline) throw new Error("MLX generation is busy");
+      if (Date.now() >= deadline) throw new MlxBusyError("MLX generation is busy");
       await new Promise<void>((resolve, reject) => {
         let settled = false;
         const finish = (error?: Error) => {
@@ -359,8 +362,8 @@ export async function ensureMlx(options: MlxOptions = {}): Promise<MlxHandle> {
   }
   const sharedHandle = (status: MlxStatus): MlxHandle => {
     let active = 0;
-    return { url: status.url!, model: modelPath, status: () => ({ ...status, active }), acquire: async (signal) => {
-      const releaseSlot = await acquireGeneration(runtimeDir, maxConcurrency, signal, readInfo);
+    return { url: status.url!, model: modelPath, status: () => ({ ...status, active }), acquire: async (signal, waitMs) => {
+      const releaseSlot = await acquireGeneration(runtimeDir, maxConcurrency, signal, readInfo, waitMs);
       active++;
       return () => { active = Math.max(0, active - 1); releaseSlot(); };
     }, close: async () => {} };
@@ -408,8 +411,8 @@ export async function ensureMlx(options: MlxOptions = {}): Promise<MlxHandle> {
         // The server is machine-shared. Relay shutdown releases this process's handle;
         // only the explicit stopMlx operation may stop the owned runtime.
         const close = async () => {};
-        return { url, model: modelPath, status: () => ({ state: "ready", url, model: modelPath, pid: child?.pid, maxInputTokens, maxConcurrency, active }), acquire: async (signal) => {
-          const releaseSlot = await acquireGeneration(runtimeDir, maxConcurrency, signal, readInfo);
+        return { url, model: modelPath, status: () => ({ state: "ready", url, model: modelPath, pid: child?.pid, maxInputTokens, maxConcurrency, active }), acquire: async (signal, waitMs) => {
+          const releaseSlot = await acquireGeneration(runtimeDir, maxConcurrency, signal, readInfo, waitMs);
           active++;
           return () => { active = Math.max(0, active - 1); releaseSlot(); };
         }, close };

package/src/models/ollama.ts

@@ -142,8 +142,8 @@ export async function ensureOllama(options: MlxOptions = {}): Promise<MlxHandle>
     url: `${baseUrl(config)}/v1`,
     model: config.model,
     status,
-    acquire: async (signal) => {
-      const releaseSlot = await acquireGeneration(config.runtimeDir, config.maxConcurrency, signal);
+    acquire: async (signal, waitMs) => {
+      const releaseSlot = await acquireGeneration(config.runtimeDir, config.maxConcurrency, signal, undefined, waitMs);
       try {
         const current = await inspectOllama(options);
         if (signal?.aborted) throw new Error("Ollama generation was cancelled");

package/src/models/relay.ts

@@ -1,8 +1,9 @@
 import { createHash, randomUUID } from "node:crypto";
 import { isIP } from "node:net";
 import type { OmniRoute, ChatMessage } from "../omniroute/client.ts";
-import { ensureMlx, type MlxHandle, type MlxOptions, type MlxStatus } from "./mlx.ts";
-import { AutoRouteSelector } from "./route/relay-selector.ts";
+import { ensureMlx, MlxBusyError, type MlxHandle, type MlxOptions, type MlxStatus } from "./mlx.ts";
+import { AutoRouteSelector, type AutoChoice, type RelayRouteEvent } from "./route/relay-selector.ts";
+import { estimateInputTokens, type StaySwitchPolicy } from "./route/stage.ts";
 
 export type ModelBackend = { kind: "mlx"; alias?: string } | { kind: "dgx"; alias: string };
 
@@ -23,6 +24,11 @@ export interface RelayBackendStatus {
   provider?: string;
   active: number;
   lastError?: string;
+  /** While the alias cools down after consecutive transport or startup failures (#199). */
+  coolingUntil?: string;
+  failures?: number;
+  /** While its last dispatch counts as failed: no load move goes to it. */
+  failingUntil?: string;
 }
 
 export interface ModelRelayStatus {
@@ -153,7 +159,22 @@ export interface ModelRelayOptions {
   enableHubAuto?: boolean;
   /** Trusted host callback. Requests without a stable session key get stateless stage selection. */
   routeSessionKey?: (request: RelayRequest) => string | undefined;
-  onRoute?: (event: { route: "hub/auto"; tier: string; source: "override" | "dimensions" | "hold" | "classifier" | "default"; score: number; ms: number }) => void;
+  onRoute?: (event: RelayRouteEvent) => void;
+  /** Read on every `hub/auto` call (#197); absent is the shadow default. */
+  staySwitch?: () => StaySwitchPolicy | undefined;
+  /** Opt-in (#199): how long a `hub/auto` request waits for a busy MLX slot before it moves to its fallback. Absent, no
+   *  load move or enforced-pin move happens; cooldown reorders still do. */
+  efficientWaitMs?: number;
+  /** A backend alias's cooldown starts or ends. */
+  onCooldown?: (event: RelayCooldownEvent) => void;
+  /** Cooldown clock; tests inject one. */
+  now?: () => number;
+  /** How long a dispatch waits for a busy MLX slot before it gives up and falls back (default 120 s); tests shorten it. */
+  slotWaitMs?: number;
+  /** A moved attempt's deadline for its response headers (default 15 s); tests shorten it. */
+  moveFirstByteMs?: number;
+  /** Whether the request runs under an execution budget, read without admitting anything; then nothing moves (#199). */
+  underBudget?: () => boolean;
   allowedDGXmodels: Record<string, string>;
   /** Trusted physical model expectations by backend alias. Gateway identifiers can include a provider
    *  prefix or route name that differs from the model reported by generation. Omission preserves the
@@ -184,11 +205,85 @@ export interface ModelRelay {
   readonly close: () => Promise<void>;
 }
 
+export interface RelayCooldownEvent { alias: string; event: "start" | "end"; failures: number; ms?: number }
+
+// ponytail: fixed policy, 3 failures and 30 s doubling up to 5 min; routing.toml keys if a backend needs other values.
+const COOLDOWN_FAILURES = 3, COOLDOWN_MS = 30_000, COOLDOWN_CAP_MS = 300_000;
+/** The longest one upstream request may take. */
+const REQUEST_DEADLINE_MS = 180_000;
+// ponytail: a moved attempt (a load move or an enforced pin, both only optimizations) gets 15 s to its response headers
+// and is then abandoned for MLX; tune it from the #199 AC5 measurement.
+const MOVE_FIRST_BYTE_MS = 15_000;
+
+/**
+ * Per-alias health in the relay (#199). A cooldown follows consecutive transport or startup failures, and any HTTP
+ * answer, a success included, ends it. Separately, a backend whose last dispatch failed in any way (an error status too)
+ * gets no load move for COOLDOWN_MS or until it succeeds.
+ */
+export class BackendCooldowns {
+  private readonly entries = new Map<string, { failures: number; until?: number; failedAt?: number; countedAt?: number }>();
+  constructor(private readonly now: () => number = Date.now, private readonly notify: (event: RelayCooldownEvent) => void = () => {}) {}
+
+  /** The alias's cooldown while it lasts. Its end is recorded when it is first seen to have passed, or at an answer. */
+  cooling(alias: string): { until: number; failures: number } | undefined {
+    const entry = this.entries.get(alias);
+    if (entry?.until === undefined) return undefined;
+    if (this.now() < entry.until) return { until: entry.until, failures: entry.failures };
+    entry.until = undefined;
+    this.notify({ alias, event: "end", failures: entry.failures });
+    return undefined;
+  }
+
+  /**
+   * A failed dispatch. `streak`: a transport or startup failure, which counts toward a cooldown unless one is running (a
+   * request already in flight, or a cooling alias tried as a fallback); otherwise it only marks the alias `failing`. A
+   * streak whose last counted failure is more than twice the cap old starts over, so an old cooldown does not double.
+   */
+  failed(alias: string, streak = true): void {
+    const cooling = this.cooling(alias);
+    const entry = this.entry(alias);
+    entry.failedAt = this.now();
+    if (cooling || !streak) return;
+    if (entry.countedAt !== undefined && this.now() - entry.countedAt > 2 * COOLDOWN_CAP_MS) entry.failures = 0;
+    entry.countedAt = this.now();
+    entry.failures++;
+    if (entry.failures < COOLDOWN_FAILURES) return;
+    const ms = Math.min(COOLDOWN_CAP_MS, COOLDOWN_MS * 2 ** (entry.failures - COOLDOWN_FAILURES));
+    entry.until = this.now() + ms;
+    this.notify({ alias, event: "start", failures: entry.failures, ms });
+  }
+
+  /** The backend answered: its transport works, so the failure streak and any cooldown end. Only a success clears `failing`. */
+  answered(alias: string, ok: boolean): void {
+    const entry = this.entry(alias);
+    if (entry.until !== undefined) this.notify({ alias, event: "end", failures: entry.failures });
+    entry.failures = 0;
+    entry.until = undefined;
+    entry.failedAt = ok ? undefined : this.now();
+  }
+
+  /** Until when the alias's last dispatch counts as failed for a load move, while it does. */
+  failing(alias: string): number | undefined {
+    const at = this.entries.get(alias)?.failedAt;
+    return at !== undefined && this.now() < at + COOLDOWN_MS ? at + COOLDOWN_MS : undefined;
+  }
+
+  private entry(alias: string) {
+    let entry = this.entries.get(alias);
+    if (!entry) this.entries.set(alias, entry = { failures: 0 });
+    return entry;
+  }
+}
+
+type Probe = { release: () => void } | { error: unknown };
+
 interface RequestJournalEntry {
   readonly record: RelayRequestRecord;
   identify(model: string, source: "header" | "stream" | "configured"): void;
   usage(value: unknown): void;
   close(outcome: RelayRequestRecord["outcome"]): void;
+  /** The relay's own deadline cut the attempt: the execution budget's, or a moved attempt's first-byte one. No cooldown count. */
+  cut?: "budget" | "move";
 }
 
 interface ActiveRequest {
@@ -225,10 +320,6 @@ function assertLoopback(host: string): void {
   if (value !== "localhost" && value !== "::1" && !(isIP(value) === 4 && value.startsWith("127."))) throw new Error("model relay host must be loopback");
 }
 
-function estimateInputTokens(messages: ChatMessage[], tools?: unknown[]): number {
-  return Math.ceil(JSON.stringify({ messages, tools }).length / 4);
-}
-
 function bearer(request: Request): string | undefined {
   const value = request.headers.get("authorization");
   return value?.startsWith("Bearer ") ? value.slice("Bearer ".length) : undefined;
@@ -337,7 +428,10 @@ export async function startModelRelay(options: ModelRelayOptions): Promise<Model
   const dgxMaxInputTokens = options.dgxMaxInputTokens ?? 262_144;
   const defaultBackend = options.defaultBackend ?? (options.mlx ? { kind: "mlx", alias: mlxAlias } : { kind: "dgx", alias: "dgx/coding" });
   const models = relayModelIds(options);
-  const autoRoute = options.enableHubAuto ? new AutoRouteSelector(options, defaultBackend, mlxAlias, estimateInputTokens) : undefined;
+  const efficientWaitMs = options.efficientWaitMs;
+  const moveFirstByteMs = options.moveFirstByteMs ?? MOVE_FIRST_BYTE_MS;
+  const cooldowns = new BackendCooldowns(options.now, (event) => { try { options.onCooldown?.(event); } catch { /* observation cannot fail routing */ } });
+  const autoRoute = options.enableHubAuto ? new AutoRouteSelector({ ...options, dgxMaxInputTokens }, defaultBackend, mlxAlias, estimateInputTokens) : undefined;
   let mlx: MlxHandle | undefined;
   let mlxStarting: Promise<MlxHandle> | undefined;
   const states = new Map<string, RelayBackendStatus>();
@@ -403,20 +497,22 @@ export async function startModelRelay(options: ModelRelayOptions): Promise<Model
     states.set(alias, { ...state(backend), ...patch, alias, kind: backend.kind });
   };
 
-  const resolve = async (body: RelayRequest): Promise<ModelBackend> => {
+  const resolve = async (body: RelayRequest): Promise<{ backend: ModelBackend; auto?: AutoChoice }> => {
     const requested = typeof body.model === "string" ? body.model : undefined;
     const automatic = options.enableHubAuto === true && requested === "hub/auto";
     if (requested && !models.includes(requested)) throw new Error("model alias is not allowed");
-    if (requested === mlxAlias && options.mlx) return { kind: "mlx", alias: mlxAlias };
-    if (requested && requested in options.allowedDGXmodels) return { kind: "dgx", alias: requested };
-    if (automatic) return autoRoute!.select(body);
+    if (requested === mlxAlias && options.mlx) return { backend: { kind: "mlx", alias: mlxAlias } };
+    if (requested && requested in options.allowedDGXmodels) return { backend: { kind: "dgx", alias: requested } };
+    if (automatic) { const auto = await autoRoute!.select(body); return { backend: auto.backend, auto }; }
     const selected = options.selectBackend ? await options.selectBackend(body) : defaultBackend;
     if (selected.kind === "mlx" && !options.mlx) throw new Error("MLX backend is not configured");
     if (selected.kind === "dgx" && !(selected.alias in options.allowedDGXmodels)) throw new Error("DGX model alias is not allowed");
-    return selected;
+    return { backend: selected };
   };
 
-  const upstream = async (request: RelayRequest, backend: ModelBackend, signal: AbortSignal, journalEntry: RequestJournalEntry): Promise<{ response: Response; release: () => void; onModel?: (model: string) => void }> => {
+  /** `pre`: the load probe's outcome for MLX, a held generation slot (released exactly once by whoever ends the request)
+   *  or the error it met, reported here without asking MLX again. */
+  const upstream = async (request: RelayRequest, backend: ModelBackend, signal: AbortSignal, journalEntry: RequestJournalEntry, pre?: Probe, firstByteMs?: number): Promise<{ response: Response; release: () => void; onModel?: (model: string) => void }> => {
     const alias = aliasOf(backend, mlxAlias);
     let base: string;
     let model: string;
@@ -436,7 +532,8 @@ export async function startModelRelay(options: ModelRelayOptions): Promise<Model
       if (options.mlx?.provider === "ollama" && options.mlxModel && options.mlxModel !== handle.model) throw new Error("Ollama model override does not match the validated model");
       model = options.mlxModel ?? handle.model;
       if (signal.aborted) throw new Error("request was cancelled before MLX generation started");
-      release = await handle.acquire(signal);
+      if (pre && "error" in pre) throw pre.error;
+      release = pre?.release ?? await handle.acquire(signal, options.slotWaitMs);
     } else {
       base = (await options.omni.base()) ?? (() => { throw new Error("DGX gateway is unavailable"); })();
       model = options.allowedDGXmodels[backend.alias]!;
@@ -455,6 +552,9 @@ export async function startModelRelay(options: ModelRelayOptions): Promise<Model
     const headers: Record<string, string> = { "content-type": "application/json" };
     if (key) Object.assign(headers, { authorization: `Bearer ${key}` }, options.omni.accessHeaders(base));
     let response: Response;
+    let budgetDeadline: AbortSignal | undefined;
+    const firstByte = firstByteMs === undefined ? undefined : new AbortController();
+    let firstByteTimer: ReturnType<typeof setTimeout> | undefined;
     try {
       const isOllama = backend.kind === "mlx" && options.mlx?.provider === "ollama";
       const boundedRequest = isOllama
@@ -462,14 +562,19 @@ export async function startModelRelay(options: ModelRelayOptions): Promise<Model
         : request;
       const decision = await options.admitRequest?.();
       if (decision && !decision.allowed) throw new ExecutionAdmissionError(decision.reason ?? "execution budget exhausted");
-      const deadline = decision?.remainingMs === undefined ? 180_000 : Math.max(1, Math.min(180_000, decision.remainingMs));
-      response = await fetch(`${base.replace(/\/$/, "")}/chat/completions`, { method: "POST", ...(isOllama ? { redirect: "error" as const } : {}), headers, body: JSON.stringify(bodyForUpstream(boundedRequest, model)), signal: AbortSignal.any([signal, AbortSignal.timeout(deadline)]) });
+      const deadline = decision?.remainingMs === undefined ? REQUEST_DEADLINE_MS : Math.max(1, Math.min(REQUEST_DEADLINE_MS, decision.remainingMs));
+      const timeout = AbortSignal.timeout(deadline);
+      if (deadline < REQUEST_DEADLINE_MS) budgetDeadline = timeout;
+      if (firstByte) firstByteTimer = setTimeout(() => firstByte.abort(new Error(`moved attempt had no answer within ${firstByteMs} ms`)), firstByteMs);
+      response = await fetch(`${base.replace(/\/$/, "")}/chat/completions`, { method: "POST", ...(isOllama ? { redirect: "error" as const } : {}), headers, body: JSON.stringify(bodyForUpstream(boundedRequest, model)), signal: AbortSignal.any([signal, timeout, ...(firstByte ? [firstByte.signal] : [])]) });
     } catch (error) {
       releaseOnce();
       journalEntry.record.failureClass = "transport";
+      if (firstByte?.signal.aborted) journalEntry.cut = "move";
+      else if (budgetDeadline?.aborted) journalEntry.cut = "budget";
       setState(backend, { state: "error", lastError: error instanceof Error ? error.message.slice(0, 160) : "upstream request failed" });
       throw error;
-    }
+    } finally { clearTimeout(firstByteTimer); }
     const provider = safeHeader(response.headers.get("x-omniroute-provider"));
     if (provider) {
       journalEntry.record.provider = provider;
@@ -524,32 +629,65 @@ export async function startModelRelay(options: ModelRelayOptions): Promise<Model
         activeRequests.delete(record);
         return Response.json({ error: error instanceof Error ? error.message : "invalid request" }, { status: 400 });
       }
-      let backend: ModelBackend;
-      try { backend = await resolve(body); } catch (error) {
+      let backend: ModelBackend, auto: AutoChoice | undefined;
+      try { ({ backend, auto } = await resolve(body)); } catch (error) {
         record.cleanup();
         activeRequests.delete(record);
         return Response.json({ error: error instanceof Error ? error.message : "backend unavailable" }, { status: 400 });
       }
-      const inputBudget = backend.kind === "mlx" ? (options.mlx?.maxInputTokens ?? (options.mlx?.provider === "ollama" ? 6000 : 16_000)) : dgxMaxInputTokens;
+      const own = backend;
+      // A refused request still records the decision, unmoved; a dispatched one records it after the ordering step.
+      const emitRoute = () => { if (auto) try { options.onRoute?.(auto.route); } catch { /* observation cannot fail routing */ } };
       const inputTokens = estimateInputTokens(body.messages, body.tools);
-      const ollama = backend.kind === "mlx" && options.mlx?.provider === "ollama";
+      const inputBudget = own.kind === "mlx" ? (options.mlx?.maxInputTokens ?? (options.mlx?.provider === "ollama" ? 6000 : 16_000)) : dgxMaxInputTokens;
+      const ollama = own.kind === "mlx" && options.mlx?.provider === "ollama";
       const contextWindow = ollama ? (options.mlx?.contextWindow ?? 8192) : undefined;
       const configuredMaxTokens = ollama ? (options.mlx?.maxTokens ?? 2048) : undefined;
       const requestedMaxTokens = body.max_tokens === undefined ? configuredMaxTokens : body.max_tokens;
       if (ollama && (!Number.isInteger(requestedMaxTokens) || (requestedMaxTokens as number) < 1 || (requestedMaxTokens as number) > configuredMaxTokens! || inputTokens + (requestedMaxTokens as number) > contextWindow!)) {
+        emitRoute();
         record.cleanup();
         activeRequests.delete(record);
         return Response.json({ error: "input and max_tokens exceed the Ollama context budget" }, { status: 400 });
       }
       if (inputTokens > inputBudget) {
+        emitRoute();
         record.cleanup();
         activeRequests.delete(record);
         return Response.json({ error: "input exceeds the model context budget" }, { status: 400 });
       }
-      const fallback = backend.kind === "mlx" && options.fallbackDGXAlias ? { kind: "dgx", alias: options.fallbackDGXAlias } as ModelBackend : undefined;
+      // #199: one ordering step, for hub/auto only (a fixed alias keeps its backend). The candidates are the route's own
+      // backend and, for MLX, its DGX fallback, exactly as before; a cooldown, an enforced pin or a busy MLX slot only
+      // reorders them. Nothing goes ahead to a fallback that is failing or cooling down.
+      const candidates: ModelBackend[] = [own, ...(own.kind === "mlx" && options.fallbackDGXAlias ? [{ kind: "dgx", alias: options.fallbackDGXAlias } as ModelBackend] : [])];
+      const second = candidates[1];
+      let moved: "optional" | "cooldown" | undefined;
+      const swap = (source: "load" | "cooldown" | "pin") => { candidates.reverse(); moved = source === "cooldown" ? "cooldown" : "optional"; auto?.moved(aliasOf(candidates[0]!, mlxAlias), source); };
+      // A load move or pin is only an optimization: opt-in (`efficientWaitMs`) until #199 AC5 measures it, bounded by its
+      // first-byte deadline, and never under an execution budget, where it could spend the model call or the time MLX
+      // needs after it (a throw counts as a budget).
+      // ponytail: no moves at all under a budget; budget-aware moves from admitPiRequest's remaining units are the upgrade.
+      let room = efficientWaitMs !== undefined;
+      try { room &&= !options.underBudget?.(); } catch { room = false; }
+      let pre: Probe | undefined;
+      if (auto && second && !cooldowns.failing(aliasOf(second, mlxAlias)) && !cooldowns.cooling(aliasOf(second, mlxAlias))) {
+        if (cooldowns.cooling(aliasOf(own, mlxAlias))) swap("cooldown");
+        else if (auto.prefer === aliasOf(second, mlxAlias) && room) swap("pin");
+        else if (auto.movable && mlx && room) {
+          try {
+            const held = await mlx.acquire(controller.signal, efficientWaitMs);
+            let holding = true;
+            pre = { release: () => { if (holding) { holding = false; held(); } } };
+          } catch (error) {
+            if (error instanceof MlxBusyError) swap("load");
+            else pre = { error }; // the MLX dispatch reports it, without inspecting Ollama a second time
+          }
+        }
+      }
+      emitRoute();
       const dispatchGroupId = randomUUID();
       let primaryDispatchId: string | undefined;
-      const dispatch = async (selected: ModelBackend, body: RelayRequest) => {
+      const dispatch = async (selected: ModelBackend, body: RelayRequest, pre?: Probe, firstByteMs?: number) => {
         const journalEntry = openRequestRecord(aliasOf(selected, mlxAlias), dispatchGroupId);
         // The surface is a property of the request as admitted: even a failed dispatch keeps what the
         // native published. A request without a tools array carries no observation, never an empty one.
@@ -565,11 +703,21 @@ export async function startModelRelay(options: ModelRelayOptions): Promise<Model
         record.closeRecord = journalEntry.close;
         let result: Awaited<ReturnType<typeof upstream>>;
         try {
-          result = await upstream(body, selected, controller.signal, journalEntry);
+          result = await upstream(body, selected, controller.signal, journalEntry, pre, firstByteMs);
         } catch (error) {
+          if (pre && "release" in pre) pre.release();
           journalEntry.record.failureClass ??= controller.signal.aborted ? "cancelled" : error instanceof ExecutionAdmissionError ? "admission" : "startup";
+          const failure = journalEntry.record.failureClass;
+          // A busy MLX slot is load, never a failure; neither is a cancelled request, an admission refusal or a budget cut.
+          if (!controller.signal.aborted && !journalEntry.cut && !(error instanceof ExecutionAdmissionError) && !(error instanceof MlxBusyError) && (failure === "transport" || failure === "startup")) {
+            // A DGX alias can fail before it has a status row (gateway or key unavailable); a cooldown must still show.
+            if (!states.has(journalEntry.record.alias)) setState(selected, { state: "error", lastError: error instanceof Error ? error.message.slice(0, 160) : "backend unavailable" });
+            cooldowns.failed(journalEntry.record.alias);
+          } else if (journalEntry.cut === "move") cooldowns.failed(journalEntry.record.alias, false); // slow, not unreachable
+          else if (failure === "http") cooldowns.answered(journalEntry.record.alias, false);
           throw error;
         }
+        cooldowns.answered(journalEntry.record.alias, true);
         let released = false;
         const release = () => {
           if (released) return;
@@ -582,30 +730,30 @@ export async function startModelRelay(options: ModelRelayOptions): Promise<Model
         const onUsage = journalEntry.usage;
         return { result, release, journalEntry, onUsage };
       };
-      try {
-        const { result, release, journalEntry, onUsage } = await dispatch(backend, body);
-        return sseResponse(result.response, release, result.onModel, (cancel) => { record.cancel = cancel; }, journalEntry.close, onUsage);
-      } catch (error) {
-        record.closeRecord?.(controller.signal.aborted ? "cancelled" : "failed");
-        if (!fallback || controller.signal.aborted || error instanceof ExecutionAdmissionError) {
-          record.cleanup();
-          activeRequests.delete(record);
-          return Response.json({ error: error instanceof Error ? error.message : "backend unavailable" }, { status: 502 });
-        }
+      let lastError: unknown;
+      for (const [i, candidate] of candidates.entries()) {
         try {
-          const { result, release, journalEntry, onUsage } = await dispatch(fallback, { ...body, model: fallback.alias });
-          return sseResponse(result.response, release, result.onModel, (cancel) => { record.cancel = cancel; }, journalEntry.close, onUsage);
-        } catch (fallbackError) {
+          const { result, release, journalEntry, onUsage } = await dispatch(candidate, i ? { ...body, model: aliasOf(candidate, mlxAlias) } : body, i ? undefined : pre, i === 0 && moved === "optional" ? moveFirstByteMs : undefined);
+          // A stream that fails after its headers cannot fall back; it still marks the alias failing, so no move follows. A
+          // client that went away can fail the pending read before the cancel arrives: that says nothing about the backend.
+          const close = (outcome: RelayRequestRecord["outcome"]) => { journalEntry.close(outcome); if (outcome === "failed" && !controller.signal.aborted) cooldowns.failed(journalEntry.record.alias, false); };
+          return sseResponse(result.response, release, result.onModel, (cancel) => { record.cancel = cancel; }, close, onUsage);
+        } catch (error) {
           record.closeRecord?.(controller.signal.aborted ? "cancelled" : "failed");
-          record.cleanup();
-          activeRequests.delete(record);
-          return Response.json({ error: fallbackError instanceof Error ? fallbackError.message : "fallback unavailable" }, { status: 502 });
+          lastError = error;
+          if (controller.signal.aborted || error instanceof ExecutionAdmissionError) break;
         }
       }
+      record.cleanup();
+      activeRequests.delete(record);
+      return Response.json({ error: lastError instanceof Error ? lastError.message : "backend unavailable" }, { status: 502 });
     },
   });
   const url = `http://${host}:${server.port}/v1`;
-  const status = (): ModelRelayStatus => ({ url, models, backends: [...states.values()].map((value) => ({ ...value })) });
+  const status = (): ModelRelayStatus => ({ url, models, backends: [...states.values()].map((value) => {
+    const cooling = cooldowns.cooling(value.alias), failing = cooldowns.failing(value.alias);
+    return { ...value, ...(cooling ? { coolingUntil: new Date(cooling.until).toISOString(), failures: cooling.failures } : {}), ...(failing ? { failingUntil: new Date(failing).toISOString() } : {}) };
+  }) });
   const requests = (): RelayRequestRecord[] => journal.map(copyRecord);
   return { url, token, models, status, requests, close: async () => {
     const closing = [...activeRequests].map(async (request) => {