@staix/agent-hub 0.12.18 → 0.12.20

package/src/local/deny.ts

@@ -4,7 +4,8 @@ import { basename, sep } from "node:path";
  * The one denylist. tools.ts checks paths against it, sandbox.ts turns it into seatbelt rules, capture.ts keeps
  * matching calls out of memory. Two lists would drift, and the gap is where a secret leaks.
  */
-export const DENY_SEGMENTS = [".maru/secrets", ".agenthub/state"];
+// `.agenthub/archive` holds the state directories `ahub reset --all` moved there: hub.db with task text, PII included.
+export const DENY_SEGMENTS = [".maru/secrets", ".agenthub/state", ".agenthub/archive"];
 /** Regex sources, matched against a file's basename. */
 export const DENY_NAMES = ["^\\.env", "\\.pem$", "\\.key$", "^id_rsa", "^id_ed25519", "^credentials(\\.|$)", "^auth\\.json$", "^\\.netrc$", "^\\.npmrc$", "^\\.pypirc$"];
 

package/src/models/mlx.ts

@@ -48,7 +48,8 @@ export interface MlxHandle {
   readonly status: () => MlxStatus;
   readonly url: string;
   readonly model: string;
-  readonly acquire: (signal?: AbortSignal) => Promise<() => void>;
+  /** `waitMs` bounds the wait for a free generation slot (default 120 s); past it, `MlxBusyError`. */
+  readonly acquire: (signal?: AbortSignal, waitMs?: number) => Promise<() => void>;
   readonly close: () => Promise<void>;
 }
 
@@ -177,6 +178,8 @@ interface GenerationOwner {
 }
 
 class GenerationSlotBusy extends Error {}
+/** Every generation slot stayed taken for the whole wait. */
+export class MlxBusyError extends Error {}
 
 function generationDatabase(runtimeDir: string): Database {
   mkdirSync(runtimeDir, { recursive: true, mode: 0o700 });
@@ -239,10 +242,10 @@ function claimGenerationSlot(db: Database, slot: number, readInfo: (pid: number)
   };
 }
 
-export async function acquireGeneration(runtimeDir: string, maxConcurrency: number, signal?: AbortSignal, readInfo: (pid: number) => ProcessSignature | undefined = processInfo): Promise<() => void> {
+export async function acquireGeneration(runtimeDir: string, maxConcurrency: number, signal?: AbortSignal, readInfo: (pid: number) => ProcessSignature | undefined = processInfo, waitMs = 120_000): Promise<() => void> {
   if (!Number.isInteger(maxConcurrency) || maxConcurrency < 1) throw new Error("MLX maxConcurrency must be a positive integer");
   if (signal?.aborted) throw new Error("MLX generation acquisition cancelled");
-  const deadline = Date.now() + 120_000;
+  const deadline = Date.now() + waitMs;
   const db = generationDatabase(runtimeDir);
   try {
     for (;;) {
@@ -251,7 +254,7 @@ export async function acquireGeneration(runtimeDir: string, maxConcurrency: numb
         if (release) return release;
       }
       if (signal?.aborted) throw new Error("MLX generation acquisition cancelled");
-      if (Date.now() >= deadline) throw new Error("MLX generation is busy");
+      if (Date.now() >= deadline) throw new MlxBusyError("MLX generation is busy");
       await new Promise<void>((resolve, reject) => {
         let settled = false;
         const finish = (error?: Error) => {
@@ -359,8 +362,8 @@ export async function ensureMlx(options: MlxOptions = {}): Promise<MlxHandle> {
   }
   const sharedHandle = (status: MlxStatus): MlxHandle => {
     let active = 0;
-    return { url: status.url!, model: modelPath, status: () => ({ ...status, active }), acquire: async (signal) => {
-      const releaseSlot = await acquireGeneration(runtimeDir, maxConcurrency, signal, readInfo);
+    return { url: status.url!, model: modelPath, status: () => ({ ...status, active }), acquire: async (signal, waitMs) => {
+      const releaseSlot = await acquireGeneration(runtimeDir, maxConcurrency, signal, readInfo, waitMs);
       active++;
       return () => { active = Math.max(0, active - 1); releaseSlot(); };
     }, close: async () => {} };
@@ -408,8 +411,8 @@ export async function ensureMlx(options: MlxOptions = {}): Promise<MlxHandle> {
         // The server is machine-shared. Relay shutdown releases this process's handle;
         // only the explicit stopMlx operation may stop the owned runtime.
         const close = async () => {};
-        return { url, model: modelPath, status: () => ({ state: "ready", url, model: modelPath, pid: child?.pid, maxInputTokens, maxConcurrency, active }), acquire: async (signal) => {
-          const releaseSlot = await acquireGeneration(runtimeDir, maxConcurrency, signal, readInfo);
+        return { url, model: modelPath, status: () => ({ state: "ready", url, model: modelPath, pid: child?.pid, maxInputTokens, maxConcurrency, active }), acquire: async (signal, waitMs) => {
+          const releaseSlot = await acquireGeneration(runtimeDir, maxConcurrency, signal, readInfo, waitMs);
           active++;
           return () => { active = Math.max(0, active - 1); releaseSlot(); };
         }, close };

package/src/models/ollama.ts

@@ -142,8 +142,8 @@ export async function ensureOllama(options: MlxOptions = {}): Promise<MlxHandle>
     url: `${baseUrl(config)}/v1`,
     model: config.model,
     status,
-    acquire: async (signal) => {
-      const releaseSlot = await acquireGeneration(config.runtimeDir, config.maxConcurrency, signal);
+    acquire: async (signal, waitMs) => {
+      const releaseSlot = await acquireGeneration(config.runtimeDir, config.maxConcurrency, signal, undefined, waitMs);
       try {
         const current = await inspectOllama(options);
         if (signal?.aborted) throw new Error("Ollama generation was cancelled");

package/src/models/relay.ts

@@ -1,8 +1,9 @@
 import { createHash, randomUUID } from "node:crypto";
 import { isIP } from "node:net";
 import type { OmniRoute, ChatMessage } from "../omniroute/client.ts";
-import { ensureMlx, type MlxHandle, type MlxOptions, type MlxStatus } from "./mlx.ts";
-import { AutoRouteSelector } from "./route/relay-selector.ts";
+import { ensureMlx, MlxBusyError, type MlxHandle, type MlxOptions, type MlxStatus } from "./mlx.ts";
+import { AutoRouteSelector, type AutoChoice, type RelayRouteEvent } from "./route/relay-selector.ts";
+import { estimateInputTokens, type StaySwitchPolicy } from "./route/stage.ts";
 
 export type ModelBackend = { kind: "mlx"; alias?: string } | { kind: "dgx"; alias: string };
 
@@ -23,6 +24,11 @@ export interface RelayBackendStatus {
   provider?: string;
   active: number;
   lastError?: string;
+  /** While the alias cools down after consecutive transport or startup failures (#199). */
+  coolingUntil?: string;
+  failures?: number;
+  /** While its last dispatch counts as failed: no load move goes to it. */
+  failingUntil?: string;
 }
 
 export interface ModelRelayStatus {
@@ -153,7 +159,22 @@ export interface ModelRelayOptions {
   enableHubAuto?: boolean;
   /** Trusted host callback. Requests without a stable session key get stateless stage selection. */
   routeSessionKey?: (request: RelayRequest) => string | undefined;
-  onRoute?: (event: { route: "hub/auto"; tier: string; source: "override" | "dimensions" | "hold" | "classifier" | "default"; score: number; ms: number }) => void;
+  onRoute?: (event: RelayRouteEvent) => void;
+  /** Read on every `hub/auto` call (#197); absent is the shadow default. */
+  staySwitch?: () => StaySwitchPolicy | undefined;
+  /** Opt-in (#199): how long a `hub/auto` request waits for a busy MLX slot before it moves to its fallback. Absent, no
+   *  load move or enforced-pin move happens; cooldown reorders still do. */
+  efficientWaitMs?: number;
+  /** A backend alias's cooldown starts or ends. */
+  onCooldown?: (event: RelayCooldownEvent) => void;
+  /** Cooldown clock; tests inject one. */
+  now?: () => number;
+  /** How long a dispatch waits for a busy MLX slot before it gives up and falls back (default 120 s); tests shorten it. */
+  slotWaitMs?: number;
+  /** A moved attempt's deadline for its response headers (default 15 s); tests shorten it. */
+  moveFirstByteMs?: number;
+  /** Whether the request runs under an execution budget, read without admitting anything; then nothing moves (#199). */
+  underBudget?: () => boolean;
   allowedDGXmodels: Record<string, string>;
   /** Trusted physical model expectations by backend alias. Gateway identifiers can include a provider
    *  prefix or route name that differs from the model reported by generation. Omission preserves the
@@ -184,11 +205,85 @@ export interface ModelRelay {
   readonly close: () => Promise<void>;
 }
 
+export interface RelayCooldownEvent { alias: string; event: "start" | "end"; failures: number; ms?: number }
+
+// ponytail: fixed policy, 3 failures and 30 s doubling up to 5 min; routing.toml keys if a backend needs other values.
+const COOLDOWN_FAILURES = 3, COOLDOWN_MS = 30_000, COOLDOWN_CAP_MS = 300_000;
+/** The longest one upstream request may take. */
+const REQUEST_DEADLINE_MS = 180_000;
+// ponytail: a moved attempt (a load move or an enforced pin, both only optimizations) gets 15 s to its response headers
+// and is then abandoned for MLX; tune it from the #199 AC5 measurement.
+const MOVE_FIRST_BYTE_MS = 15_000;
+
+/**
+ * Per-alias health in the relay (#199). A cooldown follows consecutive transport or startup failures, and any HTTP
+ * answer, a success included, ends it. Separately, a backend whose last dispatch failed in any way (an error status too)
+ * gets no load move for COOLDOWN_MS or until it succeeds.
+ */
+export class BackendCooldowns {
+  private readonly entries = new Map<string, { failures: number; until?: number; failedAt?: number; countedAt?: number }>();
+  constructor(private readonly now: () => number = Date.now, private readonly notify: (event: RelayCooldownEvent) => void = () => {}) {}
+
+  /** The alias's cooldown while it lasts. Its end is recorded when it is first seen to have passed, or at an answer. */
+  cooling(alias: string): { until: number; failures: number } | undefined {
+    const entry = this.entries.get(alias);
+    if (entry?.until === undefined) return undefined;
+    if (this.now() < entry.until) return { until: entry.until, failures: entry.failures };
+    entry.until = undefined;
+    this.notify({ alias, event: "end", failures: entry.failures });
+    return undefined;
+  }
+
+  /**
+   * A failed dispatch. `streak`: a transport or startup failure, which counts toward a cooldown unless one is running (a
+   * request already in flight, or a cooling alias tried as a fallback); otherwise it only marks the alias `failing`. A
+   * streak whose last counted failure is more than twice the cap old starts over, so an old cooldown does not double.
+   */
+  failed(alias: string, streak = true): void {
+    const cooling = this.cooling(alias);
+    const entry = this.entry(alias);
+    entry.failedAt = this.now();
+    if (cooling || !streak) return;
+    if (entry.countedAt !== undefined && this.now() - entry.countedAt > 2 * COOLDOWN_CAP_MS) entry.failures = 0;
+    entry.countedAt = this.now();
+    entry.failures++;
+    if (entry.failures < COOLDOWN_FAILURES) return;
+    const ms = Math.min(COOLDOWN_CAP_MS, COOLDOWN_MS * 2 ** (entry.failures - COOLDOWN_FAILURES));
+    entry.until = this.now() + ms;
+    this.notify({ alias, event: "start", failures: entry.failures, ms });
+  }
+
+  /** The backend answered: its transport works, so the failure streak and any cooldown end. Only a success clears `failing`. */
+  answered(alias: string, ok: boolean): void {
+    const entry = this.entry(alias);
+    if (entry.until !== undefined) this.notify({ alias, event: "end", failures: entry.failures });
+    entry.failures = 0;
+    entry.until = undefined;
+    entry.failedAt = ok ? undefined : this.now();
+  }
+
+  /** Until when the alias's last dispatch counts as failed for a load move, while it does. */
+  failing(alias: string): number | undefined {
+    const at = this.entries.get(alias)?.failedAt;
+    return at !== undefined && this.now() < at + COOLDOWN_MS ? at + COOLDOWN_MS : undefined;
+  }
+
+  private entry(alias: string) {
+    let entry = this.entries.get(alias);
+    if (!entry) this.entries.set(alias, entry = { failures: 0 });
+    return entry;
+  }
+}
+
+type Probe = { release: () => void } | { error: unknown };
+
 interface RequestJournalEntry {
   readonly record: RelayRequestRecord;
   identify(model: string, source: "header" | "stream" | "configured"): void;
   usage(value: unknown): void;
   close(outcome: RelayRequestRecord["outcome"]): void;
+  /** The relay's own deadline cut the attempt: the execution budget's, or a moved attempt's first-byte one. No cooldown count. */
+  cut?: "budget" | "move";
 }
 
 interface ActiveRequest {
@@ -225,10 +320,6 @@ function assertLoopback(host: string): void {
   if (value !== "localhost" && value !== "::1" && !(isIP(value) === 4 && value.startsWith("127."))) throw new Error("model relay host must be loopback");
 }
 
-function estimateInputTokens(messages: ChatMessage[], tools?: unknown[]): number {
-  return Math.ceil(JSON.stringify({ messages, tools }).length / 4);
-}
-
 function bearer(request: Request): string | undefined {
   const value = request.headers.get("authorization");
   return value?.startsWith("Bearer ") ? value.slice("Bearer ".length) : undefined;
@@ -337,7 +428,10 @@ export async function startModelRelay(options: ModelRelayOptions): Promise<Model
   const dgxMaxInputTokens = options.dgxMaxInputTokens ?? 262_144;
   const defaultBackend = options.defaultBackend ?? (options.mlx ? { kind: "mlx", alias: mlxAlias } : { kind: "dgx", alias: "dgx/coding" });
   const models = relayModelIds(options);
-  const autoRoute = options.enableHubAuto ? new AutoRouteSelector(options, defaultBackend, mlxAlias, estimateInputTokens) : undefined;
+  const efficientWaitMs = options.efficientWaitMs;
+  const moveFirstByteMs = options.moveFirstByteMs ?? MOVE_FIRST_BYTE_MS;
+  const cooldowns = new BackendCooldowns(options.now, (event) => { try { options.onCooldown?.(event); } catch { /* observation cannot fail routing */ } });
+  const autoRoute = options.enableHubAuto ? new AutoRouteSelector({ ...options, dgxMaxInputTokens }, defaultBackend, mlxAlias, estimateInputTokens) : undefined;
   let mlx: MlxHandle | undefined;
   let mlxStarting: Promise<MlxHandle> | undefined;
   const states = new Map<string, RelayBackendStatus>();
@@ -403,20 +497,22 @@ export async function startModelRelay(options: ModelRelayOptions): Promise<Model
     states.set(alias, { ...state(backend), ...patch, alias, kind: backend.kind });
   };
 
-  const resolve = async (body: RelayRequest): Promise<ModelBackend> => {
+  const resolve = async (body: RelayRequest): Promise<{ backend: ModelBackend; auto?: AutoChoice }> => {
     const requested = typeof body.model === "string" ? body.model : undefined;
     const automatic = options.enableHubAuto === true && requested === "hub/auto";
     if (requested && !models.includes(requested)) throw new Error("model alias is not allowed");
-    if (requested === mlxAlias && options.mlx) return { kind: "mlx", alias: mlxAlias };
-    if (requested && requested in options.allowedDGXmodels) return { kind: "dgx", alias: requested };
-    if (automatic) return autoRoute!.select(body);
+    if (requested === mlxAlias && options.mlx) return { backend: { kind: "mlx", alias: mlxAlias } };
+    if (requested && requested in options.allowedDGXmodels) return { backend: { kind: "dgx", alias: requested } };
+    if (automatic) { const auto = await autoRoute!.select(body); return { backend: auto.backend, auto }; }
     const selected = options.selectBackend ? await options.selectBackend(body) : defaultBackend;
     if (selected.kind === "mlx" && !options.mlx) throw new Error("MLX backend is not configured");
     if (selected.kind === "dgx" && !(selected.alias in options.allowedDGXmodels)) throw new Error("DGX model alias is not allowed");
-    return selected;
+    return { backend: selected };
   };
 
-  const upstream = async (request: RelayRequest, backend: ModelBackend, signal: AbortSignal, journalEntry: RequestJournalEntry): Promise<{ response: Response; release: () => void; onModel?: (model: string) => void }> => {
+  /** `pre`: the load probe's outcome for MLX, a held generation slot (released exactly once by whoever ends the request)
+   *  or the error it met, reported here without asking MLX again. */
+  const upstream = async (request: RelayRequest, backend: ModelBackend, signal: AbortSignal, journalEntry: RequestJournalEntry, pre?: Probe, firstByteMs?: number): Promise<{ response: Response; release: () => void; onModel?: (model: string) => void }> => {
     const alias = aliasOf(backend, mlxAlias);
     let base: string;
     let model: string;
@@ -436,7 +532,8 @@ export async function startModelRelay(options: ModelRelayOptions): Promise<Model
       if (options.mlx?.provider === "ollama" && options.mlxModel && options.mlxModel !== handle.model) throw new Error("Ollama model override does not match the validated model");
       model = options.mlxModel ?? handle.model;
       if (signal.aborted) throw new Error("request was cancelled before MLX generation started");
-      release = await handle.acquire(signal);
+      if (pre && "error" in pre) throw pre.error;
+      release = pre?.release ?? await handle.acquire(signal, options.slotWaitMs);
     } else {
       base = (await options.omni.base()) ?? (() => { throw new Error("DGX gateway is unavailable"); })();
       model = options.allowedDGXmodels[backend.alias]!;
@@ -455,6 +552,9 @@ export async function startModelRelay(options: ModelRelayOptions): Promise<Model
     const headers: Record<string, string> = { "content-type": "application/json" };
     if (key) Object.assign(headers, { authorization: `Bearer ${key}` }, options.omni.accessHeaders(base));
     let response: Response;
+    let budgetDeadline: AbortSignal | undefined;
+    const firstByte = firstByteMs === undefined ? undefined : new AbortController();
+    let firstByteTimer: ReturnType<typeof setTimeout> | undefined;
     try {
       const isOllama = backend.kind === "mlx" && options.mlx?.provider === "ollama";
       const boundedRequest = isOllama
@@ -462,14 +562,19 @@ export async function startModelRelay(options: ModelRelayOptions): Promise<Model
         : request;
       const decision = await options.admitRequest?.();
       if (decision && !decision.allowed) throw new ExecutionAdmissionError(decision.reason ?? "execution budget exhausted");
-      const deadline = decision?.remainingMs === undefined ? 180_000 : Math.max(1, Math.min(180_000, decision.remainingMs));
-      response = await fetch(`${base.replace(/\/$/, "")}/chat/completions`, { method: "POST", ...(isOllama ? { redirect: "error" as const } : {}), headers, body: JSON.stringify(bodyForUpstream(boundedRequest, model)), signal: AbortSignal.any([signal, AbortSignal.timeout(deadline)]) });
+      const deadline = decision?.remainingMs === undefined ? REQUEST_DEADLINE_MS : Math.max(1, Math.min(REQUEST_DEADLINE_MS, decision.remainingMs));
+      const timeout = AbortSignal.timeout(deadline);
+      if (deadline < REQUEST_DEADLINE_MS) budgetDeadline = timeout;
+      if (firstByte) firstByteTimer = setTimeout(() => firstByte.abort(new Error(`moved attempt had no answer within ${firstByteMs} ms`)), firstByteMs);
+      response = await fetch(`${base.replace(/\/$/, "")}/chat/completions`, { method: "POST", ...(isOllama ? { redirect: "error" as const } : {}), headers, body: JSON.stringify(bodyForUpstream(boundedRequest, model)), signal: AbortSignal.any([signal, timeout, ...(firstByte ? [firstByte.signal] : [])]) });
     } catch (error) {
       releaseOnce();
       journalEntry.record.failureClass = "transport";
+      if (firstByte?.signal.aborted) journalEntry.cut = "move";
+      else if (budgetDeadline?.aborted) journalEntry.cut = "budget";
       setState(backend, { state: "error", lastError: error instanceof Error ? error.message.slice(0, 160) : "upstream request failed" });
       throw error;
-    }
+    } finally { clearTimeout(firstByteTimer); }
     const provider = safeHeader(response.headers.get("x-omniroute-provider"));
     if (provider) {
       journalEntry.record.provider = provider;
@@ -524,32 +629,65 @@ export async function startModelRelay(options: ModelRelayOptions): Promise<Model
         activeRequests.delete(record);
         return Response.json({ error: error instanceof Error ? error.message : "invalid request" }, { status: 400 });
       }
-      let backend: ModelBackend;
-      try { backend = await resolve(body); } catch (error) {
+      let backend: ModelBackend, auto: AutoChoice | undefined;
+      try { ({ backend, auto } = await resolve(body)); } catch (error) {
         record.cleanup();
         activeRequests.delete(record);
         return Response.json({ error: error instanceof Error ? error.message : "backend unavailable" }, { status: 400 });
       }
-      const inputBudget = backend.kind === "mlx" ? (options.mlx?.maxInputTokens ?? (options.mlx?.provider === "ollama" ? 6000 : 16_000)) : dgxMaxInputTokens;
+      const own = backend;
+      // A refused request still records the decision, unmoved; a dispatched one records it after the ordering step.
+      const emitRoute = () => { if (auto) try { options.onRoute?.(auto.route); } catch { /* observation cannot fail routing */ } };
       const inputTokens = estimateInputTokens(body.messages, body.tools);
-      const ollama = backend.kind === "mlx" && options.mlx?.provider === "ollama";
+      const inputBudget = own.kind === "mlx" ? (options.mlx?.maxInputTokens ?? (options.mlx?.provider === "ollama" ? 6000 : 16_000)) : dgxMaxInputTokens;
+      const ollama = own.kind === "mlx" && options.mlx?.provider === "ollama";
       const contextWindow = ollama ? (options.mlx?.contextWindow ?? 8192) : undefined;
       const configuredMaxTokens = ollama ? (options.mlx?.maxTokens ?? 2048) : undefined;
       const requestedMaxTokens = body.max_tokens === undefined ? configuredMaxTokens : body.max_tokens;
       if (ollama && (!Number.isInteger(requestedMaxTokens) || (requestedMaxTokens as number) < 1 || (requestedMaxTokens as number) > configuredMaxTokens! || inputTokens + (requestedMaxTokens as number) > contextWindow!)) {
+        emitRoute();
         record.cleanup();
         activeRequests.delete(record);
         return Response.json({ error: "input and max_tokens exceed the Ollama context budget" }, { status: 400 });
       }
       if (inputTokens > inputBudget) {
+        emitRoute();
         record.cleanup();
         activeRequests.delete(record);
         return Response.json({ error: "input exceeds the model context budget" }, { status: 400 });
       }
-      const fallback = backend.kind === "mlx" && options.fallbackDGXAlias ? { kind: "dgx", alias: options.fallbackDGXAlias } as ModelBackend : undefined;
+      // #199: one ordering step, for hub/auto only (a fixed alias keeps its backend). The candidates are the route's own
+      // backend and, for MLX, its DGX fallback, exactly as before; a cooldown, an enforced pin or a busy MLX slot only
+      // reorders them. Nothing goes ahead to a fallback that is failing or cooling down.
+      const candidates: ModelBackend[] = [own, ...(own.kind === "mlx" && options.fallbackDGXAlias ? [{ kind: "dgx", alias: options.fallbackDGXAlias } as ModelBackend] : [])];
+      const second = candidates[1];
+      let moved: "optional" | "cooldown" | undefined;
+      const swap = (source: "load" | "cooldown" | "pin") => { candidates.reverse(); moved = source === "cooldown" ? "cooldown" : "optional"; auto?.moved(aliasOf(candidates[0]!, mlxAlias), source); };
+      // A load move or pin is only an optimization: opt-in (`efficientWaitMs`) until #199 AC5 measures it, bounded by its
+      // first-byte deadline, and never under an execution budget, where it could spend the model call or the time MLX
+      // needs after it (a throw counts as a budget).
+      // ponytail: no moves at all under a budget; budget-aware moves from admitPiRequest's remaining units are the upgrade.
+      let room = efficientWaitMs !== undefined;
+      try { room &&= !options.underBudget?.(); } catch { room = false; }
+      let pre: Probe | undefined;
+      if (auto && second && !cooldowns.failing(aliasOf(second, mlxAlias)) && !cooldowns.cooling(aliasOf(second, mlxAlias))) {
+        if (cooldowns.cooling(aliasOf(own, mlxAlias))) swap("cooldown");
+        else if (auto.prefer === aliasOf(second, mlxAlias) && room) swap("pin");
+        else if (auto.movable && mlx && room) {
+          try {
+            const held = await mlx.acquire(controller.signal, efficientWaitMs);
+            let holding = true;
+            pre = { release: () => { if (holding) { holding = false; held(); } } };
+          } catch (error) {
+            if (error instanceof MlxBusyError) swap("load");
+            else pre = { error }; // the MLX dispatch reports it, without inspecting Ollama a second time
+          }
+        }
+      }
+      emitRoute();
       const dispatchGroupId = randomUUID();
       let primaryDispatchId: string | undefined;
-      const dispatch = async (selected: ModelBackend, body: RelayRequest) => {
+      const dispatch = async (selected: ModelBackend, body: RelayRequest, pre?: Probe, firstByteMs?: number) => {
         const journalEntry = openRequestRecord(aliasOf(selected, mlxAlias), dispatchGroupId);
         // The surface is a property of the request as admitted: even a failed dispatch keeps what the
         // native published. A request without a tools array carries no observation, never an empty one.
@@ -565,11 +703,21 @@ export async function startModelRelay(options: ModelRelayOptions): Promise<Model
         record.closeRecord = journalEntry.close;
         let result: Awaited<ReturnType<typeof upstream>>;
         try {
-          result = await upstream(body, selected, controller.signal, journalEntry);
+          result = await upstream(body, selected, controller.signal, journalEntry, pre, firstByteMs);
         } catch (error) {
+          if (pre && "release" in pre) pre.release();
           journalEntry.record.failureClass ??= controller.signal.aborted ? "cancelled" : error instanceof ExecutionAdmissionError ? "admission" : "startup";
+          const failure = journalEntry.record.failureClass;
+          // A busy MLX slot is load, never a failure; neither is a cancelled request, an admission refusal or a budget cut.
+          if (!controller.signal.aborted && !journalEntry.cut && !(error instanceof ExecutionAdmissionError) && !(error instanceof MlxBusyError) && (failure === "transport" || failure === "startup")) {
+            // A DGX alias can fail before it has a status row (gateway or key unavailable); a cooldown must still show.
+            if (!states.has(journalEntry.record.alias)) setState(selected, { state: "error", lastError: error instanceof Error ? error.message.slice(0, 160) : "backend unavailable" });
+            cooldowns.failed(journalEntry.record.alias);
+          } else if (journalEntry.cut === "move") cooldowns.failed(journalEntry.record.alias, false); // slow, not unreachable
+          else if (failure === "http") cooldowns.answered(journalEntry.record.alias, false);
           throw error;
         }
+        cooldowns.answered(journalEntry.record.alias, true);
         let released = false;
         const release = () => {
           if (released) return;
@@ -582,30 +730,30 @@ export async function startModelRelay(options: ModelRelayOptions): Promise<Model
         const onUsage = journalEntry.usage;
         return { result, release, journalEntry, onUsage };
       };
-      try {
-        const { result, release, journalEntry, onUsage } = await dispatch(backend, body);
-        return sseResponse(result.response, release, result.onModel, (cancel) => { record.cancel = cancel; }, journalEntry.close, onUsage);
-      } catch (error) {
-        record.closeRecord?.(controller.signal.aborted ? "cancelled" : "failed");
-        if (!fallback || controller.signal.aborted || error instanceof ExecutionAdmissionError) {
-          record.cleanup();
-          activeRequests.delete(record);
-          return Response.json({ error: error instanceof Error ? error.message : "backend unavailable" }, { status: 502 });
-        }
+      let lastError: unknown;
+      for (const [i, candidate] of candidates.entries()) {
         try {
-          const { result, release, journalEntry, onUsage } = await dispatch(fallback, { ...body, model: fallback.alias });
-          return sseResponse(result.response, release, result.onModel, (cancel) => { record.cancel = cancel; }, journalEntry.close, onUsage);
-        } catch (fallbackError) {
+          const { result, release, journalEntry, onUsage } = await dispatch(candidate, i ? { ...body, model: aliasOf(candidate, mlxAlias) } : body, i ? undefined : pre, i === 0 && moved === "optional" ? moveFirstByteMs : undefined);
+          // A stream that fails after its headers cannot fall back; it still marks the alias failing, so no move follows. A
+          // client that went away can fail the pending read before the cancel arrives: that says nothing about the backend.
+          const close = (outcome: RelayRequestRecord["outcome"]) => { journalEntry.close(outcome); if (outcome === "failed" && !controller.signal.aborted) cooldowns.failed(journalEntry.record.alias, false); };
+          return sseResponse(result.response, release, result.onModel, (cancel) => { record.cancel = cancel; }, close, onUsage);
+        } catch (error) {
           record.closeRecord?.(controller.signal.aborted ? "cancelled" : "failed");
-          record.cleanup();
-          activeRequests.delete(record);
-          return Response.json({ error: fallbackError instanceof Error ? fallbackError.message : "fallback unavailable" }, { status: 502 });
+          lastError = error;
+          if (controller.signal.aborted || error instanceof ExecutionAdmissionError) break;
         }
       }
+      record.cleanup();
+      activeRequests.delete(record);
+      return Response.json({ error: lastError instanceof Error ? lastError.message : "backend unavailable" }, { status: 502 });
     },
   });
   const url = `http://${host}:${server.port}/v1`;
-  const status = (): ModelRelayStatus => ({ url, models, backends: [...states.values()].map((value) => ({ ...value })) });
+  const status = (): ModelRelayStatus => ({ url, models, backends: [...states.values()].map((value) => {
+    const cooling = cooldowns.cooling(value.alias), failing = cooldowns.failing(value.alias);
+    return { ...value, ...(cooling ? { coolingUntil: new Date(cooling.until).toISOString(), failures: cooling.failures } : {}), ...(failing ? { failingUntil: new Date(failing).toISOString() } : {}) };
+  }) });
   const requests = (): RelayRequestRecord[] => journal.map(copyRecord);
   return { url, token, models, status, requests, close: async () => {
     const closing = [...activeRequests].map(async (request) => {

package/src/models/route/relay-selector.ts

@@ -1,17 +1,24 @@
 import type { ModelBackend, ModelRelayOptions, RelayRequest } from "../relay.ts";
 import { normalizeConversation } from "./normalize.ts";
-import { extractToolSignals } from "./signals.ts";
-import { selectStage, type StageState } from "./stage.ts";
+import { extractToolSignals, turnKind } from "./signals.ts";
+import { selectStage, stayOrSwitch, type StageState, type SwitchTrace, type Tier } from "./stage.ts";
 import { SessionState } from "./state.ts";
 
-type RelaySelectorOptions = Pick<ModelRelayOptions, "mlx" | "selectBackend" | "routeSessionKey" | "onRoute" | "allowedDGXmodels">;
-type RouteSource = "override" | "dimensions" | "hold" | "classifier" | "default";
-type RelayRouteEvent = { route: "hub/auto"; tier: string; source: RouteSource; score: number; ms: number };
+type RelaySelectorOptions = Pick<ModelRelayOptions, "mlx" | "selectBackend" | "routeSessionKey" | "allowedDGXmodels" | "staySwitch"> & { dgxMaxInputTokens: number };
+type RouteSource = "override" | "dimensions" | "hold" | "classifier" | "default" | "load" | "cooldown" | "pin";
+/** `stateless`: the request carried no session key, so the planner saw a new session on every call. */
+export type RelayRouteEvent = { route: "hub/auto"; tier: string; source: RouteSource; score: number; ms: number; stateless?: boolean } & Partial<SwitchTrace>;
 type EstimateInputTokens = (messages: RelayRequest["messages"], tools?: unknown[]) => number;
+type AutoState = { stage: StageState; pin?: { tier: Tier; alias: string } };
+/**
+ * A `hub/auto` decision (#199). `backend` is the stage backend; the relay only reorders it with its fallback, trying
+ * `prefer` (an enforced tool loop's pinned backend) first when that is the fallback, and reports a reorder through `moved`.
+ */
+export interface AutoChoice { backend: ModelBackend; route: RelayRouteEvent; movable: boolean; prefer?: string; moved: (alias: string, source: "load" | "cooldown" | "pin") => void }
 
 /** Stage selection for the relay's virtual `hub/auto` model. */
 export class AutoRouteSelector {
-  private readonly states = new SessionState<StageState>(512, 60 * 60_000);
+  private readonly states = new SessionState<AutoState>(512, 60 * 60_000);
 
   constructor(
     private readonly options: RelaySelectorOptions,
@@ -20,33 +27,38 @@ export class AutoRouteSelector {
     private readonly estimateInputTokens: EstimateInputTokens,
   ) {}
 
-  async select(body: RelayRequest): Promise<ModelBackend> {
+  async select(body: RelayRequest): Promise<AutoChoice> {
     const started = performance.now();
     let sessionKey: string | undefined;
     try {
       const key = this.options.routeSessionKey?.(body);
       if (typeof key === "string" && key.length > 0 && key.length <= 512) sessionKey = key;
     } catch { /* host identity lookup is fail-open */ }
-    const priorState = sessionKey ? this.states.get(sessionKey) ?? { capableHoldTurnsRemaining: 0 } : { capableHoldTurnsRemaining: 0 };
+    const prior = sessionKey ? this.states.get(sessionKey) : undefined;
 
     let backend = this.defaultBackend;
-    let tier = this.aliasOf(backend);
     let source: RouteSource = "default";
     let score = 0;
+    let trace: SwitchTrace | undefined;
+    let next: AutoState | undefined;
+    let movable = true;
+    let prefer: string | undefined;
     try {
-      const signals = extractToolSignals(normalizeConversation(body));
-      const decision = selectStage(signals, { mode: "efficient_first", confidenceThreshold: 0.5, capableHoldTurns: 2 }, priorState);
-      if (sessionKey) this.states.set(sessionKey, decision.state);
+      const conversation = normalizeConversation(body);
+      const decision = selectStage(extractToolSignals(conversation), { mode: "efficient_first", confidenceThreshold: 0.5, capableHoldTurns: 2 }, prior?.stage ?? { capableHoldTurnsRemaining: 0 });
+      const inputTokens = this.estimateInputTokens(body.messages, body.tools);
+      const fits = (candidate: Tier) => { const staged = this.stageBackend(candidate, body, inputTokens); return !!staged && (staged.kind === "mlx" || inputTokens <= this.options.dgxMaxInputTokens); };
+      const staged = stayOrSwitch(this.options.staySwitch?.(), prior?.pin?.tier, decision, turnKind(conversation), { inputTokens, fits });
+      next = { stage: decision.state, ...(staged.pin ? { pin: { tier: staged.pin, alias: "" } } : {}) };
+      trace = staged.trace;
       score = decision.score;
       source = this.sourceOf(decision.source);
-      const selectedTier = decision.tier ?? decision.defaultTier;
-      const stagedBackend = this.stageBackend(selectedTier, body);
-      if (stagedBackend) {
-        backend = stagedBackend;
-        tier = this.aliasOf(stagedBackend);
-      } else {
-        source = "default";
-      }
+      // Enforced, a tool loop that keeps its tier keeps its backend too: a move inside the tier costs the same prefill (#199).
+      movable = !(trace.staySwitch === "enforce" && trace.turnType === "tool_result" && trace.plan === "stay" && trace.reason !== "new_pin");
+      if (!movable) prefer = prior?.pin?.alias;
+      const stagedBackend = this.stageBackend(staged.tier, body, inputTokens);
+      if (stagedBackend) backend = stagedBackend;
+      else source = "default";
     } catch {
       source = "default";
     }
@@ -56,27 +68,31 @@ export class AutoRouteSelector {
         const override = await this.options.selectBackend(body);
         if (this.isAllowed(override)) {
           backend = override;
-          tier = this.aliasOf(override);
           source = "override";
         }
       } catch { /* deterministic stage choice is the fail-open route */ }
     }
 
-    this.emit({ route: "hub/auto", tier, source, score, ms: Math.max(0, performance.now() - started) });
-    return backend;
+    if (next?.pin) next.pin.alias = this.aliasOf(backend);
+    if (sessionKey && next) this.states.set(sessionKey, next);
+    const route: RelayRouteEvent = { route: "hub/auto", tier: this.aliasOf(backend), source, score, ms: Math.max(0, performance.now() - started), ...trace, ...(sessionKey ? {} : { stateless: true }) };
+    return { backend, route, movable, ...(prefer ? { prefer } : {}), moved: (alias, moveSource) => {
+      route.tier = alias;
+      route.source = moveSource;
+      if (next?.pin) next.pin.alias = alias;
+    } };
   }
 
-  private stageBackend(tier: "capable" | "efficient", body: RelayRequest): ModelBackend | undefined {
+  private stageBackend(tier: "capable" | "efficient", body: RelayRequest, input: number): ModelBackend | undefined {
     if (tier === "capable") return "dgx/coding" in this.options.allowedDGXmodels ? { kind: "dgx", alias: "dgx/coding" } : undefined;
-    if (this.options.mlx && this.mlxFitsBudget(body)) return { kind: "mlx", alias: this.mlxAlias };
+    if (this.options.mlx && this.mlxFitsBudget(body, input)) return { kind: "mlx", alias: this.mlxAlias };
     return "dgx/fast" in this.options.allowedDGXmodels ? { kind: "dgx", alias: "dgx/fast" } : undefined;
   }
 
-  private mlxFitsBudget(body: RelayRequest): boolean {
+  private mlxFitsBudget(body: RelayRequest, input: number): boolean {
     const context = Math.min(8192, this.options.mlx?.contextWindow ?? 8192);
     const maxOutput = this.options.mlx?.maxTokens ?? 2048;
     const output = body.max_tokens ?? maxOutput;
-    const input = this.estimateInputTokens(body.messages, body.tools);
     const inputLimit = this.options.mlx?.maxInputTokens ?? (this.options.mlx?.provider === "ollama" ? 6000 : 16_000);
     return Number.isInteger(output) && output > 0 && output <= maxOutput && input <= inputLimit && input + output <= context;
   }
@@ -95,8 +111,4 @@ export class AutoRouteSelector {
     if (source === "llm-classifier") return "classifier";
     return "default";
   }
-
-  private emit(event: RelayRouteEvent): void {
-    try { this.options.onRoute?.(event); } catch { /* observation cannot fail routing */ }
-  }
 }