agent-relay-server 0.117.1 → 0.118.1

package/src/task-dispatcher.ts

@@ -19,6 +19,7 @@ import type {
   DispatchModelTarget,
   DispatchPolicy,
   DispatchRule,
+  HostUtilizationSignal,
   SpawnProvider,
   TaskDispatch,
 } from "agent-relay-sdk";
@@ -42,12 +43,31 @@ export interface DispatchContext {
   policy?: DispatchPolicy;
   bands?: ProviderQuotaBand[];
   catalog?: ProviderCatalogEntry[];
+  /**
+   * #901 degrade-never-refuse: when true, the dispatcher ALWAYS returns a usable provider —
+   * widening the search through caution → avoid-large → least-utilized blockedNow tiers before
+   * giving up. A loud warning is emitted for each degraded tier used. Non-critical paths (flag
+   * unset or false) keep the existing stricter behaviour (prefer|normal only).
+   */
+  mustResolveAvailable?: boolean;
+  /**
+   * #901 host-placement seam. Shape-only in v1 — no selection logic reads this field yet.
+   * A future host-placement phase will inject live host load here so the dispatcher can
+   * prefer the least-loaded machine when multiple orchestrators are viable.
+   */
+  hosts?: HostUtilizationSignal[];
 }
 
 export interface DispatchTarget {
   provider: SpawnProvider;
   model?: string;
   effort?: string;
+  /**
+   * #901 host-placement seam. Stays undefined in v1 — a future host-placement phase will
+   * populate this with the chosen orchestrator id when multiple hosts are viable.
+   * No selection logic writes or reads this field in v1.
+   */
+  orchestratorId?: string;
 }
 
 /** Structured trace of the inputs that produced the decision (the machine-readable rationale). */
@@ -131,7 +151,12 @@ export function dispatchTask(shape: TaskDispatch | undefined, pin: DispatchPin =
   // Band-aware rerouting (skip when the provider is explicitly pinned). Protects scarce pools
   // and load-shifts large/blocked work onto an abundant one. Re-validates footprint + a pinned
   // effort on the rerouted provider so neither is silently dropped.
-  if (!providerPinned) rerouteForBand(target, work, policy, bands, catalog, factors, warnings, { effortPinned });
+  if (!providerPinned) rerouteForBand(target, work, policy, bands, catalog, factors, warnings, { effortPinned, mustResolveAvailable: ctx.mustResolveAvailable });
+
+  // Guarantee the resolved model is enabled. If the routing policy or reroute picked a model
+  // that has been disabled/unavailable in the effective catalog, swap to the first enabled model
+  // on the chosen provider. Skip when the model was explicitly pinned — caller's choice wins.
+  if (!modelPinned) ensureEnabledModel(target, catalog, warnings);
 
   // Record the FINAL provider's band + the chosen model's window for transparency.
   factors.band = bandFor(target.provider, bands);
@@ -248,6 +273,33 @@ function escalateForFootprint(
   );
 }
 
+/** True when the provider has at least one model that is not disabled or unavailable. */
+function hasEnabledModel(catalog: ProviderCatalogEntry[], provider: SpawnProvider): boolean {
+  const entry = catalogEntry(catalog, provider);
+  return !!entry && entry.models.some((m) => !m.disabled && !m.unavailable);
+}
+
+/**
+ * Validate that `target.model` is enabled on the chosen provider. When it is disabled or
+ * unavailable, replace it with the first enabled model and emit a warning. Callers must skip
+ * this when the model was explicitly pinned (pin takes precedence, even if broken).
+ */
+function ensureEnabledModel(target: DispatchTarget, catalog: ProviderCatalogEntry[], warnings: string[]): void {
+  const entry = catalogEntry(catalog, target.provider);
+  if (!entry) return;
+  const alias = target.model ?? entry.defaultModel;
+  if (!alias) return;
+  const model = entry.models.find((m) => m.alias === alias || m.providerModel === alias);
+  if (model && !model.disabled && !model.unavailable) return;
+  const enabled = entry.models.find((m) => !m.disabled && !m.unavailable);
+  if (enabled) {
+    warnings.push(`Model ${alias} is disabled/unavailable on ${target.provider}; resolved to ${enabled.alias} instead.`);
+    target.model = enabled.alias;
+  } else {
+    warnings.push(`No enabled models on ${target.provider}; spawn may fail.`);
+  }
+}
+
 // The smallest-window model on this provider whose window meets the requirement. Prefers the
 // same model family as `alias` (e.g. opus-4.8 → opus-4.8[1m]) when one qualifies, else any family.
 function biggestSufficient(
@@ -281,7 +333,7 @@ function rerouteForBand(
   catalog: ProviderCatalogEntry[],
   factors: DispatchFactors,
   warnings: string[],
-  opts: { effortPinned: boolean },
+  opts: { effortPinned: boolean; mustResolveAvailable?: boolean },
 ): void {
   const band = bandFor(target.provider, bands);
   const blocked = isBlocked(target.provider, bands);
@@ -290,11 +342,16 @@ function rerouteForBand(
   const mustMove = blocked || band === "hard-avoid" || (isLarge && band === "avoid-large");
   if (!mustMove) return;
 
-  const alt = healthiestAlternative(target.provider, bands, catalog);
+  const alt = healthiestAlternative(target.provider, bands, catalog, opts.mustResolveAvailable);
   if (!alt) {
     warnings.push(`${target.provider} band=${band}${blocked ? " (at capacity)" : ""} and no healthier provider is available — keeping it; expect contention.`);
     return;
   }
+  if (alt.degraded) {
+    warnings.push(
+      `[CRITICAL] mustResolveAvailable: all prefer/normal providers unavailable; degraded to ${alt.provider} (${alt.degradedReason ?? "degraded tier"}). Contention likely.`,
+    );
+  }
   const altEntry = catalogEntry(catalog, alt.provider)!;
   factors.reroutedFromProvider = target.provider;
   warnings.push(`Rerouted ${target.provider} (band=${band}${blocked ? ", at capacity" : ""}) → ${alt.provider} (band=${alt.band}) to ${alt.band === "prefer" ? "use abundant quota" : "avoid the scarce pool"}.`);
@@ -323,18 +380,48 @@ function rerouteForBand(
 // catalog — not only those with a quota advisory. A provider with no quota record defaults to the
 // "normal" band (via bandFor), so a healthy catalog provider is never skipped just because it has
 // no burn data yet. Prefer "prefer" band, then "normal"; never a blocked one.
+//
+// When mustResolveAvailable is set (#901 degrade-never-refuse), widens search in tiers when
+// no prefer/normal provider is available: caution → avoid-large → least-utilized blockedNow.
+// A `degraded` flag + `degradedReason` signals to the caller to emit a louder warning.
 function healthiestAlternative(
   avoid: SpawnProvider,
   bands: ProviderQuotaBand[],
   catalog: ProviderCatalogEntry[],
-): { provider: SpawnProvider; band: QuotaBand } | undefined {
-  const ranked = catalog
+  mustResolveAvailable?: boolean,
+): { provider: SpawnProvider; band: QuotaBand; degraded?: boolean; degradedReason?: string } | undefined {
+  const all = catalog
     .map((c) => c.provider)
     .filter((provider) => provider !== avoid)
-    .map((provider) => ({ provider, band: bandFor(provider, bands), blockedNow: isBlocked(provider, bands) }))
+    .filter((provider) => hasEnabledModel(catalog, provider)) // skip providers with all models disabled
+    .map((provider) => ({ provider, band: bandFor(provider, bands), blockedNow: isBlocked(provider, bands) }));
+
+  const standard = all
     .filter((c) => !c.blockedNow && (c.band === "prefer" || c.band === "normal"))
     .sort((a, b) => bandRank(a.band) - bandRank(b.band));
-  return ranked[0];
+  if (standard[0]) return standard[0];
+
+  if (!mustResolveAvailable) return undefined;
+
+  // Tier 1 degradation: caution (over quota but not avoid/blocked)
+  const cautionAlt = all
+    .filter((c) => !c.blockedNow && c.band === "caution")
+    .sort((a, b) => bandRank(a.band) - bandRank(b.band));
+  if (cautionAlt[0]) return { ...cautionAlt[0], degraded: true, degradedReason: "caution band (tier 1 degraded)" };
+
+  // Tier 2 degradation: avoid-large
+  const avoidLargeAlt = all
+    .filter((c) => !c.blockedNow && c.band === "avoid-large")
+    .sort((a, b) => bandRank(a.band) - bandRank(b.band));
+  if (avoidLargeAlt[0]) return { ...avoidLargeAlt[0], degraded: true, degradedReason: "avoid-large band (tier 2 degraded)" };
+
+  // Tier 3 degradation: least-utilized blockedNow (last resort — emit the loudest warning).
+  // Sort by bandRank so the healthiest (lowest penalty) blocked provider is picked first,
+  // not whichever happens to be earliest in catalog order.
+  const blockedAlts = all.filter((c) => c.blockedNow).sort((a, b) => bandRank(a.band) - bandRank(b.band));
+  if (blockedAlts[0]) return { ...blockedAlts[0], degraded: true, degradedReason: "blockedNow provider (tier 3 last-resort — ALL providers at capacity)" };
+
+  return undefined;
 }
 
 function collectBandWarnings(provider: SpawnProvider, work: TaskDispatch, bands: ProviderQuotaBand[], providerPinned: boolean, warnings: string[]): void {

package/src/utils.ts

@@ -1,6 +1,12 @@
-import { isAbsolute, join, relative, resolve } from "node:path";
+import { join } from "node:path";
 import { homedir } from "node:os";
 
+// `isPathWithinBase` and `parseJson` now live in the SDK (agent-relay-sdk) as the
+// single canonical home shared with the orchestrator package; re-exported here so
+// the server's existing importers stay byte-unchanged. Never re-declare them — the
+// path-containment helper in particular has a history of duplicated-copy bugs.
+export { isPathWithinBase, parseJson } from "agent-relay-sdk";
+
 /**
  * Expand a leading `~` / `~/` to the current user's home directory. Leaves any
  * other value (absolute paths, `./rel`, embedded `~` mid-string) untouched.
@@ -14,40 +20,3 @@ export function expandTilde(value: string): string {
   if (value.startsWith("~/")) return join(homedir(), value.slice(2));
   return value;
 }
-
-/**
- * Path containment check — is `path` inside (or equal to) `baseDir`?
- *
- * Both inputs are `resolve()`d first for defensive clarity. (Note: `path.relative`
- * already `resolve()`s both args against cwd internally, so this is belt-and-
- * suspenders, not a behavior change — verified: the `security.ts` copy that
- * omitted the explicit `resolve()` produced identical results. The audit's
- * "bug #4" was a false alarm.) Returns `false` when either input is missing.
- *
- * Was hand-copied in 6 server modules (security, workspace-merge, mcp,
- * lifecycle-manager, steward, routes) under three different names. Import this;
- * never re-declare it.
- */
-export function isPathWithinBase(path: string | undefined, baseDir: string | undefined): boolean {
-  if (!path || !baseDir) return false;
-  const rel = relative(resolve(baseDir), resolve(path));
-  return rel === "" || (!!rel && !rel.startsWith("..") && !isAbsolute(rel));
-}
-
-/**
- * Parse JSON, returning `fallback` on any failure (non-string input, empty
- * string, or malformed JSON). Never throws.
- *
- * Folds the four fallback-style `parseJson` copies (db, memory-sqlite-broker,
- * provider-catalog-store, automations). The throwing / Buffer / null-returning
- * variants (memory-*-broker operation-context, control-server, insights-db) are
- * deliberately separate — different contracts.
- */
-export function parseJson<T>(raw: unknown, fallback: T): T {
-  if (typeof raw !== "string" || !raw) return fallback;
-  try {
-    return JSON.parse(raw) as T;
-  } catch {
-    return fallback;
-  }
-}

package/src/validation.ts

@@ -1,33 +1,14 @@
 import { ValidationError } from "./db/connection.ts";
 import { type TokenConstraints, type WorkspaceAutoMergePolicy, type WorkspaceLandingPolicy, type WorkspaceMetadata, type WorkspaceMode, type WorkspaceProbe, type WorkspaceStatus } from "./types";
-import { AUTO_MERGE_POLICIES, VALID_WORKSPACE_MODES, isRecord } from "agent-relay-sdk";
+import { AUTO_MERGE_POLICIES, VALID_WORKSPACE_MODES, cleanString, isRecord } from "agent-relay-sdk";
 
 const WORKSPACE_LAND_STRATEGIES = ["direct", "pr"] as const;
 
-/**
- * Trim + validate an optional string input. Throws `ValidationError` when the
- * value is present but not a string, exceeds `max`, or is required-but-empty.
- * Returns `undefined` for absent/blank non-required values.
- *
- * Single home — was byte-identical in config-store, automations, and routes.
- */
-export function cleanString(
-  value: unknown,
-  field: string,
-  opts: { required?: boolean; max?: number } = {},
-): string | undefined {
-  if (value === undefined || value === null) {
-    if (opts.required) throw new ValidationError(`${field} required`);
-    return undefined;
-  }
-  if (typeof value !== "string") throw new ValidationError(`${field} must be a string`);
-  const trimmed = value.trim();
-  if (opts.required && !trimmed) throw new ValidationError(`${field} required`);
-  if (opts.max && trimmed.length > opts.max) {
-    throw new ValidationError(`${field} must be ${opts.max} characters or fewer`);
-  }
-  return trimmed || undefined;
-}
+// `cleanString` now lives in the SDK (agent-relay-sdk) as the single canonical home
+// shared with the orchestrator package; imported above for the local validators here and
+// re-exported so existing importers stay byte-unchanged. The other validators keep their
+// server home.
+export { cleanString };
 
 /**
  * Validate a REQUIRED enum value: throws `ValidationError` if missing or not one