@humwick/server 0.0.0-stage → 0.130.1

package/src/spawn-targets.ts

@@ -0,0 +1,679 @@
+import { apiBudgetStatus } from "./db/claude-api-budget";
+import { apiBudgetStatusAsync } from "./claude-api-budget-async";
+import { isOpenAiApiModel, type ApiBudgetStatus } from "@humwick/sdk";
+import { collapseProviderQuotaRows } from "@humwick/sdk/provider-quota-order";
+import { resolveQuotaPacing, subscriptionExpiryForWindow, subscriptionExpiryPreference, type ResolvedQuotaPacing } from "./quota-pacing";
+import { countLiveSpawnedAgents, countLiveSpawnedAgentsAsync, getAgent, getAgentAsync, listOrchestrators, listOrchestratorsAsync, listProviderQuotas, providerQuotaRoutingSnapshotAsync } from "./db";
+import { listHostViews } from "./db/hosts-views";
+import { listHostViewsAsync } from "./db/hosts-async";
+import { DEFAULT_SPAWN_PROFILE_NAME, listAgentProfiles } from "./config-store";
+import { profileProviderRefusal } from "./agent-profile-catalog";
+import { liveHostDisabledProviders } from "./host-provider-store";
+import { effectiveProviderCatalogList } from "./provider-catalog-store";
+import { disabledProviderSet, resolveDefaultModel, type ProviderCatalogEntry } from "@humwick/sdk/provider-catalog";
+import { isHiddenAgentProfile } from "@humwick/sdk/types";
+import { getRoutingPolicy } from "./routing-policy-store";
+import { windowRecommendedAction } from "./quota-band-projection";
+import type { ProviderBandInfo, QuotaBand } from "./quota-band-projection";
+export type { ProviderBandInfo, QuotaBand } from "./quota-band-projection";
+import { sameHostIdentity } from "./utils";
+import { hostIdentitiesById, hostIdentityFor, spawnEligible, liveHostIdentities } from "./spawn-placement";
+import { hostAgentSlots } from "./host/capacity";
+export { resolveSpawnHostAnchor, selectSpawnOrchestrator, selectSpawnPlacement } from "./spawn-placement";
+import {
+  DEFAULT_QUOTA_PREFER_SLACK_AT,
+  DEFAULT_QUOTA_TARGET_UTILIZATION,
+  formatPaceLabel,
+  quotaOvershootPenalty,
+  quotaPaceBandForWeight,
+  quotaPaceCombinedBand,
+  quotaPaceMetrics,
+  quotaPaceWindowMetrics,
+  type QuotaPaceBand,
+  type QuotaPaceRoleSignal,
+} from "@humwick/sdk/provider-quota";
+import { isSpawnProvider, type HostIdentity, type HostView, type ProviderQuotaRoutingConfig, type RoutingPolicy } from "@humwick/sdk";
+import { providerSupportsApprovalMode, providerSupportsQuota, supportedLaunchModesForProvider } from "@humwick/providers";
+import { providerProbeAdmission } from "./spawn-admission";
+import { hostRunnableCatalog } from "./spawn-models";
+import { resolveSpawnLaunchMode } from "./launch-mode";
+import type { Orchestrator, ProviderQuotaBurnRole, ProviderQuotaBurnWindow, ProviderQuotaRecord, ProviderQuotaWindowStatus, QuotaState, SpawnProvider } from "./types";
+
+// #308 — spawn discovery and the scope-gated startup primer. Placement lives in
+// spawn-placement.ts; the legacy exports below preserve existing callers. Lives here so the
+// already-large src/mcp.ts only carries the thin auth wrappers that call in here.
+
+// === relay_spawn_targets: the live spawn capability matrix ===
+
+// Reuses the per-host catalog each orchestrator already reports on heartbeat (orchestrator-probed
+// availability, not a static config guess), so the agent learns what it can spawn (and where)
+// upfront instead of from a runtime rejection. Caller identity + quota come from the MCP auth.
+export function buildSpawnTargets(input: { callerId?: string; quota: number }): Record<string, unknown> {
+  const now = Date.now();
+  const me = input.callerId ? getAgent(input.callerId) : undefined;
+  return buildSpawnTargetsFrom(input, {
+    me,
+    openaiBudget: apiBudgetStatus(now, "openai-api"),
+    orchestrators: listOrchestrators(),
+    hostViews: listHostViews(now),
+    liveChildren: input.callerId ? countLiveSpawnedAgents(input.callerId) : 0,
+    quotaAdvisories: providerQuotaAdvisories(),
+    policy: getRoutingPolicy(),
+    catalog: effectiveProviderCatalogList(),
+    profiles: listAgentProfiles(),
+  });
+}
+
+/** Worker-safe discovery path: every durable input is read through the reader gateway. */
+export async function buildSpawnTargetsAsync(input: { callerId?: string; quota: number }): Promise<Record<string, unknown>> {
+  const now = Date.now();
+  const [me, orchestrators, hostViews, liveChildren, snapshot, openaiBudget] = await Promise.all([
+    input.callerId ? getAgentAsync(input.callerId) : Promise.resolve(null),
+    listOrchestratorsAsync(),
+    listHostViewsAsync(),
+    input.callerId ? countLiveSpawnedAgentsAsync(input.callerId) : Promise.resolve(0),
+    providerQuotaRoutingSnapshotAsync(now),
+    apiBudgetStatusAsync("openai-api"),
+  ]);
+  return buildSpawnTargetsFrom(input, {
+    me: me ?? undefined,
+    openaiBudget,
+    orchestrators,
+    hostViews,
+    liveChildren,
+    quotaAdvisories: providerQuotaAdvisoriesFor(snapshot.matrix.quotas, snapshot.policy, now),
+    policy: snapshot.policy,
+    catalog: snapshot.catalog,
+    profiles: snapshot.profiles,
+  });
+}
+
+function buildSpawnTargetsFrom(input: { callerId?: string; quota: number }, data: {
+  me?: ReturnType<typeof getAgent>;
+  openaiBudget: ApiBudgetStatus;
+  orchestrators: Orchestrator[];
+  hostViews: readonly HostView[];
+  liveChildren: number;
+  quotaAdvisories: Map<SpawnProvider, SpawnQuotaAdvisory>;
+  policy: RoutingPolicy;
+  catalog: ReturnType<typeof effectiveProviderCatalogList>;
+  profiles: ReturnType<typeof listAgentProfiles>;
+}): Record<string, unknown> {
+  const hostIdentities = hostIdentitiesById(data.hostViews);
+  const viewById = new Map(data.hostViews.map((view) => [view.id, view]));
+  const preferHost = data.me?.machine;
+  const self = preferHost ? data.orchestrators.find((o) => sameHostIdentity(o.hostname, preferHost) || sameHostIdentity(o.id, preferHost)) : undefined;
+  // #2855 — read once per matrix build, not once per host.
+  const hostDisabled = liveHostDisabledProviders();
+
+  return {
+    self: {
+      orchestratorId: self?.id,
+      host: self?.hostname ?? preferHost,
+      spawnQuota: { max: input.quota, liveChildren: data.liveChildren, remaining: Math.max(0, input.quota - data.liveChildren) },
+    },
+    orchestrators: data.orchestrators
+      .filter((o) => o.status === "online")
+      .map((o) => {
+        const identity = hostIdentityFor(o, hostIdentities);
+        const view = o.hostId ? viewById.get(o.hostId) : undefined;
+        return {
+          orchestratorId: o.id,
+          host: o.hostname,
+          online: true,
+          identity,
+          spawn_eligible: identity.matched,
+          isSelf: self ? o.id === self.id : false,
+          // #2694 — host-wide live agents vs the operator cap; #2415's per-daemon concurrent-spawn
+          // semaphore is `spawn_slots`, absent until a generation reports those fields.
+          ...(view ? { agent_slots: hostAgentSlots(view) } : {}),
+          ...(o.utilization && Number.isFinite(o.utilization.agentSlotsUsedThisDaemon) && Number.isFinite(o.utilization.agentSlotsMaxThisDaemon) && o.utilization.agentSlotsMaxThisDaemon > 0
+            ? { spawn_slots: { used: o.utilization.agentSlotsUsedThisDaemon, max: o.utilization.agentSlotsMaxThisDaemon } }
+            : {}),
+          providers: providerMatrixFor(o, data.quotaAdvisories, data.catalog, data.profiles, hostDisabled, data.openaiBudget),
+        };
+      }),
+    apiBudgets: [data.openaiBudget],
+    ...operatorDisabledProviders(data.catalog),
+    profiles: spawnProfilesSummary(data.profiles),
+    // #3327 — preferences steer automatic spawns inside the dispatcher; never copy them into explicit pins.
+    routing: {
+      schemaVersion: data.policy.schemaVersion,
+      preferences: data.policy.rules.filter((rule) => rule.enabled !== false),
+      guidance: "Omit provider/model/effort (provider:auto) and give a work shape (Task dispatch or inline dispatch); the dispatcher applies these preferences within quota bands.",
+    },
+    guidance:
+      `Omit orchestratorId to place on the eligible host with the most free capacity; set it for an intentional host pin. Shared workspaces stay on your own host. Omitting profile applies ${DEFAULT_SPAWN_PROFILE_NAME}; omitting model/effort uses the provider default (marked below). Prefer a named profile over assembling raw knobs; set spawnRequestId for idempotency. After spawning you'll be pushed agent.ready / agent.spawn_unconfirmed / agent.exited / agent.spawn_failed — an unconfirmed registration may still start, so don't re-spawn it.`,
+  };
+}
+
+// #2516 — providers at least one launch-ELIGIBLE orchestrator advertises (its heartbeat-reported
+// `providers` list, CLI-availability-driven — the same signal relay_spawn_targets' per-host
+// provider matrix and host selection use). An online host with an identity mismatch is deliberately
+// not live availability: choosing its sole provider would only fail later at the launch gate.
+// Static manifest/catalog presence is NOT live availability: claude-b is a registered manifest
+// on every install, so routing must consult this roster (not the catalog) before preferring it.
+// #2656 — a provider the effective catalog marks disabled is never live availability either.
+export function advertisedProvidersFrom(
+  orchestrators: Orchestrator[],
+  identities: ReadonlyMap<string, HostIdentity> = liveHostIdentities(),
+  catalog: readonly ProviderCatalogEntry[] = [],
+  hostDisabled: ReadonlyMap<string, ReadonlySet<SpawnProvider>> = liveHostDisabledProviders(),
+): Set<SpawnProvider> {
+  const disabled = disabledProviderSet(catalog);
+  return new Set(orchestrators.filter((o) => spawnEligible(o, identities))
+    .flatMap((o) => o.providers.filter((p) => !disabled.has(p) && !hostDisabled.get(o.id)?.has(p))));
+}
+
+/** #2656 — disabled providers vanish from the matrix; say why once so a caller isn't left guessing. */
+function operatorDisabledProviders(catalog: readonly ProviderCatalogEntry[]): { disabledProviders?: Array<Record<string, unknown>> } {
+  const disabled = catalog.filter((entry) => entry.disabled).map((entry) => ({ provider: entry.provider, ...entry.disabled }));
+  return disabled.length ? { disabledProviders: disabled } : {};
+}
+
+/** #2656 — routing-facing quota projections never advise a disabled provider. */
+function enabledProviderQuotas(records: ProviderQuotaRecord[], catalog: readonly ProviderCatalogEntry[]): ProviderQuotaRecord[] {
+  const disabled = disabledProviderSet(catalog);
+  return disabled.size ? records.filter((record) => !disabled.has(record.provider as SpawnProvider)) : records;
+}
+
+/** Live sync default. Tolerant of an uninitialized DB (pure-function unit harnesses inject
+ * `DispatchContext.advertisedProviders` instead): no DB → no online hosts → no family fallback. */
+export function advertisedProviders(): Set<SpawnProvider> {
+  try {
+    return advertisedProvidersFrom(listOrchestrators(), liveHostIdentities(), effectiveProviderCatalogList());
+  } catch {
+    return new Set();
+  }
+}
+
+// #760 — "prefer" is the underburn counterpart to the over-pace penalty ladder.
+type SpawnQuotaBurnBand = "prefer" | "normal" | "caution" | "avoid-large" | "hard-avoid";
+
+interface SpawnQuotaBurnWindow {
+  expiryPreference?: string;
+  role: ProviderQuotaBurnRole;
+  name: string;
+  accountKey: string;
+  utilization: number;
+  resetsAt: number | null;
+  timeToResetMs: number | null;
+  confidence: ProviderQuotaBurnWindow["confidence"];
+  burnRatio: number | null;
+  timeToWallMs: number | null;
+  projectedWallAt: number | null;
+  earlyWallMs: number | null;
+  // #760 — symmetric underburn signal (see ProviderQuotaBurnWindow.slack), re-derived against
+  // the per-provider target. #1098: demoted to a tiebreaker/secondary signal — still surfaced
+  // (and still the input to the "prefer" overlay below) but no longer the primary band driver.
+  projectedUtilAtReset: number | null;
+  slack: number | null;
+  // #1098 — the pace-error control signal: present position vs. the ideal linear pace line,
+  // and the legibility label ("+9% ahead of pace" beats "projected 115%"). null when timeToReset
+  // is unknown (no reset boundary to pace against).
+  paceTarget: number | null;
+  paceError: number | null;
+  paceWeight: number | null;
+  pace: string | null;
+  prefer: boolean;
+  penalty: number;
+  blockedNow: boolean;
+  // #2496 — this window's utilization is below the per-provider burnPenaltyUtilizationFloor, so it
+  // must not contribute an escalation signal (caution/avoid-large/hard-avoid) to the provider-level
+  // band — see the `signals` filter in quotaAdvisoryFor, the actual floor gate.
+  belowPenaltyFloor: boolean;
+}
+
+interface SpawnQuotaAdvisory {
+  expiryPreference?: string;
+  quotaBurn: { windows: SpawnQuotaBurnWindow[]; resetCredits: number };
+  quotaFreshness: {
+    lastUpdatedAt: number | null;
+    stale: boolean;
+    unavailable: boolean;
+    windows: Array<ProviderQuotaWindowStatus & { accountKey: string }>;
+  };
+  quotaPenalty: number;
+  quotaPenaltyBand: SpawnQuotaBurnBand;
+  // #760 — provider underburn: every selected window projects ≥ preferSlackAt headroom at reset.
+  prefer: boolean;
+  slack: number | null;
+  blockedNow: boolean;
+  recommendedAction: string;
+}
+
+// #2441 — a provider with NO quota records at all is either (a) quota-supported but genuinely
+// unprobed/stale, which really is an outage-shaped signal, or (b) quota-less by design (e.g.
+// opencode — quotaPoll.strategy "none", no probe ever writes a record for it), which is just its
+// normal steady state and must not read as unavailable/stale. Branch on the manifest, not on the
+// record count.
+function defaultQuotaAdvisory(provider: SpawnProvider): SpawnQuotaAdvisory {
+  const quotaLess = !providerSupportsQuota(provider);
+  return {
+    quotaBurn: { windows: [], resetCredits: 0 },
+    quotaFreshness: { lastUpdatedAt: null, stale: !quotaLess, unavailable: !quotaLess, windows: [] },
+    quotaPenalty: 0,
+    quotaPenaltyBand: "normal",
+    prefer: false,
+    slack: null,
+    blockedNow: false,
+    recommendedAction: quotaLess
+      ? "Provider does not report quota; spawn normally."
+      : "No quota burn projection available; spawn normally.",
+  };
+}
+
+// #1098/#1105 — resolved pacing tunables: delta/thresholds/windowLengthMs are GLOBAL (one dial per
+// relay instance, not per-provider — matches the design's "δ a tunable constant"), sourced from
+// RoutingPolicy.quotaPacing with the provider-quota.ts constants as fallback defaults, so an
+// unconfigured instance reproduces the shipped behaviour exactly.
+
+function providerMatrixFor(
+  o: Orchestrator,
+  quotaAdvisories: Map<SpawnProvider, SpawnQuotaAdvisory>,
+  global: ReturnType<typeof effectiveProviderCatalogList> = effectiveProviderCatalogList(),
+  profiles: ReturnType<typeof listAgentProfiles> = listAgentProfiles(),
+  hostDisabled: ReadonlyMap<string, ReadonlySet<SpawnProvider>> = liveHostDisabledProviders(),
+  openaiBudget?: ApiBudgetStatus,
+): Array<Record<string, unknown>> {
+  // #1254 — per-provider login-validity signal, keyed off the orchestrator's own heartbeat-reported
+  // probe status. Deliberately kept SEPARATE from `available`/`o.providers` (CLI-availability-driven):
+  // a host's CLI can run fine while its login is expired — exactly the zombie-worker bug this surfaces
+  // so a coordinator can avoid the host proactively, ahead of the spawn-time hard gate (#1254 also adds
+  // to orchestrator/src/spawn/spawn-agent.ts) rejecting it loudly.
+  const providerStatusByName = new Map((o.providerStatus ?? []).map((status) => [status.name, status]));
+  const disabledProviders = disabledProviderSet(global);
+  // #2855 — a provider the operator turned off for THIS host reads as absent
+  // here, even though the host still heartbeat-advertises it (orchestrator
+  // applies the central setting on its next pull; this is the server-side bar).
+  const hostDisabledFor = hostDisabled.get(o.id);
+  const providers = [...new Set([...o.providers, ...(o.providerStatus ?? []).filter((s) => s.degraded || s.cli?.transient || s.runner?.transient || s.auth?.transient).map((s) => s.name)])];
+  return providers.filter((provider) => !disabledProviders.has(provider) && !hostDisabledFor?.has(provider)).map((provider) => {
+    const globalEntry = global.find((e) => e.provider === provider);
+    // Relay owns model settings; the heartbeat limits what this host can run (#3202).
+    const cat = globalEntry ? hostRunnableCatalog(globalEntry, o) : undefined;
+    const quotaAdvisory = quotaAdvisories.get(provider) ?? defaultQuotaAdvisory(provider);
+    // #932 — advertise the SAME default a bare spawn would actually resolve to. resolveDefaultModel
+    // runs against the effective (DB-override-aware) catalog — the exact source spawn validation
+    // (resolveProviderSelection) reads — so the reported default can never drift from the spawned
+    // one and can never name a disabled/unavailable model. Fall back to the merged catalog's static
+    // default only when the provider is absent from the effective list (should not happen).
+    const resolvedDefault = cat ? resolveDefaultModel(cat).alias : undefined;
+    const auth = providerStatusByName.get(provider)?.auth;
+    const probe = providerProbeAdmission(o, provider);
+    return {
+      provider,
+      available: o.providers.includes(provider),
+      ...(providerStatusByName.get(provider)?.degraded ? { degraded: providerStatusByName.get(provider)!.degraded } : {}),
+      ...(probe.uncertainties.length ? { probeUncertainties: probe.uncertainties } : {}),
+      // Only advertise a named profile on a provider where its declared approval
+      // posture can actually launch. The server repeats this gate before command
+      // creation; discovery must not invite an impossible combination.
+      compatibleProfiles: profiles
+        .filter(({ value }) => !profileProviderRefusal(value.name, provider)
+          && (!value.permissions?.mode || providerSupportsApprovalMode(provider, value.permissions.mode)))
+        .map(({ value, key }) => value.name ?? key),
+      ...launchModeAdvert(provider),
+      ...(auth ? { authOk: auth.ok, ...(auth.ok ? {} : { authReason: auth.transient ? `login probe unknown: ${auth.error ?? "probe failed transiently"}; retry` : auth.error ?? "provider login invalid or expired" }) } : {}),
+      ...quotaAdvisory,
+      ...(cat
+        ? {
+            defaultModel: resolvedDefault,
+            models: cat.models
+              .map((m) => ({
+                id: m.alias,
+                providerModel: m.providerModel,
+                ...(isOpenAiApiModel(provider, m.alias) ? { backend: "openai", billing: "api", explicitOnly: true,
+                  budgetId: "openai-api", headroomMicroUsd: openaiBudget?.headroomMicroUsd ?? null,
+                  endsAt: openaiBudget?.endsAt, refusalReason: openaiBudget?.refusalReason ?? (!openaiBudget ? "budget-accounting-unknown" : undefined) } : {}),
+                default: resolvedDefault === m.alias,
+                effortLevels: m.efforts,
+                ...(m.defaultEffort ? { defaultEffort: m.defaultEffort } : {}),
+              })),
+          }
+        : { models: [] }),
+    };
+  });
+}
+
+/** #2773 — offered launch modes + the mode an omitted `launchMode` resolves to; absent for single-mode providers. */
+function launchModeAdvert(provider: SpawnProvider): { launchModes?: string[]; defaultLaunchMode?: string } {
+  const launchModes = supportedLaunchModesForProvider(provider);
+  return launchModes.length ? { launchModes, defaultLaunchMode: resolveSpawnLaunchMode(provider, undefined) } : {};
+}
+
+export function providerQuotaAdvisories(now = Date.now()): Map<SpawnProvider, SpawnQuotaAdvisory> {
+  return providerQuotaAdvisoriesFor(enabledProviderQuotas(listProviderQuotas({ historyLimit: 0, now }).quotas, effectiveProviderCatalogList(now)), getRoutingPolicy(), now);
+}
+
+/** Reader-pool live-state shell. The pure projection remains synchronous and is shared by all
+ * callers that already supply a snapshot. */
+export async function providerQuotaAdvisoriesAsync(now = Date.now()): Promise<Map<SpawnProvider, SpawnQuotaAdvisory>> {
+  const { matrix, policy, catalog } = await providerQuotaRoutingSnapshotAsync(now);
+  return providerQuotaAdvisoriesFor(enabledProviderQuotas(matrix.quotas, catalog), policy, now);
+}
+
+export function providerQuotaAdvisoriesFor(records: ProviderQuotaRecord[], policy: RoutingPolicy, now: number): Map<SpawnProvider, SpawnQuotaAdvisory> {
+  const byProvider = new Map<SpawnProvider, ProviderQuotaRecord[]>();
+  for (const record of collapseProviderQuotaRows(records)) {
+    if (!isSpawnProvider(record.provider)) continue;
+    byProvider.set(record.provider, [...(byProvider.get(record.provider) ?? []), record]);
+  }
+  const pacing = resolveQuotaPacing(policy);
+  return new Map([...byProvider.entries()].map(([provider, records]) => [
+    provider,
+    quotaAdvisoryFor(provider, records, now, pacing, policy.providerQuota?.[provider]),
+  ]));
+}
+
+function quotaAdvisoryFor(
+  provider: SpawnProvider,
+  records: ProviderQuotaRecord[],
+  now: number,
+  pacing: ResolvedQuotaPacing,
+  providerCfg?: ProviderQuotaRoutingConfig,
+): SpawnQuotaAdvisory {
+  const selected = new Map<ProviderQuotaBurnRole, SpawnQuotaBurnWindow>();
+  for (const record of records) {
+    for (const window of quotaWindowsFor(record, now, providerCfg, pacing)) {
+      const existing = selected.get(window.role);
+      if (!existing || compareQuotaWindows(window, existing) > 0) selected.set(window.role, window);
+    }
+  }
+  const windows = [...selected.values()].sort((a, b) => roleSort(a.role) - roleSort(b.role));
+  const quotaPenalty = Math.max(0, ...windows.map((window) => window.penalty));
+  const blockedNow = windows.some((window) => window.blockedNow);
+  // #1098 — the provider-level band: the MORE-BINDING window's pace weight drives the ladder;
+  // hard-avoid additionally requires the binding window's paceError beyond δ AND no other window
+  // able to absorb the slack (or blockedNow). Replaces the old max-penalty-then-threshold cliff.
+  // #2496 — the burnPenaltyUtilizationFloor gate: a window whose utilization is below the
+  // (per-provider-configurable) floor must not itself drive escalation. Previously this floor only
+  // zeroed the informational `penalty` field (#760/#1072); the #1098 pace-weight redesign moved the
+  // actual band decision onto paceWeight/paceError, which never consulted the floor — so a provider
+  // could show penalty:0 (correctly floored) yet still be labeled hard-avoid off the unfloored pace
+  // signal (round 1 of this fix). Round 2 (post-review): a below-floor window must stay IN the
+  // signal pool — dropping it entirely also removes it as an ABSORBER for a genuinely over-pace
+  // above-floor window's `canAbsorb` check (provider-quota-pace.ts), which perversely made that
+  // above-floor window's band WORSE (e.g. avoid-large → hard-avoid) once its only absorber was
+  // excluded. Round 3 (post-review again): merely clamping the below-floor window's weight to 0.5
+  // was not enough — a clamped-but-still-considered signal could still get selected as the binding
+  // window and either demote a genuinely better non-neutral signal, or (under custom thresholds)
+  // read as "caution" on its own when it was the ONLY tracked window. So `neutral` now fully
+  // excludes the window from binding/base selection (see quotaPaceCombinedBand) — its real,
+  // unclamped paceError/weight only matters for absorbing another window's hard-avoid check.
+  const signals: QuotaPaceRoleSignal[] = windows
+    .filter((window): window is SpawnQuotaBurnWindow & { paceWeight: number; paceError: number } =>
+      window.paceWeight !== null && window.paceError !== null)
+    .map((window) => ({
+      role: window.role,
+      paceError: window.paceError,
+      weight: window.paceWeight,
+      blockedNow: window.blockedNow,
+      neutral: window.belowPenaltyFloor,
+    }));
+  const { band: paceBand } = quotaPaceCombinedBand(signals, pacing.delta, pacing.thresholds);
+  // #760 — provider-level "prefer" tiebreaker: every selected window is projected (the #1072
+  // slow-EWMA trend, demoted to secondary per #1098) to land ≥ preferSlackAt below target — e.g. a
+  // just-reset window whose RAW utilization reads mildly "ahead" of a near-zero pace line (paceBand
+  // normal/caution) but whose sustained trend shows it is nowhere near the target. The binding
+  // constraint is the LEAST-slack window, so a tight 5h window vetoes prefer even if 7d is wide
+  // open. Reserved for normal/caution — avoid-large/hard-avoid are real over-pace signals that a
+  // secondary projection should not paper over.
+  const preferSlackAt = providerCfg?.preferSlackAt ?? DEFAULT_QUOTA_PREFER_SLACK_AT;
+  const slacks = windows.map((window) => window.slack).filter((slack): slack is number => slack !== null);
+  const minSlack = slacks.length === windows.length && slacks.length > 0 ? Math.min(...slacks) : null;
+  const prefer = !blockedNow
+    && (paceBand === "prefer"
+      || subscriptionExpiryPreference(windows, pacing)
+      || ((paceBand === "normal" || paceBand === "caution") && minSlack !== null && minSlack >= preferSlackAt));
+  const quotaPenaltyBand: SpawnQuotaBurnBand = prefer ? "prefer" : paceBand;
+  const expiryPreference = prefer ? windows.map((w) => w.expiryPreference).filter(Boolean).join("; ") : "";
+  return {
+    quotaBurn: {
+      windows,
+      resetCredits: records.reduce((total, record) => total + (record.codexResetCredits?.availableCount ?? 0), 0),
+    },
+    quotaFreshness: quotaFreshnessFor(provider, records),
+    quotaPenalty,
+    quotaPenaltyBand,
+    prefer,
+    ...(expiryPreference ? { expiryPreference } : {}),
+    slack: minSlack,
+    blockedNow,
+    recommendedAction: recommendedAction({ quotaPenaltyBand, blockedNow, prefer, slack: minSlack, windows }) + (expiryPreference ? ` ${expiryPreference}` : ""),
+  };
+}
+
+function quotaFreshnessFor(provider: SpawnProvider, records: ProviderQuotaRecord[]): SpawnQuotaAdvisory["quotaFreshness"] {
+  const updated = records.flatMap((record) => record.lastUpdatedAt === undefined ? [] : [record.lastUpdatedAt]);
+  const windows = records.flatMap((record) => record.windowStatuses.map((window) => ({ accountKey: record.accountKey, ...window })));
+  // Records present: report their real freshness, even when the provider is quota-less by manifest.
+  if (records.length > 0) {
+    return {
+      lastUpdatedAt: updated.length > 0 ? Math.max(...updated) : null,
+      stale: records.every((record) => record.stale),
+      unavailable: records.every((record) => record.unavailable),
+      windows,
+    };
+  }
+  // #2441 — a quota-less provider (opencode) never gets a "not stale/not unavailable" record to
+  // begin with, so `records.every(...)` over its (usually empty) set would otherwise collapse to
+  // true forever. Quota-supported providers keep the original every()/zero-records semantics
+  // unchanged — a genuinely empty or fully-stale table for THEM really is an outage signal.
+  if (!providerSupportsQuota(provider)) {
+    return { lastUpdatedAt: null, stale: false, unavailable: false, windows };
+  }
+  return {
+    lastUpdatedAt: null,
+    stale: true,
+    unavailable: true,
+    windows,
+  };
+}
+
+// #839 — compact per-provider quota band view for the task dispatcher. Projects the same
+// advisory the spawn matrix surfaces (band / prefer / blockedNow) down to the fields the
+// dispatcher needs to protect scarce pools and load-shift onto abundant ones. Returns one
+// entry per provider that has quota records; providers without records are simply absent
+// (the dispatcher then treats them as band-unknown = "normal").
+export interface ProviderQuotaBand {
+  expiryPreference?: string;
+  provider: SpawnProvider;
+  band: QuotaBand;
+  quotaPenalty: number;
+  slack: number | null;
+  prefer: boolean;
+  blockedNow: boolean;
+  recommendedAction: string;
+}
+
+export function providerQuotaBands(now = Date.now()): ProviderQuotaBand[] {
+  return providerQuotaBandsFor(enabledProviderQuotas(listProviderQuotas({ historyLimit: 0, now }).quotas, effectiveProviderCatalogList(now)), getRoutingPolicy(), now);
+}
+
+export function providerQuotaBandsFor(records: ProviderQuotaRecord[], policy: RoutingPolicy, now: number): ProviderQuotaBand[] {
+  return [...providerQuotaAdvisoriesFor(records, policy, now).entries()].map(([provider, advisory]) => ({
+    provider,
+    band: advisory.quotaPenaltyBand,
+    quotaPenalty: advisory.quotaPenalty,
+    slack: advisory.slack,
+    prefer: advisory.prefer,
+    blockedNow: advisory.blockedNow,
+    recommendedAction: advisory.recommendedAction,
+    ...(advisory.expiryPreference ? { expiryPreference: advisory.expiryPreference } : {}),
+  }));
+}
+
+function quotaWindowsFor(
+  record: ProviderQuotaRecord,
+  now: number,
+  providerCfg: ProviderQuotaRoutingConfig | undefined,
+  pacing: ResolvedQuotaPacing,
+): SpawnQuotaBurnWindow[] {
+  const quotaByName = new Map((record.quota?.windows ?? []).map((window) => [window.name, window]));
+  return (record.burn?.windows ?? [])
+    .map((burnWindow) => spawnQuotaWindow(record, burnWindow, quotaByName.get(burnWindow.name), now, providerCfg, pacing))
+    .filter((window): window is SpawnQuotaBurnWindow => Boolean(window));
+}
+
+function spawnQuotaWindow(
+  record: ProviderQuotaRecord,
+  burnWindow: ProviderQuotaBurnWindow,
+  quotaWindow: QuotaState["windows"][number] | undefined,
+  now: number,
+  providerCfg: ProviderQuotaRoutingConfig | undefined,
+  pacing: ResolvedQuotaPacing,
+): SpawnQuotaBurnWindow | undefined {
+  if (!quotaWindow) return undefined;
+  const timeToResetMs = typeof quotaWindow.resetsAt === "number" ? Math.max(0, quotaWindow.resetsAt - now) : null;
+  const hardBackstop = providerCfg?.hardBackstopUtilization;
+  // #760 — floor default is now 0 (off). The evidence-capped projection replaces the blunt
+  // utilization floor (which hid a legitimate low-util-high-burst alarm); operators may still opt
+  // back in to raw-utilization suppression by setting a positive floor.
+  const utilizationFloor = providerCfg?.burnPenaltyUtilizationFloor ?? 0;
+  const belowPenaltyFloor = quotaWindow.utilization < utilizationFloor;
+
+  // #1072 — the band projection is evidence-capped at INGEST off the slow rate and is
+  // target-independent, so keep it as-is and re-derive only the per-provider-target slack + penalty
+  // from it. burnRatio (display) re-paces the same slow rate against the per-provider target.
+  const target = providerCfg?.targetUtilization ?? DEFAULT_QUOTA_TARGET_UTILIZATION;
+  const preferSlackAt = providerCfg?.preferSlackAt ?? DEFAULT_QUOTA_PREFER_SLACK_AT;
+  const blockedNow = hardBackstop === 0 ? false : quotaWindow.utilization >= (hardBackstop ?? 0.99);
+  const projectedUtilAtReset = burnWindow.projectedUtilAtReset;
+  // #1984 — the pace window length is the window's MEASURED reset period (`cadenceMs`, derived from
+  // successive reset boundaries), NOT the name-mapped role length and NOT its time-remaining. A
+  // codex `primary` window that now resets ~weekly therefore paces against its real 7d period
+  // (→ under-pace → prefer) instead of a 5h ruler (→ spurious hard-avoid), and a reappearing 5h
+  // window paces against 5h with no code change. `null` = cadence not yet determinable ⇒ the window
+  // is ambiguous: it contributes NO pace signal (see the paceMetrics guard below and the signal
+  // filter in quotaAdvisoryFor), so it can never be escalated to hard-avoid on a guessed length.
+  const windowLengthMs = burnWindow.cadenceMs;
+  let burnRatio = burnWindow.burnRatio;
+  if (burnWindow.observedRatePerMs !== null && timeToResetMs !== null && timeToResetMs > 0 && windowLengthMs !== null) {
+    burnRatio = quotaPaceMetrics({
+      utilization: quotaWindow.utilization,
+      observedRatePerMs: burnWindow.observedRatePerMs,
+      timeToResetMs,
+      windowLengthMs,
+      target,
+    }).burnRatio;
+  }
+
+  // #1072 (kept, demoted to secondary) — the unified overshoot penalty/slack against the
+  // per-provider target. #1098 no longer uses `penalty` to drive the band, but it's still surfaced
+  // (informational) and `slack` still feeds the "prefer" tiebreaker below.
+  const confidence = burnWindow.confidenceScore ?? 0;
+  const { slack, penalty: rawPenalty } = projectedUtilAtReset !== null
+    ? quotaOvershootPenalty(projectedUtilAtReset, quotaWindow.utilization, target, confidence)
+    : { slack: burnWindow.slack, penalty: 0 };
+  const penalty = belowPenaltyFloor ? 0 : rawPenalty;
+
+  // #1098 — the pace-error control signal: present position (utilization) vs. the ideal linear
+  // pace line (elapsed/windowLength), NOT a forecast. Drives the band; see quotaAdvisoryFor for
+  // the cross-window (5h+7d, more-binding-wins) combination and the hard-avoid escalation gate.
+  const paceMetrics = timeToResetMs !== null && windowLengthMs !== null
+    ? quotaPaceWindowMetrics({
+        utilization: quotaWindow.utilization,
+        timeToResetMs,
+        windowLengthMs,
+        delta: pacing.delta,
+      })
+    : null;
+  const windowBand: QuotaPaceBand | null = paceMetrics ? quotaPaceBandForWeight(paceMetrics.weight, pacing.thresholds) : null;
+  // #1098 — per-window "prefer": either the weight ladder itself climbed past neutral into active
+  // promotion (use-it-or-lose-it, near reset with util below pace), or the window carries ample
+  // future slack per the secondary/tiebreaker projection (the #1072 target-buffer signal — e.g. a
+  // just-reset window reading mildly "ahead" of a near-zero pace line but nowhere near the target
+  // on the sustained trend). Reserved for normal/caution; avoid-large/hard-avoid are real over-pace
+  // signals a secondary projection should not paper over.
+  const expiryPreference = subscriptionExpiryForWindow(record, burnWindow, now, pacing, providerCfg);
+  const prefer = !blockedNow && windowBand !== null
+    && (windowBand === "prefer"
+      || ((windowBand === "normal" || windowBand === "caution") && slack !== null && slack >= preferSlackAt));
+  return {
+    ...(expiryPreference ? { expiryPreference } : {}),
+    role: burnWindow.role,
+    name: burnWindow.name,
+    accountKey: record.accountKey,
+    utilization: quotaWindow.utilization,
+    resetsAt: quotaWindow.resetsAt ?? null,
+    timeToResetMs,
+    confidence: burnWindow.confidence,
+    burnRatio,
+    timeToWallMs: burnWindow.timeToWallMs,
+    projectedWallAt: burnWindow.projectedWallAt,
+    earlyWallMs: burnWindow.earlyWallMs,
+    projectedUtilAtReset,
+    slack,
+    paceTarget: paceMetrics?.target ?? null,
+    paceError: paceMetrics?.paceError ?? null,
+    paceWeight: paceMetrics?.weight ?? null,
+    pace: paceMetrics ? formatPaceLabel(paceMetrics.paceError) : null,
+    prefer,
+    penalty,
+    blockedNow,
+    belowPenaltyFloor,
+  };
+}
+
+function compareQuotaWindows(a: SpawnQuotaBurnWindow, b: SpawnQuotaBurnWindow): number {
+  if (a.blockedNow !== b.blockedNow) return a.blockedNow ? 1 : -1;
+  // #1098 — the more-BINDING (lower pace weight) window is "worse"; unknown weight (no reset
+  // boundary) reads as neutral (0.5), same as an on-pace window.
+  const aw = a.paceWeight ?? 0.5;
+  const bw = b.paceWeight ?? 0.5;
+  if (aw !== bw) return bw - aw;
+  // #1098 — projectedUtilAtReset demoted to a tiebreaker: when pace weight ties, the more
+  // concerning (higher) projection wins the "worse" slot.
+  const ap = a.projectedUtilAtReset ?? -Infinity;
+  const bp = b.projectedUtilAtReset ?? -Infinity;
+  if (ap !== bp) return ap - bp;
+  return a.utilization - b.utilization;
+}
+
+function quotaPenaltyBandFor(
+  quotaPenalty: number,
+  quotaBands?: Partial<{ cautionAt: number; avoidLargeAt: number; hardAvoidAt: number }>,
+): SpawnQuotaBurnBand {
+  const hardAvoidAt = quotaBands?.hardAvoidAt ?? 1.0;
+  const avoidLargeAt = quotaBands?.avoidLargeAt ?? 0.5;
+  const cautionAt = quotaBands?.cautionAt ?? 0.15;
+  if (quotaPenalty >= hardAvoidAt) return "hard-avoid";
+  if (quotaPenalty >= avoidLargeAt) return "avoid-large";
+  if (quotaPenalty >= cautionAt) return "caution";
+  return "normal";
+}
+
+function recommendedAction(input: { quotaPenaltyBand: SpawnQuotaBurnBand; blockedNow: boolean; prefer: boolean; slack: number | null; windows: SpawnQuotaBurnWindow[] }): string {
+  if (input.blockedNow) {
+    const hasSevenDayBlock = input.windows.some((window) => window.role === "7d" && window.blockedNow);
+    // #827 — "at capacity" (util ≥ hardBackstop) is a routing-avoid advisory, not an evidence-based
+    // rate-limit hold. Use distinct wording so operators don't confuse it with an HTTP 429 block.
+    return hasSevenDayBlock
+      ? "Quota at capacity (routing-avoid); switch provider or wait for reset unless this provider was explicitly requested."
+      : "5h quota at capacity (routing-avoid); wait for reset or use another provider unless the job is urgent.";
+  }
+  return windowRecommendedAction(input.quotaPenaltyBand, input.slack);
+}
+
+/**
+ * Per-window recommended action text for a given band — used by band-transition notifications
+ * so each window's notification reflects ITS own band, not the provider-level band.
+ */
+export { windowRecommendedAction };
+
+function roleSort(role: ProviderQuotaBurnRole): number {
+  return role === "5h" ? 0 : 1;
+}
+
+function spawnProfilesSummary(profiles: ReturnType<typeof listAgentProfiles> = listAgentProfiles()): Array<Record<string, unknown>> {
+  return profiles.filter(({ key }) => !isHiddenAgentProfile(key)).map(({ key, value }) => ({
+    name: value.name ?? key,
+    description: value.description,
+    ...(value.icon ? { icon: value.icon } : {}),
+    provider: value.provider,
+    approvalMode: value.permissions?.mode,
+    ...(value.maxSpawnedAgents !== undefined ? { maxSpawnedAgents: value.maxSpawnedAgents } : {}),
+    builtIn: value.builtIn ?? false,
+  }));
+}