agent-relay-server 0.123.1 → 0.124.0

package/src/services/spawn-chain-inspector.ts

@@ -0,0 +1,134 @@
+// Real-chain spawn inspector (#1103 L1 seed) — the regression harness whose absence let two
+// dead #1096 fixes ship. It drives the ACTUAL spawn chain for a named profile, end to end, with
+// NO hand-fed config objects:
+//
+//   getAgentProfile(name)            — the real built-in/stored profile
+//   → withResolvedProvisioning       — relay-side registry resolution (needs a seeded DB)
+//   → buildSpawnCommand              — the command params the orchestrator dispatches
+//   → JSON round-trip                — the command-bus / AGENT_RELAY_AGENT_PROFILE_JSON env seam
+//   → spawnOptionsFromControl        — orchestrator re-parse of the command
+//   → ClaudeAdapter.buildSpawnArgs   — runner assembly + hook/skill/plugin MATERIALIZATION to disk
+//
+// then reads back the EFFECTIVE on-disk artifacts: the written managed `settings.json`, the
+// materialized `relay-provisioned-hooks/` tree, and the assembled argv. Two prior fixes each
+// tested one half in isolation (relay resolution OR runner materialization) with a hand-fed
+// `provisioning.hooks`, so neither caught that the bundled hook asset is never SHIPPED (the
+// `runner/hooks/**` packaging gap) and so never seeds → resolves empty on a real deployed spawn.
+// This helper runs the whole chain against the real DB + real package layout, so that class of
+// break surfaces as a red on-disk artifact instead of a green half-test.
+
+import { existsSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+import type { SpawnProvider } from "agent-relay-sdk";
+import { withResolvedProvisioning } from "../db";
+import { getAgentProfile } from "../config-store";
+import { buildSpawnCommand } from "../spawn-command";
+import { spawnOptionsFromControl } from "../../orchestrator/src/control";
+import { ClaudeAdapter } from "../../runner/src/adapters/claude";
+import { defaultProviderConfig } from "../../runner/src/config";
+import type { RunnerSpawnConfig } from "../../runner/src/adapter";
+
+/** The bundled baseline safety hook whose silent absence is the #1096 bug. */
+export const BASELINE_HOOK_REF = "agent-relay-rg-r-guard";
+export const BASELINE_HOOK_SCRIPT = "rg-r-guard.cjs";
+
+export interface SpawnChainInspection {
+  profile: string;
+  provider: SpawnProvider;
+  /** The isolated managed CLAUDE_CONFIG_DIR the runner assembled + materialized into. */
+  configHome: string | undefined;
+  /** Absolute path of the managed settings.json (whether or not it exists). */
+  settingsPath: string | undefined;
+  /** Parsed on-disk settings.json (null when relay wrote none). */
+  settings: Record<string, unknown> | null;
+  /** The effective launch argv the adapter would exec. */
+  argv: string[];
+  /** The launch command (claude / claude-rig). */
+  command: string;
+  /** relay-provisioned-hooks/<ref> dir for the baseline guard. */
+  baselineHookAssetDir: string | undefined;
+  baselineHookAssetExists: boolean;
+  baselineHookScriptExists: boolean;
+  /** True iff the WRITTEN settings.json (the documented, transit-safe hook source) carries the
+   *  baseline PreToolUse guard — the #1096 proof-of-life bar. */
+  settingsHasBaselineHook: boolean;
+  /** True iff SOME launch channel (written settings.json OR inline --settings) carries it. */
+  launchHasBaselineHook: boolean;
+}
+
+function settingsCarriesBaselineHook(settings: Record<string, unknown> | null): boolean {
+  const pre = (settings?.hooks as { PreToolUse?: unknown } | undefined)?.PreToolUse;
+  if (!Array.isArray(pre)) return false;
+  return JSON.stringify(pre).includes(BASELINE_HOOK_SCRIPT);
+}
+
+function argvCarriesBaselineHook(argv: string[]): boolean {
+  const idx = argv.findIndex((a) => a === "--settings");
+  if (idx < 0 || idx + 1 >= argv.length) return false;
+  return argv[idx + 1]!.includes(BASELINE_HOOK_SCRIPT);
+}
+
+/**
+ * Run the real spawn chain for a profile and read back the on-disk artifacts. Two preconditions,
+ * both owned by the caller so this stays a pure harness (no global/env mutation of its own):
+ *   • the DB is seeded (initDb) so registry resolution can find the bundled hook, and
+ *   • AGENT_RELAY_PROVIDER_HOME_ROOT points at a scratch dir (the runner's config module reads it
+ *     to place isolated provider homes) — set it like launch-assembly.test.ts does.
+ */
+export function inspectSpawnChain(opts: {
+  profile: string;
+  provider: SpawnProvider;
+  cwd: string;
+  approvalMode?: string;
+}): SpawnChainInspection {
+  const { profile, provider, cwd } = opts;
+
+  // 1. RELAY: resolve the named profile's provisioning against the registry.
+  const resolved = withResolvedProvisioning(getAgentProfile(profile)?.value, provider);
+  if (!resolved) throw new Error(`profile not found: ${profile}`);
+
+  // 2. RELAY: build the spawn command params the orchestrator dispatches.
+  const cmdParams = buildSpawnCommand({
+    provider, cwd, agentProfile: resolved, env: {}, requestedBy: "spawn-chain-inspector",
+  } as Parameters<typeof buildSpawnCommand>[0]);
+
+  // 3. SEAM: JSON round-trip through the command bus / AGENT_RELAY_AGENT_PROFILE_JSON env var.
+  const roundTripped = JSON.parse(JSON.stringify(cmdParams)) as Record<string, unknown>;
+
+  // 4. ORCHESTRATOR: re-parse the command into spawn options.
+  const spawnOpts = spawnOptionsFromControl(roundTripped, { providers: [provider], baseDir: cwd } as never);
+
+  // 5. RUNNER: assemble the RunnerSpawnConfig the adapter consumes.
+  const providerConfig = defaultProviderConfig(provider);
+  const config = {
+    provider, runnerId: "inspector", instanceId: `insp-${profile}`, agentId: "insp",
+    relayUrl: "http://localhost:4850", cwd, headless: true,
+    approvalMode: opts.approvalMode ?? "open", providerArgs: [], providerConfig,
+    env: {}, controlPort: 4900,
+    agentProfile: spawnOpts.agentProfile,
+  } as unknown as RunnerSpawnConfig;
+
+  // 6. RUNNER: the REAL adapter path — prepares the home, materializes provisioned assets to
+  //    disk, and returns the launch argv (exactly what the runner execs).
+  const spawnArgs = new ClaudeAdapter().buildSpawnArgs(config, providerConfig);
+
+  // 7. Read back the effective on-disk artifacts.
+  const configHome = spawnArgs.env.CLAUDE_CONFIG_DIR;
+  const settingsPath = configHome ? join(configHome, "settings.json") : undefined;
+  const settings = settingsPath && existsSync(settingsPath)
+    ? (JSON.parse(readFileSync(settingsPath, "utf8")) as Record<string, unknown>)
+    : null;
+  const baselineHookAssetDir = configHome
+    ? join(configHome, "relay-provisioned-hooks", BASELINE_HOOK_REF)
+    : undefined;
+
+  return {
+    profile, provider, configHome, settingsPath, settings,
+    argv: spawnArgs.args, command: spawnArgs.command,
+    baselineHookAssetDir,
+    baselineHookAssetExists: Boolean(baselineHookAssetDir && existsSync(baselineHookAssetDir)),
+    baselineHookScriptExists: Boolean(baselineHookAssetDir && existsSync(join(baselineHookAssetDir, BASELINE_HOOK_SCRIPT))),
+    settingsHasBaselineHook: settingsCarriesBaselineHook(settings),
+    launchHasBaselineHook: settingsCarriesBaselineHook(settings) || argvCarriesBaselineHook(spawnArgs.args),
+  };
+}

package/src/spawn-targets.ts

@@ -6,8 +6,23 @@ import { renderTemplate } from "./prompt-resolver";
 import { getRoutingPolicy } from "./routing-policy-store";
 import { isPathWithinBase } from "./utils";
 import { McpNotFoundError } from "./mcp-errors";
-import { DEFAULT_QUOTA_PREFER_SLACK_AT, DEFAULT_QUOTA_TARGET_UTILIZATION, quotaOvershootPenalty, quotaPaceMetrics } from "agent-relay-sdk/provider-quota";
-import { isSpawnProvider, type ProviderQuotaRoutingConfig } from "agent-relay-sdk";
+import {
+  DEFAULT_QUOTA_PACE_DELTA,
+  DEFAULT_QUOTA_PACE_THRESHOLDS,
+  DEFAULT_QUOTA_PREFER_SLACK_AT,
+  DEFAULT_QUOTA_TARGET_UTILIZATION,
+  formatPaceLabel,
+  QUOTA_PACE_WINDOW_LENGTH_MS,
+  quotaOvershootPenalty,
+  quotaPaceBandForWeight,
+  quotaPaceCombinedBand,
+  quotaPaceMetrics,
+  quotaPaceWindowMetrics,
+  type QuotaPaceBand,
+  type QuotaPaceRoleSignal,
+  type QuotaPaceThresholds,
+} from "agent-relay-sdk/provider-quota";
+import { isSpawnProvider, type ProviderQuotaRoutingConfig, type QuotaPacingConfig, type RoutingPolicy } from "agent-relay-sdk";
 import type { Orchestrator, ProviderCatalogSummary, ProviderQuotaBurnRole, ProviderQuotaBurnWindow, ProviderQuotaRecord, QuotaState, SpawnProvider } from "./types";
 
 // #308 — spawn discovery + target selection. Home for the live capability matrix that
@@ -83,9 +98,17 @@ interface SpawnQuotaBurnWindow {
   projectedWallAt: number | null;
   earlyWallMs: number | null;
   // #760 — symmetric underburn signal (see ProviderQuotaBurnWindow.slack), re-derived against
-  // the per-provider target, plus the "prefer" flag for this window.
+  // the per-provider target. #1098: demoted to a tiebreaker/secondary signal — still surfaced
+  // (and still the input to the "prefer" overlay below) but no longer the primary band driver.
   projectedUtilAtReset: number | null;
   slack: number | null;
+  // #1098 — the pace-error control signal: present position vs. the ideal linear pace line,
+  // and the legibility label ("+9% ahead of pace" beats "projected 115%"). null when timeToReset
+  // is unknown (no reset boundary to pace against).
+  paceTarget: number | null;
+  paceError: number | null;
+  paceWeight: number | null;
+  pace: string | null;
   prefer: boolean;
   penalty: number;
   blockedNow: boolean;
@@ -112,6 +135,32 @@ const DEFAULT_QUOTA_ADVISORY: SpawnQuotaAdvisory = {
   recommendedAction: "No quota burn projection available; spawn normally.",
 };
 
+// #1098/#1105 — resolved pacing tunables: delta/thresholds/windowLengthMs are GLOBAL (one dial per
+// relay instance, not per-provider — matches the design's "δ a tunable constant"), sourced from
+// RoutingPolicy.quotaPacing with the provider-quota.ts constants as fallback defaults, so an
+// unconfigured instance reproduces the shipped behaviour exactly.
+interface ResolvedQuotaPacing {
+  delta: number;
+  thresholds: QuotaPaceThresholds;
+  windowLengthMs: Record<ProviderQuotaBurnRole, number>;
+}
+
+function resolveQuotaPacing(policy: RoutingPolicy): ResolvedQuotaPacing {
+  const cfg: QuotaPacingConfig = policy.quotaPacing ?? {};
+  return {
+    delta: cfg.delta ?? DEFAULT_QUOTA_PACE_DELTA,
+    thresholds: {
+      preferAt: cfg.thresholds?.preferAt ?? DEFAULT_QUOTA_PACE_THRESHOLDS.preferAt,
+      normalAt: cfg.thresholds?.normalAt ?? DEFAULT_QUOTA_PACE_THRESHOLDS.normalAt,
+      cautionAt: cfg.thresholds?.cautionAt ?? DEFAULT_QUOTA_PACE_THRESHOLDS.cautionAt,
+    },
+    windowLengthMs: {
+      "5h": cfg.windowLengthMs?.["5h"] ?? QUOTA_PACE_WINDOW_LENGTH_MS["5h"],
+      "7d": cfg.windowLengthMs?.["7d"] ?? QUOTA_PACE_WINDOW_LENGTH_MS["7d"],
+    },
+  };
+}
+
 // #926 — the orchestrator's REPORTED catalog is built from the static SDK manifest only
 // (orchestrator/src/provider-probe.ts uses providerCatalogList(), which has no DB access), while
 // spawn VALIDATION resolves against effectiveProviderCatalogList() (static catalog + DB
@@ -189,21 +238,22 @@ export function providerQuotaAdvisories(now = Date.now()): Map<SpawnProvider, Sp
     const provider = record.provider;
     byProvider.set(provider, [...(byProvider.get(provider) ?? []), record]);
   }
+  const pacing = resolveQuotaPacing(policy);
   return new Map([...byProvider.entries()].map(([provider, records]) => [
     provider,
-    quotaAdvisoryFor(records, now, policy.quotaBands, policy.providerQuota?.[provider]),
+    quotaAdvisoryFor(records, now, pacing, policy.providerQuota?.[provider]),
   ]));
 }
 
 function quotaAdvisoryFor(
   records: ProviderQuotaRecord[],
   now: number,
-  quotaBands?: Partial<{ cautionAt: number; avoidLargeAt: number; hardAvoidAt: number }>,
+  pacing: ResolvedQuotaPacing,
   providerCfg?: ProviderQuotaRoutingConfig,
 ): SpawnQuotaAdvisory {
   const selected = new Map<ProviderQuotaBurnRole, SpawnQuotaBurnWindow>();
   for (const record of records) {
-    for (const window of quotaWindowsFor(record, now, providerCfg)) {
+    for (const window of quotaWindowsFor(record, now, providerCfg, pacing)) {
       const existing = selected.get(window.role);
       if (!existing || compareQuotaWindows(window, existing) > 0) selected.set(window.role, window);
     }
@@ -211,15 +261,28 @@ function quotaAdvisoryFor(
   const windows = [...selected.values()].sort((a, b) => roleSort(a.role) - roleSort(b.role));
   const quotaPenalty = Math.max(0, ...windows.map((window) => window.penalty));
   const blockedNow = windows.some((window) => window.blockedNow);
-  const penaltyBand = quotaPenaltyBandFor(quotaPenalty, quotaBands);
-  // #760 — provider-level "prefer": no over-pace penalty AND every selected window is projected
-  // to land ≥ preferSlackAt below target. The binding constraint is the LEAST-slack window, so a
-  // tight 5h window vetoes prefer even if 7d is wide open.
+  // #1098 — the provider-level band: the MORE-BINDING window's pace weight drives the ladder;
+  // hard-avoid additionally requires the binding window's paceError beyond δ AND no other window
+  // able to absorb the slack (or blockedNow). Replaces the old max-penalty-then-threshold cliff.
+  const signals: QuotaPaceRoleSignal[] = windows
+    .filter((window): window is SpawnQuotaBurnWindow & { paceWeight: number; paceError: number } =>
+      window.paceWeight !== null && window.paceError !== null)
+    .map((window) => ({ role: window.role, paceError: window.paceError, weight: window.paceWeight, blockedNow: window.blockedNow }));
+  const { band: paceBand } = quotaPaceCombinedBand(signals, pacing.delta, pacing.thresholds);
+  // #760 — provider-level "prefer" tiebreaker: every selected window is projected (the #1072
+  // slow-EWMA trend, demoted to secondary per #1098) to land ≥ preferSlackAt below target — e.g. a
+  // just-reset window whose RAW utilization reads mildly "ahead" of a near-zero pace line (paceBand
+  // normal/caution) but whose sustained trend shows it is nowhere near the target. The binding
+  // constraint is the LEAST-slack window, so a tight 5h window vetoes prefer even if 7d is wide
+  // open. Reserved for normal/caution — avoid-large/hard-avoid are real over-pace signals that a
+  // secondary projection should not paper over.
   const preferSlackAt = providerCfg?.preferSlackAt ?? DEFAULT_QUOTA_PREFER_SLACK_AT;
   const slacks = windows.map((window) => window.slack).filter((slack): slack is number => slack !== null);
   const minSlack = slacks.length === windows.length && slacks.length > 0 ? Math.min(...slacks) : null;
-  const prefer = !blockedNow && penaltyBand === "normal" && minSlack !== null && minSlack >= preferSlackAt;
-  const quotaPenaltyBand: SpawnQuotaBurnBand = prefer ? "prefer" : penaltyBand;
+  const prefer = !blockedNow
+    && (paceBand === "prefer"
+      || ((paceBand === "normal" || paceBand === "caution") && minSlack !== null && minSlack >= preferSlackAt));
+  const quotaPenaltyBand: SpawnQuotaBurnBand = prefer ? "prefer" : paceBand;
   return {
     quotaBurn: { windows },
     quotaPenalty,
@@ -254,10 +317,15 @@ export function providerQuotaBands(now = Date.now()): ProviderQuotaBand[] {
   }));
 }
 
-function quotaWindowsFor(record: ProviderQuotaRecord, now: number, providerCfg?: ProviderQuotaRoutingConfig): SpawnQuotaBurnWindow[] {
+function quotaWindowsFor(
+  record: ProviderQuotaRecord,
+  now: number,
+  providerCfg: ProviderQuotaRoutingConfig | undefined,
+  pacing: ResolvedQuotaPacing,
+): SpawnQuotaBurnWindow[] {
   const quotaByName = new Map((record.quota?.windows ?? []).map((window) => [window.name, window]));
   return (record.burn?.windows ?? [])
-    .map((burnWindow) => spawnQuotaWindow(record, burnWindow, quotaByName.get(burnWindow.name), now, providerCfg))
+    .map((burnWindow) => spawnQuotaWindow(record, burnWindow, quotaByName.get(burnWindow.name), now, providerCfg, pacing))
     .filter((window): window is SpawnQuotaBurnWindow => Boolean(window));
 }
 
@@ -266,7 +334,8 @@ function spawnQuotaWindow(
   burnWindow: ProviderQuotaBurnWindow,
   quotaWindow: QuotaState["windows"][number] | undefined,
   now: number,
-  providerCfg?: ProviderQuotaRoutingConfig,
+  providerCfg: ProviderQuotaRoutingConfig | undefined,
+  pacing: ResolvedQuotaPacing,
 ): SpawnQuotaBurnWindow | undefined {
   if (!quotaWindow) return undefined;
   const timeToResetMs = typeof quotaWindow.resetsAt === "number" ? Math.max(0, quotaWindow.resetsAt - now) : null;
@@ -288,20 +357,36 @@ function spawnQuotaWindow(
     burnRatio = quotaPaceMetrics({ utilization: quotaWindow.utilization, observedRatePerMs: burnWindow.observedRatePerMs, timeToResetMs, target }).burnRatio;
   }
 
-  // #1072 — ONE unified overshoot penalty (replaces #760 earlyFraction, the #1071 7d ×3 weight,
-  // and the #798 idle position penalty). The measured current overshoot counts at full weight; the
-  // projected trend overshoot is scaled by the continuous confidence, so thin evidence cannot
-  // hard-escalate on an extrapolated slope. Idle and active share this single formula — an idle
-  // over-target provider is held by the current-overshoot term (confidence-independent, so the
-  // band is flat tick-to-tick), an active climb escalates as the confidence-weighted trend crosses.
+  // #1072 (kept, demoted to secondary) — the unified overshoot penalty/slack against the
+  // per-provider target. #1098 no longer uses `penalty` to drive the band, but it's still surfaced
+  // (informational) and `slack` still feeds the "prefer" tiebreaker below.
   const confidence = burnWindow.confidenceScore ?? 0;
   const { slack, penalty: rawPenalty } = projectedUtilAtReset !== null
     ? quotaOvershootPenalty(projectedUtilAtReset, quotaWindow.utilization, target, confidence)
     : { slack: burnWindow.slack, penalty: 0 };
   const penalty = quotaWindow.utilization < utilizationFloor ? 0 : rawPenalty;
-  const prefer = !blockedNow && penalty === 0
-    && quotaPenaltyBandFor(penalty) === "normal"
-    && slack !== null && slack >= preferSlackAt;
+
+  // #1098 — the pace-error control signal: present position (utilization) vs. the ideal linear
+  // pace line (elapsed/windowLength), NOT a forecast. Drives the band; see quotaAdvisoryFor for
+  // the cross-window (5h+7d, more-binding-wins) combination and the hard-avoid escalation gate.
+  const paceMetrics = timeToResetMs !== null
+    ? quotaPaceWindowMetrics({
+        utilization: quotaWindow.utilization,
+        timeToResetMs,
+        windowLengthMs: pacing.windowLengthMs[burnWindow.role],
+        delta: pacing.delta,
+      })
+    : null;
+  const windowBand: QuotaPaceBand | null = paceMetrics ? quotaPaceBandForWeight(paceMetrics.weight, pacing.thresholds) : null;
+  // #1098 — per-window "prefer": either the weight ladder itself climbed past neutral into active
+  // promotion (use-it-or-lose-it, near reset with util below pace), or the window carries ample
+  // future slack per the secondary/tiebreaker projection (the #1072 target-buffer signal — e.g. a
+  // just-reset window reading mildly "ahead" of a near-zero pace line but nowhere near the target
+  // on the sustained trend). Reserved for normal/caution; avoid-large/hard-avoid are real over-pace
+  // signals a secondary projection should not paper over.
+  const prefer = !blockedNow && windowBand !== null
+    && (windowBand === "prefer"
+      || ((windowBand === "normal" || windowBand === "caution") && slack !== null && slack >= preferSlackAt));
   return {
     role: burnWindow.role,
     name: burnWindow.name,
@@ -316,6 +401,10 @@ function spawnQuotaWindow(
     earlyWallMs: burnWindow.earlyWallMs,
     projectedUtilAtReset,
     slack,
+    paceTarget: paceMetrics?.target ?? null,
+    paceError: paceMetrics?.paceError ?? null,
+    paceWeight: paceMetrics?.weight ?? null,
+    pace: paceMetrics ? formatPaceLabel(paceMetrics.paceError) : null,
     prefer,
     penalty,
     blockedNow,
@@ -357,7 +446,8 @@ export function getProviderBandInfo(provider: string, now: number): ProviderBand
   const policy = getRoutingPolicy();
   const records = listProviderQuotas({ provider, historyLimit: 0, now }).quotas;
   if (records.length === 0) return undefined;
-  const advisory = quotaAdvisoryFor(records, now, policy.quotaBands, policy.providerQuota?.[provider as SpawnProvider]);
+  const pacing = resolveQuotaPacing(policy);
+  const advisory = quotaAdvisoryFor(records, now, pacing, policy.providerQuota?.[provider as SpawnProvider]);
   const quotaBands = {
     cautionAt: policy.quotaBands?.cautionAt ?? 0.15,
     avoidLargeAt: policy.quotaBands?.avoidLargeAt ?? 0.5,
@@ -365,8 +455,11 @@ export function getProviderBandInfo(provider: string, now: number): ProviderBand
   };
   const windows: ProviderBandWindow[] = advisory.quotaBurn.windows.map((w) => ({
     role: w.role,
-    // #760 — per-window prefer overlay: w.prefer already encodes "no penalty + ≥ preferSlackAt slack".
-    band: w.prefer ? "prefer" : quotaPenaltyBandFor(w.penalty, policy.quotaBands),
+    // #1098 — per-window prefer overlay: w.prefer already encodes the pace-ladder "prefer" state OR
+    // "on pace + ≥ preferSlackAt slack". Otherwise fall back to the window's own pace-weight band
+    // (no cross-window hard-avoid escalation at this per-window level — that needs the OTHER
+    // window's state and is applied only at the provider level; see quotaAdvisoryFor).
+    band: w.prefer ? "prefer" : quotaPaceBandForWeight(w.paceWeight ?? 0.5, pacing.thresholds),
     penalty: w.penalty,
     utilization: w.utilization,
     burnRatio: w.burnRatio,
@@ -380,7 +473,16 @@ export function getProviderBandInfo(provider: string, now: number): ProviderBand
 
 function compareQuotaWindows(a: SpawnQuotaBurnWindow, b: SpawnQuotaBurnWindow): number {
   if (a.blockedNow !== b.blockedNow) return a.blockedNow ? 1 : -1;
-  if (a.penalty !== b.penalty) return a.penalty - b.penalty;
+  // #1098 — the more-BINDING (lower pace weight) window is "worse"; unknown weight (no reset
+  // boundary) reads as neutral (0.5), same as an on-pace window.
+  const aw = a.paceWeight ?? 0.5;
+  const bw = b.paceWeight ?? 0.5;
+  if (aw !== bw) return bw - aw;
+  // #1098 — projectedUtilAtReset demoted to a tiebreaker: when pace weight ties, the more
+  // concerning (higher) projection wins the "worse" slot.
+  const ap = a.projectedUtilAtReset ?? -Infinity;
+  const bp = b.projectedUtilAtReset ?? -Infinity;
+  if (ap !== bp) return ap - bp;
   return a.utilization - b.utilization;
 }