agent-nuvira 3.5.0 → 3.5.1

package/dist/learning/model-warmup.d.ts

@@ -25,6 +25,7 @@
  *   stopWarmupDaemon();
  */
 import { getModelRegistry } from './model-registry.js';
+import { type DutyLeaseInfo } from './duty-lease.js';
 import type { ConfigManager } from '../config/manager.js';
 /**
  * Every bound this module spends against a free tier, in one place and
@@ -52,6 +53,33 @@ export interface WarmupConfig {
     hotWindowMs: number;
     /** Models used within this window are "warm" (ms). */
     warmWindowMs: number;
+    /**
+     * Max VERIFIED, in-pool models re-spot-checked per cycle to keep the pool
+     * FRESH. Separate from `hotPerCycle` because the two halves spend on
+     * different things: `hotPerCycle` keeps in-use models warm, this one stops
+     * the routable pool from ageing out.
+     */
+    poolRefreshPerCycle: number;
+    /**
+     * How long a VERIFIED model may go without re-verification (ms).
+     *
+     * WHY HOURS, NOT THE 7-DAY RULE. The registry retires a model nothing
+     * re-verifies, and NOTHING was re-verifying a verified one: the exploration
+     * half selects `status === 'unverified'` rows only, so a model that had been
+     * verified once could only age — the measured pool showed verified entries at
+     * 142.8h and shrinking, while routing kept serving from a pool that was
+     * quietly expiring. Freshness IS a routing input (health ranking reads
+     * `lastVerifiedAt`, latency and errorRate), so a stale pool is a routing
+     * signal that stops describing reality.
+     *
+     * WHY THIS STAYS BOUNDED. Only VERIFIED rows are eligible (the pool the
+     * router would actually serve — never the hundreds of catalog entries), only
+     * `poolRefreshPerCycle` of them per cycle, parked rows are skipped (a park is
+     * a rest, not staleness), and a 1-token spot-check per model every 12h is a
+     * few hundred tokens a day for a pool of ~12 — not a catalog-wide sweep,
+     * which on free tiers is where RPM/TPM limits would turn warmup into 429s.
+     */
+    poolStalenessMs: number;
 }
 /**
  * Resolve the sweep's bounds (env overrides → defaults).
@@ -86,6 +114,8 @@ export interface WarmupCycleResult {
     warmed: number;
     /** NEVER-VERIFIED models spot-checked to grow the routing pool. */
     explored: number;
+    /** STALE, verified in-pool models re-spot-checked to keep the pool fresh. */
+    poolRefreshed: number;
     /** Spot-checks of either kind that came back `verified`. */
     verified: number;
     /** Hot candidates the per-cycle cap left for the next cycle. */
@@ -96,6 +126,18 @@ export interface WarmupCycleResult {
      * did noticeably more network work than its neighbours.
      */
     catalogRefreshed: boolean;
+    /**
+     * False when a cycle was REFUSED because another process holds the warmup
+     * duty lease (see `duty-lease.ts`) — every other counter is zero then.
+     *
+     * Reported rather than silent because it is the one outcome that looks
+     * identical to "there was nothing to do", and the two call for opposite
+     * responses: one is good news, the other means this process will never do the
+     * sweep on its own.
+     */
+    ran: boolean;
+    /** Who held the lease, when `ran` is false. Diagnostics only. */
+    leaseHeldBy?: DutyLeaseInfo;
 }
 /** Injectable seams for tests — the real spot-check talks to a provider. */
 export interface WarmupDeps {
@@ -123,6 +165,29 @@ export interface WarmupDeps {
  * spend its whole budget on something that cannot answer.
  */
 export declare function selectExplorationCandidates(registry: ReturnType<typeof getModelRegistry>, configManager: ConfigManager, now: number, limit: number): WarmupCandidate[];
+/**
+ * Pick VERIFIED, in-pool models that have gone STALE, oldest verification
+ * first.
+ *
+ * This is the half the module was missing: exploration (`status ===
+ * 'unverified'`) grows the pool, hot records keep THIS process's models warm,
+ * and nothing refreshed a verified model that stopped being used — so a pool
+ * model aged toward the registry's 7-day retirement and, because the health
+ * ranking reads `lastVerifiedAt`, routing kept ranking it on evidence that was
+ * no longer true.
+ *
+ * Deliberately narrow:
+ *   - VERIFIED rows only. An `unverified` row belongs to exploration, and an
+ *     `unavailable` one is not stale — it is broken, and re-probing it is
+ *     exploration's job once it is re-listed.
+ *   - Unservable providers are excluded, so a cycle never spends its budget on
+ *     a provider with no credentials.
+ *   - Quota-parked rows are skipped: a park is a rest with a known end, and
+ *     re-probing it would be exactly the extra traffic the park prevents.
+ *   - The exploration throttle still applies per model, so this can never
+ *     re-probe the same model twice in one `exploreThrottleMs` window.
+ */
+export declare function selectPoolRefreshCandidates(registry: ReturnType<typeof getModelRegistry>, configManager: ConfigManager, now: number, limit: number): WarmupCandidate[];
 /**
  * Run one warmup cycle: keep hot models warm, and grow the verified pool.
  *
@@ -132,6 +197,33 @@ export declare function selectExplorationCandidates(registry: ReturnType<typeof
  * a fresh process (the old `if (records.length === 0) return;` made the first
  * cycle of every short-lived CLI run a no-op).
  */
+/** The duty name every warmup caller contends on — one lease per machine. */
+export declare const WARMUP_DUTY_LEASE = "model-warmup";
+/**
+ * How long an un-renewed warmup lease may live before another process may steal
+ * it (ms).
+ *
+ * A live cycle re-news at half this window (see the loop below), so this is
+ * purely the CRASH-RECOVERY delay: how long a dead holder can keep the sweep
+ * switched off. It is also the floor under the interval — a lease that expired
+ * inside one cycle would let two processes sweep at once.
+ */
+export declare function warmupLeaseStaleMs(env?: NodeJS.ProcessEnv): number;
+/**
+ * Run one warmup cycle UNDER the warmup duty lease, or refuse and say who holds
+ * it.
+ *
+ * WHY THE LEASE LIVES HERE AND NOT IN THE CALLERS. Every caller wants the same
+ * thing — "one sweep at a time" — and there are four of them (the daemon timer,
+ * the first-cycle kick at startup, `nuvira models warm`, and any future
+ * dashboard trigger). Putting the guard in each one means the fifth caller, or a
+ * refactor, silently re-introduces the waste. Putting it here means the sweep
+ * cannot be started un-leased at all.
+ *
+ * The work a cycle does is a real generation per candidate, on the user's free
+ * tier — which is why two processes sweeping the same candidate list is not
+ * merely redundant, it is the exact traffic that turns warmup into 429s.
+ */
 export declare function runWarmupCycle(configManager: ConfigManager, deps?: WarmupDeps): Promise<WarmupCycleResult>;
 /**
  * Get warmup statistics (for dashboard display).

package/dist/learning/model-warmup.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"model-warmup.d.ts","sourceRoot":"","sources":["../../src/learning/model-warmup.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAEH,OAAO,EAAE,gBAAgB,EAA2B,MAAM,qBAAqB,CAAC;AAIhF,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAK1D;;;;;;;;;;GAUG;AACH,MAAM,WAAW,YAAY;IAC3B,8CAA8C;IAC9C,UAAU,EAAE,MAAM,CAAC;IACnB,2DAA2D;IAC3D,cAAc,EAAE,MAAM,CAAC;IACvB,wDAAwD;IACxD,WAAW,EAAE,MAAM,CAAC;IACpB,iFAAiF;IACjF,eAAe,EAAE,MAAM,CAAC;IACxB,0EAA0E;IAC1E,iBAAiB,EAAE,MAAM,CAAC;IAC1B,qDAAqD;IACrD,WAAW,EAAE,MAAM,CAAC;IACpB,sDAAsD;IACtD,YAAY,EAAE,MAAM,CAAC;CACtB;AASD;;;;;GAKG;AACH,wBAAgB,YAAY,CAAC,GAAG,GAAE,MAAM,CAAC,UAAwB,GAAG,YAAY,CAU/E;AAsCD;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CActE;AAID,UAAU,eAAe;IACvB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;CAClB;AAmDD;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,aAAa,EAAE,aAAa,EAAE,IAAI,GAAE,UAAe,GAAG,IAAI,CAwB3F;AAED;;GAEG;AACH,wBAAgB,gBAAgB,IAAI,IAAI,CAMvC;AAED,mFAAmF;AACnF,MAAM,WAAW,iBAAiB;IAChC,yEAAyE;IACzE,MAAM,EAAE,MAAM,CAAC;IACf,mEAAmE;IACnE,QAAQ,EAAE,MAAM,CAAC;IACjB,4DAA4D;IAC5D,QAAQ,EAAE,MAAM,CAAC;IACjB,gEAAgE;IAChE,OAAO,EAAE,MAAM,CAAC;IAChB;;;;OAIG;IACH,gBAAgB,EAAE,OAAO,CAAC;CAC3B;AAED,4EAA4E;AAC5E,MAAM,WAAW,UAAU;IACzB,SAAS,CAAC,EAAE,CACV,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,MAAM,EACb,EAAE,EAAE,aAAa,KACd,OAAO,CAAC,UAAU,GAAG,aAAa,GAAG,SAAS,GAAG,OAAO,CAAC,CAAC;IAC/D;;;;OAIG;IACH,cAAc,CAAC,EAAE,CAAC,EAAE,EAAE,aAAa,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IACzD,2EAA2E;IAC3E,sBAAsB,CAAC,EAAE,CAAC,EAAE,EAAE,aAAa,KAAK;QAAE,OAAO,EAAE,OAAO,CAAA;KAAE,CAAC;CACtE;AAYD;;;;;;;;;;GAUG;AACH,wBAAgB,2BAA2B,CACzC,QAAQ,EAAE,UAAU,CAAC,OAAO,gBAAgB,CAAC,EAC7C,aAAa,EAAE,aAAa,EAC5B,GAAG,EAAE,MAAM,EACX,KAAK,EAAE,MAAM,GACZ,eAAe,EAAE,CAyBnB;AAED;;;;;;;;GAQG;AACH,wBAAsB,cAAc,CAClC,aAAa,EAAE,aAAa,EAC5B,IAAI,GAAE,UAAe,GACpB,OAAO,CAAC,iBAAiB,CAAC,CAgF5B;AAID;;GAEG;AACH,wBAAgB,cAAc,IAAI;IAChC,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;CACrB,CAoBA;AAED;;GAEG;AACH,wBAAgB,gBAAgB,IAAI,KAAK,CAAC;IACxC,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,KAAK,GAAG,MAAM,GAAG,MAAM,CAAC;CACjC,CAAC,CAiBD"}
+{"version":3,"file":"model-warmup.d.ts","sourceRoot":"","sources":["../../src/learning/model-warmup.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAEH,OAAO,EAAE,gBAAgB,EAA2B,MAAM,qBAAqB,CAAC;AAGhF,OAAO,EAKL,KAAK,aAAa,EACnB,MAAM,iBAAiB,CAAC;AAEzB,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAK1D;;;;;;;;;;GAUG;AACH,MAAM,WAAW,YAAY;IAC3B,8CAA8C;IAC9C,UAAU,EAAE,MAAM,CAAC;IACnB,2DAA2D;IAC3D,cAAc,EAAE,MAAM,CAAC;IACvB,wDAAwD;IACxD,WAAW,EAAE,MAAM,CAAC;IACpB,iFAAiF;IACjF,eAAe,EAAE,MAAM,CAAC;IACxB,0EAA0E;IAC1E,iBAAiB,EAAE,MAAM,CAAC;IAC1B,qDAAqD;IACrD,WAAW,EAAE,MAAM,CAAC;IACpB,sDAAsD;IACtD,YAAY,EAAE,MAAM,CAAC;IACrB;;;;;OAKG;IACH,mBAAmB,EAAE,MAAM,CAAC;IAC5B;;;;;;;;;;;;;;;;;;OAkBG;IACH,eAAe,EAAE,MAAM,CAAC;CACzB;AASD;;;;;GAKG;AACH,wBAAgB,YAAY,CAAC,GAAG,GAAE,MAAM,CAAC,UAAwB,GAAG,YAAY,CAe/E;AAsCD;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CActE;AAID,UAAU,eAAe;IACvB,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;CAClB;AAmDD;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,aAAa,EAAE,aAAa,EAAE,IAAI,GAAE,UAAe,GAAG,IAAI,CAwB3F;AAED;;GAEG;AACH,wBAAgB,gBAAgB,IAAI,IAAI,CAMvC;AAED,mFAAmF;AACnF,MAAM,WAAW,iBAAiB;IAChC,yEAAyE;IACzE,MAAM,EAAE,MAAM,CAAC;IACf,mEAAmE;IACnE,QAAQ,EAAE,MAAM,CAAC;IACjB,6EAA6E;IAC7E,aAAa,EAAE,MAAM,CAAC;IACtB,4DAA4D;IAC5D,QAAQ,EAAE,MAAM,CAAC;IACjB,gEAAgE;IAChE,OAAO,EAAE,MAAM,CAAC;IAChB;;;;OAIG;IACH,gBAAgB,EAAE,OAAO,CAAC;IAC1B;;;;;;;;OAQG;IACH,GAAG,EAAE,OAAO,CAAC;IACb,iEAAiE;IACjE,WAAW,CAAC,EAAE,aAAa,CAAC;CAC7B;AAED,4EAA4E;AAC5E,MAAM,WAAW,UAAU;IACzB,SAAS,CAAC,EAAE,CACV,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,MAAM,EACb,EAAE,EAAE,aAAa,KACd,OAAO,CAAC,UAAU,GAAG,aAAa,GAAG,SAAS,GAAG,OAAO,CAAC,CAAC;IAC/D;;;;OAIG;IACH,cAAc,CAAC,EAAE,CAAC,EAAE,EAAE,aAAa,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IACzD,2EAA2E;IAC3E,sBAAsB,CAAC,EAAE,CAAC,EAAE,EAAE,aAAa,KAAK;QAAE,OAAO,EAAE,OAAO,CAAA;KAAE,CAAC;CACtE;AAYD;;;;;;;;;;GAUG;AACH,wBAAgB,2BAA2B,CACzC,QAAQ,EAAE,UAAU,CAAC,OAAO,gBAAgB,CAAC,EAC7C,aAAa,EAAE,aAAa,EAC5B,GAAG,EAAE,MAAM,EACX,KAAK,EAAE,MAAM,GACZ,eAAe,EAAE,CAyBnB;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAgB,2BAA2B,CACzC,QAAQ,EAAE,UAAU,CAAC,OAAO,gBAAgB,CAAC,EAC7C,aAAa,EAAE,aAAa,EAC5B,GAAG,EAAE,MAAM,EACX,KAAK,EAAE,MAAM,GACZ,eAAe,EAAE,CA2BnB;AAED;;;;;;;;GAQG;AACH,6EAA6E;AAC7E,eAAO,MAAM,iBAAiB,iBAAiB,CAAC;AAEhD;;;;;;;;GAQG;AACH,wBAAgB,kBAAkB,CAAC,GAAG,GAAE,MAAM,CAAC,UAAwB,GAAG,MAAM,CAE/E;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,cAAc,CAClC,aAAa,EAAE,aAAa,EAC5B,IAAI,GAAE,UAAe,GACpB,OAAO,CAAC,iBAAiB,CAAC,CAyB5B;AAuHD;;GAEG;AACH,wBAAgB,cAAc,IAAI;IAChC,aAAa,EAAE,MAAM,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;CACrB,CAoBA;AAED;;GAEG;AACH,wBAAgB,gBAAgB,IAAI,KAAK,CAAC;IACxC,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,KAAK,GAAG,MAAM,GAAG,MAAM,CAAC;CACjC,CAAC,CAiBD"}

package/dist/learning/model-warmup.js

@@ -27,6 +27,7 @@
 import { getModelRegistry } from './model-registry.js';
 import { spotCheckModel, refreshModelRegistry } from '../inference/model-probe.js';
 import { detectCredentialChange } from './credential-fingerprint.js';
+import { acquireDutyLease, releaseDutyLease, renewDutyLease, dutyLeaseStaleMs, } from './duty-lease.js';
 import { ProviderFactory } from '../inference/factory.js';
 import { logger } from '../utils/logger.js';
 /** Parse a positive-integer env override; anything else falls back. */
@@ -51,6 +52,11 @@ export function warmupConfig(env = process.env) {
         exploreThrottleMs: positiveInt(env.NUVIRA_WARMUP_EXPLORE_THROTTLE_MS, 600_000),
         hotWindowMs: positiveInt(env.NUVIRA_WARMUP_HOT_WINDOW_MS, 30_600_000),
         warmWindowMs: positiveInt(env.NUVIRA_WARMUP_WARM_WINDOW_MS, 86_400_000),
+        poolRefreshPerCycle: positiveInt(env.NUVIRA_WARMUP_POOL_PER_CYCLE, 4),
+        // 12 hours: a verified model is re-probed twice a day, so a provider that
+        // silently stopped serving one is noticed within a working session rather
+        // than at the 7-day retirement.
+        poolStalenessMs: positiveInt(env.NUVIRA_WARMUP_POOL_STALENESS_MS, 43_200_000),
     };
 }
 /** In-memory usage tracker (persisted to registry via telemetry). */
@@ -204,6 +210,58 @@ export function selectExplorationCandidates(registry, configManager, now, limit)
         a.model.localeCompare(b.model));
     return rows.slice(0, limit).map(({ provider, model, priority }) => ({ provider, model, priority }));
 }
+/**
+ * Pick VERIFIED, in-pool models that have gone STALE, oldest verification
+ * first.
+ *
+ * This is the half the module was missing: exploration (`status ===
+ * 'unverified'`) grows the pool, hot records keep THIS process's models warm,
+ * and nothing refreshed a verified model that stopped being used — so a pool
+ * model aged toward the registry's 7-day retirement and, because the health
+ * ranking reads `lastVerifiedAt`, routing kept ranking it on evidence that was
+ * no longer true.
+ *
+ * Deliberately narrow:
+ *   - VERIFIED rows only. An `unverified` row belongs to exploration, and an
+ *     `unavailable` one is not stale — it is broken, and re-probing it is
+ *     exploration's job once it is re-listed.
+ *   - Unservable providers are excluded, so a cycle never spends its budget on
+ *     a provider with no credentials.
+ *   - Quota-parked rows are skipped: a park is a rest with a known end, and
+ *     re-probing it would be exactly the extra traffic the park prevents.
+ *   - The exploration throttle still applies per model, so this can never
+ *     re-probe the same model twice in one `exploreThrottleMs` window.
+ */
+export function selectPoolRefreshCandidates(registry, configManager, now, limit) {
+    if (limit <= 0)
+        return [];
+    const cfg = warmupConfig();
+    const rows = [];
+    for (const provider of registry.getTrackedProviders()) {
+        if (!isServable(provider, configManager))
+            continue;
+        for (const entry of registry.getAllModelsForProvider(provider)) {
+            if (entry.status !== 'verified')
+                continue;
+            if (!entry.model || entry.model === 'default')
+                continue;
+            if (entry.quotaParkedUntil > now)
+                continue;
+            if (now - (entry.lastProbedAt ?? 0) < cfg.exploreThrottleMs)
+                continue;
+            const lastVerifiedAt = entry.lastVerifiedAt ?? 0;
+            if (now - lastVerifiedAt < cfg.poolStalenessMs)
+                continue;
+            rows.push({ provider, model: entry.model, priority: 0, lastVerifiedAt });
+        }
+    }
+    // Oldest verification first: the most-stale model is the one whose routing
+    // evidence is furthest from reality.
+    rows.sort((a, b) => a.lastVerifiedAt - b.lastVerifiedAt ||
+        a.provider.localeCompare(b.provider) ||
+        a.model.localeCompare(b.model));
+    return rows.slice(0, limit).map(({ provider, model, priority }) => ({ provider, model, priority }));
+}
 /**
  * Run one warmup cycle: keep hot models warm, and grow the verified pool.
  *
@@ -213,7 +271,62 @@ export function selectExplorationCandidates(registry, configManager, now, limit)
  * a fresh process (the old `if (records.length === 0) return;` made the first
  * cycle of every short-lived CLI run a no-op).
  */
+/** The duty name every warmup caller contends on — one lease per machine. */
+export const WARMUP_DUTY_LEASE = 'model-warmup';
+/**
+ * How long an un-renewed warmup lease may live before another process may steal
+ * it (ms).
+ *
+ * A live cycle re-news at half this window (see the loop below), so this is
+ * purely the CRASH-RECOVERY delay: how long a dead holder can keep the sweep
+ * switched off. It is also the floor under the interval — a lease that expired
+ * inside one cycle would let two processes sweep at once.
+ */
+export function warmupLeaseStaleMs(env = process.env) {
+    return Math.max(dutyLeaseStaleMs(env), warmupConfig(env).intervalMs * 2);
+}
+/**
+ * Run one warmup cycle UNDER the warmup duty lease, or refuse and say who holds
+ * it.
+ *
+ * WHY THE LEASE LIVES HERE AND NOT IN THE CALLERS. Every caller wants the same
+ * thing — "one sweep at a time" — and there are four of them (the daemon timer,
+ * the first-cycle kick at startup, `nuvira models warm`, and any future
+ * dashboard trigger). Putting the guard in each one means the fifth caller, or a
+ * refactor, silently re-introduces the waste. Putting it here means the sweep
+ * cannot be started un-leased at all.
+ *
+ * The work a cycle does is a real generation per candidate, on the user's free
+ * tier — which is why two processes sweeping the same candidate list is not
+ * merely redundant, it is the exact traffic that turns warmup into 429s.
+ */
 export async function runWarmupCycle(configManager, deps = {}) {
+    const staleMs = warmupLeaseStaleMs();
+    const taken = acquireDutyLease(WARMUP_DUTY_LEASE, { staleMs, purpose: 'model warmup sweep' });
+    if (!taken.owned) {
+        logger.debug(`[warmup] ${WARMUP_DUTY_LEASE} lease held by pid ${taken.holder?.pid ?? '?'} on ${taken.holder?.host ?? '?'} — skipping this cycle`);
+        return {
+            warmed: 0,
+            explored: 0,
+            poolRefreshed: 0,
+            verified: 0,
+            skipped: 0,
+            catalogRefreshed: false,
+            ran: false,
+            ...(taken.holder ? { leaseHeldBy: taken.holder } : {}),
+        };
+    }
+    try {
+        return await runWarmupCycleBody(configManager, deps, { name: WARMUP_DUTY_LEASE, staleMs });
+    }
+    finally {
+        // Released even on a throw: a failing cycle must not switch the sweep off
+        // until the lease expires.
+        releaseDutyLease(WARMUP_DUTY_LEASE);
+    }
+}
+/** The cycle itself. Call it through {@link runWarmupCycle}, which leases it. */
+async function runWarmupCycleBody(configManager, deps, lease) {
     const registry = getModelRegistry();
     const now = Date.now();
     const cfg = warmupConfig();
@@ -221,9 +334,11 @@ export async function runWarmupCycle(configManager, deps = {}) {
     const result = {
         warmed: 0,
         explored: 0,
+        poolRefreshed: 0,
         verified: 0,
         skipped: 0,
         catalogRefreshed: false,
+        ran: true,
     };
     // ── Half 0: did the KEY SET change since the last run? ──────────────────
     // The pool can only hold models a probe has verified, and there is no
@@ -263,12 +378,30 @@ export async function runWarmupCycle(configManager, deps = {}) {
     result.skipped = hot.length - toWarm.length;
     // ── Half 2: never-verified models (the pool the router cannot see) ──────
     const exploration = selectExplorationCandidates(registry, configManager, now, cfg.explorePerCycle);
+    // ── Half 3: verified pool models that have gone STALE ──────────────────
+    // WITHOUT this half a verified model could only age: exploration looks at
+    // `unverified` rows, the hot half only knows THIS process's usage, and the
+    // registry retires what nothing re-verifies (7 days). Measured pool: verified
+    // entries at 142.8h and shrinking.
+    const stalePool = selectPoolRefreshCandidates(registry, configManager, now, cfg.poolRefreshPerCycle);
     const exploreKeys = new Set(exploration.map((c) => usageKey(c.provider, c.model)));
-    const cycle = [...toWarm, ...exploration];
+    const poolKeys = new Set(stalePool.map((c) => usageKey(c.provider, c.model)));
+    const cycle = [...toWarm, ...exploration, ...stalePool];
     if (cycle.length === 0)
         return result;
-    logger.debug(`[warmup] ${toWarm.length} hot · ${exploration.length} unverified candidates`);
+    logger.debug(`[warmup] ${toWarm.length} hot · ${exploration.length} unverified · ${stalePool.length} stale pool candidates`);
+    // Keep the lease alive across a long cycle. A cycle is up to
+    // `hotPerCycle + explorePerCycle + poolRefreshPerCycle` real generations, each
+    // bounded by the probe timeout, so it can outlive the stale window — and a
+    // lease that expired MID-cycle would invite a second process to start
+    // sweeping the same models, which is the waste the lease exists to stop.
+    const renewEveryMs = Math.max(1_000, Math.floor(lease.staleMs / 2));
+    let lastRenewAt = Date.now();
     for (const candidate of cycle) {
+        if (Date.now() - lastRenewAt >= renewEveryMs) {
+            renewDutyLease(lease.name);
+            lastRenewAt = Date.now();
+        }
         const key = usageKey(candidate.provider, candidate.model);
         try {
             const outcome = await spotCheck(candidate.provider, candidate.model, configManager);
@@ -276,6 +409,8 @@ export async function runWarmupCycle(configManager, deps = {}) {
                 result.verified += 1;
             if (exploreKeys.has(key))
                 result.explored += 1;
+            else if (poolKeys.has(key))
+                result.poolRefreshed += 1;
             else
                 result.warmed += 1;
             if (outcome === 'verified') {

package/dist/learning/model-warmup.js.map

@@ -1 +1 @@
-{"version":3,"file":"model-warmup.js","sourceRoot":"","sources":["../../src/learning/model-warmup.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAEH,OAAO,EAAE,gBAAgB,EAA2B,MAAM,qBAAqB,CAAC;AAChF,OAAO,EAAE,cAAc,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AACnF,OAAO,EAAE,sBAAsB,EAAE,MAAM,6BAA6B,CAAC;AACrE,OAAO,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAE1D,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAgC5C,uEAAuE;AACvE,SAAS,WAAW,CAAC,GAAuB,EAAE,QAAgB;IAC5D,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE;QAAE,OAAO,QAAQ,CAAC;IAC5D,MAAM,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;IACtB,OAAO,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;AAChE,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,YAAY,CAAC,MAAyB,OAAO,CAAC,GAAG;IAC/D,OAAO;QACL,UAAU,EAAE,WAAW,CAAC,GAAG,CAAC,yBAAyB,EAAE,MAAM,CAAC;QAC9D,cAAc,EAAE,WAAW,CAAC,GAAG,CAAC,yBAAyB,EAAE,OAAO,CAAC;QACnE,WAAW,EAAE,WAAW,CAAC,GAAG,CAAC,2BAA2B,EAAE,EAAE,CAAC;QAC7D,eAAe,EAAE,WAAW,CAAC,GAAG,CAAC,+BAA+B,EAAE,CAAC,CAAC;QACpE,iBAAiB,EAAE,WAAW,CAAC,GAAG,CAAC,iCAAiC,EAAE,OAAO,CAAC;QAC9E,WAAW,EAAE,WAAW,CAAC,GAAG,CAAC,2BAA2B,EAAE,UAAU,CAAC;QACrE,YAAY,EAAE,WAAW,CAAC,GAAG,CAAC,4BAA4B,EAAE,UAAU,CAAC;KACxE,CAAC;AACJ,CAAC;AA+BD,qEAAqE;AACrE,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAuB,CAAC;AAEhD,SAAS,QAAQ,CAAC,QAAgB,EAAE,KAAa;IAC/C,OAAO,GAAG,QAAQ,IAAI,KAAK,EAAE,CAAC;AAChC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,QAAgB,EAAE,KAAa;IAC9D,MAAM,GAAG,GAAG,QAAQ,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;IACtC,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACnC,IAAI,QAAQ,EAAE,CAAC;QACb,QAAQ,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACjC,QAAQ,CAAC,QAAQ,EAAE,CAAC;IACtB,CAAC;SAAM,CAAC;QACN,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE;YAChB,QAAQ;YACR,KAAK;YACL,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE;YACtB,QAAQ,EAAE,CAAC;SACZ,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAUD;;;;;;;GAOG;AACH,SAAS,mBAAmB,CAC1B,MAAmB,EACnB,aAA6C;IAE7C,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAEvB,wDAAwD;IACxD,MAAM,YAAY,GAAG,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC;IAC7C,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,YAAY,GAAG,YAAY,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC;IAElF,6CAA6C;IAC7C,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC;IAE1D,sEAAsE;IACtE,IAAI,iBAAiB,GAAG,GAAG,CAAC;IAC5B,IAAI,aAAa,EAAE,CAAC;QAClB,IAAI,aAAa,CAAC,MAAM,KAAK,UAAU;YAAE,iBAAiB,GAAG,GAAG,CAAC;aAC5D,IAAI,aAAa,CAAC,MAAM,KAAK,aAAa;YAAE,iBAAiB,GAAG,GAAG,CAAC;IAC3E,CAAC;IAED,8DAA8D;IAC9D,IAAI,YAAY,GAAG,GAAG,CAAC;IACvB,IAAI,aAAa,EAAE,SAAS,EAAE,CAAC;QAC7B,wDAAwD;QACxD,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,aAAa,CAAC,SAAS,GAAG,IAAI,CAAC,CAAC;IAC7D,CAAC;IAED,4BAA4B;IAC5B,OAAO,CACL,YAAY,GAAG,IAAI;QACnB,cAAc,GAAG,IAAI;QACrB,iBAAiB,GAAG,IAAI;QACxB,YAAY,GAAG,IAAI,CACpB,CAAC;AACJ,CAAC;AAED,+EAA+E;AAE/E,IAAI,WAAW,GAA0C,IAAI,CAAC;AAC9D,IAAI,SAAS,GAAG,KAAK,CAAC;AAEtB;;;;GAIG;AACH,MAAM,UAAU,iBAAiB,CAAC,aAA4B,EAAE,OAAmB,EAAE;IACnF,IAAI,WAAW;QAAE,OAAO,CAAC,kBAAkB;IAE3C,MAAM,UAAU,GAAG,YAAY,EAAE,CAAC,UAAU,CAAC;IAC7C,MAAM,CAAC,IAAI,CAAC,gDAAgD,UAAU,KAAK,CAAC,CAAC;IAC7E,WAAW,GAAG,WAAW,CAAC,GAAG,EAAE;QAC7B,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,SAAS,GAAG,IAAI,CAAC;YACjB,cAAc,CAAC,aAAa,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,SAAS,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5E,CAAC;IACH,CAAC,EAAE,UAAU,CAAC,CAAC;IACf,+CAA+C;IAC/C,EAAE;IACF,yEAAyE;IACzE,6EAA6E;IAC7E,4EAA4E;IAC5E,yEAAyE;IACzE,6EAA6E;IAC7E,oDAAoD;IACnD,WAAiD,CAAC,KAAK,EAAE,EAAE,CAAC;IAE7D,8BAA8B;IAC9B,SAAS,GAAG,IAAI,CAAC;IACjB,cAAc,CAAC,aAAa,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,SAAS,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;AAC5E,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB;IAC9B,IAAI,WAAW,EAAE,CAAC;QAChB,aAAa,CAAC,WAAW,CAAC,CAAC;QAC3B,WAAW,GAAG,IAAI,CAAC;QACnB,MAAM,CAAC,IAAI,CAAC,sCAAsC,CAAC,CAAC;IACtD,CAAC;AACH,CAAC;AAqCD,oEAAoE;AACpE,SAAS,UAAU,CAAC,QAAgB,EAAE,aAA4B;IAChE,IAAI,CAAC;QACH,IAAI,CAAC,eAAe,CAAC,eAAe,CAAC,QAAQ,CAAC;YAAE,OAAO,KAAK,CAAC;QAC7D,OAAO,aAAa,CAAC,sBAAsB,CAAC,QAAQ,CAAC,KAAK,IAAI,CAAC;IACjE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,2BAA2B,CACzC,QAA6C,EAC7C,aAA4B,EAC5B,GAAW,EACX,KAAa;IAEb,IAAI,KAAK,IAAI,CAAC;QAAE,OAAO,EAAE,CAAC;IAC1B,MAAM,IAAI,GAA4D,EAAE,CAAC;IAEzE,MAAM,iBAAiB,GAAG,YAAY,EAAE,CAAC,iBAAiB,CAAC;IAC3D,KAAK,MAAM,QAAQ,IAAI,QAAQ,CAAC,mBAAmB,EAAE,EAAE,CAAC;QACtD,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,aAAa,CAAC;YAAE,SAAS;QACnD,MAAM,kBAAkB,GAAG,QAAQ,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC,QAAQ,CAAC;QACxE,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,uBAAuB,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC/D,IAAI,KAAK,CAAC,MAAM,KAAK,YAAY;gBAAE,SAAS;YAC5C,sEAAsE;YACtE,IAAI,CAAC,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS;gBAAE,SAAS;YACxD,IAAI,KAAK,CAAC,gBAAgB,GAAG,GAAG;gBAAE,SAAS;YAC3C,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,YAAY,IAAI,CAAC,CAAC,GAAG,iBAAiB;gBAAE,SAAS;YAClE,IAAI,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,QAAQ,EAAE,CAAC,kBAAkB,EAAE,kBAAkB,EAAE,CAAC,CAAC;QACjG,CAAC;IACH,CAAC;IAED,IAAI,CAAC,IAAI,CACP,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACP,CAAC,CAAC,kBAAkB,GAAG,CAAC,CAAC,kBAAkB;QAC3C,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC;QACpC,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,CACjC,CAAC;IACF,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC;AACtG,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,aAA4B,EAC5B,OAAmB,EAAE;IAErB,MAAM,QAAQ,GAAG,gBAAgB,EAAE,CAAC;IACpC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,GAAG,GAAG,YAAY,EAAE,CAAC;IAC3B,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,cAAc,CAAC;IACnD,MAAM,MAAM,GAAsB;QAChC,MAAM,EAAE,CAAC;QACT,QAAQ,EAAE,CAAC;QACX,QAAQ,EAAE,CAAC;QACX,OAAO,EAAE,CAAC;QACV,gBAAgB,EAAE,KAAK;KACxB,CAAC;IAEF,2EAA2E;IAC3E,sEAAsE;IACtE,6EAA6E;IAC7E,2EAA2E;IAC3E,6EAA6E;IAC7E,6EAA6E;IAC7E,mCAAmC;IACnC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,sBAAsB,IAAI,sBAAsB,CAAC;QACrE,IAAI,MAAM,CAAC,aAAa,CAAC,CAAC,OAAO,EAAE,CAAC;YAClC,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,IAAI,CAAC,CAAC,EAAiB,EAAE,EAAE,CAAC,oBAAoB,CAAC,EAAE,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;YAC9G,MAAM,CAAC,gBAAgB,GAAG,IAAI,CAAC;YAC/B,MAAM,CAAC,IAAI,CAAC,uDAAuD,CAAC,CAAC;YACrE,MAAM,OAAO,CAAC,aAAa,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,2DAA2D;IAC7D,CAAC;IAED,2EAA2E;IAC3E,MAAM,GAAG,GAAsB,EAAE,CAAC;IAClC,KAAK,MAAM,MAAM,IAAI,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC;QACvC,MAAM,KAAK,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;QAC/D,IAAI,KAAK,IAAI,KAAK,CAAC,UAAU,IAAI,GAAG,GAAG,KAAK,CAAC,UAAU,GAAG,GAAG,CAAC,cAAc;YAAE,SAAS;QACvF,IAAI,KAAK,EAAE,MAAM,KAAK,aAAa;YAAE,SAAS;QAC9C,IAAI,GAAG,GAAG,MAAM,CAAC,UAAU,GAAG,GAAG,CAAC,YAAY;YAAE,SAAS;QACzD,MAAM,QAAQ,GAAG,mBAAmB,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QACpD,IAAI,QAAQ,GAAG,GAAG;YAAE,GAAG,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC;IAC7F,CAAC;IACD,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;IAE5C,MAAM,MAAM,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,WAAW,CAAC,CAAC;IAC7C,MAAM,CAAC,OAAO,GAAG,GAAG,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;IAE5C,2EAA2E;IAC3E,MAAM,WAAW,GAAG,2BAA2B,CAC7C,QAAQ,EACR,aAAa,EACb,GAAG,EACH,GAAG,CAAC,eAAe,CACpB,CAAC;IAEF,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACnF,MAAM,KAAK,GAAG,CAAC,GAAG,MAAM,EAAE,GAAG,WAAW,CAAC,CAAC;IAC1C,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,MAAM,CAAC;IAEtC,MAAM,CAAC,KAAK,CACV,YAAY,MAAM,CAAC,MAAM,UAAU,WAAW,CAAC,MAAM,wBAAwB,CAC9E,CAAC;IAEF,KAAK,MAAM,SAAS,IAAI,KAAK,EAAE,CAAC;QAC9B,MAAM,GAAG,GAAG,QAAQ,CAAC,SAAS,CAAC,QAAQ,EAAE,SAAS,CAAC,KAAK,CAAC,CAAC;QAC1D,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,SAAS,CAAC,QAAQ,EAAE,SAAS,CAAC,KAAK,EAAE,aAAa,CAAC,CAAC;YACpF,IAAI,OAAO,KAAK,UAAU;gBAAE,MAAM,CAAC,QAAQ,IAAI,CAAC,CAAC;YACjD,IAAI,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,MAAM,CAAC,QAAQ,IAAI,CAAC,CAAC;;gBAC1C,MAAM,CAAC,MAAM,IAAI,CAAC,CAAC;YACxB,IAAI,OAAO,KAAK,UAAU,EAAE,CAAC;gBAC3B,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBACjC,IAAI,MAAM;oBAAE,MAAM,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC,kBAAkB;YACzD,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,gDAAgD;QAClD,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,+EAA+E;AAE/E;;GAEG;AACH,MAAM,UAAU,cAAc;IAM5B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,GAAG,GAAG,YAAY,EAAE,CAAC;IAC3B,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;IAE9C,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,YAAY,GAAG,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC;QAC7C,IAAI,YAAY,GAAG,GAAG,CAAC,WAAW;YAAE,SAAS,EAAE,CAAC;aAC3C,IAAI,YAAY,GAAG,GAAG,CAAC,YAAY;YAAE,UAAU,EAAE,CAAC;IACzD,CAAC;IAED,OAAO;QACL,aAAa,EAAE,OAAO,CAAC,MAAM;QAC7B,SAAS;QACT,UAAU;QACV,WAAW,EAAE,GAAG;KACjB,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB;IAO9B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,GAAG,GAAG,YAAY,EAAE,CAAC;IAC3B,OAAO,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE;QAChD,MAAM,YAAY,GAAG,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC;QAC7C,IAAI,MAAM,GAA4B,MAAM,CAAC;QAC7C,IAAI,YAAY,GAAG,GAAG,CAAC,WAAW;YAAE,MAAM,GAAG,KAAK,CAAC;aAC9C,IAAI,YAAY,GAAG,GAAG,CAAC,YAAY;YAAE,MAAM,GAAG,MAAM,CAAC;QAE1D,OAAO;YACL,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,MAAM;SACP,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC"}
+{"version":3,"file":"model-warmup.js","sourceRoot":"","sources":["../../src/learning/model-warmup.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAEH,OAAO,EAAE,gBAAgB,EAA2B,MAAM,qBAAqB,CAAC;AAChF,OAAO,EAAE,cAAc,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AACnF,OAAO,EAAE,sBAAsB,EAAE,MAAM,6BAA6B,CAAC;AACrE,OAAO,EACL,gBAAgB,EAChB,gBAAgB,EAChB,cAAc,EACd,gBAAgB,GAEjB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AAE1D,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AA2D5C,uEAAuE;AACvE,SAAS,WAAW,CAAC,GAAuB,EAAE,QAAgB;IAC5D,IAAI,GAAG,KAAK,SAAS,IAAI,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE;QAAE,OAAO,QAAQ,CAAC;IAC5D,MAAM,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC;IACtB,OAAO,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;AAChE,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,YAAY,CAAC,MAAyB,OAAO,CAAC,GAAG;IAC/D,OAAO;QACL,UAAU,EAAE,WAAW,CAAC,GAAG,CAAC,yBAAyB,EAAE,MAAM,CAAC;QAC9D,cAAc,EAAE,WAAW,CAAC,GAAG,CAAC,yBAAyB,EAAE,OAAO,CAAC;QACnE,WAAW,EAAE,WAAW,CAAC,GAAG,CAAC,2BAA2B,EAAE,EAAE,CAAC;QAC7D,eAAe,EAAE,WAAW,CAAC,GAAG,CAAC,+BAA+B,EAAE,CAAC,CAAC;QACpE,iBAAiB,EAAE,WAAW,CAAC,GAAG,CAAC,iCAAiC,EAAE,OAAO,CAAC;QAC9E,WAAW,EAAE,WAAW,CAAC,GAAG,CAAC,2BAA2B,EAAE,UAAU,CAAC;QACrE,YAAY,EAAE,WAAW,CAAC,GAAG,CAAC,4BAA4B,EAAE,UAAU,CAAC;QACvE,mBAAmB,EAAE,WAAW,CAAC,GAAG,CAAC,4BAA4B,EAAE,CAAC,CAAC;QACrE,0EAA0E;QAC1E,0EAA0E;QAC1E,gCAAgC;QAChC,eAAe,EAAE,WAAW,CAAC,GAAG,CAAC,+BAA+B,EAAE,UAAU,CAAC;KAC9E,CAAC;AACJ,CAAC;AA+BD,qEAAqE;AACrE,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAuB,CAAC;AAEhD,SAAS,QAAQ,CAAC,QAAgB,EAAE,KAAa;IAC/C,OAAO,GAAG,QAAQ,IAAI,KAAK,EAAE,CAAC;AAChC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,QAAgB,EAAE,KAAa;IAC9D,MAAM,GAAG,GAAG,QAAQ,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;IACtC,MAAM,QAAQ,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACnC,IAAI,QAAQ,EAAE,CAAC;QACb,QAAQ,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACjC,QAAQ,CAAC,QAAQ,EAAE,CAAC;IACtB,CAAC;SAAM,CAAC;QACN,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE;YAChB,QAAQ;YACR,KAAK;YACL,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE;YACtB,QAAQ,EAAE,CAAC;SACZ,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAUD;;;;;;;GAOG;AACH,SAAS,mBAAmB,CAC1B,MAAmB,EACnB,aAA6C;IAE7C,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAEvB,wDAAwD;IACxD,MAAM,YAAY,GAAG,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC;IAC7C,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,YAAY,GAAG,YAAY,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC;IAElF,6CAA6C;IAC7C,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC;IAE1D,sEAAsE;IACtE,IAAI,iBAAiB,GAAG,GAAG,CAAC;IAC5B,IAAI,aAAa,EAAE,CAAC;QAClB,IAAI,aAAa,CAAC,MAAM,KAAK,UAAU;YAAE,iBAAiB,GAAG,GAAG,CAAC;aAC5D,IAAI,aAAa,CAAC,MAAM,KAAK,aAAa;YAAE,iBAAiB,GAAG,GAAG,CAAC;IAC3E,CAAC;IAED,8DAA8D;IAC9D,IAAI,YAAY,GAAG,GAAG,CAAC;IACvB,IAAI,aAAa,EAAE,SAAS,EAAE,CAAC;QAC7B,wDAAwD;QACxD,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,aAAa,CAAC,SAAS,GAAG,IAAI,CAAC,CAAC;IAC7D,CAAC;IAED,4BAA4B;IAC5B,OAAO,CACL,YAAY,GAAG,IAAI;QACnB,cAAc,GAAG,IAAI;QACrB,iBAAiB,GAAG,IAAI;QACxB,YAAY,GAAG,IAAI,CACpB,CAAC;AACJ,CAAC;AAED,+EAA+E;AAE/E,IAAI,WAAW,GAA0C,IAAI,CAAC;AAC9D,IAAI,SAAS,GAAG,KAAK,CAAC;AAEtB;;;;GAIG;AACH,MAAM,UAAU,iBAAiB,CAAC,aAA4B,EAAE,OAAmB,EAAE;IACnF,IAAI,WAAW;QAAE,OAAO,CAAC,kBAAkB;IAE3C,MAAM,UAAU,GAAG,YAAY,EAAE,CAAC,UAAU,CAAC;IAC7C,MAAM,CAAC,IAAI,CAAC,gDAAgD,UAAU,KAAK,CAAC,CAAC;IAC7E,WAAW,GAAG,WAAW,CAAC,GAAG,EAAE;QAC7B,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,SAAS,GAAG,IAAI,CAAC;YACjB,cAAc,CAAC,aAAa,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,SAAS,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5E,CAAC;IACH,CAAC,EAAE,UAAU,CAAC,CAAC;IACf,+CAA+C;IAC/C,EAAE;IACF,yEAAyE;IACzE,6EAA6E;IAC7E,4EAA4E;IAC5E,yEAAyE;IACzE,6EAA6E;IAC7E,oDAAoD;IACnD,WAAiD,CAAC,KAAK,EAAE,EAAE,CAAC;IAE7D,8BAA8B;IAC9B,SAAS,GAAG,IAAI,CAAC;IACjB,cAAc,CAAC,aAAa,EAAE,IAAI,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,SAAS,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;AAC5E,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB;IAC9B,IAAI,WAAW,EAAE,CAAC;QAChB,aAAa,CAAC,WAAW,CAAC,CAAC;QAC3B,WAAW,GAAG,IAAI,CAAC;QACnB,MAAM,CAAC,IAAI,CAAC,sCAAsC,CAAC,CAAC;IACtD,CAAC;AACH,CAAC;AAmDD,oEAAoE;AACpE,SAAS,UAAU,CAAC,QAAgB,EAAE,aAA4B;IAChE,IAAI,CAAC;QACH,IAAI,CAAC,eAAe,CAAC,eAAe,CAAC,QAAQ,CAAC;YAAE,OAAO,KAAK,CAAC;QAC7D,OAAO,aAAa,CAAC,sBAAsB,CAAC,QAAQ,CAAC,KAAK,IAAI,CAAC;IACjE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,2BAA2B,CACzC,QAA6C,EAC7C,aAA4B,EAC5B,GAAW,EACX,KAAa;IAEb,IAAI,KAAK,IAAI,CAAC;QAAE,OAAO,EAAE,CAAC;IAC1B,MAAM,IAAI,GAA4D,EAAE,CAAC;IAEzE,MAAM,iBAAiB,GAAG,YAAY,EAAE,CAAC,iBAAiB,CAAC;IAC3D,KAAK,MAAM,QAAQ,IAAI,QAAQ,CAAC,mBAAmB,EAAE,EAAE,CAAC;QACtD,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,aAAa,CAAC;YAAE,SAAS;QACnD,MAAM,kBAAkB,GAAG,QAAQ,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC,QAAQ,CAAC;QACxE,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,uBAAuB,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC/D,IAAI,KAAK,CAAC,MAAM,KAAK,YAAY;gBAAE,SAAS;YAC5C,sEAAsE;YACtE,IAAI,CAAC,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS;gBAAE,SAAS;YACxD,IAAI,KAAK,CAAC,gBAAgB,GAAG,GAAG;gBAAE,SAAS;YAC3C,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,YAAY,IAAI,CAAC,CAAC,GAAG,iBAAiB;gBAAE,SAAS;YAClE,IAAI,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,QAAQ,EAAE,CAAC,kBAAkB,EAAE,kBAAkB,EAAE,CAAC,CAAC;QACjG,CAAC;IACH,CAAC;IAED,IAAI,CAAC,IAAI,CACP,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACP,CAAC,CAAC,kBAAkB,GAAG,CAAC,CAAC,kBAAkB;QAC3C,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC;QACpC,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,CACjC,CAAC;IACF,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC;AACtG,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,MAAM,UAAU,2BAA2B,CACzC,QAA6C,EAC7C,aAA4B,EAC5B,GAAW,EACX,KAAa;IAEb,IAAI,KAAK,IAAI,CAAC;QAAE,OAAO,EAAE,CAAC;IAC1B,MAAM,GAAG,GAAG,YAAY,EAAE,CAAC;IAC3B,MAAM,IAAI,GAAwD,EAAE,CAAC;IAErE,KAAK,MAAM,QAAQ,IAAI,QAAQ,CAAC,mBAAmB,EAAE,EAAE,CAAC;QACtD,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,aAAa,CAAC;YAAE,SAAS;QACnD,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,uBAAuB,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC/D,IAAI,KAAK,CAAC,MAAM,KAAK,UAAU;gBAAE,SAAS;YAC1C,IAAI,CAAC,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS;gBAAE,SAAS;YACxD,IAAI,KAAK,CAAC,gBAAgB,GAAG,GAAG;gBAAE,SAAS;YAC3C,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,YAAY,IAAI,CAAC,CAAC,GAAG,GAAG,CAAC,iBAAiB;gBAAE,SAAS;YACtE,MAAM,cAAc,GAAG,KAAK,CAAC,cAAc,IAAI,CAAC,CAAC;YACjD,IAAI,GAAG,GAAG,cAAc,GAAG,GAAG,CAAC,eAAe;gBAAE,SAAS;YACzD,IAAI,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,QAAQ,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,CAAC;QAC3E,CAAC;IACH,CAAC;IAED,2EAA2E;IAC3E,qCAAqC;IACrC,IAAI,CAAC,IAAI,CACP,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACP,CAAC,CAAC,cAAc,GAAG,CAAC,CAAC,cAAc;QACnC,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC;QACpC,CAAC,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,CACjC,CAAC;IACF,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC;AACtG,CAAC;AAED;;;;;;;;GAQG;AACH,6EAA6E;AAC7E,MAAM,CAAC,MAAM,iBAAiB,GAAG,cAAc,CAAC;AAEhD;;;;;;;;GAQG;AACH,MAAM,UAAU,kBAAkB,CAAC,MAAyB,OAAO,CAAC,GAAG;IACrE,OAAO,IAAI,CAAC,GAAG,CAAC,gBAAgB,CAAC,GAAG,CAAC,EAAE,YAAY,CAAC,GAAG,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC;AAC3E,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,aAA4B,EAC5B,OAAmB,EAAE;IAErB,MAAM,OAAO,GAAG,kBAAkB,EAAE,CAAC;IACrC,MAAM,KAAK,GAAG,gBAAgB,CAAC,iBAAiB,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,oBAAoB,EAAE,CAAC,CAAC;IAC9F,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QACjB,MAAM,CAAC,KAAK,CACV,YAAY,iBAAiB,sBAAsB,KAAK,CAAC,MAAM,EAAE,GAAG,IAAI,GAAG,OAAO,KAAK,CAAC,MAAM,EAAE,IAAI,IAAI,GAAG,wBAAwB,CACpI,CAAC;QACF,OAAO;YACL,MAAM,EAAE,CAAC;YACT,QAAQ,EAAE,CAAC;YACX,aAAa,EAAE,CAAC;YAChB,QAAQ,EAAE,CAAC;YACX,OAAO,EAAE,CAAC;YACV,gBAAgB,EAAE,KAAK;YACvB,GAAG,EAAE,KAAK;YACV,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACvD,CAAC;IACJ,CAAC;IACD,IAAI,CAAC;QACH,OAAO,MAAM,kBAAkB,CAAC,aAAa,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,iBAAiB,EAAE,OAAO,EAAE,CAAC,CAAC;IAC7F,CAAC;YAAS,CAAC;QACT,0EAA0E;QAC1E,2BAA2B;QAC3B,gBAAgB,CAAC,iBAAiB,CAAC,CAAC;IACtC,CAAC;AACH,CAAC;AAED,iFAAiF;AACjF,KAAK,UAAU,kBAAkB,CAC/B,aAA4B,EAC5B,IAAgB,EAChB,KAAwC;IAExC,MAAM,QAAQ,GAAG,gBAAgB,EAAE,CAAC;IACpC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,GAAG,GAAG,YAAY,EAAE,CAAC;IAC3B,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,cAAc,CAAC;IACnD,MAAM,MAAM,GAAsB;QAChC,MAAM,EAAE,CAAC;QACT,QAAQ,EAAE,CAAC;QACX,aAAa,EAAE,CAAC;QAChB,QAAQ,EAAE,CAAC;QACX,OAAO,EAAE,CAAC;QACV,gBAAgB,EAAE,KAAK;QACvB,GAAG,EAAE,IAAI;KACV,CAAC;IAEF,2EAA2E;IAC3E,sEAAsE;IACtE,6EAA6E;IAC7E,2EAA2E;IAC3E,6EAA6E;IAC7E,6EAA6E;IAC7E,mCAAmC;IACnC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,sBAAsB,IAAI,sBAAsB,CAAC;QACrE,IAAI,MAAM,CAAC,aAAa,CAAC,CAAC,OAAO,EAAE,CAAC;YAClC,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,IAAI,CAAC,CAAC,EAAiB,EAAE,EAAE,CAAC,oBAAoB,CAAC,EAAE,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;YAC9G,MAAM,CAAC,gBAAgB,GAAG,IAAI,CAAC;YAC/B,MAAM,CAAC,IAAI,CAAC,uDAAuD,CAAC,CAAC;YACrE,MAAM,OAAO,CAAC,aAAa,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,2DAA2D;IAC7D,CAAC;IAED,2EAA2E;IAC3E,MAAM,GAAG,GAAsB,EAAE,CAAC;IAClC,KAAK,MAAM,MAAM,IAAI,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC;QACvC,MAAM,KAAK,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;QAC/D,IAAI,KAAK,IAAI,KAAK,CAAC,UAAU,IAAI,GAAG,GAAG,KAAK,CAAC,UAAU,GAAG,GAAG,CAAC,cAAc;YAAE,SAAS;QACvF,IAAI,KAAK,EAAE,MAAM,KAAK,aAAa;YAAE,SAAS;QAC9C,IAAI,GAAG,GAAG,MAAM,CAAC,UAAU,GAAG,GAAG,CAAC,YAAY;YAAE,SAAS;QACzD,MAAM,QAAQ,GAAG,mBAAmB,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QACpD,IAAI,QAAQ,GAAG,GAAG;YAAE,GAAG,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC;IAC7F,CAAC;IACD,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;IAE5C,MAAM,MAAM,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,WAAW,CAAC,CAAC;IAC7C,MAAM,CAAC,OAAO,GAAG,GAAG,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;IAE5C,2EAA2E;IAC3E,MAAM,WAAW,GAAG,2BAA2B,CAC7C,QAAQ,EACR,aAAa,EACb,GAAG,EACH,GAAG,CAAC,eAAe,CACpB,CAAC;IAEF,0EAA0E;IAC1E,0EAA0E;IAC1E,2EAA2E;IAC3E,8EAA8E;IAC9E,mCAAmC;IACnC,MAAM,SAAS,GAAG,2BAA2B,CAC3C,QAAQ,EACR,aAAa,EACb,GAAG,EACH,GAAG,CAAC,mBAAmB,CACxB,CAAC;IAEF,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IACnF,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAC9E,MAAM,KAAK,GAAG,CAAC,GAAG,MAAM,EAAE,GAAG,WAAW,EAAE,GAAG,SAAS,CAAC,CAAC;IACxD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,MAAM,CAAC;IAEtC,MAAM,CAAC,KAAK,CACV,YAAY,MAAM,CAAC,MAAM,UAAU,WAAW,CAAC,MAAM,iBAAiB,SAAS,CAAC,MAAM,wBAAwB,CAC/G,CAAC;IAEF,6DAA6D;IAC7D,+EAA+E;IAC/E,2EAA2E;IAC3E,sEAAsE;IACtE,yEAAyE;IACzE,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC;IACpE,IAAI,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE7B,KAAK,MAAM,SAAS,IAAI,KAAK,EAAE,CAAC;QAC9B,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,WAAW,IAAI,YAAY,EAAE,CAAC;YAC7C,cAAc,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAC3B,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC3B,CAAC;QACD,MAAM,GAAG,GAAG,QAAQ,CAAC,SAAS,CAAC,QAAQ,EAAE,SAAS,CAAC,KAAK,CAAC,CAAC;QAC1D,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,SAAS,CAAC,QAAQ,EAAE,SAAS,CAAC,KAAK,EAAE,aAAa,CAAC,CAAC;YACpF,IAAI,OAAO,KAAK,UAAU;gBAAE,MAAM,CAAC,QAAQ,IAAI,CAAC,CAAC;YACjD,IAAI,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,MAAM,CAAC,QAAQ,IAAI,CAAC,CAAC;iBAC1C,IAAI,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,MAAM,CAAC,aAAa,IAAI,CAAC,CAAC;;gBACjD,MAAM,CAAC,MAAM,IAAI,CAAC,CAAC;YACxB,IAAI,OAAO,KAAK,UAAU,EAAE,CAAC;gBAC3B,MAAM,MAAM,GAAG,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;gBACjC,IAAI,MAAM;oBAAE,MAAM,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC,kBAAkB;YACzD,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,gDAAgD;QAClD,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,+EAA+E;AAE/E;;GAEG;AACH,MAAM,UAAU,cAAc;IAM5B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,GAAG,GAAG,YAAY,EAAE,CAAC;IAC3B,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;IAE9C,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,YAAY,GAAG,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC;QAC7C,IAAI,YAAY,GAAG,GAAG,CAAC,WAAW;YAAE,SAAS,EAAE,CAAC;aAC3C,IAAI,YAAY,GAAG,GAAG,CAAC,YAAY;YAAE,UAAU,EAAE,CAAC;IACzD,CAAC;IAED,OAAO;QACL,aAAa,EAAE,OAAO,CAAC,MAAM;QAC7B,SAAS;QACT,UAAU;QACV,WAAW,EAAE,GAAG;KACjB,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB;IAO9B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,GAAG,GAAG,YAAY,EAAE,CAAC;IAC3B,OAAO,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE;QAChD,MAAM,YAAY,GAAG,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC;QAC7C,IAAI,MAAM,GAA4B,MAAM,CAAC;QAC7C,IAAI,YAAY,GAAG,GAAG,CAAC,WAAW;YAAE,MAAM,GAAG,KAAK,CAAC;aAC9C,IAAI,YAAY,GAAG,GAAG,CAAC,YAAY;YAAE,MAAM,GAAG,MAAM,CAAC;QAE1D,OAAO;YACL,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,UAAU,EAAE,MAAM,CAAC,UAAU;YAC7B,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,MAAM;SACP,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC"}

package/dist/learning/retry-patience.d.ts

@@ -0,0 +1,139 @@
+/**
+ * Blip patience — how long to STAY on the pair that just failed, and whether
+ * walking away is even a remedy.
+ *
+ * THE OBSERVED PROBLEM THIS ENCODES. A cloud provider does not go down for a
+ * turn: it returns 200, then a few empty/5xx responses for a handful of seconds,
+ * then serves normally again (measured repeatedly on `deepseek`). Two opposite
+ * mistakes are available, and the codebase has made both:
+ *
+ *   - FAIL FAST: give up after seconds and fold onto the next candidate — which
+ *     for that blip means silently dropping from a strong model to a weak one.
+ *     The result is answered, badly, by a model the user did not choose.
+ *   - WAIT BLINDLY: sit out a long fixed backoff on a pair that is actually
+ *     dead, and make every user of a broken provider pay for it.
+ *
+ * A timeout cannot tell those apart, so a fixed schedule is the wrong shape
+ * either way. What CAN tell them apart is the evidence already in hand:
+ *
+ *   1. THE ALTERNATIVES. Walking away is only a remedy when it does not cost
+ *      quality. If every alternative sits in a LOWER capability band, then
+ *      "fail over" means "de-escalate", and for a blip that is the worse trade —
+ *      so we wait instead. If a same-or-better band is reachable, failing over is
+ *      free and waiting is pure waste.
+ *   2. THE PAIR'S TRACK RECORD. A pair that has served real traffic with a low
+ *      error rate earns patience; a never-proven id does not.
+ *   3. THE SHAPE OF THE FAILURE. One failing step is a blip. The SAME pair
+ *      failing two steps in a row is not — patience collapses there, whatever
+ *      the alternatives look like.
+ *
+ * And the schedule's SHAPE matters as much as its total: a 2-second blink is
+ * caught by a retry at t≈1s, not at t=5s, so the early delays are dense and the
+ * late ones sparse ({@link TRANSIENT_BLIP_DELAYS_MS} → attempts at ≈1, 3, 7, 15,
+ * 30, 60s). That rides out both a blink and a real minute-long spike.
+ *
+ * PURE AND DETERMINISTIC. This module reads no clock, no registry and no config:
+ * the caller gathers the evidence (it has the registry and the ranked pool) and
+ * gets back a plan it can log. That is what makes the policy reviewable and
+ * testable one row at a time — see the table in the tests.
+ */
+import { type CapabilityBand } from './model-capability.js';
+/**
+ * Transient (server/network/timeout) backoff, dense early and sparse late:
+ * attempts at ≈1s, 3s, 7s, 15s, 30s, 60s. Used exactly when WAITING beats
+ * walking away — no alternative, or only degrading ones.
+ */
+export declare const TRANSIENT_BLIP_DELAYS_MS: readonly [1000, 2000, 4000, 8000, 15000, 30000];
+/**
+ * Transient backoff when a same-or-better-band candidate IS reachable: a short
+ * courtesy retry (two blinks' worth, <2s) and then the walk. Spending longer is
+ * waste — the replacement costs nothing in quality.
+ */
+export declare const TRANSIENT_FREE_ALTERNATIVE_DELAYS_MS: readonly [500, 1500];
+/**
+ * Transient backoff for a pair we have no reason to trust and no alternative to:
+ * a token wait (≈5s), then the honest failure.
+ */
+export declare const TRANSIENT_LAST_RESORT_DELAYS_MS: readonly [1000, 4000];
+/** Empty-response patience for a proven pair — the "final verse" case. */
+export declare const EMPTY_PATIENCE_MS: readonly [2000, 5000];
+/**
+ * Empty-response patience when EVERY alternative would de-escalate. Longer, and
+ * deliberately so: this is the one situation where the choice is literally
+ * "wait a few more seconds" vs "answer with a weaker model".
+ */
+export declare const EMPTY_PATIENCE_WAIT_BEATS_FOLDING_MS: readonly [2000, 5000, 10000];
+/**
+ * Error rate at or below which a verified pair counts as a proven track record.
+ * Above it the pair has failed roughly 1 call in 5, which is not something a
+ * blip explains away.
+ */
+export declare const PROVEN_ERROR_RATE_MAX = 0.2;
+/** Registry evidence for the failing pair, as the caller reads it. */
+export interface PatienceEvidence {
+    /** `status === 'verified'` in the Model Availability Registry. */
+    verified: boolean;
+    /**
+     * Where that verification came from. Real traffic (`telemetry`) is strong
+     * evidence; a bare catalog `probe` is not (it proves the id exists, nothing
+     * about answers), so it cannot on its own earn the long patience.
+     */
+    source?: 'telemetry' | 'spot-check' | 'probe';
+    /** Rolling error rate 0–1, when recorded. */
+    errorRate?: number;
+}
+export interface PatienceInput {
+    /** The pair that just failed. */
+    current: {
+        provider: string;
+        model: string;
+    };
+    /**
+     * The pairs this surface could switch to, best first (provider + the model it
+     * would actually serve). Empty = this surface has nowhere else to go — a
+     * pinned turn, a single configured provider, the last candidate.
+     */
+    alternatives: ReadonlyArray<{
+        provider: string;
+        model: string;
+    }>;
+    /** Registry evidence for {@link current}; omit when nothing is recorded. */
+    evidence?: PatienceEvidence;
+    /** Did this pair already fail an EARLIER step of this same turn? */
+    failedEarlierStep?: boolean;
+    /** Has this turn already produced completed tool work? */
+    deliveredWorkThisTurn?: boolean;
+}
+export interface PatiencePlan {
+    /** Schedule for server/network/timeout failures (`[]` = do not retry). */
+    transientDelaysMs: readonly number[];
+    /** Schedule for an `empty-response` (`[]` = do not retry). */
+    emptyResponseDelaysMs: readonly number[];
+    /** The strongest band reachable elsewhere, when an alternative exists. */
+    bestAlternativeBand?: CapabilityBand;
+    /**
+     * True when we are deliberately WAITING rather than folding down a band —
+     * the situation the log line exists to make visible.
+     */
+    foldingWouldDegrade: boolean;
+    /** Why this plan — printed once, so a slow turn is explainable after the fact. */
+    reason: string;
+}
+/**
+ * Decide how patient to be with the pair that just failed, and say why.
+ *
+ * The whole policy, in order (see the tests for each row):
+ *
+ *   1. The pair already failed an earlier step of this turn → NOT a blip. One
+ *      token wait, no empty patience.
+ *   2. Nothing to gain by patience: a same-or-better band is reachable → short
+ *      transient retry; empty patched only for a proven pair (a good pair's
+ *      first blink is still worth two short waits).
+ *   3. A PROVEN pair facing only degrading alternatives (or none) → long, dense
+ *      transient patience, and extended empty patience. This is the row that
+ *      stops a blinking strong model from being replaced by a weak one.
+ *   4. An UNPROVEN pair with nowhere better to go → a token wait; the walk (or
+ *      the honest failure) is the remedy, not optimism about a stranger.
+ */
+export declare function planBlipPatience(input: PatienceInput): PatiencePlan;
+//# sourceMappingURL=retry-patience.d.ts.map

package/dist/learning/retry-patience.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"retry-patience.d.ts","sourceRoot":"","sources":["../../src/learning/retry-patience.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AAEH,OAAO,EAA+B,KAAK,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAEzF;;;;GAIG;AACH,eAAO,MAAM,wBAAwB,iDAAwD,CAAC;AAE9F;;;;GAIG;AACH,eAAO,MAAM,oCAAoC,sBAAwB,CAAC;AAE1E;;;GAGG;AACH,eAAO,MAAM,+BAA+B,uBAA0B,CAAC;AAEvE,0EAA0E;AAC1E,eAAO,MAAM,iBAAiB,uBAA0B,CAAC;AAEzD;;;;GAIG;AACH,eAAO,MAAM,oCAAoC,8BAAkC,CAAC;AAEpF;;;;GAIG;AACH,eAAO,MAAM,qBAAqB,MAAM,CAAC;AAEzC,sEAAsE;AACtE,MAAM,WAAW,gBAAgB;IAC/B,kEAAkE;IAClE,QAAQ,EAAE,OAAO,CAAC;IAClB;;;;OAIG;IACH,MAAM,CAAC,EAAE,WAAW,GAAG,YAAY,GAAG,OAAO,CAAC;IAC9C,6CAA6C;IAC7C,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,aAAa;IAC5B,iCAAiC;IACjC,OAAO,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IAC7C;;;;OAIG;IACH,YAAY,EAAE,aAAa,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACjE,4EAA4E;IAC5E,QAAQ,CAAC,EAAE,gBAAgB,CAAC;IAC5B,oEAAoE;IACpE,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B,0DAA0D;IAC1D,qBAAqB,CAAC,EAAE,OAAO,CAAC;CACjC;AAED,MAAM,WAAW,YAAY;IAC3B,0EAA0E;IAC1E,iBAAiB,EAAE,SAAS,MAAM,EAAE,CAAC;IACrC,8DAA8D;IAC9D,qBAAqB,EAAE,SAAS,MAAM,EAAE,CAAC;IACzC,0EAA0E;IAC1E,mBAAmB,CAAC,EAAE,cAAc,CAAC;IACrC;;;OAGG;IACH,mBAAmB,EAAE,OAAO,CAAC;IAC7B,kFAAkF;IAClF,MAAM,EAAE,MAAM,CAAC;CAChB;AAWD;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,aAAa,GAAG,YAAY,CA6EnE"}