@antseed/node 0.2.156 → 0.2.157

package/dist/health/model-health-checker.js

@@ -1,23 +1,28 @@
+import { serviceHealthOf } from '../discovery/peer-metadata.js';
 import { isNativeVideoProtocol, isRoutingProtocol } from '../types/service-api.js';
 import { debugLog, debugWarn } from '../utils/debug.js';
 import { isLiveResponseOutOfCredit, isUpstreamOutOfCredit } from './upstream-credit.js';
-/** Default sweep interval. Each sweep sends one 1-token probe per advertised service. */
+/** Default sweep interval. Each sweep sends one 1-token probe per service without recent buyer traffic. */
 export const DEFAULT_HEALTH_CHECK_INTERVAL_MS = 5 * 60_000;
-/** Consecutive probe failures before a service is unadvertised. */
+/** Consecutive failed checks before a service is announced as failing. */
 export const DEFAULT_HEALTH_CHECK_FAILURE_THRESHOLD = 3;
 /** Per-probe timeout. A model that hangs this long is treated as failing. */
 export const DEFAULT_HEALTH_CHECK_PROBE_TIMEOUT_MS = 60_000;
+/** Shortest gap between probes triggered by failed buyer requests for one service. */
+export const LIVE_FAILURE_PROBE_MIN_GAP_MS = 30_000;
+class ProbeTimeoutError extends Error {
+}
 /**
- * Periodically probes every advertised service with a minimal (1-token)
- * completion and unadvertises services that fail repeatedly, so buyers stop
- * discovering models the seller cannot actually serve. Removed services keep
- * being probed and are re-advertised on the first healthy probe.
+ * The seller's self health check. Every service stays announced; its latest
+ * result is published on `provider.serviceHealth`, which goes into signed
+ * metadata (v13+) so buyers can hide failing services from catalogs and
+ * routing and explain why a pinned request failed.
  *
- * Removal mutates `provider.services` in place — the same array reference the
- * announcer and the seller request handler read live — so it takes effect on
- * the next `/metadata` fetch and the next incoming request. If every service
- * is removed, `provider.healthCheckAvailable` becomes false so discovery omits
- * the provider instead of treating its empty service list as a wildcard.
+ * Buyer traffic is the main evidence: a successful billable response marks a
+ * service healthy, and a service that had one within the last interval is not
+ * probed. Idle services get a minimal (1-token) completion each interval, and
+ * a failed buyer request triggers an early probe. A service is failing after
+ * `failureThreshold` failed checks in a row and recovers on the first pass.
  */
 export class ModelHealthChecker {
     _targets;
@@ -36,8 +41,8 @@ export class ModelHealthChecker {
     _onSweepComplete;
     /** Each upstream's credit as last seen; unset until something said. */
     _accounts = new Map();
-    /** Services withdrawn before `onChange` could be told; reported on the first start. */
-    _withdrawnAtInit = [];
+    /** Services marked failing before `onChange` could be told; reported on the first start. */
+    _failingAtInit = [];
     constructor(config) {
         this._targets = config.targets;
         this._intervalMs = config.intervalMs ?? DEFAULT_HEALTH_CHECK_INTERVAL_MS;
@@ -47,20 +52,23 @@ export class ModelHealthChecker {
         this._onUpstreamAccount = config.onUpstreamAccount;
         this._lowBalanceUsd = config.lowBalanceUsd;
         this._onSweepComplete = config.onSweepComplete;
-        // Providers retained after an OAuth initialization failure must pass a real
-        // probe before any service is advertised. Keep their services in recovery
-        // state rather than dropping the provider permanently.
+        // Providers retained after an OAuth initialization failure stay announced
+        // as failing until a real check passes.
+        const now = Date.now();
         for (const { provider } of this._targets) {
             if (provider.healthCheckAvailable !== false)
                 continue;
-            for (const [index, service] of provider.services.entries()) {
+            for (const service of provider.services) {
                 const state = this._stateFor(provider, service);
-                state.removed = true;
-                state.removedAtIndex = index;
-                state.lastDetail = 'Awaiting successful probe after initialization failure';
-                this._withdrawnAtInit.push(state);
+                state.status = 'failing';
+                state.reason = 'init_failed';
+                state.failingSince = now;
+                state.checkedAt = now;
+                state.source = 'probe';
+                state.lastDetail = 'Initialization failed; awaiting a successful check';
+                this._publish(state);
+                this._failingAtInit.push(state);
             }
-            provider.services.splice(0);
         }
     }
     /** Start periodic sweeps. The first sweep runs immediately (async). */
@@ -68,15 +76,16 @@ export class ModelHealthChecker {
         if (this._timer)
             return;
         this._stopped = false;
-        for (const state of this._withdrawnAtInit.splice(0)) {
-            if (!state.removed)
+        for (const state of this._failingAtInit.splice(0)) {
+            if (state.status !== 'failing')
                 continue;
             this._emitChange({
                 provider: state.providerName,
                 providerId: state.providerId,
                 service: state.service,
-                status: 'removed',
+                status: 'failing',
                 consecutiveFailures: state.consecutiveFailures,
+                reason: 'init_failed',
                 detail: state.lastDetail ?? 'initialization failed',
             });
         }
@@ -94,7 +103,7 @@ export class ModelHealthChecker {
             this._timer = null;
         }
     }
-    /** Probe every advertised (and previously removed) service once. */
+    /** Check every service once, skipping probes for services with recent buyer traffic. */
     async runSweep() {
         if (this._sweepInFlight)
             return this._sweepInFlight;
@@ -116,11 +125,16 @@ export class ModelHealthChecker {
             provider: state.providerName,
             providerId: state.providerId,
             service: state.service,
-            advertised: !state.removed,
+            status: state.status,
             consecutiveFailures: state.consecutiveFailures,
+            failingSince: state.failingSince,
+            reason: state.reason,
+            checkedAt: state.checkedAt,
+            source: state.source,
             lastProbeAt: state.lastProbeAt,
             lastStatusCode: state.lastStatusCode,
             lastDetail: state.lastDetail,
+            lastTrafficAt: state.lastTrafficAt,
         }));
     }
     async _runSweep() {
@@ -128,39 +142,29 @@ export class ModelHealthChecker {
             if (this._stopped)
                 return;
             const provider = target.provider;
-            // Don't compete with paying buyers for concurrency slots.
-            try {
-                const capacity = provider.getCapacity();
-                if (capacity.current >= capacity.max) {
-                    debugLog(`[ModelHealth] Skipping ${provider.name}: at capacity (${capacity.current}/${capacity.max})`);
-                    continue;
-                }
-            }
-            catch {
-                // getCapacity should not throw; probe anyway if it does.
-            }
-            // Probe currently advertised services plus ones we removed earlier
-            // (so they can recover). Snapshot before iterating — probes mutate
-            // the live array on state transitions.
-            const services = new Set(provider.services);
-            for (const state of this._states.values()) {
-                if (state.provider === provider && state.removed) {
-                    services.add(state.service);
-                }
-            }
+            if (this._atCapacity(provider))
+                continue;
+            const services = [...provider.services];
             const hasFreeCheck = (candidate) => Boolean(candidate?.checkServiceHealth || candidate?.checkUpstreamAccount);
             const checkProvider = hasFreeCheck(target.probeProvider) ? target.probeProvider : provider;
+            // Free checks also read the account balance, so they run even with traffic.
             const checked = hasFreeCheck(checkProvider)
-                ? [...services].filter((service) => !supportsHealthProbe(resolveProbeProtocol(provider, service)))
+                ? services.filter((service) => !supportsHealthProbe(resolveProbeProtocol(provider, service)))
                 : [];
             if (checked.length > 0) {
                 await this._checkServices(target, checkProvider, checked);
             }
+            const now = Date.now();
             for (const service of services) {
                 if (this._stopped)
                     return;
                 if (checked.includes(service))
                     continue;
+                const state = this._stateFor(provider, service);
+                if (state.lastTrafficAt !== null && now - state.lastTrafficAt < this._intervalMs) {
+                    debugLog(`[ModelHealth] ${provider.name}/${service}: skipping probe, served a buyer ${Math.round((now - state.lastTrafficAt) / 1000)}s ago`);
+                    continue;
+                }
                 await this._probeService(target, service);
             }
         }
@@ -170,12 +174,13 @@ export class ModelHealthChecker {
      * upstream account check (key valid, account funded) once, and the model
      * check (model still offered) for every service. A service is unavailable
      * when either check says so, and available only when no check is unsure.
-     * `unavailable` counts as a failed check, `available` re-advertises,
+     * `unavailable` counts as a failed check, `available` as a pass,
      * `unknown` changes nothing.
      */
     async _checkServices(target, checkProvider, services) {
         const provider = target.provider;
         const timed = (label, check) => withTimeout(Promise.resolve().then(check), this._probeTimeoutMs, `${label} for ${provider.name} timed out after ${this._probeTimeoutMs}ms`);
+        const checkStartedAt = Date.now();
         const [account, models] = await Promise.all([
             checkProvider.checkUpstreamAccount
                 ? timed('account check', () => checkProvider.checkUpstreamAccount())
@@ -186,7 +191,8 @@ export class ModelHealthChecker {
                 : undefined,
         ]);
         const balance = account?.status === 'available' ? account.balanceUsd : undefined;
-        if (account?.status === 'unavailable' && (account.outOfCredit === true || isUpstreamOutOfCredit(null, account.reason))) {
+        const accountOutOfCredit = account?.status === 'unavailable' && (account.outOfCredit === true || isUpstreamOutOfCredit(null, account.reason));
+        if (accountOutOfCredit) {
             this._setAccount(provider, 'out', account.reason, null);
         }
         else if (account?.status === 'available') {
@@ -204,15 +210,17 @@ export class ModelHealthChecker {
             const model = models === undefined
                 ? undefined
                 : models?.[service] ?? { status: 'unknown', reason: 'model check gave no result' };
-            const results = [account, model].filter((result) => result !== undefined);
-            return results.find((result) => result.status === 'unavailable')
-                ?? results.find((result) => result.status === 'unknown')
-                ?? { status: 'available', balanceUsd: balance };
+            if (account?.status === 'unavailable')
+                return { result: account, reason: accountOutOfCredit ? 'out_of_credit' : reasonForAccount(account.reason) };
+            if (model?.status === 'unavailable')
+                return { result: model, reason: 'model_unavailable' };
+            const unsure = [account, model].find((result) => result?.status === 'unknown');
+            return { result: unsure ?? { status: 'available', balanceUsd: balance }, reason: 'upstream_error' };
         };
         for (const service of services) {
             if (this._stopped)
                 return;
-            const result = combine(service);
+            const { result, reason } = combine(service);
             const state = this._stateFor(provider, service);
             if (result.status === 'unknown') {
                 debugLog(`[ModelHealth] ${provider.name}/${service}: inconclusive (${result.reason ?? 'no result'})`);
@@ -222,14 +230,10 @@ export class ModelHealthChecker {
             state.lastStatusCode = null;
             if (result.status === 'unavailable') {
                 state.lastDetail = result.reason;
-                this._recordFailure(provider, state);
+                this._recordFailure(state, reason, null, checkStartedAt);
                 continue;
             }
-            state.consecutiveFailures = 0;
-            if (!state.removed)
-                continue;
-            state.lastDetail = balance === undefined ? 'Account check passed' : `Account check passed (balance $${balance.toFixed(2)})`;
-            this._restoreService(provider, state);
+            this._recordSuccess(state, 'probe', balance === undefined ? 'Account check passed' : `Account check passed (balance $${balance.toFixed(2)})`);
         }
     }
     _stateFor(provider, service) {
@@ -246,12 +250,18 @@ export class ModelHealthChecker {
                 providerName: provider.name,
                 providerId: this._targets.find((target) => target.provider === provider)?.id ?? provider.name,
                 service,
+                status: 'unknown',
                 consecutiveFailures: 0,
-                removed: false,
-                removedAtIndex: 0,
+                failingSince: null,
+                reason: null,
+                httpStatus: null,
+                checkedAt: null,
+                source: null,
                 lastProbeAt: null,
                 lastStatusCode: null,
                 lastDetail: null,
+                lastTrafficAt: null,
+                probeInFlight: false,
             };
             this._states.set(key, state);
         }
@@ -261,8 +271,13 @@ export class ModelHealthChecker {
         const provider = target.provider;
         const probeProvider = target.probeProvider ?? provider;
         const state = this._stateFor(provider, service);
-        state.lastProbeAt = Date.now();
+        if (state.probeInFlight)
+            return;
+        state.probeInFlight = true;
+        const probeStartedAt = Date.now();
+        state.lastProbeAt = probeStartedAt;
         let outcome;
+        let reason;
         let errorText = null;
         try {
             const protocol = resolveProbeProtocol(provider, service);
@@ -276,6 +291,7 @@ export class ModelHealthChecker {
             const response = await withTimeout(probeProvider.handleRequest(request), this._probeTimeoutMs, `health probe for ${service} timed out after ${this._probeTimeoutMs}ms`);
             state.lastStatusCode = response.statusCode;
             outcome = classifyProbeStatus(response.statusCode);
+            reason = reasonForStatus(response.statusCode);
             // Classified on the body itself, not the 200-byte summary: a billing code can follow a long message.
             if (outcome !== 'healthy')
                 errorText = decodeBody(response.body, 16_384);
@@ -288,17 +304,22 @@ export class ModelHealthChecker {
             state.lastDetail = err instanceof Error ? err.message : String(err);
             errorText = state.lastDetail;
             outcome = 'unhealthy';
+            reason = err instanceof ProbeTimeoutError ? 'timeout' : 'unreachable';
+        }
+        finally {
+            state.probeInFlight = false;
         }
         // A billing error is a failure whatever its status (Anthropic answers one with a 400);
         // a 404 or 422 is about the probe itself, so its text never marks the account.
         if (outcome !== 'healthy' && state.lastStatusCode !== 404 && state.lastStatusCode !== 422 && isUpstreamOutOfCredit(state.lastStatusCode, errorText)) {
             outcome = 'unhealthy';
+            reason = 'out_of_credit';
             this._setAccount(provider, 'out', state.lastDetail ?? 'out of credit', null);
         }
         if (outcome === 'inconclusive') {
             // The endpoint answered but the probe result proves nothing either way
-            // (for example, 400 for an unsupported probe parameter). Leave failure
-            // counters and advertisement untouched.
+            // (for example, 400 for an unsupported probe parameter). Leave the
+            // service's health untouched.
             debugLog(`[ModelHealth] ${provider.name}/${service}: inconclusive (${state.lastDetail})`);
             return;
         }
@@ -306,22 +327,22 @@ export class ModelHealthChecker {
             // An answered completion means the account pays again (its balance is unknown here).
             if (this._accounts.get(provider) === 'out')
                 this._setAccount(provider, 'ok', state.lastDetail ?? 'probe succeeded', null);
-            state.consecutiveFailures = 0;
-            if (state.removed) {
-                this._restoreService(provider, state);
-            }
+            this._recordSuccess(state, 'probe', state.lastDetail ?? 'probe succeeded');
             return;
         }
-        this._recordFailure(provider, state);
+        this._recordFailure(state, reason, state.lastStatusCode, probeStartedAt);
     }
     /**
-     * A live request's upstream answer (the seller's own traffic): an out-of-credit
-     * error is reported at once instead of waiting for the probes, and a billable
-     * success after one means the account pays again (polling or downloading a
-     * video already paid for is not billable and proves nothing).
+     * A live request's upstream answer (the seller's own traffic). A billable
+     * success marks `service` healthy and spares it the next probe. A failure
+     * the upstream is responsible for (auth, limits, 5xx) triggers an early
+     * probe instead of counting directly, so one buyer's odd request cannot mark
+     * a service failing. An out-of-credit error is reported at once, and a
+     * billable success after one means the account pays again (polling or
+     * downloading a video already paid for is not billable and proves nothing).
      */
-    reportUpstreamResponse(provider, statusCode, body, billable = true) {
-        const target = this._targets.find((entry) => entry.provider === provider || entry.probeProvider === provider);
+    reportUpstreamResponse(provider, statusCode, body, billable = true, service) {
+        const target = this._findTarget(provider);
         if (!target || this._stopped)
             return;
         if (statusCode >= 400 && isLiveResponseOutOfCredit(statusCode, body)) {
@@ -330,6 +351,66 @@ export class ModelHealthChecker {
         else if (statusCode < 400 && billable && this._accounts.get(target.provider) === 'out') {
             this._setAccount(target.provider, 'ok', `HTTP ${statusCode} on a request`, null);
         }
+        if (!service || !target.provider.services.includes(service))
+            return;
+        if (statusCode < 400) {
+            if (!billable)
+                return;
+            const state = this._stateFor(target.provider, service);
+            state.lastTrafficAt = Date.now();
+            state.lastStatusCode = statusCode;
+            this._recordSuccess(state, 'traffic', `HTTP ${statusCode} on a buyer request`);
+            return;
+        }
+        if (statusCode !== 404 && classifyProbeStatus(statusCode) === 'unhealthy') {
+            this._probeSoon(target, service);
+        }
+    }
+    /** A live request threw before the upstream answered (network error, timeout). */
+    reportUpstreamFailure(provider, service) {
+        const target = this._findTarget(provider);
+        if (!target || this._stopped || !target.provider.services.includes(service))
+            return;
+        this._probeSoon(target, service);
+    }
+    /** The announced health of `service`, if it has been checked. */
+    getServiceHealth(provider, service) {
+        const target = this._findTarget(provider);
+        return serviceHealthOf(target?.provider.serviceHealth, service);
+    }
+    _findTarget(provider) {
+        return this._targets.find((entry) => entry.provider === provider || entry.probeProvider === provider);
+    }
+    _probeSoon(target, service) {
+        // Image and video services are never probed (a probe would cost a real generation).
+        if (!supportsHealthProbe(resolveProbeProtocol(target.provider, service)))
+            return;
+        const state = this._stateFor(target.provider, service);
+        if (state.probeInFlight)
+            return;
+        if (state.lastProbeAt !== null && Date.now() - state.lastProbeAt < LIVE_FAILURE_PROBE_MIN_GAP_MS)
+            return;
+        // A full provider answers its own 429, which says nothing about the upstream.
+        if (this._atCapacity(target.provider))
+            return;
+        debugLog(`[ModelHealth] ${target.provider.name}/${service}: buyer request failed upstream, probing now`);
+        void this._probeService(target, service).catch((err) => {
+            debugWarn(`[ModelHealth] early probe failed: ${err instanceof Error ? err.message : err}`);
+        });
+    }
+    /** Probes never compete with paying buyers for concurrency slots. */
+    _atCapacity(provider) {
+        try {
+            const capacity = provider.getCapacity();
+            if (capacity.current < capacity.max)
+                return false;
+            debugLog(`[ModelHealth] Skipping ${provider.name}: at capacity (${capacity.current}/${capacity.max})`);
+            return true;
+        }
+        catch {
+            // getCapacity should not throw; probe anyway if it does.
+            return false;
+        }
     }
     _lowLine() {
         const value = typeof this._lowBalanceUsd === 'function' ? this._lowBalanceUsd() : this._lowBalanceUsd;
@@ -344,14 +425,13 @@ export class ModelHealthChecker {
         if (next === 'ok' && previous === undefined)
             return;
         const target = this._targets.find((entry) => entry.provider === provider);
-        const services = [...new Set([...provider.services, ...[...this._states.values()].filter((state) => state.provider === provider).map((state) => state.service)])];
         const event = {
             provider: provider.name,
             providerId: target?.id ?? provider.name,
             status: next === 'out' ? 'out_of_credit' : next === 'low' ? 'low_balance' : 'funded',
             detail,
             balanceUsd,
-            services,
+            services: [...provider.services],
         };
         debugWarn(`[ModelHealth] ${event.providerId}: upstream ${event.status} (${detail})`);
         if (!this._onUpstreamAccount)
@@ -366,59 +446,88 @@ export class ModelHealthChecker {
             debugWarn(`[ModelHealth] onUpstreamAccount failed: ${err instanceof Error ? err.message : err}`);
         }
     }
-    _recordFailure(provider, state) {
-        state.consecutiveFailures += 1;
-        debugWarn(`[ModelHealth] ${provider.name}/${state.service}: health check failed `
-            + `(${state.consecutiveFailures}/${this._failureThreshold}) — ${state.lastDetail}`);
-        if (!state.removed && state.consecutiveFailures >= this._failureThreshold) {
-            this._removeService(provider, state);
-        }
-    }
-    _removeService(provider, state) {
-        const services = provider.services;
-        const index = services.indexOf(state.service);
-        if (index < 0) {
-            // Already gone (removed externally) — track it as removed so it can recover.
-            state.removed = true;
-            if (services.length === 0) {
-                provider.healthCheckAvailable = false;
-            }
+    _recordSuccess(state, source, detail) {
+        const wasFailing = state.status === 'failing';
+        state.status = 'healthy';
+        state.consecutiveFailures = 0;
+        state.failingSince = null;
+        state.reason = null;
+        state.httpStatus = null;
+        state.checkedAt = Date.now();
+        state.source = source;
+        state.lastDetail = detail;
+        if (state.provider.healthCheckAvailable === false)
+            state.provider.healthCheckAvailable = true;
+        this._publish(state);
+        if (!wasFailing)
             return;
-        }
-        services.splice(index, 1);
-        state.removed = true;
-        state.removedAtIndex = index;
-        if (services.length === 0) {
-            provider.healthCheckAvailable = false;
-        }
         this._emitChange({
-            provider: provider.name,
+            provider: state.providerName,
             providerId: state.providerId,
             service: state.service,
-            status: 'removed',
-            consecutiveFailures: state.consecutiveFailures,
-            detail: state.lastDetail ?? 'probe failed',
+            status: 'recovered',
+            consecutiveFailures: 0,
+            detail,
         });
     }
-    _restoreService(provider, state) {
-        const services = provider.services;
-        state.removed = false;
-        if (!services.includes(state.service)) {
-            services.splice(Math.min(state.removedAtIndex, services.length), 0, state.service);
+    /** A failed check started at `checkStartedAt`; a buyer success since then outweighs it. */
+    _recordFailure(state, reason, httpStatus, checkStartedAt) {
+        if (state.lastTrafficAt !== null && state.lastTrafficAt >= checkStartedAt) {
+            debugLog(`[ModelHealth] ${state.providerName}/${state.service}: check failed, but a buyer request succeeded meanwhile`);
+            return;
         }
-        provider.healthCheckAvailable = true;
+        const now = Date.now();
+        state.consecutiveFailures += 1;
+        state.reason = reason;
+        state.httpStatus = httpStatus;
+        state.failingSince ??= now;
+        debugWarn(`[ModelHealth] ${state.providerName}/${state.service}: health check failed `
+            + `(${state.consecutiveFailures}/${this._failureThreshold}, ${reason}) — ${state.lastDetail}`);
+        const crossed = state.status !== 'failing' && state.consecutiveFailures >= this._failureThreshold;
+        if (crossed)
+            state.status = 'failing';
+        // Below the threshold the announced status and its evidence stand; only
+        // a failing status is backed by this check.
+        if (state.status !== 'failing')
+            return;
+        state.checkedAt = now;
+        state.source = 'probe';
+        this._publish(state);
+        if (!crossed)
+            return;
         this._emitChange({
-            provider: provider.name,
+            provider: state.providerName,
             providerId: state.providerId,
             service: state.service,
-            status: 'restored',
-            consecutiveFailures: 0,
-            detail: state.lastDetail ?? 'probe succeeded',
+            status: 'failing',
+            consecutiveFailures: state.consecutiveFailures,
+            reason,
+            detail: state.lastDetail ?? 'check failed',
         });
     }
+    /** Writes the announced form of `state` onto the provider. */
+    _publish(state) {
+        const health = { status: state.status };
+        if (state.checkedAt !== null)
+            health.checkedAt = state.checkedAt;
+        if (state.source !== null)
+            health.source = state.source;
+        if (state.status === 'failing') {
+            if (state.failingSince !== null)
+                health.failingSince = state.failingSince;
+            if (state.consecutiveFailures > 0)
+                health.consecutiveFailures = Math.min(0xffff, state.consecutiveFailures);
+            if (state.reason !== null)
+                health.reason = state.reason;
+            if (state.httpStatus !== null && state.httpStatus >= 100 && state.httpStatus <= 599)
+                health.httpStatus = state.httpStatus;
+        }
+        // An own entry even for a service named like an Object.prototype property ("__proto__").
+        state.provider.serviceHealth ??= Object.create(null);
+        Object.defineProperty(state.provider.serviceHealth, state.service, { value: health, enumerable: true, writable: true, configurable: true });
+    }
     _emitChange(event) {
-        const action = event.status === 'removed' ? 'unadvertised' : 're-advertised';
-        debugLog(`[ModelHealth] ${event.provider}/${event.service} ${action} (${event.detail})`);
+        debugLog(`[ModelHealth] ${event.provider}/${event.service} ${event.status === 'failing' ? 'marked failing' : 'recovered'} (${event.detail})`);
         if (!this._onChange)
             return;
         try {
@@ -434,6 +543,26 @@ export class ModelHealthChecker {
         }
     }
 }
+/** The announced reason for an account check that says the upstream cannot serve. */
+function reasonForAccount(reason) {
+    if (/balance|credit|fund|top.?up|quota/i.test(reason))
+        return 'out_of_credit';
+    if (/key|auth|credential|forbidden|permission|token/i.test(reason))
+        return 'auth';
+    return 'upstream_error';
+}
+/** The announced reason for a failed check's upstream status. */
+function reasonForStatus(statusCode) {
+    if (statusCode === 429)
+        return 'rate_limited';
+    if (statusCode === 402)
+        return 'out_of_credit';
+    if (statusCode === 401 || statusCode === 403)
+        return 'auth';
+    if (statusCode === 404)
+        return 'model_unavailable';
+    return 'upstream_error';
+}
 /** Pick the probe request shape from the service's advertised API protocol. */
 function resolveProbeProtocol(provider, service) {
     return provider.serviceApiProtocols?.[service]?.[0] ?? 'openai-chat-completions';
@@ -507,7 +636,7 @@ export function buildHealthProbeRequest(service, protocol) {
  *   collapses upstream network errors/timeouts to 502.)
  * - Everything else (400, 422, ...) — the endpoint is alive but the probe
  *   itself was rejected (for example, an unsupported parameter): inconclusive,
- *   so a working model is never unadvertised over a probe quirk.
+ *   so a working model is never marked failing over a probe quirk.
  */
 export function classifyProbeStatus(statusCode) {
     if (statusCode >= 200 && statusCode < 400)
@@ -538,7 +667,7 @@ function summarizeBody(body) {
 async function withTimeout(promise, timeoutMs, message) {
     let timer;
     const timeout = new Promise((_, reject) => {
-        timer = setTimeout(() => reject(new Error(message)), timeoutMs);
+        timer = setTimeout(() => reject(new ProbeTimeoutError(message)), timeoutMs);
         timer.unref?.();
     });
     try {