nexus-ai-pro 2.0.0 → 2.2.0

package/dist/operations/runner.js

@@ -93,6 +93,7 @@ export class OperationRunner {
             expiresAt: options.expiresAt,
             idempotencyKey: options.idempotencyKey,
             kind: options.kind,
+            ...(options.tenantId ? { tenantId: options.tenantId } : {}),
             traceContext: options.traceContext,
             metadata: options.metadata,
         };
@@ -214,7 +215,7 @@ export class OperationRunner {
             const heartbeat = this.startHeartbeat(record, handle);
             let outcome;
             try {
-                const value = await executor(this.buildContext(record, handle));
+                const value = await executor(this.buildContext(record, handle, heartbeat.beat));
                 outcome = { ok: true, value };
             }
             catch (error) {
@@ -290,7 +291,13 @@ export class OperationRunner {
     startHeartbeat(record, handle) {
         let latest = record;
         let stopped = false;
-        const beat = async () => {
+        let inFlight = Promise.resolve();
+        const beat = (details) => {
+            const next = inFlight.then(() => renew(details));
+            inFlight = next.catch(() => undefined);
+            return next;
+        };
+        const renew = async (details) => {
             if (stopped)
                 return;
             const current = await this.store.read(latest.id);
@@ -315,6 +322,8 @@ export class OperationRunner {
                 heartbeatAt: renewed.updatedAt,
             };
             renewed.progress = handle.progress() ?? current.progress;
+            if (details !== undefined)
+                renewed.heartbeatDetails = details;
             if (await this.store.update(renewed, current.sequence))
                 latest = renewed;
         };
@@ -335,24 +344,14 @@ export class OperationRunner {
             beat,
         };
     }
-    buildContext(record, handle) {
+    buildContext(record, handle, beat) {
         return {
             operationId: record.id,
             attempt: record.attempt,
             signal: handle.signal,
             report: (progress) => handle.report(progress),
-            heartbeat: async () => {
-                const current = await this.store.read(record.id);
-                if (!current || isTerminalOperationStatus(current.status))
-                    return;
-                const renewed = this.advance(current, 'running');
-                renewed.lease = {
-                    owner: this.owner,
-                    expiresAt: new Date(this.now().getTime() + this.leaseMs).toISOString(),
-                    heartbeatAt: renewed.updatedAt,
-                };
-                await this.store.update(renewed, current.sequence);
-            },
+            heartbeat: (details) => beat(details),
+            ...(record.heartbeatDetails === undefined ? {} : { previousHeartbeat: record.heartbeatDetails }),
             traceContext: record.traceContext,
             metadata: record.metadata,
         };

package/dist/operations/stats.js

@@ -3,6 +3,8 @@ export function isUnheldQueued(record, now) {
     return record.status === 'queued' && (!record.lease || record.lease.expiresAt <= now);
 }
 export function matchesFilter(record, filter) {
+    if (filter?.tenantId !== undefined && record.tenantId !== filter.tenantId)
+        return false;
     return !filter?.kindPrefix || (record.kind ?? '').startsWith(filter.kindPrefix);
 }
 export function countRecords(records, now, filter) {
@@ -23,7 +25,12 @@ export function countRecords(records, now, filter) {
 }
 export async function operationStats(store, options = {}) {
     const now = (options.now ?? new Date()).toISOString();
-    const filter = options.kindPrefix ? { kindPrefix: options.kindPrefix } : undefined;
+    const filter = options.kindPrefix || options.tenantId !== undefined
+        ? {
+            ...(options.kindPrefix ? { kindPrefix: options.kindPrefix } : {}),
+            ...(options.tenantId !== undefined ? { tenantId: options.tenantId } : {}),
+        }
+        : undefined;
     if (store.stats)
         return store.stats(now, filter);
     if (store.list)

package/dist/operations/store.d.ts

@@ -19,7 +19,11 @@ export declare class MemoryOperationStore<TResult = unknown> implements Operatio
     private readonly byIdempotencyKey;
     private readonly maxRecords;
     constructor(options?: MemoryOperationStoreOptions);
-    /** Stores a new record. Refuses records carrying raw bytes. */
+    /**
+     * Stores a new record. Throws `OperationDuplicateError` when another record already holds its
+     * idempotency key, as every shared store does, so two submissions racing under one key cannot both
+     * start. Refuses records carrying raw bytes.
+     */
     create(record: OperationRecord<TResult>): void;
     /** Reads a record. */
     read(id: string): OperationRecord<TResult> | undefined;

package/dist/operations/store.js

@@ -1,4 +1,5 @@
 import { isTerminalOperationStatus } from '../types/operations.js';
+import { OperationDuplicateError } from './errors.js';
 import { assertSerializableRecord } from './serialization.js';
 import { countRecords, isUnheldQueued, matchesFilter } from './stats.js';
 export { assertSerializableRecord };
@@ -12,6 +13,12 @@ export class MemoryOperationStore {
     }
     create(record) {
         assertSerializableRecord(record);
+        if (record.idempotencyKey) {
+            const owner = this.byIdempotencyKey.get(record.idempotencyKey);
+            if (owner !== undefined && owner !== record.id && this.records.has(owner)) {
+                throw new OperationDuplicateError(record.idempotencyKey);
+            }
+        }
         this.records.set(record.id, clone(record));
         if (record.idempotencyKey)
             this.byIdempotencyKey.set(record.idempotencyKey, record.id);

package/dist/ops/circuit-breaker.d.ts

@@ -41,6 +41,15 @@ export interface CircuitBreakerConfig {
      * `nexus-ai-pro/ops/circuit-store` and `nexus-ai-pro/postgres/circuits`.
      */
     store?: CircuitStateStore;
+    /**
+     * Counts every worker's calls together, so nine failures spread over three workers count as nine.
+     * Each worker adds the calls it saw to a rolling window in the `store` when it syncs, and opens the
+     * circuit everywhere once the shared count crosses `failureThreshold` (failures since any worker's
+     * last success) or `failureRateThreshold`. Checks stay synchronous: a worker learns the shared count
+     * on its next sync, at most `syncIntervalMs` late. Needs a store with `observe()`; every bundled one
+     * has it. Off by default, when each worker counts only its own calls.
+     */
+    shareObservations?: boolean;
     /** Names this worker in shared state. Defaults to a random id. */
     workerId?: string;
     /** Shared state is refreshed at most this often, on the next check that needs it. Defaults to 1 second. */
@@ -85,6 +94,41 @@ export interface CircuitSnapshot {
     retryAt?: string;
     /** The most recent error, as text. */
     lastError?: string;
+    /** Every worker's calls together, as of this worker's last sync, with `shareObservations`. */
+    shared?: CircuitWindow;
+}
+/** Calls one worker saw since it last reported, which `CircuitStateStore.observe()` adds to the shared window. */
+export interface CircuitObservation {
+    /** Calls and failures per window bucket, by the bucket's start in epoch milliseconds. */
+    buckets: Array<{
+        at: number;
+        calls: number;
+        failures: number;
+    }>;
+    /** Whether any of the calls succeeded, which ends the shared failure streak before `trailingFailures` is added. */
+    succeeded: boolean;
+    /** Failures after the last success, or every failure when none succeeded. */
+    trailingFailures: number;
+    /** The rolling window, in milliseconds. */
+    windowMs: number;
+    /** The width of one bucket. A bucket that ended before `now - windowMs` is dropped. */
+    bucketMs: number;
+    /** Epoch milliseconds now, by the breaker's clock. */
+    now: number;
+    /**
+     * Epoch milliseconds of the circuit's last close. Buckets that started before it are not counted, so
+     * failures that opened a circuit do not reopen it the moment a probe closes it.
+     */
+    since?: number;
+}
+/** A provider's calls across every worker, inside the rolling window. */
+export interface CircuitWindow {
+    /** Calls counted. */
+    calls: number;
+    /** Of those, failures. */
+    failures: number;
+    /** Failures since any worker last reported a success. */
+    consecutiveFailures: number;
 }
 /** A circuit decision as every worker sees it. */
 export interface SharedCircuitState {
@@ -119,6 +163,12 @@ export interface CircuitStateStore {
      * recovering provider is probed once rather than by every replica at the same moment.
      */
     claimProbe(providerName: string, owner: string, ttlMs: number): Promise<boolean> | boolean;
+    /**
+     * Adds one worker's calls to a provider's shared window, and returns the window across every
+     * worker. Optional; `shareObservations` needs it. Each call must add atomically, so two workers
+     * reporting at once both count.
+     */
+    observe?(providerName: string, observation: CircuitObservation): Promise<CircuitWindow> | CircuitWindow;
 }
 /** @internal A circuit's local state. Shared with the coordinator module, never exported publicly. */
 export interface CircuitEntry {
@@ -139,6 +189,8 @@ export interface CircuitEntry {
     probeClaimed: boolean;
     /** Why the circuit last changed state. */
     reason?: string;
+    /** The shared window from the last sync, with `shareObservations`. */
+    shared?: CircuitWindow;
 }
 /**
  * Trips routing away from a provider that is failing.

package/dist/ops/circuit-breaker.js

@@ -69,6 +69,7 @@ export class CircuitBreaker {
         this.refreshState(providerName, entry);
         this.pushWindow(entry, false);
         entry.consecutiveFailures = 0;
+        this.coordinator?.observed(providerName, false);
         if (entry.state === 'half-open') {
             entry.halfOpenInFlight = Math.max(0, entry.halfOpenInFlight - 1);
             entry.halfOpenSuccesses += 1;
@@ -90,6 +91,7 @@ export class CircuitBreaker {
         }
         this.pushWindow(entry, true);
         entry.consecutiveFailures += 1;
+        this.coordinator?.observed(providerName, true);
         entry.lastError = error instanceof Error ? error.message : error === undefined ? undefined : String(error);
         if (entry.state === 'half-open') {
             entry.halfOpenInFlight = Math.max(0, entry.halfOpenInFlight - 1);
@@ -126,6 +128,7 @@ export class CircuitBreaker {
                     ? new Date(entry.openedAt + resetTimeoutMs).toISOString()
                     : undefined,
                 lastError: entry.lastError,
+                ...(entry.shared ? { shared: { ...entry.shared } } : {}),
             };
         });
     }
@@ -176,6 +179,7 @@ export class CircuitBreaker {
             entry.openedAt = undefined;
             entry.consecutiveFailures = 0;
             entry.window = [];
+            entry.shared = undefined;
             entry.halfOpenInFlight = 0;
             entry.halfOpenSuccesses = 0;
             entry.probeClaimed = false;

package/dist/ops/circuit-store.d.ts

@@ -1,5 +1,5 @@
-import type { CircuitStateStore, SharedCircuitState } from './circuit-breaker.js';
-export type { CircuitStateStore, SharedCircuitState } from './circuit-breaker.js';
+import type { CircuitObservation, CircuitStateStore, CircuitWindow, SharedCircuitState } from './circuit-breaker.js';
+export type { CircuitObservation, CircuitStateStore, CircuitWindow, SharedCircuitState, } from './circuit-breaker.js';
 /**
  * Shared circuit state in one process.
  *
@@ -10,7 +10,10 @@ export declare class MemoryCircuitStateStore implements CircuitStateStore {
     private readonly now;
     private readonly states;
     private readonly probes;
+    private readonly windows;
     constructor(now?: () => number);
+    /** Adds one worker's calls to a provider's window and returns the window across every worker. */
+    observe(providerName: string, observation: CircuitObservation): CircuitWindow;
     /** Every provider's shared circuit state. */
     read(): SharedCircuitState[];
     /**
@@ -66,7 +69,15 @@ export declare class RedisCircuitStateStore implements CircuitStateStore {
     private readonly client;
     private readonly prefix;
     private readonly useEval;
+    /**
+     * Adds one worker's calls to a provider's shared window in one Lua call, and returns the window
+     * across every worker. Present only when the client has `eval`, since the add and the sum must
+     * happen together; without it, `shareObservations` has nothing to call and each worker counts
+     * its own calls.
+     */
+    readonly observe?: (providerName: string, observation: CircuitObservation) => Promise<CircuitWindow>;
     constructor(client: RedisCircuitLikeClient, options?: RedisCircuitStateStoreOptions);
+    private observeWithEval;
     /** Every provider's shared circuit state. Unreadable entries are skipped. */
     read(): Promise<SharedCircuitState[]>;
     /**

package/dist/ops/circuit-store.js

@@ -2,9 +2,37 @@ export class MemoryCircuitStateStore {
     now;
     states = new Map();
     probes = new Map();
+    windows = new Map();
     constructor(now = () => Date.now()) {
         this.now = now;
     }
+    observe(providerName, observation) {
+        let window = this.windows.get(providerName);
+        if (!window) {
+            window = { buckets: new Map(), streak: 0 };
+            this.windows.set(providerName, window);
+        }
+        for (const bucket of observation.buckets) {
+            const stored = window.buckets.get(bucket.at) ?? { calls: 0, failures: 0 };
+            stored.calls += bucket.calls;
+            stored.failures += bucket.failures;
+            window.buckets.set(bucket.at, stored);
+        }
+        window.streak = observation.succeeded ? observation.trailingFailures : window.streak + observation.trailingFailures;
+        const keepAfter = observation.now - observation.windowMs - observation.bucketMs;
+        const since = observation.since ?? Number.NEGATIVE_INFINITY;
+        let calls = 0;
+        let failures = 0;
+        for (const [at, bucket] of window.buckets) {
+            if (at <= keepAfter)
+                window.buckets.delete(at);
+            else if (at >= since) {
+                calls += bucket.calls;
+                failures += bucket.failures;
+            }
+        }
+        return { calls, failures, consecutiveFailures: window.streak };
+    }
     read() {
         return [...this.states.values()].map((state) => ({ ...state }));
     }
@@ -34,14 +62,54 @@ redis.call('HSET', KEYS[1], ARGV[1], ARGV[2])
 redis.call('DEL', KEYS[2])
 return 1
 `;
+const OBSERVE_SCRIPT = `
+local key = KEYS[1]
+for i = 6, #ARGV, 3 do
+  redis.call('HINCRBY', key, 'c:' .. ARGV[i], ARGV[i + 1])
+  redis.call('HINCRBY', key, 'f:' .. ARGV[i], ARGV[i + 2])
+end
+local streak
+if ARGV[1] == '1' then
+  redis.call('HSET', key, 'streak', ARGV[2])
+  streak = tonumber(ARGV[2])
+else
+  streak = tonumber(redis.call('HINCRBY', key, 'streak', ARGV[2]))
+end
+local keepAfter = tonumber(ARGV[3])
+local since = tonumber(ARGV[4])
+local calls, failures = 0, 0
+local fields = redis.call('HGETALL', key)
+for i = 1, #fields, 2 do
+  local name = fields[i]
+  local kind = string.sub(name, 1, 2)
+  if kind == 'c:' or kind == 'f:' then
+    local at = tonumber(string.sub(name, 3))
+    if at <= keepAfter then
+      redis.call('HDEL', key, name)
+    elseif at >= since then
+      if kind == 'c:' then calls = calls + tonumber(fields[i + 1]) else failures = failures + tonumber(fields[i + 1]) end
+    end
+  end
+end
+redis.call('PEXPIRE', key, ARGV[5])
+return { calls, failures, streak }
+`;
 export class RedisCircuitStateStore {
     client;
     prefix;
     useEval;
+    observe;
     constructor(client, options = {}) {
         this.client = client;
         this.prefix = options.prefix ?? 'nexus-ai-pro:circuits:';
         this.useEval = options.useEval !== false && typeof client.eval === 'function';
+        if (this.useEval)
+            this.observe = (providerName, observation) => this.observeWithEval(providerName, observation);
+    }
+    async observeWithEval(providerName, observation) {
+        const reply = (await this.client.eval(OBSERVE_SCRIPT, 1, `${this.prefix}window:${providerName}`, observation.succeeded ? '1' : '0', String(observation.trailingFailures), String(observation.now - observation.windowMs - observation.bucketMs), String(observation.since ?? -1), String(observation.windowMs * 2 + observation.bucketMs), ...observation.buckets.flatMap((bucket) => [String(bucket.at), String(bucket.calls), String(bucket.failures)])));
+        const [calls, failures, streak] = (reply ?? []).map(Number);
+        return { calls: calls ?? 0, failures: failures ?? 0, consecutiveFailures: streak ?? 0 };
     }
     async read() {
         const all = (await this.client.hgetall(this.statesKey())) ?? {};

package/dist/ops/circuit-sync.d.ts

@@ -26,6 +26,8 @@ export declare class CircuitCoordinator {
     private lastSyncAt;
     /** False after a store failure, until a sync succeeds; the breaker then decides locally. */
     private reachable;
+    /** Calls not yet added to the shared window, per provider, with `shareObservations`. */
+    private readonly unreported;
     constructor(host: CircuitHost);
     /**
      * Whether this worker may go half-open once a cooldown has elapsed. With a reachable store only
@@ -35,6 +37,11 @@ export declare class CircuitCoordinator {
     /** Starts a background sync when the last one is older than the interval. Never waits. */
     tick(): void;
     sync(): Promise<void>;
+    /**
+     * Counts one call toward the shared window, to be added on the next sync. Cheap and synchronous:
+     * a check never waits for the store.
+     */
+    observed(providerName: string, failed: boolean): void;
     /** Publishes a transition this worker made. */
     published(providerName: string, entry: CircuitEntry, to: 'open' | 'closed', reason: string): void;
     /** Publishes an operator reset, so every worker closes the circuit rather than only this one. */
@@ -45,6 +52,14 @@ export declare class CircuitCoordinator {
      */
     reconcile(): void;
     flush(): Promise<void>;
+    /**
+     * Adds this worker's unreported calls to each provider's shared window, then opens a closed circuit
+     * whose shared count crossed a threshold. The batch is taken before the first await, so calls made
+     * while it is in flight wait for the next sync rather than being lost or counted twice.
+     */
+    private report;
+    /** Opens a closed circuit whose shared window crossed the breaker's thresholds. */
+    private judge;
     /** Takes a decision another worker published, when it is newer than what this worker knows. */
     private adopt;
     /**
@@ -55,3 +70,5 @@ export declare class CircuitCoordinator {
     private publish;
     private failed;
 }
+/** The width of one shared-window bucket: a twelfth of the window, so a bucket rolling out moves the count by little. */
+export declare function bucketWidth(windowMs: number): number;

package/dist/ops/circuit-sync.js

@@ -4,6 +4,7 @@ export class CircuitCoordinator {
     syncing;
     lastSyncAt = Number.NEGATIVE_INFINITY;
     reachable = true;
+    unreported = new Map();
     constructor(host) {
         this.host = host;
     }
@@ -26,6 +27,8 @@ export class CircuitCoordinator {
         this.syncing = (async () => {
             try {
                 this.lastSyncAt = this.host.now();
+                if (this.unreported.size > 0)
+                    await this.report(store);
                 for (const shared of await store.read())
                     this.adopt(shared);
                 const resetTimeoutMs = this.host.config.resetTimeoutMs ?? 30_000;
@@ -51,6 +54,29 @@ export class CircuitCoordinator {
         })();
         return this.syncing;
     }
+    observed(providerName, failed) {
+        if (!this.host.config.shareObservations || !this.host.config.store?.observe)
+            return;
+        const bucketMs = bucketWidth(this.host.config.windowMs ?? 60_000);
+        const at = Math.floor(this.host.now() / bucketMs) * bucketMs;
+        let pending = this.unreported.get(providerName);
+        if (!pending) {
+            pending = { buckets: new Map(), succeeded: false, trailing: 0 };
+            this.unreported.set(providerName, pending);
+        }
+        const bucket = pending.buckets.get(at) ?? { calls: 0, failures: 0 };
+        bucket.calls += 1;
+        if (failed)
+            bucket.failures += 1;
+        pending.buckets.set(at, bucket);
+        if (failed)
+            pending.trailing += 1;
+        else {
+            pending.succeeded = true;
+            pending.trailing = 0;
+        }
+        this.tick();
+    }
     published(providerName, entry, to, reason) {
         entry.changedAt = this.stampAfter(entry.changedAt);
         this.publish({
@@ -72,6 +98,23 @@ export class CircuitCoordinator {
     reconcile() {
         for (const name of [...this.host.names()]) {
             const entry = this.host.entry(name);
+            if (this.host.config.shareObservations && this.host.config.store?.observe && entry.window.length > 0) {
+                const bucketMs = bucketWidth(this.host.config.windowMs ?? 60_000);
+                const buckets = new Map();
+                for (const call of entry.window) {
+                    const at = Math.floor(call.at / bucketMs) * bucketMs;
+                    const bucket = buckets.get(at) ?? { calls: 0, failures: 0 };
+                    bucket.calls += 1;
+                    if (call.failed)
+                        bucket.failures += 1;
+                    buckets.set(at, bucket);
+                }
+                this.unreported.set(name, {
+                    buckets,
+                    succeeded: entry.window.some((call) => !call.failed),
+                    trailing: entry.consecutiveFailures,
+                });
+            }
             if (entry.state === 'open' && entry.changedAt === Number.NEGATIVE_INFINITY) {
                 this.published(name, entry, 'open', entry.reason ?? 'opened while shared state was loading');
             }
@@ -81,6 +124,46 @@ export class CircuitCoordinator {
     async flush() {
         await Promise.all([...this.pending]);
     }
+    async report(store) {
+        if (!store.observe || this.unreported.size === 0)
+            return;
+        const batches = [...this.unreported];
+        this.unreported.clear();
+        const windowMs = this.host.config.windowMs ?? 60_000;
+        for (const [name, pending] of batches) {
+            const entry = this.host.entry(name);
+            const window = await store.observe(name, {
+                buckets: [...pending.buckets].map(([at, bucket]) => ({ at, calls: bucket.calls, failures: bucket.failures })),
+                succeeded: pending.succeeded,
+                trailingFailures: pending.trailing,
+                windowMs,
+                bucketMs: bucketWidth(windowMs),
+                now: this.host.now(),
+                ...(entry.state === 'closed' && Number.isFinite(entry.changedAt) ? { since: entry.changedAt } : {}),
+            });
+            entry.shared = {
+                calls: window.calls,
+                failures: window.failures,
+                consecutiveFailures: window.consecutiveFailures,
+            };
+            this.judge(name, entry, window);
+        }
+    }
+    judge(name, entry, window) {
+        if (entry.state !== 'closed')
+            return;
+        const config = this.host.config;
+        if (window.consecutiveFailures >= (config.failureThreshold ?? 5)) {
+            this.host.transition(name, entry, 'open', 'shared failure threshold reached');
+            return;
+        }
+        const rate = config.failureRateThreshold;
+        if (rate !== undefined &&
+            window.calls >= (config.minimumThroughput ?? 10) &&
+            window.failures / window.calls >= rate) {
+            this.host.transition(name, entry, 'open', 'shared failure rate threshold reached');
+        }
+    }
     adopt(shared) {
         const entry = this.host.entry(shared.providerName);
         if (shared.updatedAt <= entry.changedAt)
@@ -120,3 +203,6 @@ export class CircuitCoordinator {
         }
     }
 }
+export function bucketWidth(windowMs) {
+    return Math.max(1, Math.floor(windowMs / 12));
+}

package/dist/ops/health.d.ts

@@ -21,7 +21,18 @@ export interface ProviderHealthSnapshot {
      * latency up to 30.
      */
     score: number;
+    /**
+     * `unknown` with no recent calls to judge by, `unhealthy` past the failure threshold or under the
+     * minimum score, `degraded` when calls are failing or slow but not enough to stop routing to it,
+     * and `healthy` otherwise. Always set by `ProviderHealthMonitor`; optional so a hand-built snapshot
+     * still type-checks.
+     */
+    status?: ProviderHealthStatus;
+    /** True when the last call is older than `observationTtlMs`, so the figures above are old news. */
+    stale?: boolean;
 }
+/** Where a provider stands, by what its recent calls say. */
+export type ProviderHealthStatus = 'healthy' | 'degraded' | 'unhealthy' | 'unknown';
 /** Health tracking for providers, used to route around unhealthy ones. */
 export interface HealthConfig {
     /** Turns tracking on. Off by default, and then nothing is recorded. */
@@ -30,12 +41,22 @@ export interface HealthConfig {
     failureThreshold?: number;
     /** Score below which a provider is unhealthy. Defaults to 20. */
     minScore?: number;
+    /**
+     * How long a call's outcome counts, in milliseconds. A provider whose last call is older is
+     * `unknown`, and is routed to as a provider never seen is, rather than kept down for a failure
+     * hours ago or kept up for a success days ago. `checkProviders({ staleOnly: true })` refreshes it.
+     * Defaults to never expiring, as before.
+     */
+    observationTtlMs?: number;
+    /** Replaces the system clock, for tests. */
+    now?: () => Date;
 }
 /** Tracks provider health from call outcomes and scores each provider for routing. */
 export declare class ProviderHealthMonitor {
     private config;
     private states;
     constructor(config?: HealthConfig);
+    private nowDate;
     /** Records a successful call and its latency. */
     recordSuccess(providerName: string, latencyMs: number): void;
     /** Records a failed call. */
@@ -46,6 +67,9 @@ export declare class ProviderHealthMonitor {
     score(providerName: string): number;
     /** Whether a provider is healthy. */
     isHealthy(providerName: string): boolean;
+    /** Whether a provider's figures are too old to judge by, or there are none. */
+    isUnknown(providerName: string): boolean;
+    private isStale;
     private snapshotOne;
     private state;
 }

package/dist/ops/health.js

@@ -4,23 +4,30 @@ export class ProviderHealthMonitor {
     constructor(config = {}) {
         this.config = config;
     }
+    nowDate() {
+        return this.config.now?.() ?? new Date();
+    }
     recordSuccess(providerName, latencyMs) {
         if (!this.config.enabled)
             return;
         const state = this.state(providerName);
         state.successes += 1;
         state.consecutiveFailures = 0;
+        if (this.isStale(state))
+            Object.assign(state, freshState());
         state.avgLatencyMs = state.avgLatencyMs ? state.avgLatencyMs * 0.7 + latencyMs * 0.3 : latencyMs;
-        state.lastCheckedAt = new Date().toISOString();
+        state.lastCheckedAt = this.nowDate().toISOString();
     }
     recordFailure(providerName, error) {
         if (!this.config.enabled)
             return;
         const state = this.state(providerName);
+        if (this.isStale(state))
+            state.consecutiveFailures = 0;
         state.failures += 1;
         state.consecutiveFailures += 1;
         state.lastError = error instanceof Error ? error.message : String(error);
-        state.lastCheckedAt = new Date().toISOString();
+        state.lastCheckedAt = this.nowDate().toISOString();
     }
     snapshot(providerName) {
         const names = providerName ? [providerName] : [...this.states.keys()];
@@ -32,16 +39,39 @@ export class ProviderHealthMonitor {
     isHealthy(providerName) {
         return this.snapshotOne(providerName).healthy;
     }
+    isUnknown(providerName) {
+        return this.snapshotOne(providerName).status === 'unknown';
+    }
+    isStale(state) {
+        const ttl = this.config.observationTtlMs;
+        if (ttl === undefined || !state.lastCheckedAt)
+            return false;
+        return this.nowDate().getTime() - Date.parse(state.lastCheckedAt) > ttl;
+    }
     snapshotOne(providerName) {
-        const state = this.state(providerName);
+        const recorded = this.state(providerName);
+        const stale = this.isStale(recorded);
+        const state = stale
+            ? { ...freshState(), lastError: recorded.lastError, lastCheckedAt: recorded.lastCheckedAt }
+            : recorded;
         const failureThreshold = this.config.failureThreshold || 3;
         const failurePenalty = Math.min(60, state.consecutiveFailures * 20);
         const latencyPenalty = Math.min(30, state.avgLatencyMs / 1000);
         const score = Math.max(0, 100 - failurePenalty - latencyPenalty);
         const minScore = this.config.minScore ?? 20;
+        const healthy = state.consecutiveFailures < failureThreshold && score >= minScore;
+        const seen = recorded.successes + recorded.failures > 0;
         return {
             providerName,
-            healthy: state.consecutiveFailures < failureThreshold && score >= minScore,
+            healthy,
+            status: stale || !seen
+                ? 'unknown'
+                : !healthy
+                    ? 'unhealthy'
+                    : state.consecutiveFailures > 0 || score < 70
+                        ? 'degraded'
+                        : 'healthy',
+            stale,
             successes: state.successes,
             failures: state.failures,
             consecutiveFailures: state.consecutiveFailures,
@@ -54,14 +84,12 @@ export class ProviderHealthMonitor {
     state(providerName) {
         let state = this.states.get(providerName);
         if (!state) {
-            state = {
-                successes: 0,
-                failures: 0,
-                consecutiveFailures: 0,
-                avgLatencyMs: 0,
-            };
+            state = freshState();
             this.states.set(providerName, state);
         }
         return state;
     }
 }
+function freshState() {
+    return { successes: 0, failures: 0, consecutiveFailures: 0, avgLatencyMs: 0 };
+}