nexus-ai-pro 2.0.0 → 2.2.0

package/dist-cjs/operations/adapters.js

@@ -2,6 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.BullMQOperationDispatcher = exports.RedisOperationStore = void 0;
 const operations_js_1 = require("../types/operations.js");
+const errors_js_1 = require("./errors.js");
 const serialization_js_1 = require("./serialization.js");
 const stats_js_1 = require("./stats.js");
 const CAS_SCRIPT = `
@@ -12,21 +13,102 @@ if tonumber(decoded.sequence) ~= tonumber(ARGV[2]) then return 0 end
 redis.call('HSET', KEYS[1], ARGV[1], ARGV[3])
 return 1
 `;
+const PLACE_LUA = `
+local function place(key, score, id)
+  if score ~= '' then redis.call('ZADD', key, score, id) else redis.call('ZREM', key, id) end
+end
+`;
+const UNIQUE_KEY_LUA = `
+if ARGV[3] ~= '' then
+  local owner = redis.call('HGET', KEYS[2], ARGV[3])
+  if owner and owner ~= ARGV[1] and redis.call('HEXISTS', KEYS[1], owner) == 1 then return 0 end
+end
+`;
+const CREATE_SCRIPT = `${UNIQUE_KEY_LUA}
+redis.call('HSET', KEYS[1], ARGV[1], ARGV[2])
+if ARGV[3] ~= '' then redis.call('HSET', KEYS[2], ARGV[3], ARGV[1]) end
+return 1
+`;
+const INDEXED_CREATE_SCRIPT = `${PLACE_LUA}${UNIQUE_KEY_LUA}
+redis.call('HSET', KEYS[1], ARGV[1], ARGV[2])
+if ARGV[3] ~= '' then redis.call('HSET', KEYS[2], ARGV[3], ARGV[1]) end
+place(KEYS[3], ARGV[4], ARGV[1])
+place(KEYS[4], ARGV[5], ARGV[1])
+return 1
+`;
+const INDEXED_CAS_SCRIPT = `${PLACE_LUA}
+local current = redis.call('HGET', KEYS[1], ARGV[1])
+if current == false then return 0 end
+local decoded = cjson.decode(current)
+if tonumber(decoded.sequence) ~= tonumber(ARGV[2]) then return 0 end
+redis.call('HSET', KEYS[1], ARGV[1], ARGV[3])
+place(KEYS[2], ARGV[4], ARGV[1])
+place(KEYS[3], ARGV[5], ARGV[1])
+return 1
+`;
+const PRUNE_SCRIPT = `
+local terminal = { ${operations_js_1.TERMINAL_OPERATION_STATUSES.map((status) => `${status} = true`).join(', ')} }
+local removed = 0
+for i = 2, #ARGV do
+  local raw = redis.call('HGET', KEYS[1], ARGV[i])
+  local keep = false
+  if raw then
+    local record = cjson.decode(raw)
+    if ARGV[1] == 'queued' then
+      keep = record.status == 'queued'
+    else
+      keep = (not terminal[record.status]) and (record.lease ~= nil or record.status == 'running')
+    end
+  end
+  if not keep then
+    redis.call('ZREM', KEYS[2], ARGV[i])
+    removed = removed + 1
+  end
+end
+return removed
+`;
 class RedisOperationStore {
     client;
     prefix;
     useEval;
+    indexed;
     constructor(client, options = {}) {
         this.client = client;
         this.prefix = options.prefix ?? 'nexus-ai-pro:operations:';
         this.useEval = options.useEval !== false && typeof client.eval === 'function';
+        this.indexed = options.index === true;
+        if (this.indexed && !(client.zadd && client.zrem && client.zrangebyscore)) {
+            throw new TypeError('A Redis operation store with index: true needs zadd, zrem, and zrangebyscore');
+        }
     }
     async create(record) {
         (0, serialization_js_1.assertSerializableRecord)(record);
-        await this.client.hset(this.recordsKey(), record.id, JSON.stringify(record));
+        const encoded = JSON.stringify(record);
+        const duplicate = () => new errors_js_1.OperationDuplicateError(record.idempotencyKey);
+        if (this.useEval && this.client.eval && !this.indexed) {
+            const created = await this.client.eval(CREATE_SCRIPT, 2, this.recordsKey(), this.idempotencyKey(), record.id, encoded, record.idempotencyKey ?? '');
+            if (Number(created) !== 1)
+                throw duplicate();
+            return;
+        }
+        if (this.indexed && this.useEval && this.client.eval) {
+            const { queued, lease } = indexScores(record);
+            const created = await this.client.eval(INDEXED_CREATE_SCRIPT, 4, this.recordsKey(), this.idempotencyKey(), this.queuedKey(), this.leasesKey(), record.id, encoded, record.idempotencyKey ?? '', queued, lease);
+            if (Number(created) !== 1)
+                throw duplicate();
+            return;
+        }
+        if (record.idempotencyKey) {
+            const owner = await this.client.hget(this.idempotencyKey(), record.idempotencyKey);
+            if (owner && owner !== record.id && (await this.client.hget(this.recordsKey(), owner)))
+                throw duplicate();
+        }
+        await this.client.hset(this.recordsKey(), record.id, encoded);
         if (record.idempotencyKey) {
             await this.client.hset(this.idempotencyKey(), record.idempotencyKey, record.id);
         }
+        if (this.indexed)
+            await this.place(record);
     }
     async read(id) {
         const raw = await this.client.hget(this.recordsKey(), id);
@@ -36,6 +118,11 @@ class RedisOperationStore {
         (0, serialization_js_1.assertSerializableRecord)(record);
         const encoded = JSON.stringify(record);
         if (this.useEval && this.client.eval) {
+            if (this.indexed) {
+                const { queued, lease } = indexScores(record);
+                const result = await this.client.eval(INDEXED_CAS_SCRIPT, 3, this.recordsKey(), this.queuedKey(), this.leasesKey(), record.id, String(expectedSequence), encoded, queued, lease);
+                return Number(result) === 1;
+            }
             const result = await this.client.eval(CAS_SCRIPT, 1, this.recordsKey(), record.id, String(expectedSequence), encoded);
             return Number(result) === 1;
         }
@@ -43,6 +130,8 @@ class RedisOperationStore {
         if (!current || current.sequence !== expectedSequence)
             return false;
         await this.client.hset(this.recordsKey(), record.id, encoded);
+        if (this.indexed)
+            await this.place(record);
         return true;
     }
     async delete(id) {
@@ -50,19 +139,22 @@ class RedisOperationStore {
         if (record?.idempotencyKey)
             await this.client.hdel(this.idempotencyKey(), record.idempotencyKey);
         await this.client.hdel(this.recordsKey(), id);
+        if (this.indexed) {
+            await this.client.zrem?.(this.queuedKey(), id);
+            await this.client.zrem?.(this.leasesKey(), id);
+        }
         return record !== undefined;
     }
     async claimExpired(now, limit) {
+        if (this.indexed) {
+            return this.scanIndex('leases', Date.parse(now), limit, (record) => isExpired(record, now), isLeased);
+        }
         const records = await this.list();
         const expired = [];
         for (const record of records) {
             if (expired.length >= limit)
                 break;
-            if ((0, operations_js_1.isTerminalOperationStatus)(record.status))
-                continue;
-            if (record.lease && record.lease.expiresAt > now)
-                continue;
-            if (record.status === 'running' || record.lease)
+            if (isExpired(record, now))
                 expired.push(record);
         }
         return expired;
@@ -77,6 +169,9 @@ class RedisOperationStore {
     }
     async listQueued(limit, filter) {
         const now = new Date().toISOString();
+        if (this.indexed) {
+            return this.scanIndex('queued', Number.POSITIVE_INFINITY, limit, (record) => (0, stats_js_1.isUnheldQueued)(record, now) && (0, stats_js_1.matchesFilter)(record, filter), (record) => record.status === 'queued');
+        }
         return (await this.list())
             .filter((record) => (0, stats_js_1.isUnheldQueued)(record, now) && (0, stats_js_1.matchesFilter)(record, filter))
             .sort((a, b) => (a.createdAt < b.createdAt ? -1 : a.createdAt > b.createdAt ? 1 : 0))
@@ -85,14 +180,106 @@ class RedisOperationStore {
     async stats(now, filter) {
         return (0, stats_js_1.countRecords)(await this.list(), now, filter);
     }
+    async reindex() {
+        if (!this.indexed)
+            throw new TypeError('reindex() needs a store created with index: true');
+        const records = await this.list();
+        for (const record of records) {
+            const { queued, lease } = indexScores(record);
+            if (queued !== '')
+                await this.client.zadd?.(this.queuedKey(), Number(queued), record.id);
+            if (lease !== '')
+                await this.client.zadd?.(this.leasesKey(), Number(lease), record.id);
+        }
+        await this.prune('queued', records.filter((record) => record.status !== 'queued').map((record) => record.id));
+        await this.prune('leases', records.filter((record) => !isLeased(record)).map((record) => record.id));
+        return records.length;
+    }
+    async scanIndex(index, max, limit, keep, belongs) {
+        const found = [];
+        if (limit <= 0)
+            return found;
+        const stale = [];
+        const key = index === 'queued' ? this.queuedKey() : this.leasesKey();
+        const upper = max === Number.POSITIVE_INFINITY ? '+inf' : max;
+        const page = Math.max(16, limit * 4);
+        for (let offset = 0; found.length < limit; offset += page) {
+            const ids = (await this.client.zrangebyscore?.(key, '-inf', upper, 'LIMIT', offset, page)) ?? [];
+            const records = await this.readMany(ids);
+            for (const [position, id] of ids.entries()) {
+                const record = records[position];
+                if (!record || !belongs(record)) {
+                    stale.push(id);
+                    continue;
+                }
+                if (keep(record))
+                    found.push(record);
+                if (found.length >= limit)
+                    break;
+            }
+            if (ids.length < page)
+                break;
+        }
+        await this.prune(index, stale);
+        return found;
+    }
+    async readMany(ids) {
+        if (ids.length === 0)
+            return [];
+        const raws = this.client.hmget
+            ? await this.client.hmget(this.recordsKey(), ...ids)
+            : await Promise.all(ids.map((id) => this.client.hget(this.recordsKey(), id)));
+        return raws.map((raw) => (raw ? JSON.parse(raw) : undefined));
+    }
+    async prune(index, ids) {
+        if (ids.length === 0 || !this.useEval || !this.client.eval)
+            return;
+        const key = index === 'queued' ? this.queuedKey() : this.leasesKey();
+        for (let start = 0; start < ids.length; start += 500) {
+            await this.client.eval(PRUNE_SCRIPT, 2, this.recordsKey(), key, index, ...ids.slice(start, start + 500));
+        }
+    }
+    async place(record) {
+        const { queued, lease } = indexScores(record);
+        if (queued !== '')
+            await this.client.zadd?.(this.queuedKey(), Number(queued), record.id);
+        else
+            await this.client.zrem?.(this.queuedKey(), record.id);
+        if (lease !== '')
+            await this.client.zadd?.(this.leasesKey(), Number(lease), record.id);
+        else
+            await this.client.zrem?.(this.leasesKey(), record.id);
+    }
     recordsKey() {
         return `${this.prefix}records`;
     }
     idempotencyKey() {
         return `${this.prefix}idempotency`;
     }
+    queuedKey() {
+        return `${this.prefix}queued`;
+    }
+    leasesKey() {
+        return `${this.prefix}leases`;
+    }
 }
 exports.RedisOperationStore = RedisOperationStore;
+function isExpired(record, now) {
+    if ((0, operations_js_1.isTerminalOperationStatus)(record.status))
+        return false;
+    if (record.lease && record.lease.expiresAt > now)
+        return false;
+    return record.status === 'running' || record.lease !== undefined;
+}
+function isLeased(record) {
+    return !(0, operations_js_1.isTerminalOperationStatus)(record.status) && (record.lease !== undefined || record.status === 'running');
+}
+function indexScores(record) {
+    return {
+        queued: record.status === 'queued' ? String(Date.parse(record.createdAt)) : '',
+        lease: isLeased(record) ? String(record.lease ? Date.parse(record.lease.expiresAt) : 0) : '',
+    };
+}
 class BullMQOperationDispatcher {
     queue;
     options;

package/dist-cjs/operations/runner.js

@@ -96,6 +96,7 @@ class OperationRunner {
             expiresAt: options.expiresAt,
             idempotencyKey: options.idempotencyKey,
             kind: options.kind,
+            ...(options.tenantId ? { tenantId: options.tenantId } : {}),
             traceContext: options.traceContext,
             metadata: options.metadata,
         };
@@ -217,7 +218,7 @@ class OperationRunner {
             const heartbeat = this.startHeartbeat(record, handle);
             let outcome;
             try {
-                const value = await executor(this.buildContext(record, handle));
+                const value = await executor(this.buildContext(record, handle, heartbeat.beat));
                 outcome = { ok: true, value };
             }
             catch (error) {
@@ -293,7 +294,13 @@ class OperationRunner {
     startHeartbeat(record, handle) {
         let latest = record;
         let stopped = false;
-        const beat = async () => {
+        let inFlight = Promise.resolve();
+        const beat = (details) => {
+            const next = inFlight.then(() => renew(details));
+            inFlight = next.catch(() => undefined);
+            return next;
+        };
+        const renew = async (details) => {
             if (stopped)
                 return;
             const current = await this.store.read(latest.id);
@@ -318,6 +325,8 @@ class OperationRunner {
                 heartbeatAt: renewed.updatedAt,
             };
             renewed.progress = handle.progress() ?? current.progress;
+            if (details !== undefined)
+                renewed.heartbeatDetails = details;
             if (await this.store.update(renewed, current.sequence))
                 latest = renewed;
         };
@@ -338,24 +347,14 @@ class OperationRunner {
             beat,
         };
     }
-    buildContext(record, handle) {
+    buildContext(record, handle, beat) {
         return {
             operationId: record.id,
             attempt: record.attempt,
             signal: handle.signal,
             report: (progress) => handle.report(progress),
-            heartbeat: async () => {
-                const current = await this.store.read(record.id);
-                if (!current || (0, operations_js_1.isTerminalOperationStatus)(current.status))
-                    return;
-                const renewed = this.advance(current, 'running');
-                renewed.lease = {
-                    owner: this.owner,
-                    expiresAt: new Date(this.now().getTime() + this.leaseMs).toISOString(),
-                    heartbeatAt: renewed.updatedAt,
-                };
-                await this.store.update(renewed, current.sequence);
-            },
+            heartbeat: (details) => beat(details),
+            ...(record.heartbeatDetails === undefined ? {} : { previousHeartbeat: record.heartbeatDetails }),
             traceContext: record.traceContext,
             metadata: record.metadata,
         };

package/dist-cjs/operations/stats.js

@@ -9,6 +9,8 @@ function isUnheldQueued(record, now) {
     return record.status === 'queued' && (!record.lease || record.lease.expiresAt <= now);
 }
 function matchesFilter(record, filter) {
+    if (filter?.tenantId !== undefined && record.tenantId !== filter.tenantId)
+        return false;
     return !filter?.kindPrefix || (record.kind ?? '').startsWith(filter.kindPrefix);
 }
 function countRecords(records, now, filter) {
@@ -29,7 +31,12 @@ function countRecords(records, now, filter) {
 }
 async function operationStats(store, options = {}) {
     const now = (options.now ?? new Date()).toISOString();
-    const filter = options.kindPrefix ? { kindPrefix: options.kindPrefix } : undefined;
+    const filter = options.kindPrefix || options.tenantId !== undefined
+        ? {
+            ...(options.kindPrefix ? { kindPrefix: options.kindPrefix } : {}),
+            ...(options.tenantId !== undefined ? { tenantId: options.tenantId } : {}),
+        }
+        : undefined;
     if (store.stats)
         return store.stats(now, filter);
     if (store.list)

package/dist-cjs/operations/store.d.ts

@@ -19,7 +19,11 @@ export declare class MemoryOperationStore<TResult = unknown> implements Operatio
     private readonly byIdempotencyKey;
     private readonly maxRecords;
     constructor(options?: MemoryOperationStoreOptions);
-    /** Stores a new record. Refuses records carrying raw bytes. */
+    /**
+     * Stores a new record. Throws `OperationDuplicateError` when another record already holds its
+     * idempotency key, as every shared store does, so two submissions racing under one key cannot both
+     * start. Refuses records carrying raw bytes.
+     */
     create(record: OperationRecord<TResult>): void;
     /** Reads a record. */
     read(id: string): OperationRecord<TResult> | undefined;

package/dist-cjs/operations/store.js

@@ -2,6 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.MemoryOperationStore = exports.operationStats = exports.assertSerializableRecord = void 0;
 const operations_js_1 = require("../types/operations.js");
+const errors_js_1 = require("./errors.js");
 const serialization_js_1 = require("./serialization.js");
 Object.defineProperty(exports, "assertSerializableRecord", { enumerable: true, get: function () { return serialization_js_1.assertSerializableRecord; } });
 const stats_js_1 = require("./stats.js");
@@ -16,6 +17,12 @@ class MemoryOperationStore {
     }
     create(record) {
         (0, serialization_js_1.assertSerializableRecord)(record);
+        if (record.idempotencyKey) {
+            const owner = this.byIdempotencyKey.get(record.idempotencyKey);
+            if (owner !== undefined && owner !== record.id && this.records.has(owner)) {
+                throw new errors_js_1.OperationDuplicateError(record.idempotencyKey);
+            }
+        }
         this.records.set(record.id, clone(record));
         if (record.idempotencyKey)
             this.byIdempotencyKey.set(record.idempotencyKey, record.id);

package/dist-cjs/ops/circuit-breaker.d.ts

@@ -41,6 +41,15 @@ export interface CircuitBreakerConfig {
      * `nexus-ai-pro/ops/circuit-store` and `nexus-ai-pro/postgres/circuits`.
      */
     store?: CircuitStateStore;
+    /**
+     * Counts every worker's calls together, so nine failures spread over three workers count as nine.
+     * Each worker adds the calls it saw to a rolling window in the `store` when it syncs, and opens the
+     * circuit everywhere once the shared count crosses `failureThreshold` (failures since any worker's
+     * last success) or `failureRateThreshold`. Checks stay synchronous: a worker learns the shared count
+     * on its next sync, at most `syncIntervalMs` late. Needs a store with `observe()`; every bundled one
+     * has it. Off by default, when each worker counts only its own calls.
+     */
+    shareObservations?: boolean;
     /** Names this worker in shared state. Defaults to a random id. */
     workerId?: string;
     /** Shared state is refreshed at most this often, on the next check that needs it. Defaults to 1 second. */
@@ -85,6 +94,41 @@ export interface CircuitSnapshot {
     retryAt?: string;
     /** The most recent error, as text. */
     lastError?: string;
+    /** Every worker's calls together, as of this worker's last sync, with `shareObservations`. */
+    shared?: CircuitWindow;
+}
+/** Calls one worker saw since it last reported, which `CircuitStateStore.observe()` adds to the shared window. */
+export interface CircuitObservation {
+    /** Calls and failures per window bucket, by the bucket's start in epoch milliseconds. */
+    buckets: Array<{
+        at: number;
+        calls: number;
+        failures: number;
+    }>;
+    /** Whether any of the calls succeeded, which ends the shared failure streak before `trailingFailures` is added. */
+    succeeded: boolean;
+    /** Failures after the last success, or every failure when none succeeded. */
+    trailingFailures: number;
+    /** The rolling window, in milliseconds. */
+    windowMs: number;
+    /** The width of one bucket. A bucket that ended before `now - windowMs` is dropped. */
+    bucketMs: number;
+    /** Epoch milliseconds now, by the breaker's clock. */
+    now: number;
+    /**
+     * Epoch milliseconds of the circuit's last close. Buckets that started before it are not counted, so
+     * failures that opened a circuit do not reopen it the moment a probe closes it.
+     */
+    since?: number;
+}
+/** A provider's calls across every worker, inside the rolling window. */
+export interface CircuitWindow {
+    /** Calls counted. */
+    calls: number;
+    /** Of those, failures. */
+    failures: number;
+    /** Failures since any worker last reported a success. */
+    consecutiveFailures: number;
 }
 /** A circuit decision as every worker sees it. */
 export interface SharedCircuitState {
@@ -119,6 +163,12 @@ export interface CircuitStateStore {
      * recovering provider is probed once rather than by every replica at the same moment.
      */
     claimProbe(providerName: string, owner: string, ttlMs: number): Promise<boolean> | boolean;
+    /**
+     * Adds one worker's calls to a provider's shared window, and returns the window across every
+     * worker. Optional; `shareObservations` needs it. Each call must add atomically, so two workers
+     * reporting at once both count.
+     */
+    observe?(providerName: string, observation: CircuitObservation): Promise<CircuitWindow> | CircuitWindow;
 }
 /** @internal A circuit's local state. Shared with the coordinator module, never exported publicly. */
 export interface CircuitEntry {
@@ -139,6 +189,8 @@ export interface CircuitEntry {
     probeClaimed: boolean;
     /** Why the circuit last changed state. */
     reason?: string;
+    /** The shared window from the last sync, with `shareObservations`. */
+    shared?: CircuitWindow;
 }
 /**
  * Trips routing away from a provider that is failing.

package/dist-cjs/ops/circuit-breaker.js

@@ -105,6 +105,7 @@ class CircuitBreaker {
         this.refreshState(providerName, entry);
         this.pushWindow(entry, false);
         entry.consecutiveFailures = 0;
+        this.coordinator?.observed(providerName, false);
         if (entry.state === 'half-open') {
             entry.halfOpenInFlight = Math.max(0, entry.halfOpenInFlight - 1);
             entry.halfOpenSuccesses += 1;
@@ -126,6 +127,7 @@ class CircuitBreaker {
         }
         this.pushWindow(entry, true);
         entry.consecutiveFailures += 1;
+        this.coordinator?.observed(providerName, true);
         entry.lastError = error instanceof Error ? error.message : error === undefined ? undefined : String(error);
         if (entry.state === 'half-open') {
             entry.halfOpenInFlight = Math.max(0, entry.halfOpenInFlight - 1);
@@ -162,6 +164,7 @@ class CircuitBreaker {
                     ? new Date(entry.openedAt + resetTimeoutMs).toISOString()
                     : undefined,
                 lastError: entry.lastError,
+                ...(entry.shared ? { shared: { ...entry.shared } } : {}),
             };
         });
     }
@@ -212,6 +215,7 @@ class CircuitBreaker {
             entry.openedAt = undefined;
             entry.consecutiveFailures = 0;
             entry.window = [];
+            entry.shared = undefined;
             entry.halfOpenInFlight = 0;
             entry.halfOpenSuccesses = 0;
             entry.probeClaimed = false;

package/dist-cjs/ops/circuit-store.d.ts

@@ -1,5 +1,5 @@
-import type { CircuitStateStore, SharedCircuitState } from './circuit-breaker.js';
-export type { CircuitStateStore, SharedCircuitState } from './circuit-breaker.js';
+import type { CircuitObservation, CircuitStateStore, CircuitWindow, SharedCircuitState } from './circuit-breaker.js';
+export type { CircuitObservation, CircuitStateStore, CircuitWindow, SharedCircuitState, } from './circuit-breaker.js';
 /**
  * Shared circuit state in one process.
  *
@@ -10,7 +10,10 @@ export declare class MemoryCircuitStateStore implements CircuitStateStore {
     private readonly now;
     private readonly states;
     private readonly probes;
+    private readonly windows;
     constructor(now?: () => number);
+    /** Adds one worker's calls to a provider's window and returns the window across every worker. */
+    observe(providerName: string, observation: CircuitObservation): CircuitWindow;
     /** Every provider's shared circuit state. */
     read(): SharedCircuitState[];
     /**
@@ -66,7 +69,15 @@ export declare class RedisCircuitStateStore implements CircuitStateStore {
     private readonly client;
     private readonly prefix;
     private readonly useEval;
+    /**
+     * Adds one worker's calls to a provider's shared window in one Lua call, and returns the window
+     * across every worker. Present only when the client has `eval`, since the add and the sum must
+     * happen together; without it, `shareObservations` has nothing to call and each worker counts
+     * its own calls.
+     */
+    readonly observe?: (providerName: string, observation: CircuitObservation) => Promise<CircuitWindow>;
     constructor(client: RedisCircuitLikeClient, options?: RedisCircuitStateStoreOptions);
+    private observeWithEval;
     /** Every provider's shared circuit state. Unreadable entries are skipped. */
     read(): Promise<SharedCircuitState[]>;
     /**

package/dist-cjs/ops/circuit-store.js

@@ -5,9 +5,37 @@ class MemoryCircuitStateStore {
     now;
     states = new Map();
     probes = new Map();
+    windows = new Map();
     constructor(now = () => Date.now()) {
         this.now = now;
     }
+    observe(providerName, observation) {
+        let window = this.windows.get(providerName);
+        if (!window) {
+            window = { buckets: new Map(), streak: 0 };
+            this.windows.set(providerName, window);
+        }
+        for (const bucket of observation.buckets) {
+            const stored = window.buckets.get(bucket.at) ?? { calls: 0, failures: 0 };
+            stored.calls += bucket.calls;
+            stored.failures += bucket.failures;
+            window.buckets.set(bucket.at, stored);
+        }
+        window.streak = observation.succeeded ? observation.trailingFailures : window.streak + observation.trailingFailures;
+        const keepAfter = observation.now - observation.windowMs - observation.bucketMs;
+        const since = observation.since ?? Number.NEGATIVE_INFINITY;
+        let calls = 0;
+        let failures = 0;
+        for (const [at, bucket] of window.buckets) {
+            if (at <= keepAfter)
+                window.buckets.delete(at);
+            else if (at >= since) {
+                calls += bucket.calls;
+                failures += bucket.failures;
+            }
+        }
+        return { calls, failures, consecutiveFailures: window.streak };
+    }
     read() {
         return [...this.states.values()].map((state) => ({ ...state }));
     }
@@ -38,14 +66,54 @@ redis.call('HSET', KEYS[1], ARGV[1], ARGV[2])
 redis.call('DEL', KEYS[2])
 return 1
 `;
+const OBSERVE_SCRIPT = `
+local key = KEYS[1]
+for i = 6, #ARGV, 3 do
+  redis.call('HINCRBY', key, 'c:' .. ARGV[i], ARGV[i + 1])
+  redis.call('HINCRBY', key, 'f:' .. ARGV[i], ARGV[i + 2])
+end
+local streak
+if ARGV[1] == '1' then
+  redis.call('HSET', key, 'streak', ARGV[2])
+  streak = tonumber(ARGV[2])
+else
+  streak = tonumber(redis.call('HINCRBY', key, 'streak', ARGV[2]))
+end
+local keepAfter = tonumber(ARGV[3])
+local since = tonumber(ARGV[4])
+local calls, failures = 0, 0
+local fields = redis.call('HGETALL', key)
+for i = 1, #fields, 2 do
+  local name = fields[i]
+  local kind = string.sub(name, 1, 2)
+  if kind == 'c:' or kind == 'f:' then
+    local at = tonumber(string.sub(name, 3))
+    if at <= keepAfter then
+      redis.call('HDEL', key, name)
+    elseif at >= since then
+      if kind == 'c:' then calls = calls + tonumber(fields[i + 1]) else failures = failures + tonumber(fields[i + 1]) end
+    end
+  end
+end
+redis.call('PEXPIRE', key, ARGV[5])
+return { calls, failures, streak }
+`;
 class RedisCircuitStateStore {
     client;
     prefix;
     useEval;
+    observe;
     constructor(client, options = {}) {
         this.client = client;
         this.prefix = options.prefix ?? 'nexus-ai-pro:circuits:';
         this.useEval = options.useEval !== false && typeof client.eval === 'function';
+        if (this.useEval)
+            this.observe = (providerName, observation) => this.observeWithEval(providerName, observation);
+    }
+    async observeWithEval(providerName, observation) {
+        const reply = (await this.client.eval(OBSERVE_SCRIPT, 1, `${this.prefix}window:${providerName}`, observation.succeeded ? '1' : '0', String(observation.trailingFailures), String(observation.now - observation.windowMs - observation.bucketMs), String(observation.since ?? -1), String(observation.windowMs * 2 + observation.bucketMs), ...observation.buckets.flatMap((bucket) => [String(bucket.at), String(bucket.calls), String(bucket.failures)])));
+        const [calls, failures, streak] = (reply ?? []).map(Number);
+        return { calls: calls ?? 0, failures: failures ?? 0, consecutiveFailures: streak ?? 0 };
     }
     async read() {
         const all = (await this.client.hgetall(this.statesKey())) ?? {};