@jsonstudio/rcc 0.89.1189 → 0.89.1348

package/dist/manager/modules/quota/provider-quota-daemon.events.js

@@ -0,0 +1,237 @@
+import { applyErrorEvent as applyQuotaErrorEvent, createInitialQuotaState } from '../../quota/provider-quota-center.js';
+import { appendProviderErrorEvent, saveProviderQuotaSnapshot } from '../../quota/provider-quota-store.js';
+import { canonicalizeProviderKey } from './provider-key-normalization.js';
+import { capAutoCooldownMs, capAutoCooldownUntil, extractVirtualRouterSeriesCooldown, parseQuotaResetDelayMs } from './provider-quota-daemon.cooldown.js';
+import { isModelCapacityExhausted429, ProviderModelBackoffTracker } from './provider-quota-daemon.model-backoff.js';
+export async function handleProviderQuotaErrorEvent(ctx, event) {
+    if (!ctx.quotaRoutingEnabled) {
+        return;
+    }
+    if (!event) {
+        return;
+    }
+    const code = typeof event.code === 'string' ? event.code : '';
+    const extracted = extractProviderKey(event);
+    const providerKey = extracted ? canonicalizeProviderKey(extracted) : null;
+    if (!providerKey) {
+        return;
+    }
+    const nowMs = typeof event.timestamp === 'number' && Number.isFinite(event.timestamp) && event.timestamp > 0
+        ? event.timestamp
+        : Date.now();
+    const previous = ctx.quotaStates.get(providerKey) ??
+        createInitialQuotaState(providerKey, ctx.staticConfigs.get(providerKey), nowMs);
+    // Manual/operator blacklist is rigid: do not override it with automated error/quota signals.
+    if (previous.reason === 'blacklist' && previous.blacklistUntil && nowMs < previous.blacklistUntil) {
+        return;
+    }
+    // Upstream capacity exhaustion is not quota depletion. Cool down the entire model series immediately
+    // so the router can try other models/providers instead of hammering 429s.
+    if (isModelCapacityExhausted429(event)) {
+        ctx.modelBackoff.recordCapacity429(providerKey, event, nowMs, nowMs + 60_000);
+        const errorForQuota = {
+            providerKey,
+            httpStatus: typeof event.status === 'number' ? event.status : undefined,
+            code: typeof event.code === 'string' ? event.code : undefined,
+            fatal: false,
+            timestampMs: nowMs
+        };
+        const applied = applyQuotaErrorEvent(previous, errorForQuota, nowMs);
+        const nextState = {
+            ...applied,
+            inPool: false,
+            reason: 'cooldown',
+            cooldownUntil: nowMs + 60_000
+        };
+        ctx.quotaStates.set(providerKey, nextState);
+        ctx.schedulePersist(nowMs);
+        return;
+    }
+    // QUOTA_* 属于“确定性配额信号”，不进入错误 series 统计。
+    if (code === 'QUOTA_DEPLETED') {
+        const detailCarrier = (event.details && typeof event.details === 'object') ? event.details : {};
+        const raw = detailCarrier.virtualRouterQuotaDepleted;
+        const cooldownMs = raw && typeof raw === 'object' && typeof raw.cooldownMs === 'number'
+            ? raw.cooldownMs
+            : undefined;
+        const ttl = typeof cooldownMs === 'number' && Number.isFinite(cooldownMs) && cooldownMs > 0
+            ? cooldownMs
+            : undefined;
+        const cappedTtl = capAutoCooldownMs(ttl);
+        const nextCooldownUntil = cappedTtl ? nowMs + cappedTtl : previous.cooldownUntil;
+        const existingCooldownUntil = previous.cooldownUntil;
+        const cooldownUntil = typeof existingCooldownUntil === 'number' && typeof nextCooldownUntil === 'number' && existingCooldownUntil > nextCooldownUntil
+            ? existingCooldownUntil
+            : nextCooldownUntil;
+        const nextState = {
+            ...previous,
+            inPool: false,
+            reason: 'quotaDepleted',
+            cooldownUntil
+        };
+        ctx.quotaStates.set(providerKey, nextState);
+        ctx.schedulePersist(nowMs);
+        return;
+    }
+    if (code === 'QUOTA_RECOVERY') {
+        const withinBlacklist = previous.blacklistUntil !== null && nowMs < previous.blacklistUntil;
+        // QUOTA_RECOVERY should only flip providers that are waiting on an explicit quota snapshot:
+        // - previously quota-depleted, or
+        // - antigravity oauth "untracked" initial state (cooldown with no timers / no error series).
+        //
+        // It must NOT override active cooldown windows caused by real upstream failures
+        // (e.g. MODEL_CAPACITY_EXHAUSTED short backoff), otherwise the pool will keep hammering 429s.
+        const isUntrackedAntigravityOauthGate = previous.reason === 'cooldown' &&
+            previous.cooldownUntil === null &&
+            previous.blacklistUntil === null &&
+            previous.lastErrorSeries === null &&
+            previous.lastErrorCode === null &&
+            previous.lastErrorAtMs === null &&
+            previous.consecutiveErrorCount === 0;
+        const canRecover = previous.reason === 'quotaDepleted' || isUntrackedAntigravityOauthGate;
+        if (canRecover && !withinBlacklist) {
+            const nextState = {
+                ...previous,
+                inPool: true,
+                reason: 'ok',
+                cooldownUntil: null
+            };
+            ctx.quotaStates.set(providerKey, nextState);
+            ctx.schedulePersist(nowMs);
+        }
+        return;
+    }
+    // Gemini-family quota exhausted errors often carry quota reset delay.
+    // When present, treat as deterministic quota depletion signal rather than generic 429 backoff/blacklist.
+    if (typeof event.status === 'number' && event.status === 429) {
+        const seriesCooldown = extractVirtualRouterSeriesCooldown(event, nowMs);
+        if (seriesCooldown) {
+            const withinBlacklist = previous.blacklistUntil !== null && nowMs < previous.blacklistUntil;
+            // Do not override an active blacklist window (manual ops or policy).
+            if (withinBlacklist) {
+                return;
+            }
+            const isCapacityCooldown = typeof seriesCooldown.source === 'string' && seriesCooldown.source.toLowerCase().includes('capacity');
+            const until = capAutoCooldownUntil(seriesCooldown.until, nowMs);
+            const existingCooldownUntil = previous.cooldownUntil;
+            const cooldownUntil = typeof existingCooldownUntil === 'number' && existingCooldownUntil > until
+                ? existingCooldownUntil
+                : until;
+            const nextState = {
+                ...previous,
+                inPool: false,
+                reason: isCapacityCooldown ? 'cooldown' : 'quotaDepleted',
+                cooldownUntil,
+                lastErrorSeries: null,
+                lastErrorCode: null,
+                lastErrorAtMs: null,
+                consecutiveErrorCount: 0,
+                ...(isCapacityCooldown
+                    ? {}
+                    : {
+                        blacklistUntil: null,
+                        // deterministic quota signals should clear blacklist to avoid sticky long locks
+                        // when upstream provides explicit reset delay.
+                    })
+            };
+            ctx.quotaStates.set(providerKey, nextState);
+            ctx.schedulePersist(nowMs);
+            return;
+        }
+        const runtime = event.runtime;
+        const providerIdRaw = runtime && typeof runtime.providerId === 'string' ? runtime.providerId.trim().toLowerCase() : '';
+        const isQuotaProvider = providerIdRaw === 'antigravity' || providerIdRaw === 'gemini-cli';
+        if (isQuotaProvider) {
+            const ttl = parseQuotaResetDelayMs(event);
+            if (ttl && ttl > 0) {
+                const capped = capAutoCooldownMs(ttl);
+                const nextState = {
+                    ...previous,
+                    inPool: false,
+                    reason: 'quotaDepleted',
+                    cooldownUntil: nowMs + (capped ?? ttl),
+                    blacklistUntil: null,
+                    lastErrorSeries: null,
+                    lastErrorCode: null,
+                    lastErrorAtMs: null,
+                    consecutiveErrorCount: 0
+                };
+                ctx.quotaStates.set(providerKey, nextState);
+                ctx.schedulePersist(nowMs);
+                return;
+            }
+        }
+    }
+    const errorForQuota = {
+        providerKey,
+        httpStatus: typeof event.status === 'number' ? event.status : undefined,
+        code: typeof event.code === 'string' ? event.code : undefined,
+        fatal: isFatalForQuota(event),
+        timestampMs: nowMs
+    };
+    const nextState = applyQuotaErrorEvent(previous, errorForQuota, nowMs);
+    ctx.quotaStates.set(providerKey, nextState);
+    const tsIso = new Date(nowMs).toISOString();
+    try {
+        await appendProviderErrorEvent({
+            ts: tsIso,
+            providerKey,
+            code: typeof errorForQuota.code === 'string' ? errorForQuota.code : undefined,
+            httpStatus: typeof errorForQuota.httpStatus === 'number' ? errorForQuota.httpStatus : undefined,
+            message: event.message,
+            details: {
+                stage: event.stage,
+                routeName: event.runtime.routeName,
+                entryEndpoint: event.runtime.entryEndpoint
+            }
+        });
+    }
+    catch {
+        // logging failure is non-fatal
+    }
+    try {
+        await saveProviderQuotaSnapshot(ctx.toSnapshotObject(), new Date(nowMs));
+    }
+    catch {
+        // best-effort persistence only
+    }
+}
+function extractProviderKey(event) {
+    const runtime = event.runtime;
+    const direct = runtime && typeof runtime.providerKey === 'string' && runtime.providerKey.trim()
+        ? runtime.providerKey.trim()
+        : null;
+    if (direct) {
+        return direct;
+    }
+    const target = runtime && runtime.target;
+    if (target && typeof target === 'object') {
+        const targetKey = target.providerKey;
+        if (typeof targetKey === 'string' && targetKey.trim()) {
+            return targetKey.trim();
+        }
+    }
+    return null;
+}
+function isFatalForQuota(event) {
+    const status = typeof event.status === 'number' ? event.status : undefined;
+    const code = typeof event.code === 'string' ? event.code.toUpperCase() : '';
+    const stage = typeof event.stage === 'string' ? event.stage.toLowerCase() : '';
+    if (status === 401 || status === 402 || status === 403) {
+        return true;
+    }
+    if (code.includes('AUTH') || code.includes('UNAUTHORIZED')) {
+        return true;
+    }
+    if (code.includes('CONFIG')) {
+        return true;
+    }
+    if (stage.includes('compat')) {
+        return true;
+    }
+    if (event.recoverable === false && status !== undefined && status >= 500) {
+        return true;
+    }
+    return false;
+}
+//# sourceMappingURL=provider-quota-daemon.events.js.map

package/dist/manager/modules/quota/provider-quota-daemon.events.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"provider-quota-daemon.events.js","sourceRoot":"","sources":["../../../../src/manager/modules/quota/provider-quota-daemon.events.ts"],"names":[],"mappings":"AACA,OAAO,EACL,eAAe,IAAI,oBAAoB,EACvC,uBAAuB,EAIxB,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EAAE,wBAAwB,EAAE,yBAAyB,EAAE,MAAM,qCAAqC,CAAC;AAC1G,OAAO,EAAE,uBAAuB,EAAE,MAAM,iCAAiC,CAAC;AAC1E,OAAO,EACL,iBAAiB,EACjB,oBAAoB,EACpB,kCAAkC,EAClC,sBAAsB,EACvB,MAAM,qCAAqC,CAAC;AAC7C,OAAO,EAAE,2BAA2B,EAAE,2BAA2B,EAAE,MAAM,0CAA0C,CAAC;AAWpH,MAAM,CAAC,KAAK,UAAU,6BAA6B,CACjD,GAAoC,EACpC,KAAyB;IAEzB,IAAI,CAAC,GAAG,CAAC,mBAAmB,EAAE,CAAC;QAC7B,OAAO;IACT,CAAC;IACD,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO;IACT,CAAC;IACD,MAAM,IAAI,GAAG,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;IAE9D,MAAM,SAAS,GAAG,kBAAkB,CAAC,KAAK,CAAC,CAAC;IAC5C,MAAM,WAAW,GAAG,SAAS,CAAC,CAAC,CAAC,uBAAuB,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC1E,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,OAAO;IACT,CAAC;IAED,MAAM,KAAK,GACT,OAAO,KAAK,CAAC,SAAS,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,KAAK,CAAC,SAAS,GAAG,CAAC;QAC5F,CAAC,CAAC,KAAK,CAAC,SAAS;QACjB,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC;IAEjB,MAAM,QAAQ,GACZ,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,WAAW,CAAC;QAChC,uBAAuB,CAAC,WAAW,EAAE,GAAG,CAAC,aAAa,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,KAAK,CAAC,CAAC;IAElF,6FAA6F;IAC7F,IAAI,QAAQ,CAAC,MAAM,KAAK,WAAW,IAAI,QAAQ,CAAC,cAAc,IAAI,KAAK,GAAG,QAAQ,CAAC,cAAc,EAAE,CAAC;QAClG,OAAO;IACT,CAAC;IAED,qGAAqG;IACrG,0EAA0E;IAC1E,IAAI,2BAA2B,CAAC,KAAK,CAAC,EAAE,CAAC;QACvC,GAAG,CAAC,YAAY,CAAC,iBAAiB,CAAC,WAAW,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,GAAG,MAAM,CAAC,CAAC;QAC9E,MAAM,aAAa,GAAuB;YACxC,WAAW;YACX,UAAU,EAAE,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;YACvE,IAAI,EAAE,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS;YAC7D,KAAK,EAAE,KAAK;YACZ,WAAW,EAAE,KAAK;SACnB,CAAC;QACF,MAAM,OAAO,GAAG,oBAAoB,CAAC,QAAQ,EAAE,aAAa,EAAE,KAAK,CAAC,CAAC;QACrE,MAAM,SAAS,GAAe;YAC5B,GAAG,OAAO;YACV,MAAM,EAAE,KAAK;YACb,MAAM,EAAE,UAAU;YAClB,aAAa,EAAE,KAAK,GAAG,MAAM;SAC9B,CAAC;QACF,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;QAC5C,GAAG,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;QAC3B,OAAO;IACT,CAAC;IAED,uCAAuC;IACvC,IAAI,IAAI,KAAK,gBAAgB,EAAE,CAAC;QAC9B,MAAM,aAAa,GAAG,CAAC,KAAK,CAAC,OAAO,IAAI,OAAO,KAAK,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAE,KAAK,CAAC,OAAmC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC7H,MAAM,GAAG,GAAG,aAAa,CAAC,0BAA0B,CAAC;QACrD,MAAM,UAAU,GACd,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,OAAQ,GAAgC,CAAC,UAAU,KAAK,QAAQ;YAChG,CAAC,CAAG,GAA+B,CAAC,UAAqB;YACzD,CAAC,CAAC,SAAS,CAAC;QAChB,MAAM,GAAG,GACP,OAAO,UAAU,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,UAAU,GAAG,CAAC;YAC7E,CAAC,CAAC,UAAU;YACZ,CAAC,CAAC,SAAS,CAAC;QAChB,MAAM,SAAS,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAC;QACzC,MAAM,iBAAiB,GAAG,SAAS,CAAC,CAAC,CAAC,KAAK,GAAG,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC;QACjF,MAAM,qBAAqB,GAAG,QAAQ,CAAC,aAAa,CAAC;QACrD,MAAM,aAAa,GACjB,OAAO,qBAAqB,KAAK,QAAQ,IAAI,OAAO,iBAAiB,KAAK,QAAQ,IAAI,qBAAqB,GAAG,iBAAiB;YAC7H,CAAC,CAAC,qBAAqB;YACvB,CAAC,CAAC,iBAAiB,CAAC;QACxB,MAAM,SAAS,GAAe;YAC5B,GAAG,QAAQ;YACX,MAAM,EAAE,KAAK;YACb,MAAM,EAAE,eAAe;YACvB,aAAa;SACd,CAAC;QACF,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;QAC5C,GAAG,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;QAC3B,OAAO;IACT,CAAC;IACD,IAAI,IAAI,KAAK,gBAAgB,EAAE,CAAC;QAC9B,MAAM,eAAe,GACnB,QAAQ,CAAC,cAAc,KAAK,IAAI,IAAI,KAAK,GAAG,QAAQ,CAAC,cAAc,CAAC;QACtE,4FAA4F;QAC5F,kCAAkC;QAClC,6FAA6F;QAC7F,EAAE;QACF,gFAAgF;QAChF,8FAA8F;QAC7F,MAAM,+BAA+B,GACnC,QAAQ,CAAC,MAAM,KAAK,UAAU;YAC9B,QAAQ,CAAC,aAAa,KAAK,IAAI;YAC/B,QAAQ,CAAC,cAAc,KAAK,IAAI;YAChC,QAAQ,CAAC,eAAe,KAAK,IAAI;YACjC,QAAQ,CAAC,aAAa,KAAK,IAAI;YAC/B,QAAQ,CAAC,aAAa,KAAK,IAAI;YAC/B,QAAQ,CAAC,qBAAqB,KAAK,CAAC,CAAC;QACxC,MAAM,UAAU,GAAG,QAAQ,CAAC,MAAM,KAAK,eAAe,IAAI,+BAA+B,CAAC;QAC1F,IAAI,UAAU,IAAI,CAAC,eAAe,EAAE,CAAC;YACnC,MAAM,SAAS,GAAe;gBAC5B,GAAG,QAAQ;gBACX,MAAM,EAAE,IAAI;gBACZ,MAAM,EAAE,IAAI;gBACZ,aAAa,EAAE,IAAI;aACpB,CAAC;YACF,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;YAC5C,GAAG,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;QAC7B,CAAC;QACD,OAAO;IACT,CAAC;IAED,sEAAsE;IACtE,yGAAyG;IACzG,IAAI,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ,IAAI,KAAK,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;QAC7D,MAAM,cAAc,GAAG,kCAAkC,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QACxE,IAAI,cAAc,EAAE,CAAC;YACnB,MAAM,eAAe,GACnB,QAAQ,CAAC,cAAc,KAAK,IAAI,IAAI,KAAK,GAAG,QAAQ,CAAC,cAAc,CAAC;YACtE,qEAAqE;YACrE,IAAI,eAAe,EAAE,CAAC;gBACpB,OAAO;YACT,CAAC;YACD,MAAM,kBAAkB,GACtB,OAAO,cAAc,CAAC,MAAM,KAAK,QAAQ,IAAI,cAAc,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;YACxG,MAAM,KAAK,GAAG,oBAAoB,CAAC,cAAc,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;YAChE,MAAM,qBAAqB,GAAG,QAAQ,CAAC,aAAa,CAAC;YACrD,MAAM,aAAa,GACjB,OAAO,qBAAqB,KAAK,QAAQ,IAAI,qBAAqB,GAAG,KAAK;gBACxE,CAAC,CAAC,qBAAqB;gBACvB,CAAC,CAAC,KAAK,CAAC;YACX,MAAM,SAAS,GAAe;gBAC5B,GAAG,QAAQ;gBACX,MAAM,EAAE,KAAK;gBACb,MAAM,EAAE,kBAAkB,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,eAAe;gBACzD,aAAa;gBACb,eAAe,EAAE,IAAI;gBACrB,aAAa,EAAE,IAAI;gBACnB,aAAa,EAAE,IAAI;gBACnB,qBAAqB,EAAE,CAAC;gBACxB,GAAG,CAAC,kBAAkB;oBACpB,CAAC,CAAC,EAAE;oBACJ,CAAC,CAAC;wBACC,cAAc,EAAE,IAAI;wBACpB,gFAAgF;wBAChF,+CAA+C;qBAChD,CAAC;aACP,CAAC;YACF,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;YAC5C,GAAG,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;YAC3B,OAAO;QACT,CAAC;QACD,MAAM,OAAO,GAAG,KAAK,CAAC,OAA2E,CAAC;QAClG,MAAM,aAAa,GAAG,OAAO,IAAI,OAAO,OAAO,CAAC,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACvH,MAAM,eAAe,GAAG,aAAa,KAAK,aAAa,IAAI,aAAa,KAAK,YAAY,CAAC;QAC1F,IAAI,eAAe,EAAE,CAAC;YACpB,MAAM,GAAG,GAAG,sBAAsB,CAAC,KAAK,CAAC,CAAC;YAC1C,IAAI,GAAG,IAAI,GAAG,GAAG,CAAC,EAAE,CAAC;gBACnB,MAAM,MAAM,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAC;gBACrC,MAAM,SAAS,GAAe;oBAC5B,GAAG,QAAQ;oBACX,MAAM,EAAE,KAAK;oBACb,MAAM,EAAE,eAAe;oBACvB,aAAa,EAAE,KAAK,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC;oBACtC,cAAc,EAAE,IAAI;oBACpB,eAAe,EAAE,IAAI;oBACrB,aAAa,EAAE,IAAI;oBACnB,aAAa,EAAE,IAAI;oBACnB,qBAAqB,EAAE,CAAC;iBACzB,CAAC;gBACH,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;gBAC5C,GAAG,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;gBAC3B,OAAO;YACT,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,aAAa,GAAuB;QACxC,WAAW;QACX,UAAU,EAAE,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;QACvE,IAAI,EAAE,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS;QAC7D,KAAK,EAAE,eAAe,CAAC,KAAK,CAAC;QAC7B,WAAW,EAAE,KAAK;KACnB,CAAC;IAEF,MAAM,SAAS,GAAG,oBAAoB,CAAC,QAAQ,EAAE,aAAa,EAAE,KAAK,CAAC,CAAC;IACvE,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;IAE5C,MAAM,KAAK,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC;IAC5C,IAAI,CAAC;QACH,MAAM,wBAAwB,CAAC;YAC7B,EAAE,EAAE,KAAK;YACT,WAAW;YACX,IAAI,EAAE,OAAO,aAAa,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS;YAC7E,UAAU,EAAE,OAAO,aAAa,CAAC,UAAU,KAAK,QAAQ,CAAC,CAAC,CAAC,aAAa,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS;YAC/F,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,OAAO,EAAE;gBACP,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,SAAS,EAAG,KAAK,CAAC,OAAkC,CAAC,SAAS;gBAC9D,aAAa,EAAG,KAAK,CAAC,OAAsC,CAAC,aAAa;aAC3E;SACF,CAAC,CAAC;IACL,CAAC;IAAC,MAAM,CAAC;QACP,+BAA+B;IACjC,CAAC;IAED,IAAI,CAAC;QACH,MAAM,yBAAyB,CAAC,GAAG,CAAC,gBAAgB,EAAE,EAAE,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;IAC3E,CAAC;IAAC,MAAM,CAAC;QACP,+BAA+B;IACjC,CAAC;AACH,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAyB;IACnD,MAAM,OAAO,GAAG,KAAK,CAAC,OAAkE,CAAC;IACzF,MAAM,MAAM,GACV,OAAO,IAAI,OAAO,OAAO,CAAC,WAAW,KAAK,QAAQ,IAAI,OAAO,CAAC,WAAW,CAAC,IAAI,EAAE;QAC9E,CAAC,CAAC,OAAO,CAAC,WAAW,CAAC,IAAI,EAAE;QAC5B,CAAC,CAAC,IAAI,CAAC;IACX,IAAI,MAAM,EAAE,CAAC;QACX,OAAO,MAAM,CAAC;IAChB,CAAC;IACD,MAAM,MAAM,GAAG,OAAO,IAAI,OAAO,CAAC,MAAM,CAAC;IACzC,IAAI,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;QACzC,MAAM,SAAS,GAAI,MAAoC,CAAC,WAAW,CAAC;QACpE,IAAI,OAAO,SAAS,KAAK,QAAQ,IAAI,SAAS,CAAC,IAAI,EAAE,EAAE,CAAC;YACtD,OAAO,SAAS,CAAC,IAAI,EAAE,CAAC;QAC1B,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,eAAe,CAAC,KAAyB;IAChD,MAAM,MAAM,GAAG,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;IAC3E,MAAM,IAAI,GAAG,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAC5E,MAAM,KAAK,GAAG,OAAO,KAAK,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IAE/E,IAAI,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;QACvD,OAAO,IAAI,CAAC;IACd,CAAC;IACD,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,cAAc,CAAC,EAAE,CAAC;QAC3D,OAAO,IAAI,CAAC;IACd,CAAC;IACD,IAAI,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC5B,OAAO,IAAI,CAAC;IACd,CAAC;IACD,IAAI,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC7B,OAAO,IAAI,CAAC;IACd,CAAC;IACD,IAAI,KAAK,CAAC,WAAW,KAAK,KAAK,IAAI,MAAM,KAAK,SAAS,IAAI,MAAM,IAAI,GAAG,EAAE,CAAC;QACzE,OAAO,IAAI,CAAC;IACd,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC"}

package/dist/manager/modules/quota/provider-quota-daemon.js

@@ -0,0 +1,404 @@
+import { getProviderErrorCenter } from '../../../modules/llmswitch/bridge.js';
+import { applySuccessEvent as applyQuotaSuccessEvent, applyUsageEvent as applyQuotaUsageEvent, createInitialQuotaState, tickQuotaStateTime } from '../../quota/provider-quota-center.js';
+import { saveProviderQuotaSnapshot } from '../../quota/provider-quota-store.js';
+import { canonicalizeProviderKey } from './provider-key-normalization.js';
+import { handleProviderQuotaErrorEvent } from './provider-quota-daemon.events.js';
+import { loadProviderQuotaStates } from './provider-quota-daemon.snapshot.js';
+import { buildQuotaViewEntry } from './provider-quota-daemon.view.js';
+import { ProviderModelBackoffTracker } from './provider-quota-daemon.model-backoff.js';
+const ERROR_PRIORITY_WINDOW_MS = readPositiveNumberFromEnv('ROUTECODEX_QUOTA_ERROR_PRIORITY_WINDOW_MS', 10 * 60_000);
+export class ProviderQuotaDaemonModule {
+    id = 'provider-quota';
+    quotaStates = new Map();
+    staticConfigs = new Map();
+    modelBackoff = new ProviderModelBackoffTracker();
+    unsubscribe = null;
+    maintenanceTimer = null;
+    persistTimer = null;
+    quotaRoutingEnabled = true;
+    async loadSnapshotIntoMemory() {
+        const { quotaStates, seeded, needsPersist } = await loadProviderQuotaStates({
+            staticConfigs: this.staticConfigs
+        });
+        this.quotaStates = quotaStates;
+        const nowMs = Date.now();
+        if (seeded || needsPersist) {
+            this.schedulePersist(nowMs);
+        }
+    }
+    async init(context) {
+        this.quotaRoutingEnabled = context.quotaRoutingEnabled !== false;
+        try {
+            await this.loadSnapshotIntoMemory();
+        }
+        catch {
+            this.quotaStates = new Map();
+        }
+    }
+    async reloadFromDisk() {
+        await this.loadSnapshotIntoMemory();
+        return { loadedAt: Date.now(), providerCount: this.quotaStates.size };
+    }
+    async reset(options = {}) {
+        const nowMs = Date.now();
+        this.quotaStates = new Map();
+        if (this.staticConfigs.size) {
+            for (const [providerKey, cfg] of this.staticConfigs.entries()) {
+                this.quotaStates.set(providerKey, createInitialQuotaState(providerKey, cfg, nowMs));
+            }
+        }
+        const persisted = options.persist !== false;
+        if (persisted) {
+            try {
+                await saveProviderQuotaSnapshot(this.toSnapshotObject(), new Date(nowMs));
+            }
+            catch {
+                // ignore persistence failure
+            }
+        }
+        return { resetAt: nowMs, persisted };
+    }
+    async resetProvider(providerKey) {
+        const raw = typeof providerKey === 'string' ? providerKey.trim() : '';
+        const key = raw ? canonicalizeProviderKey(raw) : '';
+        if (!key) {
+            return null;
+        }
+        const nowMs = Date.now();
+        const next = createInitialQuotaState(key, this.staticConfigs.get(key), nowMs);
+        this.quotaStates.set(key, next);
+        try {
+            await saveProviderQuotaSnapshot(this.toSnapshotObject(), new Date(nowMs));
+        }
+        catch {
+            // ignore persistence failure
+        }
+        return { providerKey: key, state: next };
+    }
+    async recoverProvider(providerKey) {
+        const raw = typeof providerKey === 'string' ? providerKey.trim() : '';
+        const key = raw ? canonicalizeProviderKey(raw) : '';
+        if (!key) {
+            return null;
+        }
+        const nowMs = Date.now();
+        const previous = this.quotaStates.get(key) ?? createInitialQuotaState(key, this.staticConfigs.get(key), nowMs);
+        const next = {
+            ...previous,
+            inPool: true,
+            reason: 'ok',
+            cooldownUntil: null,
+            blacklistUntil: null,
+            lastErrorSeries: null,
+            lastErrorCode: null,
+            lastErrorAtMs: null,
+            consecutiveErrorCount: 0
+        };
+        this.quotaStates.set(key, next);
+        try {
+            await saveProviderQuotaSnapshot(this.toSnapshotObject(), new Date(nowMs));
+        }
+        catch {
+            // ignore persistence failure
+        }
+        return { providerKey: key, state: next };
+    }
+    async disableProvider(options) {
+        const raw = typeof options?.providerKey === 'string' ? options.providerKey.trim() : '';
+        const key = raw ? canonicalizeProviderKey(raw) : '';
+        if (!key) {
+            return null;
+        }
+        const durationMs = typeof options.durationMs === 'number' && Number.isFinite(options.durationMs) && options.durationMs > 0
+            ? Math.floor(options.durationMs)
+            : 0;
+        if (!durationMs) {
+            return null;
+        }
+        const mode = options.mode === 'blacklist' ? 'blacklist' : 'cooldown';
+        const nowMs = Date.now();
+        const previous = this.quotaStates.get(key) ?? createInitialQuotaState(key, this.staticConfigs.get(key), nowMs);
+        const next = mode === 'blacklist'
+            ? {
+                ...previous,
+                inPool: false,
+                reason: 'blacklist',
+                blacklistUntil: nowMs + durationMs,
+                cooldownUntil: null
+            }
+            : {
+                ...previous,
+                inPool: false,
+                reason: 'cooldown',
+                cooldownUntil: nowMs + durationMs
+            };
+        this.quotaStates.set(key, next);
+        try {
+            await saveProviderQuotaSnapshot(this.toSnapshotObject(), new Date(nowMs));
+        }
+        catch {
+            // ignore persistence failure
+        }
+        return { providerKey: key, state: next };
+    }
+    getAdminSnapshot() {
+        return this.toSnapshotObject();
+    }
+    async start() {
+        if (!this.quotaRoutingEnabled) {
+            return;
+        }
+        let center = null;
+        try {
+            center = await getProviderErrorCenter();
+        }
+        catch {
+            center = null;
+        }
+        if (center && typeof center.subscribe === 'function') {
+            this.unsubscribe = center.subscribe((event) => {
+                void this.handleProviderErrorEvent(event).catch(() => {
+                    // swallow handler errors; quota updates are best-effort
+                });
+            });
+        }
+        const intervalMs = readPositiveNumberFromEnv('ROUTECODEX_QUOTA_DAEMON_INTERVAL_MS', 60_000);
+        if (intervalMs > 0) {
+            this.maintenanceTimer = setInterval(() => {
+                void this.runMaintenanceTick().catch(() => {
+                    // ignore maintenance failures
+                });
+            }, intervalMs);
+        }
+        void this.runMaintenanceTick().catch(() => {
+            // ignore immediate tick failures
+        });
+    }
+    async stop() {
+        if (this.unsubscribe) {
+            try {
+                this.unsubscribe();
+            }
+            catch {
+                // ignore unsubscribe failures
+            }
+            this.unsubscribe = null;
+        }
+        if (this.maintenanceTimer) {
+            clearInterval(this.maintenanceTimer);
+            this.maintenanceTimer = null;
+        }
+        if (this.persistTimer) {
+            clearTimeout(this.persistTimer);
+            this.persistTimer = null;
+        }
+        if (!this.quotaRoutingEnabled) {
+            return;
+        }
+        try {
+            await saveProviderQuotaSnapshot(this.toSnapshotObject(), new Date());
+        }
+        catch {
+            // best-effort persistence
+        }
+    }
+    recordProviderUsage(event) {
+        if (!this.quotaRoutingEnabled) {
+            return;
+        }
+        const rawKey = typeof event?.providerKey === 'string' ? event.providerKey.trim() : '';
+        const providerKey = rawKey ? canonicalizeProviderKey(rawKey) : '';
+        if (!providerKey) {
+            return;
+        }
+        const nowMs = typeof event.timestampMs === 'number' && Number.isFinite(event.timestampMs) && event.timestampMs > 0
+            ? event.timestampMs
+            : Date.now();
+        const requestedTokens = typeof event.requestedTokens === 'number' && Number.isFinite(event.requestedTokens) && event.requestedTokens > 0
+            ? event.requestedTokens
+            : 0;
+        const previous = this.quotaStates.get(providerKey) ??
+            createInitialQuotaState(providerKey, this.staticConfigs.get(providerKey), nowMs);
+        const nextState = applyQuotaUsageEvent(previous, { providerKey, requestedTokens, timestampMs: nowMs }, nowMs);
+        this.quotaStates.set(providerKey, nextState);
+        this.schedulePersist(nowMs);
+    }
+    recordProviderSuccess(event) {
+        if (!this.quotaRoutingEnabled) {
+            return;
+        }
+        const rawKey = typeof event?.providerKey === 'string' ? event.providerKey.trim() : '';
+        const providerKey = rawKey ? canonicalizeProviderKey(rawKey) : '';
+        if (!providerKey) {
+            return;
+        }
+        const nowMs = typeof event.timestampMs === 'number' && Number.isFinite(event.timestampMs) && event.timestampMs > 0
+            ? event.timestampMs
+            : Date.now();
+        const usedTokens = typeof event.usedTokens === 'number' && Number.isFinite(event.usedTokens) && event.usedTokens > 0
+            ? event.usedTokens
+            : 0;
+        const previous = this.quotaStates.get(providerKey) ??
+            createInitialQuotaState(providerKey, this.staticConfigs.get(providerKey), nowMs);
+        const nextState = applyQuotaSuccessEvent(previous, { providerKey, usedTokens, timestampMs: nowMs }, nowMs);
+        this.quotaStates.set(providerKey, nextState);
+        this.modelBackoff.recordSuccess(providerKey);
+        this.schedulePersist(nowMs);
+    }
+    registerProviderStaticConfig(providerKey, config = {}) {
+        if (!this.quotaRoutingEnabled) {
+            return;
+        }
+        const raw = typeof providerKey === 'string' ? providerKey.trim() : '';
+        const key = raw ? canonicalizeProviderKey(raw) : '';
+        if (!key) {
+            return;
+        }
+        const authTypeRaw = typeof config.authType === 'string' ? config.authType.trim().toLowerCase() : '';
+        const authType = authTypeRaw === 'apikey' ? 'apikey' : authTypeRaw === 'oauth' ? 'oauth' : 'unknown';
+        const staticConfig = {
+            ...(typeof config.priorityTier === 'number' && Number.isFinite(config.priorityTier)
+                ? { priorityTier: config.priorityTier }
+                : {}),
+            authType
+        };
+        this.staticConfigs.set(key, staticConfig);
+        const nowMs = Date.now();
+        const existing = this.quotaStates.get(key);
+        if (existing) {
+            const merged = {
+                ...existing,
+                ...(staticConfig.authType ? { authType: staticConfig.authType } : {}),
+                ...(typeof staticConfig.priorityTier === 'number' ? { priorityTier: staticConfig.priorityTier } : {})
+            };
+            this.quotaStates.set(key, merged);
+            this.schedulePersist(nowMs);
+            return;
+        }
+        const initial = createInitialQuotaState(key, staticConfig, nowMs);
+        const isAntigravity = key.toLowerCase().startsWith('antigravity.');
+        if (isAntigravity && authType === 'oauth') {
+            this.quotaStates.set(key, {
+                ...initial,
+                inPool: false,
+                reason: 'cooldown',
+                cooldownUntil: null
+            });
+        }
+        else {
+            this.quotaStates.set(key, initial);
+        }
+        this.schedulePersist(nowMs);
+    }
+    async handleProviderErrorEvent(event) {
+        await handleProviderQuotaErrorEvent({
+            quotaStates: this.quotaStates,
+            staticConfigs: this.staticConfigs,
+            quotaRoutingEnabled: this.quotaRoutingEnabled,
+            modelBackoff: this.modelBackoff,
+            schedulePersist: (nowMs) => this.schedulePersist(nowMs),
+            toSnapshotObject: () => this.toSnapshotObject()
+        }, event);
+    }
+    async runMaintenanceTick() {
+        if (!this.quotaStates.size) {
+            return;
+        }
+        const nowMs = Date.now();
+        const updated = new Map();
+        for (const [providerKey, state] of this.quotaStates.entries()) {
+            const next = tickQuotaStateTime(state, nowMs);
+            updated.set(providerKey, next);
+        }
+        this.quotaStates = updated;
+        try {
+            await saveProviderQuotaSnapshot(this.toSnapshotObject(), new Date(nowMs));
+        }
+        catch {
+            // ignore persistence errors
+        }
+    }
+    schedulePersist(_nowMs) {
+        if (this.persistTimer) {
+            return;
+        }
+        const debounceMs = readPositiveNumberFromEnv('ROUTECODEX_QUOTA_PERSIST_DEBOUNCE_MS', 5_000);
+        this.persistTimer = setTimeout(() => {
+            this.persistTimer = null;
+            void saveProviderQuotaSnapshot(this.toSnapshotObject(), new Date()).catch(() => {
+                // ignore persistence errors
+            });
+        }, debounceMs);
+    }
+    toSnapshotObject() {
+        const result = {};
+        for (const [key, state] of this.quotaStates.entries()) {
+            result[key] = state;
+        }
+        return result;
+    }
+    getQuotaView() {
+        if (!this.quotaRoutingEnabled) {
+            return () => null;
+        }
+        return (providerKey) => {
+            const raw = typeof providerKey === 'string' ? providerKey.trim() : '';
+            const key = raw ? canonicalizeProviderKey(raw) : '';
+            if (!key) {
+                return null;
+            }
+            const state = this.quotaStates.get(key);
+            if (!state) {
+                return null;
+            }
+            const nowMs = Date.now();
+            const normalized = tickQuotaStateTime(state, nowMs);
+            if (normalized !== state) {
+                this.quotaStates.set(key, normalized);
+                this.schedulePersist(nowMs);
+            }
+            return buildQuotaViewEntry({
+                state: normalized,
+                nowMs,
+                modelBackoff: this.modelBackoff,
+                errorPriorityWindowMs: ERROR_PRIORITY_WINDOW_MS
+            });
+        };
+    }
+    getQuotaViewReadOnly() {
+        if (!this.quotaRoutingEnabled) {
+            return () => null;
+        }
+        return (providerKey) => {
+            const raw = typeof providerKey === 'string' ? providerKey.trim() : '';
+            const key = raw ? canonicalizeProviderKey(raw) : '';
+            if (!key) {
+                return null;
+            }
+            const state = this.quotaStates.get(key);
+            if (!state) {
+                return null;
+            }
+            const nowMs = Date.now();
+            const effective = tickQuotaStateTime(state, nowMs);
+            return buildQuotaViewEntry({
+                state: effective,
+                nowMs,
+                modelBackoff: this.modelBackoff,
+                errorPriorityWindowMs: ERROR_PRIORITY_WINDOW_MS
+            });
+        };
+    }
+}
+function readPositiveNumberFromEnv(name, fallback) {
+    const raw = process.env[name];
+    if (!raw) {
+        return fallback;
+    }
+    const parsed = Number(raw);
+    if (!Number.isFinite(parsed) || parsed <= 0) {
+        return fallback;
+    }
+    return parsed;
+}
+//# sourceMappingURL=provider-quota-daemon.js.map