@timo972/cc-router 0.10.1 → 0.11.0

package/dist/proxy/server.js

@@ -4,12 +4,14 @@ import { ServerResponse } from "http";
 import { timingSafeEqual } from "crypto";
 import { TokenPool } from "./token-pool.js";
 import { needsRefresh, refreshAccountIfCurrent, saveAccounts, startRefreshLoop } from "./token-refresher.js";
-import { loadAccounts, loadOpenAIAccounts, saveOpenAIAccountsToPath, accountsFileExists, readAccountsFromPath, readConfig, writeConfig, getProxyRequestTimeoutMs, migrateLegacyAccountProviders, setProviderAccountsEnabled } from "../config/manager.js";
+import { loadAccounts, loadOpenAIAccounts, saveOpenAIAccountsToPath, accountsFileExists, readAccountsFromPath, readConfig, writeConfig, getAutoFailoverEnabled, getProxyRequestTimeoutMs, migrateLegacyAccountProviders, setProviderAccountsEnabled } from "../config/manager.js";
 import { checkForUpdate, performUpdate, restartSelf, printUpdateBanner, getCurrentVersion } from "../utils/self-update.js";
 import { trackEvent, startHeartbeat } from "../utils/telemetry.js";
 import { loadTelemetryState } from "../config/telemetry.js";
 import { logRoute, logError, logStartup } from "./logger.js";
 import { createLocalRoutingErrorLog, stats } from "./stats.js";
+import { applyRateLimitHeaders } from "../providers/anthropic/rate-limit-headers.js";
+import { mountAnthropicMessagesRoute, withOAuthBeta } from "./anthropic-messages-route.js";
 import { PROXY_PORT, LITELLM_URL, ACCOUNTS_PATH } from "../config/paths.js";
 import { writePid, removePid, managesPidFile } from "../daemon/pid.js";
 import { applyOpenAIAccountPatch, validateAccountPatchBody } from "./account-patch.js";
@@ -26,14 +28,13 @@ import { SessionRouter } from "./session-router.js";
 import { createAnthropicProxy } from "./anthropic-proxy.js";
 import { AnthropicUsageRefresher } from "../providers/anthropic/usage-refresher.js";
 import { OpenAIUsageRefresher } from "../providers/openai/usage-fetch.js";
-import { createAnthropicUsageCapture } from "./usage-capture.js";
+import { attachAnthropicResponseCapture } from "./anthropic-response-capture.js";
 import { canUseExtraUsage } from "../providers/anthropic/usage.js";
 import { applyUpstreamFailureRoutingDetailed, reconcileAmbiguousRateLimitCooldown, routeFailureDetails, routeReasonDetails, } from "./lease-lifecycle.js";
 import { persistProviderEnabledState } from "./provider-routing.js";
 import { accountDeletionStatusCode, deleteAnthropicAccountTransaction, deleteOpenAIAccountTransaction, } from "./account-deletion.js";
 import { addOpenAIAccountTransaction } from "./account-add.js";
 import { createAnthropicRefreshMiddleware, createAnthropicRoutingMiddleware, } from "./anthropic-routing.js";
-import { createStreamLifecycleTracker } from "./stream-lifecycle.js";
 const zeroRoutingMetrics = () => ({
     inFlightRequests: 0,
     activeSessions: 0,
@@ -139,6 +140,8 @@ function publicUsageSnapshot(usage) {
         fetchStatus: usage.fetchStatus,
     };
 }
+// An unreported utilization surfaces as 0 here, which is what the dashboard
+// has always shown for it; only release decisions need the distinction.
 function publicWindow(window) {
     return { utilization: publicUtilization(window.utilization), resetAt: publicTimestamp(window.resetAt) };
 }
@@ -277,57 +280,9 @@ function providerStatus(accounts) {
         enabled: accounts.filter(a => a.enabled !== false).length,
     };
 }
-// Mutates entry and updates aggregate counters with token usage from Anthropic's
-// response. Called asynchronously after the log entry is already stored,
-// so the dashboard picks up the values on the next poll.
-function applyInputUsage(entry, usage) {
-    entry.cacheReadTokens = usage["cache_read_input_tokens"] ?? 0;
-    entry.cacheCreationTokens = usage["cache_creation_input_tokens"] ?? 0;
-    entry.inputTokens = usage["input_tokens"] ?? 0;
-    stats.totalCacheReadTokens += entry.cacheReadTokens;
-    stats.totalCacheCreationTokens += entry.cacheCreationTokens;
-    stats.totalInputTokens += entry.inputTokens;
-}
-function applyOutputUsage(entry, usage) {
-    entry.outputTokens = usage["output_tokens"] ?? 0;
-    stats.totalOutputTokens += entry.outputTokens;
-}
-// ─── Rate limit header extraction ──────────────────────────────────────────
-function inferPlan(requestsLimit) {
-    if (requestsLimit <= 0)
-        return "";
-    if (requestsLimit <= 100)
-        return "Pro";
-    if (requestsLimit <= 500)
-        return "Max 5x";
-    return "Max 20x";
-}
-function extractRateLimits(headers) {
-    const h = (name) => String(headers[name] ?? "");
-    const status = h("anthropic-ratelimit-unified-status");
-    if (!status)
-        return null; // No unified headers in this response
-    const requestsLimit = parseInt(h("anthropic-ratelimit-requests-limit"), 10) || 0;
-    return {
-        status: status === "rate_limited" ? "rate_limited" : "allowed",
-        fiveHourUtil: parseFloat(h("anthropic-ratelimit-unified-5h-utilization")) || 0,
-        fiveHourReset: parseInt(h("anthropic-ratelimit-unified-5h-reset"), 10) || 0,
-        sevenDayUtil: parseFloat(h("anthropic-ratelimit-unified-7d-utilization")) || 0,
-        sevenDayReset: parseInt(h("anthropic-ratelimit-unified-7d-reset"), 10) || 0,
-        claim: h("anthropic-ratelimit-unified-representative-claim"),
-        plan: inferPlan(requestsLimit),
-        requestsLimit,
-        lastUpdated: Date.now(),
-    };
-}
-/** Apply upstream rate-limit headers without discarding the usage snapshot. */
-export function applyRateLimitHeaders(account, headers) {
-    const rateLimits = extractRateLimits(headers);
-    if (!rateLimits)
-        return false;
-    account.rateLimits = { ...account.rateLimits, ...rateLimits };
-    return true;
-}
+// Re-exported so existing importers keep working; the implementation moved to
+// providers/anthropic so both Anthropic transports share it.
+export { applyRateLimitHeaders } from "../providers/anthropic/rate-limit-headers.js";
 /**
  * Build the single function through which this server writes OpenAI accounts.
  *
@@ -442,6 +397,13 @@ export async function startServer(opts = {}) {
     openAIUsageRefresher.start();
     const app = express();
     const proxyRequestTimeoutMs = getProxyRequestTimeoutMs();
+    // Router-side 429 failover / 5xx retry is on by default; `"autoFailover":
+    // false` in config.json opts out for anyone who cannot work with the
+    // trade-off (a committed retry abandons the original failure response).
+    // A single-attempt budget IS the off switch: both transports then relay
+    // every upstream failure unchanged, exactly as before the feature existed.
+    const autoFailover = getAutoFailoverEnabled();
+    const upstreamAttempts = autoFailover ? {} : { maxAttempts: 1 };
     // ─── Proxy auth middleware ─────────────────────────────────────────────────
     // If a proxySecret is configured, all requests must present it as EITHER
     //   "Authorization: Bearer <secret>" (Claude Code CLI, HTTP clients)
@@ -923,6 +885,7 @@ export async function startServer(opts = {}) {
         prepareOpenAIAccount: (account) => prepareOpenAIAccountForRequest(account, openAIAccounts, persistOpenAIAccounts),
         modelRouting,
         onUpstreamAuthFailure: onOpenAIUpstreamAuthFailure,
+        ...upstreamAttempts,
     });
     mountMessagesCrossProviderRoute(app, {
         openAIRouter,
@@ -930,6 +893,58 @@ export async function startServer(opts = {}) {
         prepareOpenAIAccount: (account) => prepareOpenAIAccountForRequest(account, openAIAccounts, persistOpenAIAccounts),
         modelRouting,
         onUpstreamAuthFailure: onOpenAIUpstreamAuthFailure,
+        ...upstreamAttempts,
+    });
+    // Shared between the retrying /v1/messages route and the generic /v1 chain
+    // so a locally rejected request is reported identically on both.
+    const onAnthropicEmptyPool = (err, _req, res) => {
+        stats.totalErrors++;
+        logError("proxy", 503, err.message);
+        res.status(503).json({
+            type: "error",
+            error: { type: "no_accounts", message: err.message },
+        });
+    };
+    const onAnthropicNoEligibleAccount = (err, req) => {
+        stats.totalErrors++;
+        const entry = createLocalRoutingErrorLog(err.reason, req._ccRouteContext?.modelFamily);
+        stats.addLog(entry);
+        logError(entry.accountId, entry.statusCode ?? 0, entry.details ?? "no-eligible");
+    };
+    const onAnthropicRefreshFailure = (account) => {
+        stats.totalErrors++;
+        logError(account.id, 401, "Token refresh failed");
+    };
+    // Claude-bound POST /v1/messages goes through its own transport with
+    // router-side 429 failover and 5xx retry; every other /v1 endpoint stays on
+    // the generic byte-transparent proxy below.
+    mountAnthropicMessagesRoute(app, {
+        target,
+        timeoutMs: proxyRequestTimeoutMs,
+        pool,
+        sessionRouter,
+        ...upstreamAttempts,
+        needsRefresh,
+        refresh: account => refreshAccountIfCurrent(account, pool),
+        onRefreshFailure: onAnthropicRefreshFailure,
+        onEmptyPool: onAnthropicEmptyPool,
+        onNoEligibleAccount: onAnthropicNoEligibleAccount,
+        // A relayed 401 means the token is stale — refresh in the background so
+        // the next request succeeds without making this client wait on it.
+        onUpstream401: account => {
+            void refreshAccountIfCurrent(account, pool).catch(console.error);
+        },
+        // Refresh in the background to narrow only ambiguity-owned global state
+        // when fresh usage proves a requested-model exhaustion.
+        onRateLimited: (route, ambiguousCooldownToken) => {
+            queueMicrotask(() => {
+                void usageRefresher.refreshAfterCurrent(route.account).then(result => {
+                    if (result.ok) {
+                        reconcileAmbiguousRateLimitCooldown(route, pool, ambiguousCooldownToken);
+                    }
+                });
+            });
+        },
     });
     // ─── Proxy middleware ──────────────────────────────────────────────────────
     // IMPORTANT: selfHandleResponse must be false (default) for SSE streaming to
@@ -952,15 +967,9 @@ export async function startServer(opts = {}) {
                 // CRITICAL: api.anthropic.com requires the "oauth-2025-04-20" beta flag to
                 // accept OAuth tokens (sk-ant-oat01-*). Without it the request is rejected
                 // with "OAuth authentication is currently not supported."
-                // APPEND — do NOT replace — so existing betas (tools, computer-use, etc.) are preserved.
-                const existingBeta = proxyReq.getHeader("anthropic-beta");
-                const betas = existingBeta
-                    ? String(existingBeta).split(",").map(b => b.trim()).filter(Boolean)
-                    : [];
-                if (!betas.includes("oauth-2025-04-20")) {
-                    betas.push("oauth-2025-04-20");
-                    proxyReq.setHeader("anthropic-beta", betas.join(","));
-                }
+                // APPEND — do NOT replace — so existing betas (tools, computer-use, etc.)
+                // are preserved. Shared with the retrying /v1/messages transport.
+                proxyReq.setHeader("anthropic-beta", withOAuthBeta(proxyReq.getHeader("anthropic-beta")));
                 // All other headers are forwarded automatically by http-proxy-middleware:
                 //   anthropic-version         — required by Anthropic API
                 //   X-Claude-Code-Session-Id  — session aggregation header sent by Claude Code
@@ -1061,34 +1070,9 @@ export async function startServer(opts = {}) {
                 const entry = pendingLog;
                 stats.addLog(entry);
                 // ── Capture token usage from Anthropic response body ─────────────────
-                // SSE streams carry usage across two events:
-                //   message_start  → input_tokens, cache_read/creation_input_tokens
-                //   message_delta   → output_tokens
-                // Non-streaming JSON carries all fields in a single usage object.
-                // The proxy is byte-transparent and the client's accept-encoding makes
-                // upstream compress, so the capture decompresses its own copy of the
-                // stream (see usage-capture.ts) — previously compressed responses were
-                // skipped, which in practice was EVERY response: no cache rate or
-                // token counts ever appeared on Anthropic activity rows.
-                const contentType = String(proxyRes.headers["content-type"] ?? "");
-                const encoding = String(proxyRes.headers["content-encoding"] ?? "");
-                const isCompressed = /gzip|br|deflate/.test(encoding);
-                const streamTracker = createStreamLifecycleTracker(req._startTime ?? Date.now(), !isCompressed && contentType.includes("text/event-stream"));
-                entry.streamLifecycle = streamTracker.state;
-                streamTracker.attach(proxyRes, response);
-                proxyRes.on("data", (chunk) => streamTracker.observeChunk(chunk));
-                const usageCapture = createAnthropicUsageCapture({
-                    contentType,
-                    contentEncoding: encoding,
-                    // Mutates the already-logged entry in place; the dashboard picks the
-                    // values up on its next poll.
-                    onInputUsage: (usage) => applyInputUsage(entry, usage),
-                    onOutputUsage: (usage) => applyOutputUsage(entry, usage),
-                });
-                if (usageCapture) {
-                    proxyRes.on("data", (chunk) => usageCapture.write(chunk));
-                    proxyRes.on("end", () => usageCapture.end());
-                }
+                // Passive stream-lifecycle + token-usage taps, shared with the
+                // retrying /v1/messages transport (see anthropic-response-capture.ts).
+                attachAnthropicResponseCapture(proxyRes, response, entry, req._startTime ?? Date.now());
             },
             error: (err, _req, res) => {
                 const request = _req;
@@ -1125,27 +1109,12 @@ export async function startServer(opts = {}) {
     // and breaks SSE streaming passthrough.
     app.use("/v1", createAnthropicRoutingMiddleware({
         sessionRouter,
-        onEmptyPool: (err, _req, res) => {
-            stats.totalErrors++;
-            logError("proxy", 503, err.message);
-            res.status(503).json({
-                type: "error",
-                error: { type: "no_accounts", message: err.message },
-            });
-        },
-        onNoEligibleAccount: (err, req) => {
-            stats.totalErrors++;
-            const entry = createLocalRoutingErrorLog(err.reason, req._ccRouteContext?.modelFamily);
-            stats.addLog(entry);
-            logError(entry.accountId, entry.statusCode ?? 0, entry.details ?? "no-eligible");
-        },
+        onEmptyPool: onAnthropicEmptyPool,
+        onNoEligibleAccount: onAnthropicNoEligibleAccount,
     }), createAnthropicRefreshMiddleware({
         needsRefresh,
         refresh: account => refreshAccountIfCurrent(account, pool),
-        onRefreshFailure: (account) => {
-            stats.totalErrors++;
-            logError(account.id, 401, "Token refresh failed");
-        },
+        onRefreshFailure: onAnthropicRefreshFailure,
     }), (req, _res, next) => {
         const route = req._ccRoute;
         const account = route.account;
@@ -1258,6 +1227,9 @@ export async function startServer(opts = {}) {
         console.log(autoUpdate
             ? chalk.gray("  Auto-update: enabled (patch/minor)")
             : chalk.gray("  Auto-update: off (notify-only) — run 'cc-router update' to install"));
+        console.log(autoFailover
+            ? chalk.gray("  Auto-failover: on — 429/5xx retried across accounts before the first relayed byte")
+            : chalk.gray("  Auto-failover: off — upstream failures pass through; clients own retries"));
         // Anonymous telemetry — fire-and-forget, never blocks proxy startup.
         try {
             const telemetryState = loadTelemetryState();

package/dist/proxy/stats.js

@@ -52,14 +52,33 @@ class ProxyStats {
 }
 // Singleton — shared across server and health endpoint
 export const stats = new ProxyStats();
+/**
+ * Record Anthropic input-side usage (message_start, or a non-streaming JSON
+ * body) on both the request's log entry and the running totals. Mutates an
+ * entry that is typically already stored — the dashboard picks the values up
+ * on its next poll.
+ */
+export function applyAnthropicInputUsage(entry, usage) {
+    entry.cacheReadTokens = usage["cache_read_input_tokens"] ?? 0;
+    entry.cacheCreationTokens = usage["cache_creation_input_tokens"] ?? 0;
+    entry.inputTokens = usage["input_tokens"] ?? 0;
+    stats.totalCacheReadTokens += entry.cacheReadTokens;
+    stats.totalCacheCreationTokens += entry.cacheCreationTokens;
+    stats.totalInputTokens += entry.inputTokens;
+}
+/** Record Anthropic output-side usage (message_delta) — see input counterpart. */
+export function applyAnthropicOutputUsage(entry, usage) {
+    entry.outputTokens = usage["output_tokens"] ?? 0;
+    stats.totalOutputTokens += entry.outputTokens;
+}
 /** Record Codex token usage on both the request's log entry and the running totals. */
 export function applyCodexUsage(entry, usage) {
     if (!usage)
         return;
-    entry.inputTokens = usage.inputTokens;
+    entry.inputTokens = Math.max(0, usage.inputTokens - usage.cachedInputTokens);
     entry.outputTokens = usage.outputTokens;
     entry.cacheReadTokens = usage.cachedInputTokens;
-    stats.totalInputTokens += usage.inputTokens;
+    stats.totalInputTokens += entry.inputTokens;
     stats.totalOutputTokens += usage.outputTokens;
     stats.totalCacheReadTokens += usage.cachedInputTokens;
 }