@juspay/neurolink 12.7.3 → 12.7.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +2 -2
- package/dist/browser/neurolink.min.js +399 -399
- package/dist/cli/commands/auth.js +252 -125
- package/dist/cli/commands/proxy.js +110 -91
- package/dist/cli/factories/authCommandFactory.js +8 -11
- package/dist/proxy/codexAccountUsage.d.ts +1 -1
- package/dist/proxy/codexAccountUsage.js +13 -1
- package/dist/proxy/proxyActivity.js +12 -1
- package/dist/proxy/usageStats.d.ts +2 -2
- package/dist/proxy/usageStats.js +66 -10
- package/dist/server/routes/claudeProxyRoutes.d.ts +2 -1
- package/dist/server/routes/claudeProxyRoutes.js +62 -12
- package/dist/server/routes/codexProxyRoutes.d.ts +8 -2
- package/dist/server/routes/codexProxyRoutes.js +191 -58
- package/dist/types/cli.d.ts +37 -5
- package/dist/types/proxy.d.ts +37 -0
- package/package.json +1 -1
|
@@ -15,7 +15,7 @@ import { join } from "node:path";
|
|
|
15
15
|
import { Agent } from "undici";
|
|
16
16
|
import { buildStableClaudeCodeBillingHeader, CLAUDE_CLI_USER_AGENT, CLAUDE_CODE_OAUTH_BETAS, getOrCreateClaudeCodeIdentity, parseClaudeCodeUserId, } from "../../auth/anthropicOAuth.js";
|
|
17
17
|
import { clearAccountCooldown, loadAccountCooldowns, saveAccountCooldown, } from "../../proxy/accountCooldown.js";
|
|
18
|
-
import { anthropicAccountKeysEqual, ENV_ANTHROPIC_ACCOUNT_KEY, isAccountAllowed, LEGACY_ANTHROPIC_ACCOUNT_KEY, shouldLoadFallbackCredential, } from "../../proxy/accountSelection.js";
|
|
18
|
+
import { anthropicAccountKeysEqual, ENV_ANTHROPIC_ACCOUNT_KEY, isAccountAllowed, LEGACY_ANTHROPIC_ACCOUNT_KEY, normalizeAnthropicAccountKey, shouldLoadFallbackCredential, } from "../../proxy/accountSelection.js";
|
|
19
19
|
import { getUnifiedRateLimitStatus, isQuotaOverageAvailable, loadAccountQuotas, mergeQuotaSnapshot, modelFamilyToken, parseQuotaHeaders, saveAccountQuota, } from "../../proxy/accountQuota.js";
|
|
20
20
|
import { fetchAccountUsage, listAnthropicAccountsForUsage, usageToQuota, } from "../../proxy/accountUsage.js";
|
|
21
21
|
import { AccountQuotaRefreshCoordinator } from "../../proxy/accountQuotaRefreshCoordinator.js";
|
|
@@ -72,6 +72,30 @@ const BLOCKED_UPSTREAM_HEADERS = new Set([
|
|
|
72
72
|
]);
|
|
73
73
|
const PROXY_INTERNAL_ACCOUNT_LABEL = "proxy/internal";
|
|
74
74
|
const PROXY_INTERNAL_ACCOUNT_TYPE = "internal";
|
|
75
|
+
function resolveRequestLogAccountIdentity(accountLabel, accountType) {
|
|
76
|
+
if (!accountLabel) {
|
|
77
|
+
return {};
|
|
78
|
+
}
|
|
79
|
+
if (accountType === "codex-oauth") {
|
|
80
|
+
return {
|
|
81
|
+
accountKey: accountLabel.startsWith("codex:")
|
|
82
|
+
? accountLabel
|
|
83
|
+
: `codex:${accountLabel}`,
|
|
84
|
+
provider: "openai",
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
if (accountType === "oauth" ||
|
|
88
|
+
accountType === "api_key" ||
|
|
89
|
+
accountType === "passthrough") {
|
|
90
|
+
return {
|
|
91
|
+
accountKey: accountType === "passthrough"
|
|
92
|
+
? accountLabel
|
|
93
|
+
: normalizeAnthropicAccountKey(accountLabel),
|
|
94
|
+
provider: "anthropic",
|
|
95
|
+
};
|
|
96
|
+
}
|
|
97
|
+
return {};
|
|
98
|
+
}
|
|
75
99
|
// ---------------------------------------------------------------------------
|
|
76
100
|
// Module-level state
|
|
77
101
|
// ---------------------------------------------------------------------------
|
|
@@ -634,7 +658,8 @@ function reconcileCooldownFromQuota(state, quota, now, policy = overagePolicy) {
|
|
|
634
658
|
}
|
|
635
659
|
/**
|
|
636
660
|
* Seed each account's runtime quota from the persisted snapshots in
|
|
637
|
-
* ~/.neurolink/account-quotas.json (keyed by
|
|
661
|
+
* ~/.neurolink/account-quotas.json (keyed by provider-qualified account key).
|
|
662
|
+
* Runtime state is
|
|
638
663
|
* in-memory only, so without this the quota-aware ordering is blind after a
|
|
639
664
|
* proxy restart: all accounts tie, selection falls back to token-store
|
|
640
665
|
* enumeration order, and the first account served becomes self-reinforcing
|
|
@@ -652,8 +677,13 @@ async function seedRuntimeQuotasFromDisk(accounts) {
|
|
|
652
677
|
const now = Date.now();
|
|
653
678
|
for (const account of accounts) {
|
|
654
679
|
const state = getOrCreateRuntimeState(account.key);
|
|
655
|
-
if (!state.quota
|
|
656
|
-
|
|
680
|
+
if (!state.quota) {
|
|
681
|
+
// Before provider identity was introduced, Anthropic snapshots were
|
|
682
|
+
// written under the bare display label. Keep that reading as an
|
|
683
|
+
// upgrade bridge, but never write new data back under the ambiguous
|
|
684
|
+
// key: a Codex login can legitimately use the same email.
|
|
685
|
+
state.quota =
|
|
686
|
+
persistedQuotas[account.key] ?? persistedQuotas[account.label];
|
|
657
687
|
}
|
|
658
688
|
const persistedCooldown = persistedCooldowns[account.key];
|
|
659
689
|
if (persistedCooldown?.coolingUntil > now &&
|
|
@@ -708,7 +738,7 @@ async function applyAccountUsageResult(account, fetchResult, observedAt, prior)
|
|
|
708
738
|
// Non-fatal: the next successful response will reconcile again.
|
|
709
739
|
});
|
|
710
740
|
}
|
|
711
|
-
await saveAccountQuota(account.
|
|
741
|
+
await saveAccountQuota(account.key, quota).catch(() => {
|
|
712
742
|
// Non-fatal: quota persistence is best-effort.
|
|
713
743
|
});
|
|
714
744
|
return quota;
|
|
@@ -759,7 +789,11 @@ async function refreshAccountLimits(options = {}) {
|
|
|
759
789
|
key: account.key,
|
|
760
790
|
type: account.type,
|
|
761
791
|
status,
|
|
762
|
-
quota: quota ??
|
|
792
|
+
quota: quota ??
|
|
793
|
+
state?.quota ??
|
|
794
|
+
persisted[account.key] ??
|
|
795
|
+
persisted[account.label] ??
|
|
796
|
+
null,
|
|
763
797
|
};
|
|
764
798
|
if (error !== undefined) {
|
|
765
799
|
result.error = error;
|
|
@@ -815,7 +849,7 @@ async function refreshAccountLimits(options = {}) {
|
|
|
815
849
|
results[index] = buildResult(account, "error", null, fetchResult.error);
|
|
816
850
|
continue;
|
|
817
851
|
}
|
|
818
|
-
const quota = await applyAccountUsageResult(account, fetchResult, refresh.startedAt, persisted[account.label] ?? null);
|
|
852
|
+
const quota = await applyAccountUsageResult(account, fetchResult, refresh.startedAt, persisted[account.key] ?? persisted[account.label] ?? null);
|
|
819
853
|
if (!quota) {
|
|
820
854
|
results[index] = buildResult(account, "error", null, "usage payload had no recognizable limit windows");
|
|
821
855
|
continue;
|
|
@@ -2832,7 +2866,7 @@ async function auditCompleteShareHeartbeat(grant, reportedCoins) {
|
|
|
2832
2866
|
return { paused: false, detail: "no provisioned account recorded" };
|
|
2833
2867
|
}
|
|
2834
2868
|
const state = accountRuntimeState.get(`anthropic:${accountLabel}`);
|
|
2835
|
-
const stats = getAccountStats(accountLabel);
|
|
2869
|
+
const stats = getAccountStats(accountLabel, "oauth");
|
|
2836
2870
|
const { verdict, shouldPause } = await recordAuditObservation({
|
|
2837
2871
|
grantId: grant.id,
|
|
2838
2872
|
accountLabel,
|
|
@@ -3453,6 +3487,7 @@ async function executeClaudeCodexFallback(args) {
|
|
|
3453
3487
|
query: {},
|
|
3454
3488
|
params: {},
|
|
3455
3489
|
body: convertClaudeRequestToCodex(body, model),
|
|
3490
|
+
metadata: { ...ctx.metadata, "neurolink.codexFallback": true },
|
|
3456
3491
|
// Keep the child attribution isolated until its stream has passed
|
|
3457
3492
|
// validation. A failed Codex attempt must not look like a served request.
|
|
3458
3493
|
responseHeaders: {},
|
|
@@ -4059,7 +4094,7 @@ async function handleAnthropicSuccessfulResponse(args) {
|
|
|
4059
4094
|
// Non-fatal: the next successful response will reconcile again.
|
|
4060
4095
|
});
|
|
4061
4096
|
}
|
|
4062
|
-
saveAccountQuota(account.
|
|
4097
|
+
saveAccountQuota(account.key, quota).catch(() => {
|
|
4063
4098
|
// Non-fatal: quota persistence is best-effort
|
|
4064
4099
|
});
|
|
4065
4100
|
}
|
|
@@ -4725,7 +4760,7 @@ async function handleAnthropicSuccessfulNonStreamRetryResponse(args) {
|
|
|
4725
4760
|
// Non-fatal: the next successful response will reconcile again.
|
|
4726
4761
|
});
|
|
4727
4762
|
}
|
|
4728
|
-
saveAccountQuota(account.
|
|
4763
|
+
saveAccountQuota(account.key, retryQuota).catch((error) => {
|
|
4729
4764
|
logger.debug("[proxy] Failed to persist account quota after auth retry", {
|
|
4730
4765
|
account: account.label,
|
|
4731
4766
|
error: error instanceof Error ? error.message : String(error),
|
|
@@ -5016,7 +5051,7 @@ async function handleAnthropicAuthRetry(args) {
|
|
|
5016
5051
|
accountState.coolingReason = retryPlan.reason;
|
|
5017
5052
|
}
|
|
5018
5053
|
if (retryQuota429) {
|
|
5019
|
-
saveAccountQuota(account.
|
|
5054
|
+
saveAccountQuota(account.key, retryQuota429).catch(() => {
|
|
5020
5055
|
// Non-fatal: routing already has the in-memory snapshot.
|
|
5021
5056
|
});
|
|
5022
5057
|
}
|
|
@@ -5580,9 +5615,13 @@ function createClaudeRequestRuntimeContext(args) {
|
|
|
5580
5615
|
: status >= 400
|
|
5581
5616
|
? PROXY_INTERNAL_ACCOUNT_TYPE
|
|
5582
5617
|
: undefined;
|
|
5618
|
+
const finalAccountIdentity = resolveRequestLogAccountIdentity(finalAccountLabel, finalAccountType);
|
|
5583
5619
|
if (status >= 400) {
|
|
5584
5620
|
recordFinalError(status, finalAccountLabel, finalAccountType, {
|
|
5585
5621
|
requestId: ctx.requestId,
|
|
5622
|
+
...(finalAccountIdentity.accountKey
|
|
5623
|
+
? { accountKey: finalAccountIdentity.accountKey }
|
|
5624
|
+
: {}),
|
|
5586
5625
|
errorType,
|
|
5587
5626
|
terminalOutcome: errorType === "client_cancelled"
|
|
5588
5627
|
? "client_cancelled"
|
|
@@ -5606,7 +5645,13 @@ function createClaudeRequestRuntimeContext(args) {
|
|
|
5606
5645
|
stream: !!body.stream,
|
|
5607
5646
|
toolCount: Array.isArray(body.tools) ? body.tools.length : 0,
|
|
5608
5647
|
account: finalAccountLabel ?? "",
|
|
5648
|
+
...(finalAccountIdentity.accountKey
|
|
5649
|
+
? { accountKey: finalAccountIdentity.accountKey }
|
|
5650
|
+
: {}),
|
|
5609
5651
|
accountType: finalAccountType ?? "",
|
|
5652
|
+
...(finalAccountIdentity.provider
|
|
5653
|
+
? { provider: finalAccountIdentity.provider }
|
|
5654
|
+
: {}),
|
|
5610
5655
|
...buildClientAttribution(ctx.headers),
|
|
5611
5656
|
responseStatus: status,
|
|
5612
5657
|
responseTimeMs: Date.now() - requestStartTime,
|
|
@@ -5669,6 +5714,7 @@ function createClaudeRequestRuntimeContext(args) {
|
|
|
5669
5714
|
function createAnthropicAttemptLogger(args) {
|
|
5670
5715
|
const { ctx, body, toolCount, requestStart, tracer, account, attemptNumber } = args;
|
|
5671
5716
|
const attemptStartedAt = Date.now();
|
|
5717
|
+
const accountIdentity = resolveRequestLogAccountIdentity(account.label, account.type);
|
|
5672
5718
|
return (status, errorType, errorMessage, extra) => {
|
|
5673
5719
|
const attemptCompletedAt = Date.now();
|
|
5674
5720
|
const traceCtx = tracer?.getTraceContext();
|
|
@@ -5682,7 +5728,11 @@ function createAnthropicAttemptLogger(args) {
|
|
|
5682
5728
|
stream: !!body.stream,
|
|
5683
5729
|
toolCount,
|
|
5684
5730
|
account: account.label,
|
|
5731
|
+
accountKey: account.key,
|
|
5685
5732
|
accountType: account.type,
|
|
5733
|
+
...(accountIdentity.provider
|
|
5734
|
+
? { provider: accountIdentity.provider }
|
|
5735
|
+
: {}),
|
|
5686
5736
|
responseStatus: status,
|
|
5687
5737
|
responseTimeMs: attemptCompletedAt - requestStart,
|
|
5688
5738
|
attemptDurationMs: extra?.attemptDurationMs ?? attemptCompletedAt - attemptStartedAt,
|
|
@@ -6335,7 +6385,7 @@ async function handleAnthropicRoutedClaudeRequest(args) {
|
|
|
6335
6385
|
// Refresh the account's quota snapshot for proactive selection.
|
|
6336
6386
|
if (fetchResult.quota) {
|
|
6337
6387
|
accountState.quota = mergeQuotaSnapshot(accountState.quota, fetchResult.quota);
|
|
6338
|
-
saveAccountQuota(account.
|
|
6388
|
+
saveAccountQuota(account.key, fetchResult.quota).catch(() => {
|
|
6339
6389
|
// Non-fatal: routing already has the in-memory snapshot.
|
|
6340
6390
|
});
|
|
6341
6391
|
}
|
|
@@ -17,7 +17,12 @@
|
|
|
17
17
|
* never collides with anthropic entries) and does pre-commit rotation only, not
|
|
18
18
|
* the full transient-budget / admission machinery.
|
|
19
19
|
*/
|
|
20
|
-
import type {
|
|
20
|
+
import type { AccountQuota, CodexRefreshTokenStore, CodexRuntimeAccount, CodexTokenRefresher, RateLimitCoolingReason, RouteGroup, ServerContext } from "../../types/index.js";
|
|
21
|
+
declare function refreshCodexTokenOnceWithDependencies(key: string, refreshToken: string, store: CodexRefreshTokenStore, refresh: CodexTokenRefresher): Promise<{
|
|
22
|
+
accessToken: string;
|
|
23
|
+
refreshToken: string;
|
|
24
|
+
expiresAt?: number;
|
|
25
|
+
}>;
|
|
21
26
|
/** Refresh an account's token at most once at a time. */
|
|
22
27
|
declare function refreshCodexTokenOnce(key: string, refreshToken: string): Promise<{
|
|
23
28
|
accessToken: string;
|
|
@@ -44,7 +49,7 @@ declare function buildCodexUpstreamHeaders(clientHeaders: Record<string, string>
|
|
|
44
49
|
*/
|
|
45
50
|
declare function planCodexCooldown(quota: AccountQuota | null, retryAfterMs: number, now: number): {
|
|
46
51
|
coolingUntil: number;
|
|
47
|
-
reason:
|
|
52
|
+
reason: RateLimitCoolingReason;
|
|
48
53
|
};
|
|
49
54
|
/** Core pooled handler for POST /backend-api/codex/responses. */
|
|
50
55
|
export declare function handleCodexResponsesRequest(ctx: ServerContext): Promise<Response>;
|
|
@@ -61,6 +66,7 @@ export declare const __testHooks: {
|
|
|
61
66
|
buildCodexUpstreamHeaders: typeof buildCodexUpstreamHeaders;
|
|
62
67
|
planCodexCooldown: typeof planCodexCooldown;
|
|
63
68
|
refreshCodexTokenOnce: typeof refreshCodexTokenOnce;
|
|
69
|
+
refreshCodexTokenOnceWithDependencies: typeof refreshCodexTokenOnceWithDependencies;
|
|
64
70
|
codexRefreshInFlightSize: () => number;
|
|
65
71
|
};
|
|
66
72
|
export {};
|
|
@@ -24,8 +24,10 @@ import { loadAccountQuotas, saveAccountQuota, } from "../../proxy/accountQuota.j
|
|
|
24
24
|
import { createCodexUsageTap } from "../../proxy/codexUsage.js";
|
|
25
25
|
import { CODEX_ACCOUNT_PREFIX, parseCodexRateLimitHeaders, } from "../../proxy/codexAccountUsage.js";
|
|
26
26
|
import { buildClientAttribution } from "../../proxy/clientAttribution.js";
|
|
27
|
-
import {
|
|
27
|
+
import { trackProxyResponse } from "../../proxy/proxyActivity.js";
|
|
28
|
+
import { logRequest, logRequestAttempt } from "../../proxy/requestLogger.js";
|
|
28
29
|
import { parseRetryAfterMs } from "../../proxy/routingPolicy.js";
|
|
30
|
+
import { recordAttempt, recordAttemptError, recordFinalError, recordFinalSuccess, } from "../../proxy/usageStats.js";
|
|
29
31
|
import { sanitizeForLog } from "../../utils/logSanitize.js";
|
|
30
32
|
import { logger } from "../../utils/logger.js";
|
|
31
33
|
const CODEX_UPSTREAM_TIMEOUT_MS = 15 * 60 * 1000; // 15 min, matches Claude path
|
|
@@ -33,6 +35,32 @@ const DEFAULT_TRANSIENT_COOLDOWN_MS = 60_000;
|
|
|
33
35
|
const MAX_TRANSIENT_COOLDOWN_MS = 15 * 60 * 1000;
|
|
34
36
|
/** Brief park after a refresh attempt that never reached a verdict. */
|
|
35
37
|
const CODEX_AUTH_COOLDOWN_MS = 60_000;
|
|
38
|
+
const CODEX_ACCOUNT_TYPE = "codex-oauth";
|
|
39
|
+
const CODEX_FALLBACK_METADATA_KEY = "neurolink.codexFallback";
|
|
40
|
+
function getCodexTransportErrorCode(error) {
|
|
41
|
+
if (!error || typeof error !== "object") {
|
|
42
|
+
return undefined;
|
|
43
|
+
}
|
|
44
|
+
const directCode = error.code;
|
|
45
|
+
if (typeof directCode === "string") {
|
|
46
|
+
return directCode;
|
|
47
|
+
}
|
|
48
|
+
const cause = error.cause;
|
|
49
|
+
if (!cause || typeof cause !== "object") {
|
|
50
|
+
return undefined;
|
|
51
|
+
}
|
|
52
|
+
const causeCode = cause.code;
|
|
53
|
+
return typeof causeCode === "string" ? causeCode : undefined;
|
|
54
|
+
}
|
|
55
|
+
function codexTransportScope(error) {
|
|
56
|
+
const code = getCodexTransportErrorCode(error);
|
|
57
|
+
return code === "ENOTFOUND" || code === "EAI_AGAIN"
|
|
58
|
+
? "shared_provider_transport"
|
|
59
|
+
: "connection_transport";
|
|
60
|
+
}
|
|
61
|
+
function summarizeCodexUpstreamError(errorText, fallback) {
|
|
62
|
+
return sanitizeForLog(errorText).slice(0, 200) || fallback;
|
|
63
|
+
}
|
|
36
64
|
/**
|
|
37
65
|
* In-flight proactive refreshes, keyed by account.
|
|
38
66
|
*
|
|
@@ -43,8 +71,7 @@ const CODEX_AUTH_COOLDOWN_MS = 60_000;
|
|
|
43
71
|
* that is perfectly healthy.
|
|
44
72
|
*/
|
|
45
73
|
const codexRefreshInFlight = new Map();
|
|
46
|
-
|
|
47
|
-
async function refreshCodexTokenOnce(key, refreshToken) {
|
|
74
|
+
async function refreshCodexTokenOnceWithDependencies(key, refreshToken, store, refresh) {
|
|
48
75
|
const existing = codexRefreshInFlight.get(key);
|
|
49
76
|
if (existing) {
|
|
50
77
|
return existing;
|
|
@@ -54,20 +81,36 @@ async function refreshCodexTokenOnce(key, refreshToken) {
|
|
|
54
81
|
// request that captured the pool just before a previous refresh completed
|
|
55
82
|
// holds a token that has since been rotated; using it would spend a real
|
|
56
83
|
// attempt on a grant the server has already invalidated.
|
|
57
|
-
const latest = await
|
|
84
|
+
const latest = await store.peekTokens(key).catch(() => null);
|
|
58
85
|
const current = latest?.refreshToken ?? refreshToken;
|
|
59
|
-
const refreshed = await
|
|
60
|
-
|
|
86
|
+
const refreshed = await refresh(current);
|
|
87
|
+
const resolved = {
|
|
61
88
|
accessToken: refreshed.accessToken,
|
|
62
89
|
refreshToken: refreshed.refreshToken ?? current,
|
|
63
|
-
expiresAt: refreshed.expiresAt,
|
|
90
|
+
expiresAt: refreshed.expiresAt ?? latest?.expiresAt ?? Date.now() + 3_600_000,
|
|
64
91
|
};
|
|
92
|
+
// Hold the single-flight slot through persistence. Releasing it after the
|
|
93
|
+
// OAuth response but before this write lets a third request read the old
|
|
94
|
+
// rotating refresh token, receive an invalid_grant, and disable an account
|
|
95
|
+
// another request has already healed.
|
|
96
|
+
if (latest) {
|
|
97
|
+
await store.saveTokens(key, {
|
|
98
|
+
...resolved,
|
|
99
|
+
tokenType: "Bearer",
|
|
100
|
+
...(latest.scope ? { scope: latest.scope } : {}),
|
|
101
|
+
});
|
|
102
|
+
}
|
|
103
|
+
return resolved;
|
|
65
104
|
})().finally(() => {
|
|
66
105
|
codexRefreshInFlight.delete(key);
|
|
67
106
|
});
|
|
68
107
|
codexRefreshInFlight.set(key, pending);
|
|
69
108
|
return pending;
|
|
70
109
|
}
|
|
110
|
+
/** Refresh an account's token at most once at a time. */
|
|
111
|
+
async function refreshCodexTokenOnce(key, refreshToken) {
|
|
112
|
+
return refreshCodexTokenOnceWithDependencies(key, refreshToken, tokenStore, refreshCodexToken);
|
|
113
|
+
}
|
|
71
114
|
// Headers we never forward upstream (hop-by-hop, client creds, or things we
|
|
72
115
|
// re-derive). The client's own auth is replaced with the pooled account's.
|
|
73
116
|
const BLOCKED_UPSTREAM_HEADERS = new Set([
|
|
@@ -114,13 +157,6 @@ async function loadCodexProxyAccounts() {
|
|
|
114
157
|
const refreshed = await refreshCodexTokenOnce(key, tokens.refreshToken);
|
|
115
158
|
accessToken = refreshed.accessToken;
|
|
116
159
|
expiresAt = refreshed.expiresAt ?? expiresAt;
|
|
117
|
-
await tokenStore.saveTokens(key, {
|
|
118
|
-
accessToken,
|
|
119
|
-
refreshToken: refreshed.refreshToken,
|
|
120
|
-
expiresAt: expiresAt ?? Date.now() + 3_600_000,
|
|
121
|
-
tokenType: "Bearer",
|
|
122
|
-
scope: tokens.scope,
|
|
123
|
-
});
|
|
124
160
|
}
|
|
125
161
|
catch (error) {
|
|
126
162
|
// Keep the stale token; a 401 upstream will trigger rotation.
|
|
@@ -241,7 +277,11 @@ export async function handleCodexResponsesRequest(ctx) {
|
|
|
241
277
|
const model = typeof body.model === "string"
|
|
242
278
|
? body.model
|
|
243
279
|
: "-";
|
|
244
|
-
|
|
280
|
+
// A Codex call made as an inner Anthropic fallback is an upstream attempt,
|
|
281
|
+
// not an independently final client request. The parent fallback owns the
|
|
282
|
+
// final status and can still recover with a later provider.
|
|
283
|
+
const isFallbackRequest = ctx.metadata?.[CODEX_FALLBACK_METADATA_KEY] === true;
|
|
284
|
+
const writeFinalLog = (account, responseStatus, extra = {}) => logRequest({
|
|
245
285
|
timestamp: new Date().toISOString(),
|
|
246
286
|
requestId: ctx.requestId,
|
|
247
287
|
method: ctx.method,
|
|
@@ -251,16 +291,63 @@ export async function handleCodexResponsesRequest(ctx) {
|
|
|
251
291
|
toolCount: Array.isArray(body.tools)
|
|
252
292
|
? body.tools.length
|
|
253
293
|
: 0,
|
|
254
|
-
account,
|
|
255
|
-
|
|
294
|
+
account: account?.label ?? "",
|
|
295
|
+
...(account ? { accountKey: account.key } : {}),
|
|
296
|
+
accountType: account ? CODEX_ACCOUNT_TYPE : "",
|
|
297
|
+
// This is the cost provider. accountKey and the response header identify
|
|
298
|
+
// the actual Codex pool that supplied the credential.
|
|
299
|
+
provider: "openai",
|
|
256
300
|
...buildClientAttribution(ctx.headers),
|
|
257
301
|
responseStatus,
|
|
258
302
|
responseTimeMs: Date.now() - requestStartTime,
|
|
259
303
|
...extra,
|
|
260
304
|
});
|
|
305
|
+
let finalOutcomeRecorded = false;
|
|
306
|
+
const recordFinalOutcome = async (account, responseStatus, extra = {}) => {
|
|
307
|
+
if (isFallbackRequest || finalOutcomeRecorded) {
|
|
308
|
+
return;
|
|
309
|
+
}
|
|
310
|
+
finalOutcomeRecorded = true;
|
|
311
|
+
if (responseStatus >= 400) {
|
|
312
|
+
recordFinalError(responseStatus, account?.label, account ? CODEX_ACCOUNT_TYPE : undefined, {
|
|
313
|
+
requestId: ctx.requestId,
|
|
314
|
+
...(account ? { accountKey: account.key } : {}),
|
|
315
|
+
errorType: extra.errorType,
|
|
316
|
+
terminalOutcome: extra.terminalOutcome ?? "handler_error",
|
|
317
|
+
message: extra.errorMessage,
|
|
318
|
+
errorCode: extra.errorCode,
|
|
319
|
+
});
|
|
320
|
+
}
|
|
321
|
+
else {
|
|
322
|
+
recordFinalSuccess(account?.label, account ? CODEX_ACCOUNT_TYPE : undefined);
|
|
323
|
+
}
|
|
324
|
+
await writeFinalLog(account, responseStatus, extra);
|
|
325
|
+
};
|
|
326
|
+
const writeAttempt = (account, attempt, startedAt, responseStatus, extra = {}) => {
|
|
327
|
+
void logRequestAttempt({
|
|
328
|
+
timestamp: new Date().toISOString(),
|
|
329
|
+
requestId: ctx.requestId,
|
|
330
|
+
attempt,
|
|
331
|
+
method: ctx.method,
|
|
332
|
+
path: ctx.path,
|
|
333
|
+
model,
|
|
334
|
+
stream: true,
|
|
335
|
+
toolCount: Array.isArray(body.tools)
|
|
336
|
+
? body.tools.length
|
|
337
|
+
: 0,
|
|
338
|
+
account: account.label,
|
|
339
|
+
accountKey: account.key,
|
|
340
|
+
accountType: CODEX_ACCOUNT_TYPE,
|
|
341
|
+
provider: "openai",
|
|
342
|
+
responseStatus,
|
|
343
|
+
responseTimeMs: Date.now() - requestStartTime,
|
|
344
|
+
attemptDurationMs: Date.now() - startedAt,
|
|
345
|
+
...extra,
|
|
346
|
+
}).catch(() => undefined);
|
|
347
|
+
};
|
|
261
348
|
const accounts = await loadCodexProxyAccounts();
|
|
262
349
|
if (accounts.length === 0) {
|
|
263
|
-
await
|
|
350
|
+
await recordFinalOutcome(undefined, 401, {
|
|
264
351
|
errorType: "no_accounts",
|
|
265
352
|
errorMessage: "No Codex accounts",
|
|
266
353
|
});
|
|
@@ -280,7 +367,7 @@ export async function handleCodexResponsesRequest(ctx) {
|
|
|
280
367
|
const retryAfterSec = soonest
|
|
281
368
|
? Math.max(1, Math.ceil((soonest - now) / 1000))
|
|
282
369
|
: 60;
|
|
283
|
-
await
|
|
370
|
+
await recordFinalOutcome(undefined, 429, {
|
|
284
371
|
errorType: "all_accounts_cooling",
|
|
285
372
|
errorMessage: "All Codex accounts are rate-limited",
|
|
286
373
|
});
|
|
@@ -300,11 +387,15 @@ export async function handleCodexResponsesRequest(ctx) {
|
|
|
300
387
|
let attempt = 0;
|
|
301
388
|
let lastErrorMessage = "All Codex accounts failed";
|
|
302
389
|
let lastErrorStatus = 502;
|
|
390
|
+
let lastAttemptedAccount;
|
|
303
391
|
for (const account of eligible) {
|
|
304
|
-
attempt += 1;
|
|
305
392
|
let authRetried = false;
|
|
306
393
|
// Same-account loop only re-runs once, for a post-401 token refresh.
|
|
307
394
|
for (;;) {
|
|
395
|
+
attempt += 1;
|
|
396
|
+
const attemptStartedAt = Date.now();
|
|
397
|
+
lastAttemptedAccount = account;
|
|
398
|
+
recordAttempt(account.label, CODEX_ACCOUNT_TYPE);
|
|
308
399
|
let upstream;
|
|
309
400
|
try {
|
|
310
401
|
upstream = await fetch(CODEX_RESPONSES_URL, {
|
|
@@ -320,6 +411,17 @@ export async function handleCodexResponsesRequest(ctx) {
|
|
|
320
411
|
// can act on. Keep the detail in the log and return a fixed string, so
|
|
321
412
|
// internal topology never reaches the client.
|
|
322
413
|
logger.debug(`Codex upstream fetch failed (${account.label}): ${sanitizeForLog(error instanceof Error ? error.message : String(error))}`);
|
|
414
|
+
const errorMessage = summarizeCodexUpstreamError(error instanceof Error ? error.message : String(error), "Codex upstream request failed");
|
|
415
|
+
const errorCode = getCodexTransportErrorCode(error);
|
|
416
|
+
const transportScope = codexTransportScope(error);
|
|
417
|
+
recordAttemptError(account.label, CODEX_ACCOUNT_TYPE, 502);
|
|
418
|
+
writeAttempt(account, attempt, attemptStartedAt, 502, {
|
|
419
|
+
errorType: "network_error",
|
|
420
|
+
errorMessage,
|
|
421
|
+
...(errorCode ? { errorCode } : {}),
|
|
422
|
+
transportScope,
|
|
423
|
+
retryable: true,
|
|
424
|
+
});
|
|
323
425
|
lastErrorMessage = "Codex upstream request failed";
|
|
324
426
|
lastErrorStatus = 502;
|
|
325
427
|
break; // rotate to next account
|
|
@@ -336,50 +438,72 @@ export async function handleCodexResponsesRequest(ctx) {
|
|
|
336
438
|
clearAccountCooldown(account.key, account.expiredCooldownUntil).catch(() => undefined);
|
|
337
439
|
}
|
|
338
440
|
publishCodexHeaders(ctx, account, attempt, quota);
|
|
339
|
-
|
|
441
|
+
writeAttempt(account, attempt, attemptStartedAt, upstream.status);
|
|
340
442
|
const headers = {
|
|
341
443
|
"content-type": upstream.headers.get("content-type") ?? "text/event-stream",
|
|
342
444
|
"cache-control": "no-cache",
|
|
343
445
|
connection: "keep-alive",
|
|
344
446
|
...(ctx.responseHeaders ?? {}),
|
|
345
447
|
};
|
|
346
|
-
// Tap the relay for token usage. The log above is written first and
|
|
347
|
-
// unconditionally so a request is never lost when a client hangs up
|
|
348
|
-
// mid-stream; this emits a second record for the same requestId
|
|
349
|
-
// carrying the counts, which proxyAnalysis merges. If the stream shape
|
|
350
|
-
// is not recognised, usage resolves null and nothing extra is written —
|
|
351
|
-
// i.e. exactly the previous behaviour.
|
|
352
448
|
if (!upstream.body) {
|
|
449
|
+
await recordFinalOutcome(account, upstream.status, {
|
|
450
|
+
terminalOutcome: "bodyless",
|
|
451
|
+
});
|
|
353
452
|
return new Response(upstream.body, {
|
|
354
453
|
status: upstream.status,
|
|
355
454
|
headers,
|
|
356
455
|
});
|
|
357
456
|
}
|
|
358
457
|
const { stream: usageTap, usage: usageSeen } = createCodexUsageTap();
|
|
359
|
-
|
|
360
|
-
.then((usage) => {
|
|
361
|
-
if (!usage) {
|
|
362
|
-
return;
|
|
363
|
-
}
|
|
364
|
-
return writeLog(account.label, upstream.status, {
|
|
365
|
-
provider: "openai",
|
|
366
|
-
inputTokens: usage.inputTokens,
|
|
367
|
-
outputTokens: usage.outputTokens,
|
|
368
|
-
cacheReadTokens: usage.cacheReadTokens,
|
|
369
|
-
cacheCreationTokens: usage.cacheCreationTokens,
|
|
370
|
-
});
|
|
371
|
-
})
|
|
372
|
-
.catch(() => undefined);
|
|
373
|
-
return new Response(upstream.body.pipeThrough(usageTap), {
|
|
458
|
+
const relay = new Response(upstream.body.pipeThrough(usageTap), {
|
|
374
459
|
status: upstream.status,
|
|
375
460
|
headers,
|
|
376
461
|
});
|
|
462
|
+
return trackProxyResponse(relay, () => undefined, {
|
|
463
|
+
onTerminal: ({ outcome }) => {
|
|
464
|
+
void usageSeen
|
|
465
|
+
.then((usage) => {
|
|
466
|
+
const usageExtra = usage
|
|
467
|
+
? {
|
|
468
|
+
inputTokens: usage.inputTokens,
|
|
469
|
+
outputTokens: usage.outputTokens,
|
|
470
|
+
cacheReadTokens: usage.cacheReadTokens,
|
|
471
|
+
cacheCreationTokens: usage.cacheCreationTokens,
|
|
472
|
+
}
|
|
473
|
+
: {};
|
|
474
|
+
if (outcome === "completed" || outcome === "bodyless") {
|
|
475
|
+
return recordFinalOutcome(account, upstream.status, {
|
|
476
|
+
terminalOutcome: outcome,
|
|
477
|
+
...usageExtra,
|
|
478
|
+
});
|
|
479
|
+
}
|
|
480
|
+
return recordFinalOutcome(account, outcome === "client_cancelled" ? 499 : 502, {
|
|
481
|
+
errorType: outcome === "client_cancelled"
|
|
482
|
+
? "client_cancelled"
|
|
483
|
+
: "stream_error",
|
|
484
|
+
errorMessage: outcome === "client_cancelled"
|
|
485
|
+
? "Client cancelled Codex stream"
|
|
486
|
+
: "Codex upstream stream failed",
|
|
487
|
+
terminalOutcome: outcome,
|
|
488
|
+
...usageExtra,
|
|
489
|
+
});
|
|
490
|
+
})
|
|
491
|
+
.catch(() => undefined);
|
|
492
|
+
},
|
|
493
|
+
});
|
|
377
494
|
}
|
|
378
495
|
const errText = await upstream.text().catch(() => "");
|
|
379
496
|
// 401/403 → try a forced token refresh once, then rotate.
|
|
380
497
|
if ((upstream.status === 401 || upstream.status === 403) &&
|
|
381
498
|
!authRetried &&
|
|
382
499
|
account.refreshToken) {
|
|
500
|
+
const errorMessage = summarizeCodexUpstreamError(errText, "Codex authentication rejected upstream");
|
|
501
|
+
recordAttemptError(account.label, CODEX_ACCOUNT_TYPE, upstream.status);
|
|
502
|
+
writeAttempt(account, attempt, attemptStartedAt, upstream.status, {
|
|
503
|
+
errorType: "authentication_error",
|
|
504
|
+
errorMessage,
|
|
505
|
+
retryable: true,
|
|
506
|
+
});
|
|
383
507
|
authRetried = true;
|
|
384
508
|
const staleTokens = {
|
|
385
509
|
accessToken: account.token,
|
|
@@ -387,17 +511,11 @@ export async function handleCodexResponsesRequest(ctx) {
|
|
|
387
511
|
expiresAt: account.expiresAt ?? 0,
|
|
388
512
|
};
|
|
389
513
|
try {
|
|
390
|
-
const refreshed = await
|
|
514
|
+
const refreshed = await refreshCodexTokenOnce(account.key, account.refreshToken);
|
|
391
515
|
account.token = refreshed.accessToken;
|
|
392
516
|
account.refreshToken = refreshed.refreshToken ?? account.refreshToken;
|
|
393
517
|
account.expiresAt = refreshed.expiresAt ?? account.expiresAt;
|
|
394
518
|
account.accountId = resolveCodexAccountId(refreshed.accessToken);
|
|
395
|
-
await tokenStore.saveTokens(account.key, {
|
|
396
|
-
accessToken: account.token,
|
|
397
|
-
refreshToken: account.refreshToken,
|
|
398
|
-
expiresAt: account.expiresAt ?? Date.now() + 3_600_000,
|
|
399
|
-
tokenType: "Bearer",
|
|
400
|
-
});
|
|
401
519
|
continue; // retry same account with the fresh token
|
|
402
520
|
}
|
|
403
521
|
catch (error) {
|
|
@@ -432,6 +550,16 @@ export async function handleCodexResponsesRequest(ctx) {
|
|
|
432
550
|
const retryAfterMs = parseRetryAfterMs(upstream.headers.get("retry-after"));
|
|
433
551
|
const plan = planCodexCooldown(quota, retryAfterMs, Date.now());
|
|
434
552
|
await saveAccountCooldown(account.key, plan.coolingUntil, plan.reason).catch(() => undefined);
|
|
553
|
+
const rateLimitKind = plan.reason === "transient" ? "transient" : "quota";
|
|
554
|
+
const errorMessage = summarizeCodexUpstreamError(errText, "Codex account rate-limited");
|
|
555
|
+
recordAttemptError(account.label, CODEX_ACCOUNT_TYPE, upstream.status, rateLimitKind);
|
|
556
|
+
writeAttempt(account, attempt, attemptStartedAt, upstream.status, {
|
|
557
|
+
errorType: "rate_limit_error",
|
|
558
|
+
errorMessage,
|
|
559
|
+
retryable: true,
|
|
560
|
+
rateLimitKind,
|
|
561
|
+
cooldownReason: plan.reason,
|
|
562
|
+
});
|
|
435
563
|
lastErrorStatus = 429;
|
|
436
564
|
lastErrorMessage = "Codex account rate-limited";
|
|
437
565
|
break; // rotate
|
|
@@ -443,12 +571,22 @@ export async function handleCodexResponsesRequest(ctx) {
|
|
|
443
571
|
// of letting it stay first in line with unknown quota.
|
|
444
572
|
await saveAccountCooldown(account.key, Date.now() + CODEX_AUTH_COOLDOWN_MS, "auth").catch(() => undefined);
|
|
445
573
|
}
|
|
574
|
+
const errorMessage = summarizeCodexUpstreamError(errText, "Codex error");
|
|
575
|
+
const errorType = upstream.status === 401 || upstream.status === 403
|
|
576
|
+
? "authentication_error"
|
|
577
|
+
: "api_error";
|
|
578
|
+
recordAttemptError(account.label, CODEX_ACCOUNT_TYPE, upstream.status);
|
|
579
|
+
writeAttempt(account, attempt, attemptStartedAt, upstream.status, {
|
|
580
|
+
errorType,
|
|
581
|
+
errorMessage,
|
|
582
|
+
retryable: upstream.status >= 500,
|
|
583
|
+
});
|
|
446
584
|
lastErrorStatus = upstream.status >= 500 ? 502 : upstream.status;
|
|
447
|
-
lastErrorMessage =
|
|
585
|
+
lastErrorMessage = errorMessage;
|
|
448
586
|
break; // rotate
|
|
449
587
|
}
|
|
450
588
|
}
|
|
451
|
-
await
|
|
589
|
+
await recordFinalOutcome(lastAttemptedAccount, lastErrorStatus, {
|
|
452
590
|
errorType: "all_accounts_failed",
|
|
453
591
|
errorMessage: lastErrorMessage,
|
|
454
592
|
});
|
|
@@ -529,18 +667,12 @@ async function handleCodexModelsRequest(ctx) {
|
|
|
529
667
|
if (!authRetried && account.refreshToken) {
|
|
530
668
|
authRetried = true;
|
|
531
669
|
try {
|
|
532
|
-
const refreshed = await
|
|
670
|
+
const refreshed = await refreshCodexTokenOnce(account.key, account.refreshToken);
|
|
533
671
|
account.token = refreshed.accessToken;
|
|
534
672
|
account.refreshToken =
|
|
535
673
|
refreshed.refreshToken ?? account.refreshToken;
|
|
536
674
|
account.expiresAt = refreshed.expiresAt ?? account.expiresAt;
|
|
537
675
|
account.accountId = resolveCodexAccountId(refreshed.accessToken);
|
|
538
|
-
await tokenStore.saveTokens(account.key, {
|
|
539
|
-
accessToken: account.token,
|
|
540
|
-
refreshToken: account.refreshToken,
|
|
541
|
-
expiresAt: account.expiresAt ?? Date.now() + 3_600_000,
|
|
542
|
-
tokenType: "Bearer",
|
|
543
|
-
});
|
|
544
676
|
continue; // retry this account with the fresh token
|
|
545
677
|
}
|
|
546
678
|
catch {
|
|
@@ -605,5 +737,6 @@ export const __testHooks = {
|
|
|
605
737
|
buildCodexUpstreamHeaders,
|
|
606
738
|
planCodexCooldown,
|
|
607
739
|
refreshCodexTokenOnce,
|
|
740
|
+
refreshCodexTokenOnceWithDependencies,
|
|
608
741
|
codexRefreshInFlightSize: () => codexRefreshInFlight.size,
|
|
609
742
|
};
|