@juspay/neurolink 12.7.3 → 12.7.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -15,7 +15,7 @@ import { join } from "node:path";
15
15
  import { Agent } from "undici";
16
16
  import { buildStableClaudeCodeBillingHeader, CLAUDE_CLI_USER_AGENT, CLAUDE_CODE_OAUTH_BETAS, getOrCreateClaudeCodeIdentity, parseClaudeCodeUserId, } from "../../auth/anthropicOAuth.js";
17
17
  import { clearAccountCooldown, loadAccountCooldowns, saveAccountCooldown, } from "../../proxy/accountCooldown.js";
18
- import { anthropicAccountKeysEqual, ENV_ANTHROPIC_ACCOUNT_KEY, isAccountAllowed, LEGACY_ANTHROPIC_ACCOUNT_KEY, shouldLoadFallbackCredential, } from "../../proxy/accountSelection.js";
18
+ import { anthropicAccountKeysEqual, ENV_ANTHROPIC_ACCOUNT_KEY, isAccountAllowed, LEGACY_ANTHROPIC_ACCOUNT_KEY, normalizeAnthropicAccountKey, shouldLoadFallbackCredential, } from "../../proxy/accountSelection.js";
19
19
  import { getUnifiedRateLimitStatus, isQuotaOverageAvailable, loadAccountQuotas, mergeQuotaSnapshot, modelFamilyToken, parseQuotaHeaders, saveAccountQuota, } from "../../proxy/accountQuota.js";
20
20
  import { fetchAccountUsage, listAnthropicAccountsForUsage, usageToQuota, } from "../../proxy/accountUsage.js";
21
21
  import { AccountQuotaRefreshCoordinator } from "../../proxy/accountQuotaRefreshCoordinator.js";
@@ -72,6 +72,30 @@ const BLOCKED_UPSTREAM_HEADERS = new Set([
72
72
  ]);
73
73
  const PROXY_INTERNAL_ACCOUNT_LABEL = "proxy/internal";
74
74
  const PROXY_INTERNAL_ACCOUNT_TYPE = "internal";
75
+ function resolveRequestLogAccountIdentity(accountLabel, accountType) {
76
+ if (!accountLabel) {
77
+ return {};
78
+ }
79
+ if (accountType === "codex-oauth") {
80
+ return {
81
+ accountKey: accountLabel.startsWith("codex:")
82
+ ? accountLabel
83
+ : `codex:${accountLabel}`,
84
+ provider: "openai",
85
+ };
86
+ }
87
+ if (accountType === "oauth" ||
88
+ accountType === "api_key" ||
89
+ accountType === "passthrough") {
90
+ return {
91
+ accountKey: accountType === "passthrough"
92
+ ? accountLabel
93
+ : normalizeAnthropicAccountKey(accountLabel),
94
+ provider: "anthropic",
95
+ };
96
+ }
97
+ return {};
98
+ }
75
99
  // ---------------------------------------------------------------------------
76
100
  // Module-level state
77
101
  // ---------------------------------------------------------------------------
@@ -634,7 +658,8 @@ function reconcileCooldownFromQuota(state, quota, now, policy = overagePolicy) {
634
658
  }
635
659
  /**
636
660
  * Seed each account's runtime quota from the persisted snapshots in
637
- * ~/.neurolink/account-quotas.json (keyed by label). Runtime state is
661
+ * ~/.neurolink/account-quotas.json (keyed by provider-qualified account key).
662
+ * Runtime state is
638
663
  * in-memory only, so without this the quota-aware ordering is blind after a
639
664
  * proxy restart: all accounts tie, selection falls back to token-store
640
665
  * enumeration order, and the first account served becomes self-reinforcing
@@ -652,8 +677,13 @@ async function seedRuntimeQuotasFromDisk(accounts) {
652
677
  const now = Date.now();
653
678
  for (const account of accounts) {
654
679
  const state = getOrCreateRuntimeState(account.key);
655
- if (!state.quota && persistedQuotas[account.label]) {
656
- state.quota = persistedQuotas[account.label];
680
+ if (!state.quota) {
681
+ // Before provider identity was introduced, Anthropic snapshots were
682
+ // written under the bare display label. Keep that reading as an
683
+ // upgrade bridge, but never write new data back under the ambiguous
684
+ // key: a Codex login can legitimately use the same email.
685
+ state.quota =
686
+ persistedQuotas[account.key] ?? persistedQuotas[account.label];
657
687
  }
658
688
  const persistedCooldown = persistedCooldowns[account.key];
659
689
  if (persistedCooldown?.coolingUntil > now &&
@@ -708,7 +738,7 @@ async function applyAccountUsageResult(account, fetchResult, observedAt, prior)
708
738
  // Non-fatal: the next successful response will reconcile again.
709
739
  });
710
740
  }
711
- await saveAccountQuota(account.label, quota).catch(() => {
741
+ await saveAccountQuota(account.key, quota).catch(() => {
712
742
  // Non-fatal: quota persistence is best-effort.
713
743
  });
714
744
  return quota;
@@ -759,7 +789,11 @@ async function refreshAccountLimits(options = {}) {
759
789
  key: account.key,
760
790
  type: account.type,
761
791
  status,
762
- quota: quota ?? state?.quota ?? persisted[account.label] ?? null,
792
+ quota: quota ??
793
+ state?.quota ??
794
+ persisted[account.key] ??
795
+ persisted[account.label] ??
796
+ null,
763
797
  };
764
798
  if (error !== undefined) {
765
799
  result.error = error;
@@ -815,7 +849,7 @@ async function refreshAccountLimits(options = {}) {
815
849
  results[index] = buildResult(account, "error", null, fetchResult.error);
816
850
  continue;
817
851
  }
818
- const quota = await applyAccountUsageResult(account, fetchResult, refresh.startedAt, persisted[account.label] ?? null);
852
+ const quota = await applyAccountUsageResult(account, fetchResult, refresh.startedAt, persisted[account.key] ?? persisted[account.label] ?? null);
819
853
  if (!quota) {
820
854
  results[index] = buildResult(account, "error", null, "usage payload had no recognizable limit windows");
821
855
  continue;
@@ -2832,7 +2866,7 @@ async function auditCompleteShareHeartbeat(grant, reportedCoins) {
2832
2866
  return { paused: false, detail: "no provisioned account recorded" };
2833
2867
  }
2834
2868
  const state = accountRuntimeState.get(`anthropic:${accountLabel}`);
2835
- const stats = getAccountStats(accountLabel);
2869
+ const stats = getAccountStats(accountLabel, "oauth");
2836
2870
  const { verdict, shouldPause } = await recordAuditObservation({
2837
2871
  grantId: grant.id,
2838
2872
  accountLabel,
@@ -3453,6 +3487,7 @@ async function executeClaudeCodexFallback(args) {
3453
3487
  query: {},
3454
3488
  params: {},
3455
3489
  body: convertClaudeRequestToCodex(body, model),
3490
+ metadata: { ...ctx.metadata, "neurolink.codexFallback": true },
3456
3491
  // Keep the child attribution isolated until its stream has passed
3457
3492
  // validation. A failed Codex attempt must not look like a served request.
3458
3493
  responseHeaders: {},
@@ -4059,7 +4094,7 @@ async function handleAnthropicSuccessfulResponse(args) {
4059
4094
  // Non-fatal: the next successful response will reconcile again.
4060
4095
  });
4061
4096
  }
4062
- saveAccountQuota(account.label, quota).catch(() => {
4097
+ saveAccountQuota(account.key, quota).catch(() => {
4063
4098
  // Non-fatal: quota persistence is best-effort
4064
4099
  });
4065
4100
  }
@@ -4725,7 +4760,7 @@ async function handleAnthropicSuccessfulNonStreamRetryResponse(args) {
4725
4760
  // Non-fatal: the next successful response will reconcile again.
4726
4761
  });
4727
4762
  }
4728
- saveAccountQuota(account.label, retryQuota).catch((error) => {
4763
+ saveAccountQuota(account.key, retryQuota).catch((error) => {
4729
4764
  logger.debug("[proxy] Failed to persist account quota after auth retry", {
4730
4765
  account: account.label,
4731
4766
  error: error instanceof Error ? error.message : String(error),
@@ -5016,7 +5051,7 @@ async function handleAnthropicAuthRetry(args) {
5016
5051
  accountState.coolingReason = retryPlan.reason;
5017
5052
  }
5018
5053
  if (retryQuota429) {
5019
- saveAccountQuota(account.label, retryQuota429).catch(() => {
5054
+ saveAccountQuota(account.key, retryQuota429).catch(() => {
5020
5055
  // Non-fatal: routing already has the in-memory snapshot.
5021
5056
  });
5022
5057
  }
@@ -5580,9 +5615,13 @@ function createClaudeRequestRuntimeContext(args) {
5580
5615
  : status >= 400
5581
5616
  ? PROXY_INTERNAL_ACCOUNT_TYPE
5582
5617
  : undefined;
5618
+ const finalAccountIdentity = resolveRequestLogAccountIdentity(finalAccountLabel, finalAccountType);
5583
5619
  if (status >= 400) {
5584
5620
  recordFinalError(status, finalAccountLabel, finalAccountType, {
5585
5621
  requestId: ctx.requestId,
5622
+ ...(finalAccountIdentity.accountKey
5623
+ ? { accountKey: finalAccountIdentity.accountKey }
5624
+ : {}),
5586
5625
  errorType,
5587
5626
  terminalOutcome: errorType === "client_cancelled"
5588
5627
  ? "client_cancelled"
@@ -5606,7 +5645,13 @@ function createClaudeRequestRuntimeContext(args) {
5606
5645
  stream: !!body.stream,
5607
5646
  toolCount: Array.isArray(body.tools) ? body.tools.length : 0,
5608
5647
  account: finalAccountLabel ?? "",
5648
+ ...(finalAccountIdentity.accountKey
5649
+ ? { accountKey: finalAccountIdentity.accountKey }
5650
+ : {}),
5609
5651
  accountType: finalAccountType ?? "",
5652
+ ...(finalAccountIdentity.provider
5653
+ ? { provider: finalAccountIdentity.provider }
5654
+ : {}),
5610
5655
  ...buildClientAttribution(ctx.headers),
5611
5656
  responseStatus: status,
5612
5657
  responseTimeMs: Date.now() - requestStartTime,
@@ -5669,6 +5714,7 @@ function createClaudeRequestRuntimeContext(args) {
5669
5714
  function createAnthropicAttemptLogger(args) {
5670
5715
  const { ctx, body, toolCount, requestStart, tracer, account, attemptNumber } = args;
5671
5716
  const attemptStartedAt = Date.now();
5717
+ const accountIdentity = resolveRequestLogAccountIdentity(account.label, account.type);
5672
5718
  return (status, errorType, errorMessage, extra) => {
5673
5719
  const attemptCompletedAt = Date.now();
5674
5720
  const traceCtx = tracer?.getTraceContext();
@@ -5682,7 +5728,11 @@ function createAnthropicAttemptLogger(args) {
5682
5728
  stream: !!body.stream,
5683
5729
  toolCount,
5684
5730
  account: account.label,
5731
+ accountKey: account.key,
5685
5732
  accountType: account.type,
5733
+ ...(accountIdentity.provider
5734
+ ? { provider: accountIdentity.provider }
5735
+ : {}),
5686
5736
  responseStatus: status,
5687
5737
  responseTimeMs: attemptCompletedAt - requestStart,
5688
5738
  attemptDurationMs: extra?.attemptDurationMs ?? attemptCompletedAt - attemptStartedAt,
@@ -6335,7 +6385,7 @@ async function handleAnthropicRoutedClaudeRequest(args) {
6335
6385
  // Refresh the account's quota snapshot for proactive selection.
6336
6386
  if (fetchResult.quota) {
6337
6387
  accountState.quota = mergeQuotaSnapshot(accountState.quota, fetchResult.quota);
6338
- saveAccountQuota(account.label, fetchResult.quota).catch(() => {
6388
+ saveAccountQuota(account.key, fetchResult.quota).catch(() => {
6339
6389
  // Non-fatal: routing already has the in-memory snapshot.
6340
6390
  });
6341
6391
  }
@@ -17,7 +17,12 @@
17
17
  * never collides with anthropic entries) and does pre-commit rotation only, not
18
18
  * the full transient-budget / admission machinery.
19
19
  */
20
- import type { AccountCoolingReason, AccountQuota, CodexRuntimeAccount, RouteGroup, ServerContext } from "../../types/index.js";
20
+ import type { AccountQuota, CodexRefreshTokenStore, CodexRuntimeAccount, CodexTokenRefresher, RateLimitCoolingReason, RouteGroup, ServerContext } from "../../types/index.js";
21
+ declare function refreshCodexTokenOnceWithDependencies(key: string, refreshToken: string, store: CodexRefreshTokenStore, refresh: CodexTokenRefresher): Promise<{
22
+ accessToken: string;
23
+ refreshToken: string;
24
+ expiresAt?: number;
25
+ }>;
21
26
  /** Refresh an account's token at most once at a time. */
22
27
  declare function refreshCodexTokenOnce(key: string, refreshToken: string): Promise<{
23
28
  accessToken: string;
@@ -44,7 +49,7 @@ declare function buildCodexUpstreamHeaders(clientHeaders: Record<string, string>
44
49
  */
45
50
  declare function planCodexCooldown(quota: AccountQuota | null, retryAfterMs: number, now: number): {
46
51
  coolingUntil: number;
47
- reason: AccountCoolingReason;
52
+ reason: RateLimitCoolingReason;
48
53
  };
49
54
  /** Core pooled handler for POST /backend-api/codex/responses. */
50
55
  export declare function handleCodexResponsesRequest(ctx: ServerContext): Promise<Response>;
@@ -61,6 +66,7 @@ export declare const __testHooks: {
61
66
  buildCodexUpstreamHeaders: typeof buildCodexUpstreamHeaders;
62
67
  planCodexCooldown: typeof planCodexCooldown;
63
68
  refreshCodexTokenOnce: typeof refreshCodexTokenOnce;
69
+ refreshCodexTokenOnceWithDependencies: typeof refreshCodexTokenOnceWithDependencies;
64
70
  codexRefreshInFlightSize: () => number;
65
71
  };
66
72
  export {};
@@ -24,8 +24,10 @@ import { loadAccountQuotas, saveAccountQuota, } from "../../proxy/accountQuota.j
24
24
  import { createCodexUsageTap } from "../../proxy/codexUsage.js";
25
25
  import { CODEX_ACCOUNT_PREFIX, parseCodexRateLimitHeaders, } from "../../proxy/codexAccountUsage.js";
26
26
  import { buildClientAttribution } from "../../proxy/clientAttribution.js";
27
- import { logRequest } from "../../proxy/requestLogger.js";
27
+ import { trackProxyResponse } from "../../proxy/proxyActivity.js";
28
+ import { logRequest, logRequestAttempt } from "../../proxy/requestLogger.js";
28
29
  import { parseRetryAfterMs } from "../../proxy/routingPolicy.js";
30
+ import { recordAttempt, recordAttemptError, recordFinalError, recordFinalSuccess, } from "../../proxy/usageStats.js";
29
31
  import { sanitizeForLog } from "../../utils/logSanitize.js";
30
32
  import { logger } from "../../utils/logger.js";
31
33
  const CODEX_UPSTREAM_TIMEOUT_MS = 15 * 60 * 1000; // 15 min, matches Claude path
@@ -33,6 +35,32 @@ const DEFAULT_TRANSIENT_COOLDOWN_MS = 60_000;
33
35
  const MAX_TRANSIENT_COOLDOWN_MS = 15 * 60 * 1000;
34
36
  /** Brief park after a refresh attempt that never reached a verdict. */
35
37
  const CODEX_AUTH_COOLDOWN_MS = 60_000;
38
+ const CODEX_ACCOUNT_TYPE = "codex-oauth";
39
+ const CODEX_FALLBACK_METADATA_KEY = "neurolink.codexFallback";
40
+ function getCodexTransportErrorCode(error) {
41
+ if (!error || typeof error !== "object") {
42
+ return undefined;
43
+ }
44
+ const directCode = error.code;
45
+ if (typeof directCode === "string") {
46
+ return directCode;
47
+ }
48
+ const cause = error.cause;
49
+ if (!cause || typeof cause !== "object") {
50
+ return undefined;
51
+ }
52
+ const causeCode = cause.code;
53
+ return typeof causeCode === "string" ? causeCode : undefined;
54
+ }
55
+ function codexTransportScope(error) {
56
+ const code = getCodexTransportErrorCode(error);
57
+ return code === "ENOTFOUND" || code === "EAI_AGAIN"
58
+ ? "shared_provider_transport"
59
+ : "connection_transport";
60
+ }
61
+ function summarizeCodexUpstreamError(errorText, fallback) {
62
+ return sanitizeForLog(errorText).slice(0, 200) || fallback;
63
+ }
36
64
  /**
37
65
  * In-flight proactive refreshes, keyed by account.
38
66
  *
@@ -43,8 +71,7 @@ const CODEX_AUTH_COOLDOWN_MS = 60_000;
43
71
  * that is perfectly healthy.
44
72
  */
45
73
  const codexRefreshInFlight = new Map();
46
- /** Refresh an account's token at most once at a time. */
47
- async function refreshCodexTokenOnce(key, refreshToken) {
74
+ async function refreshCodexTokenOnceWithDependencies(key, refreshToken, store, refresh) {
48
75
  const existing = codexRefreshInFlight.get(key);
49
76
  if (existing) {
50
77
  return existing;
@@ -54,20 +81,36 @@ async function refreshCodexTokenOnce(key, refreshToken) {
54
81
  // request that captured the pool just before a previous refresh completed
55
82
  // holds a token that has since been rotated; using it would spend a real
56
83
  // attempt on a grant the server has already invalidated.
57
- const latest = await tokenStore.peekTokens(key).catch(() => null);
84
+ const latest = await store.peekTokens(key).catch(() => null);
58
85
  const current = latest?.refreshToken ?? refreshToken;
59
- const refreshed = await refreshCodexToken(current);
60
- return {
86
+ const refreshed = await refresh(current);
87
+ const resolved = {
61
88
  accessToken: refreshed.accessToken,
62
89
  refreshToken: refreshed.refreshToken ?? current,
63
- expiresAt: refreshed.expiresAt,
90
+ expiresAt: refreshed.expiresAt ?? latest?.expiresAt ?? Date.now() + 3_600_000,
64
91
  };
92
+ // Hold the single-flight slot through persistence. Releasing it after the
93
+ // OAuth response but before this write lets a third request read the old
94
+ // rotating refresh token, receive an invalid_grant, and disable an account
95
+ // another request has already healed.
96
+ if (latest) {
97
+ await store.saveTokens(key, {
98
+ ...resolved,
99
+ tokenType: "Bearer",
100
+ ...(latest.scope ? { scope: latest.scope } : {}),
101
+ });
102
+ }
103
+ return resolved;
65
104
  })().finally(() => {
66
105
  codexRefreshInFlight.delete(key);
67
106
  });
68
107
  codexRefreshInFlight.set(key, pending);
69
108
  return pending;
70
109
  }
110
+ /** Refresh an account's token at most once at a time. */
111
+ async function refreshCodexTokenOnce(key, refreshToken) {
112
+ return refreshCodexTokenOnceWithDependencies(key, refreshToken, tokenStore, refreshCodexToken);
113
+ }
71
114
  // Headers we never forward upstream (hop-by-hop, client creds, or things we
72
115
  // re-derive). The client's own auth is replaced with the pooled account's.
73
116
  const BLOCKED_UPSTREAM_HEADERS = new Set([
@@ -114,13 +157,6 @@ async function loadCodexProxyAccounts() {
114
157
  const refreshed = await refreshCodexTokenOnce(key, tokens.refreshToken);
115
158
  accessToken = refreshed.accessToken;
116
159
  expiresAt = refreshed.expiresAt ?? expiresAt;
117
- await tokenStore.saveTokens(key, {
118
- accessToken,
119
- refreshToken: refreshed.refreshToken,
120
- expiresAt: expiresAt ?? Date.now() + 3_600_000,
121
- tokenType: "Bearer",
122
- scope: tokens.scope,
123
- });
124
160
  }
125
161
  catch (error) {
126
162
  // Keep the stale token; a 401 upstream will trigger rotation.
@@ -241,7 +277,11 @@ export async function handleCodexResponsesRequest(ctx) {
241
277
  const model = typeof body.model === "string"
242
278
  ? body.model
243
279
  : "-";
244
- const writeLog = (account, responseStatus, extra = {}) => logRequest({
280
+ // A Codex call made as an inner Anthropic fallback is an upstream attempt,
281
+ // not an independently final client request. The parent fallback owns the
282
+ // final status and can still recover with a later provider.
283
+ const isFallbackRequest = ctx.metadata?.[CODEX_FALLBACK_METADATA_KEY] === true;
284
+ const writeFinalLog = (account, responseStatus, extra = {}) => logRequest({
245
285
  timestamp: new Date().toISOString(),
246
286
  requestId: ctx.requestId,
247
287
  method: ctx.method,
@@ -251,16 +291,63 @@ export async function handleCodexResponsesRequest(ctx) {
251
291
  toolCount: Array.isArray(body.tools)
252
292
  ? body.tools.length
253
293
  : 0,
254
- account,
255
- accountType: "codex-oauth",
294
+ account: account?.label ?? "",
295
+ ...(account ? { accountKey: account.key } : {}),
296
+ accountType: account ? CODEX_ACCOUNT_TYPE : "",
297
+ // This is the cost provider. accountKey and the response header identify
298
+ // the actual Codex pool that supplied the credential.
299
+ provider: "openai",
256
300
  ...buildClientAttribution(ctx.headers),
257
301
  responseStatus,
258
302
  responseTimeMs: Date.now() - requestStartTime,
259
303
  ...extra,
260
304
  });
305
+ let finalOutcomeRecorded = false;
306
+ const recordFinalOutcome = async (account, responseStatus, extra = {}) => {
307
+ if (isFallbackRequest || finalOutcomeRecorded) {
308
+ return;
309
+ }
310
+ finalOutcomeRecorded = true;
311
+ if (responseStatus >= 400) {
312
+ recordFinalError(responseStatus, account?.label, account ? CODEX_ACCOUNT_TYPE : undefined, {
313
+ requestId: ctx.requestId,
314
+ ...(account ? { accountKey: account.key } : {}),
315
+ errorType: extra.errorType,
316
+ terminalOutcome: extra.terminalOutcome ?? "handler_error",
317
+ message: extra.errorMessage,
318
+ errorCode: extra.errorCode,
319
+ });
320
+ }
321
+ else {
322
+ recordFinalSuccess(account?.label, account ? CODEX_ACCOUNT_TYPE : undefined);
323
+ }
324
+ await writeFinalLog(account, responseStatus, extra);
325
+ };
326
+ const writeAttempt = (account, attempt, startedAt, responseStatus, extra = {}) => {
327
+ void logRequestAttempt({
328
+ timestamp: new Date().toISOString(),
329
+ requestId: ctx.requestId,
330
+ attempt,
331
+ method: ctx.method,
332
+ path: ctx.path,
333
+ model,
334
+ stream: true,
335
+ toolCount: Array.isArray(body.tools)
336
+ ? body.tools.length
337
+ : 0,
338
+ account: account.label,
339
+ accountKey: account.key,
340
+ accountType: CODEX_ACCOUNT_TYPE,
341
+ provider: "openai",
342
+ responseStatus,
343
+ responseTimeMs: Date.now() - requestStartTime,
344
+ attemptDurationMs: Date.now() - startedAt,
345
+ ...extra,
346
+ }).catch(() => undefined);
347
+ };
261
348
  const accounts = await loadCodexProxyAccounts();
262
349
  if (accounts.length === 0) {
263
- await writeLog("", 401, {
350
+ await recordFinalOutcome(undefined, 401, {
264
351
  errorType: "no_accounts",
265
352
  errorMessage: "No Codex accounts",
266
353
  });
@@ -280,7 +367,7 @@ export async function handleCodexResponsesRequest(ctx) {
280
367
  const retryAfterSec = soonest
281
368
  ? Math.max(1, Math.ceil((soonest - now) / 1000))
282
369
  : 60;
283
- await writeLog("", 429, {
370
+ await recordFinalOutcome(undefined, 429, {
284
371
  errorType: "all_accounts_cooling",
285
372
  errorMessage: "All Codex accounts are rate-limited",
286
373
  });
@@ -300,11 +387,15 @@ export async function handleCodexResponsesRequest(ctx) {
300
387
  let attempt = 0;
301
388
  let lastErrorMessage = "All Codex accounts failed";
302
389
  let lastErrorStatus = 502;
390
+ let lastAttemptedAccount;
303
391
  for (const account of eligible) {
304
- attempt += 1;
305
392
  let authRetried = false;
306
393
  // Same-account loop only re-runs once, for a post-401 token refresh.
307
394
  for (;;) {
395
+ attempt += 1;
396
+ const attemptStartedAt = Date.now();
397
+ lastAttemptedAccount = account;
398
+ recordAttempt(account.label, CODEX_ACCOUNT_TYPE);
308
399
  let upstream;
309
400
  try {
310
401
  upstream = await fetch(CODEX_RESPONSES_URL, {
@@ -320,6 +411,17 @@ export async function handleCodexResponsesRequest(ctx) {
320
411
  // can act on. Keep the detail in the log and return a fixed string, so
321
412
  // internal topology never reaches the client.
322
413
  logger.debug(`Codex upstream fetch failed (${account.label}): ${sanitizeForLog(error instanceof Error ? error.message : String(error))}`);
414
+ const errorMessage = summarizeCodexUpstreamError(error instanceof Error ? error.message : String(error), "Codex upstream request failed");
415
+ const errorCode = getCodexTransportErrorCode(error);
416
+ const transportScope = codexTransportScope(error);
417
+ recordAttemptError(account.label, CODEX_ACCOUNT_TYPE, 502);
418
+ writeAttempt(account, attempt, attemptStartedAt, 502, {
419
+ errorType: "network_error",
420
+ errorMessage,
421
+ ...(errorCode ? { errorCode } : {}),
422
+ transportScope,
423
+ retryable: true,
424
+ });
323
425
  lastErrorMessage = "Codex upstream request failed";
324
426
  lastErrorStatus = 502;
325
427
  break; // rotate to next account
@@ -336,50 +438,72 @@ export async function handleCodexResponsesRequest(ctx) {
336
438
  clearAccountCooldown(account.key, account.expiredCooldownUntil).catch(() => undefined);
337
439
  }
338
440
  publishCodexHeaders(ctx, account, attempt, quota);
339
- await writeLog(account.label, upstream.status);
441
+ writeAttempt(account, attempt, attemptStartedAt, upstream.status);
340
442
  const headers = {
341
443
  "content-type": upstream.headers.get("content-type") ?? "text/event-stream",
342
444
  "cache-control": "no-cache",
343
445
  connection: "keep-alive",
344
446
  ...(ctx.responseHeaders ?? {}),
345
447
  };
346
- // Tap the relay for token usage. The log above is written first and
347
- // unconditionally so a request is never lost when a client hangs up
348
- // mid-stream; this emits a second record for the same requestId
349
- // carrying the counts, which proxyAnalysis merges. If the stream shape
350
- // is not recognised, usage resolves null and nothing extra is written —
351
- // i.e. exactly the previous behaviour.
352
448
  if (!upstream.body) {
449
+ await recordFinalOutcome(account, upstream.status, {
450
+ terminalOutcome: "bodyless",
451
+ });
353
452
  return new Response(upstream.body, {
354
453
  status: upstream.status,
355
454
  headers,
356
455
  });
357
456
  }
358
457
  const { stream: usageTap, usage: usageSeen } = createCodexUsageTap();
359
- usageSeen
360
- .then((usage) => {
361
- if (!usage) {
362
- return;
363
- }
364
- return writeLog(account.label, upstream.status, {
365
- provider: "openai",
366
- inputTokens: usage.inputTokens,
367
- outputTokens: usage.outputTokens,
368
- cacheReadTokens: usage.cacheReadTokens,
369
- cacheCreationTokens: usage.cacheCreationTokens,
370
- });
371
- })
372
- .catch(() => undefined);
373
- return new Response(upstream.body.pipeThrough(usageTap), {
458
+ const relay = new Response(upstream.body.pipeThrough(usageTap), {
374
459
  status: upstream.status,
375
460
  headers,
376
461
  });
462
+ return trackProxyResponse(relay, () => undefined, {
463
+ onTerminal: ({ outcome }) => {
464
+ void usageSeen
465
+ .then((usage) => {
466
+ const usageExtra = usage
467
+ ? {
468
+ inputTokens: usage.inputTokens,
469
+ outputTokens: usage.outputTokens,
470
+ cacheReadTokens: usage.cacheReadTokens,
471
+ cacheCreationTokens: usage.cacheCreationTokens,
472
+ }
473
+ : {};
474
+ if (outcome === "completed" || outcome === "bodyless") {
475
+ return recordFinalOutcome(account, upstream.status, {
476
+ terminalOutcome: outcome,
477
+ ...usageExtra,
478
+ });
479
+ }
480
+ return recordFinalOutcome(account, outcome === "client_cancelled" ? 499 : 502, {
481
+ errorType: outcome === "client_cancelled"
482
+ ? "client_cancelled"
483
+ : "stream_error",
484
+ errorMessage: outcome === "client_cancelled"
485
+ ? "Client cancelled Codex stream"
486
+ : "Codex upstream stream failed",
487
+ terminalOutcome: outcome,
488
+ ...usageExtra,
489
+ });
490
+ })
491
+ .catch(() => undefined);
492
+ },
493
+ });
377
494
  }
378
495
  const errText = await upstream.text().catch(() => "");
379
496
  // 401/403 → try a forced token refresh once, then rotate.
380
497
  if ((upstream.status === 401 || upstream.status === 403) &&
381
498
  !authRetried &&
382
499
  account.refreshToken) {
500
+ const errorMessage = summarizeCodexUpstreamError(errText, "Codex authentication rejected upstream");
501
+ recordAttemptError(account.label, CODEX_ACCOUNT_TYPE, upstream.status);
502
+ writeAttempt(account, attempt, attemptStartedAt, upstream.status, {
503
+ errorType: "authentication_error",
504
+ errorMessage,
505
+ retryable: true,
506
+ });
383
507
  authRetried = true;
384
508
  const staleTokens = {
385
509
  accessToken: account.token,
@@ -387,17 +511,11 @@ export async function handleCodexResponsesRequest(ctx) {
387
511
  expiresAt: account.expiresAt ?? 0,
388
512
  };
389
513
  try {
390
- const refreshed = await refreshCodexToken(account.refreshToken);
514
+ const refreshed = await refreshCodexTokenOnce(account.key, account.refreshToken);
391
515
  account.token = refreshed.accessToken;
392
516
  account.refreshToken = refreshed.refreshToken ?? account.refreshToken;
393
517
  account.expiresAt = refreshed.expiresAt ?? account.expiresAt;
394
518
  account.accountId = resolveCodexAccountId(refreshed.accessToken);
395
- await tokenStore.saveTokens(account.key, {
396
- accessToken: account.token,
397
- refreshToken: account.refreshToken,
398
- expiresAt: account.expiresAt ?? Date.now() + 3_600_000,
399
- tokenType: "Bearer",
400
- });
401
519
  continue; // retry same account with the fresh token
402
520
  }
403
521
  catch (error) {
@@ -432,6 +550,16 @@ export async function handleCodexResponsesRequest(ctx) {
432
550
  const retryAfterMs = parseRetryAfterMs(upstream.headers.get("retry-after"));
433
551
  const plan = planCodexCooldown(quota, retryAfterMs, Date.now());
434
552
  await saveAccountCooldown(account.key, plan.coolingUntil, plan.reason).catch(() => undefined);
553
+ const rateLimitKind = plan.reason === "transient" ? "transient" : "quota";
554
+ const errorMessage = summarizeCodexUpstreamError(errText, "Codex account rate-limited");
555
+ recordAttemptError(account.label, CODEX_ACCOUNT_TYPE, upstream.status, rateLimitKind);
556
+ writeAttempt(account, attempt, attemptStartedAt, upstream.status, {
557
+ errorType: "rate_limit_error",
558
+ errorMessage,
559
+ retryable: true,
560
+ rateLimitKind,
561
+ cooldownReason: plan.reason,
562
+ });
435
563
  lastErrorStatus = 429;
436
564
  lastErrorMessage = "Codex account rate-limited";
437
565
  break; // rotate
@@ -443,12 +571,22 @@ export async function handleCodexResponsesRequest(ctx) {
443
571
  // of letting it stay first in line with unknown quota.
444
572
  await saveAccountCooldown(account.key, Date.now() + CODEX_AUTH_COOLDOWN_MS, "auth").catch(() => undefined);
445
573
  }
574
+ const errorMessage = summarizeCodexUpstreamError(errText, "Codex error");
575
+ const errorType = upstream.status === 401 || upstream.status === 403
576
+ ? "authentication_error"
577
+ : "api_error";
578
+ recordAttemptError(account.label, CODEX_ACCOUNT_TYPE, upstream.status);
579
+ writeAttempt(account, attempt, attemptStartedAt, upstream.status, {
580
+ errorType,
581
+ errorMessage,
582
+ retryable: upstream.status >= 500,
583
+ });
446
584
  lastErrorStatus = upstream.status >= 500 ? 502 : upstream.status;
447
- lastErrorMessage = sanitizeForLog(errText).slice(0, 200) || "Codex error";
585
+ lastErrorMessage = errorMessage;
448
586
  break; // rotate
449
587
  }
450
588
  }
451
- await writeLog("", lastErrorStatus, {
589
+ await recordFinalOutcome(lastAttemptedAccount, lastErrorStatus, {
452
590
  errorType: "all_accounts_failed",
453
591
  errorMessage: lastErrorMessage,
454
592
  });
@@ -529,18 +667,12 @@ async function handleCodexModelsRequest(ctx) {
529
667
  if (!authRetried && account.refreshToken) {
530
668
  authRetried = true;
531
669
  try {
532
- const refreshed = await refreshCodexToken(account.refreshToken);
670
+ const refreshed = await refreshCodexTokenOnce(account.key, account.refreshToken);
533
671
  account.token = refreshed.accessToken;
534
672
  account.refreshToken =
535
673
  refreshed.refreshToken ?? account.refreshToken;
536
674
  account.expiresAt = refreshed.expiresAt ?? account.expiresAt;
537
675
  account.accountId = resolveCodexAccountId(refreshed.accessToken);
538
- await tokenStore.saveTokens(account.key, {
539
- accessToken: account.token,
540
- refreshToken: account.refreshToken,
541
- expiresAt: account.expiresAt ?? Date.now() + 3_600_000,
542
- tokenType: "Bearer",
543
- });
544
676
  continue; // retry this account with the fresh token
545
677
  }
546
678
  catch {
@@ -605,5 +737,6 @@ export const __testHooks = {
605
737
  buildCodexUpstreamHeaders,
606
738
  planCodexCooldown,
607
739
  refreshCodexTokenOnce,
740
+ refreshCodexTokenOnceWithDependencies,
608
741
  codexRefreshInFlightSize: () => codexRefreshInFlight.size,
609
742
  };