@juspay/neurolink 12.59.0 → 13.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (146) hide show
  1. package/CHANGELOG.md +3 -3
  2. package/dist/auth/anthropicOAuth.js +3 -3
  3. package/dist/auth/middleware/AuthMiddleware.d.ts +1 -1
  4. package/dist/auth/sessionManager.js +4 -4
  5. package/dist/auth/tokenStore.js +4 -4
  6. package/dist/browser/neurolink.min.js +474 -473
  7. package/dist/cli/commands/observability.d.ts +1 -6
  8. package/dist/cli/commands/observability.js +0 -106
  9. package/dist/cli/commands/proxy.js +9 -18
  10. package/dist/cli/commands/telemetry.d.ts +3 -13
  11. package/dist/cli/commands/telemetry.js +7 -354
  12. package/dist/cli/loop/session.js +1 -1
  13. package/dist/client/httpClient.js +1 -1
  14. package/dist/client/sseClient.js +1 -1
  15. package/dist/client/streamingClient.js +2 -2
  16. package/dist/client/wsClient.js +1 -1
  17. package/dist/constants/langfuse.d.ts +25 -0
  18. package/dist/constants/langfuse.js +38 -0
  19. package/dist/constants/telemetry.d.ts +20 -0
  20. package/dist/constants/telemetry.js +34 -0
  21. package/dist/core/baseProvider.d.ts +2 -1
  22. package/dist/core/baseProvider.js +4 -11
  23. package/dist/core/conversationMemoryManager.js +5 -8
  24. package/dist/core/factory.js +1 -1
  25. package/dist/core/geminiLoopAdapter.d.ts +8 -1
  26. package/dist/core/geminiLoopAdapter.js +25 -1
  27. package/dist/core/loopEngine.js +69 -40
  28. package/dist/core/modules/MessageBuilder.js +2 -2
  29. package/dist/core/modules/StreamHandler.js +2 -2
  30. package/dist/core/modules/TelemetryHandler.d.ts +2 -18
  31. package/dist/core/modules/TelemetryHandler.js +1 -66
  32. package/dist/core/modules/ToolsManager.js +40 -31
  33. package/dist/core/nativeGenerateLoop.js +78 -8
  34. package/dist/core/redisConversationMemoryManager.js +8 -15
  35. package/dist/core/toolExecutionGuards.js +1 -13
  36. package/dist/index.d.ts +5 -12
  37. package/dist/index.js +5 -37
  38. package/dist/mcp/batching/requestBatcher.js +1 -1
  39. package/dist/mcp/externalServerManager.js +3 -21
  40. package/dist/mcp/toolDiscoveryService.js +4 -16
  41. package/dist/mcp/toolRegistry.js +2 -5
  42. package/dist/memory/memoryRetrievalTools.js +1 -1
  43. package/dist/neurolink.d.ts +16 -27
  44. package/dist/neurolink.js +788 -792
  45. package/dist/observability/exporterSpanProcessor.d.ts +50 -0
  46. package/dist/observability/exporterSpanProcessor.js +181 -0
  47. package/dist/observability/exporters/langfuseExporter.d.ts +13 -34
  48. package/dist/observability/exporters/langfuseExporter.js +101 -154
  49. package/dist/observability/index.d.ts +1 -0
  50. package/dist/observability/index.js +1 -0
  51. package/dist/observability/langfuseMasking.d.ts +8 -0
  52. package/dist/observability/langfuseMasking.js +47 -0
  53. package/dist/observability/otlpEncoding.d.ts +20 -0
  54. package/dist/observability/otlpEncoding.js +85 -0
  55. package/dist/providers/amazonBedrock/client.js +142 -176
  56. package/dist/providers/amazonBedrock/loopAdapter.js +14 -1
  57. package/dist/providers/amazonSagemaker.js +118 -80
  58. package/dist/providers/anthropic/client.js +3 -52
  59. package/dist/providers/anthropic/loopAdapter.js +24 -1
  60. package/dist/providers/anthropic/rateLimitCapture.js +9 -12
  61. package/dist/providers/googleAiStudio/client.d.ts +4 -5
  62. package/dist/providers/googleAiStudio/client.js +38 -40
  63. package/dist/providers/googleVertex/client.d.ts +2 -7
  64. package/dist/providers/googleVertex/client.js +282 -378
  65. package/dist/providers/googleVertex/constants.js +2 -2
  66. package/dist/providers/litellm/client.d.ts +2 -9
  67. package/dist/providers/litellm/client.js +1 -63
  68. package/dist/providers/openAI/client.d.ts +1 -8
  69. package/dist/providers/openAI/client.js +0 -65
  70. package/dist/providers/openaiChatCompletionsBase.d.ts +1 -0
  71. package/dist/providers/openaiChatCompletionsBase.js +61 -4
  72. package/dist/providers/replicate.js +22 -4
  73. package/dist/proxy/codexUsage.js +7 -0
  74. package/dist/proxy/proxyFetch.js +19 -25
  75. package/dist/proxy/proxyTracer.d.ts +14 -7
  76. package/dist/proxy/proxyTracer.js +129 -83
  77. package/dist/proxy/proxyTranslationEngine.js +0 -1
  78. package/dist/proxy/requestLogger.d.ts +1 -1
  79. package/dist/proxy/requestLogger.js +39 -45
  80. package/dist/server/middleware/auth.d.ts +5 -1
  81. package/dist/server/middleware/auth.js +10 -3
  82. package/dist/server/middleware/common.js +1 -1
  83. package/dist/server/routes/agentRoutes.js +4 -4
  84. package/dist/server/routes/claudeProxyRoutes.js +99 -196
  85. package/dist/server/routes/codexProxyRoutes.js +621 -591
  86. package/dist/server/routes/healthRoutes.js +6 -6
  87. package/dist/server/routes/mcpRoutes.js +7 -7
  88. package/dist/server/routes/memoryRoutes.js +7 -7
  89. package/dist/server/routes/openApiRoutes.js +3 -3
  90. package/dist/server/routes/toolRoutes.js +5 -5
  91. package/dist/server/routes/transcribeRoutes.js +1 -1
  92. package/dist/server/utils/validation.d.ts +3 -3
  93. package/dist/services/server/ai/observability/instrumentation.d.ts +34 -201
  94. package/dist/services/server/ai/observability/instrumentation.js +220 -1267
  95. package/dist/services/server/ai/observability/langfuseSetup.d.ts +14 -0
  96. package/dist/services/server/ai/observability/langfuseSetup.js +55 -0
  97. package/dist/services/server/ai/observability/otlpSetup.d.ts +16 -0
  98. package/dist/services/server/ai/observability/otlpSetup.js +58 -0
  99. package/dist/session/globalSessionState.js +0 -11
  100. package/dist/telemetry/attributes.d.ts +23 -46
  101. package/dist/telemetry/attributes.js +76 -162
  102. package/dist/telemetry/chatMessages.d.ts +13 -0
  103. package/dist/telemetry/chatMessages.js +59 -0
  104. package/dist/telemetry/index.d.ts +5 -19
  105. package/dist/telemetry/index.js +5 -26
  106. package/dist/telemetry/observation.d.ts +37 -0
  107. package/dist/telemetry/observation.js +112 -0
  108. package/dist/telemetry/observationNames.d.ts +18 -0
  109. package/dist/telemetry/observationNames.js +24 -0
  110. package/dist/telemetry/spanSanitizer.d.ts +39 -0
  111. package/dist/telemetry/spanSanitizer.js +92 -0
  112. package/dist/telemetry/spanSerialization.d.ts +13 -0
  113. package/dist/telemetry/spanSerialization.js +46 -0
  114. package/dist/telemetry/tracers.d.ts +37 -18
  115. package/dist/telemetry/tracers.js +97 -19
  116. package/dist/telemetry/withSpan.d.ts +47 -25
  117. package/dist/telemetry/withSpan.js +158 -71
  118. package/dist/types/cli.d.ts +0 -11
  119. package/dist/types/decision.d.ts +3 -0
  120. package/dist/types/exporter.d.ts +54 -0
  121. package/dist/types/exporter.js +0 -4
  122. package/dist/types/generate.d.ts +4 -1
  123. package/dist/types/isolatedAgent.d.ts +5 -4
  124. package/dist/types/loopEngine.d.ts +18 -28
  125. package/dist/types/observability.d.ts +127 -160
  126. package/dist/types/providers.d.ts +15 -0
  127. package/dist/types/proxy.d.ts +41 -2
  128. package/dist/types/server.d.ts +9 -5
  129. package/dist/utils/anthropicTraceSanitizer.d.ts +2 -2
  130. package/dist/utils/anthropicTraceSanitizer.js +3 -1
  131. package/dist/utils/conversationMemory.js +4 -4
  132. package/dist/utils/fileDetector.js +1 -1
  133. package/dist/utils/messageBuilder.js +1 -1
  134. package/dist/utils/noOutputSentinel.d.ts +4 -16
  135. package/dist/utils/noOutputSentinel.js +12 -28
  136. package/dist/utils/providerRetry.js +4 -4
  137. package/dist/voice/livekit/brain.js +1 -1
  138. package/dist/voice/livekit/realtimeMemory.d.ts +1 -1
  139. package/dist/voice/livekit/realtimeMemory.js +66 -1
  140. package/dist/voice/livekit/realtimeVoiceAgent.js +1 -1
  141. package/docs-site/static/search-index.json +42 -64
  142. package/package.json +5 -4
  143. package/dist/telemetry/telemetryService.d.ts +0 -79
  144. package/dist/telemetry/telemetryService.js +0 -572
  145. package/dist/utils/observabilityHelpers.d.ts +0 -27
  146. package/dist/utils/observabilityHelpers.js +0 -47
@@ -1024,39 +1024,35 @@ function captureCodexResponse(ctx, response, phase, account, attempt) {
1024
1024
  headers: response.headers,
1025
1025
  });
1026
1026
  }
1027
- /**
1028
- * Serve one native `/backend-api/codex/responses` turn.
1029
- *
1030
- * Owns the whole hop: account selection and rotation, the upstream request,
1031
- * the SSE relay back to the caller, and the terminal accounting that turns the
1032
- * observed wire usage into request-log and tracer records. Usage counts the
1033
- * provider did not report are carried as unobserved rather than zero, so a
1034
- * silent cache breakdown is never recorded as a cache miss.
1035
- */
1036
- async function executeCodexResponsesRequest(ctx,
1037
- // Read by the outbound-fallback insertion points below (stage-c-trigger.md)
1038
- // to gate/target a Codex-outbound fallback attempt when the native Codex
1039
- // pool itself fails.
1040
- runtimeConfigProvider,
1041
- // Threaded through from `createCodexProxyRoutes` to the anthropic-target
1042
- // loopback leg of the outbound-fallback dispatcher (mirrors the
1043
- // OpenAI->Anthropic bridge's `loopbackPort`/`internalDispatch`).
1044
- loopbackPort, internalDispatch) {
1045
- const requestStartTime = Date.now();
1046
- let body = (ctx.body ?? {});
1047
- let bodyStr;
1048
- let preparedContext;
1049
- let contextPreflight;
1050
- let budgetLease;
1051
- let budgetDispatched = false;
1027
+ function startCodexTracer(ctx, model, body, isFallbackRequest) {
1028
+ try {
1029
+ return ProxyTracer.startRequest({
1030
+ requestId: ctx.requestId,
1031
+ method: ctx.method,
1032
+ path: ctx.path,
1033
+ model,
1034
+ stream: true,
1035
+ toolCount: Array.isArray(body.tools) ? body.tools.length : 0,
1036
+ provider: "openai",
1037
+ userAgent: ctx.headers["user-agent"],
1038
+ recordRequestMetrics: !isFallbackRequest,
1039
+ recordUsageMetrics: true,
1040
+ }, ctx.headers);
1041
+ }
1042
+ catch {
1043
+ return undefined;
1044
+ }
1045
+ }
1046
+ function createCodexRequestRecorder(input) {
1047
+ const { ctx, model, tracer, requestStartTime, isFallbackRequest, reasoningEffort, state, } = input;
1052
1048
  const settleBudget = async (actualTokens) => {
1053
- if (!budgetLease) {
1049
+ if (!state.budgetLease) {
1054
1050
  return;
1055
1051
  }
1056
1052
  try {
1057
- if (budgetDispatched) {
1058
- await settleProxyTokenBudget(budgetLease, actualTokens);
1059
- if (budgetLease.snapshot.settlement === "unconfirmed") {
1053
+ if (state.budgetDispatched) {
1054
+ await settleProxyTokenBudget(state.budgetLease, actualTokens);
1055
+ if (state.budgetLease.snapshot.settlement === "unconfirmed") {
1060
1056
  emitProxyOtelEvent("token_budget", {
1061
1057
  requestId: ctx.requestId,
1062
1058
  event: "settlement_unconfirmed",
@@ -1064,7 +1060,7 @@ loopbackPort, internalDispatch) {
1064
1060
  }
1065
1061
  }
1066
1062
  else {
1067
- await budgetLease.cancelBeforeDispatch();
1063
+ await state.budgetLease.cancelBeforeDispatch();
1068
1064
  }
1069
1065
  }
1070
1066
  catch (error) {
@@ -1076,40 +1072,6 @@ loopbackPort, internalDispatch) {
1076
1072
  });
1077
1073
  }
1078
1074
  };
1079
- const model = typeof body.model === "string"
1080
- ? body.model
1081
- : "-";
1082
- // A Codex call made as an inner Anthropic fallback is an upstream attempt,
1083
- // not an independently final client request. The parent fallback owns the
1084
- // final status and can still recover with a later provider.
1085
- const isFallbackRequest = ctx.metadata?.[CODEX_FALLBACK_METADATA_KEY] === true;
1086
- const reasoning = body.reasoning;
1087
- const reasoningEffort = reasoning &&
1088
- typeof reasoning === "object" &&
1089
- "effort" in reasoning &&
1090
- typeof reasoning.effort === "string"
1091
- ? reasoning.effort
1092
- : undefined;
1093
- let tracer;
1094
- try {
1095
- tracer = ProxyTracer.startRequest({
1096
- requestId: ctx.requestId,
1097
- method: ctx.method,
1098
- path: ctx.path,
1099
- model,
1100
- stream: true,
1101
- toolCount: Array.isArray(body.tools)
1102
- ? body.tools.length
1103
- : 0,
1104
- provider: "openai",
1105
- userAgent: ctx.headers["user-agent"],
1106
- recordRequestMetrics: !isFallbackRequest,
1107
- recordUsageMetrics: true,
1108
- }, ctx.headers);
1109
- }
1110
- catch {
1111
- // Instrumentation must not change provider request handling.
1112
- }
1113
1075
  const writeFinalLog = (account, responseStatus, extra = {}) => logRequest({
1114
1076
  timestamp: new Date().toISOString(),
1115
1077
  requestId: ctx.requestId,
@@ -1117,12 +1079,12 @@ loopbackPort, internalDispatch) {
1117
1079
  path: ctx.path,
1118
1080
  model,
1119
1081
  requestedModel: model,
1120
- contextPreflight,
1121
- tokenBudget: budgetLease ? { ...budgetLease.snapshot } : undefined,
1082
+ contextPreflight: state.contextPreflight,
1083
+ tokenBudget: state.budgetLease
1084
+ ? { ...state.budgetLease.snapshot }
1085
+ : undefined,
1122
1086
  stream: true,
1123
- toolCount: Array.isArray(body.tools)
1124
- ? body.tools.length
1125
- : 0,
1087
+ toolCount: Array.isArray(state.body.tools) ? state.body.tools.length : 0,
1126
1088
  account: account?.label ?? "",
1127
1089
  ...(account ? { accountKey: account.key } : {}),
1128
1090
  accountType: account ? CODEX_ACCOUNT_TYPE : "",
@@ -1138,15 +1100,14 @@ loopbackPort, internalDispatch) {
1138
1100
  responseTimeMs: Date.now() - requestStartTime,
1139
1101
  ...extra,
1140
1102
  });
1141
- let finalOutcomeRecorded = false;
1142
1103
  const recordFinalOutcome = async (account, responseStatus, extra = {}) => {
1143
1104
  await settleBudget(extra.inputTokens !== undefined && extra.outputTokens !== undefined
1144
1105
  ? extra.inputTokens + extra.outputTokens
1145
1106
  : undefined);
1146
- if (finalOutcomeRecorded) {
1107
+ if (state.finalOutcomeRecorded) {
1147
1108
  return;
1148
1109
  }
1149
- finalOutcomeRecorded = true;
1110
+ state.finalOutcomeRecorded = true;
1150
1111
  if (isProxyRequestFinalized(ctx.requestId)) {
1151
1112
  // Runtime owns the client final, but this route still owns its span.
1152
1113
  const status = ctx.abortSignal?.aborted
@@ -1203,12 +1164,12 @@ loopbackPort, internalDispatch) {
1203
1164
  path: ctx.path,
1204
1165
  model,
1205
1166
  requestedModel: model,
1206
- contextPreflight,
1207
- tokenBudget: budgetLease ? { ...budgetLease.snapshot } : undefined,
1167
+ contextPreflight: state.contextPreflight,
1168
+ tokenBudget: state.budgetLease
1169
+ ? { ...state.budgetLease.snapshot }
1170
+ : undefined,
1208
1171
  stream: true,
1209
- toolCount: Array.isArray(body.tools)
1210
- ? body.tools.length
1211
- : 0,
1172
+ toolCount: Array.isArray(state.body.tools) ? state.body.tools.length : 0,
1212
1173
  account: account.label,
1213
1174
  accountKey: account.key,
1214
1175
  accountType: CODEX_ACCOUNT_TYPE,
@@ -1219,22 +1180,526 @@ loopbackPort, internalDispatch) {
1219
1180
  ...extra,
1220
1181
  }).catch(() => undefined);
1221
1182
  };
1183
+ return { settleBudget, recordFinalOutcome, writeAttempt };
1184
+ }
1185
+ async function resolveCodexOutbound(rt, failureInput, account) {
1186
+ const { state } = rt;
1187
+ const outbound = await attemptCodexOutboundFallback({
1188
+ ctx: rt.ctx,
1189
+ body: state.body,
1190
+ model: rt.model,
1191
+ isFallbackRequest: rt.isFallbackRequest,
1192
+ runtimeConfigProvider: rt.runtimeConfigProvider,
1193
+ loopbackPort: rt.loopbackPort,
1194
+ internalDispatch: rt.internalDispatch,
1195
+ tracer: rt.tracer,
1196
+ recordFinalOutcome: rt.recordFinalOutcome,
1197
+ failureInput,
1198
+ });
1199
+ if (outbound.kind === "success") {
1200
+ await rt.settleBudget();
1201
+ state.budgetLease = undefined;
1202
+ state.budgetDispatched = false;
1203
+ return outbound.response;
1204
+ }
1205
+ if (outbound.kind === "request_too_large") {
1206
+ await rt.recordFinalOutcome(account, 413, {
1207
+ errorType: "request_too_large",
1208
+ errorMessage: outbound.message,
1209
+ terminalOutcome: "handler_error",
1210
+ });
1211
+ return buildCodexErrorResponse(413, outbound.message, "request_too_large");
1212
+ }
1213
+ return undefined;
1214
+ }
1215
+ async function cancelCodexRequest(rt, account) {
1216
+ await rt.recordFinalOutcome(account, 499, {
1217
+ errorType: "client_cancelled",
1218
+ errorMessage: "Client cancelled Codex request",
1219
+ terminalOutcome: "client_cancelled",
1220
+ });
1221
+ return buildCodexErrorResponse(499, "Client cancelled Codex request");
1222
+ }
1223
+ async function respondCodexNoAccounts(rt) {
1224
+ const fallback = await resolveCodexOutbound(rt, { failureClass: "no_accounts" }, undefined);
1225
+ if (fallback) {
1226
+ return fallback;
1227
+ }
1228
+ await rt.recordFinalOutcome(undefined, 401, {
1229
+ errorType: "no_accounts",
1230
+ errorMessage: "No Codex accounts",
1231
+ });
1232
+ return buildCodexErrorResponse(401, "No Codex accounts configured. Run `neurolink auth login codex`.");
1233
+ }
1234
+ async function respondCodexPoolCooling(rt, ordered, now) {
1235
+ const { recordFinalOutcome } = rt;
1236
+ // Every account is cooling; surface the soonest recovery as retry-after.
1237
+ const soonest = ordered.reduce((min, a) => {
1238
+ if (a.coolingUntil === undefined) {
1239
+ return min;
1240
+ }
1241
+ return min === undefined ? a.coolingUntil : Math.min(min, a.coolingUntil);
1242
+ }, undefined);
1243
+ const retryAfterSec = soonest
1244
+ ? Math.max(1, Math.ceil((soonest - now) / 1000))
1245
+ : 60;
1246
+ const fallback = await resolveCodexOutbound(rt, { failureClass: "pool_exhausted" }, undefined);
1247
+ if (fallback) {
1248
+ return fallback;
1249
+ }
1250
+ const resetAt = soonest ? new Date(soonest).toISOString() : undefined;
1251
+ // A refresh that failed transiently, or a 401 with no refresh token to
1252
+ // retry with, parks the account as "auth". Reporting that as spent quota
1253
+ // would tell the user the plan is exhausted when it is not.
1254
+ if (ordered.every((account) => account.coolingReason === "auth")) {
1255
+ await recordFinalOutcome(undefined, 503, {
1256
+ errorType: "all_accounts_auth_cooling",
1257
+ errorMessage: "All Codex accounts are cooling after an auth failure",
1258
+ });
1259
+ return buildCodexPoolCoolingResponse(resetAt
1260
+ ? `Codex authentication is temporarily unavailable for every account. Retry after ${resetAt}, or run \`neurolink auth login codex\` if it persists.`
1261
+ : "Codex authentication is temporarily unavailable for every account. Retry shortly, or run `neurolink auth login codex` if it persists.", retryAfterSec, "server_error");
1262
+ }
1263
+ await recordFinalOutcome(undefined, 429, {
1264
+ errorType: "all_accounts_cooling",
1265
+ errorMessage: "All Codex accounts are rate-limited",
1266
+ });
1267
+ return buildCodexPoolCoolingResponse(resetAt
1268
+ ? `Codex quota exhausted: all accounts are rate-limited. Resets at ${resetAt}.`
1269
+ : "Codex quota exhausted: all accounts are rate-limited.", retryAfterSec, "insufficient_quota");
1270
+ }
1271
+ async function relayCodexSuccess(rt, args) {
1272
+ const { ctx, writeAttempt, recordFinalOutcome } = rt;
1273
+ const { account, attempt, attemptStartedAt, upstream } = args;
1274
+ const quota = parseCodexRateLimitHeaders(upstream.headers);
1275
+ if (quota) {
1276
+ saveAccountQuota(account.key, quota).catch(() => undefined);
1277
+ }
1278
+ // A prior cooldown that has expired is cleared on success. The
1279
+ // compare-and-swap guards against wiping a longer cooldown that another
1280
+ // in-flight request set while this one was upstream.
1281
+ if (account.expiredCooldownUntil !== undefined) {
1282
+ clearAccountCooldown(account.key, account.expiredCooldownUntil).catch(() => undefined);
1283
+ }
1284
+ publishCodexHeaders(ctx, account, attempt, quota);
1285
+ writeAttempt(account, attempt, attemptStartedAt, upstream.status);
1286
+ const headers = {
1287
+ "content-type": upstream.headers.get("content-type") ?? "text/event-stream",
1288
+ "cache-control": "no-cache",
1289
+ connection: "keep-alive",
1290
+ ...(ctx.responseHeaders ?? {}),
1291
+ };
1292
+ if (!upstream.body) {
1293
+ recordAttemptError(account.label, CODEX_ACCOUNT_TYPE, 502);
1294
+ writeAttempt(account, attempt, attemptStartedAt, 502, {
1295
+ errorType: "incomplete_stream",
1296
+ errorMessage: "Codex returned no response stream",
1297
+ retryable: false,
1298
+ });
1299
+ await recordFinalOutcome(account, 502, {
1300
+ terminalOutcome: "stream_error",
1301
+ errorType: "incomplete_stream",
1302
+ errorMessage: "Codex returned no response stream",
1303
+ });
1304
+ return new Response(upstream.body, {
1305
+ status: upstream.status,
1306
+ headers,
1307
+ });
1308
+ }
1309
+ const { stream: usageTap, usage: usageSeen, evidence, } = createCodexUsageTap();
1310
+ const relay = new Response(upstream.body.pipeThrough(usageTap), {
1311
+ status: upstream.status,
1312
+ headers,
1313
+ });
1314
+ registerProxyResponseObserver(ctx.metadata, {
1315
+ onTerminal: createCodexTerminalObserver(rt, {
1316
+ account,
1317
+ attempt,
1318
+ attemptStartedAt,
1319
+ upstreamStatus: upstream.status,
1320
+ usageSeen,
1321
+ evidence,
1322
+ }),
1323
+ });
1324
+ return relay;
1325
+ }
1326
+ async function handleCodexTransportFailure(rt, args) {
1327
+ const { ctx, writeAttempt, recordFinalOutcome } = rt;
1328
+ const { error, account, attempt, attemptStartedAt } = args;
1329
+ if (ctx.abortSignal?.aborted) {
1330
+ writeAttempt(account, attempt, attemptStartedAt, 499, {
1331
+ errorType: "client_cancelled",
1332
+ errorMessage: "Client cancelled Codex request",
1333
+ retryable: false,
1334
+ });
1335
+ return cancelCodexRequest(rt, account);
1336
+ }
1337
+ // A transport failure message is derived from local state — resolved
1338
+ // hostnames, socket paths, Node internals — and says nothing the caller
1339
+ // can act on. Keep the detail in the log and return a fixed string, so
1340
+ // internal topology never reaches the client.
1341
+ logger.debug(`Codex upstream fetch failed (${account.label}): ${sanitizeForLog(error instanceof Error ? error.message : String(error))}`);
1342
+ const errorMessage = summarizeCodexUpstreamError(error instanceof Error ? error.message : String(error), "Codex upstream request failed");
1343
+ const errorCode = getCodexTransportErrorCode(error);
1344
+ const transportScope = codexTransportScope(error);
1345
+ // These codes prove failure before HTTP dispatch. Socket resets,
1346
+ // EPIPE and generic timeouts may follow dispatch and must not replay.
1347
+ const retryable = [
1348
+ "UND_ERR_CONNECT_TIMEOUT",
1349
+ "ECONNREFUSED",
1350
+ "ENOTFOUND",
1351
+ "EAI_AGAIN",
1352
+ ].includes(errorCode ?? "");
1353
+ recordAttemptError(account.label, CODEX_ACCOUNT_TYPE, 502);
1354
+ writeAttempt(account, attempt, attemptStartedAt, 502, {
1355
+ errorType: "network_error",
1356
+ errorMessage,
1357
+ ...(errorCode ? { errorCode } : {}),
1358
+ transportScope,
1359
+ retryable,
1360
+ });
1361
+ const lastFailure = {
1362
+ errorType: "network_error",
1363
+ errorMessage,
1364
+ errorCode,
1365
+ transportScope,
1366
+ };
1367
+ if (retryable) {
1368
+ return {
1369
+ lastFailure,
1370
+ lastErrorStatus: 502,
1371
+ lastErrorCode: errorCode,
1372
+ lastErrorMessage: "Codex upstream request failed",
1373
+ };
1374
+ }
1375
+ const fallback = await resolveCodexOutbound(rt, {
1376
+ failureClass: "non_retryable_transport",
1377
+ transportErrorCode: errorCode,
1378
+ }, account);
1379
+ if (fallback) {
1380
+ return fallback;
1381
+ }
1382
+ await recordFinalOutcome(account, 502, { ...lastFailure, errorMessage });
1383
+ return buildCodexErrorResponse(502, "Codex upstream request failed");
1384
+ }
1385
+ async function handleCodexUpstreamFailure(rt, args) {
1386
+ const { writeAttempt } = rt;
1387
+ const { account, attempt, attemptStartedAt, upstream, authRetried } = args;
1388
+ const errText = await upstream.text().catch(() => "");
1389
+ // 401/403 → try a forced token refresh once, then rotate.
1390
+ if ((upstream.status === 401 || upstream.status === 403) &&
1391
+ !authRetried &&
1392
+ account.refreshToken) {
1393
+ const errorMessage = summarizeCodexUpstreamError(errText, "Codex authentication rejected upstream");
1394
+ recordAttemptError(account.label, CODEX_ACCOUNT_TYPE, upstream.status);
1395
+ writeAttempt(account, attempt, attemptStartedAt, upstream.status, {
1396
+ errorType: "authentication_error",
1397
+ errorMessage,
1398
+ retryable: true,
1399
+ });
1400
+ const staleTokens = {
1401
+ accessToken: account.token,
1402
+ refreshToken: account.refreshToken,
1403
+ expiresAt: account.expiresAt ?? 0,
1404
+ };
1405
+ try {
1406
+ const refreshed = await refreshCodexTokenOnce(account.key, account.refreshToken);
1407
+ account.token = refreshed.accessToken;
1408
+ account.refreshToken = refreshed.refreshToken ?? account.refreshToken;
1409
+ account.expiresAt = refreshed.expiresAt ?? account.expiresAt;
1410
+ account.accountId = resolveCodexAccountId(refreshed.accessToken);
1411
+ return "retry";
1412
+ }
1413
+ catch (error) {
1414
+ if (isPermanentCodexRefreshFailure(error)) {
1415
+ // Compare-and-swap: the pool is rebuilt per request with no shared
1416
+ // state, so a concurrent request may already have rotated this
1417
+ // credential. Disabling unconditionally would kill the account that
1418
+ // the other request just healed.
1419
+ const disabled = await tokenStore.markDisabledIfCurrent(account.key, staleTokens, "refresh_invalid");
1420
+ if (disabled) {
1421
+ logger.always(`[proxy] codex account=${account.label} disabled until re-authentication. Run: neurolink auth login codex --label ${account.label}`);
1422
+ }
1423
+ return {
1424
+ lastFailure: {
1425
+ errorType: "authentication_error",
1426
+ errorCode: "refresh_invalid",
1427
+ },
1428
+ lastErrorStatus: 401,
1429
+ lastErrorCode: "refresh_invalid",
1430
+ lastErrorMessage: "Codex token refresh failed; re-login required",
1431
+ };
1432
+ }
1433
+ // No verdict on the credential — cool briefly and try the next
1434
+ // account, so a 5xx or a timeout cannot cost the user a login.
1435
+ await saveAccountCooldown(account.key, Date.now() + CODEX_AUTH_COOLDOWN_MS, "auth").catch(() => undefined);
1436
+ logger.debug(`[proxy] codex account=${account.label} refresh failed transiently; cooling and rotating`);
1437
+ const errorCode = getCodexTransportErrorCode(error);
1438
+ return {
1439
+ lastFailure: { errorType: "auth_refresh_unavailable", errorCode },
1440
+ lastErrorStatus: 503,
1441
+ lastErrorCode: errorCode,
1442
+ lastErrorMessage: "Codex token refresh temporarily unavailable",
1443
+ };
1444
+ }
1445
+ }
1446
+ // 429 → cooldown + rotate.
1447
+ if (upstream.status === 429) {
1448
+ const quota = parseCodexRateLimitHeaders(upstream.headers);
1449
+ if (quota) {
1450
+ saveAccountQuota(account.key, quota).catch(() => undefined);
1451
+ }
1452
+ const retryAfterMs = parseRetryAfterMs(upstream.headers.get("retry-after"));
1453
+ const now = Date.now();
1454
+ const bodyQuota = parseCodexQuotaError(errText, now);
1455
+ const plan = planCodexCooldown(quota, retryAfterMs, now, errText);
1456
+ await saveAccountCooldown(account.key, plan.coolingUntil, plan.reason).catch(() => undefined);
1457
+ const rateLimitKind = plan.reason === "transient" ? "transient" : "quota";
1458
+ const errorMessage = summarizeCodexUpstreamError(errText, "Codex account rate-limited");
1459
+ recordAttemptError(account.label, CODEX_ACCOUNT_TYPE, upstream.status, rateLimitKind);
1460
+ writeAttempt(account, attempt, attemptStartedAt, upstream.status, {
1461
+ errorType: "rate_limit_error",
1462
+ errorMessage,
1463
+ retryable: rateLimitKind === "transient",
1464
+ rateLimitKind,
1465
+ cooldownReason: plan.reason,
1466
+ ...(bodyQuota
1467
+ ? {
1468
+ errorCode: bodyQuota.errorCode,
1469
+ quotaResetAt: bodyQuota.resetAt || undefined,
1470
+ quotaScope: bodyQuota.scope,
1471
+ }
1472
+ : {}),
1473
+ });
1474
+ return {
1475
+ lastFailure: {
1476
+ errorType: "rate_limit_error",
1477
+ errorCode: bodyQuota?.errorCode,
1478
+ },
1479
+ lastErrorStatus: 429,
1480
+ lastErrorCode: bodyQuota?.errorCode,
1481
+ lastErrorMessage: "Codex account rate-limited",
1482
+ };
1483
+ }
1484
+ // Other non-ok → record and rotate.
1485
+ if (upstream.status === 401 || upstream.status === 403) {
1486
+ // Reached only when the account has no refresh token to retry with, so
1487
+ // it will fail identically on the next request. Park it briefly instead
1488
+ // of letting it stay first in line with unknown quota.
1489
+ await saveAccountCooldown(account.key, Date.now() + CODEX_AUTH_COOLDOWN_MS, "auth").catch(() => undefined);
1490
+ }
1491
+ const errorMessage = summarizeCodexUpstreamError(errText, "Codex error");
1492
+ const errorType = upstream.status === 401 || upstream.status === 403
1493
+ ? "authentication_error"
1494
+ : "api_error";
1495
+ recordAttemptError(account.label, CODEX_ACCOUNT_TYPE, upstream.status);
1496
+ writeAttempt(account, attempt, attemptStartedAt, upstream.status, {
1497
+ errorType,
1498
+ errorMessage,
1499
+ retryable: upstream.status >= 500,
1500
+ });
1501
+ return {
1502
+ lastFailure: { errorType },
1503
+ lastErrorStatus: upstream.status >= 500 ? 502 : upstream.status,
1504
+ lastErrorCode: getProxyUpstreamFailure({ responseBody: errText })?.code,
1505
+ lastErrorMessage: errorMessage,
1506
+ };
1507
+ }
1508
+ function createCodexTerminalObserver(rt, args) {
1509
+ const { tracer, requestStartTime, writeAttempt, recordFinalOutcome } = rt;
1510
+ const { account, attempt, attemptStartedAt, upstreamStatus, usageSeen, evidence, } = args;
1511
+ return ({ outcome, error, observedBodyBytes }) => {
1512
+ return usageSeen
1513
+ .then((usage) => {
1514
+ const semantic = evidence();
1515
+ const completedFrameDelivered = semantic.completed && observedBodyBytes >= semantic.terminalBytes;
1516
+ const failed = semantic.errorType ||
1517
+ ((outcome === "completed" || outcome === "bodyless") &&
1518
+ !semantic.completed);
1519
+ const usageExtra = usage
1520
+ ? {
1521
+ inputTokens: usage.inputTokensObserved
1522
+ ? usage.inputTokens
1523
+ : undefined,
1524
+ outputTokens: usage.outputTokensObserved
1525
+ ? usage.outputTokens
1526
+ : undefined,
1527
+ cacheReadTokens: usage.cacheReadTokensObserved
1528
+ ? usage.cacheReadTokens
1529
+ : undefined,
1530
+ cacheCreationTokens: usage.cacheCreationTokensObserved
1531
+ ? usage.cacheCreationTokens
1532
+ : undefined,
1533
+ reasoningTokens: usage.reasoningTokensObserved
1534
+ ? usage.reasoningTokens
1535
+ : undefined,
1536
+ }
1537
+ : {};
1538
+ tracer?.setResponseInfo(codexResponseInfo(semantic));
1539
+ if (usage?.inputTokensObserved && usage.outputTokensObserved) {
1540
+ try {
1541
+ tracer?.setUsage({
1542
+ ...usage,
1543
+ reasoningTokens: usage.reasoningTokensObserved
1544
+ ? usage.reasoningTokens
1545
+ : undefined,
1546
+ cacheReadTokensObserved: usage.cacheReadTokensObserved === true,
1547
+ cacheCreationTokensObserved: usage.cacheCreationTokensObserved === true,
1548
+ inputIncludesCachedTokens: true,
1549
+ });
1550
+ }
1551
+ catch {
1552
+ // Pricing/metrics must never change the stream outcome.
1553
+ }
1554
+ }
1555
+ const timing = semantic.firstUsefulOutputAt === undefined ||
1556
+ semantic.observationIncomplete
1557
+ ? {
1558
+ firstUsefulOutputStatus: semantic.completed && !semantic.observationIncomplete
1559
+ ? "no_useful_output"
1560
+ : "not_observed",
1561
+ }
1562
+ : {
1563
+ firstUsefulOutputStatus: "observed",
1564
+ firstUsefulOutputEvent: semantic.firstUsefulOutputEvent,
1565
+ firstUsefulOutputMs: Math.max(0, semantic.firstUsefulOutputAt - requestStartTime),
1566
+ };
1567
+ if (failed) {
1568
+ recordAttemptError(account.label, CODEX_ACCOUNT_TYPE, 502);
1569
+ writeAttempt(account, attempt, attemptStartedAt, 502, {
1570
+ // An internal fallback has no separate final row. Keep
1571
+ // its observed usage on this attempt when a later
1572
+ // provider owns the client final.
1573
+ ...usageExtra,
1574
+ inputIncludesCachedTokens: true,
1575
+ errorType: semantic.errorType ?? "incomplete_stream",
1576
+ errorCode: semantic.errorCode,
1577
+ errorMessage: semantic.errorMessage ??
1578
+ "Codex stream ended without a completion event",
1579
+ retryable: false,
1580
+ });
1581
+ return recordFinalOutcome(account, 502, {
1582
+ ...usageExtra,
1583
+ ...timing,
1584
+ terminalOutcome: "stream_error",
1585
+ errorType: semantic.errorType ?? "incomplete_stream",
1586
+ errorCode: semantic.errorCode,
1587
+ errorMessage: semantic.errorMessage ??
1588
+ "Codex stream ended without a completion event",
1589
+ });
1590
+ }
1591
+ if (outcome === "completed" ||
1592
+ outcome === "bodyless" ||
1593
+ (outcome === "client_cancelled" && completedFrameDelivered)) {
1594
+ return recordFinalOutcome(account, upstreamStatus, {
1595
+ terminalOutcome: "completed",
1596
+ ...usageExtra,
1597
+ ...timing,
1598
+ });
1599
+ }
1600
+ if (outcome === "stream_error") {
1601
+ recordAttemptError(account.label, CODEX_ACCOUNT_TYPE, 502);
1602
+ writeAttempt(account, attempt, attemptStartedAt, 502, {
1603
+ errorType: "stream_error",
1604
+ errorCode: getCodexTransportErrorCode(error),
1605
+ errorMessage: summarizeCodexUpstreamError(error instanceof Error ? error.message : "", "Codex upstream stream failed"),
1606
+ retryable: false,
1607
+ });
1608
+ }
1609
+ return recordFinalOutcome(account, outcome === "client_cancelled" ? 499 : 502, {
1610
+ errorType: outcome === "client_cancelled"
1611
+ ? "client_cancelled"
1612
+ : "stream_error",
1613
+ errorMessage: outcome === "client_cancelled"
1614
+ ? "Client cancelled Codex stream"
1615
+ : summarizeCodexUpstreamError(error instanceof Error ? error.message : "", "Codex upstream stream failed"),
1616
+ ...(outcome === "stream_error"
1617
+ ? { errorCode: getCodexTransportErrorCode(error) }
1618
+ : {}),
1619
+ terminalOutcome: outcome,
1620
+ ...usageExtra,
1621
+ ...timing,
1622
+ });
1623
+ })
1624
+ .catch(() => undefined);
1625
+ };
1626
+ }
1627
+ /**
1628
+ * Serve one native `/backend-api/codex/responses` turn.
1629
+ *
1630
+ * Owns the whole hop: account selection and rotation, the upstream request,
1631
+ * the SSE relay back to the caller, and the terminal accounting that turns the
1632
+ * observed wire usage into request-log and tracer records. Usage counts the
1633
+ * provider did not report are carried as unobserved rather than zero, so a
1634
+ * silent cache breakdown is never recorded as a cache miss.
1635
+ */
1636
+ async function executeCodexResponsesRequest(ctx,
1637
+ // Read by the outbound-fallback insertion points below (stage-c-trigger.md)
1638
+ // to gate/target a Codex-outbound fallback attempt when the native Codex
1639
+ // pool itself fails.
1640
+ runtimeConfigProvider,
1641
+ // Threaded through from `createCodexProxyRoutes` to the anthropic-target
1642
+ // loopback leg of the outbound-fallback dispatcher (mirrors the
1643
+ // OpenAI->Anthropic bridge's `loopbackPort`/`internalDispatch`).
1644
+ loopbackPort, internalDispatch) {
1645
+ const requestStartTime = Date.now();
1646
+ const state = {
1647
+ body: (ctx.body ?? {}),
1648
+ bodyStr: "",
1649
+ budgetDispatched: false,
1650
+ finalOutcomeRecorded: false,
1651
+ };
1652
+ const model = typeof state.body.model === "string" ? state.body.model : "-";
1653
+ // A Codex call made as an inner Anthropic fallback is an upstream attempt,
1654
+ // not an independently final client request. The parent fallback owns the
1655
+ // final status and can still recover with a later provider.
1656
+ const isFallbackRequest = ctx.metadata?.[CODEX_FALLBACK_METADATA_KEY] === true;
1657
+ const reasoning = state.body.reasoning;
1658
+ const reasoningEffort = reasoning &&
1659
+ typeof reasoning === "object" &&
1660
+ "effort" in reasoning &&
1661
+ typeof reasoning.effort === "string"
1662
+ ? reasoning.effort
1663
+ : undefined;
1664
+ const tracer = startCodexTracer(ctx, model, state.body, isFallbackRequest);
1665
+ const rt = {
1666
+ ctx,
1667
+ model,
1668
+ tracer,
1669
+ requestStartTime,
1670
+ isFallbackRequest,
1671
+ state,
1672
+ runtimeConfigProvider,
1673
+ loopbackPort,
1674
+ internalDispatch,
1675
+ ...createCodexRequestRecorder({
1676
+ ctx,
1677
+ model,
1678
+ tracer,
1679
+ requestStartTime,
1680
+ isFallbackRequest,
1681
+ reasoningEffort,
1682
+ state,
1683
+ }),
1684
+ };
1685
+ const { settleBudget, recordFinalOutcome } = rt;
1222
1686
  const dispatch = async () => {
1687
+ let preparedContext;
1223
1688
  try {
1224
1689
  preparedContext = prepareProxyRequestContext({
1225
1690
  provider: "codex",
1226
1691
  model,
1227
- body,
1692
+ body: state.body,
1228
1693
  });
1229
- contextPreflight = preparedContext.evidence;
1230
- body = preparedContext.body;
1231
- bodyStr = JSON.stringify(body);
1694
+ state.contextPreflight = preparedContext.evidence;
1695
+ state.body = preparedContext.body;
1696
+ state.bodyStr = JSON.stringify(state.body);
1232
1697
  }
1233
1698
  catch (error) {
1234
1699
  if (!(error instanceof ProxyContextPreflightError)) {
1235
1700
  throw error;
1236
1701
  }
1237
- contextPreflight = error.evidence;
1702
+ state.contextPreflight = error.evidence;
1238
1703
  await recordFinalOutcome(undefined, error.status, {
1239
1704
  errorType: "context_preflight",
1240
1705
  errorCode: error.code,
@@ -1245,118 +1710,17 @@ loopbackPort, internalDispatch) {
1245
1710
  return buildCodexErrorResponse(error.status, error.message, error.code);
1246
1711
  }
1247
1712
  const accounts = await loadCodexProxyAccounts();
1248
- const cancelRequest = async (account) => {
1249
- await recordFinalOutcome(account, 499, {
1250
- errorType: "client_cancelled",
1251
- errorMessage: "Client cancelled Codex request",
1252
- terminalOutcome: "client_cancelled",
1253
- });
1254
- return buildCodexErrorResponse(499, "Client cancelled Codex request");
1255
- };
1256
1713
  if (ctx.abortSignal?.aborted) {
1257
- return cancelRequest();
1714
+ return cancelCodexRequest(rt);
1258
1715
  }
1259
1716
  if (accounts.length === 0) {
1260
- const outbound = await attemptCodexOutboundFallback({
1261
- ctx,
1262
- body,
1263
- model,
1264
- isFallbackRequest,
1265
- runtimeConfigProvider,
1266
- loopbackPort,
1267
- internalDispatch,
1268
- tracer,
1269
- recordFinalOutcome,
1270
- failureInput: { failureClass: "no_accounts" },
1271
- });
1272
- if (outbound.kind === "success") {
1273
- // No Codex account was ever selected on this path, so budgetLease is
1274
- // always undefined here — settleBudget() is a defensive no-op. Kept
1275
- // for consistency with the other three insertion points below.
1276
- await settleBudget();
1277
- budgetLease = undefined;
1278
- budgetDispatched = false;
1279
- return outbound.response;
1280
- }
1281
- if (outbound.kind === "request_too_large") {
1282
- await recordFinalOutcome(undefined, 413, {
1283
- errorType: "request_too_large",
1284
- errorMessage: outbound.message,
1285
- terminalOutcome: "handler_error",
1286
- });
1287
- return buildCodexErrorResponse(413, outbound.message, "request_too_large");
1288
- }
1289
- await recordFinalOutcome(undefined, 401, {
1290
- errorType: "no_accounts",
1291
- errorMessage: "No Codex accounts",
1292
- });
1293
- return buildCodexErrorResponse(401, "No Codex accounts configured. Run `neurolink auth login codex`.");
1717
+ return respondCodexNoAccounts(rt);
1294
1718
  }
1295
1719
  const now = Date.now();
1296
1720
  const ordered = orderCodexAccounts(accounts, now);
1297
1721
  const eligible = ordered.filter((a) => !(a.coolingUntil !== undefined && a.coolingUntil > now));
1298
1722
  if (eligible.length === 0) {
1299
- // Every account is cooling; surface the soonest recovery as retry-after.
1300
- const soonest = ordered.reduce((min, a) => {
1301
- if (a.coolingUntil === undefined) {
1302
- return min;
1303
- }
1304
- return min === undefined
1305
- ? a.coolingUntil
1306
- : Math.min(min, a.coolingUntil);
1307
- }, undefined);
1308
- const retryAfterSec = soonest
1309
- ? Math.max(1, Math.ceil((soonest - now) / 1000))
1310
- : 60;
1311
- const outbound = await attemptCodexOutboundFallback({
1312
- ctx,
1313
- body,
1314
- model,
1315
- isFallbackRequest,
1316
- runtimeConfigProvider,
1317
- loopbackPort,
1318
- internalDispatch,
1319
- tracer,
1320
- recordFinalOutcome,
1321
- failureInput: { failureClass: "pool_exhausted" },
1322
- });
1323
- if (outbound.kind === "success") {
1324
- // No Codex account was ever selected on this path, so budgetLease is
1325
- // always undefined here — settleBudget() is a defensive no-op. Kept
1326
- // for consistency with the other three insertion points below.
1327
- await settleBudget();
1328
- budgetLease = undefined;
1329
- budgetDispatched = false;
1330
- return outbound.response;
1331
- }
1332
- if (outbound.kind === "request_too_large") {
1333
- await recordFinalOutcome(undefined, 413, {
1334
- errorType: "request_too_large",
1335
- errorMessage: outbound.message,
1336
- terminalOutcome: "handler_error",
1337
- });
1338
- return buildCodexErrorResponse(413, outbound.message, "request_too_large");
1339
- }
1340
- const resetAt = soonest ? new Date(soonest).toISOString() : undefined;
1341
- // A refresh that failed transiently, or a 401 with no refresh token to
1342
- // retry with, parks the account as "auth". Reporting that as spent quota
1343
- // would tell the user the plan is exhausted when it is not.
1344
- if (ordered.every((account) => account.coolingReason === "auth")) {
1345
- await recordFinalOutcome(undefined, 503, {
1346
- errorType: "all_accounts_auth_cooling",
1347
- errorMessage: "All Codex accounts are cooling after an auth failure",
1348
- });
1349
- return buildCodexPoolCoolingResponse(resetAt
1350
- ? `Codex authentication is temporarily unavailable for every account. Retry after ${resetAt}, or run \`neurolink auth login codex\` if it persists.`
1351
- : "Codex authentication is temporarily unavailable for every account. Retry shortly, or run `neurolink auth login codex` if it persists.", retryAfterSec, "server_error");
1352
- }
1353
- await recordFinalOutcome(undefined, 429, {
1354
- errorType: "all_accounts_cooling",
1355
- errorMessage: "All Codex accounts are rate-limited",
1356
- });
1357
- return buildCodexPoolCoolingResponse(resetAt
1358
- ? `Codex quota exhausted: all accounts are rate-limited. Resets at ${resetAt}.`
1359
- : "Codex quota exhausted: all accounts are rate-limited.", retryAfterSec, "insufficient_quota");
1723
+ return respondCodexPoolCooling(rt, ordered, now);
1360
1724
  }
1361
1725
  let attempt = 0;
1362
1726
  let lastErrorMessage = "All Codex accounts failed";
@@ -1383,13 +1747,13 @@ loopbackPort, internalDispatch) {
1383
1747
  // Same-account loop only re-runs once, for a post-401 token refresh.
1384
1748
  for (;;) {
1385
1749
  if (ctx.abortSignal?.aborted) {
1386
- return cancelRequest(lastAttemptedAccount);
1750
+ return cancelCodexRequest(rt, lastAttemptedAccount);
1387
1751
  }
1388
1752
  await settleBudget();
1389
- budgetLease = undefined;
1390
- budgetDispatched = false;
1753
+ state.budgetLease = undefined;
1754
+ state.budgetDispatched = false;
1391
1755
  try {
1392
- budgetLease = await reserveProxyTokenBudget({
1756
+ state.budgetLease = await reserveProxyTokenBudget({
1393
1757
  provider: "codex",
1394
1758
  accountKey: account.key,
1395
1759
  sessionKey: getProxyTokenBudgetSessionKey(new Headers(ctx.headers)),
@@ -1413,7 +1777,7 @@ loopbackPort, internalDispatch) {
1413
1777
  return buildCodexErrorResponse(budgetError.status, budgetError.message, budgetError.code);
1414
1778
  }
1415
1779
  if (ctx.abortSignal?.aborted) {
1416
- return cancelRequest(account);
1780
+ return cancelCodexRequest(rt, account);
1417
1781
  }
1418
1782
  attempt += 1;
1419
1783
  const attemptStartedAt = Date.now();
@@ -1428,7 +1792,7 @@ loopbackPort, internalDispatch) {
1428
1792
  phase: "upstream_request",
1429
1793
  model,
1430
1794
  stream: true,
1431
- body: bodyStr,
1795
+ body: state.bodyStr,
1432
1796
  headers: upstreamHeaders,
1433
1797
  contentType: "application/json",
1434
1798
  account: account.label,
@@ -1436,427 +1800,72 @@ loopbackPort, internalDispatch) {
1436
1800
  attempt,
1437
1801
  ...tracer?.getTraceContext(),
1438
1802
  });
1439
- budgetDispatched = true;
1803
+ const upstreamSpan = tracer?.startUpstreamAttempt({
1804
+ attempt,
1805
+ account: account.label,
1806
+ polyfillHeaders: false,
1807
+ polyfillBody: false,
1808
+ upstreamUrl: CODEX_RESPONSES_URL,
1809
+ });
1810
+ state.budgetDispatched = true;
1440
1811
  upstream = await fetch(CODEX_RESPONSES_URL, {
1441
1812
  method: "POST",
1442
1813
  headers: upstreamHeaders,
1443
- body: bodyStr,
1814
+ body: state.bodyStr,
1444
1815
  signal: ctx.abortSignal
1445
1816
  ? AbortSignal.any([
1446
1817
  ctx.abortSignal,
1447
1818
  AbortSignal.timeout(CODEX_UPSTREAM_TIMEOUT_MS),
1448
1819
  ])
1449
1820
  : AbortSignal.timeout(CODEX_UPSTREAM_TIMEOUT_MS),
1450
- });
1821
+ }).finally(() => upstreamSpan?.end());
1451
1822
  }
1452
1823
  catch (error) {
1453
- if (ctx.abortSignal?.aborted) {
1454
- writeAttempt(account, attempt, attemptStartedAt, 499, {
1455
- errorType: "client_cancelled",
1456
- errorMessage: "Client cancelled Codex request",
1457
- retryable: false,
1458
- });
1459
- return cancelRequest(account);
1460
- }
1461
- // A transport failure message is derived from local state — resolved
1462
- // hostnames, socket paths, Node internals — and says nothing the caller
1463
- // can act on. Keep the detail in the log and return a fixed string, so
1464
- // internal topology never reaches the client.
1465
- logger.debug(`Codex upstream fetch failed (${account.label}): ${sanitizeForLog(error instanceof Error ? error.message : String(error))}`);
1466
- const errorMessage = summarizeCodexUpstreamError(error instanceof Error ? error.message : String(error), "Codex upstream request failed");
1467
- const errorCode = getCodexTransportErrorCode(error);
1468
- const transportScope = codexTransportScope(error);
1469
- recordAttemptError(account.label, CODEX_ACCOUNT_TYPE, 502);
1470
- writeAttempt(account, attempt, attemptStartedAt, 502, {
1471
- errorType: "network_error",
1472
- errorMessage,
1473
- ...(errorCode ? { errorCode } : {}),
1474
- transportScope,
1475
- // These codes prove failure before HTTP dispatch. Socket resets,
1476
- // EPIPE and generic timeouts may follow dispatch and must not replay.
1477
- retryable: [
1478
- "UND_ERR_CONNECT_TIMEOUT",
1479
- "ECONNREFUSED",
1480
- "ENOTFOUND",
1481
- "EAI_AGAIN",
1482
- ].includes(errorCode ?? ""),
1824
+ const failure = await handleCodexTransportFailure(rt, {
1825
+ error: error instanceof Error ? error : new Error(String(error)),
1826
+ account,
1827
+ attempt,
1828
+ attemptStartedAt,
1483
1829
  });
1484
- lastFailure = {
1485
- errorType: "network_error",
1486
- errorMessage,
1487
- errorCode,
1488
- transportScope,
1489
- };
1490
- if (![
1491
- "UND_ERR_CONNECT_TIMEOUT",
1492
- "ECONNREFUSED",
1493
- "ENOTFOUND",
1494
- "EAI_AGAIN",
1495
- ].includes(errorCode ?? "")) {
1496
- const outbound = await attemptCodexOutboundFallback({
1497
- ctx,
1498
- body,
1499
- model,
1500
- isFallbackRequest,
1501
- runtimeConfigProvider,
1502
- loopbackPort,
1503
- internalDispatch,
1504
- tracer,
1505
- recordFinalOutcome,
1506
- failureInput: {
1507
- failureClass: "non_retryable_transport",
1508
- transportErrorCode: errorCode,
1509
- },
1510
- });
1511
- if (outbound.kind === "success") {
1512
- // This Codex account's budget lease (reserved for the attempt
1513
- // that just failed transport-level) belongs to a request this
1514
- // turn is no longer serving via Codex at all — hold it open no
1515
- // longer than necessary rather than leaving it pending for the
1516
- // unrelated fallback stream's full duration.
1517
- await settleBudget();
1518
- budgetLease = undefined;
1519
- budgetDispatched = false;
1520
- return outbound.response;
1521
- }
1522
- if (outbound.kind === "request_too_large") {
1523
- await recordFinalOutcome(account, 413, {
1524
- errorType: "request_too_large",
1525
- errorMessage: outbound.message,
1526
- terminalOutcome: "handler_error",
1527
- });
1528
- return buildCodexErrorResponse(413, outbound.message, "request_too_large");
1529
- }
1530
- await recordFinalOutcome(account, 502, {
1531
- ...lastFailure,
1532
- errorMessage,
1533
- });
1534
- return buildCodexErrorResponse(502, "Codex upstream request failed");
1830
+ if (failure instanceof Response) {
1831
+ return failure;
1535
1832
  }
1536
- lastErrorMessage = "Codex upstream request failed";
1537
- lastErrorStatus = 502;
1538
- lastErrorCode = lastFailure.errorCode;
1539
- break; // rotate to next account
1833
+ ({ lastFailure, lastErrorStatus, lastErrorCode, lastErrorMessage } =
1834
+ failure);
1835
+ break;
1540
1836
  }
1541
1837
  upstream = captureCodexResponse(ctx, upstream, "upstream_response", account, attempt);
1542
1838
  if (upstream.ok) {
1543
- const quota = parseCodexRateLimitHeaders(upstream.headers);
1544
- if (quota) {
1545
- saveAccountQuota(account.key, quota).catch(() => undefined);
1546
- }
1547
- // A prior cooldown that has expired is cleared on success. The
1548
- // compare-and-swap guards against wiping a longer cooldown that another
1549
- // in-flight request set while this one was upstream.
1550
- if (account.expiredCooldownUntil !== undefined) {
1551
- clearAccountCooldown(account.key, account.expiredCooldownUntil).catch(() => undefined);
1552
- }
1553
- publishCodexHeaders(ctx, account, attempt, quota);
1554
- writeAttempt(account, attempt, attemptStartedAt, upstream.status);
1555
- const headers = {
1556
- "content-type": upstream.headers.get("content-type") ?? "text/event-stream",
1557
- "cache-control": "no-cache",
1558
- connection: "keep-alive",
1559
- ...(ctx.responseHeaders ?? {}),
1560
- };
1561
- if (!upstream.body) {
1562
- recordAttemptError(account.label, CODEX_ACCOUNT_TYPE, 502);
1563
- writeAttempt(account, attempt, attemptStartedAt, 502, {
1564
- errorType: "incomplete_stream",
1565
- errorMessage: "Codex returned no response stream",
1566
- retryable: false,
1567
- });
1568
- await recordFinalOutcome(account, 502, {
1569
- terminalOutcome: "stream_error",
1570
- errorType: "incomplete_stream",
1571
- errorMessage: "Codex returned no response stream",
1572
- });
1573
- return new Response(upstream.body, {
1574
- status: upstream.status,
1575
- headers,
1576
- });
1577
- }
1578
- const { stream: usageTap, usage: usageSeen, evidence, } = createCodexUsageTap();
1579
- const relay = new Response(upstream.body.pipeThrough(usageTap), {
1580
- status: upstream.status,
1581
- headers,
1582
- });
1583
- registerProxyResponseObserver(ctx.metadata, {
1584
- onTerminal: ({ outcome, error, observedBodyBytes }) => {
1585
- return usageSeen
1586
- .then((usage) => {
1587
- const semantic = evidence();
1588
- const completedFrameDelivered = semantic.completed &&
1589
- observedBodyBytes >= semantic.terminalBytes;
1590
- const failed = semantic.errorType ||
1591
- ((outcome === "completed" || outcome === "bodyless") &&
1592
- !semantic.completed);
1593
- const usageExtra = usage
1594
- ? {
1595
- inputTokens: usage.inputTokensObserved
1596
- ? usage.inputTokens
1597
- : undefined,
1598
- outputTokens: usage.outputTokensObserved
1599
- ? usage.outputTokens
1600
- : undefined,
1601
- cacheReadTokens: usage.cacheReadTokensObserved
1602
- ? usage.cacheReadTokens
1603
- : undefined,
1604
- cacheCreationTokens: usage.cacheCreationTokensObserved
1605
- ? usage.cacheCreationTokens
1606
- : undefined,
1607
- reasoningTokens: usage.reasoningTokensObserved
1608
- ? usage.reasoningTokens
1609
- : undefined,
1610
- }
1611
- : {};
1612
- if (usage?.inputTokensObserved &&
1613
- usage.outputTokensObserved) {
1614
- try {
1615
- tracer?.setUsage({
1616
- ...usage,
1617
- reasoningTokens: usage.reasoningTokensObserved
1618
- ? usage.reasoningTokens
1619
- : undefined,
1620
- cacheReadTokensObserved: usage.cacheReadTokensObserved === true,
1621
- cacheCreationTokensObserved: usage.cacheCreationTokensObserved === true,
1622
- inputIncludesCachedTokens: true,
1623
- });
1624
- }
1625
- catch {
1626
- // Pricing/metrics must never change the stream outcome.
1627
- }
1628
- }
1629
- const timing = semantic.firstUsefulOutputAt === undefined ||
1630
- semantic.observationIncomplete
1631
- ? {
1632
- firstUsefulOutputStatus: semantic.completed &&
1633
- !semantic.observationIncomplete
1634
- ? "no_useful_output"
1635
- : "not_observed",
1636
- }
1637
- : {
1638
- firstUsefulOutputStatus: "observed",
1639
- firstUsefulOutputEvent: semantic.firstUsefulOutputEvent,
1640
- firstUsefulOutputMs: Math.max(0, semantic.firstUsefulOutputAt - requestStartTime),
1641
- };
1642
- if (failed) {
1643
- recordAttemptError(account.label, CODEX_ACCOUNT_TYPE, 502);
1644
- writeAttempt(account, attempt, attemptStartedAt, 502, {
1645
- // An internal fallback has no separate final row. Keep
1646
- // its observed usage on this attempt when a later
1647
- // provider owns the client final.
1648
- ...usageExtra,
1649
- inputIncludesCachedTokens: true,
1650
- errorType: semantic.errorType ?? "incomplete_stream",
1651
- errorCode: semantic.errorCode,
1652
- errorMessage: semantic.errorMessage ??
1653
- "Codex stream ended without a completion event",
1654
- retryable: false,
1655
- });
1656
- return recordFinalOutcome(account, 502, {
1657
- ...usageExtra,
1658
- ...timing,
1659
- terminalOutcome: "stream_error",
1660
- errorType: semantic.errorType ?? "incomplete_stream",
1661
- errorCode: semantic.errorCode,
1662
- errorMessage: semantic.errorMessage ??
1663
- "Codex stream ended without a completion event",
1664
- });
1665
- }
1666
- if (outcome === "completed" ||
1667
- outcome === "bodyless" ||
1668
- (outcome === "client_cancelled" && completedFrameDelivered)) {
1669
- return recordFinalOutcome(account, upstream.status, {
1670
- terminalOutcome: "completed",
1671
- ...usageExtra,
1672
- ...timing,
1673
- });
1674
- }
1675
- if (outcome === "stream_error") {
1676
- recordAttemptError(account.label, CODEX_ACCOUNT_TYPE, 502);
1677
- writeAttempt(account, attempt, attemptStartedAt, 502, {
1678
- errorType: "stream_error",
1679
- errorCode: getCodexTransportErrorCode(error),
1680
- errorMessage: summarizeCodexUpstreamError(error instanceof Error ? error.message : "", "Codex upstream stream failed"),
1681
- retryable: false,
1682
- });
1683
- }
1684
- return recordFinalOutcome(account, outcome === "client_cancelled" ? 499 : 502, {
1685
- errorType: outcome === "client_cancelled"
1686
- ? "client_cancelled"
1687
- : "stream_error",
1688
- errorMessage: outcome === "client_cancelled"
1689
- ? "Client cancelled Codex stream"
1690
- : summarizeCodexUpstreamError(error instanceof Error ? error.message : "", "Codex upstream stream failed"),
1691
- ...(outcome === "stream_error"
1692
- ? { errorCode: getCodexTransportErrorCode(error) }
1693
- : {}),
1694
- terminalOutcome: outcome,
1695
- ...usageExtra,
1696
- ...timing,
1697
- });
1698
- })
1699
- .catch(() => undefined);
1700
- },
1839
+ return relayCodexSuccess(rt, {
1840
+ account,
1841
+ attempt,
1842
+ attemptStartedAt,
1843
+ upstream,
1701
1844
  });
1702
- return relay;
1703
1845
  }
1704
- const errText = await upstream.text().catch(() => "");
1705
- // 401/403 → try a forced token refresh once, then rotate.
1706
- if ((upstream.status === 401 || upstream.status === 403) &&
1707
- !authRetried &&
1708
- account.refreshToken) {
1709
- const errorMessage = summarizeCodexUpstreamError(errText, "Codex authentication rejected upstream");
1710
- recordAttemptError(account.label, CODEX_ACCOUNT_TYPE, upstream.status);
1711
- writeAttempt(account, attempt, attemptStartedAt, upstream.status, {
1712
- errorType: "authentication_error",
1713
- errorMessage,
1714
- retryable: true,
1715
- });
1846
+ const failure = await handleCodexUpstreamFailure(rt, {
1847
+ account,
1848
+ attempt,
1849
+ attemptStartedAt,
1850
+ upstream,
1851
+ authRetried,
1852
+ });
1853
+ if (failure === "retry") {
1716
1854
  authRetried = true;
1717
- const staleTokens = {
1718
- accessToken: account.token,
1719
- refreshToken: account.refreshToken,
1720
- expiresAt: account.expiresAt ?? 0,
1721
- };
1722
- try {
1723
- const refreshed = await refreshCodexTokenOnce(account.key, account.refreshToken);
1724
- account.token = refreshed.accessToken;
1725
- account.refreshToken =
1726
- refreshed.refreshToken ?? account.refreshToken;
1727
- account.expiresAt = refreshed.expiresAt ?? account.expiresAt;
1728
- account.accountId = resolveCodexAccountId(refreshed.accessToken);
1729
- continue; // retry same account with the fresh token
1730
- }
1731
- catch (error) {
1732
- if (isPermanentCodexRefreshFailure(error)) {
1733
- // Compare-and-swap: the pool is rebuilt per request with no shared
1734
- // state, so a concurrent request may already have rotated this
1735
- // credential. Disabling unconditionally would kill the account that
1736
- // the other request just healed.
1737
- const disabled = await tokenStore.markDisabledIfCurrent(account.key, staleTokens, "refresh_invalid");
1738
- if (disabled) {
1739
- logger.always(`[proxy] codex account=${account.label} disabled until re-authentication. Run: neurolink auth login codex --label ${account.label}`);
1740
- }
1741
- lastFailure = {
1742
- errorType: "authentication_error",
1743
- errorCode: "refresh_invalid",
1744
- };
1745
- lastErrorStatus = 401;
1746
- lastErrorCode = lastFailure.errorCode;
1747
- lastErrorMessage =
1748
- "Codex token refresh failed; re-login required";
1749
- break;
1750
- }
1751
- // No verdict on the credential — cool briefly and try the next
1752
- // account, so a 5xx or a timeout cannot cost the user a login.
1753
- await saveAccountCooldown(account.key, Date.now() + CODEX_AUTH_COOLDOWN_MS, "auth").catch(() => undefined);
1754
- logger.debug(`[proxy] codex account=${account.label} refresh failed transiently; cooling and rotating`);
1755
- lastFailure = {
1756
- errorType: "auth_refresh_unavailable",
1757
- errorCode: getCodexTransportErrorCode(error),
1758
- };
1759
- lastErrorStatus = 503;
1760
- lastErrorCode = lastFailure.errorCode;
1761
- lastErrorMessage = "Codex token refresh temporarily unavailable";
1762
- break;
1763
- }
1855
+ continue; // retry same account with the fresh token
1764
1856
  }
1765
- // 429 → cooldown + rotate.
1766
- if (upstream.status === 429) {
1767
- const quota = parseCodexRateLimitHeaders(upstream.headers);
1768
- if (quota) {
1769
- saveAccountQuota(account.key, quota).catch(() => undefined);
1770
- }
1771
- const retryAfterMs = parseRetryAfterMs(upstream.headers.get("retry-after"));
1772
- const now = Date.now();
1773
- const bodyQuota = parseCodexQuotaError(errText, now);
1774
- const plan = planCodexCooldown(quota, retryAfterMs, now, errText);
1775
- await saveAccountCooldown(account.key, plan.coolingUntil, plan.reason).catch(() => undefined);
1776
- const rateLimitKind = plan.reason === "transient" ? "transient" : "quota";
1777
- const errorMessage = summarizeCodexUpstreamError(errText, "Codex account rate-limited");
1778
- recordAttemptError(account.label, CODEX_ACCOUNT_TYPE, upstream.status, rateLimitKind);
1779
- writeAttempt(account, attempt, attemptStartedAt, upstream.status, {
1780
- errorType: "rate_limit_error",
1781
- errorMessage,
1782
- retryable: rateLimitKind === "transient",
1783
- rateLimitKind,
1784
- cooldownReason: plan.reason,
1785
- ...(bodyQuota
1786
- ? {
1787
- errorCode: bodyQuota.errorCode,
1788
- quotaResetAt: bodyQuota.resetAt || undefined,
1789
- quotaScope: bodyQuota.scope,
1790
- }
1791
- : {}),
1792
- });
1793
- lastFailure = {
1794
- errorType: "rate_limit_error",
1795
- errorCode: bodyQuota?.errorCode,
1796
- };
1797
- lastErrorStatus = 429;
1798
- lastErrorCode = lastFailure.errorCode;
1799
- lastErrorMessage = "Codex account rate-limited";
1800
- break; // rotate
1801
- }
1802
- // Other non-ok → record and rotate.
1803
- if (upstream.status === 401 || upstream.status === 403) {
1804
- // Reached only when the account has no refresh token to retry with, so
1805
- // it will fail identically on the next request. Park it briefly instead
1806
- // of letting it stay first in line with unknown quota.
1807
- await saveAccountCooldown(account.key, Date.now() + CODEX_AUTH_COOLDOWN_MS, "auth").catch(() => undefined);
1808
- }
1809
- const errorMessage = summarizeCodexUpstreamError(errText, "Codex error");
1810
- const errorType = upstream.status === 401 || upstream.status === 403
1811
- ? "authentication_error"
1812
- : "api_error";
1813
- recordAttemptError(account.label, CODEX_ACCOUNT_TYPE, upstream.status);
1814
- writeAttempt(account, attempt, attemptStartedAt, upstream.status, {
1815
- errorType,
1816
- errorMessage,
1817
- retryable: upstream.status >= 500,
1818
- });
1819
- lastFailure = { errorType };
1820
- lastErrorStatus = upstream.status >= 500 ? 502 : upstream.status;
1821
- lastErrorCode = getProxyUpstreamFailure({
1822
- responseBody: errText,
1823
- })?.code;
1824
- lastErrorMessage = errorMessage;
1825
- break; // rotate
1857
+ ({ lastFailure, lastErrorStatus, lastErrorCode, lastErrorMessage } =
1858
+ failure);
1859
+ break;
1826
1860
  }
1827
1861
  }
1828
- const outbound = await attemptCodexOutboundFallback({
1829
- ctx,
1830
- body,
1831
- model,
1832
- isFallbackRequest,
1833
- runtimeConfigProvider,
1834
- loopbackPort,
1835
- internalDispatch,
1836
- tracer,
1837
- recordFinalOutcome,
1838
- failureInput: {
1839
- failureClass: "loop_fallthrough",
1840
- lastStatus: lastErrorStatus,
1841
- lastErrorCode,
1842
- },
1843
- });
1844
- if (outbound.kind === "success") {
1845
- // Same reasoning as the transport-catch insertion point above: settle
1846
- // the exhausted Codex account's budget lease immediately rather than
1847
- // holding it open for the unrelated fallback stream's duration.
1848
- await settleBudget();
1849
- budgetLease = undefined;
1850
- budgetDispatched = false;
1851
- return outbound.response;
1852
- }
1853
- if (outbound.kind === "request_too_large") {
1854
- await recordFinalOutcome(lastAttemptedAccount, 413, {
1855
- errorType: "request_too_large",
1856
- errorMessage: outbound.message,
1857
- terminalOutcome: "handler_error",
1858
- });
1859
- return buildCodexErrorResponse(413, outbound.message, "request_too_large");
1862
+ const fallback = await resolveCodexOutbound(rt, {
1863
+ failureClass: "loop_fallthrough",
1864
+ lastStatus: lastErrorStatus,
1865
+ lastErrorCode,
1866
+ }, lastAttemptedAccount);
1867
+ if (fallback) {
1868
+ return fallback;
1860
1869
  }
1861
1870
  await recordFinalOutcome(lastAttemptedAccount, lastErrorStatus, {
1862
1871
  ...lastFailure,
@@ -2137,3 +2146,24 @@ export const __testHooks = {
2137
2146
  },
2138
2147
  wrapCodexOutboundResponseStream,
2139
2148
  };
2149
+ function codexResponseInfo(evidence) {
2150
+ const response = evidence.response;
2151
+ const output = Array.isArray(response?.output) ? response.output : undefined;
2152
+ const toolCalls = (output ?? []).flatMap((item) => item &&
2153
+ typeof item === "object" &&
2154
+ "name" in item &&
2155
+ typeof item.name === "string" &&
2156
+ "type" in item &&
2157
+ (item.type === "function_call" || item.type === "custom_tool_call")
2158
+ ? [item.name]
2159
+ : []);
2160
+ return {
2161
+ responseModel: typeof response?.model === "string" ? response.model : undefined,
2162
+ finishReason: typeof response?.status === "string" ? response.status : undefined,
2163
+ output,
2164
+ firstOutputAt: evidence.observationIncomplete
2165
+ ? undefined
2166
+ : evidence.firstUsefulOutputAt,
2167
+ ...(toolCalls.length > 0 && { toolCalls }),
2168
+ };
2169
+ }