@juspay/neurolink 12.58.1 → 13.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -3
- package/dist/auth/anthropicOAuth.js +3 -3
- package/dist/auth/middleware/AuthMiddleware.d.ts +1 -1
- package/dist/auth/sessionManager.js +4 -4
- package/dist/auth/tokenStore.js +4 -4
- package/dist/browser/neurolink.min.js +474 -473
- package/dist/cli/commands/observability.d.ts +1 -6
- package/dist/cli/commands/observability.js +0 -106
- package/dist/cli/commands/proxy.js +9 -18
- package/dist/cli/commands/telemetry.d.ts +3 -13
- package/dist/cli/commands/telemetry.js +7 -354
- package/dist/cli/loop/session.js +1 -1
- package/dist/client/httpClient.js +1 -1
- package/dist/client/sseClient.js +1 -1
- package/dist/client/streamingClient.js +2 -2
- package/dist/client/wsClient.js +1 -1
- package/dist/constants/langfuse.d.ts +25 -0
- package/dist/constants/langfuse.js +38 -0
- package/dist/constants/telemetry.d.ts +20 -0
- package/dist/constants/telemetry.js +34 -0
- package/dist/core/baseProvider.d.ts +2 -1
- package/dist/core/baseProvider.js +4 -11
- package/dist/core/conversationMemoryManager.js +5 -8
- package/dist/core/factory.js +1 -1
- package/dist/core/geminiLoopAdapter.d.ts +8 -1
- package/dist/core/geminiLoopAdapter.js +25 -1
- package/dist/core/loopEngine.js +69 -40
- package/dist/core/modules/MessageBuilder.js +2 -2
- package/dist/core/modules/StreamHandler.js +2 -2
- package/dist/core/modules/TelemetryHandler.d.ts +2 -18
- package/dist/core/modules/TelemetryHandler.js +1 -66
- package/dist/core/modules/ToolsManager.js +40 -31
- package/dist/core/nativeGenerateLoop.js +78 -8
- package/dist/core/redisConversationMemoryManager.js +8 -15
- package/dist/core/toolExecutionGuards.js +1 -13
- package/dist/index.d.ts +5 -12
- package/dist/index.js +5 -37
- package/dist/mcp/batching/requestBatcher.js +1 -1
- package/dist/mcp/externalServerManager.js +3 -21
- package/dist/mcp/toolDiscoveryService.js +4 -16
- package/dist/mcp/toolRegistry.js +2 -5
- package/dist/memory/memoryRetrievalTools.js +1 -1
- package/dist/neurolink.d.ts +16 -27
- package/dist/neurolink.js +788 -792
- package/dist/observability/exporterSpanProcessor.d.ts +50 -0
- package/dist/observability/exporterSpanProcessor.js +181 -0
- package/dist/observability/exporters/langfuseExporter.d.ts +13 -34
- package/dist/observability/exporters/langfuseExporter.js +101 -154
- package/dist/observability/index.d.ts +1 -0
- package/dist/observability/index.js +1 -0
- package/dist/observability/langfuseMasking.d.ts +8 -0
- package/dist/observability/langfuseMasking.js +47 -0
- package/dist/observability/otlpEncoding.d.ts +20 -0
- package/dist/observability/otlpEncoding.js +85 -0
- package/dist/providers/amazonBedrock/client.js +142 -176
- package/dist/providers/amazonBedrock/loopAdapter.js +14 -1
- package/dist/providers/amazonSagemaker.js +118 -80
- package/dist/providers/anthropic/client.js +3 -52
- package/dist/providers/anthropic/loopAdapter.js +24 -1
- package/dist/providers/anthropic/rateLimitCapture.js +9 -12
- package/dist/providers/googleAiStudio/client.d.ts +4 -5
- package/dist/providers/googleAiStudio/client.js +38 -40
- package/dist/providers/googleVertex/client.d.ts +2 -7
- package/dist/providers/googleVertex/client.js +282 -378
- package/dist/providers/googleVertex/constants.js +2 -2
- package/dist/providers/litellm/client.d.ts +2 -9
- package/dist/providers/litellm/client.js +1 -63
- package/dist/providers/openAI/client.d.ts +1 -8
- package/dist/providers/openAI/client.js +0 -65
- package/dist/providers/openaiChatCompletionsBase.d.ts +1 -0
- package/dist/providers/openaiChatCompletionsBase.js +61 -4
- package/dist/providers/replicate.js +22 -4
- package/dist/proxy/codexUsage.js +7 -0
- package/dist/proxy/proxyFetch.js +19 -25
- package/dist/proxy/proxyTracer.d.ts +14 -7
- package/dist/proxy/proxyTracer.js +129 -83
- package/dist/proxy/proxyTranslationEngine.js +0 -1
- package/dist/proxy/requestLogger.d.ts +1 -1
- package/dist/proxy/requestLogger.js +39 -45
- package/dist/server/middleware/common.js +1 -1
- package/dist/server/routes/agentRoutes.js +4 -4
- package/dist/server/routes/claudeProxyRoutes.js +99 -196
- package/dist/server/routes/codexProxyRoutes.js +621 -591
- package/dist/server/routes/healthRoutes.js +6 -6
- package/dist/server/routes/mcpRoutes.js +7 -7
- package/dist/server/routes/memoryRoutes.js +7 -7
- package/dist/server/routes/openApiRoutes.js +3 -3
- package/dist/server/routes/toolRoutes.js +5 -5
- package/dist/server/routes/transcribeRoutes.js +1 -1
- package/dist/server/utils/validation.d.ts +3 -3
- package/dist/services/server/ai/observability/instrumentation.d.ts +34 -201
- package/dist/services/server/ai/observability/instrumentation.js +220 -1267
- package/dist/services/server/ai/observability/langfuseSetup.d.ts +14 -0
- package/dist/services/server/ai/observability/langfuseSetup.js +55 -0
- package/dist/services/server/ai/observability/otlpSetup.d.ts +16 -0
- package/dist/services/server/ai/observability/otlpSetup.js +58 -0
- package/dist/session/globalSessionState.js +0 -11
- package/dist/telemetry/attributes.d.ts +23 -46
- package/dist/telemetry/attributes.js +76 -162
- package/dist/telemetry/chatMessages.d.ts +13 -0
- package/dist/telemetry/chatMessages.js +59 -0
- package/dist/telemetry/index.d.ts +5 -19
- package/dist/telemetry/index.js +5 -26
- package/dist/telemetry/observation.d.ts +37 -0
- package/dist/telemetry/observation.js +112 -0
- package/dist/telemetry/observationNames.d.ts +18 -0
- package/dist/telemetry/observationNames.js +24 -0
- package/dist/telemetry/spanSanitizer.d.ts +39 -0
- package/dist/telemetry/spanSanitizer.js +92 -0
- package/dist/telemetry/spanSerialization.d.ts +13 -0
- package/dist/telemetry/spanSerialization.js +46 -0
- package/dist/telemetry/tracers.d.ts +37 -18
- package/dist/telemetry/tracers.js +97 -19
- package/dist/telemetry/withSpan.d.ts +47 -25
- package/dist/telemetry/withSpan.js +158 -71
- package/dist/types/cli.d.ts +0 -11
- package/dist/types/decision.d.ts +3 -0
- package/dist/types/exporter.d.ts +54 -0
- package/dist/types/exporter.js +0 -4
- package/dist/types/generate.d.ts +4 -1
- package/dist/types/isolatedAgent.d.ts +5 -4
- package/dist/types/loopEngine.d.ts +18 -28
- package/dist/types/observability.d.ts +127 -160
- package/dist/types/providers.d.ts +15 -0
- package/dist/types/proxy.d.ts +41 -2
- package/dist/utils/anthropicTraceSanitizer.d.ts +2 -2
- package/dist/utils/anthropicTraceSanitizer.js +3 -1
- package/dist/utils/conversationMemory.js +4 -4
- package/dist/utils/fileDetector.js +1 -1
- package/dist/utils/messageBuilder.js +1 -1
- package/dist/utils/noOutputSentinel.d.ts +4 -16
- package/dist/utils/noOutputSentinel.js +12 -28
- package/dist/utils/providerRetry.js +4 -4
- package/dist/voice/livekit/brain.js +1 -1
- package/dist/voice/livekit/realtimeMemory.d.ts +1 -1
- package/dist/voice/livekit/realtimeMemory.js +66 -1
- package/dist/voice/livekit/realtimeVoiceAgent.js +1 -1
- package/docs-site/static/search-index.json +42 -63
- package/package.json +10 -4
- package/dist/telemetry/telemetryService.d.ts +0 -79
- package/dist/telemetry/telemetryService.js +0 -572
- package/dist/utils/observabilityHelpers.d.ts +0 -27
- package/dist/utils/observabilityHelpers.js +0 -47
|
@@ -1024,39 +1024,35 @@ function captureCodexResponse(ctx, response, phase, account, attempt) {
|
|
|
1024
1024
|
headers: response.headers,
|
|
1025
1025
|
});
|
|
1026
1026
|
}
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
let preparedContext;
|
|
1049
|
-
let contextPreflight;
|
|
1050
|
-
let budgetLease;
|
|
1051
|
-
let budgetDispatched = false;
|
|
1027
|
+
function startCodexTracer(ctx, model, body, isFallbackRequest) {
|
|
1028
|
+
try {
|
|
1029
|
+
return ProxyTracer.startRequest({
|
|
1030
|
+
requestId: ctx.requestId,
|
|
1031
|
+
method: ctx.method,
|
|
1032
|
+
path: ctx.path,
|
|
1033
|
+
model,
|
|
1034
|
+
stream: true,
|
|
1035
|
+
toolCount: Array.isArray(body.tools) ? body.tools.length : 0,
|
|
1036
|
+
provider: "openai",
|
|
1037
|
+
userAgent: ctx.headers["user-agent"],
|
|
1038
|
+
recordRequestMetrics: !isFallbackRequest,
|
|
1039
|
+
recordUsageMetrics: true,
|
|
1040
|
+
}, ctx.headers);
|
|
1041
|
+
}
|
|
1042
|
+
catch {
|
|
1043
|
+
return undefined;
|
|
1044
|
+
}
|
|
1045
|
+
}
|
|
1046
|
+
function createCodexRequestRecorder(input) {
|
|
1047
|
+
const { ctx, model, tracer, requestStartTime, isFallbackRequest, reasoningEffort, state, } = input;
|
|
1052
1048
|
const settleBudget = async (actualTokens) => {
|
|
1053
|
-
if (!budgetLease) {
|
|
1049
|
+
if (!state.budgetLease) {
|
|
1054
1050
|
return;
|
|
1055
1051
|
}
|
|
1056
1052
|
try {
|
|
1057
|
-
if (budgetDispatched) {
|
|
1058
|
-
await settleProxyTokenBudget(budgetLease, actualTokens);
|
|
1059
|
-
if (budgetLease.snapshot.settlement === "unconfirmed") {
|
|
1053
|
+
if (state.budgetDispatched) {
|
|
1054
|
+
await settleProxyTokenBudget(state.budgetLease, actualTokens);
|
|
1055
|
+
if (state.budgetLease.snapshot.settlement === "unconfirmed") {
|
|
1060
1056
|
emitProxyOtelEvent("token_budget", {
|
|
1061
1057
|
requestId: ctx.requestId,
|
|
1062
1058
|
event: "settlement_unconfirmed",
|
|
@@ -1064,7 +1060,7 @@ loopbackPort, internalDispatch) {
|
|
|
1064
1060
|
}
|
|
1065
1061
|
}
|
|
1066
1062
|
else {
|
|
1067
|
-
await budgetLease.cancelBeforeDispatch();
|
|
1063
|
+
await state.budgetLease.cancelBeforeDispatch();
|
|
1068
1064
|
}
|
|
1069
1065
|
}
|
|
1070
1066
|
catch (error) {
|
|
@@ -1076,40 +1072,6 @@ loopbackPort, internalDispatch) {
|
|
|
1076
1072
|
});
|
|
1077
1073
|
}
|
|
1078
1074
|
};
|
|
1079
|
-
const model = typeof body.model === "string"
|
|
1080
|
-
? body.model
|
|
1081
|
-
: "-";
|
|
1082
|
-
// A Codex call made as an inner Anthropic fallback is an upstream attempt,
|
|
1083
|
-
// not an independently final client request. The parent fallback owns the
|
|
1084
|
-
// final status and can still recover with a later provider.
|
|
1085
|
-
const isFallbackRequest = ctx.metadata?.[CODEX_FALLBACK_METADATA_KEY] === true;
|
|
1086
|
-
const reasoning = body.reasoning;
|
|
1087
|
-
const reasoningEffort = reasoning &&
|
|
1088
|
-
typeof reasoning === "object" &&
|
|
1089
|
-
"effort" in reasoning &&
|
|
1090
|
-
typeof reasoning.effort === "string"
|
|
1091
|
-
? reasoning.effort
|
|
1092
|
-
: undefined;
|
|
1093
|
-
let tracer;
|
|
1094
|
-
try {
|
|
1095
|
-
tracer = ProxyTracer.startRequest({
|
|
1096
|
-
requestId: ctx.requestId,
|
|
1097
|
-
method: ctx.method,
|
|
1098
|
-
path: ctx.path,
|
|
1099
|
-
model,
|
|
1100
|
-
stream: true,
|
|
1101
|
-
toolCount: Array.isArray(body.tools)
|
|
1102
|
-
? body.tools.length
|
|
1103
|
-
: 0,
|
|
1104
|
-
provider: "openai",
|
|
1105
|
-
userAgent: ctx.headers["user-agent"],
|
|
1106
|
-
recordRequestMetrics: !isFallbackRequest,
|
|
1107
|
-
recordUsageMetrics: true,
|
|
1108
|
-
}, ctx.headers);
|
|
1109
|
-
}
|
|
1110
|
-
catch {
|
|
1111
|
-
// Instrumentation must not change provider request handling.
|
|
1112
|
-
}
|
|
1113
1075
|
const writeFinalLog = (account, responseStatus, extra = {}) => logRequest({
|
|
1114
1076
|
timestamp: new Date().toISOString(),
|
|
1115
1077
|
requestId: ctx.requestId,
|
|
@@ -1117,12 +1079,12 @@ loopbackPort, internalDispatch) {
|
|
|
1117
1079
|
path: ctx.path,
|
|
1118
1080
|
model,
|
|
1119
1081
|
requestedModel: model,
|
|
1120
|
-
contextPreflight,
|
|
1121
|
-
tokenBudget: budgetLease
|
|
1082
|
+
contextPreflight: state.contextPreflight,
|
|
1083
|
+
tokenBudget: state.budgetLease
|
|
1084
|
+
? { ...state.budgetLease.snapshot }
|
|
1085
|
+
: undefined,
|
|
1122
1086
|
stream: true,
|
|
1123
|
-
toolCount: Array.isArray(body.tools)
|
|
1124
|
-
? body.tools.length
|
|
1125
|
-
: 0,
|
|
1087
|
+
toolCount: Array.isArray(state.body.tools) ? state.body.tools.length : 0,
|
|
1126
1088
|
account: account?.label ?? "",
|
|
1127
1089
|
...(account ? { accountKey: account.key } : {}),
|
|
1128
1090
|
accountType: account ? CODEX_ACCOUNT_TYPE : "",
|
|
@@ -1138,15 +1100,14 @@ loopbackPort, internalDispatch) {
|
|
|
1138
1100
|
responseTimeMs: Date.now() - requestStartTime,
|
|
1139
1101
|
...extra,
|
|
1140
1102
|
});
|
|
1141
|
-
let finalOutcomeRecorded = false;
|
|
1142
1103
|
const recordFinalOutcome = async (account, responseStatus, extra = {}) => {
|
|
1143
1104
|
await settleBudget(extra.inputTokens !== undefined && extra.outputTokens !== undefined
|
|
1144
1105
|
? extra.inputTokens + extra.outputTokens
|
|
1145
1106
|
: undefined);
|
|
1146
|
-
if (finalOutcomeRecorded) {
|
|
1107
|
+
if (state.finalOutcomeRecorded) {
|
|
1147
1108
|
return;
|
|
1148
1109
|
}
|
|
1149
|
-
finalOutcomeRecorded = true;
|
|
1110
|
+
state.finalOutcomeRecorded = true;
|
|
1150
1111
|
if (isProxyRequestFinalized(ctx.requestId)) {
|
|
1151
1112
|
// Runtime owns the client final, but this route still owns its span.
|
|
1152
1113
|
const status = ctx.abortSignal?.aborted
|
|
@@ -1203,12 +1164,12 @@ loopbackPort, internalDispatch) {
|
|
|
1203
1164
|
path: ctx.path,
|
|
1204
1165
|
model,
|
|
1205
1166
|
requestedModel: model,
|
|
1206
|
-
contextPreflight,
|
|
1207
|
-
tokenBudget: budgetLease
|
|
1167
|
+
contextPreflight: state.contextPreflight,
|
|
1168
|
+
tokenBudget: state.budgetLease
|
|
1169
|
+
? { ...state.budgetLease.snapshot }
|
|
1170
|
+
: undefined,
|
|
1208
1171
|
stream: true,
|
|
1209
|
-
toolCount: Array.isArray(body.tools)
|
|
1210
|
-
? body.tools.length
|
|
1211
|
-
: 0,
|
|
1172
|
+
toolCount: Array.isArray(state.body.tools) ? state.body.tools.length : 0,
|
|
1212
1173
|
account: account.label,
|
|
1213
1174
|
accountKey: account.key,
|
|
1214
1175
|
accountType: CODEX_ACCOUNT_TYPE,
|
|
@@ -1219,22 +1180,526 @@ loopbackPort, internalDispatch) {
|
|
|
1219
1180
|
...extra,
|
|
1220
1181
|
}).catch(() => undefined);
|
|
1221
1182
|
};
|
|
1183
|
+
return { settleBudget, recordFinalOutcome, writeAttempt };
|
|
1184
|
+
}
|
|
1185
|
+
async function resolveCodexOutbound(rt, failureInput, account) {
|
|
1186
|
+
const { state } = rt;
|
|
1187
|
+
const outbound = await attemptCodexOutboundFallback({
|
|
1188
|
+
ctx: rt.ctx,
|
|
1189
|
+
body: state.body,
|
|
1190
|
+
model: rt.model,
|
|
1191
|
+
isFallbackRequest: rt.isFallbackRequest,
|
|
1192
|
+
runtimeConfigProvider: rt.runtimeConfigProvider,
|
|
1193
|
+
loopbackPort: rt.loopbackPort,
|
|
1194
|
+
internalDispatch: rt.internalDispatch,
|
|
1195
|
+
tracer: rt.tracer,
|
|
1196
|
+
recordFinalOutcome: rt.recordFinalOutcome,
|
|
1197
|
+
failureInput,
|
|
1198
|
+
});
|
|
1199
|
+
if (outbound.kind === "success") {
|
|
1200
|
+
await rt.settleBudget();
|
|
1201
|
+
state.budgetLease = undefined;
|
|
1202
|
+
state.budgetDispatched = false;
|
|
1203
|
+
return outbound.response;
|
|
1204
|
+
}
|
|
1205
|
+
if (outbound.kind === "request_too_large") {
|
|
1206
|
+
await rt.recordFinalOutcome(account, 413, {
|
|
1207
|
+
errorType: "request_too_large",
|
|
1208
|
+
errorMessage: outbound.message,
|
|
1209
|
+
terminalOutcome: "handler_error",
|
|
1210
|
+
});
|
|
1211
|
+
return buildCodexErrorResponse(413, outbound.message, "request_too_large");
|
|
1212
|
+
}
|
|
1213
|
+
return undefined;
|
|
1214
|
+
}
|
|
1215
|
+
async function cancelCodexRequest(rt, account) {
|
|
1216
|
+
await rt.recordFinalOutcome(account, 499, {
|
|
1217
|
+
errorType: "client_cancelled",
|
|
1218
|
+
errorMessage: "Client cancelled Codex request",
|
|
1219
|
+
terminalOutcome: "client_cancelled",
|
|
1220
|
+
});
|
|
1221
|
+
return buildCodexErrorResponse(499, "Client cancelled Codex request");
|
|
1222
|
+
}
|
|
1223
|
+
async function respondCodexNoAccounts(rt) {
|
|
1224
|
+
const fallback = await resolveCodexOutbound(rt, { failureClass: "no_accounts" }, undefined);
|
|
1225
|
+
if (fallback) {
|
|
1226
|
+
return fallback;
|
|
1227
|
+
}
|
|
1228
|
+
await rt.recordFinalOutcome(undefined, 401, {
|
|
1229
|
+
errorType: "no_accounts",
|
|
1230
|
+
errorMessage: "No Codex accounts",
|
|
1231
|
+
});
|
|
1232
|
+
return buildCodexErrorResponse(401, "No Codex accounts configured. Run `neurolink auth login codex`.");
|
|
1233
|
+
}
|
|
1234
|
+
async function respondCodexPoolCooling(rt, ordered, now) {
|
|
1235
|
+
const { recordFinalOutcome } = rt;
|
|
1236
|
+
// Every account is cooling; surface the soonest recovery as retry-after.
|
|
1237
|
+
const soonest = ordered.reduce((min, a) => {
|
|
1238
|
+
if (a.coolingUntil === undefined) {
|
|
1239
|
+
return min;
|
|
1240
|
+
}
|
|
1241
|
+
return min === undefined ? a.coolingUntil : Math.min(min, a.coolingUntil);
|
|
1242
|
+
}, undefined);
|
|
1243
|
+
const retryAfterSec = soonest
|
|
1244
|
+
? Math.max(1, Math.ceil((soonest - now) / 1000))
|
|
1245
|
+
: 60;
|
|
1246
|
+
const fallback = await resolveCodexOutbound(rt, { failureClass: "pool_exhausted" }, undefined);
|
|
1247
|
+
if (fallback) {
|
|
1248
|
+
return fallback;
|
|
1249
|
+
}
|
|
1250
|
+
const resetAt = soonest ? new Date(soonest).toISOString() : undefined;
|
|
1251
|
+
// A refresh that failed transiently, or a 401 with no refresh token to
|
|
1252
|
+
// retry with, parks the account as "auth". Reporting that as spent quota
|
|
1253
|
+
// would tell the user the plan is exhausted when it is not.
|
|
1254
|
+
if (ordered.every((account) => account.coolingReason === "auth")) {
|
|
1255
|
+
await recordFinalOutcome(undefined, 503, {
|
|
1256
|
+
errorType: "all_accounts_auth_cooling",
|
|
1257
|
+
errorMessage: "All Codex accounts are cooling after an auth failure",
|
|
1258
|
+
});
|
|
1259
|
+
return buildCodexPoolCoolingResponse(resetAt
|
|
1260
|
+
? `Codex authentication is temporarily unavailable for every account. Retry after ${resetAt}, or run \`neurolink auth login codex\` if it persists.`
|
|
1261
|
+
: "Codex authentication is temporarily unavailable for every account. Retry shortly, or run `neurolink auth login codex` if it persists.", retryAfterSec, "server_error");
|
|
1262
|
+
}
|
|
1263
|
+
await recordFinalOutcome(undefined, 429, {
|
|
1264
|
+
errorType: "all_accounts_cooling",
|
|
1265
|
+
errorMessage: "All Codex accounts are rate-limited",
|
|
1266
|
+
});
|
|
1267
|
+
return buildCodexPoolCoolingResponse(resetAt
|
|
1268
|
+
? `Codex quota exhausted: all accounts are rate-limited. Resets at ${resetAt}.`
|
|
1269
|
+
: "Codex quota exhausted: all accounts are rate-limited.", retryAfterSec, "insufficient_quota");
|
|
1270
|
+
}
|
|
1271
|
+
async function relayCodexSuccess(rt, args) {
|
|
1272
|
+
const { ctx, writeAttempt, recordFinalOutcome } = rt;
|
|
1273
|
+
const { account, attempt, attemptStartedAt, upstream } = args;
|
|
1274
|
+
const quota = parseCodexRateLimitHeaders(upstream.headers);
|
|
1275
|
+
if (quota) {
|
|
1276
|
+
saveAccountQuota(account.key, quota).catch(() => undefined);
|
|
1277
|
+
}
|
|
1278
|
+
// A prior cooldown that has expired is cleared on success. The
|
|
1279
|
+
// compare-and-swap guards against wiping a longer cooldown that another
|
|
1280
|
+
// in-flight request set while this one was upstream.
|
|
1281
|
+
if (account.expiredCooldownUntil !== undefined) {
|
|
1282
|
+
clearAccountCooldown(account.key, account.expiredCooldownUntil).catch(() => undefined);
|
|
1283
|
+
}
|
|
1284
|
+
publishCodexHeaders(ctx, account, attempt, quota);
|
|
1285
|
+
writeAttempt(account, attempt, attemptStartedAt, upstream.status);
|
|
1286
|
+
const headers = {
|
|
1287
|
+
"content-type": upstream.headers.get("content-type") ?? "text/event-stream",
|
|
1288
|
+
"cache-control": "no-cache",
|
|
1289
|
+
connection: "keep-alive",
|
|
1290
|
+
...(ctx.responseHeaders ?? {}),
|
|
1291
|
+
};
|
|
1292
|
+
if (!upstream.body) {
|
|
1293
|
+
recordAttemptError(account.label, CODEX_ACCOUNT_TYPE, 502);
|
|
1294
|
+
writeAttempt(account, attempt, attemptStartedAt, 502, {
|
|
1295
|
+
errorType: "incomplete_stream",
|
|
1296
|
+
errorMessage: "Codex returned no response stream",
|
|
1297
|
+
retryable: false,
|
|
1298
|
+
});
|
|
1299
|
+
await recordFinalOutcome(account, 502, {
|
|
1300
|
+
terminalOutcome: "stream_error",
|
|
1301
|
+
errorType: "incomplete_stream",
|
|
1302
|
+
errorMessage: "Codex returned no response stream",
|
|
1303
|
+
});
|
|
1304
|
+
return new Response(upstream.body, {
|
|
1305
|
+
status: upstream.status,
|
|
1306
|
+
headers,
|
|
1307
|
+
});
|
|
1308
|
+
}
|
|
1309
|
+
const { stream: usageTap, usage: usageSeen, evidence, } = createCodexUsageTap();
|
|
1310
|
+
const relay = new Response(upstream.body.pipeThrough(usageTap), {
|
|
1311
|
+
status: upstream.status,
|
|
1312
|
+
headers,
|
|
1313
|
+
});
|
|
1314
|
+
registerProxyResponseObserver(ctx.metadata, {
|
|
1315
|
+
onTerminal: createCodexTerminalObserver(rt, {
|
|
1316
|
+
account,
|
|
1317
|
+
attempt,
|
|
1318
|
+
attemptStartedAt,
|
|
1319
|
+
upstreamStatus: upstream.status,
|
|
1320
|
+
usageSeen,
|
|
1321
|
+
evidence,
|
|
1322
|
+
}),
|
|
1323
|
+
});
|
|
1324
|
+
return relay;
|
|
1325
|
+
}
|
|
1326
|
+
async function handleCodexTransportFailure(rt, args) {
|
|
1327
|
+
const { ctx, writeAttempt, recordFinalOutcome } = rt;
|
|
1328
|
+
const { error, account, attempt, attemptStartedAt } = args;
|
|
1329
|
+
if (ctx.abortSignal?.aborted) {
|
|
1330
|
+
writeAttempt(account, attempt, attemptStartedAt, 499, {
|
|
1331
|
+
errorType: "client_cancelled",
|
|
1332
|
+
errorMessage: "Client cancelled Codex request",
|
|
1333
|
+
retryable: false,
|
|
1334
|
+
});
|
|
1335
|
+
return cancelCodexRequest(rt, account);
|
|
1336
|
+
}
|
|
1337
|
+
// A transport failure message is derived from local state — resolved
|
|
1338
|
+
// hostnames, socket paths, Node internals — and says nothing the caller
|
|
1339
|
+
// can act on. Keep the detail in the log and return a fixed string, so
|
|
1340
|
+
// internal topology never reaches the client.
|
|
1341
|
+
logger.debug(`Codex upstream fetch failed (${account.label}): ${sanitizeForLog(error instanceof Error ? error.message : String(error))}`);
|
|
1342
|
+
const errorMessage = summarizeCodexUpstreamError(error instanceof Error ? error.message : String(error), "Codex upstream request failed");
|
|
1343
|
+
const errorCode = getCodexTransportErrorCode(error);
|
|
1344
|
+
const transportScope = codexTransportScope(error);
|
|
1345
|
+
// These codes prove failure before HTTP dispatch. Socket resets,
|
|
1346
|
+
// EPIPE and generic timeouts may follow dispatch and must not replay.
|
|
1347
|
+
const retryable = [
|
|
1348
|
+
"UND_ERR_CONNECT_TIMEOUT",
|
|
1349
|
+
"ECONNREFUSED",
|
|
1350
|
+
"ENOTFOUND",
|
|
1351
|
+
"EAI_AGAIN",
|
|
1352
|
+
].includes(errorCode ?? "");
|
|
1353
|
+
recordAttemptError(account.label, CODEX_ACCOUNT_TYPE, 502);
|
|
1354
|
+
writeAttempt(account, attempt, attemptStartedAt, 502, {
|
|
1355
|
+
errorType: "network_error",
|
|
1356
|
+
errorMessage,
|
|
1357
|
+
...(errorCode ? { errorCode } : {}),
|
|
1358
|
+
transportScope,
|
|
1359
|
+
retryable,
|
|
1360
|
+
});
|
|
1361
|
+
const lastFailure = {
|
|
1362
|
+
errorType: "network_error",
|
|
1363
|
+
errorMessage,
|
|
1364
|
+
errorCode,
|
|
1365
|
+
transportScope,
|
|
1366
|
+
};
|
|
1367
|
+
if (retryable) {
|
|
1368
|
+
return {
|
|
1369
|
+
lastFailure,
|
|
1370
|
+
lastErrorStatus: 502,
|
|
1371
|
+
lastErrorCode: errorCode,
|
|
1372
|
+
lastErrorMessage: "Codex upstream request failed",
|
|
1373
|
+
};
|
|
1374
|
+
}
|
|
1375
|
+
const fallback = await resolveCodexOutbound(rt, {
|
|
1376
|
+
failureClass: "non_retryable_transport",
|
|
1377
|
+
transportErrorCode: errorCode,
|
|
1378
|
+
}, account);
|
|
1379
|
+
if (fallback) {
|
|
1380
|
+
return fallback;
|
|
1381
|
+
}
|
|
1382
|
+
await recordFinalOutcome(account, 502, { ...lastFailure, errorMessage });
|
|
1383
|
+
return buildCodexErrorResponse(502, "Codex upstream request failed");
|
|
1384
|
+
}
|
|
1385
|
+
async function handleCodexUpstreamFailure(rt, args) {
|
|
1386
|
+
const { writeAttempt } = rt;
|
|
1387
|
+
const { account, attempt, attemptStartedAt, upstream, authRetried } = args;
|
|
1388
|
+
const errText = await upstream.text().catch(() => "");
|
|
1389
|
+
// 401/403 → try a forced token refresh once, then rotate.
|
|
1390
|
+
if ((upstream.status === 401 || upstream.status === 403) &&
|
|
1391
|
+
!authRetried &&
|
|
1392
|
+
account.refreshToken) {
|
|
1393
|
+
const errorMessage = summarizeCodexUpstreamError(errText, "Codex authentication rejected upstream");
|
|
1394
|
+
recordAttemptError(account.label, CODEX_ACCOUNT_TYPE, upstream.status);
|
|
1395
|
+
writeAttempt(account, attempt, attemptStartedAt, upstream.status, {
|
|
1396
|
+
errorType: "authentication_error",
|
|
1397
|
+
errorMessage,
|
|
1398
|
+
retryable: true,
|
|
1399
|
+
});
|
|
1400
|
+
const staleTokens = {
|
|
1401
|
+
accessToken: account.token,
|
|
1402
|
+
refreshToken: account.refreshToken,
|
|
1403
|
+
expiresAt: account.expiresAt ?? 0,
|
|
1404
|
+
};
|
|
1405
|
+
try {
|
|
1406
|
+
const refreshed = await refreshCodexTokenOnce(account.key, account.refreshToken);
|
|
1407
|
+
account.token = refreshed.accessToken;
|
|
1408
|
+
account.refreshToken = refreshed.refreshToken ?? account.refreshToken;
|
|
1409
|
+
account.expiresAt = refreshed.expiresAt ?? account.expiresAt;
|
|
1410
|
+
account.accountId = resolveCodexAccountId(refreshed.accessToken);
|
|
1411
|
+
return "retry";
|
|
1412
|
+
}
|
|
1413
|
+
catch (error) {
|
|
1414
|
+
if (isPermanentCodexRefreshFailure(error)) {
|
|
1415
|
+
// Compare-and-swap: the pool is rebuilt per request with no shared
|
|
1416
|
+
// state, so a concurrent request may already have rotated this
|
|
1417
|
+
// credential. Disabling unconditionally would kill the account that
|
|
1418
|
+
// the other request just healed.
|
|
1419
|
+
const disabled = await tokenStore.markDisabledIfCurrent(account.key, staleTokens, "refresh_invalid");
|
|
1420
|
+
if (disabled) {
|
|
1421
|
+
logger.always(`[proxy] codex account=${account.label} disabled until re-authentication. Run: neurolink auth login codex --label ${account.label}`);
|
|
1422
|
+
}
|
|
1423
|
+
return {
|
|
1424
|
+
lastFailure: {
|
|
1425
|
+
errorType: "authentication_error",
|
|
1426
|
+
errorCode: "refresh_invalid",
|
|
1427
|
+
},
|
|
1428
|
+
lastErrorStatus: 401,
|
|
1429
|
+
lastErrorCode: "refresh_invalid",
|
|
1430
|
+
lastErrorMessage: "Codex token refresh failed; re-login required",
|
|
1431
|
+
};
|
|
1432
|
+
}
|
|
1433
|
+
// No verdict on the credential — cool briefly and try the next
|
|
1434
|
+
// account, so a 5xx or a timeout cannot cost the user a login.
|
|
1435
|
+
await saveAccountCooldown(account.key, Date.now() + CODEX_AUTH_COOLDOWN_MS, "auth").catch(() => undefined);
|
|
1436
|
+
logger.debug(`[proxy] codex account=${account.label} refresh failed transiently; cooling and rotating`);
|
|
1437
|
+
const errorCode = getCodexTransportErrorCode(error);
|
|
1438
|
+
return {
|
|
1439
|
+
lastFailure: { errorType: "auth_refresh_unavailable", errorCode },
|
|
1440
|
+
lastErrorStatus: 503,
|
|
1441
|
+
lastErrorCode: errorCode,
|
|
1442
|
+
lastErrorMessage: "Codex token refresh temporarily unavailable",
|
|
1443
|
+
};
|
|
1444
|
+
}
|
|
1445
|
+
}
|
|
1446
|
+
// 429 → cooldown + rotate.
|
|
1447
|
+
if (upstream.status === 429) {
|
|
1448
|
+
const quota = parseCodexRateLimitHeaders(upstream.headers);
|
|
1449
|
+
if (quota) {
|
|
1450
|
+
saveAccountQuota(account.key, quota).catch(() => undefined);
|
|
1451
|
+
}
|
|
1452
|
+
const retryAfterMs = parseRetryAfterMs(upstream.headers.get("retry-after"));
|
|
1453
|
+
const now = Date.now();
|
|
1454
|
+
const bodyQuota = parseCodexQuotaError(errText, now);
|
|
1455
|
+
const plan = planCodexCooldown(quota, retryAfterMs, now, errText);
|
|
1456
|
+
await saveAccountCooldown(account.key, plan.coolingUntil, plan.reason).catch(() => undefined);
|
|
1457
|
+
const rateLimitKind = plan.reason === "transient" ? "transient" : "quota";
|
|
1458
|
+
const errorMessage = summarizeCodexUpstreamError(errText, "Codex account rate-limited");
|
|
1459
|
+
recordAttemptError(account.label, CODEX_ACCOUNT_TYPE, upstream.status, rateLimitKind);
|
|
1460
|
+
writeAttempt(account, attempt, attemptStartedAt, upstream.status, {
|
|
1461
|
+
errorType: "rate_limit_error",
|
|
1462
|
+
errorMessage,
|
|
1463
|
+
retryable: rateLimitKind === "transient",
|
|
1464
|
+
rateLimitKind,
|
|
1465
|
+
cooldownReason: plan.reason,
|
|
1466
|
+
...(bodyQuota
|
|
1467
|
+
? {
|
|
1468
|
+
errorCode: bodyQuota.errorCode,
|
|
1469
|
+
quotaResetAt: bodyQuota.resetAt || undefined,
|
|
1470
|
+
quotaScope: bodyQuota.scope,
|
|
1471
|
+
}
|
|
1472
|
+
: {}),
|
|
1473
|
+
});
|
|
1474
|
+
return {
|
|
1475
|
+
lastFailure: {
|
|
1476
|
+
errorType: "rate_limit_error",
|
|
1477
|
+
errorCode: bodyQuota?.errorCode,
|
|
1478
|
+
},
|
|
1479
|
+
lastErrorStatus: 429,
|
|
1480
|
+
lastErrorCode: bodyQuota?.errorCode,
|
|
1481
|
+
lastErrorMessage: "Codex account rate-limited",
|
|
1482
|
+
};
|
|
1483
|
+
}
|
|
1484
|
+
// Other non-ok → record and rotate.
|
|
1485
|
+
if (upstream.status === 401 || upstream.status === 403) {
|
|
1486
|
+
// Reached only when the account has no refresh token to retry with, so
|
|
1487
|
+
// it will fail identically on the next request. Park it briefly instead
|
|
1488
|
+
// of letting it stay first in line with unknown quota.
|
|
1489
|
+
await saveAccountCooldown(account.key, Date.now() + CODEX_AUTH_COOLDOWN_MS, "auth").catch(() => undefined);
|
|
1490
|
+
}
|
|
1491
|
+
const errorMessage = summarizeCodexUpstreamError(errText, "Codex error");
|
|
1492
|
+
const errorType = upstream.status === 401 || upstream.status === 403
|
|
1493
|
+
? "authentication_error"
|
|
1494
|
+
: "api_error";
|
|
1495
|
+
recordAttemptError(account.label, CODEX_ACCOUNT_TYPE, upstream.status);
|
|
1496
|
+
writeAttempt(account, attempt, attemptStartedAt, upstream.status, {
|
|
1497
|
+
errorType,
|
|
1498
|
+
errorMessage,
|
|
1499
|
+
retryable: upstream.status >= 500,
|
|
1500
|
+
});
|
|
1501
|
+
return {
|
|
1502
|
+
lastFailure: { errorType },
|
|
1503
|
+
lastErrorStatus: upstream.status >= 500 ? 502 : upstream.status,
|
|
1504
|
+
lastErrorCode: getProxyUpstreamFailure({ responseBody: errText })?.code,
|
|
1505
|
+
lastErrorMessage: errorMessage,
|
|
1506
|
+
};
|
|
1507
|
+
}
|
|
1508
|
+
function createCodexTerminalObserver(rt, args) {
|
|
1509
|
+
const { tracer, requestStartTime, writeAttempt, recordFinalOutcome } = rt;
|
|
1510
|
+
const { account, attempt, attemptStartedAt, upstreamStatus, usageSeen, evidence, } = args;
|
|
1511
|
+
return ({ outcome, error, observedBodyBytes }) => {
|
|
1512
|
+
return usageSeen
|
|
1513
|
+
.then((usage) => {
|
|
1514
|
+
const semantic = evidence();
|
|
1515
|
+
const completedFrameDelivered = semantic.completed && observedBodyBytes >= semantic.terminalBytes;
|
|
1516
|
+
const failed = semantic.errorType ||
|
|
1517
|
+
((outcome === "completed" || outcome === "bodyless") &&
|
|
1518
|
+
!semantic.completed);
|
|
1519
|
+
const usageExtra = usage
|
|
1520
|
+
? {
|
|
1521
|
+
inputTokens: usage.inputTokensObserved
|
|
1522
|
+
? usage.inputTokens
|
|
1523
|
+
: undefined,
|
|
1524
|
+
outputTokens: usage.outputTokensObserved
|
|
1525
|
+
? usage.outputTokens
|
|
1526
|
+
: undefined,
|
|
1527
|
+
cacheReadTokens: usage.cacheReadTokensObserved
|
|
1528
|
+
? usage.cacheReadTokens
|
|
1529
|
+
: undefined,
|
|
1530
|
+
cacheCreationTokens: usage.cacheCreationTokensObserved
|
|
1531
|
+
? usage.cacheCreationTokens
|
|
1532
|
+
: undefined,
|
|
1533
|
+
reasoningTokens: usage.reasoningTokensObserved
|
|
1534
|
+
? usage.reasoningTokens
|
|
1535
|
+
: undefined,
|
|
1536
|
+
}
|
|
1537
|
+
: {};
|
|
1538
|
+
tracer?.setResponseInfo(codexResponseInfo(semantic));
|
|
1539
|
+
if (usage?.inputTokensObserved && usage.outputTokensObserved) {
|
|
1540
|
+
try {
|
|
1541
|
+
tracer?.setUsage({
|
|
1542
|
+
...usage,
|
|
1543
|
+
reasoningTokens: usage.reasoningTokensObserved
|
|
1544
|
+
? usage.reasoningTokens
|
|
1545
|
+
: undefined,
|
|
1546
|
+
cacheReadTokensObserved: usage.cacheReadTokensObserved === true,
|
|
1547
|
+
cacheCreationTokensObserved: usage.cacheCreationTokensObserved === true,
|
|
1548
|
+
inputIncludesCachedTokens: true,
|
|
1549
|
+
});
|
|
1550
|
+
}
|
|
1551
|
+
catch {
|
|
1552
|
+
// Pricing/metrics must never change the stream outcome.
|
|
1553
|
+
}
|
|
1554
|
+
}
|
|
1555
|
+
const timing = semantic.firstUsefulOutputAt === undefined ||
|
|
1556
|
+
semantic.observationIncomplete
|
|
1557
|
+
? {
|
|
1558
|
+
firstUsefulOutputStatus: semantic.completed && !semantic.observationIncomplete
|
|
1559
|
+
? "no_useful_output"
|
|
1560
|
+
: "not_observed",
|
|
1561
|
+
}
|
|
1562
|
+
: {
|
|
1563
|
+
firstUsefulOutputStatus: "observed",
|
|
1564
|
+
firstUsefulOutputEvent: semantic.firstUsefulOutputEvent,
|
|
1565
|
+
firstUsefulOutputMs: Math.max(0, semantic.firstUsefulOutputAt - requestStartTime),
|
|
1566
|
+
};
|
|
1567
|
+
if (failed) {
|
|
1568
|
+
recordAttemptError(account.label, CODEX_ACCOUNT_TYPE, 502);
|
|
1569
|
+
writeAttempt(account, attempt, attemptStartedAt, 502, {
|
|
1570
|
+
// An internal fallback has no separate final row. Keep
|
|
1571
|
+
// its observed usage on this attempt when a later
|
|
1572
|
+
// provider owns the client final.
|
|
1573
|
+
...usageExtra,
|
|
1574
|
+
inputIncludesCachedTokens: true,
|
|
1575
|
+
errorType: semantic.errorType ?? "incomplete_stream",
|
|
1576
|
+
errorCode: semantic.errorCode,
|
|
1577
|
+
errorMessage: semantic.errorMessage ??
|
|
1578
|
+
"Codex stream ended without a completion event",
|
|
1579
|
+
retryable: false,
|
|
1580
|
+
});
|
|
1581
|
+
return recordFinalOutcome(account, 502, {
|
|
1582
|
+
...usageExtra,
|
|
1583
|
+
...timing,
|
|
1584
|
+
terminalOutcome: "stream_error",
|
|
1585
|
+
errorType: semantic.errorType ?? "incomplete_stream",
|
|
1586
|
+
errorCode: semantic.errorCode,
|
|
1587
|
+
errorMessage: semantic.errorMessage ??
|
|
1588
|
+
"Codex stream ended without a completion event",
|
|
1589
|
+
});
|
|
1590
|
+
}
|
|
1591
|
+
if (outcome === "completed" ||
|
|
1592
|
+
outcome === "bodyless" ||
|
|
1593
|
+
(outcome === "client_cancelled" && completedFrameDelivered)) {
|
|
1594
|
+
return recordFinalOutcome(account, upstreamStatus, {
|
|
1595
|
+
terminalOutcome: "completed",
|
|
1596
|
+
...usageExtra,
|
|
1597
|
+
...timing,
|
|
1598
|
+
});
|
|
1599
|
+
}
|
|
1600
|
+
if (outcome === "stream_error") {
|
|
1601
|
+
recordAttemptError(account.label, CODEX_ACCOUNT_TYPE, 502);
|
|
1602
|
+
writeAttempt(account, attempt, attemptStartedAt, 502, {
|
|
1603
|
+
errorType: "stream_error",
|
|
1604
|
+
errorCode: getCodexTransportErrorCode(error),
|
|
1605
|
+
errorMessage: summarizeCodexUpstreamError(error instanceof Error ? error.message : "", "Codex upstream stream failed"),
|
|
1606
|
+
retryable: false,
|
|
1607
|
+
});
|
|
1608
|
+
}
|
|
1609
|
+
return recordFinalOutcome(account, outcome === "client_cancelled" ? 499 : 502, {
|
|
1610
|
+
errorType: outcome === "client_cancelled"
|
|
1611
|
+
? "client_cancelled"
|
|
1612
|
+
: "stream_error",
|
|
1613
|
+
errorMessage: outcome === "client_cancelled"
|
|
1614
|
+
? "Client cancelled Codex stream"
|
|
1615
|
+
: summarizeCodexUpstreamError(error instanceof Error ? error.message : "", "Codex upstream stream failed"),
|
|
1616
|
+
...(outcome === "stream_error"
|
|
1617
|
+
? { errorCode: getCodexTransportErrorCode(error) }
|
|
1618
|
+
: {}),
|
|
1619
|
+
terminalOutcome: outcome,
|
|
1620
|
+
...usageExtra,
|
|
1621
|
+
...timing,
|
|
1622
|
+
});
|
|
1623
|
+
})
|
|
1624
|
+
.catch(() => undefined);
|
|
1625
|
+
};
|
|
1626
|
+
}
|
|
1627
|
+
/**
|
|
1628
|
+
* Serve one native `/backend-api/codex/responses` turn.
|
|
1629
|
+
*
|
|
1630
|
+
* Owns the whole hop: account selection and rotation, the upstream request,
|
|
1631
|
+
* the SSE relay back to the caller, and the terminal accounting that turns the
|
|
1632
|
+
* observed wire usage into request-log and tracer records. Usage counts the
|
|
1633
|
+
* provider did not report are carried as unobserved rather than zero, so a
|
|
1634
|
+
* silent cache breakdown is never recorded as a cache miss.
|
|
1635
|
+
*/
|
|
1636
|
+
async function executeCodexResponsesRequest(ctx,
|
|
1637
|
+
// Read by the outbound-fallback insertion points below (stage-c-trigger.md)
|
|
1638
|
+
// to gate/target a Codex-outbound fallback attempt when the native Codex
|
|
1639
|
+
// pool itself fails.
|
|
1640
|
+
runtimeConfigProvider,
|
|
1641
|
+
// Threaded through from `createCodexProxyRoutes` to the anthropic-target
|
|
1642
|
+
// loopback leg of the outbound-fallback dispatcher (mirrors the
|
|
1643
|
+
// OpenAI->Anthropic bridge's `loopbackPort`/`internalDispatch`).
|
|
1644
|
+
loopbackPort, internalDispatch) {
|
|
1645
|
+
const requestStartTime = Date.now();
|
|
1646
|
+
const state = {
|
|
1647
|
+
body: (ctx.body ?? {}),
|
|
1648
|
+
bodyStr: "",
|
|
1649
|
+
budgetDispatched: false,
|
|
1650
|
+
finalOutcomeRecorded: false,
|
|
1651
|
+
};
|
|
1652
|
+
const model = typeof state.body.model === "string" ? state.body.model : "-";
|
|
1653
|
+
// A Codex call made as an inner Anthropic fallback is an upstream attempt,
|
|
1654
|
+
// not an independently final client request. The parent fallback owns the
|
|
1655
|
+
// final status and can still recover with a later provider.
|
|
1656
|
+
const isFallbackRequest = ctx.metadata?.[CODEX_FALLBACK_METADATA_KEY] === true;
|
|
1657
|
+
const reasoning = state.body.reasoning;
|
|
1658
|
+
const reasoningEffort = reasoning &&
|
|
1659
|
+
typeof reasoning === "object" &&
|
|
1660
|
+
"effort" in reasoning &&
|
|
1661
|
+
typeof reasoning.effort === "string"
|
|
1662
|
+
? reasoning.effort
|
|
1663
|
+
: undefined;
|
|
1664
|
+
const tracer = startCodexTracer(ctx, model, state.body, isFallbackRequest);
|
|
1665
|
+
const rt = {
|
|
1666
|
+
ctx,
|
|
1667
|
+
model,
|
|
1668
|
+
tracer,
|
|
1669
|
+
requestStartTime,
|
|
1670
|
+
isFallbackRequest,
|
|
1671
|
+
state,
|
|
1672
|
+
runtimeConfigProvider,
|
|
1673
|
+
loopbackPort,
|
|
1674
|
+
internalDispatch,
|
|
1675
|
+
...createCodexRequestRecorder({
|
|
1676
|
+
ctx,
|
|
1677
|
+
model,
|
|
1678
|
+
tracer,
|
|
1679
|
+
requestStartTime,
|
|
1680
|
+
isFallbackRequest,
|
|
1681
|
+
reasoningEffort,
|
|
1682
|
+
state,
|
|
1683
|
+
}),
|
|
1684
|
+
};
|
|
1685
|
+
const { settleBudget, recordFinalOutcome } = rt;
|
|
1222
1686
|
const dispatch = async () => {
|
|
1687
|
+
let preparedContext;
|
|
1223
1688
|
try {
|
|
1224
1689
|
preparedContext = prepareProxyRequestContext({
|
|
1225
1690
|
provider: "codex",
|
|
1226
1691
|
model,
|
|
1227
|
-
body,
|
|
1692
|
+
body: state.body,
|
|
1228
1693
|
});
|
|
1229
|
-
contextPreflight = preparedContext.evidence;
|
|
1230
|
-
body = preparedContext.body;
|
|
1231
|
-
bodyStr = JSON.stringify(body);
|
|
1694
|
+
state.contextPreflight = preparedContext.evidence;
|
|
1695
|
+
state.body = preparedContext.body;
|
|
1696
|
+
state.bodyStr = JSON.stringify(state.body);
|
|
1232
1697
|
}
|
|
1233
1698
|
catch (error) {
|
|
1234
1699
|
if (!(error instanceof ProxyContextPreflightError)) {
|
|
1235
1700
|
throw error;
|
|
1236
1701
|
}
|
|
1237
|
-
contextPreflight = error.evidence;
|
|
1702
|
+
state.contextPreflight = error.evidence;
|
|
1238
1703
|
await recordFinalOutcome(undefined, error.status, {
|
|
1239
1704
|
errorType: "context_preflight",
|
|
1240
1705
|
errorCode: error.code,
|
|
@@ -1245,118 +1710,17 @@ loopbackPort, internalDispatch) {
|
|
|
1245
1710
|
return buildCodexErrorResponse(error.status, error.message, error.code);
|
|
1246
1711
|
}
|
|
1247
1712
|
const accounts = await loadCodexProxyAccounts();
|
|
1248
|
-
const cancelRequest = async (account) => {
|
|
1249
|
-
await recordFinalOutcome(account, 499, {
|
|
1250
|
-
errorType: "client_cancelled",
|
|
1251
|
-
errorMessage: "Client cancelled Codex request",
|
|
1252
|
-
terminalOutcome: "client_cancelled",
|
|
1253
|
-
});
|
|
1254
|
-
return buildCodexErrorResponse(499, "Client cancelled Codex request");
|
|
1255
|
-
};
|
|
1256
1713
|
if (ctx.abortSignal?.aborted) {
|
|
1257
|
-
return
|
|
1714
|
+
return cancelCodexRequest(rt);
|
|
1258
1715
|
}
|
|
1259
1716
|
if (accounts.length === 0) {
|
|
1260
|
-
|
|
1261
|
-
ctx,
|
|
1262
|
-
body,
|
|
1263
|
-
model,
|
|
1264
|
-
isFallbackRequest,
|
|
1265
|
-
runtimeConfigProvider,
|
|
1266
|
-
loopbackPort,
|
|
1267
|
-
internalDispatch,
|
|
1268
|
-
tracer,
|
|
1269
|
-
recordFinalOutcome,
|
|
1270
|
-
failureInput: { failureClass: "no_accounts" },
|
|
1271
|
-
});
|
|
1272
|
-
if (outbound.kind === "success") {
|
|
1273
|
-
// No Codex account was ever selected on this path, so budgetLease is
|
|
1274
|
-
// always undefined here — settleBudget() is a defensive no-op. Kept
|
|
1275
|
-
// for consistency with the other three insertion points below.
|
|
1276
|
-
await settleBudget();
|
|
1277
|
-
budgetLease = undefined;
|
|
1278
|
-
budgetDispatched = false;
|
|
1279
|
-
return outbound.response;
|
|
1280
|
-
}
|
|
1281
|
-
if (outbound.kind === "request_too_large") {
|
|
1282
|
-
await recordFinalOutcome(undefined, 413, {
|
|
1283
|
-
errorType: "request_too_large",
|
|
1284
|
-
errorMessage: outbound.message,
|
|
1285
|
-
terminalOutcome: "handler_error",
|
|
1286
|
-
});
|
|
1287
|
-
return buildCodexErrorResponse(413, outbound.message, "request_too_large");
|
|
1288
|
-
}
|
|
1289
|
-
await recordFinalOutcome(undefined, 401, {
|
|
1290
|
-
errorType: "no_accounts",
|
|
1291
|
-
errorMessage: "No Codex accounts",
|
|
1292
|
-
});
|
|
1293
|
-
return buildCodexErrorResponse(401, "No Codex accounts configured. Run `neurolink auth login codex`.");
|
|
1717
|
+
return respondCodexNoAccounts(rt);
|
|
1294
1718
|
}
|
|
1295
1719
|
const now = Date.now();
|
|
1296
1720
|
const ordered = orderCodexAccounts(accounts, now);
|
|
1297
1721
|
const eligible = ordered.filter((a) => !(a.coolingUntil !== undefined && a.coolingUntil > now));
|
|
1298
1722
|
if (eligible.length === 0) {
|
|
1299
|
-
|
|
1300
|
-
const soonest = ordered.reduce((min, a) => {
|
|
1301
|
-
if (a.coolingUntil === undefined) {
|
|
1302
|
-
return min;
|
|
1303
|
-
}
|
|
1304
|
-
return min === undefined
|
|
1305
|
-
? a.coolingUntil
|
|
1306
|
-
: Math.min(min, a.coolingUntil);
|
|
1307
|
-
}, undefined);
|
|
1308
|
-
const retryAfterSec = soonest
|
|
1309
|
-
? Math.max(1, Math.ceil((soonest - now) / 1000))
|
|
1310
|
-
: 60;
|
|
1311
|
-
const outbound = await attemptCodexOutboundFallback({
|
|
1312
|
-
ctx,
|
|
1313
|
-
body,
|
|
1314
|
-
model,
|
|
1315
|
-
isFallbackRequest,
|
|
1316
|
-
runtimeConfigProvider,
|
|
1317
|
-
loopbackPort,
|
|
1318
|
-
internalDispatch,
|
|
1319
|
-
tracer,
|
|
1320
|
-
recordFinalOutcome,
|
|
1321
|
-
failureInput: { failureClass: "pool_exhausted" },
|
|
1322
|
-
});
|
|
1323
|
-
if (outbound.kind === "success") {
|
|
1324
|
-
// No Codex account was ever selected on this path, so budgetLease is
|
|
1325
|
-
// always undefined here — settleBudget() is a defensive no-op. Kept
|
|
1326
|
-
// for consistency with the other three insertion points below.
|
|
1327
|
-
await settleBudget();
|
|
1328
|
-
budgetLease = undefined;
|
|
1329
|
-
budgetDispatched = false;
|
|
1330
|
-
return outbound.response;
|
|
1331
|
-
}
|
|
1332
|
-
if (outbound.kind === "request_too_large") {
|
|
1333
|
-
await recordFinalOutcome(undefined, 413, {
|
|
1334
|
-
errorType: "request_too_large",
|
|
1335
|
-
errorMessage: outbound.message,
|
|
1336
|
-
terminalOutcome: "handler_error",
|
|
1337
|
-
});
|
|
1338
|
-
return buildCodexErrorResponse(413, outbound.message, "request_too_large");
|
|
1339
|
-
}
|
|
1340
|
-
const resetAt = soonest ? new Date(soonest).toISOString() : undefined;
|
|
1341
|
-
// A refresh that failed transiently, or a 401 with no refresh token to
|
|
1342
|
-
// retry with, parks the account as "auth". Reporting that as spent quota
|
|
1343
|
-
// would tell the user the plan is exhausted when it is not.
|
|
1344
|
-
if (ordered.every((account) => account.coolingReason === "auth")) {
|
|
1345
|
-
await recordFinalOutcome(undefined, 503, {
|
|
1346
|
-
errorType: "all_accounts_auth_cooling",
|
|
1347
|
-
errorMessage: "All Codex accounts are cooling after an auth failure",
|
|
1348
|
-
});
|
|
1349
|
-
return buildCodexPoolCoolingResponse(resetAt
|
|
1350
|
-
? `Codex authentication is temporarily unavailable for every account. Retry after ${resetAt}, or run \`neurolink auth login codex\` if it persists.`
|
|
1351
|
-
: "Codex authentication is temporarily unavailable for every account. Retry shortly, or run `neurolink auth login codex` if it persists.", retryAfterSec, "server_error");
|
|
1352
|
-
}
|
|
1353
|
-
await recordFinalOutcome(undefined, 429, {
|
|
1354
|
-
errorType: "all_accounts_cooling",
|
|
1355
|
-
errorMessage: "All Codex accounts are rate-limited",
|
|
1356
|
-
});
|
|
1357
|
-
return buildCodexPoolCoolingResponse(resetAt
|
|
1358
|
-
? `Codex quota exhausted: all accounts are rate-limited. Resets at ${resetAt}.`
|
|
1359
|
-
: "Codex quota exhausted: all accounts are rate-limited.", retryAfterSec, "insufficient_quota");
|
|
1723
|
+
return respondCodexPoolCooling(rt, ordered, now);
|
|
1360
1724
|
}
|
|
1361
1725
|
let attempt = 0;
|
|
1362
1726
|
let lastErrorMessage = "All Codex accounts failed";
|
|
@@ -1383,13 +1747,13 @@ loopbackPort, internalDispatch) {
|
|
|
1383
1747
|
// Same-account loop only re-runs once, for a post-401 token refresh.
|
|
1384
1748
|
for (;;) {
|
|
1385
1749
|
if (ctx.abortSignal?.aborted) {
|
|
1386
|
-
return
|
|
1750
|
+
return cancelCodexRequest(rt, lastAttemptedAccount);
|
|
1387
1751
|
}
|
|
1388
1752
|
await settleBudget();
|
|
1389
|
-
budgetLease = undefined;
|
|
1390
|
-
budgetDispatched = false;
|
|
1753
|
+
state.budgetLease = undefined;
|
|
1754
|
+
state.budgetDispatched = false;
|
|
1391
1755
|
try {
|
|
1392
|
-
budgetLease = await reserveProxyTokenBudget({
|
|
1756
|
+
state.budgetLease = await reserveProxyTokenBudget({
|
|
1393
1757
|
provider: "codex",
|
|
1394
1758
|
accountKey: account.key,
|
|
1395
1759
|
sessionKey: getProxyTokenBudgetSessionKey(new Headers(ctx.headers)),
|
|
@@ -1413,7 +1777,7 @@ loopbackPort, internalDispatch) {
|
|
|
1413
1777
|
return buildCodexErrorResponse(budgetError.status, budgetError.message, budgetError.code);
|
|
1414
1778
|
}
|
|
1415
1779
|
if (ctx.abortSignal?.aborted) {
|
|
1416
|
-
return
|
|
1780
|
+
return cancelCodexRequest(rt, account);
|
|
1417
1781
|
}
|
|
1418
1782
|
attempt += 1;
|
|
1419
1783
|
const attemptStartedAt = Date.now();
|
|
@@ -1428,7 +1792,7 @@ loopbackPort, internalDispatch) {
|
|
|
1428
1792
|
phase: "upstream_request",
|
|
1429
1793
|
model,
|
|
1430
1794
|
stream: true,
|
|
1431
|
-
body: bodyStr,
|
|
1795
|
+
body: state.bodyStr,
|
|
1432
1796
|
headers: upstreamHeaders,
|
|
1433
1797
|
contentType: "application/json",
|
|
1434
1798
|
account: account.label,
|
|
@@ -1436,427 +1800,72 @@ loopbackPort, internalDispatch) {
|
|
|
1436
1800
|
attempt,
|
|
1437
1801
|
...tracer?.getTraceContext(),
|
|
1438
1802
|
});
|
|
1439
|
-
|
|
1803
|
+
const upstreamSpan = tracer?.startUpstreamAttempt({
|
|
1804
|
+
attempt,
|
|
1805
|
+
account: account.label,
|
|
1806
|
+
polyfillHeaders: false,
|
|
1807
|
+
polyfillBody: false,
|
|
1808
|
+
upstreamUrl: CODEX_RESPONSES_URL,
|
|
1809
|
+
});
|
|
1810
|
+
state.budgetDispatched = true;
|
|
1440
1811
|
upstream = await fetch(CODEX_RESPONSES_URL, {
|
|
1441
1812
|
method: "POST",
|
|
1442
1813
|
headers: upstreamHeaders,
|
|
1443
|
-
body: bodyStr,
|
|
1814
|
+
body: state.bodyStr,
|
|
1444
1815
|
signal: ctx.abortSignal
|
|
1445
1816
|
? AbortSignal.any([
|
|
1446
1817
|
ctx.abortSignal,
|
|
1447
1818
|
AbortSignal.timeout(CODEX_UPSTREAM_TIMEOUT_MS),
|
|
1448
1819
|
])
|
|
1449
1820
|
: AbortSignal.timeout(CODEX_UPSTREAM_TIMEOUT_MS),
|
|
1450
|
-
});
|
|
1821
|
+
}).finally(() => upstreamSpan?.end());
|
|
1451
1822
|
}
|
|
1452
1823
|
catch (error) {
|
|
1453
|
-
|
|
1454
|
-
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
});
|
|
1459
|
-
return cancelRequest(account);
|
|
1460
|
-
}
|
|
1461
|
-
// A transport failure message is derived from local state — resolved
|
|
1462
|
-
// hostnames, socket paths, Node internals — and says nothing the caller
|
|
1463
|
-
// can act on. Keep the detail in the log and return a fixed string, so
|
|
1464
|
-
// internal topology never reaches the client.
|
|
1465
|
-
logger.debug(`Codex upstream fetch failed (${account.label}): ${sanitizeForLog(error instanceof Error ? error.message : String(error))}`);
|
|
1466
|
-
const errorMessage = summarizeCodexUpstreamError(error instanceof Error ? error.message : String(error), "Codex upstream request failed");
|
|
1467
|
-
const errorCode = getCodexTransportErrorCode(error);
|
|
1468
|
-
const transportScope = codexTransportScope(error);
|
|
1469
|
-
recordAttemptError(account.label, CODEX_ACCOUNT_TYPE, 502);
|
|
1470
|
-
writeAttempt(account, attempt, attemptStartedAt, 502, {
|
|
1471
|
-
errorType: "network_error",
|
|
1472
|
-
errorMessage,
|
|
1473
|
-
...(errorCode ? { errorCode } : {}),
|
|
1474
|
-
transportScope,
|
|
1475
|
-
// These codes prove failure before HTTP dispatch. Socket resets,
|
|
1476
|
-
// EPIPE and generic timeouts may follow dispatch and must not replay.
|
|
1477
|
-
retryable: [
|
|
1478
|
-
"UND_ERR_CONNECT_TIMEOUT",
|
|
1479
|
-
"ECONNREFUSED",
|
|
1480
|
-
"ENOTFOUND",
|
|
1481
|
-
"EAI_AGAIN",
|
|
1482
|
-
].includes(errorCode ?? ""),
|
|
1824
|
+
const failure = await handleCodexTransportFailure(rt, {
|
|
1825
|
+
error: error instanceof Error ? error : new Error(String(error)),
|
|
1826
|
+
account,
|
|
1827
|
+
attempt,
|
|
1828
|
+
attemptStartedAt,
|
|
1483
1829
|
});
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
errorMessage,
|
|
1487
|
-
errorCode,
|
|
1488
|
-
transportScope,
|
|
1489
|
-
};
|
|
1490
|
-
if (![
|
|
1491
|
-
"UND_ERR_CONNECT_TIMEOUT",
|
|
1492
|
-
"ECONNREFUSED",
|
|
1493
|
-
"ENOTFOUND",
|
|
1494
|
-
"EAI_AGAIN",
|
|
1495
|
-
].includes(errorCode ?? "")) {
|
|
1496
|
-
const outbound = await attemptCodexOutboundFallback({
|
|
1497
|
-
ctx,
|
|
1498
|
-
body,
|
|
1499
|
-
model,
|
|
1500
|
-
isFallbackRequest,
|
|
1501
|
-
runtimeConfigProvider,
|
|
1502
|
-
loopbackPort,
|
|
1503
|
-
internalDispatch,
|
|
1504
|
-
tracer,
|
|
1505
|
-
recordFinalOutcome,
|
|
1506
|
-
failureInput: {
|
|
1507
|
-
failureClass: "non_retryable_transport",
|
|
1508
|
-
transportErrorCode: errorCode,
|
|
1509
|
-
},
|
|
1510
|
-
});
|
|
1511
|
-
if (outbound.kind === "success") {
|
|
1512
|
-
// This Codex account's budget lease (reserved for the attempt
|
|
1513
|
-
// that just failed transport-level) belongs to a request this
|
|
1514
|
-
// turn is no longer serving via Codex at all — hold it open no
|
|
1515
|
-
// longer than necessary rather than leaving it pending for the
|
|
1516
|
-
// unrelated fallback stream's full duration.
|
|
1517
|
-
await settleBudget();
|
|
1518
|
-
budgetLease = undefined;
|
|
1519
|
-
budgetDispatched = false;
|
|
1520
|
-
return outbound.response;
|
|
1521
|
-
}
|
|
1522
|
-
if (outbound.kind === "request_too_large") {
|
|
1523
|
-
await recordFinalOutcome(account, 413, {
|
|
1524
|
-
errorType: "request_too_large",
|
|
1525
|
-
errorMessage: outbound.message,
|
|
1526
|
-
terminalOutcome: "handler_error",
|
|
1527
|
-
});
|
|
1528
|
-
return buildCodexErrorResponse(413, outbound.message, "request_too_large");
|
|
1529
|
-
}
|
|
1530
|
-
await recordFinalOutcome(account, 502, {
|
|
1531
|
-
...lastFailure,
|
|
1532
|
-
errorMessage,
|
|
1533
|
-
});
|
|
1534
|
-
return buildCodexErrorResponse(502, "Codex upstream request failed");
|
|
1830
|
+
if (failure instanceof Response) {
|
|
1831
|
+
return failure;
|
|
1535
1832
|
}
|
|
1536
|
-
|
|
1537
|
-
|
|
1538
|
-
|
|
1539
|
-
break; // rotate to next account
|
|
1833
|
+
({ lastFailure, lastErrorStatus, lastErrorCode, lastErrorMessage } =
|
|
1834
|
+
failure);
|
|
1835
|
+
break;
|
|
1540
1836
|
}
|
|
1541
1837
|
upstream = captureCodexResponse(ctx, upstream, "upstream_response", account, attempt);
|
|
1542
1838
|
if (upstream.ok) {
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
|
|
1548
|
-
// compare-and-swap guards against wiping a longer cooldown that another
|
|
1549
|
-
// in-flight request set while this one was upstream.
|
|
1550
|
-
if (account.expiredCooldownUntil !== undefined) {
|
|
1551
|
-
clearAccountCooldown(account.key, account.expiredCooldownUntil).catch(() => undefined);
|
|
1552
|
-
}
|
|
1553
|
-
publishCodexHeaders(ctx, account, attempt, quota);
|
|
1554
|
-
writeAttempt(account, attempt, attemptStartedAt, upstream.status);
|
|
1555
|
-
const headers = {
|
|
1556
|
-
"content-type": upstream.headers.get("content-type") ?? "text/event-stream",
|
|
1557
|
-
"cache-control": "no-cache",
|
|
1558
|
-
connection: "keep-alive",
|
|
1559
|
-
...(ctx.responseHeaders ?? {}),
|
|
1560
|
-
};
|
|
1561
|
-
if (!upstream.body) {
|
|
1562
|
-
recordAttemptError(account.label, CODEX_ACCOUNT_TYPE, 502);
|
|
1563
|
-
writeAttempt(account, attempt, attemptStartedAt, 502, {
|
|
1564
|
-
errorType: "incomplete_stream",
|
|
1565
|
-
errorMessage: "Codex returned no response stream",
|
|
1566
|
-
retryable: false,
|
|
1567
|
-
});
|
|
1568
|
-
await recordFinalOutcome(account, 502, {
|
|
1569
|
-
terminalOutcome: "stream_error",
|
|
1570
|
-
errorType: "incomplete_stream",
|
|
1571
|
-
errorMessage: "Codex returned no response stream",
|
|
1572
|
-
});
|
|
1573
|
-
return new Response(upstream.body, {
|
|
1574
|
-
status: upstream.status,
|
|
1575
|
-
headers,
|
|
1576
|
-
});
|
|
1577
|
-
}
|
|
1578
|
-
const { stream: usageTap, usage: usageSeen, evidence, } = createCodexUsageTap();
|
|
1579
|
-
const relay = new Response(upstream.body.pipeThrough(usageTap), {
|
|
1580
|
-
status: upstream.status,
|
|
1581
|
-
headers,
|
|
1582
|
-
});
|
|
1583
|
-
registerProxyResponseObserver(ctx.metadata, {
|
|
1584
|
-
onTerminal: ({ outcome, error, observedBodyBytes }) => {
|
|
1585
|
-
return usageSeen
|
|
1586
|
-
.then((usage) => {
|
|
1587
|
-
const semantic = evidence();
|
|
1588
|
-
const completedFrameDelivered = semantic.completed &&
|
|
1589
|
-
observedBodyBytes >= semantic.terminalBytes;
|
|
1590
|
-
const failed = semantic.errorType ||
|
|
1591
|
-
((outcome === "completed" || outcome === "bodyless") &&
|
|
1592
|
-
!semantic.completed);
|
|
1593
|
-
const usageExtra = usage
|
|
1594
|
-
? {
|
|
1595
|
-
inputTokens: usage.inputTokensObserved
|
|
1596
|
-
? usage.inputTokens
|
|
1597
|
-
: undefined,
|
|
1598
|
-
outputTokens: usage.outputTokensObserved
|
|
1599
|
-
? usage.outputTokens
|
|
1600
|
-
: undefined,
|
|
1601
|
-
cacheReadTokens: usage.cacheReadTokensObserved
|
|
1602
|
-
? usage.cacheReadTokens
|
|
1603
|
-
: undefined,
|
|
1604
|
-
cacheCreationTokens: usage.cacheCreationTokensObserved
|
|
1605
|
-
? usage.cacheCreationTokens
|
|
1606
|
-
: undefined,
|
|
1607
|
-
reasoningTokens: usage.reasoningTokensObserved
|
|
1608
|
-
? usage.reasoningTokens
|
|
1609
|
-
: undefined,
|
|
1610
|
-
}
|
|
1611
|
-
: {};
|
|
1612
|
-
if (usage?.inputTokensObserved &&
|
|
1613
|
-
usage.outputTokensObserved) {
|
|
1614
|
-
try {
|
|
1615
|
-
tracer?.setUsage({
|
|
1616
|
-
...usage,
|
|
1617
|
-
reasoningTokens: usage.reasoningTokensObserved
|
|
1618
|
-
? usage.reasoningTokens
|
|
1619
|
-
: undefined,
|
|
1620
|
-
cacheReadTokensObserved: usage.cacheReadTokensObserved === true,
|
|
1621
|
-
cacheCreationTokensObserved: usage.cacheCreationTokensObserved === true,
|
|
1622
|
-
inputIncludesCachedTokens: true,
|
|
1623
|
-
});
|
|
1624
|
-
}
|
|
1625
|
-
catch {
|
|
1626
|
-
// Pricing/metrics must never change the stream outcome.
|
|
1627
|
-
}
|
|
1628
|
-
}
|
|
1629
|
-
const timing = semantic.firstUsefulOutputAt === undefined ||
|
|
1630
|
-
semantic.observationIncomplete
|
|
1631
|
-
? {
|
|
1632
|
-
firstUsefulOutputStatus: semantic.completed &&
|
|
1633
|
-
!semantic.observationIncomplete
|
|
1634
|
-
? "no_useful_output"
|
|
1635
|
-
: "not_observed",
|
|
1636
|
-
}
|
|
1637
|
-
: {
|
|
1638
|
-
firstUsefulOutputStatus: "observed",
|
|
1639
|
-
firstUsefulOutputEvent: semantic.firstUsefulOutputEvent,
|
|
1640
|
-
firstUsefulOutputMs: Math.max(0, semantic.firstUsefulOutputAt - requestStartTime),
|
|
1641
|
-
};
|
|
1642
|
-
if (failed) {
|
|
1643
|
-
recordAttemptError(account.label, CODEX_ACCOUNT_TYPE, 502);
|
|
1644
|
-
writeAttempt(account, attempt, attemptStartedAt, 502, {
|
|
1645
|
-
// An internal fallback has no separate final row. Keep
|
|
1646
|
-
// its observed usage on this attempt when a later
|
|
1647
|
-
// provider owns the client final.
|
|
1648
|
-
...usageExtra,
|
|
1649
|
-
inputIncludesCachedTokens: true,
|
|
1650
|
-
errorType: semantic.errorType ?? "incomplete_stream",
|
|
1651
|
-
errorCode: semantic.errorCode,
|
|
1652
|
-
errorMessage: semantic.errorMessage ??
|
|
1653
|
-
"Codex stream ended without a completion event",
|
|
1654
|
-
retryable: false,
|
|
1655
|
-
});
|
|
1656
|
-
return recordFinalOutcome(account, 502, {
|
|
1657
|
-
...usageExtra,
|
|
1658
|
-
...timing,
|
|
1659
|
-
terminalOutcome: "stream_error",
|
|
1660
|
-
errorType: semantic.errorType ?? "incomplete_stream",
|
|
1661
|
-
errorCode: semantic.errorCode,
|
|
1662
|
-
errorMessage: semantic.errorMessage ??
|
|
1663
|
-
"Codex stream ended without a completion event",
|
|
1664
|
-
});
|
|
1665
|
-
}
|
|
1666
|
-
if (outcome === "completed" ||
|
|
1667
|
-
outcome === "bodyless" ||
|
|
1668
|
-
(outcome === "client_cancelled" && completedFrameDelivered)) {
|
|
1669
|
-
return recordFinalOutcome(account, upstream.status, {
|
|
1670
|
-
terminalOutcome: "completed",
|
|
1671
|
-
...usageExtra,
|
|
1672
|
-
...timing,
|
|
1673
|
-
});
|
|
1674
|
-
}
|
|
1675
|
-
if (outcome === "stream_error") {
|
|
1676
|
-
recordAttemptError(account.label, CODEX_ACCOUNT_TYPE, 502);
|
|
1677
|
-
writeAttempt(account, attempt, attemptStartedAt, 502, {
|
|
1678
|
-
errorType: "stream_error",
|
|
1679
|
-
errorCode: getCodexTransportErrorCode(error),
|
|
1680
|
-
errorMessage: summarizeCodexUpstreamError(error instanceof Error ? error.message : "", "Codex upstream stream failed"),
|
|
1681
|
-
retryable: false,
|
|
1682
|
-
});
|
|
1683
|
-
}
|
|
1684
|
-
return recordFinalOutcome(account, outcome === "client_cancelled" ? 499 : 502, {
|
|
1685
|
-
errorType: outcome === "client_cancelled"
|
|
1686
|
-
? "client_cancelled"
|
|
1687
|
-
: "stream_error",
|
|
1688
|
-
errorMessage: outcome === "client_cancelled"
|
|
1689
|
-
? "Client cancelled Codex stream"
|
|
1690
|
-
: summarizeCodexUpstreamError(error instanceof Error ? error.message : "", "Codex upstream stream failed"),
|
|
1691
|
-
...(outcome === "stream_error"
|
|
1692
|
-
? { errorCode: getCodexTransportErrorCode(error) }
|
|
1693
|
-
: {}),
|
|
1694
|
-
terminalOutcome: outcome,
|
|
1695
|
-
...usageExtra,
|
|
1696
|
-
...timing,
|
|
1697
|
-
});
|
|
1698
|
-
})
|
|
1699
|
-
.catch(() => undefined);
|
|
1700
|
-
},
|
|
1839
|
+
return relayCodexSuccess(rt, {
|
|
1840
|
+
account,
|
|
1841
|
+
attempt,
|
|
1842
|
+
attemptStartedAt,
|
|
1843
|
+
upstream,
|
|
1701
1844
|
});
|
|
1702
|
-
return relay;
|
|
1703
1845
|
}
|
|
1704
|
-
const
|
|
1705
|
-
|
|
1706
|
-
|
|
1707
|
-
|
|
1708
|
-
|
|
1709
|
-
|
|
1710
|
-
|
|
1711
|
-
|
|
1712
|
-
errorType: "authentication_error",
|
|
1713
|
-
errorMessage,
|
|
1714
|
-
retryable: true,
|
|
1715
|
-
});
|
|
1846
|
+
const failure = await handleCodexUpstreamFailure(rt, {
|
|
1847
|
+
account,
|
|
1848
|
+
attempt,
|
|
1849
|
+
attemptStartedAt,
|
|
1850
|
+
upstream,
|
|
1851
|
+
authRetried,
|
|
1852
|
+
});
|
|
1853
|
+
if (failure === "retry") {
|
|
1716
1854
|
authRetried = true;
|
|
1717
|
-
|
|
1718
|
-
accessToken: account.token,
|
|
1719
|
-
refreshToken: account.refreshToken,
|
|
1720
|
-
expiresAt: account.expiresAt ?? 0,
|
|
1721
|
-
};
|
|
1722
|
-
try {
|
|
1723
|
-
const refreshed = await refreshCodexTokenOnce(account.key, account.refreshToken);
|
|
1724
|
-
account.token = refreshed.accessToken;
|
|
1725
|
-
account.refreshToken =
|
|
1726
|
-
refreshed.refreshToken ?? account.refreshToken;
|
|
1727
|
-
account.expiresAt = refreshed.expiresAt ?? account.expiresAt;
|
|
1728
|
-
account.accountId = resolveCodexAccountId(refreshed.accessToken);
|
|
1729
|
-
continue; // retry same account with the fresh token
|
|
1730
|
-
}
|
|
1731
|
-
catch (error) {
|
|
1732
|
-
if (isPermanentCodexRefreshFailure(error)) {
|
|
1733
|
-
// Compare-and-swap: the pool is rebuilt per request with no shared
|
|
1734
|
-
// state, so a concurrent request may already have rotated this
|
|
1735
|
-
// credential. Disabling unconditionally would kill the account that
|
|
1736
|
-
// the other request just healed.
|
|
1737
|
-
const disabled = await tokenStore.markDisabledIfCurrent(account.key, staleTokens, "refresh_invalid");
|
|
1738
|
-
if (disabled) {
|
|
1739
|
-
logger.always(`[proxy] codex account=${account.label} disabled until re-authentication. Run: neurolink auth login codex --label ${account.label}`);
|
|
1740
|
-
}
|
|
1741
|
-
lastFailure = {
|
|
1742
|
-
errorType: "authentication_error",
|
|
1743
|
-
errorCode: "refresh_invalid",
|
|
1744
|
-
};
|
|
1745
|
-
lastErrorStatus = 401;
|
|
1746
|
-
lastErrorCode = lastFailure.errorCode;
|
|
1747
|
-
lastErrorMessage =
|
|
1748
|
-
"Codex token refresh failed; re-login required";
|
|
1749
|
-
break;
|
|
1750
|
-
}
|
|
1751
|
-
// No verdict on the credential — cool briefly and try the next
|
|
1752
|
-
// account, so a 5xx or a timeout cannot cost the user a login.
|
|
1753
|
-
await saveAccountCooldown(account.key, Date.now() + CODEX_AUTH_COOLDOWN_MS, "auth").catch(() => undefined);
|
|
1754
|
-
logger.debug(`[proxy] codex account=${account.label} refresh failed transiently; cooling and rotating`);
|
|
1755
|
-
lastFailure = {
|
|
1756
|
-
errorType: "auth_refresh_unavailable",
|
|
1757
|
-
errorCode: getCodexTransportErrorCode(error),
|
|
1758
|
-
};
|
|
1759
|
-
lastErrorStatus = 503;
|
|
1760
|
-
lastErrorCode = lastFailure.errorCode;
|
|
1761
|
-
lastErrorMessage = "Codex token refresh temporarily unavailable";
|
|
1762
|
-
break;
|
|
1763
|
-
}
|
|
1855
|
+
continue; // retry same account with the fresh token
|
|
1764
1856
|
}
|
|
1765
|
-
|
|
1766
|
-
|
|
1767
|
-
|
|
1768
|
-
if (quota) {
|
|
1769
|
-
saveAccountQuota(account.key, quota).catch(() => undefined);
|
|
1770
|
-
}
|
|
1771
|
-
const retryAfterMs = parseRetryAfterMs(upstream.headers.get("retry-after"));
|
|
1772
|
-
const now = Date.now();
|
|
1773
|
-
const bodyQuota = parseCodexQuotaError(errText, now);
|
|
1774
|
-
const plan = planCodexCooldown(quota, retryAfterMs, now, errText);
|
|
1775
|
-
await saveAccountCooldown(account.key, plan.coolingUntil, plan.reason).catch(() => undefined);
|
|
1776
|
-
const rateLimitKind = plan.reason === "transient" ? "transient" : "quota";
|
|
1777
|
-
const errorMessage = summarizeCodexUpstreamError(errText, "Codex account rate-limited");
|
|
1778
|
-
recordAttemptError(account.label, CODEX_ACCOUNT_TYPE, upstream.status, rateLimitKind);
|
|
1779
|
-
writeAttempt(account, attempt, attemptStartedAt, upstream.status, {
|
|
1780
|
-
errorType: "rate_limit_error",
|
|
1781
|
-
errorMessage,
|
|
1782
|
-
retryable: rateLimitKind === "transient",
|
|
1783
|
-
rateLimitKind,
|
|
1784
|
-
cooldownReason: plan.reason,
|
|
1785
|
-
...(bodyQuota
|
|
1786
|
-
? {
|
|
1787
|
-
errorCode: bodyQuota.errorCode,
|
|
1788
|
-
quotaResetAt: bodyQuota.resetAt || undefined,
|
|
1789
|
-
quotaScope: bodyQuota.scope,
|
|
1790
|
-
}
|
|
1791
|
-
: {}),
|
|
1792
|
-
});
|
|
1793
|
-
lastFailure = {
|
|
1794
|
-
errorType: "rate_limit_error",
|
|
1795
|
-
errorCode: bodyQuota?.errorCode,
|
|
1796
|
-
};
|
|
1797
|
-
lastErrorStatus = 429;
|
|
1798
|
-
lastErrorCode = lastFailure.errorCode;
|
|
1799
|
-
lastErrorMessage = "Codex account rate-limited";
|
|
1800
|
-
break; // rotate
|
|
1801
|
-
}
|
|
1802
|
-
// Other non-ok → record and rotate.
|
|
1803
|
-
if (upstream.status === 401 || upstream.status === 403) {
|
|
1804
|
-
// Reached only when the account has no refresh token to retry with, so
|
|
1805
|
-
// it will fail identically on the next request. Park it briefly instead
|
|
1806
|
-
// of letting it stay first in line with unknown quota.
|
|
1807
|
-
await saveAccountCooldown(account.key, Date.now() + CODEX_AUTH_COOLDOWN_MS, "auth").catch(() => undefined);
|
|
1808
|
-
}
|
|
1809
|
-
const errorMessage = summarizeCodexUpstreamError(errText, "Codex error");
|
|
1810
|
-
const errorType = upstream.status === 401 || upstream.status === 403
|
|
1811
|
-
? "authentication_error"
|
|
1812
|
-
: "api_error";
|
|
1813
|
-
recordAttemptError(account.label, CODEX_ACCOUNT_TYPE, upstream.status);
|
|
1814
|
-
writeAttempt(account, attempt, attemptStartedAt, upstream.status, {
|
|
1815
|
-
errorType,
|
|
1816
|
-
errorMessage,
|
|
1817
|
-
retryable: upstream.status >= 500,
|
|
1818
|
-
});
|
|
1819
|
-
lastFailure = { errorType };
|
|
1820
|
-
lastErrorStatus = upstream.status >= 500 ? 502 : upstream.status;
|
|
1821
|
-
lastErrorCode = getProxyUpstreamFailure({
|
|
1822
|
-
responseBody: errText,
|
|
1823
|
-
})?.code;
|
|
1824
|
-
lastErrorMessage = errorMessage;
|
|
1825
|
-
break; // rotate
|
|
1857
|
+
({ lastFailure, lastErrorStatus, lastErrorCode, lastErrorMessage } =
|
|
1858
|
+
failure);
|
|
1859
|
+
break;
|
|
1826
1860
|
}
|
|
1827
1861
|
}
|
|
1828
|
-
const
|
|
1829
|
-
|
|
1830
|
-
|
|
1831
|
-
|
|
1832
|
-
|
|
1833
|
-
|
|
1834
|
-
|
|
1835
|
-
internalDispatch,
|
|
1836
|
-
tracer,
|
|
1837
|
-
recordFinalOutcome,
|
|
1838
|
-
failureInput: {
|
|
1839
|
-
failureClass: "loop_fallthrough",
|
|
1840
|
-
lastStatus: lastErrorStatus,
|
|
1841
|
-
lastErrorCode,
|
|
1842
|
-
},
|
|
1843
|
-
});
|
|
1844
|
-
if (outbound.kind === "success") {
|
|
1845
|
-
// Same reasoning as the transport-catch insertion point above: settle
|
|
1846
|
-
// the exhausted Codex account's budget lease immediately rather than
|
|
1847
|
-
// holding it open for the unrelated fallback stream's duration.
|
|
1848
|
-
await settleBudget();
|
|
1849
|
-
budgetLease = undefined;
|
|
1850
|
-
budgetDispatched = false;
|
|
1851
|
-
return outbound.response;
|
|
1852
|
-
}
|
|
1853
|
-
if (outbound.kind === "request_too_large") {
|
|
1854
|
-
await recordFinalOutcome(lastAttemptedAccount, 413, {
|
|
1855
|
-
errorType: "request_too_large",
|
|
1856
|
-
errorMessage: outbound.message,
|
|
1857
|
-
terminalOutcome: "handler_error",
|
|
1858
|
-
});
|
|
1859
|
-
return buildCodexErrorResponse(413, outbound.message, "request_too_large");
|
|
1862
|
+
const fallback = await resolveCodexOutbound(rt, {
|
|
1863
|
+
failureClass: "loop_fallthrough",
|
|
1864
|
+
lastStatus: lastErrorStatus,
|
|
1865
|
+
lastErrorCode,
|
|
1866
|
+
}, lastAttemptedAccount);
|
|
1867
|
+
if (fallback) {
|
|
1868
|
+
return fallback;
|
|
1860
1869
|
}
|
|
1861
1870
|
await recordFinalOutcome(lastAttemptedAccount, lastErrorStatus, {
|
|
1862
1871
|
...lastFailure,
|
|
@@ -2137,3 +2146,24 @@ export const __testHooks = {
|
|
|
2137
2146
|
},
|
|
2138
2147
|
wrapCodexOutboundResponseStream,
|
|
2139
2148
|
};
|
|
2149
|
+
function codexResponseInfo(evidence) {
|
|
2150
|
+
const response = evidence.response;
|
|
2151
|
+
const output = Array.isArray(response?.output) ? response.output : undefined;
|
|
2152
|
+
const toolCalls = (output ?? []).flatMap((item) => item &&
|
|
2153
|
+
typeof item === "object" &&
|
|
2154
|
+
"name" in item &&
|
|
2155
|
+
typeof item.name === "string" &&
|
|
2156
|
+
"type" in item &&
|
|
2157
|
+
(item.type === "function_call" || item.type === "custom_tool_call")
|
|
2158
|
+
? [item.name]
|
|
2159
|
+
: []);
|
|
2160
|
+
return {
|
|
2161
|
+
responseModel: typeof response?.model === "string" ? response.model : undefined,
|
|
2162
|
+
finishReason: typeof response?.status === "string" ? response.status : undefined,
|
|
2163
|
+
output,
|
|
2164
|
+
firstOutputAt: evidence.observationIncomplete
|
|
2165
|
+
? undefined
|
|
2166
|
+
: evidence.firstUsefulOutputAt,
|
|
2167
|
+
...(toolCalls.length > 0 && { toolCalls }),
|
|
2168
|
+
};
|
|
2169
|
+
}
|