@juspay/neurolink 12.14.5 → 12.14.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +2 -2
- package/dist/browser/neurolink.min.js +412 -412
- package/dist/cli/commands/proxy.d.ts +1 -1
- package/dist/cli/commands/proxy.js +123 -14
- package/dist/cli/commands/proxyRestart.d.ts +3 -0
- package/dist/cli/commands/proxyRestart.js +88 -0
- package/dist/cli/parser.js +3 -1
- package/dist/proxy/bodyCaptureWorker.js +26 -1
- package/dist/proxy/codexUsage.js +83 -2
- package/dist/proxy/otelLogSink.d.ts +13 -0
- package/dist/proxy/otelLogSink.js +62 -1
- package/dist/proxy/proxyTraceContext.d.ts +21 -0
- package/dist/proxy/proxyTraceContext.js +47 -0
- package/dist/proxy/proxyTracer.d.ts +9 -5
- package/dist/proxy/proxyTracer.js +81 -2
- package/dist/proxy/requestLogger.js +24 -18
- package/dist/proxy/restartControl.d.ts +12 -0
- package/dist/proxy/restartControl.js +283 -0
- package/dist/proxy/rollingWorkerSupervisor.js +8 -0
- package/dist/server/routes/codexProxyRoutes.js +414 -332
- package/dist/types/cli.d.ts +7 -1
- package/dist/types/index.d.ts +1 -0
- package/dist/types/index.js +1 -0
- package/dist/types/proxy.d.ts +119 -3
- package/dist/types/proxyRestart.d.ts +49 -0
- package/dist/types/proxyRestart.js +1 -0
- package/docs-site/static/search-index.json +1 -1
- package/package.json +2 -1
- package/scripts/observability/check-proxy-telemetry.mjs +29 -241
- package/scripts/observability/proxy-telemetry-backend.mjs +202 -0
- package/scripts/observability/proxy-telemetry-check.mjs +544 -0
- package/scripts/observability/query-proxy-history.mjs +30 -19
|
@@ -26,6 +26,7 @@ import { CODEX_ACCOUNT_PREFIX, parseCodexRateLimitHeaders, } from "../../proxy/c
|
|
|
26
26
|
import { buildClientAttribution } from "../../proxy/clientAttribution.js";
|
|
27
27
|
import { registerProxyResponseObserver } from "../../proxy/proxyActivity.js";
|
|
28
28
|
import { logRequest, logRequestAttempt } from "../../proxy/requestLogger.js";
|
|
29
|
+
import { ProxyTracer } from "../../proxy/proxyTracer.js";
|
|
29
30
|
import { parseRetryAfterMs } from "../../proxy/routingPolicy.js";
|
|
30
31
|
import { recordAttempt, recordAttemptError, recordFinalError, recordFinalSuccess, } from "../../proxy/usageStats.js";
|
|
31
32
|
import { sanitizeForLog } from "../../utils/logSanitize.js";
|
|
@@ -288,6 +289,25 @@ export async function handleCodexResponsesRequest(ctx) {
|
|
|
288
289
|
typeof reasoning.effort === "string"
|
|
289
290
|
? reasoning.effort
|
|
290
291
|
: undefined;
|
|
292
|
+
let tracer;
|
|
293
|
+
try {
|
|
294
|
+
tracer = ProxyTracer.startRequest({
|
|
295
|
+
requestId: ctx.requestId,
|
|
296
|
+
method: ctx.method,
|
|
297
|
+
path: ctx.path,
|
|
298
|
+
model,
|
|
299
|
+
stream: true,
|
|
300
|
+
toolCount: Array.isArray(body.tools)
|
|
301
|
+
? body.tools.length
|
|
302
|
+
: 0,
|
|
303
|
+
provider: "openai",
|
|
304
|
+
userAgent: ctx.headers["user-agent"],
|
|
305
|
+
recordRequestMetrics: !isFallbackRequest,
|
|
306
|
+
}, ctx.headers);
|
|
307
|
+
}
|
|
308
|
+
catch {
|
|
309
|
+
// Instrumentation must not change provider request handling.
|
|
310
|
+
}
|
|
291
311
|
const writeFinalLog = (account, responseStatus, extra = {}) => logRequest({
|
|
292
312
|
timestamp: new Date().toISOString(),
|
|
293
313
|
requestId: ctx.requestId,
|
|
@@ -304,6 +324,9 @@ export async function handleCodexResponsesRequest(ctx) {
|
|
|
304
324
|
// This is the cost provider. accountKey and the response header identify
|
|
305
325
|
// the actual Codex pool that supplied the credential.
|
|
306
326
|
provider: "openai",
|
|
327
|
+
...tracer?.getTraceContext(),
|
|
328
|
+
...(reasoningEffort ? { reasoningEffort } : {}),
|
|
329
|
+
firstUsefulOutputStatus: extra.firstUsefulOutputMs !== undefined ? "observed" : "not_observed",
|
|
307
330
|
...buildClientAttribution(ctx.headers),
|
|
308
331
|
responseStatus,
|
|
309
332
|
responseTimeMs: Date.now() - requestStartTime,
|
|
@@ -311,10 +334,22 @@ export async function handleCodexResponsesRequest(ctx) {
|
|
|
311
334
|
});
|
|
312
335
|
let finalOutcomeRecorded = false;
|
|
313
336
|
const recordFinalOutcome = async (account, responseStatus, extra = {}) => {
|
|
314
|
-
if (
|
|
337
|
+
if (finalOutcomeRecorded) {
|
|
315
338
|
return;
|
|
316
339
|
}
|
|
317
340
|
finalOutcomeRecorded = true;
|
|
341
|
+
try {
|
|
342
|
+
if (extra.errorType) {
|
|
343
|
+
tracer?.setError(extra.errorType, extra.errorMessage ?? extra.errorType);
|
|
344
|
+
}
|
|
345
|
+
tracer?.end(responseStatus, Date.now() - requestStartTime);
|
|
346
|
+
}
|
|
347
|
+
catch {
|
|
348
|
+
// End bookkeeping is best effort; the client outcome remains authoritative.
|
|
349
|
+
}
|
|
350
|
+
if (isFallbackRequest) {
|
|
351
|
+
return;
|
|
352
|
+
}
|
|
318
353
|
if (responseStatus >= 400) {
|
|
319
354
|
recordFinalError(responseStatus, account?.label, account ? CODEX_ACCOUNT_TYPE : undefined, {
|
|
320
355
|
requestId: ctx.requestId,
|
|
@@ -335,6 +370,7 @@ export async function handleCodexResponsesRequest(ctx) {
|
|
|
335
370
|
timestamp: new Date().toISOString(),
|
|
336
371
|
requestId: ctx.requestId,
|
|
337
372
|
attempt,
|
|
373
|
+
...tracer?.getTraceContext(),
|
|
338
374
|
...(isFallbackRequest
|
|
339
375
|
? { parentRequestId: ctx.requestId.replace(/:codex-fallback$/, "") }
|
|
340
376
|
: {}),
|
|
@@ -356,370 +392,416 @@ export async function handleCodexResponsesRequest(ctx) {
|
|
|
356
392
|
...extra,
|
|
357
393
|
}).catch(() => undefined);
|
|
358
394
|
};
|
|
359
|
-
const
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
let
|
|
416
|
-
|
|
417
|
-
for (
|
|
418
|
-
if (ctx.abortSignal?.aborted) {
|
|
419
|
-
return cancelRequest(lastAttemptedAccount);
|
|
420
|
-
}
|
|
421
|
-
attempt += 1;
|
|
422
|
-
const attemptStartedAt = Date.now();
|
|
423
|
-
lastAttemptedAccount = account;
|
|
424
|
-
recordAttempt(account.label, CODEX_ACCOUNT_TYPE);
|
|
425
|
-
let upstream;
|
|
395
|
+
const dispatch = async () => {
|
|
396
|
+
const accounts = await loadCodexProxyAccounts();
|
|
397
|
+
const cancelRequest = async (account) => {
|
|
398
|
+
await recordFinalOutcome(account, 499, {
|
|
399
|
+
errorType: "client_cancelled",
|
|
400
|
+
errorMessage: "Client cancelled Codex request",
|
|
401
|
+
terminalOutcome: "client_cancelled",
|
|
402
|
+
});
|
|
403
|
+
return buildCodexErrorResponse(499, "Client cancelled Codex request");
|
|
404
|
+
};
|
|
405
|
+
if (ctx.abortSignal?.aborted) {
|
|
406
|
+
return cancelRequest();
|
|
407
|
+
}
|
|
408
|
+
if (accounts.length === 0) {
|
|
409
|
+
await recordFinalOutcome(undefined, 401, {
|
|
410
|
+
errorType: "no_accounts",
|
|
411
|
+
errorMessage: "No Codex accounts",
|
|
412
|
+
});
|
|
413
|
+
return buildCodexErrorResponse(401, "No Codex accounts configured. Run `neurolink auth login codex`.");
|
|
414
|
+
}
|
|
415
|
+
const now = Date.now();
|
|
416
|
+
const ordered = orderCodexAccounts(accounts, now);
|
|
417
|
+
const eligible = ordered.filter((a) => !(a.coolingUntil !== undefined && a.coolingUntil > now));
|
|
418
|
+
if (eligible.length === 0) {
|
|
419
|
+
// Every account is cooling; surface the soonest recovery as retry-after.
|
|
420
|
+
const soonest = ordered.reduce((min, a) => {
|
|
421
|
+
if (a.coolingUntil === undefined) {
|
|
422
|
+
return min;
|
|
423
|
+
}
|
|
424
|
+
return min === undefined
|
|
425
|
+
? a.coolingUntil
|
|
426
|
+
: Math.min(min, a.coolingUntil);
|
|
427
|
+
}, undefined);
|
|
428
|
+
const retryAfterSec = soonest
|
|
429
|
+
? Math.max(1, Math.ceil((soonest - now) / 1000))
|
|
430
|
+
: 60;
|
|
431
|
+
await recordFinalOutcome(undefined, 429, {
|
|
432
|
+
errorType: "all_accounts_cooling",
|
|
433
|
+
errorMessage: "All Codex accounts are rate-limited",
|
|
434
|
+
});
|
|
435
|
+
return new Response(JSON.stringify({
|
|
436
|
+
error: {
|
|
437
|
+
type: "rate_limit_error",
|
|
438
|
+
message: "All Codex accounts are currently rate-limited",
|
|
439
|
+
},
|
|
440
|
+
}), {
|
|
441
|
+
status: 429,
|
|
442
|
+
headers: {
|
|
443
|
+
"content-type": "application/json",
|
|
444
|
+
"retry-after": String(retryAfterSec),
|
|
445
|
+
},
|
|
446
|
+
});
|
|
447
|
+
}
|
|
448
|
+
let attempt = 0;
|
|
449
|
+
let lastErrorMessage = "All Codex accounts failed";
|
|
450
|
+
let lastErrorStatus = 502;
|
|
451
|
+
let lastFailure = { errorType: "all_accounts_failed" };
|
|
452
|
+
let lastAttemptedAccount;
|
|
453
|
+
for (const account of eligible) {
|
|
426
454
|
try {
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
ctx.abortSignal,
|
|
434
|
-
AbortSignal.timeout(CODEX_UPSTREAM_TIMEOUT_MS),
|
|
435
|
-
])
|
|
436
|
-
: AbortSignal.timeout(CODEX_UPSTREAM_TIMEOUT_MS),
|
|
455
|
+
tracer?.setAccountSelection({
|
|
456
|
+
strategy: "codex-quota-order",
|
|
457
|
+
accountsTotal: accounts.length,
|
|
458
|
+
accountsHealthy: eligible.length,
|
|
459
|
+
selectedAccount: account.label,
|
|
460
|
+
accountType: CODEX_ACCOUNT_TYPE,
|
|
437
461
|
});
|
|
438
462
|
}
|
|
439
|
-
catch
|
|
463
|
+
catch {
|
|
464
|
+
// Account attribution cannot affect routing.
|
|
465
|
+
}
|
|
466
|
+
let authRetried = false;
|
|
467
|
+
// Same-account loop only re-runs once, for a post-401 token refresh.
|
|
468
|
+
for (;;) {
|
|
440
469
|
if (ctx.abortSignal?.aborted) {
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
470
|
+
return cancelRequest(lastAttemptedAccount);
|
|
471
|
+
}
|
|
472
|
+
attempt += 1;
|
|
473
|
+
const attemptStartedAt = Date.now();
|
|
474
|
+
lastAttemptedAccount = account;
|
|
475
|
+
recordAttempt(account.label, CODEX_ACCOUNT_TYPE);
|
|
476
|
+
let upstream;
|
|
477
|
+
try {
|
|
478
|
+
upstream = await fetch(CODEX_RESPONSES_URL, {
|
|
479
|
+
method: "POST",
|
|
480
|
+
headers: buildCodexUpstreamHeaders(ctx.headers, account),
|
|
481
|
+
body: bodyStr,
|
|
482
|
+
signal: ctx.abortSignal
|
|
483
|
+
? AbortSignal.any([
|
|
484
|
+
ctx.abortSignal,
|
|
485
|
+
AbortSignal.timeout(CODEX_UPSTREAM_TIMEOUT_MS),
|
|
486
|
+
])
|
|
487
|
+
: AbortSignal.timeout(CODEX_UPSTREAM_TIMEOUT_MS),
|
|
445
488
|
});
|
|
446
|
-
return cancelRequest(account);
|
|
447
489
|
}
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
490
|
+
catch (error) {
|
|
491
|
+
if (ctx.abortSignal?.aborted) {
|
|
492
|
+
writeAttempt(account, attempt, attemptStartedAt, 499, {
|
|
493
|
+
errorType: "client_cancelled",
|
|
494
|
+
errorMessage: "Client cancelled Codex request",
|
|
495
|
+
retryable: false,
|
|
496
|
+
});
|
|
497
|
+
return cancelRequest(account);
|
|
498
|
+
}
|
|
499
|
+
// A transport failure message is derived from local state — resolved
|
|
500
|
+
// hostnames, socket paths, Node internals — and says nothing the caller
|
|
501
|
+
// can act on. Keep the detail in the log and return a fixed string, so
|
|
502
|
+
// internal topology never reaches the client.
|
|
503
|
+
logger.debug(`Codex upstream fetch failed (${account.label}): ${sanitizeForLog(error instanceof Error ? error.message : String(error))}`);
|
|
504
|
+
const errorMessage = summarizeCodexUpstreamError(error instanceof Error ? error.message : String(error), "Codex upstream request failed");
|
|
505
|
+
const errorCode = getCodexTransportErrorCode(error);
|
|
506
|
+
const transportScope = codexTransportScope(error);
|
|
507
|
+
recordAttemptError(account.label, CODEX_ACCOUNT_TYPE, 502);
|
|
508
|
+
writeAttempt(account, attempt, attemptStartedAt, 502, {
|
|
509
|
+
errorType: "network_error",
|
|
510
|
+
errorMessage,
|
|
511
|
+
...(errorCode ? { errorCode } : {}),
|
|
512
|
+
transportScope,
|
|
513
|
+
// These codes prove failure before HTTP dispatch. Socket resets,
|
|
514
|
+
// EPIPE and generic timeouts may follow dispatch and must not replay.
|
|
515
|
+
retryable: [
|
|
516
|
+
"UND_ERR_CONNECT_TIMEOUT",
|
|
517
|
+
"ECONNREFUSED",
|
|
518
|
+
"ENOTFOUND",
|
|
519
|
+
"EAI_AGAIN",
|
|
520
|
+
].includes(errorCode ?? ""),
|
|
521
|
+
});
|
|
522
|
+
lastFailure = {
|
|
523
|
+
errorType: "network_error",
|
|
524
|
+
errorMessage,
|
|
525
|
+
errorCode,
|
|
526
|
+
transportScope,
|
|
527
|
+
};
|
|
528
|
+
if (![
|
|
465
529
|
"UND_ERR_CONNECT_TIMEOUT",
|
|
466
530
|
"ECONNREFUSED",
|
|
467
531
|
"ENOTFOUND",
|
|
468
532
|
"EAI_AGAIN",
|
|
469
|
-
].includes(errorCode ?? "")
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
"ECONNREFUSED",
|
|
480
|
-
"ENOTFOUND",
|
|
481
|
-
"EAI_AGAIN",
|
|
482
|
-
].includes(errorCode ?? "")) {
|
|
483
|
-
await recordFinalOutcome(account, 502, {
|
|
484
|
-
...lastFailure,
|
|
485
|
-
errorMessage,
|
|
486
|
-
});
|
|
487
|
-
return buildCodexErrorResponse(502, "Codex upstream request failed");
|
|
488
|
-
}
|
|
489
|
-
lastErrorMessage = "Codex upstream request failed";
|
|
490
|
-
lastErrorStatus = 502;
|
|
491
|
-
break; // rotate to next account
|
|
492
|
-
}
|
|
493
|
-
if (upstream.ok) {
|
|
494
|
-
const quota = parseCodexRateLimitHeaders(upstream.headers);
|
|
495
|
-
if (quota) {
|
|
496
|
-
saveAccountQuota(account.key, quota).catch(() => undefined);
|
|
497
|
-
}
|
|
498
|
-
// A prior cooldown that has expired is cleared on success. The
|
|
499
|
-
// compare-and-swap guards against wiping a longer cooldown that another
|
|
500
|
-
// in-flight request set while this one was upstream.
|
|
501
|
-
if (account.expiredCooldownUntil !== undefined) {
|
|
502
|
-
clearAccountCooldown(account.key, account.expiredCooldownUntil).catch(() => undefined);
|
|
533
|
+
].includes(errorCode ?? "")) {
|
|
534
|
+
await recordFinalOutcome(account, 502, {
|
|
535
|
+
...lastFailure,
|
|
536
|
+
errorMessage,
|
|
537
|
+
});
|
|
538
|
+
return buildCodexErrorResponse(502, "Codex upstream request failed");
|
|
539
|
+
}
|
|
540
|
+
lastErrorMessage = "Codex upstream request failed";
|
|
541
|
+
lastErrorStatus = 502;
|
|
542
|
+
break; // rotate to next account
|
|
503
543
|
}
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
544
|
+
if (upstream.ok) {
|
|
545
|
+
const quota = parseCodexRateLimitHeaders(upstream.headers);
|
|
546
|
+
if (quota) {
|
|
547
|
+
saveAccountQuota(account.key, quota).catch(() => undefined);
|
|
548
|
+
}
|
|
549
|
+
// A prior cooldown that has expired is cleared on success. The
|
|
550
|
+
// compare-and-swap guards against wiping a longer cooldown that another
|
|
551
|
+
// in-flight request set while this one was upstream.
|
|
552
|
+
if (account.expiredCooldownUntil !== undefined) {
|
|
553
|
+
clearAccountCooldown(account.key, account.expiredCooldownUntil).catch(() => undefined);
|
|
554
|
+
}
|
|
555
|
+
publishCodexHeaders(ctx, account, attempt, quota);
|
|
556
|
+
writeAttempt(account, attempt, attemptStartedAt, upstream.status);
|
|
557
|
+
const headers = {
|
|
558
|
+
"content-type": upstream.headers.get("content-type") ?? "text/event-stream",
|
|
559
|
+
"cache-control": "no-cache",
|
|
560
|
+
connection: "keep-alive",
|
|
561
|
+
...(ctx.responseHeaders ?? {}),
|
|
562
|
+
};
|
|
563
|
+
if (!upstream.body) {
|
|
564
|
+
recordAttemptError(account.label, CODEX_ACCOUNT_TYPE, 502);
|
|
565
|
+
writeAttempt(account, attempt, attemptStartedAt, 502, {
|
|
566
|
+
errorType: "incomplete_stream",
|
|
567
|
+
errorMessage: "Codex returned no response stream",
|
|
568
|
+
retryable: false,
|
|
569
|
+
});
|
|
570
|
+
await recordFinalOutcome(account, 502, {
|
|
571
|
+
terminalOutcome: "stream_error",
|
|
572
|
+
errorType: "incomplete_stream",
|
|
573
|
+
errorMessage: "Codex returned no response stream",
|
|
574
|
+
});
|
|
575
|
+
return new Response(upstream.body, {
|
|
576
|
+
status: upstream.status,
|
|
577
|
+
headers,
|
|
578
|
+
});
|
|
579
|
+
}
|
|
580
|
+
const { stream: usageTap, usage: usageSeen, evidence, } = createCodexUsageTap();
|
|
581
|
+
const relay = new Response(upstream.body.pipeThrough(usageTap), {
|
|
525
582
|
status: upstream.status,
|
|
526
583
|
headers,
|
|
527
584
|
});
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
585
|
+
registerProxyResponseObserver(ctx.metadata, {
|
|
586
|
+
onTerminal: ({ outcome, error, observedBodyBytes }) => {
|
|
587
|
+
return usageSeen
|
|
588
|
+
.then((usage) => {
|
|
589
|
+
const semantic = evidence();
|
|
590
|
+
const completedFrameDelivered = semantic.completed &&
|
|
591
|
+
observedBodyBytes >= semantic.terminalBytes;
|
|
592
|
+
const failed = semantic.errorType ||
|
|
593
|
+
((outcome === "completed" || outcome === "bodyless") &&
|
|
594
|
+
!semantic.completed);
|
|
595
|
+
const usageExtra = usage
|
|
596
|
+
? {
|
|
597
|
+
inputTokens: usage.inputTokens,
|
|
598
|
+
outputTokens: usage.outputTokens,
|
|
599
|
+
cacheReadTokens: usage.cacheReadTokens,
|
|
600
|
+
cacheCreationTokens: usage.cacheCreationTokens,
|
|
601
|
+
}
|
|
602
|
+
: {};
|
|
603
|
+
if (usage) {
|
|
604
|
+
try {
|
|
605
|
+
tracer?.setUsage(usage);
|
|
606
|
+
}
|
|
607
|
+
catch {
|
|
608
|
+
// Pricing/metrics must never change the stream outcome.
|
|
609
|
+
}
|
|
550
610
|
}
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
611
|
+
const timing = semantic.firstUsefulOutputAt === undefined ||
|
|
612
|
+
semantic.observationIncomplete
|
|
613
|
+
? {
|
|
614
|
+
firstUsefulOutputStatus: semantic.completed &&
|
|
615
|
+
!semantic.observationIncomplete
|
|
616
|
+
? "no_useful_output"
|
|
617
|
+
: "not_observed",
|
|
618
|
+
}
|
|
619
|
+
: {
|
|
620
|
+
firstUsefulOutputStatus: "observed",
|
|
621
|
+
firstUsefulOutputEvent: semantic.firstUsefulOutputEvent,
|
|
622
|
+
firstUsefulOutputMs: Math.max(0, semantic.firstUsefulOutputAt - requestStartTime),
|
|
623
|
+
};
|
|
624
|
+
if (failed) {
|
|
625
|
+
recordAttemptError(account.label, CODEX_ACCOUNT_TYPE, 502);
|
|
626
|
+
writeAttempt(account, attempt, attemptStartedAt, 502, {
|
|
627
|
+
errorType: semantic.errorType ?? "incomplete_stream",
|
|
628
|
+
errorCode: semantic.errorCode,
|
|
629
|
+
errorMessage: semantic.errorMessage ??
|
|
630
|
+
"Codex stream ended without a completion event",
|
|
631
|
+
retryable: false,
|
|
632
|
+
});
|
|
633
|
+
return recordFinalOutcome(account, 502, {
|
|
634
|
+
...usageExtra,
|
|
635
|
+
...timing,
|
|
636
|
+
terminalOutcome: "stream_error",
|
|
637
|
+
errorType: semantic.errorType ?? "incomplete_stream",
|
|
638
|
+
errorCode: semantic.errorCode,
|
|
639
|
+
errorMessage: semantic.errorMessage ??
|
|
640
|
+
"Codex stream ended without a completion event",
|
|
641
|
+
});
|
|
642
|
+
}
|
|
643
|
+
if (outcome === "completed" ||
|
|
644
|
+
outcome === "bodyless" ||
|
|
645
|
+
(outcome === "client_cancelled" && completedFrameDelivered)) {
|
|
646
|
+
return recordFinalOutcome(account, upstream.status, {
|
|
647
|
+
terminalOutcome: "completed",
|
|
648
|
+
...usageExtra,
|
|
649
|
+
...timing,
|
|
650
|
+
});
|
|
651
|
+
}
|
|
652
|
+
if (outcome === "stream_error") {
|
|
653
|
+
recordAttemptError(account.label, CODEX_ACCOUNT_TYPE, 502);
|
|
654
|
+
writeAttempt(account, attempt, attemptStartedAt, 502, {
|
|
655
|
+
errorType: "stream_error",
|
|
656
|
+
errorCode: getCodexTransportErrorCode(error),
|
|
657
|
+
errorMessage: summarizeCodexUpstreamError(error instanceof Error ? error.message : "", "Codex upstream stream failed"),
|
|
658
|
+
retryable: false,
|
|
659
|
+
});
|
|
660
|
+
}
|
|
661
|
+
return recordFinalOutcome(account, outcome === "client_cancelled" ? 499 : 502, {
|
|
662
|
+
errorType: outcome === "client_cancelled"
|
|
663
|
+
? "client_cancelled"
|
|
664
|
+
: "stream_error",
|
|
665
|
+
errorMessage: outcome === "client_cancelled"
|
|
666
|
+
? "Client cancelled Codex stream"
|
|
667
|
+
: summarizeCodexUpstreamError(error instanceof Error ? error.message : "", "Codex upstream stream failed"),
|
|
668
|
+
...(outcome === "stream_error"
|
|
669
|
+
? { errorCode: getCodexTransportErrorCode(error) }
|
|
670
|
+
: {}),
|
|
671
|
+
terminalOutcome: outcome,
|
|
581
672
|
...usageExtra,
|
|
582
673
|
...timing,
|
|
583
674
|
});
|
|
584
|
-
}
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
errorCode: getCodexTransportErrorCode(error),
|
|
590
|
-
errorMessage: summarizeCodexUpstreamError(error instanceof Error ? error.message : "", "Codex upstream stream failed"),
|
|
591
|
-
retryable: false,
|
|
592
|
-
});
|
|
593
|
-
}
|
|
594
|
-
return recordFinalOutcome(account, outcome === "client_cancelled" ? 499 : 502, {
|
|
595
|
-
errorType: outcome === "client_cancelled"
|
|
596
|
-
? "client_cancelled"
|
|
597
|
-
: "stream_error",
|
|
598
|
-
errorMessage: outcome === "client_cancelled"
|
|
599
|
-
? "Client cancelled Codex stream"
|
|
600
|
-
: summarizeCodexUpstreamError(error instanceof Error ? error.message : "", "Codex upstream stream failed"),
|
|
601
|
-
...(outcome === "stream_error"
|
|
602
|
-
? { errorCode: getCodexTransportErrorCode(error) }
|
|
603
|
-
: {}),
|
|
604
|
-
terminalOutcome: outcome,
|
|
605
|
-
...usageExtra,
|
|
606
|
-
...timing,
|
|
607
|
-
});
|
|
608
|
-
})
|
|
609
|
-
.catch(() => undefined);
|
|
610
|
-
},
|
|
611
|
-
});
|
|
612
|
-
return relay;
|
|
613
|
-
}
|
|
614
|
-
const errText = await upstream.text().catch(() => "");
|
|
615
|
-
// 401/403 → try a forced token refresh once, then rotate.
|
|
616
|
-
if ((upstream.status === 401 || upstream.status === 403) &&
|
|
617
|
-
!authRetried &&
|
|
618
|
-
account.refreshToken) {
|
|
619
|
-
const errorMessage = summarizeCodexUpstreamError(errText, "Codex authentication rejected upstream");
|
|
620
|
-
recordAttemptError(account.label, CODEX_ACCOUNT_TYPE, upstream.status);
|
|
621
|
-
writeAttempt(account, attempt, attemptStartedAt, upstream.status, {
|
|
622
|
-
errorType: "authentication_error",
|
|
623
|
-
errorMessage,
|
|
624
|
-
retryable: true,
|
|
625
|
-
});
|
|
626
|
-
authRetried = true;
|
|
627
|
-
const staleTokens = {
|
|
628
|
-
accessToken: account.token,
|
|
629
|
-
refreshToken: account.refreshToken,
|
|
630
|
-
expiresAt: account.expiresAt ?? 0,
|
|
631
|
-
};
|
|
632
|
-
try {
|
|
633
|
-
const refreshed = await refreshCodexTokenOnce(account.key, account.refreshToken);
|
|
634
|
-
account.token = refreshed.accessToken;
|
|
635
|
-
account.refreshToken = refreshed.refreshToken ?? account.refreshToken;
|
|
636
|
-
account.expiresAt = refreshed.expiresAt ?? account.expiresAt;
|
|
637
|
-
account.accountId = resolveCodexAccountId(refreshed.accessToken);
|
|
638
|
-
continue; // retry same account with the fresh token
|
|
675
|
+
})
|
|
676
|
+
.catch(() => undefined);
|
|
677
|
+
},
|
|
678
|
+
});
|
|
679
|
+
return relay;
|
|
639
680
|
}
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
681
|
+
const errText = await upstream.text().catch(() => "");
|
|
682
|
+
// 401/403 → try a forced token refresh once, then rotate.
|
|
683
|
+
if ((upstream.status === 401 || upstream.status === 403) &&
|
|
684
|
+
!authRetried &&
|
|
685
|
+
account.refreshToken) {
|
|
686
|
+
const errorMessage = summarizeCodexUpstreamError(errText, "Codex authentication rejected upstream");
|
|
687
|
+
recordAttemptError(account.label, CODEX_ACCOUNT_TYPE, upstream.status);
|
|
688
|
+
writeAttempt(account, attempt, attemptStartedAt, upstream.status, {
|
|
689
|
+
errorType: "authentication_error",
|
|
690
|
+
errorMessage,
|
|
691
|
+
retryable: true,
|
|
692
|
+
});
|
|
693
|
+
authRetried = true;
|
|
694
|
+
const staleTokens = {
|
|
695
|
+
accessToken: account.token,
|
|
696
|
+
refreshToken: account.refreshToken,
|
|
697
|
+
expiresAt: account.expiresAt ?? 0,
|
|
698
|
+
};
|
|
699
|
+
try {
|
|
700
|
+
const refreshed = await refreshCodexTokenOnce(account.key, account.refreshToken);
|
|
701
|
+
account.token = refreshed.accessToken;
|
|
702
|
+
account.refreshToken =
|
|
703
|
+
refreshed.refreshToken ?? account.refreshToken;
|
|
704
|
+
account.expiresAt = refreshed.expiresAt ?? account.expiresAt;
|
|
705
|
+
account.accountId = resolveCodexAccountId(refreshed.accessToken);
|
|
706
|
+
continue; // retry same account with the fresh token
|
|
707
|
+
}
|
|
708
|
+
catch (error) {
|
|
709
|
+
if (isPermanentCodexRefreshFailure(error)) {
|
|
710
|
+
// Compare-and-swap: the pool is rebuilt per request with no shared
|
|
711
|
+
// state, so a concurrent request may already have rotated this
|
|
712
|
+
// credential. Disabling unconditionally would kill the account that
|
|
713
|
+
// the other request just healed.
|
|
714
|
+
const disabled = await tokenStore.markDisabledIfCurrent(account.key, staleTokens, "refresh_invalid");
|
|
715
|
+
if (disabled) {
|
|
716
|
+
logger.always(`[proxy] codex account=${account.label} disabled until re-authentication. Run: neurolink auth login codex --label ${account.label}`);
|
|
717
|
+
}
|
|
718
|
+
lastFailure = {
|
|
719
|
+
errorType: "authentication_error",
|
|
720
|
+
errorCode: "refresh_invalid",
|
|
721
|
+
};
|
|
722
|
+
lastErrorStatus = 401;
|
|
723
|
+
lastErrorMessage =
|
|
724
|
+
"Codex token refresh failed; re-login required";
|
|
725
|
+
break;
|
|
649
726
|
}
|
|
727
|
+
// No verdict on the credential — cool briefly and try the next
|
|
728
|
+
// account, so a 5xx or a timeout cannot cost the user a login.
|
|
729
|
+
await saveAccountCooldown(account.key, Date.now() + CODEX_AUTH_COOLDOWN_MS, "auth").catch(() => undefined);
|
|
730
|
+
logger.debug(`[proxy] codex account=${account.label} refresh failed transiently; cooling and rotating`);
|
|
650
731
|
lastFailure = {
|
|
651
|
-
errorType: "
|
|
652
|
-
errorCode:
|
|
732
|
+
errorType: "auth_refresh_unavailable",
|
|
733
|
+
errorCode: getCodexTransportErrorCode(error),
|
|
653
734
|
};
|
|
654
|
-
lastErrorStatus =
|
|
655
|
-
lastErrorMessage = "Codex token refresh
|
|
735
|
+
lastErrorStatus = 503;
|
|
736
|
+
lastErrorMessage = "Codex token refresh temporarily unavailable";
|
|
656
737
|
break;
|
|
657
738
|
}
|
|
658
|
-
// No verdict on the credential — cool briefly and try the next
|
|
659
|
-
// account, so a 5xx or a timeout cannot cost the user a login.
|
|
660
|
-
await saveAccountCooldown(account.key, Date.now() + CODEX_AUTH_COOLDOWN_MS, "auth").catch(() => undefined);
|
|
661
|
-
logger.debug(`[proxy] codex account=${account.label} refresh failed transiently; cooling and rotating`);
|
|
662
|
-
lastFailure = {
|
|
663
|
-
errorType: "auth_refresh_unavailable",
|
|
664
|
-
errorCode: getCodexTransportErrorCode(error),
|
|
665
|
-
};
|
|
666
|
-
lastErrorStatus = 503;
|
|
667
|
-
lastErrorMessage = "Codex token refresh temporarily unavailable";
|
|
668
|
-
break;
|
|
669
739
|
}
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
740
|
+
// 429 → cooldown + rotate.
|
|
741
|
+
if (upstream.status === 429) {
|
|
742
|
+
const quota = parseCodexRateLimitHeaders(upstream.headers);
|
|
743
|
+
if (quota) {
|
|
744
|
+
saveAccountQuota(account.key, quota).catch(() => undefined);
|
|
745
|
+
}
|
|
746
|
+
const retryAfterMs = parseRetryAfterMs(upstream.headers.get("retry-after"));
|
|
747
|
+
const plan = planCodexCooldown(quota, retryAfterMs, Date.now());
|
|
748
|
+
await saveAccountCooldown(account.key, plan.coolingUntil, plan.reason).catch(() => undefined);
|
|
749
|
+
const rateLimitKind = plan.reason === "transient" ? "transient" : "quota";
|
|
750
|
+
const errorMessage = summarizeCodexUpstreamError(errText, "Codex account rate-limited");
|
|
751
|
+
recordAttemptError(account.label, CODEX_ACCOUNT_TYPE, upstream.status, rateLimitKind);
|
|
752
|
+
writeAttempt(account, attempt, attemptStartedAt, upstream.status, {
|
|
753
|
+
errorType: "rate_limit_error",
|
|
754
|
+
errorMessage,
|
|
755
|
+
retryable: true,
|
|
756
|
+
rateLimitKind,
|
|
757
|
+
cooldownReason: plan.reason,
|
|
758
|
+
});
|
|
759
|
+
lastFailure = { errorType: "rate_limit_error" };
|
|
760
|
+
lastErrorStatus = 429;
|
|
761
|
+
lastErrorMessage = "Codex account rate-limited";
|
|
762
|
+
break; // rotate
|
|
763
|
+
}
|
|
764
|
+
// Other non-ok → record and rotate.
|
|
765
|
+
if (upstream.status === 401 || upstream.status === 403) {
|
|
766
|
+
// Reached only when the account has no refresh token to retry with, so
|
|
767
|
+
// it will fail identically on the next request. Park it briefly instead
|
|
768
|
+
// of letting it stay first in line with unknown quota.
|
|
769
|
+
await saveAccountCooldown(account.key, Date.now() + CODEX_AUTH_COOLDOWN_MS, "auth").catch(() => undefined);
|
|
676
770
|
}
|
|
677
|
-
const
|
|
678
|
-
const
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
recordAttemptError(account.label, CODEX_ACCOUNT_TYPE, upstream.status, rateLimitKind);
|
|
771
|
+
const errorMessage = summarizeCodexUpstreamError(errText, "Codex error");
|
|
772
|
+
const errorType = upstream.status === 401 || upstream.status === 403
|
|
773
|
+
? "authentication_error"
|
|
774
|
+
: "api_error";
|
|
775
|
+
recordAttemptError(account.label, CODEX_ACCOUNT_TYPE, upstream.status);
|
|
683
776
|
writeAttempt(account, attempt, attemptStartedAt, upstream.status, {
|
|
684
|
-
errorType
|
|
777
|
+
errorType,
|
|
685
778
|
errorMessage,
|
|
686
|
-
retryable:
|
|
687
|
-
rateLimitKind,
|
|
688
|
-
cooldownReason: plan.reason,
|
|
779
|
+
retryable: upstream.status >= 500,
|
|
689
780
|
});
|
|
690
|
-
lastFailure = { errorType
|
|
691
|
-
lastErrorStatus =
|
|
692
|
-
lastErrorMessage =
|
|
781
|
+
lastFailure = { errorType };
|
|
782
|
+
lastErrorStatus = upstream.status >= 500 ? 502 : upstream.status;
|
|
783
|
+
lastErrorMessage = errorMessage;
|
|
693
784
|
break; // rotate
|
|
694
785
|
}
|
|
695
|
-
// Other non-ok → record and rotate.
|
|
696
|
-
if (upstream.status === 401 || upstream.status === 403) {
|
|
697
|
-
// Reached only when the account has no refresh token to retry with, so
|
|
698
|
-
// it will fail identically on the next request. Park it briefly instead
|
|
699
|
-
// of letting it stay first in line with unknown quota.
|
|
700
|
-
await saveAccountCooldown(account.key, Date.now() + CODEX_AUTH_COOLDOWN_MS, "auth").catch(() => undefined);
|
|
701
|
-
}
|
|
702
|
-
const errorMessage = summarizeCodexUpstreamError(errText, "Codex error");
|
|
703
|
-
const errorType = upstream.status === 401 || upstream.status === 403
|
|
704
|
-
? "authentication_error"
|
|
705
|
-
: "api_error";
|
|
706
|
-
recordAttemptError(account.label, CODEX_ACCOUNT_TYPE, upstream.status);
|
|
707
|
-
writeAttempt(account, attempt, attemptStartedAt, upstream.status, {
|
|
708
|
-
errorType,
|
|
709
|
-
errorMessage,
|
|
710
|
-
retryable: upstream.status >= 500,
|
|
711
|
-
});
|
|
712
|
-
lastFailure = { errorType };
|
|
713
|
-
lastErrorStatus = upstream.status >= 500 ? 502 : upstream.status;
|
|
714
|
-
lastErrorMessage = errorMessage;
|
|
715
|
-
break; // rotate
|
|
716
786
|
}
|
|
787
|
+
await recordFinalOutcome(lastAttemptedAccount, lastErrorStatus, {
|
|
788
|
+
...lastFailure,
|
|
789
|
+
errorMessage: lastFailure.errorMessage ?? lastErrorMessage,
|
|
790
|
+
});
|
|
791
|
+
return buildCodexErrorResponse(lastErrorStatus, lastErrorMessage);
|
|
792
|
+
};
|
|
793
|
+
try {
|
|
794
|
+
return await dispatch();
|
|
795
|
+
}
|
|
796
|
+
catch (error) {
|
|
797
|
+
try {
|
|
798
|
+
tracer?.end(502, Date.now() - requestStartTime);
|
|
799
|
+
}
|
|
800
|
+
catch {
|
|
801
|
+
// Shared HTTP error handling owns the client outcome and final log.
|
|
802
|
+
}
|
|
803
|
+
throw error;
|
|
717
804
|
}
|
|
718
|
-
await recordFinalOutcome(lastAttemptedAccount, lastErrorStatus, {
|
|
719
|
-
...lastFailure,
|
|
720
|
-
errorMessage: lastFailure.errorMessage ?? lastErrorMessage,
|
|
721
|
-
});
|
|
722
|
-
return buildCodexErrorResponse(lastErrorStatus, lastErrorMessage);
|
|
723
805
|
}
|
|
724
806
|
/**
|
|
725
807
|
* Relay Codex model discovery upstream.
|