@juspay/neurolink 12.14.6 → 12.14.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. package/CHANGELOG.md +2 -2
  2. package/dist/browser/neurolink.min.js +411 -411
  3. package/dist/cli/commands/proxy.js +83 -14
  4. package/dist/proxy/bodyCaptureProcessing.js +47 -3
  5. package/dist/proxy/bodyCaptureWorker.js +39 -10
  6. package/dist/proxy/claudeOutputObservation.d.ts +9 -0
  7. package/dist/proxy/claudeOutputObservation.js +85 -0
  8. package/dist/proxy/codexUsage.js +86 -3
  9. package/dist/proxy/otelLogSink.d.ts +13 -0
  10. package/dist/proxy/otelLogSink.js +63 -2
  11. package/dist/proxy/processTelemetry.d.ts +3 -0
  12. package/dist/proxy/processTelemetry.js +25 -0
  13. package/dist/proxy/proxyTraceContext.d.ts +21 -0
  14. package/dist/proxy/proxyTraceContext.js +47 -0
  15. package/dist/proxy/proxyTracer.d.ts +9 -5
  16. package/dist/proxy/proxyTracer.js +81 -2
  17. package/dist/proxy/rawStreamCapture.js +17 -26
  18. package/dist/proxy/requestLogger.d.ts +2 -0
  19. package/dist/proxy/requestLogger.js +29 -19
  20. package/dist/proxy/restartControl.d.ts +3 -1
  21. package/dist/proxy/restartControl.js +118 -1
  22. package/dist/proxy/sseInterceptor.js +71 -3
  23. package/dist/server/routes/claudeProxyRoutes.d.ts +0 -11
  24. package/dist/server/routes/claudeProxyRoutes.js +27 -12
  25. package/dist/server/routes/codexProxyRoutes.js +496 -333
  26. package/dist/types/cli.d.ts +7 -1
  27. package/dist/types/proxy.d.ts +133 -3
  28. package/dist/types/proxyRestart.d.ts +25 -0
  29. package/dist/utils/logSanitize.js +1 -1
  30. package/docs-site/static/search-index.json +1 -1
  31. package/package.json +1 -1
  32. package/scripts/observability/check-proxy-telemetry.mjs +29 -241
  33. package/scripts/observability/proxy-telemetry-backend.mjs +202 -0
  34. package/scripts/observability/proxy-telemetry-check.mjs +578 -0
  35. package/scripts/observability/query-proxy-history.mjs +30 -19
@@ -25,7 +25,11 @@ import { createCodexUsageTap } from "../../proxy/codexUsage.js";
25
25
  import { CODEX_ACCOUNT_PREFIX, parseCodexRateLimitHeaders, } from "../../proxy/codexAccountUsage.js";
26
26
  import { buildClientAttribution } from "../../proxy/clientAttribution.js";
27
27
  import { registerProxyResponseObserver } from "../../proxy/proxyActivity.js";
28
- import { logRequest, logRequestAttempt } from "../../proxy/requestLogger.js";
28
+ import { logRequest, logRequestAttempt, logBodyCapture, isProxyBodyCaptureEnabled, } from "../../proxy/requestLogger.js";
29
+ import { createRawStreamCapture } from "../../proxy/rawStreamCapture.js";
30
+ import { resolveProxyLogTraceContext } from "../../proxy/proxyTraceContext.js";
31
+ import { isBorrowedRequest } from "../../proxy/shareContext.js";
32
+ import { ProxyTracer } from "../../proxy/proxyTracer.js";
29
33
  import { parseRetryAfterMs } from "../../proxy/routingPolicy.js";
30
34
  import { recordAttempt, recordAttemptError, recordFinalError, recordFinalSuccess, } from "../../proxy/usageStats.js";
31
35
  import { sanitizeForLog } from "../../utils/logSanitize.js";
@@ -271,6 +275,68 @@ function publishCodexHeaders(ctx, account, attempt, quota) {
271
275
  }
272
276
  /** Core pooled handler for POST /backend-api/codex/responses. */
273
277
  export async function handleCodexResponsesRequest(ctx) {
278
+ if (ctx.metadata?.[CODEX_FALLBACK_METADATA_KEY] !== true) {
279
+ void logBodyCapture({
280
+ timestamp: new Date().toISOString(),
281
+ requestId: ctx.requestId,
282
+ phase: "client_request",
283
+ model: codexCaptureModel(ctx),
284
+ stream: true,
285
+ headers: ctx.headers,
286
+ body: ctx.body,
287
+ contentType: "application/json",
288
+ ...resolveProxyLogTraceContext({ requestId: ctx.requestId }),
289
+ });
290
+ }
291
+ const response = await executeCodexResponsesRequest(ctx);
292
+ return ctx.metadata?.[CODEX_FALLBACK_METADATA_KEY] === true
293
+ ? response
294
+ : captureCodexResponse(ctx, response, "client_response");
295
+ }
296
+ function codexCaptureModel(ctx) {
297
+ const body = ctx.body;
298
+ return typeof body?.model === "string" ? body.model : "-";
299
+ }
300
+ /** Observe bytes through the existing bounded pass-through; cancellation still reaches upstream. */
301
+ function captureCodexResponse(ctx, response, phase, account, attempt) {
302
+ if (!isProxyBodyCaptureEnabled()) {
303
+ return response;
304
+ }
305
+ const metadata = {
306
+ timestamp: new Date().toISOString(),
307
+ requestId: ctx.requestId,
308
+ phase,
309
+ model: codexCaptureModel(ctx),
310
+ stream: true,
311
+ headers: Object.fromEntries(response.headers.entries()),
312
+ contentType: response.headers.get("content-type") ?? undefined,
313
+ responseStatus: response.status,
314
+ account: account?.label,
315
+ accountType: account ? CODEX_ACCOUNT_TYPE : undefined,
316
+ attempt,
317
+ ...resolveProxyLogTraceContext({ requestId: ctx.requestId }),
318
+ };
319
+ if (!response.body || isBorrowedRequest()) {
320
+ void logBodyCapture(metadata);
321
+ return response;
322
+ }
323
+ const observed = createRawStreamCapture();
324
+ void observed.capture
325
+ .then((capture) => logBodyCapture({
326
+ ...metadata,
327
+ timestamp: new Date().toISOString(),
328
+ body: capture.text,
329
+ bodySize: capture.totalBytes,
330
+ sourceTruncated: capture.truncated,
331
+ }))
332
+ .catch(() => undefined);
333
+ return new Response(response.body.pipeThrough(observed.stream), {
334
+ status: response.status,
335
+ statusText: response.statusText,
336
+ headers: response.headers,
337
+ });
338
+ }
339
+ async function executeCodexResponsesRequest(ctx) {
274
340
  const requestStartTime = Date.now();
275
341
  const body = ctx.body ?? {};
276
342
  const bodyStr = JSON.stringify(body);
@@ -288,6 +354,25 @@ export async function handleCodexResponsesRequest(ctx) {
288
354
  typeof reasoning.effort === "string"
289
355
  ? reasoning.effort
290
356
  : undefined;
357
+ let tracer;
358
+ try {
359
+ tracer = ProxyTracer.startRequest({
360
+ requestId: ctx.requestId,
361
+ method: ctx.method,
362
+ path: ctx.path,
363
+ model,
364
+ stream: true,
365
+ toolCount: Array.isArray(body.tools)
366
+ ? body.tools.length
367
+ : 0,
368
+ provider: "openai",
369
+ userAgent: ctx.headers["user-agent"],
370
+ recordRequestMetrics: !isFallbackRequest,
371
+ }, ctx.headers);
372
+ }
373
+ catch {
374
+ // Instrumentation must not change provider request handling.
375
+ }
291
376
  const writeFinalLog = (account, responseStatus, extra = {}) => logRequest({
292
377
  timestamp: new Date().toISOString(),
293
378
  requestId: ctx.requestId,
@@ -304,6 +389,9 @@ export async function handleCodexResponsesRequest(ctx) {
304
389
  // This is the cost provider. accountKey and the response header identify
305
390
  // the actual Codex pool that supplied the credential.
306
391
  provider: "openai",
392
+ ...tracer?.getTraceContext(),
393
+ ...(reasoningEffort ? { reasoningEffort } : {}),
394
+ firstUsefulOutputStatus: extra.firstUsefulOutputMs !== undefined ? "observed" : "not_observed",
307
395
  ...buildClientAttribution(ctx.headers),
308
396
  responseStatus,
309
397
  responseTimeMs: Date.now() - requestStartTime,
@@ -311,10 +399,22 @@ export async function handleCodexResponsesRequest(ctx) {
311
399
  });
312
400
  let finalOutcomeRecorded = false;
313
401
  const recordFinalOutcome = async (account, responseStatus, extra = {}) => {
314
- if (isFallbackRequest || finalOutcomeRecorded) {
402
+ if (finalOutcomeRecorded) {
315
403
  return;
316
404
  }
317
405
  finalOutcomeRecorded = true;
406
+ try {
407
+ if (extra.errorType) {
408
+ tracer?.setError(extra.errorType, extra.errorMessage ?? extra.errorType);
409
+ }
410
+ tracer?.end(responseStatus, Date.now() - requestStartTime);
411
+ }
412
+ catch {
413
+ // End bookkeeping is best effort; the client outcome remains authoritative.
414
+ }
415
+ if (isFallbackRequest) {
416
+ return;
417
+ }
318
418
  if (responseStatus >= 400) {
319
419
  recordFinalError(responseStatus, account?.label, account ? CODEX_ACCOUNT_TYPE : undefined, {
320
420
  requestId: ctx.requestId,
@@ -335,6 +435,7 @@ export async function handleCodexResponsesRequest(ctx) {
335
435
  timestamp: new Date().toISOString(),
336
436
  requestId: ctx.requestId,
337
437
  attempt,
438
+ ...tracer?.getTraceContext(),
338
439
  ...(isFallbackRequest
339
440
  ? { parentRequestId: ctx.requestId.replace(/:codex-fallback$/, "") }
340
441
  : {}),
@@ -356,370 +457,432 @@ export async function handleCodexResponsesRequest(ctx) {
356
457
  ...extra,
357
458
  }).catch(() => undefined);
358
459
  };
359
- const accounts = await loadCodexProxyAccounts();
360
- const cancelRequest = async (account) => {
361
- await recordFinalOutcome(account, 499, {
362
- errorType: "client_cancelled",
363
- errorMessage: "Client cancelled Codex request",
364
- terminalOutcome: "client_cancelled",
365
- });
366
- return buildCodexErrorResponse(499, "Client cancelled Codex request");
367
- };
368
- if (ctx.abortSignal?.aborted) {
369
- return cancelRequest();
370
- }
371
- if (accounts.length === 0) {
372
- await recordFinalOutcome(undefined, 401, {
373
- errorType: "no_accounts",
374
- errorMessage: "No Codex accounts",
375
- });
376
- return buildCodexErrorResponse(401, "No Codex accounts configured. Run `neurolink auth login codex`.");
377
- }
378
- const now = Date.now();
379
- const ordered = orderCodexAccounts(accounts, now);
380
- const eligible = ordered.filter((a) => !(a.coolingUntil !== undefined && a.coolingUntil > now));
381
- if (eligible.length === 0) {
382
- // Every account is cooling; surface the soonest recovery as retry-after.
383
- const soonest = ordered.reduce((min, a) => {
384
- if (a.coolingUntil === undefined) {
385
- return min;
386
- }
387
- return min === undefined ? a.coolingUntil : Math.min(min, a.coolingUntil);
388
- }, undefined);
389
- const retryAfterSec = soonest
390
- ? Math.max(1, Math.ceil((soonest - now) / 1000))
391
- : 60;
392
- await recordFinalOutcome(undefined, 429, {
393
- errorType: "all_accounts_cooling",
394
- errorMessage: "All Codex accounts are rate-limited",
395
- });
396
- return new Response(JSON.stringify({
397
- error: {
398
- type: "rate_limit_error",
399
- message: "All Codex accounts are currently rate-limited",
400
- },
401
- }), {
402
- status: 429,
403
- headers: {
404
- "content-type": "application/json",
405
- "retry-after": String(retryAfterSec),
406
- },
407
- });
408
- }
409
- let attempt = 0;
410
- let lastErrorMessage = "All Codex accounts failed";
411
- let lastErrorStatus = 502;
412
- let lastFailure = { errorType: "all_accounts_failed" };
413
- let lastAttemptedAccount;
414
- for (const account of eligible) {
415
- let authRetried = false;
416
- // Same-account loop only re-runs once, for a post-401 token refresh.
417
- for (;;) {
418
- if (ctx.abortSignal?.aborted) {
419
- return cancelRequest(lastAttemptedAccount);
420
- }
421
- attempt += 1;
422
- const attemptStartedAt = Date.now();
423
- lastAttemptedAccount = account;
424
- recordAttempt(account.label, CODEX_ACCOUNT_TYPE);
425
- let upstream;
460
+ const dispatch = async () => {
461
+ const accounts = await loadCodexProxyAccounts();
462
+ const cancelRequest = async (account) => {
463
+ await recordFinalOutcome(account, 499, {
464
+ errorType: "client_cancelled",
465
+ errorMessage: "Client cancelled Codex request",
466
+ terminalOutcome: "client_cancelled",
467
+ });
468
+ return buildCodexErrorResponse(499, "Client cancelled Codex request");
469
+ };
470
+ if (ctx.abortSignal?.aborted) {
471
+ return cancelRequest();
472
+ }
473
+ if (accounts.length === 0) {
474
+ await recordFinalOutcome(undefined, 401, {
475
+ errorType: "no_accounts",
476
+ errorMessage: "No Codex accounts",
477
+ });
478
+ return buildCodexErrorResponse(401, "No Codex accounts configured. Run `neurolink auth login codex`.");
479
+ }
480
+ const now = Date.now();
481
+ const ordered = orderCodexAccounts(accounts, now);
482
+ const eligible = ordered.filter((a) => !(a.coolingUntil !== undefined && a.coolingUntil > now));
483
+ if (eligible.length === 0) {
484
+ // Every account is cooling; surface the soonest recovery as retry-after.
485
+ const soonest = ordered.reduce((min, a) => {
486
+ if (a.coolingUntil === undefined) {
487
+ return min;
488
+ }
489
+ return min === undefined
490
+ ? a.coolingUntil
491
+ : Math.min(min, a.coolingUntil);
492
+ }, undefined);
493
+ const retryAfterSec = soonest
494
+ ? Math.max(1, Math.ceil((soonest - now) / 1000))
495
+ : 60;
496
+ await recordFinalOutcome(undefined, 429, {
497
+ errorType: "all_accounts_cooling",
498
+ errorMessage: "All Codex accounts are rate-limited",
499
+ });
500
+ return new Response(JSON.stringify({
501
+ error: {
502
+ type: "rate_limit_error",
503
+ message: "All Codex accounts are currently rate-limited",
504
+ },
505
+ }), {
506
+ status: 429,
507
+ headers: {
508
+ "content-type": "application/json",
509
+ "retry-after": String(retryAfterSec),
510
+ },
511
+ });
512
+ }
513
+ let attempt = 0;
514
+ let lastErrorMessage = "All Codex accounts failed";
515
+ let lastErrorStatus = 502;
516
+ let lastFailure = { errorType: "all_accounts_failed" };
517
+ let lastAttemptedAccount;
518
+ for (const account of eligible) {
426
519
  try {
427
- upstream = await fetch(CODEX_RESPONSES_URL, {
428
- method: "POST",
429
- headers: buildCodexUpstreamHeaders(ctx.headers, account),
430
- body: bodyStr,
431
- signal: ctx.abortSignal
432
- ? AbortSignal.any([
433
- ctx.abortSignal,
434
- AbortSignal.timeout(CODEX_UPSTREAM_TIMEOUT_MS),
435
- ])
436
- : AbortSignal.timeout(CODEX_UPSTREAM_TIMEOUT_MS),
520
+ tracer?.setAccountSelection({
521
+ strategy: "codex-quota-order",
522
+ accountsTotal: accounts.length,
523
+ accountsHealthy: eligible.length,
524
+ selectedAccount: account.label,
525
+ accountType: CODEX_ACCOUNT_TYPE,
437
526
  });
438
527
  }
439
- catch (error) {
528
+ catch {
529
+ // Account attribution cannot affect routing.
530
+ }
531
+ let authRetried = false;
532
+ // Same-account loop only re-runs once, for a post-401 token refresh.
533
+ for (;;) {
440
534
  if (ctx.abortSignal?.aborted) {
441
- writeAttempt(account, attempt, attemptStartedAt, 499, {
442
- errorType: "client_cancelled",
443
- errorMessage: "Client cancelled Codex request",
444
- retryable: false,
535
+ return cancelRequest(lastAttemptedAccount);
536
+ }
537
+ attempt += 1;
538
+ const attemptStartedAt = Date.now();
539
+ lastAttemptedAccount = account;
540
+ recordAttempt(account.label, CODEX_ACCOUNT_TYPE);
541
+ let upstream;
542
+ try {
543
+ const upstreamHeaders = buildCodexUpstreamHeaders(ctx.headers, account);
544
+ void logBodyCapture({
545
+ timestamp: new Date().toISOString(),
546
+ requestId: ctx.requestId,
547
+ phase: "upstream_request",
548
+ model,
549
+ stream: true,
550
+ body: bodyStr,
551
+ headers: upstreamHeaders,
552
+ contentType: "application/json",
553
+ account: account.label,
554
+ accountType: CODEX_ACCOUNT_TYPE,
555
+ attempt,
556
+ ...tracer?.getTraceContext(),
557
+ });
558
+ upstream = await fetch(CODEX_RESPONSES_URL, {
559
+ method: "POST",
560
+ headers: upstreamHeaders,
561
+ body: bodyStr,
562
+ signal: ctx.abortSignal
563
+ ? AbortSignal.any([
564
+ ctx.abortSignal,
565
+ AbortSignal.timeout(CODEX_UPSTREAM_TIMEOUT_MS),
566
+ ])
567
+ : AbortSignal.timeout(CODEX_UPSTREAM_TIMEOUT_MS),
445
568
  });
446
- return cancelRequest(account);
447
569
  }
448
- // A transport failure message is derived from local state — resolved
449
- // hostnames, socket paths, Node internals — and says nothing the caller
450
- // can act on. Keep the detail in the log and return a fixed string, so
451
- // internal topology never reaches the client.
452
- logger.debug(`Codex upstream fetch failed (${account.label}): ${sanitizeForLog(error instanceof Error ? error.message : String(error))}`);
453
- const errorMessage = summarizeCodexUpstreamError(error instanceof Error ? error.message : String(error), "Codex upstream request failed");
454
- const errorCode = getCodexTransportErrorCode(error);
455
- const transportScope = codexTransportScope(error);
456
- recordAttemptError(account.label, CODEX_ACCOUNT_TYPE, 502);
457
- writeAttempt(account, attempt, attemptStartedAt, 502, {
458
- errorType: "network_error",
459
- errorMessage,
460
- ...(errorCode ? { errorCode } : {}),
461
- transportScope,
462
- // These codes prove failure before HTTP dispatch. Socket resets,
463
- // EPIPE and generic timeouts may follow dispatch and must not replay.
464
- retryable: [
570
+ catch (error) {
571
+ if (ctx.abortSignal?.aborted) {
572
+ writeAttempt(account, attempt, attemptStartedAt, 499, {
573
+ errorType: "client_cancelled",
574
+ errorMessage: "Client cancelled Codex request",
575
+ retryable: false,
576
+ });
577
+ return cancelRequest(account);
578
+ }
579
+ // A transport failure message is derived from local state — resolved
580
+ // hostnames, socket paths, Node internals — and says nothing the caller
581
+ // can act on. Keep the detail in the log and return a fixed string, so
582
+ // internal topology never reaches the client.
583
+ logger.debug(`Codex upstream fetch failed (${account.label}): ${sanitizeForLog(error instanceof Error ? error.message : String(error))}`);
584
+ const errorMessage = summarizeCodexUpstreamError(error instanceof Error ? error.message : String(error), "Codex upstream request failed");
585
+ const errorCode = getCodexTransportErrorCode(error);
586
+ const transportScope = codexTransportScope(error);
587
+ recordAttemptError(account.label, CODEX_ACCOUNT_TYPE, 502);
588
+ writeAttempt(account, attempt, attemptStartedAt, 502, {
589
+ errorType: "network_error",
590
+ errorMessage,
591
+ ...(errorCode ? { errorCode } : {}),
592
+ transportScope,
593
+ // These codes prove failure before HTTP dispatch. Socket resets,
594
+ // EPIPE and generic timeouts may follow dispatch and must not replay.
595
+ retryable: [
596
+ "UND_ERR_CONNECT_TIMEOUT",
597
+ "ECONNREFUSED",
598
+ "ENOTFOUND",
599
+ "EAI_AGAIN",
600
+ ].includes(errorCode ?? ""),
601
+ });
602
+ lastFailure = {
603
+ errorType: "network_error",
604
+ errorMessage,
605
+ errorCode,
606
+ transportScope,
607
+ };
608
+ if (![
465
609
  "UND_ERR_CONNECT_TIMEOUT",
466
610
  "ECONNREFUSED",
467
611
  "ENOTFOUND",
468
612
  "EAI_AGAIN",
469
- ].includes(errorCode ?? ""),
470
- });
471
- lastFailure = {
472
- errorType: "network_error",
473
- errorMessage,
474
- errorCode,
475
- transportScope,
476
- };
477
- if (![
478
- "UND_ERR_CONNECT_TIMEOUT",
479
- "ECONNREFUSED",
480
- "ENOTFOUND",
481
- "EAI_AGAIN",
482
- ].includes(errorCode ?? "")) {
483
- await recordFinalOutcome(account, 502, {
484
- ...lastFailure,
485
- errorMessage,
486
- });
487
- return buildCodexErrorResponse(502, "Codex upstream request failed");
488
- }
489
- lastErrorMessage = "Codex upstream request failed";
490
- lastErrorStatus = 502;
491
- break; // rotate to next account
492
- }
493
- if (upstream.ok) {
494
- const quota = parseCodexRateLimitHeaders(upstream.headers);
495
- if (quota) {
496
- saveAccountQuota(account.key, quota).catch(() => undefined);
497
- }
498
- // A prior cooldown that has expired is cleared on success. The
499
- // compare-and-swap guards against wiping a longer cooldown that another
500
- // in-flight request set while this one was upstream.
501
- if (account.expiredCooldownUntil !== undefined) {
502
- clearAccountCooldown(account.key, account.expiredCooldownUntil).catch(() => undefined);
613
+ ].includes(errorCode ?? "")) {
614
+ await recordFinalOutcome(account, 502, {
615
+ ...lastFailure,
616
+ errorMessage,
617
+ });
618
+ return buildCodexErrorResponse(502, "Codex upstream request failed");
619
+ }
620
+ lastErrorMessage = "Codex upstream request failed";
621
+ lastErrorStatus = 502;
622
+ break; // rotate to next account
503
623
  }
504
- publishCodexHeaders(ctx, account, attempt, quota);
505
- writeAttempt(account, attempt, attemptStartedAt, upstream.status);
506
- const headers = {
507
- "content-type": upstream.headers.get("content-type") ?? "text/event-stream",
508
- "cache-control": "no-cache",
509
- connection: "keep-alive",
510
- ...(ctx.responseHeaders ?? {}),
511
- };
512
- if (!upstream.body) {
513
- recordAttemptError(account.label, CODEX_ACCOUNT_TYPE, 502);
514
- writeAttempt(account, attempt, attemptStartedAt, 502, {
515
- errorType: "incomplete_stream",
516
- errorMessage: "Codex returned no response stream",
517
- retryable: false,
518
- });
519
- await recordFinalOutcome(account, 502, {
520
- terminalOutcome: "stream_error",
521
- errorType: "incomplete_stream",
522
- errorMessage: "Codex returned no response stream",
523
- });
524
- return new Response(upstream.body, {
624
+ upstream = captureCodexResponse(ctx, upstream, "upstream_response", account, attempt);
625
+ if (upstream.ok) {
626
+ const quota = parseCodexRateLimitHeaders(upstream.headers);
627
+ if (quota) {
628
+ saveAccountQuota(account.key, quota).catch(() => undefined);
629
+ }
630
+ // A prior cooldown that has expired is cleared on success. The
631
+ // compare-and-swap guards against wiping a longer cooldown that another
632
+ // in-flight request set while this one was upstream.
633
+ if (account.expiredCooldownUntil !== undefined) {
634
+ clearAccountCooldown(account.key, account.expiredCooldownUntil).catch(() => undefined);
635
+ }
636
+ publishCodexHeaders(ctx, account, attempt, quota);
637
+ writeAttempt(account, attempt, attemptStartedAt, upstream.status);
638
+ const headers = {
639
+ "content-type": upstream.headers.get("content-type") ?? "text/event-stream",
640
+ "cache-control": "no-cache",
641
+ connection: "keep-alive",
642
+ ...(ctx.responseHeaders ?? {}),
643
+ };
644
+ if (!upstream.body) {
645
+ recordAttemptError(account.label, CODEX_ACCOUNT_TYPE, 502);
646
+ writeAttempt(account, attempt, attemptStartedAt, 502, {
647
+ errorType: "incomplete_stream",
648
+ errorMessage: "Codex returned no response stream",
649
+ retryable: false,
650
+ });
651
+ await recordFinalOutcome(account, 502, {
652
+ terminalOutcome: "stream_error",
653
+ errorType: "incomplete_stream",
654
+ errorMessage: "Codex returned no response stream",
655
+ });
656
+ return new Response(upstream.body, {
657
+ status: upstream.status,
658
+ headers,
659
+ });
660
+ }
661
+ const { stream: usageTap, usage: usageSeen, evidence, } = createCodexUsageTap();
662
+ const relay = new Response(upstream.body.pipeThrough(usageTap), {
525
663
  status: upstream.status,
526
664
  headers,
527
665
  });
528
- }
529
- const { stream: usageTap, usage: usageSeen, evidence, } = createCodexUsageTap();
530
- const relay = new Response(upstream.body.pipeThrough(usageTap), {
531
- status: upstream.status,
532
- headers,
533
- });
534
- registerProxyResponseObserver(ctx.metadata, {
535
- onTerminal: ({ outcome, error, observedBodyBytes }) => {
536
- return usageSeen
537
- .then((usage) => {
538
- const semantic = evidence();
539
- const completedFrameDelivered = semantic.completed &&
540
- observedBodyBytes >= semantic.terminalBytes;
541
- const failed = semantic.errorType ||
542
- ((outcome === "completed" || outcome === "bodyless") &&
543
- !semantic.completed);
544
- const usageExtra = usage
545
- ? {
546
- inputTokens: usage.inputTokens,
547
- outputTokens: usage.outputTokens,
548
- cacheReadTokens: usage.cacheReadTokens,
549
- cacheCreationTokens: usage.cacheCreationTokens,
666
+ registerProxyResponseObserver(ctx.metadata, {
667
+ onTerminal: ({ outcome, error, observedBodyBytes }) => {
668
+ return usageSeen
669
+ .then((usage) => {
670
+ const semantic = evidence();
671
+ const completedFrameDelivered = semantic.completed &&
672
+ observedBodyBytes >= semantic.terminalBytes;
673
+ const failed = semantic.errorType ||
674
+ ((outcome === "completed" || outcome === "bodyless") &&
675
+ !semantic.completed);
676
+ const usageExtra = usage
677
+ ? {
678
+ inputTokens: usage.inputTokens,
679
+ outputTokens: usage.outputTokens,
680
+ cacheReadTokens: usage.cacheReadTokens,
681
+ cacheCreationTokens: usage.cacheCreationTokens,
682
+ }
683
+ : {};
684
+ if (usage) {
685
+ try {
686
+ tracer?.setUsage(usage);
687
+ }
688
+ catch {
689
+ // Pricing/metrics must never change the stream outcome.
690
+ }
550
691
  }
551
- : {};
552
- const timing = semantic.firstUsefulOutputAt === undefined
553
- ? {}
554
- : {
555
- firstUsefulOutputMs: Math.max(0, semantic.firstUsefulOutputAt - requestStartTime),
556
- };
557
- if (failed) {
558
- recordAttemptError(account.label, CODEX_ACCOUNT_TYPE, 502);
559
- writeAttempt(account, attempt, attemptStartedAt, 502, {
560
- errorType: semantic.errorType ?? "incomplete_stream",
561
- errorCode: semantic.errorCode,
562
- errorMessage: semantic.errorMessage ??
563
- "Codex stream ended without a completion event",
564
- retryable: false,
565
- });
566
- return recordFinalOutcome(account, 502, {
567
- ...usageExtra,
568
- ...timing,
569
- terminalOutcome: "stream_error",
570
- errorType: semantic.errorType ?? "incomplete_stream",
571
- errorCode: semantic.errorCode,
572
- errorMessage: semantic.errorMessage ??
573
- "Codex stream ended without a completion event",
574
- });
575
- }
576
- if (outcome === "completed" ||
577
- outcome === "bodyless" ||
578
- (outcome === "client_cancelled" && completedFrameDelivered)) {
579
- return recordFinalOutcome(account, upstream.status, {
580
- terminalOutcome: "completed",
692
+ const timing = semantic.firstUsefulOutputAt === undefined ||
693
+ semantic.observationIncomplete
694
+ ? {
695
+ firstUsefulOutputStatus: semantic.completed &&
696
+ !semantic.observationIncomplete
697
+ ? "no_useful_output"
698
+ : "not_observed",
699
+ }
700
+ : {
701
+ firstUsefulOutputStatus: "observed",
702
+ firstUsefulOutputEvent: semantic.firstUsefulOutputEvent,
703
+ firstUsefulOutputMs: Math.max(0, semantic.firstUsefulOutputAt - requestStartTime),
704
+ };
705
+ if (failed) {
706
+ recordAttemptError(account.label, CODEX_ACCOUNT_TYPE, 502);
707
+ writeAttempt(account, attempt, attemptStartedAt, 502, {
708
+ errorType: semantic.errorType ?? "incomplete_stream",
709
+ errorCode: semantic.errorCode,
710
+ errorMessage: semantic.errorMessage ??
711
+ "Codex stream ended without a completion event",
712
+ retryable: false,
713
+ });
714
+ return recordFinalOutcome(account, 502, {
715
+ ...usageExtra,
716
+ ...timing,
717
+ terminalOutcome: "stream_error",
718
+ errorType: semantic.errorType ?? "incomplete_stream",
719
+ errorCode: semantic.errorCode,
720
+ errorMessage: semantic.errorMessage ??
721
+ "Codex stream ended without a completion event",
722
+ });
723
+ }
724
+ if (outcome === "completed" ||
725
+ outcome === "bodyless" ||
726
+ (outcome === "client_cancelled" && completedFrameDelivered)) {
727
+ return recordFinalOutcome(account, upstream.status, {
728
+ terminalOutcome: "completed",
729
+ ...usageExtra,
730
+ ...timing,
731
+ });
732
+ }
733
+ if (outcome === "stream_error") {
734
+ recordAttemptError(account.label, CODEX_ACCOUNT_TYPE, 502);
735
+ writeAttempt(account, attempt, attemptStartedAt, 502, {
736
+ errorType: "stream_error",
737
+ errorCode: getCodexTransportErrorCode(error),
738
+ errorMessage: summarizeCodexUpstreamError(error instanceof Error ? error.message : "", "Codex upstream stream failed"),
739
+ retryable: false,
740
+ });
741
+ }
742
+ return recordFinalOutcome(account, outcome === "client_cancelled" ? 499 : 502, {
743
+ errorType: outcome === "client_cancelled"
744
+ ? "client_cancelled"
745
+ : "stream_error",
746
+ errorMessage: outcome === "client_cancelled"
747
+ ? "Client cancelled Codex stream"
748
+ : summarizeCodexUpstreamError(error instanceof Error ? error.message : "", "Codex upstream stream failed"),
749
+ ...(outcome === "stream_error"
750
+ ? { errorCode: getCodexTransportErrorCode(error) }
751
+ : {}),
752
+ terminalOutcome: outcome,
581
753
  ...usageExtra,
582
754
  ...timing,
583
755
  });
584
- }
585
- if (outcome === "stream_error") {
586
- recordAttemptError(account.label, CODEX_ACCOUNT_TYPE, 502);
587
- writeAttempt(account, attempt, attemptStartedAt, 502, {
588
- errorType: "stream_error",
589
- errorCode: getCodexTransportErrorCode(error),
590
- errorMessage: summarizeCodexUpstreamError(error instanceof Error ? error.message : "", "Codex upstream stream failed"),
591
- retryable: false,
592
- });
593
- }
594
- return recordFinalOutcome(account, outcome === "client_cancelled" ? 499 : 502, {
595
- errorType: outcome === "client_cancelled"
596
- ? "client_cancelled"
597
- : "stream_error",
598
- errorMessage: outcome === "client_cancelled"
599
- ? "Client cancelled Codex stream"
600
- : summarizeCodexUpstreamError(error instanceof Error ? error.message : "", "Codex upstream stream failed"),
601
- ...(outcome === "stream_error"
602
- ? { errorCode: getCodexTransportErrorCode(error) }
603
- : {}),
604
- terminalOutcome: outcome,
605
- ...usageExtra,
606
- ...timing,
607
- });
608
- })
609
- .catch(() => undefined);
610
- },
611
- });
612
- return relay;
613
- }
614
- const errText = await upstream.text().catch(() => "");
615
- // 401/403 → try a forced token refresh once, then rotate.
616
- if ((upstream.status === 401 || upstream.status === 403) &&
617
- !authRetried &&
618
- account.refreshToken) {
619
- const errorMessage = summarizeCodexUpstreamError(errText, "Codex authentication rejected upstream");
620
- recordAttemptError(account.label, CODEX_ACCOUNT_TYPE, upstream.status);
621
- writeAttempt(account, attempt, attemptStartedAt, upstream.status, {
622
- errorType: "authentication_error",
623
- errorMessage,
624
- retryable: true,
625
- });
626
- authRetried = true;
627
- const staleTokens = {
628
- accessToken: account.token,
629
- refreshToken: account.refreshToken,
630
- expiresAt: account.expiresAt ?? 0,
631
- };
632
- try {
633
- const refreshed = await refreshCodexTokenOnce(account.key, account.refreshToken);
634
- account.token = refreshed.accessToken;
635
- account.refreshToken = refreshed.refreshToken ?? account.refreshToken;
636
- account.expiresAt = refreshed.expiresAt ?? account.expiresAt;
637
- account.accountId = resolveCodexAccountId(refreshed.accessToken);
638
- continue; // retry same account with the fresh token
756
+ })
757
+ .catch(() => undefined);
758
+ },
759
+ });
760
+ return relay;
639
761
  }
640
- catch (error) {
641
- if (isPermanentCodexRefreshFailure(error)) {
642
- // Compare-and-swap: the pool is rebuilt per request with no shared
643
- // state, so a concurrent request may already have rotated this
644
- // credential. Disabling unconditionally would kill the account that
645
- // the other request just healed.
646
- const disabled = await tokenStore.markDisabledIfCurrent(account.key, staleTokens, "refresh_invalid");
647
- if (disabled) {
648
- logger.always(`[proxy] codex account=${account.label} disabled until re-authentication. Run: neurolink auth login codex --label ${account.label}`);
762
+ const errText = await upstream.text().catch(() => "");
763
+ // 401/403 → try a forced token refresh once, then rotate.
764
+ if ((upstream.status === 401 || upstream.status === 403) &&
765
+ !authRetried &&
766
+ account.refreshToken) {
767
+ const errorMessage = summarizeCodexUpstreamError(errText, "Codex authentication rejected upstream");
768
+ recordAttemptError(account.label, CODEX_ACCOUNT_TYPE, upstream.status);
769
+ writeAttempt(account, attempt, attemptStartedAt, upstream.status, {
770
+ errorType: "authentication_error",
771
+ errorMessage,
772
+ retryable: true,
773
+ });
774
+ authRetried = true;
775
+ const staleTokens = {
776
+ accessToken: account.token,
777
+ refreshToken: account.refreshToken,
778
+ expiresAt: account.expiresAt ?? 0,
779
+ };
780
+ try {
781
+ const refreshed = await refreshCodexTokenOnce(account.key, account.refreshToken);
782
+ account.token = refreshed.accessToken;
783
+ account.refreshToken =
784
+ refreshed.refreshToken ?? account.refreshToken;
785
+ account.expiresAt = refreshed.expiresAt ?? account.expiresAt;
786
+ account.accountId = resolveCodexAccountId(refreshed.accessToken);
787
+ continue; // retry same account with the fresh token
788
+ }
789
+ catch (error) {
790
+ if (isPermanentCodexRefreshFailure(error)) {
791
+ // Compare-and-swap: the pool is rebuilt per request with no shared
792
+ // state, so a concurrent request may already have rotated this
793
+ // credential. Disabling unconditionally would kill the account that
794
+ // the other request just healed.
795
+ const disabled = await tokenStore.markDisabledIfCurrent(account.key, staleTokens, "refresh_invalid");
796
+ if (disabled) {
797
+ logger.always(`[proxy] codex account=${account.label} disabled until re-authentication. Run: neurolink auth login codex --label ${account.label}`);
798
+ }
799
+ lastFailure = {
800
+ errorType: "authentication_error",
801
+ errorCode: "refresh_invalid",
802
+ };
803
+ lastErrorStatus = 401;
804
+ lastErrorMessage =
805
+ "Codex token refresh failed; re-login required";
806
+ break;
649
807
  }
808
+ // No verdict on the credential — cool briefly and try the next
809
+ // account, so a 5xx or a timeout cannot cost the user a login.
810
+ await saveAccountCooldown(account.key, Date.now() + CODEX_AUTH_COOLDOWN_MS, "auth").catch(() => undefined);
811
+ logger.debug(`[proxy] codex account=${account.label} refresh failed transiently; cooling and rotating`);
650
812
  lastFailure = {
651
- errorType: "authentication_error",
652
- errorCode: "refresh_invalid",
813
+ errorType: "auth_refresh_unavailable",
814
+ errorCode: getCodexTransportErrorCode(error),
653
815
  };
654
- lastErrorStatus = 401;
655
- lastErrorMessage = "Codex token refresh failed; re-login required";
816
+ lastErrorStatus = 503;
817
+ lastErrorMessage = "Codex token refresh temporarily unavailable";
656
818
  break;
657
819
  }
658
- // No verdict on the credential — cool briefly and try the next
659
- // account, so a 5xx or a timeout cannot cost the user a login.
660
- await saveAccountCooldown(account.key, Date.now() + CODEX_AUTH_COOLDOWN_MS, "auth").catch(() => undefined);
661
- logger.debug(`[proxy] codex account=${account.label} refresh failed transiently; cooling and rotating`);
662
- lastFailure = {
663
- errorType: "auth_refresh_unavailable",
664
- errorCode: getCodexTransportErrorCode(error),
665
- };
666
- lastErrorStatus = 503;
667
- lastErrorMessage = "Codex token refresh temporarily unavailable";
668
- break;
669
820
  }
670
- }
671
- // 429 → cooldown + rotate.
672
- if (upstream.status === 429) {
673
- const quota = parseCodexRateLimitHeaders(upstream.headers);
674
- if (quota) {
675
- saveAccountQuota(account.key, quota).catch(() => undefined);
821
+ // 429 → cooldown + rotate.
822
+ if (upstream.status === 429) {
823
+ const quota = parseCodexRateLimitHeaders(upstream.headers);
824
+ if (quota) {
825
+ saveAccountQuota(account.key, quota).catch(() => undefined);
826
+ }
827
+ const retryAfterMs = parseRetryAfterMs(upstream.headers.get("retry-after"));
828
+ const plan = planCodexCooldown(quota, retryAfterMs, Date.now());
829
+ await saveAccountCooldown(account.key, plan.coolingUntil, plan.reason).catch(() => undefined);
830
+ const rateLimitKind = plan.reason === "transient" ? "transient" : "quota";
831
+ const errorMessage = summarizeCodexUpstreamError(errText, "Codex account rate-limited");
832
+ recordAttemptError(account.label, CODEX_ACCOUNT_TYPE, upstream.status, rateLimitKind);
833
+ writeAttempt(account, attempt, attemptStartedAt, upstream.status, {
834
+ errorType: "rate_limit_error",
835
+ errorMessage,
836
+ retryable: true,
837
+ rateLimitKind,
838
+ cooldownReason: plan.reason,
839
+ });
840
+ lastFailure = { errorType: "rate_limit_error" };
841
+ lastErrorStatus = 429;
842
+ lastErrorMessage = "Codex account rate-limited";
843
+ break; // rotate
676
844
  }
677
- const retryAfterMs = parseRetryAfterMs(upstream.headers.get("retry-after"));
678
- const plan = planCodexCooldown(quota, retryAfterMs, Date.now());
679
- await saveAccountCooldown(account.key, plan.coolingUntil, plan.reason).catch(() => undefined);
680
- const rateLimitKind = plan.reason === "transient" ? "transient" : "quota";
681
- const errorMessage = summarizeCodexUpstreamError(errText, "Codex account rate-limited");
682
- recordAttemptError(account.label, CODEX_ACCOUNT_TYPE, upstream.status, rateLimitKind);
845
+ // Other non-ok → record and rotate.
846
+ if (upstream.status === 401 || upstream.status === 403) {
847
+ // Reached only when the account has no refresh token to retry with, so
848
+ // it will fail identically on the next request. Park it briefly instead
849
+ // of letting it stay first in line with unknown quota.
850
+ await saveAccountCooldown(account.key, Date.now() + CODEX_AUTH_COOLDOWN_MS, "auth").catch(() => undefined);
851
+ }
852
+ const errorMessage = summarizeCodexUpstreamError(errText, "Codex error");
853
+ const errorType = upstream.status === 401 || upstream.status === 403
854
+ ? "authentication_error"
855
+ : "api_error";
856
+ recordAttemptError(account.label, CODEX_ACCOUNT_TYPE, upstream.status);
683
857
  writeAttempt(account, attempt, attemptStartedAt, upstream.status, {
684
- errorType: "rate_limit_error",
858
+ errorType,
685
859
  errorMessage,
686
- retryable: true,
687
- rateLimitKind,
688
- cooldownReason: plan.reason,
860
+ retryable: upstream.status >= 500,
689
861
  });
690
- lastFailure = { errorType: "rate_limit_error" };
691
- lastErrorStatus = 429;
692
- lastErrorMessage = "Codex account rate-limited";
862
+ lastFailure = { errorType };
863
+ lastErrorStatus = upstream.status >= 500 ? 502 : upstream.status;
864
+ lastErrorMessage = errorMessage;
693
865
  break; // rotate
694
866
  }
695
- // Other non-ok → record and rotate.
696
- if (upstream.status === 401 || upstream.status === 403) {
697
- // Reached only when the account has no refresh token to retry with, so
698
- // it will fail identically on the next request. Park it briefly instead
699
- // of letting it stay first in line with unknown quota.
700
- await saveAccountCooldown(account.key, Date.now() + CODEX_AUTH_COOLDOWN_MS, "auth").catch(() => undefined);
701
- }
702
- const errorMessage = summarizeCodexUpstreamError(errText, "Codex error");
703
- const errorType = upstream.status === 401 || upstream.status === 403
704
- ? "authentication_error"
705
- : "api_error";
706
- recordAttemptError(account.label, CODEX_ACCOUNT_TYPE, upstream.status);
707
- writeAttempt(account, attempt, attemptStartedAt, upstream.status, {
708
- errorType,
709
- errorMessage,
710
- retryable: upstream.status >= 500,
711
- });
712
- lastFailure = { errorType };
713
- lastErrorStatus = upstream.status >= 500 ? 502 : upstream.status;
714
- lastErrorMessage = errorMessage;
715
- break; // rotate
716
867
  }
868
+ await recordFinalOutcome(lastAttemptedAccount, lastErrorStatus, {
869
+ ...lastFailure,
870
+ errorMessage: lastFailure.errorMessage ?? lastErrorMessage,
871
+ });
872
+ return buildCodexErrorResponse(lastErrorStatus, lastErrorMessage);
873
+ };
874
+ try {
875
+ return await dispatch();
876
+ }
877
+ catch (error) {
878
+ try {
879
+ tracer?.end(502, Date.now() - requestStartTime);
880
+ }
881
+ catch {
882
+ // Shared HTTP error handling owns the client outcome and final log.
883
+ }
884
+ throw error;
717
885
  }
718
- await recordFinalOutcome(lastAttemptedAccount, lastErrorStatus, {
719
- ...lastFailure,
720
- errorMessage: lastFailure.errorMessage ?? lastErrorMessage,
721
- });
722
- return buildCodexErrorResponse(lastErrorStatus, lastErrorMessage);
723
886
  }
724
887
  /**
725
888
  * Relay Codex model discovery upstream.