@juspay/neurolink 12.14.6 → 12.14.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -12,6 +12,7 @@ import { applyAllClients, restoreAllClients, } from "../proxy-clients/registry.j
12
12
  import { resolveProxyConfigPath } from "../../proxy/proxyConfig.js";
13
13
  import { redactUrlsInText, sanitizeForLog, } from "../../utils/logSanitize.js";
14
14
  import { withTimeout } from "../../utils/async/withTimeout.js";
15
+ import { startProxyHttpTrace } from "../../proxy/proxyTracer.js";
15
16
  import { formatUptime, isProcessRunning, StateFileManager, } from "../utils/serverUtils.js";
16
17
  import { configureProxyKeepAliveDispatcher } from "../../proxy/proxyDispatcher.js";
17
18
  import { ProxyRuntimeConfigStore } from "../../proxy/runtimeConfig.js";
@@ -837,13 +838,36 @@ function spawnProxyUpdater(host, port, parentPid, rollingSupervisor = false) {
837
838
  workerLog.close();
838
839
  }
839
840
  }
840
- async function runProxyTelemetryManager(command) {
841
+ async function runProxyTelemetryManager(command, argv) {
841
842
  const { existsSync } = await import("fs");
842
843
  if (!existsSync(PROXY_TELEMETRY_SCRIPT_PATH)) {
843
844
  throw new Error("Proxy telemetry helper files were not found in this installation. Reinstall NeuroLink with observability assets included.");
844
845
  }
845
846
  await new Promise((resolve, reject) => {
846
- const child = spawn("bash", [PROXY_TELEMETRY_SCRIPT_PATH, command], {
847
+ const queryCommand = command === "doctor" || command === "query";
848
+ const queryArgs = [];
849
+ if (queryCommand && argv) {
850
+ for (const [key, option] of [
851
+ ["since", "--since"],
852
+ ["until", "--until"],
853
+ ["kind", "--kind"],
854
+ ["format", "--format"],
855
+ ["maxRows", "--max-rows"],
856
+ ["proxyUrl", "--proxy-url"],
857
+ ]) {
858
+ if (argv[key] !== undefined) {
859
+ queryArgs.push(option, String(argv[key]));
860
+ }
861
+ }
862
+ }
863
+ const child = spawn(queryCommand ? process.execPath : "bash", queryCommand
864
+ ? [
865
+ join(dirname(PROXY_TELEMETRY_SCRIPT_PATH), command === "doctor"
866
+ ? "check-proxy-telemetry.mjs"
867
+ : "query-proxy-history.mjs"),
868
+ ...queryArgs,
869
+ ]
870
+ : [PROXY_TELEMETRY_SCRIPT_PATH, command], {
847
871
  stdio: "inherit",
848
872
  env: process.env,
849
873
  });
@@ -1078,6 +1102,7 @@ function registerProxyRequestTracking(app, requestMetadata, readiness) {
1078
1102
  rejectForUpdate: readiness.drainingForUpdate,
1079
1103
  };
1080
1104
  requestMetadata.set(c.req.raw, metadata);
1105
+ const httpTrace = startProxyHttpTrace(metadata, Object.fromEntries(c.req.raw.headers));
1081
1106
  const stopObservingFinalLog = observeProxyFinalLog(metadata.requestId, (entry) => {
1082
1107
  metadata.terminalResult = entry;
1083
1108
  }, (entry) => {
@@ -1087,6 +1112,7 @@ function registerProxyRequestTracking(app, requestMetadata, readiness) {
1087
1112
  ? () => undefined
1088
1113
  : beginProxyRequest();
1089
1114
  const finish = () => {
1115
+ httpTrace.end(metadata.terminalResult?.responseStatus ?? c.res.status, metadata.terminalResult?.terminalOutcome ?? "unknown", metadata.terminalErrorType);
1090
1116
  stopObservingFinalLog();
1091
1117
  finishActivity();
1092
1118
  // Borrowed traffic holds a concurrency slot for the lifetime of the
@@ -1099,7 +1125,7 @@ function registerProxyRequestTracking(app, requestMetadata, readiness) {
1099
1125
  // them at acceptance instead of publishing misleading placeholder values;
1100
1126
  // subsequent events carry the parsed metadata under the same request ID.
1101
1127
  try {
1102
- await persistProxyLifecycleAcceptance({
1128
+ await httpTrace.run(() => persistProxyLifecycleAcceptance({
1103
1129
  requestId: metadata.requestId,
1104
1130
  method: metadata.method,
1105
1131
  path: metadata.path,
@@ -1107,8 +1133,8 @@ function registerProxyRequestTracking(app, requestMetadata, readiness) {
1107
1133
  requestBytes,
1108
1134
  elapsedMs: 0,
1109
1135
  monotonicMs: startedMonotonicMs,
1110
- });
1111
- await next();
1136
+ }));
1137
+ await httpTrace.run(next);
1112
1138
  const responseStatus = c.res.status;
1113
1139
  logProxyLifecycleEvent({
1114
1140
  event: "response_headers",
@@ -1169,12 +1195,12 @@ function registerProxyRequestTracking(app, requestMetadata, readiness) {
1169
1195
  let accountingTimedOut = false;
1170
1196
  let accountingFailed = false;
1171
1197
  try {
1172
- accountingFailed = await notifyRouteTerminal({
1198
+ accountingFailed = await httpTrace.run(() => notifyRouteTerminal({
1173
1199
  outcome,
1174
1200
  error,
1175
1201
  observedBodyBytes,
1176
1202
  responseChunks,
1177
- });
1203
+ }));
1178
1204
  }
1179
1205
  catch {
1180
1206
  accountingTimedOut = true;
@@ -1231,6 +1257,7 @@ function registerProxyRequestTracking(app, requestMetadata, readiness) {
1231
1257
  errorCode: final?.errorCode ?? metadata.terminalErrorCode,
1232
1258
  });
1233
1259
  stopObservingFinalLog();
1260
+ httpTrace.end(final?.responseStatus ?? responseStatus, terminalOutcome, final?.errorType ?? metadata.terminalErrorType);
1234
1261
  },
1235
1262
  });
1236
1263
  }
@@ -1259,6 +1286,9 @@ function registerProxyRequestTracking(app, requestMetadata, readiness) {
1259
1286
  errorType: error instanceof Error ? error.name : "unknown_error",
1260
1287
  errorCode: getProxyRuntimeErrorCode(error),
1261
1288
  });
1289
+ httpTrace.end(getProxyRuntimeErrorCode(error) === "PROXY_TELEMETRY_UNAVAILABLE"
1290
+ ? 503
1291
+ : 502, "handler_error", error instanceof Error ? error.name : "unknown_error");
1262
1292
  throw error;
1263
1293
  }
1264
1294
  };
@@ -1331,8 +1361,9 @@ export async function createProxyStartApp(params) {
1331
1361
  accountKey: attempt?.accountKey,
1332
1362
  provider: attempt?.provider,
1333
1363
  transportScope: attempt?.transportScope,
1334
- traceId: attempt?.traceId,
1335
- spanId: attempt?.spanId,
1364
+ traceId: attempt?.traceId ?? metadata.traceId,
1365
+ spanId: attempt?.spanId ?? metadata.spanId,
1366
+ traceFlags: attempt?.traceFlags ?? metadata.traceFlags,
1336
1367
  responseStatus: status,
1337
1368
  responseTimeMs: Date.now() - metadata.startedAt,
1338
1369
  errorType,
@@ -1342,6 +1373,9 @@ export async function createProxyStartApp(params) {
1342
1373
  logBodyCapture({
1343
1374
  timestamp: new Date().toISOString(),
1344
1375
  requestId: metadata.requestId,
1376
+ traceId: attempt?.traceId ?? metadata.traceId,
1377
+ spanId: attempt?.spanId ?? metadata.spanId,
1378
+ traceFlags: attempt?.traceFlags ?? metadata.traceFlags,
1345
1379
  model: metadata.model,
1346
1380
  stream: metadata.stream,
1347
1381
  phase: "client_response",
@@ -3553,6 +3587,8 @@ export const proxyStatusCommand = {
3553
3587
  // PROXY TELEMETRY COMMAND
3554
3588
  // =============================================================================
3555
3589
  const PROXY_TELEMETRY_ACTIONS = [
3590
+ "doctor",
3591
+ "query",
3556
3592
  "setup",
3557
3593
  "start",
3558
3594
  "stop",
@@ -3562,12 +3598,37 @@ const PROXY_TELEMETRY_ACTIONS = [
3562
3598
  ];
3563
3599
  export const proxyTelemetryCommand = {
3564
3600
  command: "telemetry <action>",
3565
- describe: "Manage the local OpenObserve stack and dashboard for proxy observability",
3601
+ describe: "Query and verify stored OTel telemetry or manage the local observability stack",
3566
3602
  builder: (yargs) => yargs
3567
3603
  .positional("action", {
3568
3604
  type: "string",
3569
3605
  choices: [...PROXY_TELEMETRY_ACTIONS],
3570
- describe: "Telemetry action: setup, start, stop, status, logs, or import-dashboard",
3606
+ describe: "Telemetry action: doctor, query, setup, start, stop, status, logs, or import-dashboard",
3607
+ })
3608
+ .option("since", {
3609
+ type: "string",
3610
+ description: "Inclusive ISO timestamp for stored telemetry",
3611
+ })
3612
+ .option("until", {
3613
+ type: "string",
3614
+ description: "Exclusive ISO timestamp for stored telemetry",
3615
+ })
3616
+ .option("kind", {
3617
+ type: "string",
3618
+ description: "Metadata kind for query, such as request_final or telemetry_delivery",
3619
+ })
3620
+ .option("format", {
3621
+ type: "string",
3622
+ choices: ["json", "text"],
3623
+ description: "Doctor report format; query always returns JSON",
3624
+ })
3625
+ .option("max-rows", {
3626
+ type: "number",
3627
+ description: "Explicit bounded history record limit",
3628
+ })
3629
+ .option("proxy-url", {
3630
+ type: "string",
3631
+ description: "Proxy endpoint used by the read-only doctor",
3571
3632
  })
3572
3633
  .option("quiet", {
3573
3634
  type: "boolean",
@@ -3575,19 +3636,21 @@ export const proxyTelemetryCommand = {
3575
3636
  default: false,
3576
3637
  description: "Suppress the local CLI spinner and delegate directly",
3577
3638
  })
3639
+ .example("neurolink proxy telemetry doctor --format json", "Verify field coverage, stored captures, trace correlation, and delivery evidence")
3640
+ .example("neurolink proxy telemetry query --since 2026-09-15T00:00:00Z --kind request_final", "Read stored OTLP metadata without scanning proxy log files")
3578
3641
  .example("neurolink proxy telemetry setup", "Start OpenObserve, start the OTEL collector, and import the dashboard")
3579
3642
  .example("neurolink proxy telemetry start", "Start the local proxy telemetry stack without re-importing the dashboard")
3580
3643
  .example("neurolink proxy telemetry stop", "Stop the local OpenObserve and OTEL collector containers"),
3581
3644
  handler: async (argv) => {
3582
3645
  const action = argv.action;
3583
- const spinner = argv.quiet
3646
+ const spinner = argv.quiet || action === "doctor" || action === "query"
3584
3647
  ? null
3585
3648
  : ora(`Running proxy telemetry ${action}...`).start();
3586
3649
  try {
3587
3650
  if (spinner) {
3588
3651
  spinner.stop();
3589
3652
  }
3590
- await runProxyTelemetryManager(action);
3653
+ await runProxyTelemetryManager(action, argv);
3591
3654
  if (spinner) {
3592
3655
  spinner.succeed(`proxy telemetry ${action} completed`);
3593
3656
  }
@@ -18,6 +18,8 @@ const snapshot = {
18
18
  pendingBytes: 0,
19
19
  maxPending: MAX_PENDING,
20
20
  maxPendingBytes: MAX_PENDING_BYTES,
21
+ highWaterPending: 0,
22
+ highWaterBytes: 0,
21
23
  rejectionReasons: {},
22
24
  };
23
25
  const pending = new Map();
@@ -161,9 +163,27 @@ export async function captureProxyBody(entry, logDir, consume) {
161
163
  ? "body_worker_backoff"
162
164
  : "body_capture_queue_full";
163
165
  snapshot.lastError = error;
166
+ snapshot.lastRejectedAt = new Date().toISOString();
164
167
  snapshot.rejectionReasons[error] =
165
168
  (snapshot.rejectionReasons[error] ?? 0) + 1;
166
- return consume({ error, stored: { bodyWriteFailed: true } });
169
+ return consume({
170
+ error,
171
+ stored: { bodyWriteFailed: true },
172
+ admission: {
173
+ limitingResource: bytes > MAX_ENTRY_BYTES
174
+ ? "entry"
175
+ : Date.now() < retryAfter
176
+ ? "worker"
177
+ : snapshot.pending >= MAX_PENDING
178
+ ? "captures"
179
+ : "bytes",
180
+ estimatedBytes: Number.isFinite(bytes) ? bytes : undefined,
181
+ pending: snapshot.pending,
182
+ pendingBytes: snapshot.pendingBytes,
183
+ maxPending: MAX_PENDING,
184
+ maxPendingBytes: MAX_PENDING_BYTES,
185
+ },
186
+ });
167
187
  }
168
188
  let current;
169
189
  try {
@@ -207,7 +227,9 @@ export async function captureProxyBody(entry, logDir, consume) {
207
227
  },
208
228
  });
209
229
  snapshot.pending += 1;
230
+ snapshot.highWaterPending = Math.max(snapshot.highWaterPending, snapshot.pending);
210
231
  snapshot.pendingBytes += bytes;
232
+ snapshot.highWaterBytes = Math.max(snapshot.highWaterBytes, snapshot.pendingBytes);
211
233
  current.ref();
212
234
  try {
213
235
  current.postMessage({ id, entry, logDir, queuedAt: Date.now() });
@@ -248,6 +270,9 @@ export const __bodyCaptureWorkerTestHooks = {
248
270
  pending: 0,
249
271
  pendingBytes: 0,
250
272
  lastError: undefined,
273
+ lastRejectedAt: undefined,
274
+ highWaterPending: 0,
275
+ highWaterBytes: 0,
251
276
  rejectionReasons: {},
252
277
  });
253
278
  },
@@ -45,6 +45,34 @@ import { sanitizeForLog } from "../utils/logSanitize.js";
45
45
  const nonNegativeInt = (value) => typeof value === "number" && Number.isFinite(value) && value > 0
46
46
  ? Math.floor(value)
47
47
  : 0;
48
+ function usefulOutputItem(item) {
49
+ if (!item || typeof item !== "object") {
50
+ return false;
51
+ }
52
+ if ("type" in item) {
53
+ if (item.type === "function_call") {
54
+ return ("arguments" in item &&
55
+ typeof item.arguments === "string" &&
56
+ item.arguments.length > 0);
57
+ }
58
+ if (item.type === "custom_tool_call") {
59
+ return ("input" in item &&
60
+ typeof item.input === "string" &&
61
+ item.input.length > 0);
62
+ }
63
+ if (item.type === "output_text") {
64
+ return ("text" in item && typeof item.text === "string" && item.text.length > 0);
65
+ }
66
+ if (item.type === "refusal") {
67
+ return ("refusal" in item &&
68
+ typeof item.refusal === "string" &&
69
+ item.refusal.length > 0);
70
+ }
71
+ }
72
+ return ("content" in item &&
73
+ Array.isArray(item.content) &&
74
+ item.content.some(usefulOutputItem));
75
+ }
48
76
  /**
49
77
  * Pull usage out of one parsed SSE `data:` payload.
50
78
  *
@@ -125,7 +153,7 @@ const CAPTURE_LIMIT_BYTES = 256 * 1024;
125
153
  */
126
154
  function createCaptureSink() {
127
155
  const target = process.env.NEUROLINK_PROXY_CODEX_CAPTURE;
128
- if (!target) {
156
+ if (!target || process.env.NEUROLINK_PROXY_LOG_SINK === "otel") {
129
157
  return null;
130
158
  }
131
159
  let written = 0;
@@ -167,9 +195,13 @@ export function createCodexUsageTap() {
167
195
  let totalBytes = 0;
168
196
  const inspectEvidence = (events) => {
169
197
  for (const frame of events) {
198
+ if (frame.data.trim() === "[DONE]") {
199
+ continue;
200
+ }
170
201
  try {
171
202
  const event = JSON.parse(frame.data);
172
203
  if (!event || typeof event !== "object") {
204
+ evidence.observationIncomplete = true;
173
205
  continue;
174
206
  }
175
207
  const seen = extractCodexUsage(event);
@@ -178,10 +210,53 @@ export function createCodexUsageTap() {
178
210
  }
179
211
  const type = event.type ?? frame.event;
180
212
  if ((type === "response.output_text.delta" ||
213
+ type === "response.refusal.delta" ||
181
214
  type === "response.function_call_arguments.delta") &&
182
215
  typeof event.delta === "string" &&
183
216
  event.delta.length > 0) {
184
- evidence.firstUsefulOutputAt ??= Date.now();
217
+ if (evidence.firstUsefulOutputAt === undefined) {
218
+ evidence.firstUsefulOutputAt = Date.now();
219
+ evidence.firstUsefulOutputEvent = String(type);
220
+ }
221
+ }
222
+ // Some clients/providers emit complete output items without deltas.
223
+ // Reasoning/control events are not useful client output. Recognize
224
+ // actual tool calls and text only, without inventing an earlier time.
225
+ const usefulItem = usefulOutputItem(event.item);
226
+ const completedResponse = event.response;
227
+ const usefulCompletion = type === "response.completed" &&
228
+ completedResponse &&
229
+ typeof completedResponse === "object" &&
230
+ "output" in completedResponse &&
231
+ Array.isArray(completedResponse.output) &&
232
+ completedResponse.output.some(usefulOutputItem);
233
+ const usefulDone = (type === "response.output_text.done" &&
234
+ typeof event.text === "string" &&
235
+ event.text.length > 0) ||
236
+ (type === "response.refusal.done" &&
237
+ typeof event.refusal === "string" &&
238
+ event.refusal.length > 0) ||
239
+ (type === "response.function_call_arguments.done" &&
240
+ typeof event.arguments === "string" &&
241
+ event.arguments.length > 0) ||
242
+ (type === "response.custom_tool_call_input.done" &&
243
+ typeof event.input === "string" &&
244
+ event.input.length > 0) ||
245
+ ((type === "response.content_part.added" ||
246
+ type === "response.content_part.done") &&
247
+ usefulOutputItem(event.part));
248
+ const usefulToolDelta = type === "response.custom_tool_call_input.delta" &&
249
+ typeof event.delta === "string" &&
250
+ event.delta.length > 0;
251
+ if (evidence.firstUsefulOutputAt === undefined &&
252
+ (usefulDone ||
253
+ usefulCompletion ||
254
+ usefulToolDelta ||
255
+ ((type === "response.output_item.added" ||
256
+ type === "response.output_item.done") &&
257
+ usefulItem))) {
258
+ evidence.firstUsefulOutputAt = Date.now();
259
+ evidence.firstUsefulOutputEvent = String(type);
185
260
  }
186
261
  if (type === "response.completed") {
187
262
  evidence.completed = true;
@@ -222,6 +297,7 @@ export function createCodexUsageTap() {
222
297
  }
223
298
  catch {
224
299
  // Unknown frames cannot establish successful completion.
300
+ evidence.observationIncomplete = true;
225
301
  }
226
302
  }
227
303
  };
@@ -268,16 +344,21 @@ export function createCodexUsageTap() {
268
344
  carry = remainder;
269
345
  inspectEvidence(events);
270
346
  if (carry.length > CARRY_LIMIT_CHARS) {
347
+ evidence.observationIncomplete = true;
271
348
  carry = carry.slice(-2);
272
349
  discardingEvent = true;
273
350
  }
274
351
  }
275
352
  catch {
276
353
  // Telemetry must never break the relay.
354
+ evidence.observationIncomplete = true;
277
355
  }
278
356
  },
279
357
  flush() {
280
358
  // An event without its dispatch delimiter is incomplete on the wire.
359
+ if (discardingEvent || carry.trim()) {
360
+ evidence.observationIncomplete = true;
361
+ }
281
362
  settle(latest);
282
363
  },
283
364
  /**
@@ -34,6 +34,18 @@ export declare function getProxyOtelLogSnapshot(): {
34
34
  maxPendingBytes: number;
35
35
  };
36
36
  queues: {
37
+ recentFailures: {
38
+ records: {
39
+ eventId: string;
40
+ kind?: string;
41
+ requestId?: string;
42
+ captureId?: string;
43
+ }[];
44
+ id: string;
45
+ at: string;
46
+ reason: "export_unconfirmed" | "queue_full";
47
+ error?: string;
48
+ }[];
37
49
  attempted: number;
38
50
  submitted: number;
39
51
  transportAcknowledged: number;
@@ -43,6 +55,7 @@ export declare function getProxyOtelLogSnapshot(): {
43
55
  lastAcknowledgedAt: string | undefined;
44
56
  lastFailureAt: string | undefined;
45
57
  highWaterOutstanding: number;
58
+ failureHistoryEvicted: number;
46
59
  kind: "metadata" | "bodies";
47
60
  capacity: number;
48
61
  }[];
@@ -1,5 +1,7 @@
1
1
  /* eslint-disable no-console -- This proxy-only sink replaces console methods with OTLP emission. */
2
2
  import { inspect } from "node:util";
3
+ import { randomUUID } from "node:crypto";
4
+ import { getProxyRequestTraceContext, proxyLogContext, } from "./proxyTraceContext.js";
3
5
  import { setImmediate as yieldToRequests } from "node:timers/promises";
4
6
  import { SeverityNumber } from "@opentelemetry/api-logs";
5
7
  import { ExportResultCode } from "@opentelemetry/core";
@@ -45,6 +47,52 @@ function createTrackedProcessor(url, capacity, kind) {
45
47
  lastAcknowledgedAt: undefined,
46
48
  lastFailureAt: undefined,
47
49
  highWaterOutstanding: 0,
50
+ recentFailures: [],
51
+ failureHistoryEvicted: 0,
52
+ };
53
+ const diagnostics = [];
54
+ let diagnosticScheduled = false;
55
+ const rememberFailure = (records, reason, error) => {
56
+ const stringAttribute = (record, name) => {
57
+ const value = record.attributes[name];
58
+ return typeof value === "string" ? value.slice(0, 128) : undefined;
59
+ };
60
+ const failure = {
61
+ id: randomUUID(),
62
+ at: new Date().toISOString(),
63
+ reason,
64
+ ...(error ? { error: sanitizeForLog(error.message).slice(0, 256) } : {}),
65
+ records: records.slice(0, 64).map((record) => ({
66
+ eventId: stringAttribute(record, "proxy.event_id") ?? "unavailable",
67
+ kind: stringAttribute(record, "proxy.record_kind"),
68
+ requestId: stringAttribute(record, "request.id"),
69
+ captureId: stringAttribute(record, "body.capture_id"),
70
+ })),
71
+ };
72
+ if (state.recentFailures.length === 16) {
73
+ state.recentFailures.shift();
74
+ state.failureHistoryEvicted++;
75
+ }
76
+ state.recentFailures.push(failure);
77
+ // A failed diagnostic must not generate another diagnostic recursively.
78
+ if (records.some((record) => record.attributes["proxy.record_kind"] !== "telemetry_delivery")) {
79
+ if (diagnostics.length === 16) {
80
+ diagnostics.shift();
81
+ }
82
+ diagnostics.push(failure);
83
+ }
84
+ };
85
+ const publishRecoveredDiagnostics = () => {
86
+ if (diagnosticScheduled || !diagnostics.length || shuttingDown) {
87
+ return;
88
+ }
89
+ diagnosticScheduled = true;
90
+ queueMicrotask(() => {
91
+ diagnosticScheduled = false;
92
+ for (const failure of diagnostics.splice(0)) {
93
+ emitProxyOtelEvent("telemetry_delivery", { queue: kind, ...failure });
94
+ }
95
+ });
48
96
  };
49
97
  const transport = new OTLPLogExporter({ url, timeoutMillis: 5000 });
50
98
  const exporter = {
@@ -65,10 +113,12 @@ function createTrackedProcessor(url, capacity, kind) {
65
113
  if (result.code === ExportResultCode.SUCCESS) {
66
114
  state.transportAcknowledged += records.length;
67
115
  state.lastAcknowledgedAt = new Date().toISOString();
116
+ publishRecoveredDiagnostics();
68
117
  }
69
118
  else {
70
119
  state.exportUnconfirmed += records.length;
71
120
  state.lastFailureAt = new Date().toISOString();
121
+ rememberFailure(records, "export_unconfirmed", result.error);
72
122
  }
73
123
  for (const record of records) {
74
124
  unsettled.delete(record);
@@ -109,6 +159,7 @@ function createTrackedProcessor(url, capacity, kind) {
109
159
  });
110
160
  const processor = {
111
161
  onEmit(record) {
162
+ record.attributes["proxy.event_id"] ??= randomUUID();
112
163
  state.attempted++;
113
164
  const id = record.attributes?.["body.capture_id"];
114
165
  const publication = typeof id === "string" ? bodyPublications.get(id) : undefined;
@@ -117,6 +168,7 @@ function createTrackedProcessor(url, capacity, kind) {
117
168
  }
118
169
  if (state.outstanding >= capacity) {
119
170
  state.dropped++;
171
+ rememberFailure([record], "queue_full");
120
172
  if (publication) {
121
173
  publication.dropped++;
122
174
  publication.notify?.();
@@ -355,12 +407,17 @@ export function emitProxyOtelEvent(kind, record) {
355
407
  return;
356
408
  }
357
409
  try {
410
+ const ids = typeof record.requestId === "string" && !record.traceId
411
+ ? getProxyRequestTraceContext(record.requestId)
412
+ : undefined;
413
+ const correlated = ids ? { ...record, ...ids } : record;
358
414
  initializeProxyOtelLogs()
359
415
  ?.getLogger("neurolink-proxy-events")
360
416
  .emit({
417
+ context: proxyLogContext(correlated),
361
418
  severityNumber: SeverityNumber.INFO,
362
419
  severityText: "INFO",
363
- body: JSON.stringify(record),
420
+ body: JSON.stringify(correlated),
364
421
  attributes: {
365
422
  "proxy.record_kind": kind,
366
423
  "event.name": `proxy.${kind}`,
@@ -444,6 +501,10 @@ export function getProxyOtelLogSnapshot() {
444
501
  kind: q.kind,
445
502
  capacity: q.capacity,
446
503
  ...q.state,
504
+ recentFailures: q.state.recentFailures.map((failure) => ({
505
+ ...failure,
506
+ records: failure.records.map((record) => ({ ...record })),
507
+ })),
447
508
  })),
448
509
  };
449
510
  }
@@ -0,0 +1,21 @@
1
+ import type { ProxyLogTraceContext } from "../types/index.js";
2
+ /** Correlation belongs to the in-flight HTTP request, not to an async callback. */
3
+ export declare function registerProxyRequestTraceContext(requestId: string, ids: ProxyLogTraceContext): void;
4
+ /** Release once transport and terminal accounting have settled. */
5
+ export declare function releaseProxyRequestTraceContext(requestId: string): void;
6
+ /** Internal fallback records share their parent request's trace. */
7
+ export declare function getProxyRequestTraceContext(requestId: string): ProxyLogTraceContext | undefined;
8
+ /** Retain IDs and sampling flags before deferred processing leaves the request. */
9
+ export declare function resolveProxyLogTraceContext(record: {
10
+ traceId?: unknown;
11
+ spanId?: unknown;
12
+ traceFlags?: unknown;
13
+ requestId?: unknown;
14
+ }): ProxyLogTraceContext | undefined;
15
+ /** Populate native OTLP correlation, including valid unsampled contexts. */
16
+ export declare function proxyLogContext(record: {
17
+ traceId?: unknown;
18
+ spanId?: unknown;
19
+ traceFlags?: unknown;
20
+ requestId?: unknown;
21
+ }): import("@opentelemetry/api").Context;
@@ -0,0 +1,47 @@
1
+ import { context, ROOT_CONTEXT, trace, isSpanContextValid, } from "@opentelemetry/api";
2
+ const requests = new Map();
3
+ /** Correlation belongs to the in-flight HTTP request, not to an async callback. */
4
+ export function registerProxyRequestTraceContext(requestId, ids) {
5
+ requests.set(requestId, ids);
6
+ }
7
+ /** Release once transport and terminal accounting have settled. */
8
+ export function releaseProxyRequestTraceContext(requestId) {
9
+ requests.delete(requestId);
10
+ }
11
+ /** Internal fallback records share their parent request's trace. */
12
+ export function getProxyRequestTraceContext(requestId) {
13
+ return (requests.get(requestId) ??
14
+ requests.get(requestId.replace(/:codex-fallback$/, "")));
15
+ }
16
+ /** Retain IDs and sampling flags before deferred processing leaves the request. */
17
+ export function resolveProxyLogTraceContext(record) {
18
+ const saved = typeof record.requestId === "string"
19
+ ? getProxyRequestTraceContext(record.requestId)
20
+ : undefined;
21
+ const active = trace.getSpanContext(context.active());
22
+ const ids = typeof record.traceId === "string" && typeof record.spanId === "string"
23
+ ? { traceId: record.traceId, spanId: record.spanId }
24
+ : (saved ?? active);
25
+ if (!ids) {
26
+ return undefined;
27
+ }
28
+ const traceFlags = typeof record.traceFlags === "number" &&
29
+ Number.isInteger(record.traceFlags) &&
30
+ record.traceFlags >= 0 &&
31
+ record.traceFlags <= 255
32
+ ? record.traceFlags
33
+ : saved?.traceId === ids.traceId
34
+ ? saved.traceFlags
35
+ : active?.traceId === ids.traceId
36
+ ? active.traceFlags
37
+ : 0;
38
+ const result = { ...ids, traceFlags };
39
+ return isSpanContextValid(result)
40
+ ? { traceId: result.traceId, spanId: result.spanId, traceFlags }
41
+ : undefined;
42
+ }
43
+ /** Populate native OTLP correlation, including valid unsampled contexts. */
44
+ export function proxyLogContext(record) {
45
+ const ids = resolveProxyLogTraceContext(record);
46
+ return ids ? trace.setSpanContext(ROOT_CONTEXT, ids) : context.active();
47
+ }
@@ -14,7 +14,7 @@
14
14
  * - TelemetryService for metrics recording
15
15
  */
16
16
  import { type Span } from "@opentelemetry/api";
17
- import type { AccountSelectionContext, ProxyRequestContext, ResponseInfoContext, UpstreamAttemptContext, UsageContext } from "../types/index.js";
17
+ import type { AccountSelectionContext, ProxyRequestContext, ResponseInfoContext, UpstreamAttemptContext, UsageContext, RuntimeRequestMetadata, ProxyLogTraceContext } from "../types/index.js";
18
18
  declare class ProxyTracer {
19
19
  private readonly rootSpan;
20
20
  private readonly proxyTracer;
@@ -34,6 +34,8 @@ declare class ProxyTracer {
34
34
  private billingProvider;
35
35
  private readonly startTime;
36
36
  private readonly isStream;
37
+ private ended;
38
+ private recordRequestMetrics;
37
39
  private accountEmail?;
38
40
  private usage?;
39
41
  private mode;
@@ -102,10 +104,7 @@ declare class ProxyTracer {
102
104
  /** Record request and/or response body sizes for bandwidth tracking. */
103
105
  recordBodySizes(requestBytes?: number, responseBytes?: number): void;
104
106
  /** Return the OTel trace/span IDs for this request (for log correlation). */
105
- getTraceContext(): {
106
- traceId: string;
107
- spanId: string;
108
- };
107
+ getTraceContext(): ProxyLogTraceContext;
109
108
  /** Return the captured usage (set by setUsage). */
110
109
  getUsage(): UsageContext | undefined;
111
110
  /** End the root span with final HTTP status and duration, and emit OTEL metrics. */
@@ -126,3 +125,8 @@ export declare function recordFallbackAttempt(attrs: {
126
125
  durationMs: number;
127
126
  }): void;
128
127
  export { ProxyTracer };
128
+ /** Standard SERVER span covers every proxy door, including parsing and admission failures. */
129
+ export declare function startProxyHttpTrace(metadata: RuntimeRequestMetadata, headers: Record<string, string>): {
130
+ run: <T>(fn: () => T) => T;
131
+ end: (status: number, outcome: string, errorType?: string) => void;
132
+ };