@juspay/neurolink 12.14.6 → 12.14.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +2 -2
- package/dist/browser/neurolink.min.js +412 -412
- package/dist/cli/commands/proxy.js +76 -13
- package/dist/proxy/bodyCaptureWorker.js +26 -1
- package/dist/proxy/codexUsage.js +83 -2
- package/dist/proxy/otelLogSink.d.ts +13 -0
- package/dist/proxy/otelLogSink.js +62 -1
- package/dist/proxy/proxyTraceContext.d.ts +21 -0
- package/dist/proxy/proxyTraceContext.js +47 -0
- package/dist/proxy/proxyTracer.d.ts +9 -5
- package/dist/proxy/proxyTracer.js +81 -2
- package/dist/proxy/requestLogger.js +24 -18
- package/dist/server/routes/codexProxyRoutes.js +414 -332
- package/dist/types/cli.d.ts +7 -1
- package/dist/types/proxy.d.ts +119 -3
- package/docs-site/static/search-index.json +1 -1
- package/package.json +1 -1
- package/scripts/observability/check-proxy-telemetry.mjs +29 -241
- package/scripts/observability/proxy-telemetry-backend.mjs +202 -0
- package/scripts/observability/proxy-telemetry-check.mjs +544 -0
- package/scripts/observability/query-proxy-history.mjs +30 -19
|
@@ -12,6 +12,7 @@ import { applyAllClients, restoreAllClients, } from "../proxy-clients/registry.j
|
|
|
12
12
|
import { resolveProxyConfigPath } from "../../proxy/proxyConfig.js";
|
|
13
13
|
import { redactUrlsInText, sanitizeForLog, } from "../../utils/logSanitize.js";
|
|
14
14
|
import { withTimeout } from "../../utils/async/withTimeout.js";
|
|
15
|
+
import { startProxyHttpTrace } from "../../proxy/proxyTracer.js";
|
|
15
16
|
import { formatUptime, isProcessRunning, StateFileManager, } from "../utils/serverUtils.js";
|
|
16
17
|
import { configureProxyKeepAliveDispatcher } from "../../proxy/proxyDispatcher.js";
|
|
17
18
|
import { ProxyRuntimeConfigStore } from "../../proxy/runtimeConfig.js";
|
|
@@ -837,13 +838,36 @@ function spawnProxyUpdater(host, port, parentPid, rollingSupervisor = false) {
|
|
|
837
838
|
workerLog.close();
|
|
838
839
|
}
|
|
839
840
|
}
|
|
840
|
-
async function runProxyTelemetryManager(command) {
|
|
841
|
+
async function runProxyTelemetryManager(command, argv) {
|
|
841
842
|
const { existsSync } = await import("fs");
|
|
842
843
|
if (!existsSync(PROXY_TELEMETRY_SCRIPT_PATH)) {
|
|
843
844
|
throw new Error("Proxy telemetry helper files were not found in this installation. Reinstall NeuroLink with observability assets included.");
|
|
844
845
|
}
|
|
845
846
|
await new Promise((resolve, reject) => {
|
|
846
|
-
const
|
|
847
|
+
const queryCommand = command === "doctor" || command === "query";
|
|
848
|
+
const queryArgs = [];
|
|
849
|
+
if (queryCommand && argv) {
|
|
850
|
+
for (const [key, option] of [
|
|
851
|
+
["since", "--since"],
|
|
852
|
+
["until", "--until"],
|
|
853
|
+
["kind", "--kind"],
|
|
854
|
+
["format", "--format"],
|
|
855
|
+
["maxRows", "--max-rows"],
|
|
856
|
+
["proxyUrl", "--proxy-url"],
|
|
857
|
+
]) {
|
|
858
|
+
if (argv[key] !== undefined) {
|
|
859
|
+
queryArgs.push(option, String(argv[key]));
|
|
860
|
+
}
|
|
861
|
+
}
|
|
862
|
+
}
|
|
863
|
+
const child = spawn(queryCommand ? process.execPath : "bash", queryCommand
|
|
864
|
+
? [
|
|
865
|
+
join(dirname(PROXY_TELEMETRY_SCRIPT_PATH), command === "doctor"
|
|
866
|
+
? "check-proxy-telemetry.mjs"
|
|
867
|
+
: "query-proxy-history.mjs"),
|
|
868
|
+
...queryArgs,
|
|
869
|
+
]
|
|
870
|
+
: [PROXY_TELEMETRY_SCRIPT_PATH, command], {
|
|
847
871
|
stdio: "inherit",
|
|
848
872
|
env: process.env,
|
|
849
873
|
});
|
|
@@ -1078,6 +1102,7 @@ function registerProxyRequestTracking(app, requestMetadata, readiness) {
|
|
|
1078
1102
|
rejectForUpdate: readiness.drainingForUpdate,
|
|
1079
1103
|
};
|
|
1080
1104
|
requestMetadata.set(c.req.raw, metadata);
|
|
1105
|
+
const httpTrace = startProxyHttpTrace(metadata, Object.fromEntries(c.req.raw.headers));
|
|
1081
1106
|
const stopObservingFinalLog = observeProxyFinalLog(metadata.requestId, (entry) => {
|
|
1082
1107
|
metadata.terminalResult = entry;
|
|
1083
1108
|
}, (entry) => {
|
|
@@ -1087,6 +1112,7 @@ function registerProxyRequestTracking(app, requestMetadata, readiness) {
|
|
|
1087
1112
|
? () => undefined
|
|
1088
1113
|
: beginProxyRequest();
|
|
1089
1114
|
const finish = () => {
|
|
1115
|
+
httpTrace.end(metadata.terminalResult?.responseStatus ?? c.res.status, metadata.terminalResult?.terminalOutcome ?? "unknown", metadata.terminalErrorType);
|
|
1090
1116
|
stopObservingFinalLog();
|
|
1091
1117
|
finishActivity();
|
|
1092
1118
|
// Borrowed traffic holds a concurrency slot for the lifetime of the
|
|
@@ -1099,7 +1125,7 @@ function registerProxyRequestTracking(app, requestMetadata, readiness) {
|
|
|
1099
1125
|
// them at acceptance instead of publishing misleading placeholder values;
|
|
1100
1126
|
// subsequent events carry the parsed metadata under the same request ID.
|
|
1101
1127
|
try {
|
|
1102
|
-
await persistProxyLifecycleAcceptance({
|
|
1128
|
+
await httpTrace.run(() => persistProxyLifecycleAcceptance({
|
|
1103
1129
|
requestId: metadata.requestId,
|
|
1104
1130
|
method: metadata.method,
|
|
1105
1131
|
path: metadata.path,
|
|
@@ -1107,8 +1133,8 @@ function registerProxyRequestTracking(app, requestMetadata, readiness) {
|
|
|
1107
1133
|
requestBytes,
|
|
1108
1134
|
elapsedMs: 0,
|
|
1109
1135
|
monotonicMs: startedMonotonicMs,
|
|
1110
|
-
});
|
|
1111
|
-
await next
|
|
1136
|
+
}));
|
|
1137
|
+
await httpTrace.run(next);
|
|
1112
1138
|
const responseStatus = c.res.status;
|
|
1113
1139
|
logProxyLifecycleEvent({
|
|
1114
1140
|
event: "response_headers",
|
|
@@ -1169,12 +1195,12 @@ function registerProxyRequestTracking(app, requestMetadata, readiness) {
|
|
|
1169
1195
|
let accountingTimedOut = false;
|
|
1170
1196
|
let accountingFailed = false;
|
|
1171
1197
|
try {
|
|
1172
|
-
accountingFailed = await notifyRouteTerminal({
|
|
1198
|
+
accountingFailed = await httpTrace.run(() => notifyRouteTerminal({
|
|
1173
1199
|
outcome,
|
|
1174
1200
|
error,
|
|
1175
1201
|
observedBodyBytes,
|
|
1176
1202
|
responseChunks,
|
|
1177
|
-
});
|
|
1203
|
+
}));
|
|
1178
1204
|
}
|
|
1179
1205
|
catch {
|
|
1180
1206
|
accountingTimedOut = true;
|
|
@@ -1231,6 +1257,7 @@ function registerProxyRequestTracking(app, requestMetadata, readiness) {
|
|
|
1231
1257
|
errorCode: final?.errorCode ?? metadata.terminalErrorCode,
|
|
1232
1258
|
});
|
|
1233
1259
|
stopObservingFinalLog();
|
|
1260
|
+
httpTrace.end(final?.responseStatus ?? responseStatus, terminalOutcome, final?.errorType ?? metadata.terminalErrorType);
|
|
1234
1261
|
},
|
|
1235
1262
|
});
|
|
1236
1263
|
}
|
|
@@ -1259,6 +1286,9 @@ function registerProxyRequestTracking(app, requestMetadata, readiness) {
|
|
|
1259
1286
|
errorType: error instanceof Error ? error.name : "unknown_error",
|
|
1260
1287
|
errorCode: getProxyRuntimeErrorCode(error),
|
|
1261
1288
|
});
|
|
1289
|
+
httpTrace.end(getProxyRuntimeErrorCode(error) === "PROXY_TELEMETRY_UNAVAILABLE"
|
|
1290
|
+
? 503
|
|
1291
|
+
: 502, "handler_error", error instanceof Error ? error.name : "unknown_error");
|
|
1262
1292
|
throw error;
|
|
1263
1293
|
}
|
|
1264
1294
|
};
|
|
@@ -1331,8 +1361,9 @@ export async function createProxyStartApp(params) {
|
|
|
1331
1361
|
accountKey: attempt?.accountKey,
|
|
1332
1362
|
provider: attempt?.provider,
|
|
1333
1363
|
transportScope: attempt?.transportScope,
|
|
1334
|
-
traceId: attempt?.traceId,
|
|
1335
|
-
spanId: attempt?.spanId,
|
|
1364
|
+
traceId: attempt?.traceId ?? metadata.traceId,
|
|
1365
|
+
spanId: attempt?.spanId ?? metadata.spanId,
|
|
1366
|
+
traceFlags: attempt?.traceFlags ?? metadata.traceFlags,
|
|
1336
1367
|
responseStatus: status,
|
|
1337
1368
|
responseTimeMs: Date.now() - metadata.startedAt,
|
|
1338
1369
|
errorType,
|
|
@@ -1342,6 +1373,9 @@ export async function createProxyStartApp(params) {
|
|
|
1342
1373
|
logBodyCapture({
|
|
1343
1374
|
timestamp: new Date().toISOString(),
|
|
1344
1375
|
requestId: metadata.requestId,
|
|
1376
|
+
traceId: attempt?.traceId ?? metadata.traceId,
|
|
1377
|
+
spanId: attempt?.spanId ?? metadata.spanId,
|
|
1378
|
+
traceFlags: attempt?.traceFlags ?? metadata.traceFlags,
|
|
1345
1379
|
model: metadata.model,
|
|
1346
1380
|
stream: metadata.stream,
|
|
1347
1381
|
phase: "client_response",
|
|
@@ -3553,6 +3587,8 @@ export const proxyStatusCommand = {
|
|
|
3553
3587
|
// PROXY TELEMETRY COMMAND
|
|
3554
3588
|
// =============================================================================
|
|
3555
3589
|
const PROXY_TELEMETRY_ACTIONS = [
|
|
3590
|
+
"doctor",
|
|
3591
|
+
"query",
|
|
3556
3592
|
"setup",
|
|
3557
3593
|
"start",
|
|
3558
3594
|
"stop",
|
|
@@ -3562,12 +3598,37 @@ const PROXY_TELEMETRY_ACTIONS = [
|
|
|
3562
3598
|
];
|
|
3563
3599
|
export const proxyTelemetryCommand = {
|
|
3564
3600
|
command: "telemetry <action>",
|
|
3565
|
-
describe: "
|
|
3601
|
+
describe: "Query and verify stored OTel telemetry or manage the local observability stack",
|
|
3566
3602
|
builder: (yargs) => yargs
|
|
3567
3603
|
.positional("action", {
|
|
3568
3604
|
type: "string",
|
|
3569
3605
|
choices: [...PROXY_TELEMETRY_ACTIONS],
|
|
3570
|
-
describe: "Telemetry action: setup, start, stop, status, logs, or import-dashboard",
|
|
3606
|
+
describe: "Telemetry action: doctor, query, setup, start, stop, status, logs, or import-dashboard",
|
|
3607
|
+
})
|
|
3608
|
+
.option("since", {
|
|
3609
|
+
type: "string",
|
|
3610
|
+
description: "Inclusive ISO timestamp for stored telemetry",
|
|
3611
|
+
})
|
|
3612
|
+
.option("until", {
|
|
3613
|
+
type: "string",
|
|
3614
|
+
description: "Exclusive ISO timestamp for stored telemetry",
|
|
3615
|
+
})
|
|
3616
|
+
.option("kind", {
|
|
3617
|
+
type: "string",
|
|
3618
|
+
description: "Metadata kind for query, such as request_final or telemetry_delivery",
|
|
3619
|
+
})
|
|
3620
|
+
.option("format", {
|
|
3621
|
+
type: "string",
|
|
3622
|
+
choices: ["json", "text"],
|
|
3623
|
+
description: "Doctor report format; query always returns JSON",
|
|
3624
|
+
})
|
|
3625
|
+
.option("max-rows", {
|
|
3626
|
+
type: "number",
|
|
3627
|
+
description: "Explicit bounded history record limit",
|
|
3628
|
+
})
|
|
3629
|
+
.option("proxy-url", {
|
|
3630
|
+
type: "string",
|
|
3631
|
+
description: "Proxy endpoint used by the read-only doctor",
|
|
3571
3632
|
})
|
|
3572
3633
|
.option("quiet", {
|
|
3573
3634
|
type: "boolean",
|
|
@@ -3575,19 +3636,21 @@ export const proxyTelemetryCommand = {
|
|
|
3575
3636
|
default: false,
|
|
3576
3637
|
description: "Suppress the local CLI spinner and delegate directly",
|
|
3577
3638
|
})
|
|
3639
|
+
.example("neurolink proxy telemetry doctor --format json", "Verify field coverage, stored captures, trace correlation, and delivery evidence")
|
|
3640
|
+
.example("neurolink proxy telemetry query --since 2026-09-15T00:00:00Z --kind request_final", "Read stored OTLP metadata without scanning proxy log files")
|
|
3578
3641
|
.example("neurolink proxy telemetry setup", "Start OpenObserve, start the OTEL collector, and import the dashboard")
|
|
3579
3642
|
.example("neurolink proxy telemetry start", "Start the local proxy telemetry stack without re-importing the dashboard")
|
|
3580
3643
|
.example("neurolink proxy telemetry stop", "Stop the local OpenObserve and OTEL collector containers"),
|
|
3581
3644
|
handler: async (argv) => {
|
|
3582
3645
|
const action = argv.action;
|
|
3583
|
-
const spinner = argv.quiet
|
|
3646
|
+
const spinner = argv.quiet || action === "doctor" || action === "query"
|
|
3584
3647
|
? null
|
|
3585
3648
|
: ora(`Running proxy telemetry ${action}...`).start();
|
|
3586
3649
|
try {
|
|
3587
3650
|
if (spinner) {
|
|
3588
3651
|
spinner.stop();
|
|
3589
3652
|
}
|
|
3590
|
-
await runProxyTelemetryManager(action);
|
|
3653
|
+
await runProxyTelemetryManager(action, argv);
|
|
3591
3654
|
if (spinner) {
|
|
3592
3655
|
spinner.succeed(`proxy telemetry ${action} completed`);
|
|
3593
3656
|
}
|
|
@@ -18,6 +18,8 @@ const snapshot = {
|
|
|
18
18
|
pendingBytes: 0,
|
|
19
19
|
maxPending: MAX_PENDING,
|
|
20
20
|
maxPendingBytes: MAX_PENDING_BYTES,
|
|
21
|
+
highWaterPending: 0,
|
|
22
|
+
highWaterBytes: 0,
|
|
21
23
|
rejectionReasons: {},
|
|
22
24
|
};
|
|
23
25
|
const pending = new Map();
|
|
@@ -161,9 +163,27 @@ export async function captureProxyBody(entry, logDir, consume) {
|
|
|
161
163
|
? "body_worker_backoff"
|
|
162
164
|
: "body_capture_queue_full";
|
|
163
165
|
snapshot.lastError = error;
|
|
166
|
+
snapshot.lastRejectedAt = new Date().toISOString();
|
|
164
167
|
snapshot.rejectionReasons[error] =
|
|
165
168
|
(snapshot.rejectionReasons[error] ?? 0) + 1;
|
|
166
|
-
return consume({
|
|
169
|
+
return consume({
|
|
170
|
+
error,
|
|
171
|
+
stored: { bodyWriteFailed: true },
|
|
172
|
+
admission: {
|
|
173
|
+
limitingResource: bytes > MAX_ENTRY_BYTES
|
|
174
|
+
? "entry"
|
|
175
|
+
: Date.now() < retryAfter
|
|
176
|
+
? "worker"
|
|
177
|
+
: snapshot.pending >= MAX_PENDING
|
|
178
|
+
? "captures"
|
|
179
|
+
: "bytes",
|
|
180
|
+
estimatedBytes: Number.isFinite(bytes) ? bytes : undefined,
|
|
181
|
+
pending: snapshot.pending,
|
|
182
|
+
pendingBytes: snapshot.pendingBytes,
|
|
183
|
+
maxPending: MAX_PENDING,
|
|
184
|
+
maxPendingBytes: MAX_PENDING_BYTES,
|
|
185
|
+
},
|
|
186
|
+
});
|
|
167
187
|
}
|
|
168
188
|
let current;
|
|
169
189
|
try {
|
|
@@ -207,7 +227,9 @@ export async function captureProxyBody(entry, logDir, consume) {
|
|
|
207
227
|
},
|
|
208
228
|
});
|
|
209
229
|
snapshot.pending += 1;
|
|
230
|
+
snapshot.highWaterPending = Math.max(snapshot.highWaterPending, snapshot.pending);
|
|
210
231
|
snapshot.pendingBytes += bytes;
|
|
232
|
+
snapshot.highWaterBytes = Math.max(snapshot.highWaterBytes, snapshot.pendingBytes);
|
|
211
233
|
current.ref();
|
|
212
234
|
try {
|
|
213
235
|
current.postMessage({ id, entry, logDir, queuedAt: Date.now() });
|
|
@@ -248,6 +270,9 @@ export const __bodyCaptureWorkerTestHooks = {
|
|
|
248
270
|
pending: 0,
|
|
249
271
|
pendingBytes: 0,
|
|
250
272
|
lastError: undefined,
|
|
273
|
+
lastRejectedAt: undefined,
|
|
274
|
+
highWaterPending: 0,
|
|
275
|
+
highWaterBytes: 0,
|
|
251
276
|
rejectionReasons: {},
|
|
252
277
|
});
|
|
253
278
|
},
|
package/dist/proxy/codexUsage.js
CHANGED
|
@@ -45,6 +45,34 @@ import { sanitizeForLog } from "../utils/logSanitize.js";
|
|
|
45
45
|
const nonNegativeInt = (value) => typeof value === "number" && Number.isFinite(value) && value > 0
|
|
46
46
|
? Math.floor(value)
|
|
47
47
|
: 0;
|
|
48
|
+
function usefulOutputItem(item) {
|
|
49
|
+
if (!item || typeof item !== "object") {
|
|
50
|
+
return false;
|
|
51
|
+
}
|
|
52
|
+
if ("type" in item) {
|
|
53
|
+
if (item.type === "function_call") {
|
|
54
|
+
return ("arguments" in item &&
|
|
55
|
+
typeof item.arguments === "string" &&
|
|
56
|
+
item.arguments.length > 0);
|
|
57
|
+
}
|
|
58
|
+
if (item.type === "custom_tool_call") {
|
|
59
|
+
return ("input" in item &&
|
|
60
|
+
typeof item.input === "string" &&
|
|
61
|
+
item.input.length > 0);
|
|
62
|
+
}
|
|
63
|
+
if (item.type === "output_text") {
|
|
64
|
+
return ("text" in item && typeof item.text === "string" && item.text.length > 0);
|
|
65
|
+
}
|
|
66
|
+
if (item.type === "refusal") {
|
|
67
|
+
return ("refusal" in item &&
|
|
68
|
+
typeof item.refusal === "string" &&
|
|
69
|
+
item.refusal.length > 0);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
return ("content" in item &&
|
|
73
|
+
Array.isArray(item.content) &&
|
|
74
|
+
item.content.some(usefulOutputItem));
|
|
75
|
+
}
|
|
48
76
|
/**
|
|
49
77
|
* Pull usage out of one parsed SSE `data:` payload.
|
|
50
78
|
*
|
|
@@ -125,7 +153,7 @@ const CAPTURE_LIMIT_BYTES = 256 * 1024;
|
|
|
125
153
|
*/
|
|
126
154
|
function createCaptureSink() {
|
|
127
155
|
const target = process.env.NEUROLINK_PROXY_CODEX_CAPTURE;
|
|
128
|
-
if (!target) {
|
|
156
|
+
if (!target || process.env.NEUROLINK_PROXY_LOG_SINK === "otel") {
|
|
129
157
|
return null;
|
|
130
158
|
}
|
|
131
159
|
let written = 0;
|
|
@@ -167,9 +195,13 @@ export function createCodexUsageTap() {
|
|
|
167
195
|
let totalBytes = 0;
|
|
168
196
|
const inspectEvidence = (events) => {
|
|
169
197
|
for (const frame of events) {
|
|
198
|
+
if (frame.data.trim() === "[DONE]") {
|
|
199
|
+
continue;
|
|
200
|
+
}
|
|
170
201
|
try {
|
|
171
202
|
const event = JSON.parse(frame.data);
|
|
172
203
|
if (!event || typeof event !== "object") {
|
|
204
|
+
evidence.observationIncomplete = true;
|
|
173
205
|
continue;
|
|
174
206
|
}
|
|
175
207
|
const seen = extractCodexUsage(event);
|
|
@@ -178,10 +210,53 @@ export function createCodexUsageTap() {
|
|
|
178
210
|
}
|
|
179
211
|
const type = event.type ?? frame.event;
|
|
180
212
|
if ((type === "response.output_text.delta" ||
|
|
213
|
+
type === "response.refusal.delta" ||
|
|
181
214
|
type === "response.function_call_arguments.delta") &&
|
|
182
215
|
typeof event.delta === "string" &&
|
|
183
216
|
event.delta.length > 0) {
|
|
184
|
-
evidence.firstUsefulOutputAt
|
|
217
|
+
if (evidence.firstUsefulOutputAt === undefined) {
|
|
218
|
+
evidence.firstUsefulOutputAt = Date.now();
|
|
219
|
+
evidence.firstUsefulOutputEvent = String(type);
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
// Some clients/providers emit complete output items without deltas.
|
|
223
|
+
// Reasoning/control events are not useful client output. Recognize
|
|
224
|
+
// actual tool calls and text only, without inventing an earlier time.
|
|
225
|
+
const usefulItem = usefulOutputItem(event.item);
|
|
226
|
+
const completedResponse = event.response;
|
|
227
|
+
const usefulCompletion = type === "response.completed" &&
|
|
228
|
+
completedResponse &&
|
|
229
|
+
typeof completedResponse === "object" &&
|
|
230
|
+
"output" in completedResponse &&
|
|
231
|
+
Array.isArray(completedResponse.output) &&
|
|
232
|
+
completedResponse.output.some(usefulOutputItem);
|
|
233
|
+
const usefulDone = (type === "response.output_text.done" &&
|
|
234
|
+
typeof event.text === "string" &&
|
|
235
|
+
event.text.length > 0) ||
|
|
236
|
+
(type === "response.refusal.done" &&
|
|
237
|
+
typeof event.refusal === "string" &&
|
|
238
|
+
event.refusal.length > 0) ||
|
|
239
|
+
(type === "response.function_call_arguments.done" &&
|
|
240
|
+
typeof event.arguments === "string" &&
|
|
241
|
+
event.arguments.length > 0) ||
|
|
242
|
+
(type === "response.custom_tool_call_input.done" &&
|
|
243
|
+
typeof event.input === "string" &&
|
|
244
|
+
event.input.length > 0) ||
|
|
245
|
+
((type === "response.content_part.added" ||
|
|
246
|
+
type === "response.content_part.done") &&
|
|
247
|
+
usefulOutputItem(event.part));
|
|
248
|
+
const usefulToolDelta = type === "response.custom_tool_call_input.delta" &&
|
|
249
|
+
typeof event.delta === "string" &&
|
|
250
|
+
event.delta.length > 0;
|
|
251
|
+
if (evidence.firstUsefulOutputAt === undefined &&
|
|
252
|
+
(usefulDone ||
|
|
253
|
+
usefulCompletion ||
|
|
254
|
+
usefulToolDelta ||
|
|
255
|
+
((type === "response.output_item.added" ||
|
|
256
|
+
type === "response.output_item.done") &&
|
|
257
|
+
usefulItem))) {
|
|
258
|
+
evidence.firstUsefulOutputAt = Date.now();
|
|
259
|
+
evidence.firstUsefulOutputEvent = String(type);
|
|
185
260
|
}
|
|
186
261
|
if (type === "response.completed") {
|
|
187
262
|
evidence.completed = true;
|
|
@@ -222,6 +297,7 @@ export function createCodexUsageTap() {
|
|
|
222
297
|
}
|
|
223
298
|
catch {
|
|
224
299
|
// Unknown frames cannot establish successful completion.
|
|
300
|
+
evidence.observationIncomplete = true;
|
|
225
301
|
}
|
|
226
302
|
}
|
|
227
303
|
};
|
|
@@ -268,16 +344,21 @@ export function createCodexUsageTap() {
|
|
|
268
344
|
carry = remainder;
|
|
269
345
|
inspectEvidence(events);
|
|
270
346
|
if (carry.length > CARRY_LIMIT_CHARS) {
|
|
347
|
+
evidence.observationIncomplete = true;
|
|
271
348
|
carry = carry.slice(-2);
|
|
272
349
|
discardingEvent = true;
|
|
273
350
|
}
|
|
274
351
|
}
|
|
275
352
|
catch {
|
|
276
353
|
// Telemetry must never break the relay.
|
|
354
|
+
evidence.observationIncomplete = true;
|
|
277
355
|
}
|
|
278
356
|
},
|
|
279
357
|
flush() {
|
|
280
358
|
// An event without its dispatch delimiter is incomplete on the wire.
|
|
359
|
+
if (discardingEvent || carry.trim()) {
|
|
360
|
+
evidence.observationIncomplete = true;
|
|
361
|
+
}
|
|
281
362
|
settle(latest);
|
|
282
363
|
},
|
|
283
364
|
/**
|
|
@@ -34,6 +34,18 @@ export declare function getProxyOtelLogSnapshot(): {
|
|
|
34
34
|
maxPendingBytes: number;
|
|
35
35
|
};
|
|
36
36
|
queues: {
|
|
37
|
+
recentFailures: {
|
|
38
|
+
records: {
|
|
39
|
+
eventId: string;
|
|
40
|
+
kind?: string;
|
|
41
|
+
requestId?: string;
|
|
42
|
+
captureId?: string;
|
|
43
|
+
}[];
|
|
44
|
+
id: string;
|
|
45
|
+
at: string;
|
|
46
|
+
reason: "export_unconfirmed" | "queue_full";
|
|
47
|
+
error?: string;
|
|
48
|
+
}[];
|
|
37
49
|
attempted: number;
|
|
38
50
|
submitted: number;
|
|
39
51
|
transportAcknowledged: number;
|
|
@@ -43,6 +55,7 @@ export declare function getProxyOtelLogSnapshot(): {
|
|
|
43
55
|
lastAcknowledgedAt: string | undefined;
|
|
44
56
|
lastFailureAt: string | undefined;
|
|
45
57
|
highWaterOutstanding: number;
|
|
58
|
+
failureHistoryEvicted: number;
|
|
46
59
|
kind: "metadata" | "bodies";
|
|
47
60
|
capacity: number;
|
|
48
61
|
}[];
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
/* eslint-disable no-console -- This proxy-only sink replaces console methods with OTLP emission. */
|
|
2
2
|
import { inspect } from "node:util";
|
|
3
|
+
import { randomUUID } from "node:crypto";
|
|
4
|
+
import { getProxyRequestTraceContext, proxyLogContext, } from "./proxyTraceContext.js";
|
|
3
5
|
import { setImmediate as yieldToRequests } from "node:timers/promises";
|
|
4
6
|
import { SeverityNumber } from "@opentelemetry/api-logs";
|
|
5
7
|
import { ExportResultCode } from "@opentelemetry/core";
|
|
@@ -45,6 +47,52 @@ function createTrackedProcessor(url, capacity, kind) {
|
|
|
45
47
|
lastAcknowledgedAt: undefined,
|
|
46
48
|
lastFailureAt: undefined,
|
|
47
49
|
highWaterOutstanding: 0,
|
|
50
|
+
recentFailures: [],
|
|
51
|
+
failureHistoryEvicted: 0,
|
|
52
|
+
};
|
|
53
|
+
const diagnostics = [];
|
|
54
|
+
let diagnosticScheduled = false;
|
|
55
|
+
const rememberFailure = (records, reason, error) => {
|
|
56
|
+
const stringAttribute = (record, name) => {
|
|
57
|
+
const value = record.attributes[name];
|
|
58
|
+
return typeof value === "string" ? value.slice(0, 128) : undefined;
|
|
59
|
+
};
|
|
60
|
+
const failure = {
|
|
61
|
+
id: randomUUID(),
|
|
62
|
+
at: new Date().toISOString(),
|
|
63
|
+
reason,
|
|
64
|
+
...(error ? { error: sanitizeForLog(error.message).slice(0, 256) } : {}),
|
|
65
|
+
records: records.slice(0, 64).map((record) => ({
|
|
66
|
+
eventId: stringAttribute(record, "proxy.event_id") ?? "unavailable",
|
|
67
|
+
kind: stringAttribute(record, "proxy.record_kind"),
|
|
68
|
+
requestId: stringAttribute(record, "request.id"),
|
|
69
|
+
captureId: stringAttribute(record, "body.capture_id"),
|
|
70
|
+
})),
|
|
71
|
+
};
|
|
72
|
+
if (state.recentFailures.length === 16) {
|
|
73
|
+
state.recentFailures.shift();
|
|
74
|
+
state.failureHistoryEvicted++;
|
|
75
|
+
}
|
|
76
|
+
state.recentFailures.push(failure);
|
|
77
|
+
// A failed diagnostic must not generate another diagnostic recursively.
|
|
78
|
+
if (records.some((record) => record.attributes["proxy.record_kind"] !== "telemetry_delivery")) {
|
|
79
|
+
if (diagnostics.length === 16) {
|
|
80
|
+
diagnostics.shift();
|
|
81
|
+
}
|
|
82
|
+
diagnostics.push(failure);
|
|
83
|
+
}
|
|
84
|
+
};
|
|
85
|
+
const publishRecoveredDiagnostics = () => {
|
|
86
|
+
if (diagnosticScheduled || !diagnostics.length || shuttingDown) {
|
|
87
|
+
return;
|
|
88
|
+
}
|
|
89
|
+
diagnosticScheduled = true;
|
|
90
|
+
queueMicrotask(() => {
|
|
91
|
+
diagnosticScheduled = false;
|
|
92
|
+
for (const failure of diagnostics.splice(0)) {
|
|
93
|
+
emitProxyOtelEvent("telemetry_delivery", { queue: kind, ...failure });
|
|
94
|
+
}
|
|
95
|
+
});
|
|
48
96
|
};
|
|
49
97
|
const transport = new OTLPLogExporter({ url, timeoutMillis: 5000 });
|
|
50
98
|
const exporter = {
|
|
@@ -65,10 +113,12 @@ function createTrackedProcessor(url, capacity, kind) {
|
|
|
65
113
|
if (result.code === ExportResultCode.SUCCESS) {
|
|
66
114
|
state.transportAcknowledged += records.length;
|
|
67
115
|
state.lastAcknowledgedAt = new Date().toISOString();
|
|
116
|
+
publishRecoveredDiagnostics();
|
|
68
117
|
}
|
|
69
118
|
else {
|
|
70
119
|
state.exportUnconfirmed += records.length;
|
|
71
120
|
state.lastFailureAt = new Date().toISOString();
|
|
121
|
+
rememberFailure(records, "export_unconfirmed", result.error);
|
|
72
122
|
}
|
|
73
123
|
for (const record of records) {
|
|
74
124
|
unsettled.delete(record);
|
|
@@ -109,6 +159,7 @@ function createTrackedProcessor(url, capacity, kind) {
|
|
|
109
159
|
});
|
|
110
160
|
const processor = {
|
|
111
161
|
onEmit(record) {
|
|
162
|
+
record.attributes["proxy.event_id"] ??= randomUUID();
|
|
112
163
|
state.attempted++;
|
|
113
164
|
const id = record.attributes?.["body.capture_id"];
|
|
114
165
|
const publication = typeof id === "string" ? bodyPublications.get(id) : undefined;
|
|
@@ -117,6 +168,7 @@ function createTrackedProcessor(url, capacity, kind) {
|
|
|
117
168
|
}
|
|
118
169
|
if (state.outstanding >= capacity) {
|
|
119
170
|
state.dropped++;
|
|
171
|
+
rememberFailure([record], "queue_full");
|
|
120
172
|
if (publication) {
|
|
121
173
|
publication.dropped++;
|
|
122
174
|
publication.notify?.();
|
|
@@ -355,12 +407,17 @@ export function emitProxyOtelEvent(kind, record) {
|
|
|
355
407
|
return;
|
|
356
408
|
}
|
|
357
409
|
try {
|
|
410
|
+
const ids = typeof record.requestId === "string" && !record.traceId
|
|
411
|
+
? getProxyRequestTraceContext(record.requestId)
|
|
412
|
+
: undefined;
|
|
413
|
+
const correlated = ids ? { ...record, ...ids } : record;
|
|
358
414
|
initializeProxyOtelLogs()
|
|
359
415
|
?.getLogger("neurolink-proxy-events")
|
|
360
416
|
.emit({
|
|
417
|
+
context: proxyLogContext(correlated),
|
|
361
418
|
severityNumber: SeverityNumber.INFO,
|
|
362
419
|
severityText: "INFO",
|
|
363
|
-
body: JSON.stringify(
|
|
420
|
+
body: JSON.stringify(correlated),
|
|
364
421
|
attributes: {
|
|
365
422
|
"proxy.record_kind": kind,
|
|
366
423
|
"event.name": `proxy.${kind}`,
|
|
@@ -444,6 +501,10 @@ export function getProxyOtelLogSnapshot() {
|
|
|
444
501
|
kind: q.kind,
|
|
445
502
|
capacity: q.capacity,
|
|
446
503
|
...q.state,
|
|
504
|
+
recentFailures: q.state.recentFailures.map((failure) => ({
|
|
505
|
+
...failure,
|
|
506
|
+
records: failure.records.map((record) => ({ ...record })),
|
|
507
|
+
})),
|
|
447
508
|
})),
|
|
448
509
|
};
|
|
449
510
|
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import type { ProxyLogTraceContext } from "../types/index.js";
|
|
2
|
+
/** Correlation belongs to the in-flight HTTP request, not to an async callback. */
|
|
3
|
+
export declare function registerProxyRequestTraceContext(requestId: string, ids: ProxyLogTraceContext): void;
|
|
4
|
+
/** Release once transport and terminal accounting have settled. */
|
|
5
|
+
export declare function releaseProxyRequestTraceContext(requestId: string): void;
|
|
6
|
+
/** Internal fallback records share their parent request's trace. */
|
|
7
|
+
export declare function getProxyRequestTraceContext(requestId: string): ProxyLogTraceContext | undefined;
|
|
8
|
+
/** Retain IDs and sampling flags before deferred processing leaves the request. */
|
|
9
|
+
export declare function resolveProxyLogTraceContext(record: {
|
|
10
|
+
traceId?: unknown;
|
|
11
|
+
spanId?: unknown;
|
|
12
|
+
traceFlags?: unknown;
|
|
13
|
+
requestId?: unknown;
|
|
14
|
+
}): ProxyLogTraceContext | undefined;
|
|
15
|
+
/** Populate native OTLP correlation, including valid unsampled contexts. */
|
|
16
|
+
export declare function proxyLogContext(record: {
|
|
17
|
+
traceId?: unknown;
|
|
18
|
+
spanId?: unknown;
|
|
19
|
+
traceFlags?: unknown;
|
|
20
|
+
requestId?: unknown;
|
|
21
|
+
}): import("@opentelemetry/api").Context;
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import { context, ROOT_CONTEXT, trace, isSpanContextValid, } from "@opentelemetry/api";
|
|
2
|
+
const requests = new Map();
|
|
3
|
+
/** Correlation belongs to the in-flight HTTP request, not to an async callback. */
|
|
4
|
+
export function registerProxyRequestTraceContext(requestId, ids) {
|
|
5
|
+
requests.set(requestId, ids);
|
|
6
|
+
}
|
|
7
|
+
/** Release once transport and terminal accounting have settled. */
|
|
8
|
+
export function releaseProxyRequestTraceContext(requestId) {
|
|
9
|
+
requests.delete(requestId);
|
|
10
|
+
}
|
|
11
|
+
/** Internal fallback records share their parent request's trace. */
|
|
12
|
+
export function getProxyRequestTraceContext(requestId) {
|
|
13
|
+
return (requests.get(requestId) ??
|
|
14
|
+
requests.get(requestId.replace(/:codex-fallback$/, "")));
|
|
15
|
+
}
|
|
16
|
+
/** Retain IDs and sampling flags before deferred processing leaves the request. */
|
|
17
|
+
export function resolveProxyLogTraceContext(record) {
|
|
18
|
+
const saved = typeof record.requestId === "string"
|
|
19
|
+
? getProxyRequestTraceContext(record.requestId)
|
|
20
|
+
: undefined;
|
|
21
|
+
const active = trace.getSpanContext(context.active());
|
|
22
|
+
const ids = typeof record.traceId === "string" && typeof record.spanId === "string"
|
|
23
|
+
? { traceId: record.traceId, spanId: record.spanId }
|
|
24
|
+
: (saved ?? active);
|
|
25
|
+
if (!ids) {
|
|
26
|
+
return undefined;
|
|
27
|
+
}
|
|
28
|
+
const traceFlags = typeof record.traceFlags === "number" &&
|
|
29
|
+
Number.isInteger(record.traceFlags) &&
|
|
30
|
+
record.traceFlags >= 0 &&
|
|
31
|
+
record.traceFlags <= 255
|
|
32
|
+
? record.traceFlags
|
|
33
|
+
: saved?.traceId === ids.traceId
|
|
34
|
+
? saved.traceFlags
|
|
35
|
+
: active?.traceId === ids.traceId
|
|
36
|
+
? active.traceFlags
|
|
37
|
+
: 0;
|
|
38
|
+
const result = { ...ids, traceFlags };
|
|
39
|
+
return isSpanContextValid(result)
|
|
40
|
+
? { traceId: result.traceId, spanId: result.spanId, traceFlags }
|
|
41
|
+
: undefined;
|
|
42
|
+
}
|
|
43
|
+
/** Populate native OTLP correlation, including valid unsampled contexts. */
|
|
44
|
+
export function proxyLogContext(record) {
|
|
45
|
+
const ids = resolveProxyLogTraceContext(record);
|
|
46
|
+
return ids ? trace.setSpanContext(ROOT_CONTEXT, ids) : context.active();
|
|
47
|
+
}
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
* - TelemetryService for metrics recording
|
|
15
15
|
*/
|
|
16
16
|
import { type Span } from "@opentelemetry/api";
|
|
17
|
-
import type { AccountSelectionContext, ProxyRequestContext, ResponseInfoContext, UpstreamAttemptContext, UsageContext } from "../types/index.js";
|
|
17
|
+
import type { AccountSelectionContext, ProxyRequestContext, ResponseInfoContext, UpstreamAttemptContext, UsageContext, RuntimeRequestMetadata, ProxyLogTraceContext } from "../types/index.js";
|
|
18
18
|
declare class ProxyTracer {
|
|
19
19
|
private readonly rootSpan;
|
|
20
20
|
private readonly proxyTracer;
|
|
@@ -34,6 +34,8 @@ declare class ProxyTracer {
|
|
|
34
34
|
private billingProvider;
|
|
35
35
|
private readonly startTime;
|
|
36
36
|
private readonly isStream;
|
|
37
|
+
private ended;
|
|
38
|
+
private recordRequestMetrics;
|
|
37
39
|
private accountEmail?;
|
|
38
40
|
private usage?;
|
|
39
41
|
private mode;
|
|
@@ -102,10 +104,7 @@ declare class ProxyTracer {
|
|
|
102
104
|
/** Record request and/or response body sizes for bandwidth tracking. */
|
|
103
105
|
recordBodySizes(requestBytes?: number, responseBytes?: number): void;
|
|
104
106
|
/** Return the OTel trace/span IDs for this request (for log correlation). */
|
|
105
|
-
getTraceContext():
|
|
106
|
-
traceId: string;
|
|
107
|
-
spanId: string;
|
|
108
|
-
};
|
|
107
|
+
getTraceContext(): ProxyLogTraceContext;
|
|
109
108
|
/** Return the captured usage (set by setUsage). */
|
|
110
109
|
getUsage(): UsageContext | undefined;
|
|
111
110
|
/** End the root span with final HTTP status and duration, and emit OTEL metrics. */
|
|
@@ -126,3 +125,8 @@ export declare function recordFallbackAttempt(attrs: {
|
|
|
126
125
|
durationMs: number;
|
|
127
126
|
}): void;
|
|
128
127
|
export { ProxyTracer };
|
|
128
|
+
/** Standard SERVER span covers every proxy door, including parsing and admission failures. */
|
|
129
|
+
export declare function startProxyHttpTrace(metadata: RuntimeRequestMetadata, headers: Record<string, string>): {
|
|
130
|
+
run: <T>(fn: () => T) => T;
|
|
131
|
+
end: (status: number, outcome: string, errorType?: string) => void;
|
|
132
|
+
};
|