@juspay/neurolink 12.14.6 → 12.14.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +2 -2
- package/dist/browser/neurolink.min.js +411 -411
- package/dist/cli/commands/proxy.js +83 -14
- package/dist/proxy/bodyCaptureProcessing.js +47 -3
- package/dist/proxy/bodyCaptureWorker.js +39 -10
- package/dist/proxy/claudeOutputObservation.d.ts +9 -0
- package/dist/proxy/claudeOutputObservation.js +85 -0
- package/dist/proxy/codexUsage.js +86 -3
- package/dist/proxy/otelLogSink.d.ts +13 -0
- package/dist/proxy/otelLogSink.js +63 -2
- package/dist/proxy/processTelemetry.d.ts +3 -0
- package/dist/proxy/processTelemetry.js +25 -0
- package/dist/proxy/proxyTraceContext.d.ts +21 -0
- package/dist/proxy/proxyTraceContext.js +47 -0
- package/dist/proxy/proxyTracer.d.ts +9 -5
- package/dist/proxy/proxyTracer.js +81 -2
- package/dist/proxy/rawStreamCapture.js +17 -26
- package/dist/proxy/requestLogger.d.ts +2 -0
- package/dist/proxy/requestLogger.js +29 -19
- package/dist/proxy/restartControl.d.ts +3 -1
- package/dist/proxy/restartControl.js +118 -1
- package/dist/proxy/sseInterceptor.js +71 -3
- package/dist/server/routes/claudeProxyRoutes.d.ts +0 -11
- package/dist/server/routes/claudeProxyRoutes.js +27 -12
- package/dist/server/routes/codexProxyRoutes.js +496 -333
- package/dist/types/cli.d.ts +7 -1
- package/dist/types/proxy.d.ts +133 -3
- package/dist/types/proxyRestart.d.ts +25 -0
- package/dist/utils/logSanitize.js +1 -1
- package/docs-site/static/search-index.json +1 -1
- package/package.json +1 -1
- package/scripts/observability/check-proxy-telemetry.mjs +29 -241
- package/scripts/observability/proxy-telemetry-backend.mjs +202 -0
- package/scripts/observability/proxy-telemetry-check.mjs +578 -0
- package/scripts/observability/query-proxy-history.mjs +30 -19
package/dist/proxy/codexUsage.js
CHANGED
|
@@ -45,6 +45,36 @@ import { sanitizeForLog } from "../utils/logSanitize.js";
|
|
|
45
45
|
const nonNegativeInt = (value) => typeof value === "number" && Number.isFinite(value) && value > 0
|
|
46
46
|
? Math.floor(value)
|
|
47
47
|
: 0;
|
|
48
|
+
function usefulOutputItem(item) {
|
|
49
|
+
if (!item || typeof item !== "object") {
|
|
50
|
+
return false;
|
|
51
|
+
}
|
|
52
|
+
if ("type" in item) {
|
|
53
|
+
if (item.type === "function_call") {
|
|
54
|
+
return ("arguments" in item &&
|
|
55
|
+
typeof item.arguments === "string" &&
|
|
56
|
+
item.arguments.trim().length > 0);
|
|
57
|
+
}
|
|
58
|
+
if (item.type === "custom_tool_call") {
|
|
59
|
+
return ("input" in item &&
|
|
60
|
+
typeof item.input === "string" &&
|
|
61
|
+
item.input.trim().length > 0);
|
|
62
|
+
}
|
|
63
|
+
if (item.type === "output_text") {
|
|
64
|
+
return ("text" in item &&
|
|
65
|
+
typeof item.text === "string" &&
|
|
66
|
+
item.text.trim().length > 0);
|
|
67
|
+
}
|
|
68
|
+
if (item.type === "refusal") {
|
|
69
|
+
return ("refusal" in item &&
|
|
70
|
+
typeof item.refusal === "string" &&
|
|
71
|
+
item.refusal.trim().length > 0);
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
return ("content" in item &&
|
|
75
|
+
Array.isArray(item.content) &&
|
|
76
|
+
item.content.some(usefulOutputItem));
|
|
77
|
+
}
|
|
48
78
|
/**
|
|
49
79
|
* Pull usage out of one parsed SSE `data:` payload.
|
|
50
80
|
*
|
|
@@ -125,7 +155,7 @@ const CAPTURE_LIMIT_BYTES = 256 * 1024;
|
|
|
125
155
|
*/
|
|
126
156
|
function createCaptureSink() {
|
|
127
157
|
const target = process.env.NEUROLINK_PROXY_CODEX_CAPTURE;
|
|
128
|
-
if (!target) {
|
|
158
|
+
if (!target || process.env.NEUROLINK_PROXY_LOG_SINK === "otel") {
|
|
129
159
|
return null;
|
|
130
160
|
}
|
|
131
161
|
let written = 0;
|
|
@@ -167,9 +197,13 @@ export function createCodexUsageTap() {
|
|
|
167
197
|
let totalBytes = 0;
|
|
168
198
|
const inspectEvidence = (events) => {
|
|
169
199
|
for (const frame of events) {
|
|
200
|
+
if (frame.data.trim() === "[DONE]") {
|
|
201
|
+
continue;
|
|
202
|
+
}
|
|
170
203
|
try {
|
|
171
204
|
const event = JSON.parse(frame.data);
|
|
172
205
|
if (!event || typeof event !== "object") {
|
|
206
|
+
evidence.observationIncomplete = true;
|
|
173
207
|
continue;
|
|
174
208
|
}
|
|
175
209
|
const seen = extractCodexUsage(event);
|
|
@@ -178,10 +212,53 @@ export function createCodexUsageTap() {
|
|
|
178
212
|
}
|
|
179
213
|
const type = event.type ?? frame.event;
|
|
180
214
|
if ((type === "response.output_text.delta" ||
|
|
215
|
+
type === "response.refusal.delta" ||
|
|
181
216
|
type === "response.function_call_arguments.delta") &&
|
|
182
217
|
typeof event.delta === "string" &&
|
|
183
|
-
event.delta.length > 0) {
|
|
184
|
-
evidence.firstUsefulOutputAt
|
|
218
|
+
event.delta.trim().length > 0) {
|
|
219
|
+
if (evidence.firstUsefulOutputAt === undefined) {
|
|
220
|
+
evidence.firstUsefulOutputAt = Date.now();
|
|
221
|
+
evidence.firstUsefulOutputEvent = String(type);
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
// Some clients/providers emit complete output items without deltas.
|
|
225
|
+
// Reasoning/control events are not useful client output. Recognize
|
|
226
|
+
// actual tool calls and text only, without inventing an earlier time.
|
|
227
|
+
const usefulItem = usefulOutputItem(event.item);
|
|
228
|
+
const completedResponse = event.response;
|
|
229
|
+
const usefulCompletion = type === "response.completed" &&
|
|
230
|
+
completedResponse &&
|
|
231
|
+
typeof completedResponse === "object" &&
|
|
232
|
+
"output" in completedResponse &&
|
|
233
|
+
Array.isArray(completedResponse.output) &&
|
|
234
|
+
completedResponse.output.some(usefulOutputItem);
|
|
235
|
+
const usefulDone = (type === "response.output_text.done" &&
|
|
236
|
+
typeof event.text === "string" &&
|
|
237
|
+
event.text.trim().length > 0) ||
|
|
238
|
+
(type === "response.refusal.done" &&
|
|
239
|
+
typeof event.refusal === "string" &&
|
|
240
|
+
event.refusal.trim().length > 0) ||
|
|
241
|
+
(type === "response.function_call_arguments.done" &&
|
|
242
|
+
typeof event.arguments === "string" &&
|
|
243
|
+
event.arguments.trim().length > 0) ||
|
|
244
|
+
(type === "response.custom_tool_call_input.done" &&
|
|
245
|
+
typeof event.input === "string" &&
|
|
246
|
+
event.input.trim().length > 0) ||
|
|
247
|
+
((type === "response.content_part.added" ||
|
|
248
|
+
type === "response.content_part.done") &&
|
|
249
|
+
usefulOutputItem(event.part));
|
|
250
|
+
const usefulToolDelta = type === "response.custom_tool_call_input.delta" &&
|
|
251
|
+
typeof event.delta === "string" &&
|
|
252
|
+
event.delta.trim().length > 0;
|
|
253
|
+
if (evidence.firstUsefulOutputAt === undefined &&
|
|
254
|
+
(usefulDone ||
|
|
255
|
+
usefulCompletion ||
|
|
256
|
+
usefulToolDelta ||
|
|
257
|
+
((type === "response.output_item.added" ||
|
|
258
|
+
type === "response.output_item.done") &&
|
|
259
|
+
usefulItem))) {
|
|
260
|
+
evidence.firstUsefulOutputAt = Date.now();
|
|
261
|
+
evidence.firstUsefulOutputEvent = String(type);
|
|
185
262
|
}
|
|
186
263
|
if (type === "response.completed") {
|
|
187
264
|
evidence.completed = true;
|
|
@@ -222,6 +299,7 @@ export function createCodexUsageTap() {
|
|
|
222
299
|
}
|
|
223
300
|
catch {
|
|
224
301
|
// Unknown frames cannot establish successful completion.
|
|
302
|
+
evidence.observationIncomplete = true;
|
|
225
303
|
}
|
|
226
304
|
}
|
|
227
305
|
};
|
|
@@ -268,16 +346,21 @@ export function createCodexUsageTap() {
|
|
|
268
346
|
carry = remainder;
|
|
269
347
|
inspectEvidence(events);
|
|
270
348
|
if (carry.length > CARRY_LIMIT_CHARS) {
|
|
349
|
+
evidence.observationIncomplete = true;
|
|
271
350
|
carry = carry.slice(-2);
|
|
272
351
|
discardingEvent = true;
|
|
273
352
|
}
|
|
274
353
|
}
|
|
275
354
|
catch {
|
|
276
355
|
// Telemetry must never break the relay.
|
|
356
|
+
evidence.observationIncomplete = true;
|
|
277
357
|
}
|
|
278
358
|
},
|
|
279
359
|
flush() {
|
|
280
360
|
// An event without its dispatch delimiter is incomplete on the wire.
|
|
361
|
+
if (discardingEvent || carry.trim()) {
|
|
362
|
+
evidence.observationIncomplete = true;
|
|
363
|
+
}
|
|
281
364
|
settle(latest);
|
|
282
365
|
},
|
|
283
366
|
/**
|
|
@@ -34,6 +34,18 @@ export declare function getProxyOtelLogSnapshot(): {
|
|
|
34
34
|
maxPendingBytes: number;
|
|
35
35
|
};
|
|
36
36
|
queues: {
|
|
37
|
+
recentFailures: {
|
|
38
|
+
records: {
|
|
39
|
+
eventId: string;
|
|
40
|
+
kind?: string;
|
|
41
|
+
requestId?: string;
|
|
42
|
+
captureId?: string;
|
|
43
|
+
}[];
|
|
44
|
+
id: string;
|
|
45
|
+
at: string;
|
|
46
|
+
reason: "export_unconfirmed" | "queue_full";
|
|
47
|
+
error?: string;
|
|
48
|
+
}[];
|
|
37
49
|
attempted: number;
|
|
38
50
|
submitted: number;
|
|
39
51
|
transportAcknowledged: number;
|
|
@@ -43,6 +55,7 @@ export declare function getProxyOtelLogSnapshot(): {
|
|
|
43
55
|
lastAcknowledgedAt: string | undefined;
|
|
44
56
|
lastFailureAt: string | undefined;
|
|
45
57
|
highWaterOutstanding: number;
|
|
58
|
+
failureHistoryEvicted: number;
|
|
46
59
|
kind: "metadata" | "bodies";
|
|
47
60
|
capacity: number;
|
|
48
61
|
}[];
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
/* eslint-disable no-console -- This proxy-only sink replaces console methods with OTLP emission. */
|
|
2
2
|
import { inspect } from "node:util";
|
|
3
|
+
import { randomUUID } from "node:crypto";
|
|
4
|
+
import { getProxyRequestTraceContext, proxyLogContext, } from "./proxyTraceContext.js";
|
|
3
5
|
import { setImmediate as yieldToRequests } from "node:timers/promises";
|
|
4
6
|
import { SeverityNumber } from "@opentelemetry/api-logs";
|
|
5
7
|
import { ExportResultCode } from "@opentelemetry/core";
|
|
@@ -24,7 +26,7 @@ const bodyDelivery = {
|
|
|
24
26
|
pendingBytes: 0,
|
|
25
27
|
highWaterPending: 0,
|
|
26
28
|
highWaterBytes: 0,
|
|
27
|
-
maxPending:
|
|
29
|
+
maxPending: 64,
|
|
28
30
|
maxPendingBytes: 32 * 1024 * 1024,
|
|
29
31
|
};
|
|
30
32
|
/** Explicit opt-in; configuration never silently falls back to file logging. */
|
|
@@ -45,6 +47,52 @@ function createTrackedProcessor(url, capacity, kind) {
|
|
|
45
47
|
lastAcknowledgedAt: undefined,
|
|
46
48
|
lastFailureAt: undefined,
|
|
47
49
|
highWaterOutstanding: 0,
|
|
50
|
+
recentFailures: [],
|
|
51
|
+
failureHistoryEvicted: 0,
|
|
52
|
+
};
|
|
53
|
+
const diagnostics = [];
|
|
54
|
+
let diagnosticScheduled = false;
|
|
55
|
+
const rememberFailure = (records, reason, error) => {
|
|
56
|
+
const stringAttribute = (record, name) => {
|
|
57
|
+
const value = record.attributes[name];
|
|
58
|
+
return typeof value === "string" ? value.slice(0, 128) : undefined;
|
|
59
|
+
};
|
|
60
|
+
const failure = {
|
|
61
|
+
id: randomUUID(),
|
|
62
|
+
at: new Date().toISOString(),
|
|
63
|
+
reason,
|
|
64
|
+
...(error ? { error: sanitizeForLog(error.message).slice(0, 256) } : {}),
|
|
65
|
+
records: records.slice(0, 64).map((record) => ({
|
|
66
|
+
eventId: stringAttribute(record, "proxy.event_id") ?? "unavailable",
|
|
67
|
+
kind: stringAttribute(record, "proxy.record_kind"),
|
|
68
|
+
requestId: stringAttribute(record, "request.id"),
|
|
69
|
+
captureId: stringAttribute(record, "body.capture_id"),
|
|
70
|
+
})),
|
|
71
|
+
};
|
|
72
|
+
if (state.recentFailures.length === 16) {
|
|
73
|
+
state.recentFailures.shift();
|
|
74
|
+
state.failureHistoryEvicted++;
|
|
75
|
+
}
|
|
76
|
+
state.recentFailures.push(failure);
|
|
77
|
+
// A failed diagnostic must not generate another diagnostic recursively.
|
|
78
|
+
if (records.some((record) => record.attributes["proxy.record_kind"] !== "telemetry_delivery")) {
|
|
79
|
+
if (diagnostics.length === 16) {
|
|
80
|
+
diagnostics.shift();
|
|
81
|
+
}
|
|
82
|
+
diagnostics.push(failure);
|
|
83
|
+
}
|
|
84
|
+
};
|
|
85
|
+
const publishRecoveredDiagnostics = () => {
|
|
86
|
+
if (diagnosticScheduled || !diagnostics.length || shuttingDown) {
|
|
87
|
+
return;
|
|
88
|
+
}
|
|
89
|
+
diagnosticScheduled = true;
|
|
90
|
+
queueMicrotask(() => {
|
|
91
|
+
diagnosticScheduled = false;
|
|
92
|
+
for (const failure of diagnostics.splice(0)) {
|
|
93
|
+
emitProxyOtelEvent("telemetry_delivery", { queue: kind, ...failure });
|
|
94
|
+
}
|
|
95
|
+
});
|
|
48
96
|
};
|
|
49
97
|
const transport = new OTLPLogExporter({ url, timeoutMillis: 5000 });
|
|
50
98
|
const exporter = {
|
|
@@ -65,10 +113,12 @@ function createTrackedProcessor(url, capacity, kind) {
|
|
|
65
113
|
if (result.code === ExportResultCode.SUCCESS) {
|
|
66
114
|
state.transportAcknowledged += records.length;
|
|
67
115
|
state.lastAcknowledgedAt = new Date().toISOString();
|
|
116
|
+
publishRecoveredDiagnostics();
|
|
68
117
|
}
|
|
69
118
|
else {
|
|
70
119
|
state.exportUnconfirmed += records.length;
|
|
71
120
|
state.lastFailureAt = new Date().toISOString();
|
|
121
|
+
rememberFailure(records, "export_unconfirmed", result.error);
|
|
72
122
|
}
|
|
73
123
|
for (const record of records) {
|
|
74
124
|
unsettled.delete(record);
|
|
@@ -109,6 +159,7 @@ function createTrackedProcessor(url, capacity, kind) {
|
|
|
109
159
|
});
|
|
110
160
|
const processor = {
|
|
111
161
|
onEmit(record) {
|
|
162
|
+
record.attributes["proxy.event_id"] ??= randomUUID();
|
|
112
163
|
state.attempted++;
|
|
113
164
|
const id = record.attributes?.["body.capture_id"];
|
|
114
165
|
const publication = typeof id === "string" ? bodyPublications.get(id) : undefined;
|
|
@@ -117,6 +168,7 @@ function createTrackedProcessor(url, capacity, kind) {
|
|
|
117
168
|
}
|
|
118
169
|
if (state.outstanding >= capacity) {
|
|
119
170
|
state.dropped++;
|
|
171
|
+
rememberFailure([record], "queue_full");
|
|
120
172
|
if (publication) {
|
|
121
173
|
publication.dropped++;
|
|
122
174
|
publication.notify?.();
|
|
@@ -355,12 +407,17 @@ export function emitProxyOtelEvent(kind, record) {
|
|
|
355
407
|
return;
|
|
356
408
|
}
|
|
357
409
|
try {
|
|
410
|
+
const ids = typeof record.requestId === "string" && !record.traceId
|
|
411
|
+
? getProxyRequestTraceContext(record.requestId)
|
|
412
|
+
: undefined;
|
|
413
|
+
const correlated = ids ? { ...record, ...ids } : record;
|
|
358
414
|
initializeProxyOtelLogs()
|
|
359
415
|
?.getLogger("neurolink-proxy-events")
|
|
360
416
|
.emit({
|
|
417
|
+
context: proxyLogContext(correlated),
|
|
361
418
|
severityNumber: SeverityNumber.INFO,
|
|
362
419
|
severityText: "INFO",
|
|
363
|
-
body: JSON.stringify(
|
|
420
|
+
body: JSON.stringify(correlated),
|
|
364
421
|
attributes: {
|
|
365
422
|
"proxy.record_kind": kind,
|
|
366
423
|
"event.name": `proxy.${kind}`,
|
|
@@ -444,6 +501,10 @@ export function getProxyOtelLogSnapshot() {
|
|
|
444
501
|
kind: q.kind,
|
|
445
502
|
capacity: q.capacity,
|
|
446
503
|
...q.state,
|
|
504
|
+
recentFailures: q.state.recentFailures.map((failure) => ({
|
|
505
|
+
...failure,
|
|
506
|
+
records: failure.records.map((record) => ({ ...record })),
|
|
507
|
+
})),
|
|
447
508
|
})),
|
|
448
509
|
};
|
|
449
510
|
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { fstatSync } from "node:fs";
|
|
2
|
+
import { getProxyOtelLogSnapshot, isProxyOtelOnly } from "./otelLogSink.js";
|
|
3
|
+
import { getProxyLifecycleLoggerSnapshot } from "./proxyLifecycle.js";
|
|
4
|
+
/** Inspect this process, not the launcher configuration intended for its successor. */
|
|
5
|
+
export function getProxyProcessTelemetry() {
|
|
6
|
+
const descriptor = (fd) => {
|
|
7
|
+
try {
|
|
8
|
+
return fstatSync(fd).isFile() ? "file" : "non_file";
|
|
9
|
+
}
|
|
10
|
+
catch {
|
|
11
|
+
return "unavailable";
|
|
12
|
+
}
|
|
13
|
+
};
|
|
14
|
+
const logs = getProxyOtelLogSnapshot();
|
|
15
|
+
return {
|
|
16
|
+
pid: process.pid,
|
|
17
|
+
checkedAt: new Date().toISOString(),
|
|
18
|
+
configuredSink: isProxyOtelOnly() ? "otel" : "file",
|
|
19
|
+
lifecycleSink: getProxyLifecycleLoggerSnapshot().sink ?? "unavailable",
|
|
20
|
+
otelInitialized: logs.initialized,
|
|
21
|
+
stdio: { stdout: descriptor(1), stderr: descriptor(2) },
|
|
22
|
+
exportDropped: logs.queues.reduce((n, q) => n + q.dropped, 0),
|
|
23
|
+
exportUnconfirmed: logs.queues.reduce((n, q) => n + q.exportUnconfirmed, 0),
|
|
24
|
+
};
|
|
25
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import type { ProxyLogTraceContext } from "../types/index.js";
|
|
2
|
+
/** Correlation belongs to the in-flight HTTP request, not to an async callback. */
|
|
3
|
+
export declare function registerProxyRequestTraceContext(requestId: string, ids: ProxyLogTraceContext): void;
|
|
4
|
+
/** Release once transport and terminal accounting have settled. */
|
|
5
|
+
export declare function releaseProxyRequestTraceContext(requestId: string): void;
|
|
6
|
+
/** Internal fallback records share their parent request's trace. */
|
|
7
|
+
export declare function getProxyRequestTraceContext(requestId: string): ProxyLogTraceContext | undefined;
|
|
8
|
+
/** Retain IDs and sampling flags before deferred processing leaves the request. */
|
|
9
|
+
export declare function resolveProxyLogTraceContext(record: {
|
|
10
|
+
traceId?: unknown;
|
|
11
|
+
spanId?: unknown;
|
|
12
|
+
traceFlags?: unknown;
|
|
13
|
+
requestId?: unknown;
|
|
14
|
+
}): ProxyLogTraceContext | undefined;
|
|
15
|
+
/** Populate native OTLP correlation, including valid unsampled contexts. */
|
|
16
|
+
export declare function proxyLogContext(record: {
|
|
17
|
+
traceId?: unknown;
|
|
18
|
+
spanId?: unknown;
|
|
19
|
+
traceFlags?: unknown;
|
|
20
|
+
requestId?: unknown;
|
|
21
|
+
}): import("@opentelemetry/api").Context;
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import { context, ROOT_CONTEXT, trace, isSpanContextValid, } from "@opentelemetry/api";
|
|
2
|
+
const requests = new Map();
|
|
3
|
+
/** Correlation belongs to the in-flight HTTP request, not to an async callback. */
|
|
4
|
+
export function registerProxyRequestTraceContext(requestId, ids) {
|
|
5
|
+
requests.set(requestId, ids);
|
|
6
|
+
}
|
|
7
|
+
/** Release once transport and terminal accounting have settled. */
|
|
8
|
+
export function releaseProxyRequestTraceContext(requestId) {
|
|
9
|
+
requests.delete(requestId);
|
|
10
|
+
}
|
|
11
|
+
/** Internal fallback records share their parent request's trace. */
|
|
12
|
+
export function getProxyRequestTraceContext(requestId) {
|
|
13
|
+
return (requests.get(requestId) ??
|
|
14
|
+
requests.get(requestId.replace(/:codex-fallback$/, "")));
|
|
15
|
+
}
|
|
16
|
+
/** Retain IDs and sampling flags before deferred processing leaves the request. */
|
|
17
|
+
export function resolveProxyLogTraceContext(record) {
|
|
18
|
+
const saved = typeof record.requestId === "string"
|
|
19
|
+
? getProxyRequestTraceContext(record.requestId)
|
|
20
|
+
: undefined;
|
|
21
|
+
const active = trace.getSpanContext(context.active());
|
|
22
|
+
const ids = typeof record.traceId === "string" && typeof record.spanId === "string"
|
|
23
|
+
? { traceId: record.traceId, spanId: record.spanId }
|
|
24
|
+
: (saved ?? active);
|
|
25
|
+
if (!ids) {
|
|
26
|
+
return undefined;
|
|
27
|
+
}
|
|
28
|
+
const traceFlags = typeof record.traceFlags === "number" &&
|
|
29
|
+
Number.isInteger(record.traceFlags) &&
|
|
30
|
+
record.traceFlags >= 0 &&
|
|
31
|
+
record.traceFlags <= 255
|
|
32
|
+
? record.traceFlags
|
|
33
|
+
: saved?.traceId === ids.traceId
|
|
34
|
+
? saved.traceFlags
|
|
35
|
+
: active?.traceId === ids.traceId
|
|
36
|
+
? active.traceFlags
|
|
37
|
+
: 0;
|
|
38
|
+
const result = { ...ids, traceFlags };
|
|
39
|
+
return isSpanContextValid(result)
|
|
40
|
+
? { traceId: result.traceId, spanId: result.spanId, traceFlags }
|
|
41
|
+
: undefined;
|
|
42
|
+
}
|
|
43
|
+
/** Populate native OTLP correlation, including valid unsampled contexts. */
|
|
44
|
+
export function proxyLogContext(record) {
|
|
45
|
+
const ids = resolveProxyLogTraceContext(record);
|
|
46
|
+
return ids ? trace.setSpanContext(ROOT_CONTEXT, ids) : context.active();
|
|
47
|
+
}
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
* - TelemetryService for metrics recording
|
|
15
15
|
*/
|
|
16
16
|
import { type Span } from "@opentelemetry/api";
|
|
17
|
-
import type { AccountSelectionContext, ProxyRequestContext, ResponseInfoContext, UpstreamAttemptContext, UsageContext } from "../types/index.js";
|
|
17
|
+
import type { AccountSelectionContext, ProxyRequestContext, ResponseInfoContext, UpstreamAttemptContext, UsageContext, RuntimeRequestMetadata, ProxyLogTraceContext } from "../types/index.js";
|
|
18
18
|
declare class ProxyTracer {
|
|
19
19
|
private readonly rootSpan;
|
|
20
20
|
private readonly proxyTracer;
|
|
@@ -34,6 +34,8 @@ declare class ProxyTracer {
|
|
|
34
34
|
private billingProvider;
|
|
35
35
|
private readonly startTime;
|
|
36
36
|
private readonly isStream;
|
|
37
|
+
private ended;
|
|
38
|
+
private recordRequestMetrics;
|
|
37
39
|
private accountEmail?;
|
|
38
40
|
private usage?;
|
|
39
41
|
private mode;
|
|
@@ -102,10 +104,7 @@ declare class ProxyTracer {
|
|
|
102
104
|
/** Record request and/or response body sizes for bandwidth tracking. */
|
|
103
105
|
recordBodySizes(requestBytes?: number, responseBytes?: number): void;
|
|
104
106
|
/** Return the OTel trace/span IDs for this request (for log correlation). */
|
|
105
|
-
getTraceContext():
|
|
106
|
-
traceId: string;
|
|
107
|
-
spanId: string;
|
|
108
|
-
};
|
|
107
|
+
getTraceContext(): ProxyLogTraceContext;
|
|
109
108
|
/** Return the captured usage (set by setUsage). */
|
|
110
109
|
getUsage(): UsageContext | undefined;
|
|
111
110
|
/** End the root span with final HTTP status and duration, and emit OTEL metrics. */
|
|
@@ -126,3 +125,8 @@ export declare function recordFallbackAttempt(attrs: {
|
|
|
126
125
|
durationMs: number;
|
|
127
126
|
}): void;
|
|
128
127
|
export { ProxyTracer };
|
|
128
|
+
/** Standard SERVER span covers every proxy door, including parsing and admission failures. */
|
|
129
|
+
export declare function startProxyHttpTrace(metadata: RuntimeRequestMetadata, headers: Record<string, string>): {
|
|
130
|
+
run: <T>(fn: () => T) => T;
|
|
131
|
+
end: (status: number, outcome: string, errorType?: string) => void;
|
|
132
|
+
};
|
|
@@ -13,12 +13,13 @@
|
|
|
13
13
|
* - calculateCost() from pricing.ts for cost tracking
|
|
14
14
|
* - TelemetryService for metrics recording
|
|
15
15
|
*/
|
|
16
|
-
import { SpanStatusCode, context, metrics, trace, } from "@opentelemetry/api";
|
|
16
|
+
import { SpanStatusCode, SpanKind, ROOT_CONTEXT, isSpanContextValid, propagation, context, metrics, trace, } from "@opentelemetry/api";
|
|
17
17
|
import { getTracer, setLangfuseContext, } from "../services/server/ai/observability/instrumentation.js";
|
|
18
18
|
import { OtelBridge } from "../observability/otelBridge.js";
|
|
19
19
|
import { calculateCost } from "../utils/pricing.js";
|
|
20
20
|
import { TelemetryService } from "../telemetry/telemetryService.js";
|
|
21
21
|
import { logger } from "../utils/logger.js";
|
|
22
|
+
import { registerProxyRequestTraceContext, releaseProxyRequestTraceContext, } from "./proxyTraceContext.js";
|
|
22
23
|
const LOG_PREFIX = "[ProxyTracer]";
|
|
23
24
|
// ---------------------------------------------------------------------------
|
|
24
25
|
// OTEL Metric Instruments — lazy singleton
|
|
@@ -205,6 +206,8 @@ class ProxyTracer {
|
|
|
205
206
|
billingProvider;
|
|
206
207
|
startTime;
|
|
207
208
|
isStream;
|
|
209
|
+
ended = false;
|
|
210
|
+
recordRequestMetrics = true;
|
|
208
211
|
accountEmail;
|
|
209
212
|
usage;
|
|
210
213
|
mode = "full";
|
|
@@ -226,7 +229,7 @@ class ProxyTracer {
|
|
|
226
229
|
const tracer = getTracer("neurolink.proxy");
|
|
227
230
|
// Extract parent context from incoming headers (Claude Code may send traceparent)
|
|
228
231
|
let parentContext = context.active();
|
|
229
|
-
if (incomingHeaders) {
|
|
232
|
+
if (incomingHeaders && !trace.getSpan(context.active())) {
|
|
230
233
|
const bridge = new OtelBridge();
|
|
231
234
|
const extracted = bridge.extractContext(incomingHeaders);
|
|
232
235
|
if (extracted) {
|
|
@@ -270,6 +273,7 @@ class ProxyTracer {
|
|
|
270
273
|
rootSpan.setAttribute("neurolink.conversation_id", nlConversationId);
|
|
271
274
|
}
|
|
272
275
|
const instance = new ProxyTracer(rootSpan, ctx.requestId, ctx.model, ctx.stream, ctx.provider ?? "anthropic");
|
|
276
|
+
instance.recordRequestMetrics = ctx.recordRequestMetrics !== false;
|
|
273
277
|
// Set Langfuse context (fire-and-forget — non-blocking)
|
|
274
278
|
// Prefer NeuroLink session/user from calling SDK over Claude Code session
|
|
275
279
|
setLangfuseContext({
|
|
@@ -607,6 +611,7 @@ class ProxyTracer {
|
|
|
607
611
|
return {
|
|
608
612
|
traceId: spanCtx.traceId,
|
|
609
613
|
spanId: spanCtx.spanId,
|
|
614
|
+
traceFlags: spanCtx.traceFlags,
|
|
610
615
|
};
|
|
611
616
|
}
|
|
612
617
|
/** Return the captured usage (set by setUsage). */
|
|
@@ -618,6 +623,10 @@ class ProxyTracer {
|
|
|
618
623
|
// -------------------------------------------------------------------------
|
|
619
624
|
/** End the root span with final HTTP status and duration, and emit OTEL metrics. */
|
|
620
625
|
end(responseStatus, durationMs) {
|
|
626
|
+
if (this.ended) {
|
|
627
|
+
return;
|
|
628
|
+
}
|
|
629
|
+
this.ended = true;
|
|
621
630
|
this.rootSpan.setAttributes({
|
|
622
631
|
"http.status_code": responseStatus,
|
|
623
632
|
"proxy.duration_ms": durationMs,
|
|
@@ -636,6 +645,9 @@ class ProxyTracer {
|
|
|
636
645
|
this.rootSpan.setStatus({ code: SpanStatusCode.OK });
|
|
637
646
|
}
|
|
638
647
|
this.rootSpan.end();
|
|
648
|
+
if (!this.recordRequestMetrics) {
|
|
649
|
+
return;
|
|
650
|
+
}
|
|
639
651
|
// ---- Emit OTEL metrics (lazy-init instruments) ----
|
|
640
652
|
const m = getProxyMetrics();
|
|
641
653
|
const labels = {
|
|
@@ -745,3 +757,70 @@ export function recordFallbackAttempt(attrs) {
|
|
|
745
757
|
}
|
|
746
758
|
}
|
|
747
759
|
export { ProxyTracer };
|
|
760
|
+
/** Standard SERVER span covers every proxy door, including parsing and admission failures. */
|
|
761
|
+
export function startProxyHttpTrace(metadata, headers) {
|
|
762
|
+
try {
|
|
763
|
+
// Match OtelBridge's existing compatibility policy for HTTP-combined
|
|
764
|
+
// traceparent values: retain the first injected parent.
|
|
765
|
+
const normalizedHeaders = { ...headers };
|
|
766
|
+
if (normalizedHeaders.traceparent?.includes(",")) {
|
|
767
|
+
normalizedHeaders.traceparent = normalizedHeaders.traceparent
|
|
768
|
+
.split(",", 1)[0]
|
|
769
|
+
.trim();
|
|
770
|
+
}
|
|
771
|
+
const parent = propagation.extract(ROOT_CONTEXT, normalizedHeaders);
|
|
772
|
+
const span = getTracer("neurolink.proxy").startSpan("proxy.http.request", {
|
|
773
|
+
kind: SpanKind.SERVER,
|
|
774
|
+
attributes: {
|
|
775
|
+
"http.request.method": metadata.method,
|
|
776
|
+
"url.path": metadata.path,
|
|
777
|
+
"proxy.request_id": metadata.requestId,
|
|
778
|
+
},
|
|
779
|
+
}, parent);
|
|
780
|
+
const spanContext = span.spanContext();
|
|
781
|
+
if (isSpanContextValid(spanContext)) {
|
|
782
|
+
metadata.traceId = spanContext.traceId;
|
|
783
|
+
metadata.spanId = spanContext.spanId;
|
|
784
|
+
metadata.traceFlags = spanContext.traceFlags;
|
|
785
|
+
registerProxyRequestTraceContext(metadata.requestId, {
|
|
786
|
+
traceId: spanContext.traceId,
|
|
787
|
+
spanId: spanContext.spanId,
|
|
788
|
+
traceFlags: spanContext.traceFlags,
|
|
789
|
+
});
|
|
790
|
+
}
|
|
791
|
+
const active = trace.setSpan(parent, span);
|
|
792
|
+
let ended = false;
|
|
793
|
+
return {
|
|
794
|
+
run: (fn) => context.with(active, fn),
|
|
795
|
+
end: (status, outcome, errorType) => {
|
|
796
|
+
if (ended) {
|
|
797
|
+
return;
|
|
798
|
+
}
|
|
799
|
+
ended = true;
|
|
800
|
+
try {
|
|
801
|
+
span.setAttributes({
|
|
802
|
+
"http.response.status_code": status,
|
|
803
|
+
"proxy.terminal_outcome": outcome,
|
|
804
|
+
...(errorType ? { "error.type": errorType } : {}),
|
|
805
|
+
});
|
|
806
|
+
if (status >= 500 ||
|
|
807
|
+
outcome === "stream_error" ||
|
|
808
|
+
outcome === "unknown") {
|
|
809
|
+
span.setStatus({ code: SpanStatusCode.ERROR });
|
|
810
|
+
}
|
|
811
|
+
span.end();
|
|
812
|
+
}
|
|
813
|
+
catch {
|
|
814
|
+
// Telemetry must not interrupt transport cleanup.
|
|
815
|
+
}
|
|
816
|
+
finally {
|
|
817
|
+
releaseProxyRequestTraceContext(metadata.requestId);
|
|
818
|
+
}
|
|
819
|
+
},
|
|
820
|
+
};
|
|
821
|
+
}
|
|
822
|
+
catch {
|
|
823
|
+
releaseProxyRequestTraceContext(metadata.requestId);
|
|
824
|
+
return { run: (fn) => fn(), end: () => undefined };
|
|
825
|
+
}
|
|
826
|
+
}
|
|
@@ -1,5 +1,9 @@
|
|
|
1
1
|
/** Maximum bytes to capture before stopping accumulation (1 MB). */
|
|
2
2
|
const MAX_CAPTURE_BYTES = 1024 * 1024;
|
|
3
|
+
// Payload accounting across concurrent stream observers, independent of the
|
|
4
|
+
// worker's clone/publication pool. Never retain an unbounded number of prefixes.
|
|
5
|
+
const MAX_PENDING_CAPTURE_BYTES = 16 * 1024 * 1024;
|
|
6
|
+
let pendingCaptureBytes = 0;
|
|
3
7
|
const TRUNCATION_MARKER = "\n...[TRUNCATED]";
|
|
4
8
|
export function createRawStreamCapture() {
|
|
5
9
|
const decoder = new TextDecoder();
|
|
@@ -17,21 +21,13 @@ export function createRawStreamCapture() {
|
|
|
17
21
|
return;
|
|
18
22
|
}
|
|
19
23
|
resolved = true;
|
|
20
|
-
const
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
chunks.push(finalChunk.slice(0, remainingBytes));
|
|
25
|
-
capturedBytes += Math.min(finalChunk.length, remainingBytes);
|
|
26
|
-
}
|
|
27
|
-
else if (!truncated) {
|
|
28
|
-
chunks.push(TRUNCATION_MARKER);
|
|
29
|
-
truncated = true;
|
|
30
|
-
}
|
|
31
|
-
}
|
|
24
|
+
const text = chunks.map((chunk) => decoder.decode(chunk, { stream: true })).join("") +
|
|
25
|
+
(truncated ? "" : decoder.decode());
|
|
26
|
+
pendingCaptureBytes -= capturedBytes;
|
|
27
|
+
chunks.length = 0;
|
|
32
28
|
resolveCapture({
|
|
33
29
|
totalBytes,
|
|
34
|
-
text:
|
|
30
|
+
text: text + (truncated ? TRUNCATION_MARKER : ""),
|
|
35
31
|
truncated,
|
|
36
32
|
});
|
|
37
33
|
}
|
|
@@ -39,21 +35,16 @@ export function createRawStreamCapture() {
|
|
|
39
35
|
transform(chunk, controller) {
|
|
40
36
|
controller.enqueue(chunk);
|
|
41
37
|
totalBytes += chunk.byteLength;
|
|
42
|
-
if (capturedBytes < MAX_CAPTURE_BYTES) {
|
|
43
|
-
const
|
|
44
|
-
|
|
45
|
-
const slice =
|
|
38
|
+
if (!truncated && capturedBytes < MAX_CAPTURE_BYTES) {
|
|
39
|
+
const remainingBytes = Math.min(MAX_CAPTURE_BYTES - capturedBytes, Math.max(0, MAX_PENDING_CAPTURE_BYTES - pendingCaptureBytes));
|
|
40
|
+
// Buffer.slice is a view; copy to avoid retaining the whole wire chunk.
|
|
41
|
+
const slice = Uint8Array.from(chunk.subarray(0, remainingBytes));
|
|
46
42
|
chunks.push(slice);
|
|
47
|
-
capturedBytes +=
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
truncated = true;
|
|
51
|
-
}
|
|
52
|
-
}
|
|
53
|
-
else if (!truncated) {
|
|
54
|
-
chunks.push(TRUNCATION_MARKER);
|
|
55
|
-
truncated = true;
|
|
43
|
+
capturedBytes += slice.byteLength;
|
|
44
|
+
pendingCaptureBytes += slice.byteLength;
|
|
45
|
+
truncated = slice.byteLength < chunk.byteLength;
|
|
56
46
|
}
|
|
47
|
+
truncated ||= totalBytes > MAX_CAPTURE_BYTES;
|
|
57
48
|
},
|
|
58
49
|
flush() {
|
|
59
50
|
settle();
|
|
@@ -35,6 +35,8 @@ export declare function prepareProxyBodyForLogging(body: unknown): {
|
|
|
35
35
|
truncated: boolean;
|
|
36
36
|
originalBytes?: number;
|
|
37
37
|
};
|
|
38
|
+
/** Cheap gate for response observers; avoid copying diagnostics on the serving path. */
|
|
39
|
+
export declare function isProxyBodyCaptureEnabled(): boolean;
|
|
38
40
|
/** Capture an owned request body with bounded processing and tracked index/export publication. */
|
|
39
41
|
export declare function logBodyCapture(entry: ProxyBodyCaptureEntry): Promise<void>;
|
|
40
42
|
/**
|