@juspay/neurolink 12.14.6 → 12.14.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +2 -2
- package/dist/browser/neurolink.min.js +411 -411
- package/dist/cli/commands/proxy.js +83 -14
- package/dist/proxy/bodyCaptureProcessing.js +47 -3
- package/dist/proxy/bodyCaptureWorker.js +39 -10
- package/dist/proxy/claudeOutputObservation.d.ts +9 -0
- package/dist/proxy/claudeOutputObservation.js +85 -0
- package/dist/proxy/codexUsage.js +86 -3
- package/dist/proxy/otelLogSink.d.ts +13 -0
- package/dist/proxy/otelLogSink.js +63 -2
- package/dist/proxy/processTelemetry.d.ts +3 -0
- package/dist/proxy/processTelemetry.js +25 -0
- package/dist/proxy/proxyTraceContext.d.ts +21 -0
- package/dist/proxy/proxyTraceContext.js +47 -0
- package/dist/proxy/proxyTracer.d.ts +9 -5
- package/dist/proxy/proxyTracer.js +81 -2
- package/dist/proxy/rawStreamCapture.js +17 -26
- package/dist/proxy/requestLogger.d.ts +2 -0
- package/dist/proxy/requestLogger.js +29 -19
- package/dist/proxy/restartControl.d.ts +3 -1
- package/dist/proxy/restartControl.js +118 -1
- package/dist/proxy/sseInterceptor.js +71 -3
- package/dist/server/routes/claudeProxyRoutes.d.ts +0 -11
- package/dist/server/routes/claudeProxyRoutes.js +27 -12
- package/dist/server/routes/codexProxyRoutes.js +496 -333
- package/dist/types/cli.d.ts +7 -1
- package/dist/types/proxy.d.ts +133 -3
- package/dist/types/proxyRestart.d.ts +25 -0
- package/dist/utils/logSanitize.js +1 -1
- package/docs-site/static/search-index.json +1 -1
- package/package.json +1 -1
- package/scripts/observability/check-proxy-telemetry.mjs +29 -241
- package/scripts/observability/proxy-telemetry-backend.mjs +202 -0
- package/scripts/observability/proxy-telemetry-check.mjs +578 -0
- package/scripts/observability/query-proxy-history.mjs +30 -19
|
@@ -12,6 +12,7 @@ import { applyAllClients, restoreAllClients, } from "../proxy-clients/registry.j
|
|
|
12
12
|
import { resolveProxyConfigPath } from "../../proxy/proxyConfig.js";
|
|
13
13
|
import { redactUrlsInText, sanitizeForLog, } from "../../utils/logSanitize.js";
|
|
14
14
|
import { withTimeout } from "../../utils/async/withTimeout.js";
|
|
15
|
+
import { startProxyHttpTrace } from "../../proxy/proxyTracer.js";
|
|
15
16
|
import { formatUptime, isProcessRunning, StateFileManager, } from "../utils/serverUtils.js";
|
|
16
17
|
import { configureProxyKeepAliveDispatcher } from "../../proxy/proxyDispatcher.js";
|
|
17
18
|
import { ProxyRuntimeConfigStore } from "../../proxy/runtimeConfig.js";
|
|
@@ -24,7 +25,8 @@ import { describeInstallFailure, getGlobalInstallArgs, isTransientInstallFailure
|
|
|
24
25
|
import { startUpdaterWorkerSupervisor } from "../../proxy/updaterSupervisor.js";
|
|
25
26
|
import { openProxyWorkerLog } from "../../proxy/workerLog.js";
|
|
26
27
|
import { startRollingProxyServer } from "../../proxy/rollingProxyServer.js";
|
|
27
|
-
import { startProxyRestartControl } from "../../proxy/restartControl.js";
|
|
28
|
+
import { startProxyRestartControl, requestProxySupervisorTelemetry, } from "../../proxy/restartControl.js";
|
|
29
|
+
import { getProxyProcessTelemetry } from "../../proxy/processTelemetry.js";
|
|
28
30
|
import { isProxyAuxiliaryRequest } from "../../proxy/proxyRequestKind.js";
|
|
29
31
|
import { spawnProxySocketWorker } from "../../proxy/rollingWorkerProcess.js";
|
|
30
32
|
import { PROXY_ROLLING_SUPERVISOR_ENV, PROXY_SOCKET_WORKER_ENV, } from "../../proxy/rollingWorkerProtocol.js";
|
|
@@ -837,13 +839,36 @@ function spawnProxyUpdater(host, port, parentPid, rollingSupervisor = false) {
|
|
|
837
839
|
workerLog.close();
|
|
838
840
|
}
|
|
839
841
|
}
|
|
840
|
-
async function runProxyTelemetryManager(command) {
|
|
842
|
+
async function runProxyTelemetryManager(command, argv) {
|
|
841
843
|
const { existsSync } = await import("fs");
|
|
842
844
|
if (!existsSync(PROXY_TELEMETRY_SCRIPT_PATH)) {
|
|
843
845
|
throw new Error("Proxy telemetry helper files were not found in this installation. Reinstall NeuroLink with observability assets included.");
|
|
844
846
|
}
|
|
845
847
|
await new Promise((resolve, reject) => {
|
|
846
|
-
const
|
|
848
|
+
const queryCommand = command === "doctor" || command === "query";
|
|
849
|
+
const queryArgs = [];
|
|
850
|
+
if (queryCommand && argv) {
|
|
851
|
+
for (const [key, option] of [
|
|
852
|
+
["since", "--since"],
|
|
853
|
+
["until", "--until"],
|
|
854
|
+
["kind", "--kind"],
|
|
855
|
+
["format", "--format"],
|
|
856
|
+
["maxRows", "--max-rows"],
|
|
857
|
+
["proxyUrl", "--proxy-url"],
|
|
858
|
+
]) {
|
|
859
|
+
if (argv[key] !== undefined) {
|
|
860
|
+
queryArgs.push(option, String(argv[key]));
|
|
861
|
+
}
|
|
862
|
+
}
|
|
863
|
+
}
|
|
864
|
+
const child = spawn(queryCommand ? process.execPath : "bash", queryCommand
|
|
865
|
+
? [
|
|
866
|
+
join(dirname(PROXY_TELEMETRY_SCRIPT_PATH), command === "doctor"
|
|
867
|
+
? "check-proxy-telemetry.mjs"
|
|
868
|
+
: "query-proxy-history.mjs"),
|
|
869
|
+
...queryArgs,
|
|
870
|
+
]
|
|
871
|
+
: [PROXY_TELEMETRY_SCRIPT_PATH, command], {
|
|
847
872
|
stdio: "inherit",
|
|
848
873
|
env: process.env,
|
|
849
874
|
});
|
|
@@ -1078,6 +1103,7 @@ function registerProxyRequestTracking(app, requestMetadata, readiness) {
|
|
|
1078
1103
|
rejectForUpdate: readiness.drainingForUpdate,
|
|
1079
1104
|
};
|
|
1080
1105
|
requestMetadata.set(c.req.raw, metadata);
|
|
1106
|
+
const httpTrace = startProxyHttpTrace(metadata, Object.fromEntries(c.req.raw.headers));
|
|
1081
1107
|
const stopObservingFinalLog = observeProxyFinalLog(metadata.requestId, (entry) => {
|
|
1082
1108
|
metadata.terminalResult = entry;
|
|
1083
1109
|
}, (entry) => {
|
|
@@ -1087,6 +1113,7 @@ function registerProxyRequestTracking(app, requestMetadata, readiness) {
|
|
|
1087
1113
|
? () => undefined
|
|
1088
1114
|
: beginProxyRequest();
|
|
1089
1115
|
const finish = () => {
|
|
1116
|
+
httpTrace.end(metadata.terminalResult?.responseStatus ?? c.res.status, metadata.terminalResult?.terminalOutcome ?? "unknown", metadata.terminalErrorType);
|
|
1090
1117
|
stopObservingFinalLog();
|
|
1091
1118
|
finishActivity();
|
|
1092
1119
|
// Borrowed traffic holds a concurrency slot for the lifetime of the
|
|
@@ -1099,7 +1126,7 @@ function registerProxyRequestTracking(app, requestMetadata, readiness) {
|
|
|
1099
1126
|
// them at acceptance instead of publishing misleading placeholder values;
|
|
1100
1127
|
// subsequent events carry the parsed metadata under the same request ID.
|
|
1101
1128
|
try {
|
|
1102
|
-
await persistProxyLifecycleAcceptance({
|
|
1129
|
+
await httpTrace.run(() => persistProxyLifecycleAcceptance({
|
|
1103
1130
|
requestId: metadata.requestId,
|
|
1104
1131
|
method: metadata.method,
|
|
1105
1132
|
path: metadata.path,
|
|
@@ -1107,8 +1134,8 @@ function registerProxyRequestTracking(app, requestMetadata, readiness) {
|
|
|
1107
1134
|
requestBytes,
|
|
1108
1135
|
elapsedMs: 0,
|
|
1109
1136
|
monotonicMs: startedMonotonicMs,
|
|
1110
|
-
});
|
|
1111
|
-
await next
|
|
1137
|
+
}));
|
|
1138
|
+
await httpTrace.run(next);
|
|
1112
1139
|
const responseStatus = c.res.status;
|
|
1113
1140
|
logProxyLifecycleEvent({
|
|
1114
1141
|
event: "response_headers",
|
|
@@ -1169,12 +1196,12 @@ function registerProxyRequestTracking(app, requestMetadata, readiness) {
|
|
|
1169
1196
|
let accountingTimedOut = false;
|
|
1170
1197
|
let accountingFailed = false;
|
|
1171
1198
|
try {
|
|
1172
|
-
accountingFailed = await notifyRouteTerminal({
|
|
1199
|
+
accountingFailed = await httpTrace.run(() => notifyRouteTerminal({
|
|
1173
1200
|
outcome,
|
|
1174
1201
|
error,
|
|
1175
1202
|
observedBodyBytes,
|
|
1176
1203
|
responseChunks,
|
|
1177
|
-
});
|
|
1204
|
+
}));
|
|
1178
1205
|
}
|
|
1179
1206
|
catch {
|
|
1180
1207
|
accountingTimedOut = true;
|
|
@@ -1231,6 +1258,7 @@ function registerProxyRequestTracking(app, requestMetadata, readiness) {
|
|
|
1231
1258
|
errorCode: final?.errorCode ?? metadata.terminalErrorCode,
|
|
1232
1259
|
});
|
|
1233
1260
|
stopObservingFinalLog();
|
|
1261
|
+
httpTrace.end(final?.responseStatus ?? responseStatus, terminalOutcome, final?.errorType ?? metadata.terminalErrorType);
|
|
1234
1262
|
},
|
|
1235
1263
|
});
|
|
1236
1264
|
}
|
|
@@ -1259,6 +1287,9 @@ function registerProxyRequestTracking(app, requestMetadata, readiness) {
|
|
|
1259
1287
|
errorType: error instanceof Error ? error.name : "unknown_error",
|
|
1260
1288
|
errorCode: getProxyRuntimeErrorCode(error),
|
|
1261
1289
|
});
|
|
1290
|
+
httpTrace.end(getProxyRuntimeErrorCode(error) === "PROXY_TELEMETRY_UNAVAILABLE"
|
|
1291
|
+
? 503
|
|
1292
|
+
: 502, "handler_error", error instanceof Error ? error.name : "unknown_error");
|
|
1262
1293
|
throw error;
|
|
1263
1294
|
}
|
|
1264
1295
|
};
|
|
@@ -1331,8 +1362,9 @@ export async function createProxyStartApp(params) {
|
|
|
1331
1362
|
accountKey: attempt?.accountKey,
|
|
1332
1363
|
provider: attempt?.provider,
|
|
1333
1364
|
transportScope: attempt?.transportScope,
|
|
1334
|
-
traceId: attempt?.traceId,
|
|
1335
|
-
spanId: attempt?.spanId,
|
|
1365
|
+
traceId: attempt?.traceId ?? metadata.traceId,
|
|
1366
|
+
spanId: attempt?.spanId ?? metadata.spanId,
|
|
1367
|
+
traceFlags: attempt?.traceFlags ?? metadata.traceFlags,
|
|
1336
1368
|
responseStatus: status,
|
|
1337
1369
|
responseTimeMs: Date.now() - metadata.startedAt,
|
|
1338
1370
|
errorType,
|
|
@@ -1342,6 +1374,9 @@ export async function createProxyStartApp(params) {
|
|
|
1342
1374
|
logBodyCapture({
|
|
1343
1375
|
timestamp: new Date().toISOString(),
|
|
1344
1376
|
requestId: metadata.requestId,
|
|
1377
|
+
traceId: attempt?.traceId ?? metadata.traceId,
|
|
1378
|
+
spanId: attempt?.spanId ?? metadata.spanId,
|
|
1379
|
+
traceFlags: attempt?.traceFlags ?? metadata.traceFlags,
|
|
1345
1380
|
model: metadata.model,
|
|
1346
1381
|
stream: metadata.stream,
|
|
1347
1382
|
phase: "client_response",
|
|
@@ -2042,6 +2077,10 @@ export async function createProxyStartApp(params) {
|
|
|
2042
2077
|
};
|
|
2043
2078
|
})(),
|
|
2044
2079
|
observability: {
|
|
2080
|
+
process: getProxyProcessTelemetry(),
|
|
2081
|
+
supervisor: supervisorState?.pid
|
|
2082
|
+
? await requestProxySupervisorTelemetry(supervisorState.restartControl, supervisorState.pid)
|
|
2083
|
+
: { status: "not_applicable" },
|
|
2045
2084
|
lifecycle: getProxyLifecycleLoggerSnapshot(),
|
|
2046
2085
|
requestLogs: getRequestLoggerSnapshot(),
|
|
2047
2086
|
},
|
|
@@ -2859,6 +2898,7 @@ async function runLaunchdProxySupervisor(argv, spinner) {
|
|
|
2859
2898
|
restartControl = await startProxyRestartControl({
|
|
2860
2899
|
stateDir: join(homedir(), ".neurolink"),
|
|
2861
2900
|
server: rollingServer,
|
|
2901
|
+
getTelemetry: getProxyProcessTelemetry,
|
|
2862
2902
|
log: (message) => logger.warn(message),
|
|
2863
2903
|
isUpdatePending: () => !!rollingReplacement || !!loadUpdateState()?.pendingRestartVersion,
|
|
2864
2904
|
getInstalledVersion: async () => {
|
|
@@ -3553,6 +3593,8 @@ export const proxyStatusCommand = {
|
|
|
3553
3593
|
// PROXY TELEMETRY COMMAND
|
|
3554
3594
|
// =============================================================================
|
|
3555
3595
|
const PROXY_TELEMETRY_ACTIONS = [
|
|
3596
|
+
"doctor",
|
|
3597
|
+
"query",
|
|
3556
3598
|
"setup",
|
|
3557
3599
|
"start",
|
|
3558
3600
|
"stop",
|
|
@@ -3562,12 +3604,37 @@ const PROXY_TELEMETRY_ACTIONS = [
|
|
|
3562
3604
|
];
|
|
3563
3605
|
export const proxyTelemetryCommand = {
|
|
3564
3606
|
command: "telemetry <action>",
|
|
3565
|
-
describe: "
|
|
3607
|
+
describe: "Query and verify stored OTel telemetry or manage the local observability stack",
|
|
3566
3608
|
builder: (yargs) => yargs
|
|
3567
3609
|
.positional("action", {
|
|
3568
3610
|
type: "string",
|
|
3569
3611
|
choices: [...PROXY_TELEMETRY_ACTIONS],
|
|
3570
|
-
describe: "Telemetry action: setup, start, stop, status, logs, or import-dashboard",
|
|
3612
|
+
describe: "Telemetry action: doctor, query, setup, start, stop, status, logs, or import-dashboard",
|
|
3613
|
+
})
|
|
3614
|
+
.option("since", {
|
|
3615
|
+
type: "string",
|
|
3616
|
+
description: "Inclusive ISO timestamp for stored telemetry",
|
|
3617
|
+
})
|
|
3618
|
+
.option("until", {
|
|
3619
|
+
type: "string",
|
|
3620
|
+
description: "Exclusive ISO timestamp for stored telemetry",
|
|
3621
|
+
})
|
|
3622
|
+
.option("kind", {
|
|
3623
|
+
type: "string",
|
|
3624
|
+
description: "Metadata kind for query, such as request_final or telemetry_delivery",
|
|
3625
|
+
})
|
|
3626
|
+
.option("format", {
|
|
3627
|
+
type: "string",
|
|
3628
|
+
choices: ["json", "text"],
|
|
3629
|
+
description: "Doctor report format; query always returns JSON",
|
|
3630
|
+
})
|
|
3631
|
+
.option("max-rows", {
|
|
3632
|
+
type: "number",
|
|
3633
|
+
description: "Explicit bounded history record limit",
|
|
3634
|
+
})
|
|
3635
|
+
.option("proxy-url", {
|
|
3636
|
+
type: "string",
|
|
3637
|
+
description: "Proxy endpoint used by the read-only doctor",
|
|
3571
3638
|
})
|
|
3572
3639
|
.option("quiet", {
|
|
3573
3640
|
type: "boolean",
|
|
@@ -3575,19 +3642,21 @@ export const proxyTelemetryCommand = {
|
|
|
3575
3642
|
default: false,
|
|
3576
3643
|
description: "Suppress the local CLI spinner and delegate directly",
|
|
3577
3644
|
})
|
|
3645
|
+
.example("neurolink proxy telemetry doctor --format json", "Verify field coverage, stored captures, trace correlation, and delivery evidence")
|
|
3646
|
+
.example("neurolink proxy telemetry query --since 2026-09-15T00:00:00Z --kind request_final", "Read stored OTLP metadata without scanning proxy log files")
|
|
3578
3647
|
.example("neurolink proxy telemetry setup", "Start OpenObserve, start the OTEL collector, and import the dashboard")
|
|
3579
3648
|
.example("neurolink proxy telemetry start", "Start the local proxy telemetry stack without re-importing the dashboard")
|
|
3580
3649
|
.example("neurolink proxy telemetry stop", "Stop the local OpenObserve and OTEL collector containers"),
|
|
3581
3650
|
handler: async (argv) => {
|
|
3582
3651
|
const action = argv.action;
|
|
3583
|
-
const spinner = argv.quiet
|
|
3652
|
+
const spinner = argv.quiet || action === "doctor" || action === "query"
|
|
3584
3653
|
? null
|
|
3585
3654
|
: ora(`Running proxy telemetry ${action}...`).start();
|
|
3586
3655
|
try {
|
|
3587
3656
|
if (spinner) {
|
|
3588
3657
|
spinner.stop();
|
|
3589
3658
|
}
|
|
3590
|
-
await runProxyTelemetryManager(action);
|
|
3659
|
+
await runProxyTelemetryManager(action, argv);
|
|
3591
3660
|
if (spinner) {
|
|
3592
3661
|
spinner.succeed(`proxy telemetry ${action} completed`);
|
|
3593
3662
|
}
|
|
@@ -2,6 +2,7 @@ import { join } from "node:path";
|
|
|
2
2
|
import { mkdir, chmod, writeFile } from "node:fs/promises";
|
|
3
3
|
import { createHash, randomUUID } from "node:crypto";
|
|
4
4
|
import { promisify } from "node:util";
|
|
5
|
+
import { sanitizeForLog } from "../utils/logSanitize.js";
|
|
5
6
|
import { gzip as gzipCallback } from "node:zlib";
|
|
6
7
|
const REQUEST_LOG_IO_TIMEOUT_MS = 5_000;
|
|
7
8
|
/** Maximum redacted body bytes persisted per capture entry. */
|
|
@@ -43,6 +44,15 @@ function redactHeaders(headers) {
|
|
|
43
44
|
return redacted;
|
|
44
45
|
}
|
|
45
46
|
const SENSITIVE_BODY_KEY = /^(?:password|access_token|refresh_token|api_key|apiKey|secret|authorization|token|credential|x-api-key)$/i;
|
|
47
|
+
/** Error messages may contain nested JSON encoded as a string. */
|
|
48
|
+
function redactNestedValue(key, value) {
|
|
49
|
+
if (SENSITIVE_BODY_KEY.test(key)) {
|
|
50
|
+
return "[REDACTED]";
|
|
51
|
+
}
|
|
52
|
+
return typeof value === "string"
|
|
53
|
+
? sanitizeForLog(value.replace(SENSITIVE_BODY_KEYS, '$1"[REDACTED]"'), value.length)
|
|
54
|
+
: value;
|
|
55
|
+
}
|
|
46
56
|
/** Redact every value under a sensitive key, including objects and arrays. */
|
|
47
57
|
function redactBody(body) {
|
|
48
58
|
if (body === undefined || body === null) {
|
|
@@ -54,11 +64,44 @@ function redactBody(body) {
|
|
|
54
64
|
value = JSON.parse(body);
|
|
55
65
|
}
|
|
56
66
|
catch {
|
|
57
|
-
|
|
58
|
-
|
|
67
|
+
if (/^(?:event|data):/m.test(body)) {
|
|
68
|
+
// Redact whole SSE data values, including nested credentials. Never
|
|
69
|
+
// retain a truncated/invalid JSON tail whose secret cannot be parsed.
|
|
70
|
+
return body
|
|
71
|
+
.split(/\r\n\r\n|\n\n|\r\r/)
|
|
72
|
+
.map((frame) => {
|
|
73
|
+
const lines = frame.split(/\r\n|\n|\r/);
|
|
74
|
+
const data = lines
|
|
75
|
+
.filter((line) => line.startsWith("data:"))
|
|
76
|
+
.map((line) => line.slice(5).trimStart())
|
|
77
|
+
.join("\n");
|
|
78
|
+
if (!data || data === "[DONE]") {
|
|
79
|
+
return frame;
|
|
80
|
+
}
|
|
81
|
+
let redacted;
|
|
82
|
+
try {
|
|
83
|
+
redacted = JSON.stringify(JSON.parse(data), redactNestedValue);
|
|
84
|
+
}
|
|
85
|
+
catch {
|
|
86
|
+
redacted = "[UNPARSEABLE DATA REDACTED]";
|
|
87
|
+
}
|
|
88
|
+
return [
|
|
89
|
+
...lines.filter((line) => !line.startsWith("data:")),
|
|
90
|
+
`data: ${redacted}`,
|
|
91
|
+
].join("\n");
|
|
92
|
+
})
|
|
93
|
+
.join("\n\n");
|
|
94
|
+
}
|
|
95
|
+
// Invalid or truncated structured JSON must not expose an unterminated
|
|
96
|
+
// credential value that a quoted-string regex cannot safely redact.
|
|
97
|
+
if (/^\s*[[{]/.test(body)) {
|
|
98
|
+
return "[UNPARSEABLE DATA REDACTED]";
|
|
99
|
+
}
|
|
100
|
+
// Plain-text bodies retain the existing credential-field redaction.
|
|
101
|
+
return sanitizeForLog(body.replace(SENSITIVE_BODY_KEYS, '$1"[REDACTED]"'), body.length);
|
|
59
102
|
}
|
|
60
103
|
}
|
|
61
|
-
return JSON.stringify(value,
|
|
104
|
+
return JSON.stringify(value, redactNestedValue);
|
|
62
105
|
}
|
|
63
106
|
/**
|
|
64
107
|
* Restrict request and phase identifiers to characters safe for artifact
|
|
@@ -208,6 +251,7 @@ export async function processProxyBodyCapture(entry, logDir) {
|
|
|
208
251
|
const headers = redactHeaders(entry.headers);
|
|
209
252
|
const limit = logDir === null ? MAX_OTEL_CAPTURED_BODY_BYTES : MAX_CAPTURED_BODY_BYTES;
|
|
210
253
|
const prepared = prepareRedactedBody(entry.body, limit);
|
|
254
|
+
prepared.truncated ||= entry.sourceTruncated === true;
|
|
211
255
|
if (logDir === null) {
|
|
212
256
|
return {
|
|
213
257
|
headers,
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { Worker } from "node:worker_threads";
|
|
2
|
-
const MAX_PENDING =
|
|
2
|
+
const MAX_PENDING = 64;
|
|
3
3
|
const MAX_PENDING_BYTES = 32 * 1024 * 1024;
|
|
4
4
|
// Bound retained UTF-16 strings rather than a 3x UTF-8 guess. This accommodates
|
|
5
5
|
// the observed 7.3 MB JSON requests while retaining a 32 MiB aggregate pool.
|
|
@@ -18,6 +18,8 @@ const snapshot = {
|
|
|
18
18
|
pendingBytes: 0,
|
|
19
19
|
maxPending: MAX_PENDING,
|
|
20
20
|
maxPendingBytes: MAX_PENDING_BYTES,
|
|
21
|
+
highWaterPending: 0,
|
|
22
|
+
highWaterBytes: 0,
|
|
21
23
|
rejectionReasons: {},
|
|
22
24
|
};
|
|
23
25
|
const pending = new Map();
|
|
@@ -27,7 +29,7 @@ const pending = new Map();
|
|
|
27
29
|
* Conservatively bound clone size and traversal work without invoking
|
|
28
30
|
* getters or serializers.
|
|
29
31
|
*/
|
|
30
|
-
function estimateCloneBytes(value) {
|
|
32
|
+
function estimateCloneBytes(value, maxEntryBytes) {
|
|
31
33
|
const stack = [value];
|
|
32
34
|
const seen = new Set();
|
|
33
35
|
let bytes = 0, nodes = 0;
|
|
@@ -35,8 +37,8 @@ function estimateCloneBytes(value) {
|
|
|
35
37
|
if (++nodes > 100_000) {
|
|
36
38
|
throw new Error("body_capture_traversal_limit");
|
|
37
39
|
}
|
|
38
|
-
if (bytes >
|
|
39
|
-
|
|
40
|
+
if (bytes > maxEntryBytes) {
|
|
41
|
+
return bytes;
|
|
40
42
|
}
|
|
41
43
|
const item = stack.pop();
|
|
42
44
|
if (typeof item === "string") {
|
|
@@ -66,8 +68,8 @@ function estimateCloneBytes(value) {
|
|
|
66
68
|
if (stack.length > 100_000) {
|
|
67
69
|
throw new Error("body_capture_traversal_limit");
|
|
68
70
|
}
|
|
69
|
-
if (bytes >
|
|
70
|
-
|
|
71
|
+
if (bytes > maxEntryBytes) {
|
|
72
|
+
return bytes;
|
|
71
73
|
}
|
|
72
74
|
}
|
|
73
75
|
}
|
|
@@ -133,10 +135,13 @@ function getWorker() {
|
|
|
133
135
|
/** Bounded bulk capture. Failures are indexed; never fall back to blocking work. */
|
|
134
136
|
export async function captureProxyBody(entry, logDir, consume) {
|
|
135
137
|
snapshot.attempted += 1;
|
|
138
|
+
// An OTel capture can use the existing byte pool when it is otherwise idle.
|
|
139
|
+
// Raising the slot count does not raise the aggregate retained-memory bound.
|
|
140
|
+
const maxEntryBytes = logDir === null ? MAX_PENDING_BYTES : MAX_ENTRY_BYTES;
|
|
136
141
|
let bytes;
|
|
137
142
|
let admissionError;
|
|
138
143
|
try {
|
|
139
|
-
bytes = estimateCloneBytes(entry);
|
|
144
|
+
bytes = estimateCloneBytes(entry, maxEntryBytes);
|
|
140
145
|
}
|
|
141
146
|
catch (error) {
|
|
142
147
|
bytes = Infinity;
|
|
@@ -150,20 +155,39 @@ export async function captureProxyBody(entry, logDir, consume) {
|
|
|
150
155
|
? error.message
|
|
151
156
|
: "body_capture_unsupported_value";
|
|
152
157
|
}
|
|
153
|
-
if (bytes >
|
|
158
|
+
if (bytes > maxEntryBytes ||
|
|
154
159
|
snapshot.pending >= MAX_PENDING ||
|
|
155
160
|
snapshot.pendingBytes + bytes > MAX_PENDING_BYTES ||
|
|
156
161
|
Date.now() < retryAfter) {
|
|
157
162
|
snapshot.rejected += 1;
|
|
158
|
-
const error = bytes >
|
|
163
|
+
const error = bytes > maxEntryBytes
|
|
159
164
|
? (admissionError ?? "body_capture_entry_too_large")
|
|
160
165
|
: Date.now() < retryAfter
|
|
161
166
|
? "body_worker_backoff"
|
|
162
167
|
: "body_capture_queue_full";
|
|
163
168
|
snapshot.lastError = error;
|
|
169
|
+
snapshot.lastRejectedAt = new Date().toISOString();
|
|
164
170
|
snapshot.rejectionReasons[error] =
|
|
165
171
|
(snapshot.rejectionReasons[error] ?? 0) + 1;
|
|
166
|
-
return consume({
|
|
172
|
+
return consume({
|
|
173
|
+
error,
|
|
174
|
+
stored: { bodyWriteFailed: true },
|
|
175
|
+
admission: {
|
|
176
|
+
limitingResource: bytes > maxEntryBytes
|
|
177
|
+
? "entry"
|
|
178
|
+
: Date.now() < retryAfter
|
|
179
|
+
? "worker"
|
|
180
|
+
: snapshot.pending >= MAX_PENDING
|
|
181
|
+
? "captures"
|
|
182
|
+
: "bytes",
|
|
183
|
+
estimatedBytes: Number.isFinite(bytes) ? bytes : undefined,
|
|
184
|
+
maxEntryBytes,
|
|
185
|
+
pending: snapshot.pending,
|
|
186
|
+
pendingBytes: snapshot.pendingBytes,
|
|
187
|
+
maxPending: MAX_PENDING,
|
|
188
|
+
maxPendingBytes: MAX_PENDING_BYTES,
|
|
189
|
+
},
|
|
190
|
+
});
|
|
167
191
|
}
|
|
168
192
|
let current;
|
|
169
193
|
try {
|
|
@@ -207,7 +231,9 @@ export async function captureProxyBody(entry, logDir, consume) {
|
|
|
207
231
|
},
|
|
208
232
|
});
|
|
209
233
|
snapshot.pending += 1;
|
|
234
|
+
snapshot.highWaterPending = Math.max(snapshot.highWaterPending, snapshot.pending);
|
|
210
235
|
snapshot.pendingBytes += bytes;
|
|
236
|
+
snapshot.highWaterBytes = Math.max(snapshot.highWaterBytes, snapshot.pendingBytes);
|
|
211
237
|
current.ref();
|
|
212
238
|
try {
|
|
213
239
|
current.postMessage({ id, entry, logDir, queuedAt: Date.now() });
|
|
@@ -248,6 +274,9 @@ export const __bodyCaptureWorkerTestHooks = {
|
|
|
248
274
|
pending: 0,
|
|
249
275
|
pendingBytes: 0,
|
|
250
276
|
lastError: undefined,
|
|
277
|
+
lastRejectedAt: undefined,
|
|
278
|
+
highWaterPending: 0,
|
|
279
|
+
highWaterBytes: 0,
|
|
251
280
|
rejectionReasons: {},
|
|
252
281
|
});
|
|
253
282
|
},
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { RequestLogEntry, SSETelemetry } from "../types/index.js";
|
|
2
|
+
/** Content visible to the caller; thinking and empty text are not useful output. */
|
|
3
|
+
export declare function hasUsefulClaudeContent(value: unknown): boolean;
|
|
4
|
+
/** A complete JSON response becomes observable when its body has been read. */
|
|
5
|
+
export declare function observeClaudeJsonOutput(metadata: Record<string, unknown>, value: unknown, at?: number): void;
|
|
6
|
+
/** Keep unavailable observations distinct from a complete stream with no output. */
|
|
7
|
+
export declare function observeClaudeStreamOutput(metadata: Record<string, unknown>, data: SSETelemetry): void;
|
|
8
|
+
/** Final records always explain whether useful-output latency was observed. */
|
|
9
|
+
export declare function claudeOutputLogFields(metadata: Record<string, unknown>, requestStartTime: number): Pick<RequestLogEntry, "firstUsefulOutputMs" | "firstUsefulOutputStatus" | "firstUsefulOutputEvent" | "firstUsefulOutputUnavailableReason">;
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
/** Content visible to the caller; thinking and empty text are not useful output. */
|
|
2
|
+
export function hasUsefulClaudeContent(value) {
|
|
3
|
+
if (!value || typeof value !== "object") {
|
|
4
|
+
return false;
|
|
5
|
+
}
|
|
6
|
+
const block = value;
|
|
7
|
+
if (Array.isArray(block.content)) {
|
|
8
|
+
return block.content.some(hasUsefulClaudeContent);
|
|
9
|
+
}
|
|
10
|
+
if (block.type === "thinking" || block.type === "redacted_thinking") {
|
|
11
|
+
return false;
|
|
12
|
+
}
|
|
13
|
+
if (typeof block.text === "string" && block.text.trim().length > 0) {
|
|
14
|
+
return true;
|
|
15
|
+
}
|
|
16
|
+
if (typeof block.refusal === "string" && block.refusal.trim().length > 0) {
|
|
17
|
+
return true;
|
|
18
|
+
}
|
|
19
|
+
return ((block.type === "tool_use" || block.type === "server_tool_use") &&
|
|
20
|
+
typeof block.name === "string" &&
|
|
21
|
+
block.name.length > 0 &&
|
|
22
|
+
block.input !== undefined);
|
|
23
|
+
}
|
|
24
|
+
/** A complete JSON response becomes observable when its body has been read. */
|
|
25
|
+
export function observeClaudeJsonOutput(metadata, value, at = Date.now()) {
|
|
26
|
+
if (!value ||
|
|
27
|
+
typeof value !== "object" ||
|
|
28
|
+
!Array.isArray(value.content)) {
|
|
29
|
+
metadata.firstUsefulOutputStatus = "not_observed";
|
|
30
|
+
metadata.firstUsefulOutputUnavailableReason = "unrecognized_response";
|
|
31
|
+
return;
|
|
32
|
+
}
|
|
33
|
+
if (hasUsefulClaudeContent(value)) {
|
|
34
|
+
metadata.firstUsefulOutputAt = at;
|
|
35
|
+
metadata.firstUsefulOutputStatus = "observed";
|
|
36
|
+
metadata.firstUsefulOutputEvent = "json_response.content";
|
|
37
|
+
}
|
|
38
|
+
else {
|
|
39
|
+
metadata.firstUsefulOutputStatus = "no_useful_output";
|
|
40
|
+
delete metadata.firstUsefulOutputAt;
|
|
41
|
+
delete metadata.firstUsefulOutputEvent;
|
|
42
|
+
}
|
|
43
|
+
delete metadata.firstUsefulOutputUnavailableReason;
|
|
44
|
+
}
|
|
45
|
+
/** Keep unavailable observations distinct from a complete stream with no output. */
|
|
46
|
+
export function observeClaudeStreamOutput(metadata, data) {
|
|
47
|
+
delete metadata.firstUsefulOutputAt;
|
|
48
|
+
delete metadata.firstUsefulOutputEvent;
|
|
49
|
+
delete metadata.firstUsefulOutputUnavailableReason;
|
|
50
|
+
if (!data.observationIncomplete && data.firstUsefulOutputAt !== undefined) {
|
|
51
|
+
metadata.firstUsefulOutputAt = data.firstUsefulOutputAt;
|
|
52
|
+
metadata.firstUsefulOutputEvent = data.firstUsefulOutputEvent;
|
|
53
|
+
metadata.firstUsefulOutputStatus = "observed";
|
|
54
|
+
}
|
|
55
|
+
else if (!data.observationIncomplete && data.messageStopReceived) {
|
|
56
|
+
metadata.firstUsefulOutputStatus = "no_useful_output";
|
|
57
|
+
}
|
|
58
|
+
else {
|
|
59
|
+
metadata.firstUsefulOutputStatus = "not_observed";
|
|
60
|
+
metadata.firstUsefulOutputUnavailableReason = data.observationIncomplete
|
|
61
|
+
? "incomplete_observation"
|
|
62
|
+
: "stream_incomplete";
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
/** Final records always explain whether useful-output latency was observed. */
|
|
66
|
+
export function claudeOutputLogFields(metadata, requestStartTime) {
|
|
67
|
+
const at = metadata.firstUsefulOutputAt;
|
|
68
|
+
if (typeof at === "number" && Number.isFinite(at)) {
|
|
69
|
+
return {
|
|
70
|
+
firstUsefulOutputMs: Math.max(0, at - requestStartTime),
|
|
71
|
+
firstUsefulOutputStatus: "observed",
|
|
72
|
+
firstUsefulOutputEvent: typeof metadata.firstUsefulOutputEvent === "string"
|
|
73
|
+
? metadata.firstUsefulOutputEvent
|
|
74
|
+
: "translated_output",
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
return metadata.firstUsefulOutputStatus === "no_useful_output"
|
|
78
|
+
? { firstUsefulOutputStatus: "no_useful_output" }
|
|
79
|
+
: {
|
|
80
|
+
firstUsefulOutputStatus: "not_observed",
|
|
81
|
+
firstUsefulOutputUnavailableReason: typeof metadata.firstUsefulOutputUnavailableReason === "string"
|
|
82
|
+
? metadata.firstUsefulOutputUnavailableReason
|
|
83
|
+
: "response_not_observed",
|
|
84
|
+
};
|
|
85
|
+
}
|