@juspay/neurolink 12.14.7 → 12.14.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +2 -2
- package/dist/browser/neurolink.min.js +357 -357
- package/dist/cli/commands/proxy.js +7 -1
- package/dist/proxy/bodyCaptureProcessing.js +47 -3
- package/dist/proxy/bodyCaptureWorker.js +14 -10
- package/dist/proxy/claudeOutputObservation.d.ts +9 -0
- package/dist/proxy/claudeOutputObservation.js +85 -0
- package/dist/proxy/codexUsage.js +12 -10
- package/dist/proxy/otelLogSink.js +1 -1
- package/dist/proxy/processTelemetry.d.ts +3 -0
- package/dist/proxy/processTelemetry.js +25 -0
- package/dist/proxy/rawStreamCapture.js +17 -26
- package/dist/proxy/requestLogger.d.ts +2 -0
- package/dist/proxy/requestLogger.js +5 -1
- package/dist/proxy/restartControl.d.ts +3 -1
- package/dist/proxy/restartControl.js +118 -1
- package/dist/proxy/sseInterceptor.js +71 -3
- package/dist/server/routes/claudeProxyRoutes.d.ts +0 -11
- package/dist/server/routes/claudeProxyRoutes.js +27 -12
- package/dist/server/routes/codexProxyRoutes.js +83 -2
- package/dist/types/proxy.d.ts +14 -0
- package/dist/types/proxyRestart.d.ts +25 -0
- package/dist/utils/logSanitize.js +1 -1
- package/docs-site/static/search-index.json +1 -1
- package/package.json +1 -1
- package/scripts/observability/proxy-telemetry-check.mjs +39 -5
|
@@ -25,7 +25,8 @@ import { describeInstallFailure, getGlobalInstallArgs, isTransientInstallFailure
|
|
|
25
25
|
import { startUpdaterWorkerSupervisor } from "../../proxy/updaterSupervisor.js";
|
|
26
26
|
import { openProxyWorkerLog } from "../../proxy/workerLog.js";
|
|
27
27
|
import { startRollingProxyServer } from "../../proxy/rollingProxyServer.js";
|
|
28
|
-
import { startProxyRestartControl } from "../../proxy/restartControl.js";
|
|
28
|
+
import { startProxyRestartControl, requestProxySupervisorTelemetry, } from "../../proxy/restartControl.js";
|
|
29
|
+
import { getProxyProcessTelemetry } from "../../proxy/processTelemetry.js";
|
|
29
30
|
import { isProxyAuxiliaryRequest } from "../../proxy/proxyRequestKind.js";
|
|
30
31
|
import { spawnProxySocketWorker } from "../../proxy/rollingWorkerProcess.js";
|
|
31
32
|
import { PROXY_ROLLING_SUPERVISOR_ENV, PROXY_SOCKET_WORKER_ENV, } from "../../proxy/rollingWorkerProtocol.js";
|
|
@@ -2076,6 +2077,10 @@ export async function createProxyStartApp(params) {
|
|
|
2076
2077
|
};
|
|
2077
2078
|
})(),
|
|
2078
2079
|
observability: {
|
|
2080
|
+
process: getProxyProcessTelemetry(),
|
|
2081
|
+
supervisor: supervisorState?.pid
|
|
2082
|
+
? await requestProxySupervisorTelemetry(supervisorState.restartControl, supervisorState.pid)
|
|
2083
|
+
: { status: "not_applicable" },
|
|
2079
2084
|
lifecycle: getProxyLifecycleLoggerSnapshot(),
|
|
2080
2085
|
requestLogs: getRequestLoggerSnapshot(),
|
|
2081
2086
|
},
|
|
@@ -2893,6 +2898,7 @@ async function runLaunchdProxySupervisor(argv, spinner) {
|
|
|
2893
2898
|
restartControl = await startProxyRestartControl({
|
|
2894
2899
|
stateDir: join(homedir(), ".neurolink"),
|
|
2895
2900
|
server: rollingServer,
|
|
2901
|
+
getTelemetry: getProxyProcessTelemetry,
|
|
2896
2902
|
log: (message) => logger.warn(message),
|
|
2897
2903
|
isUpdatePending: () => !!rollingReplacement || !!loadUpdateState()?.pendingRestartVersion,
|
|
2898
2904
|
getInstalledVersion: async () => {
|
|
@@ -2,6 +2,7 @@ import { join } from "node:path";
|
|
|
2
2
|
import { mkdir, chmod, writeFile } from "node:fs/promises";
|
|
3
3
|
import { createHash, randomUUID } from "node:crypto";
|
|
4
4
|
import { promisify } from "node:util";
|
|
5
|
+
import { sanitizeForLog } from "../utils/logSanitize.js";
|
|
5
6
|
import { gzip as gzipCallback } from "node:zlib";
|
|
6
7
|
const REQUEST_LOG_IO_TIMEOUT_MS = 5_000;
|
|
7
8
|
/** Maximum redacted body bytes persisted per capture entry. */
|
|
@@ -43,6 +44,15 @@ function redactHeaders(headers) {
|
|
|
43
44
|
return redacted;
|
|
44
45
|
}
|
|
45
46
|
const SENSITIVE_BODY_KEY = /^(?:password|access_token|refresh_token|api_key|apiKey|secret|authorization|token|credential|x-api-key)$/i;
|
|
47
|
+
/** Error messages may contain nested JSON encoded as a string. */
|
|
48
|
+
function redactNestedValue(key, value) {
|
|
49
|
+
if (SENSITIVE_BODY_KEY.test(key)) {
|
|
50
|
+
return "[REDACTED]";
|
|
51
|
+
}
|
|
52
|
+
return typeof value === "string"
|
|
53
|
+
? sanitizeForLog(value.replace(SENSITIVE_BODY_KEYS, '$1"[REDACTED]"'), value.length)
|
|
54
|
+
: value;
|
|
55
|
+
}
|
|
46
56
|
/** Redact every value under a sensitive key, including objects and arrays. */
|
|
47
57
|
function redactBody(body) {
|
|
48
58
|
if (body === undefined || body === null) {
|
|
@@ -54,11 +64,44 @@ function redactBody(body) {
|
|
|
54
64
|
value = JSON.parse(body);
|
|
55
65
|
}
|
|
56
66
|
catch {
|
|
57
|
-
|
|
58
|
-
|
|
67
|
+
if (/^(?:event|data):/m.test(body)) {
|
|
68
|
+
// Redact whole SSE data values, including nested credentials. Never
|
|
69
|
+
// retain a truncated/invalid JSON tail whose secret cannot be parsed.
|
|
70
|
+
return body
|
|
71
|
+
.split(/\r\n\r\n|\n\n|\r\r/)
|
|
72
|
+
.map((frame) => {
|
|
73
|
+
const lines = frame.split(/\r\n|\n|\r/);
|
|
74
|
+
const data = lines
|
|
75
|
+
.filter((line) => line.startsWith("data:"))
|
|
76
|
+
.map((line) => line.slice(5).trimStart())
|
|
77
|
+
.join("\n");
|
|
78
|
+
if (!data || data === "[DONE]") {
|
|
79
|
+
return frame;
|
|
80
|
+
}
|
|
81
|
+
let redacted;
|
|
82
|
+
try {
|
|
83
|
+
redacted = JSON.stringify(JSON.parse(data), redactNestedValue);
|
|
84
|
+
}
|
|
85
|
+
catch {
|
|
86
|
+
redacted = "[UNPARSEABLE DATA REDACTED]";
|
|
87
|
+
}
|
|
88
|
+
return [
|
|
89
|
+
...lines.filter((line) => !line.startsWith("data:")),
|
|
90
|
+
`data: ${redacted}`,
|
|
91
|
+
].join("\n");
|
|
92
|
+
})
|
|
93
|
+
.join("\n\n");
|
|
94
|
+
}
|
|
95
|
+
// Invalid or truncated structured JSON must not expose an unterminated
|
|
96
|
+
// credential value that a quoted-string regex cannot safely redact.
|
|
97
|
+
if (/^\s*[[{]/.test(body)) {
|
|
98
|
+
return "[UNPARSEABLE DATA REDACTED]";
|
|
99
|
+
}
|
|
100
|
+
// Plain-text bodies retain the existing credential-field redaction.
|
|
101
|
+
return sanitizeForLog(body.replace(SENSITIVE_BODY_KEYS, '$1"[REDACTED]"'), body.length);
|
|
59
102
|
}
|
|
60
103
|
}
|
|
61
|
-
return JSON.stringify(value,
|
|
104
|
+
return JSON.stringify(value, redactNestedValue);
|
|
62
105
|
}
|
|
63
106
|
/**
|
|
64
107
|
* Restrict request and phase identifiers to characters safe for artifact
|
|
@@ -208,6 +251,7 @@ export async function processProxyBodyCapture(entry, logDir) {
|
|
|
208
251
|
const headers = redactHeaders(entry.headers);
|
|
209
252
|
const limit = logDir === null ? MAX_OTEL_CAPTURED_BODY_BYTES : MAX_CAPTURED_BODY_BYTES;
|
|
210
253
|
const prepared = prepareRedactedBody(entry.body, limit);
|
|
254
|
+
prepared.truncated ||= entry.sourceTruncated === true;
|
|
211
255
|
if (logDir === null) {
|
|
212
256
|
return {
|
|
213
257
|
headers,
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { Worker } from "node:worker_threads";
|
|
2
|
-
const MAX_PENDING =
|
|
2
|
+
const MAX_PENDING = 64;
|
|
3
3
|
const MAX_PENDING_BYTES = 32 * 1024 * 1024;
|
|
4
4
|
// Bound retained UTF-16 strings rather than a 3x UTF-8 guess. This accommodates
|
|
5
5
|
// the observed 7.3 MB JSON requests while retaining a 32 MiB aggregate pool.
|
|
@@ -29,7 +29,7 @@ const pending = new Map();
|
|
|
29
29
|
* Conservatively bound clone size and traversal work without invoking
|
|
30
30
|
* getters or serializers.
|
|
31
31
|
*/
|
|
32
|
-
function estimateCloneBytes(value) {
|
|
32
|
+
function estimateCloneBytes(value, maxEntryBytes) {
|
|
33
33
|
const stack = [value];
|
|
34
34
|
const seen = new Set();
|
|
35
35
|
let bytes = 0, nodes = 0;
|
|
@@ -37,8 +37,8 @@ function estimateCloneBytes(value) {
|
|
|
37
37
|
if (++nodes > 100_000) {
|
|
38
38
|
throw new Error("body_capture_traversal_limit");
|
|
39
39
|
}
|
|
40
|
-
if (bytes >
|
|
41
|
-
|
|
40
|
+
if (bytes > maxEntryBytes) {
|
|
41
|
+
return bytes;
|
|
42
42
|
}
|
|
43
43
|
const item = stack.pop();
|
|
44
44
|
if (typeof item === "string") {
|
|
@@ -68,8 +68,8 @@ function estimateCloneBytes(value) {
|
|
|
68
68
|
if (stack.length > 100_000) {
|
|
69
69
|
throw new Error("body_capture_traversal_limit");
|
|
70
70
|
}
|
|
71
|
-
if (bytes >
|
|
72
|
-
|
|
71
|
+
if (bytes > maxEntryBytes) {
|
|
72
|
+
return bytes;
|
|
73
73
|
}
|
|
74
74
|
}
|
|
75
75
|
}
|
|
@@ -135,10 +135,13 @@ function getWorker() {
|
|
|
135
135
|
/** Bounded bulk capture. Failures are indexed; never fall back to blocking work. */
|
|
136
136
|
export async function captureProxyBody(entry, logDir, consume) {
|
|
137
137
|
snapshot.attempted += 1;
|
|
138
|
+
// An OTel capture can use the existing byte pool when it is otherwise idle.
|
|
139
|
+
// Raising the slot count does not raise the aggregate retained-memory bound.
|
|
140
|
+
const maxEntryBytes = logDir === null ? MAX_PENDING_BYTES : MAX_ENTRY_BYTES;
|
|
138
141
|
let bytes;
|
|
139
142
|
let admissionError;
|
|
140
143
|
try {
|
|
141
|
-
bytes = estimateCloneBytes(entry);
|
|
144
|
+
bytes = estimateCloneBytes(entry, maxEntryBytes);
|
|
142
145
|
}
|
|
143
146
|
catch (error) {
|
|
144
147
|
bytes = Infinity;
|
|
@@ -152,12 +155,12 @@ export async function captureProxyBody(entry, logDir, consume) {
|
|
|
152
155
|
? error.message
|
|
153
156
|
: "body_capture_unsupported_value";
|
|
154
157
|
}
|
|
155
|
-
if (bytes >
|
|
158
|
+
if (bytes > maxEntryBytes ||
|
|
156
159
|
snapshot.pending >= MAX_PENDING ||
|
|
157
160
|
snapshot.pendingBytes + bytes > MAX_PENDING_BYTES ||
|
|
158
161
|
Date.now() < retryAfter) {
|
|
159
162
|
snapshot.rejected += 1;
|
|
160
|
-
const error = bytes >
|
|
163
|
+
const error = bytes > maxEntryBytes
|
|
161
164
|
? (admissionError ?? "body_capture_entry_too_large")
|
|
162
165
|
: Date.now() < retryAfter
|
|
163
166
|
? "body_worker_backoff"
|
|
@@ -170,7 +173,7 @@ export async function captureProxyBody(entry, logDir, consume) {
|
|
|
170
173
|
error,
|
|
171
174
|
stored: { bodyWriteFailed: true },
|
|
172
175
|
admission: {
|
|
173
|
-
limitingResource: bytes >
|
|
176
|
+
limitingResource: bytes > maxEntryBytes
|
|
174
177
|
? "entry"
|
|
175
178
|
: Date.now() < retryAfter
|
|
176
179
|
? "worker"
|
|
@@ -178,6 +181,7 @@ export async function captureProxyBody(entry, logDir, consume) {
|
|
|
178
181
|
? "captures"
|
|
179
182
|
: "bytes",
|
|
180
183
|
estimatedBytes: Number.isFinite(bytes) ? bytes : undefined,
|
|
184
|
+
maxEntryBytes,
|
|
181
185
|
pending: snapshot.pending,
|
|
182
186
|
pendingBytes: snapshot.pendingBytes,
|
|
183
187
|
maxPending: MAX_PENDING,
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { RequestLogEntry, SSETelemetry } from "../types/index.js";
|
|
2
|
+
/** Content visible to the caller; thinking and empty text are not useful output. */
|
|
3
|
+
export declare function hasUsefulClaudeContent(value: unknown): boolean;
|
|
4
|
+
/** A complete JSON response becomes observable when its body has been read. */
|
|
5
|
+
export declare function observeClaudeJsonOutput(metadata: Record<string, unknown>, value: unknown, at?: number): void;
|
|
6
|
+
/** Keep unavailable observations distinct from a complete stream with no output. */
|
|
7
|
+
export declare function observeClaudeStreamOutput(metadata: Record<string, unknown>, data: SSETelemetry): void;
|
|
8
|
+
/** Final records always explain whether useful-output latency was observed. */
|
|
9
|
+
export declare function claudeOutputLogFields(metadata: Record<string, unknown>, requestStartTime: number): Pick<RequestLogEntry, "firstUsefulOutputMs" | "firstUsefulOutputStatus" | "firstUsefulOutputEvent" | "firstUsefulOutputUnavailableReason">;
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
/** Content visible to the caller; thinking and empty text are not useful output. */
|
|
2
|
+
export function hasUsefulClaudeContent(value) {
|
|
3
|
+
if (!value || typeof value !== "object") {
|
|
4
|
+
return false;
|
|
5
|
+
}
|
|
6
|
+
const block = value;
|
|
7
|
+
if (Array.isArray(block.content)) {
|
|
8
|
+
return block.content.some(hasUsefulClaudeContent);
|
|
9
|
+
}
|
|
10
|
+
if (block.type === "thinking" || block.type === "redacted_thinking") {
|
|
11
|
+
return false;
|
|
12
|
+
}
|
|
13
|
+
if (typeof block.text === "string" && block.text.trim().length > 0) {
|
|
14
|
+
return true;
|
|
15
|
+
}
|
|
16
|
+
if (typeof block.refusal === "string" && block.refusal.trim().length > 0) {
|
|
17
|
+
return true;
|
|
18
|
+
}
|
|
19
|
+
return ((block.type === "tool_use" || block.type === "server_tool_use") &&
|
|
20
|
+
typeof block.name === "string" &&
|
|
21
|
+
block.name.length > 0 &&
|
|
22
|
+
block.input !== undefined);
|
|
23
|
+
}
|
|
24
|
+
/** A complete JSON response becomes observable when its body has been read. */
|
|
25
|
+
export function observeClaudeJsonOutput(metadata, value, at = Date.now()) {
|
|
26
|
+
if (!value ||
|
|
27
|
+
typeof value !== "object" ||
|
|
28
|
+
!Array.isArray(value.content)) {
|
|
29
|
+
metadata.firstUsefulOutputStatus = "not_observed";
|
|
30
|
+
metadata.firstUsefulOutputUnavailableReason = "unrecognized_response";
|
|
31
|
+
return;
|
|
32
|
+
}
|
|
33
|
+
if (hasUsefulClaudeContent(value)) {
|
|
34
|
+
metadata.firstUsefulOutputAt = at;
|
|
35
|
+
metadata.firstUsefulOutputStatus = "observed";
|
|
36
|
+
metadata.firstUsefulOutputEvent = "json_response.content";
|
|
37
|
+
}
|
|
38
|
+
else {
|
|
39
|
+
metadata.firstUsefulOutputStatus = "no_useful_output";
|
|
40
|
+
delete metadata.firstUsefulOutputAt;
|
|
41
|
+
delete metadata.firstUsefulOutputEvent;
|
|
42
|
+
}
|
|
43
|
+
delete metadata.firstUsefulOutputUnavailableReason;
|
|
44
|
+
}
|
|
45
|
+
/** Keep unavailable observations distinct from a complete stream with no output. */
|
|
46
|
+
export function observeClaudeStreamOutput(metadata, data) {
|
|
47
|
+
delete metadata.firstUsefulOutputAt;
|
|
48
|
+
delete metadata.firstUsefulOutputEvent;
|
|
49
|
+
delete metadata.firstUsefulOutputUnavailableReason;
|
|
50
|
+
if (!data.observationIncomplete && data.firstUsefulOutputAt !== undefined) {
|
|
51
|
+
metadata.firstUsefulOutputAt = data.firstUsefulOutputAt;
|
|
52
|
+
metadata.firstUsefulOutputEvent = data.firstUsefulOutputEvent;
|
|
53
|
+
metadata.firstUsefulOutputStatus = "observed";
|
|
54
|
+
}
|
|
55
|
+
else if (!data.observationIncomplete && data.messageStopReceived) {
|
|
56
|
+
metadata.firstUsefulOutputStatus = "no_useful_output";
|
|
57
|
+
}
|
|
58
|
+
else {
|
|
59
|
+
metadata.firstUsefulOutputStatus = "not_observed";
|
|
60
|
+
metadata.firstUsefulOutputUnavailableReason = data.observationIncomplete
|
|
61
|
+
? "incomplete_observation"
|
|
62
|
+
: "stream_incomplete";
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
/** Final records always explain whether useful-output latency was observed. */
|
|
66
|
+
export function claudeOutputLogFields(metadata, requestStartTime) {
|
|
67
|
+
const at = metadata.firstUsefulOutputAt;
|
|
68
|
+
if (typeof at === "number" && Number.isFinite(at)) {
|
|
69
|
+
return {
|
|
70
|
+
firstUsefulOutputMs: Math.max(0, at - requestStartTime),
|
|
71
|
+
firstUsefulOutputStatus: "observed",
|
|
72
|
+
firstUsefulOutputEvent: typeof metadata.firstUsefulOutputEvent === "string"
|
|
73
|
+
? metadata.firstUsefulOutputEvent
|
|
74
|
+
: "translated_output",
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
return metadata.firstUsefulOutputStatus === "no_useful_output"
|
|
78
|
+
? { firstUsefulOutputStatus: "no_useful_output" }
|
|
79
|
+
: {
|
|
80
|
+
firstUsefulOutputStatus: "not_observed",
|
|
81
|
+
firstUsefulOutputUnavailableReason: typeof metadata.firstUsefulOutputUnavailableReason === "string"
|
|
82
|
+
? metadata.firstUsefulOutputUnavailableReason
|
|
83
|
+
: "response_not_observed",
|
|
84
|
+
};
|
|
85
|
+
}
|
package/dist/proxy/codexUsage.js
CHANGED
|
@@ -53,20 +53,22 @@ function usefulOutputItem(item) {
|
|
|
53
53
|
if (item.type === "function_call") {
|
|
54
54
|
return ("arguments" in item &&
|
|
55
55
|
typeof item.arguments === "string" &&
|
|
56
|
-
item.arguments.length > 0);
|
|
56
|
+
item.arguments.trim().length > 0);
|
|
57
57
|
}
|
|
58
58
|
if (item.type === "custom_tool_call") {
|
|
59
59
|
return ("input" in item &&
|
|
60
60
|
typeof item.input === "string" &&
|
|
61
|
-
item.input.length > 0);
|
|
61
|
+
item.input.trim().length > 0);
|
|
62
62
|
}
|
|
63
63
|
if (item.type === "output_text") {
|
|
64
|
-
return ("text" in item &&
|
|
64
|
+
return ("text" in item &&
|
|
65
|
+
typeof item.text === "string" &&
|
|
66
|
+
item.text.trim().length > 0);
|
|
65
67
|
}
|
|
66
68
|
if (item.type === "refusal") {
|
|
67
69
|
return ("refusal" in item &&
|
|
68
70
|
typeof item.refusal === "string" &&
|
|
69
|
-
item.refusal.length > 0);
|
|
71
|
+
item.refusal.trim().length > 0);
|
|
70
72
|
}
|
|
71
73
|
}
|
|
72
74
|
return ("content" in item &&
|
|
@@ -213,7 +215,7 @@ export function createCodexUsageTap() {
|
|
|
213
215
|
type === "response.refusal.delta" ||
|
|
214
216
|
type === "response.function_call_arguments.delta") &&
|
|
215
217
|
typeof event.delta === "string" &&
|
|
216
|
-
event.delta.length > 0) {
|
|
218
|
+
event.delta.trim().length > 0) {
|
|
217
219
|
if (evidence.firstUsefulOutputAt === undefined) {
|
|
218
220
|
evidence.firstUsefulOutputAt = Date.now();
|
|
219
221
|
evidence.firstUsefulOutputEvent = String(type);
|
|
@@ -232,22 +234,22 @@ export function createCodexUsageTap() {
|
|
|
232
234
|
completedResponse.output.some(usefulOutputItem);
|
|
233
235
|
const usefulDone = (type === "response.output_text.done" &&
|
|
234
236
|
typeof event.text === "string" &&
|
|
235
|
-
event.text.length > 0) ||
|
|
237
|
+
event.text.trim().length > 0) ||
|
|
236
238
|
(type === "response.refusal.done" &&
|
|
237
239
|
typeof event.refusal === "string" &&
|
|
238
|
-
event.refusal.length > 0) ||
|
|
240
|
+
event.refusal.trim().length > 0) ||
|
|
239
241
|
(type === "response.function_call_arguments.done" &&
|
|
240
242
|
typeof event.arguments === "string" &&
|
|
241
|
-
event.arguments.length > 0) ||
|
|
243
|
+
event.arguments.trim().length > 0) ||
|
|
242
244
|
(type === "response.custom_tool_call_input.done" &&
|
|
243
245
|
typeof event.input === "string" &&
|
|
244
|
-
event.input.length > 0) ||
|
|
246
|
+
event.input.trim().length > 0) ||
|
|
245
247
|
((type === "response.content_part.added" ||
|
|
246
248
|
type === "response.content_part.done") &&
|
|
247
249
|
usefulOutputItem(event.part));
|
|
248
250
|
const usefulToolDelta = type === "response.custom_tool_call_input.delta" &&
|
|
249
251
|
typeof event.delta === "string" &&
|
|
250
|
-
event.delta.length > 0;
|
|
252
|
+
event.delta.trim().length > 0;
|
|
251
253
|
if (evidence.firstUsefulOutputAt === undefined &&
|
|
252
254
|
(usefulDone ||
|
|
253
255
|
usefulCompletion ||
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { fstatSync } from "node:fs";
|
|
2
|
+
import { getProxyOtelLogSnapshot, isProxyOtelOnly } from "./otelLogSink.js";
|
|
3
|
+
import { getProxyLifecycleLoggerSnapshot } from "./proxyLifecycle.js";
|
|
4
|
+
/** Inspect this process, not the launcher configuration intended for its successor. */
|
|
5
|
+
export function getProxyProcessTelemetry() {
|
|
6
|
+
const descriptor = (fd) => {
|
|
7
|
+
try {
|
|
8
|
+
return fstatSync(fd).isFile() ? "file" : "non_file";
|
|
9
|
+
}
|
|
10
|
+
catch {
|
|
11
|
+
return "unavailable";
|
|
12
|
+
}
|
|
13
|
+
};
|
|
14
|
+
const logs = getProxyOtelLogSnapshot();
|
|
15
|
+
return {
|
|
16
|
+
pid: process.pid,
|
|
17
|
+
checkedAt: new Date().toISOString(),
|
|
18
|
+
configuredSink: isProxyOtelOnly() ? "otel" : "file",
|
|
19
|
+
lifecycleSink: getProxyLifecycleLoggerSnapshot().sink ?? "unavailable",
|
|
20
|
+
otelInitialized: logs.initialized,
|
|
21
|
+
stdio: { stdout: descriptor(1), stderr: descriptor(2) },
|
|
22
|
+
exportDropped: logs.queues.reduce((n, q) => n + q.dropped, 0),
|
|
23
|
+
exportUnconfirmed: logs.queues.reduce((n, q) => n + q.exportUnconfirmed, 0),
|
|
24
|
+
};
|
|
25
|
+
}
|
|
@@ -1,5 +1,9 @@
|
|
|
1
1
|
/** Maximum bytes to capture before stopping accumulation (1 MB). */
|
|
2
2
|
const MAX_CAPTURE_BYTES = 1024 * 1024;
|
|
3
|
+
// Payload accounting across concurrent stream observers, independent of the
|
|
4
|
+
// worker's clone/publication pool. Never retain an unbounded number of prefixes.
|
|
5
|
+
const MAX_PENDING_CAPTURE_BYTES = 16 * 1024 * 1024;
|
|
6
|
+
let pendingCaptureBytes = 0;
|
|
3
7
|
const TRUNCATION_MARKER = "\n...[TRUNCATED]";
|
|
4
8
|
export function createRawStreamCapture() {
|
|
5
9
|
const decoder = new TextDecoder();
|
|
@@ -17,21 +21,13 @@ export function createRawStreamCapture() {
|
|
|
17
21
|
return;
|
|
18
22
|
}
|
|
19
23
|
resolved = true;
|
|
20
|
-
const
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
chunks.push(finalChunk.slice(0, remainingBytes));
|
|
25
|
-
capturedBytes += Math.min(finalChunk.length, remainingBytes);
|
|
26
|
-
}
|
|
27
|
-
else if (!truncated) {
|
|
28
|
-
chunks.push(TRUNCATION_MARKER);
|
|
29
|
-
truncated = true;
|
|
30
|
-
}
|
|
31
|
-
}
|
|
24
|
+
const text = chunks.map((chunk) => decoder.decode(chunk, { stream: true })).join("") +
|
|
25
|
+
(truncated ? "" : decoder.decode());
|
|
26
|
+
pendingCaptureBytes -= capturedBytes;
|
|
27
|
+
chunks.length = 0;
|
|
32
28
|
resolveCapture({
|
|
33
29
|
totalBytes,
|
|
34
|
-
text:
|
|
30
|
+
text: text + (truncated ? TRUNCATION_MARKER : ""),
|
|
35
31
|
truncated,
|
|
36
32
|
});
|
|
37
33
|
}
|
|
@@ -39,21 +35,16 @@ export function createRawStreamCapture() {
|
|
|
39
35
|
transform(chunk, controller) {
|
|
40
36
|
controller.enqueue(chunk);
|
|
41
37
|
totalBytes += chunk.byteLength;
|
|
42
|
-
if (capturedBytes < MAX_CAPTURE_BYTES) {
|
|
43
|
-
const
|
|
44
|
-
|
|
45
|
-
const slice =
|
|
38
|
+
if (!truncated && capturedBytes < MAX_CAPTURE_BYTES) {
|
|
39
|
+
const remainingBytes = Math.min(MAX_CAPTURE_BYTES - capturedBytes, Math.max(0, MAX_PENDING_CAPTURE_BYTES - pendingCaptureBytes));
|
|
40
|
+
// Buffer.slice is a view; copy to avoid retaining the whole wire chunk.
|
|
41
|
+
const slice = Uint8Array.from(chunk.subarray(0, remainingBytes));
|
|
46
42
|
chunks.push(slice);
|
|
47
|
-
capturedBytes +=
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
truncated = true;
|
|
51
|
-
}
|
|
52
|
-
}
|
|
53
|
-
else if (!truncated) {
|
|
54
|
-
chunks.push(TRUNCATION_MARKER);
|
|
55
|
-
truncated = true;
|
|
43
|
+
capturedBytes += slice.byteLength;
|
|
44
|
+
pendingCaptureBytes += slice.byteLength;
|
|
45
|
+
truncated = slice.byteLength < chunk.byteLength;
|
|
56
46
|
}
|
|
47
|
+
truncated ||= totalBytes > MAX_CAPTURE_BYTES;
|
|
57
48
|
},
|
|
58
49
|
flush() {
|
|
59
50
|
settle();
|
|
@@ -35,6 +35,8 @@ export declare function prepareProxyBodyForLogging(body: unknown): {
|
|
|
35
35
|
truncated: boolean;
|
|
36
36
|
originalBytes?: number;
|
|
37
37
|
};
|
|
38
|
+
/** Cheap gate for response observers; avoid copying diagnostics on the serving path. */
|
|
39
|
+
export declare function isProxyBodyCaptureEnabled(): boolean;
|
|
38
40
|
/** Capture an owned request body with bounded processing and tracked index/export publication. */
|
|
39
41
|
export declare function logBodyCapture(entry: ProxyBodyCaptureEntry): Promise<void>;
|
|
40
42
|
/**
|
|
@@ -514,9 +514,13 @@ function emitOtlpBodyLogRecord(entry, stored) {
|
|
|
514
514
|
return undefined;
|
|
515
515
|
});
|
|
516
516
|
}
|
|
517
|
+
/** Cheap gate for response observers; avoid copying diagnostics on the serving path. */
|
|
518
|
+
export function isProxyBodyCaptureEnabled() {
|
|
519
|
+
return logEnabled && (Boolean(logDir) || isProxyOtelOnly());
|
|
520
|
+
}
|
|
517
521
|
/** Capture an owned request body with bounded processing and tracked index/export publication. */
|
|
518
522
|
export async function logBodyCapture(entry) {
|
|
519
|
-
if (!
|
|
523
|
+
if (!isProxyBodyCaptureEnabled()) {
|
|
520
524
|
return;
|
|
521
525
|
}
|
|
522
526
|
// Borrowed traffic is somebody else's conversation. Capturing it would leave
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { ProxyRestartControlIdentity, ProxyRestartControlOptions, ProxyRestartResult } from "../types/index.js";
|
|
1
|
+
import type { ProxyRestartControlIdentity, ProxyRestartControlOptions, ProxyRestartResult, ProxySupervisorTelemetry } from "../types/index.js";
|
|
2
2
|
/**
|
|
3
3
|
* Own restart completion in the supervisor, so a disconnected CLI cannot leave
|
|
4
4
|
* admission closed. The control socket is private to the service's OS user.
|
|
@@ -8,5 +8,7 @@ export declare function startProxyRestartControl(options: ProxyRestartControlOpt
|
|
|
8
8
|
identity: ProxyRestartControlIdentity;
|
|
9
9
|
close: () => Promise<void>;
|
|
10
10
|
}>;
|
|
11
|
+
/** Query the private supervisor endpoint with a short, bounded read; never restart. */
|
|
12
|
+
export declare function requestProxySupervisorTelemetry(identity: ProxyRestartControlIdentity | undefined, expectedPid: number): Promise<ProxySupervisorTelemetry>;
|
|
11
13
|
/** Perform one local control operation; never fall back to killing a process. */
|
|
12
14
|
export declare function requestProxyRestart(identity: ProxyRestartControlIdentity, check: boolean): Promise<ProxyRestartResult>;
|
|
@@ -1,8 +1,35 @@
|
|
|
1
1
|
import { createServer, request } from "node:http";
|
|
2
|
-
import { chmod, mkdir, rm } from "node:fs/promises";
|
|
2
|
+
import { chmod, mkdir, rm, lstat } from "node:fs/promises";
|
|
3
3
|
import { join } from "node:path";
|
|
4
4
|
import { randomUUID } from "node:crypto";
|
|
5
5
|
import { z } from "zod";
|
|
6
|
+
const processTelemetrySchema = z.object({
|
|
7
|
+
pid: z.number().int().positive(),
|
|
8
|
+
checkedAt: z.string().datetime(),
|
|
9
|
+
configuredSink: z.enum(["otel", "file"]),
|
|
10
|
+
lifecycleSink: z.string(),
|
|
11
|
+
otelInitialized: z.boolean(),
|
|
12
|
+
stdio: z.object({
|
|
13
|
+
stdout: z.enum(["file", "non_file", "unavailable"]),
|
|
14
|
+
stderr: z.enum(["file", "non_file", "unavailable"]),
|
|
15
|
+
}),
|
|
16
|
+
exportDropped: z.number().int().nonnegative(),
|
|
17
|
+
exportUnconfirmed: z.number().int().nonnegative(),
|
|
18
|
+
});
|
|
19
|
+
/** A worker-only restart cannot repair inherited supervisor log descriptors. */
|
|
20
|
+
function assertSupervisorLogging(status, snapshot) {
|
|
21
|
+
if (status.observability.requestLogs.diskEnabled === false &&
|
|
22
|
+
(!snapshot ||
|
|
23
|
+
snapshot.configuredSink !== "otel" ||
|
|
24
|
+
snapshot.lifecycleSink !== "otel" ||
|
|
25
|
+
!snapshot.otelInitialized ||
|
|
26
|
+
snapshot.stdio.stdout !== "non_file" ||
|
|
27
|
+
snapshot.stdio.stderr !== "non_file")) {
|
|
28
|
+
throw new Error(snapshot
|
|
29
|
+
? "Supervisor logging is not OTel-only. Worker replacement cannot complete the logging cutover; activate the configured supervisor service first."
|
|
30
|
+
: "Supervisor telemetry is unavailable, so OTel-only logging cannot be proven. Activate the configured supervisor service first.");
|
|
31
|
+
}
|
|
32
|
+
}
|
|
6
33
|
const statusSchema = z.object({
|
|
7
34
|
pid: z.number().int().positive(),
|
|
8
35
|
version: z.string(),
|
|
@@ -99,6 +126,7 @@ export async function startProxyRestartControl(options) {
|
|
|
99
126
|
}
|
|
100
127
|
previousWorkerPid = active.pid;
|
|
101
128
|
const status = statusSchema.parse(await options.getStatus());
|
|
129
|
+
assertSupervisorLogging(status, options.getTelemetry?.());
|
|
102
130
|
if (status.pid !== previousWorkerPid ||
|
|
103
131
|
status.version !== active.version) {
|
|
104
132
|
throw new Error("Serving worker identity changed during preflight; retry the check.");
|
|
@@ -181,6 +209,26 @@ export async function startProxyRestartControl(options) {
|
|
|
181
209
|
res.writeHead(400, { connection: "close" }).end();
|
|
182
210
|
return;
|
|
183
211
|
}
|
|
212
|
+
if (req.method === "GET" && req.url === "/telemetry") {
|
|
213
|
+
req.resume();
|
|
214
|
+
try {
|
|
215
|
+
const snapshot = options.getTelemetry?.();
|
|
216
|
+
if (!snapshot) {
|
|
217
|
+
res.writeHead(404).end();
|
|
218
|
+
return;
|
|
219
|
+
}
|
|
220
|
+
const payload = JSON.stringify(snapshot);
|
|
221
|
+
res.writeHead(200, {
|
|
222
|
+
"content-type": "application/json",
|
|
223
|
+
connection: "close",
|
|
224
|
+
});
|
|
225
|
+
res.end(payload);
|
|
226
|
+
}
|
|
227
|
+
catch {
|
|
228
|
+
res.writeHead(503, { connection: "close" }).end();
|
|
229
|
+
}
|
|
230
|
+
return;
|
|
231
|
+
}
|
|
184
232
|
if (!((req.method === "GET" && req.url === "/check") ||
|
|
185
233
|
(req.method === "POST" && req.url === "/restart"))) {
|
|
186
234
|
res.writeHead(404).end();
|
|
@@ -238,6 +286,75 @@ export async function startProxyRestartControl(options) {
|
|
|
238
286
|
},
|
|
239
287
|
};
|
|
240
288
|
}
|
|
289
|
+
/** Query the private supervisor endpoint with a short, bounded read; never restart. */
|
|
290
|
+
export async function requestProxySupervisorTelemetry(identity, expectedPid) {
|
|
291
|
+
if (!identity) {
|
|
292
|
+
return { status: "unavailable", reason: "supervisor_upgrade_required" };
|
|
293
|
+
}
|
|
294
|
+
try {
|
|
295
|
+
if (!identity.socketPath.endsWith(`/restart-${expectedPid}-${identity.instanceId}.sock`)) {
|
|
296
|
+
throw new Error("identity");
|
|
297
|
+
}
|
|
298
|
+
const socket = await lstat(identity.socketPath);
|
|
299
|
+
if (!socket.isSocket() ||
|
|
300
|
+
(socket.mode & 0o077) !== 0 ||
|
|
301
|
+
(process.getuid && socket.uid !== process.getuid())) {
|
|
302
|
+
throw new Error("socket");
|
|
303
|
+
}
|
|
304
|
+
const snapshot = await new Promise((resolve, reject) => {
|
|
305
|
+
const req = request({
|
|
306
|
+
socketPath: identity.socketPath,
|
|
307
|
+
path: "/telemetry",
|
|
308
|
+
method: "GET",
|
|
309
|
+
headers: {
|
|
310
|
+
"x-neurolink-instance": identity.instanceId,
|
|
311
|
+
connection: "close",
|
|
312
|
+
},
|
|
313
|
+
});
|
|
314
|
+
const timer = setTimeout(() => req.destroy(new Error("timeout")), 1000);
|
|
315
|
+
req.once("error", (error) => {
|
|
316
|
+
clearTimeout(timer);
|
|
317
|
+
reject(error);
|
|
318
|
+
});
|
|
319
|
+
req.once("response", (res) => {
|
|
320
|
+
let body = "";
|
|
321
|
+
res.setEncoding("utf8");
|
|
322
|
+
res.on("data", (chunk) => {
|
|
323
|
+
body += chunk;
|
|
324
|
+
if (body.length > 8192) {
|
|
325
|
+
req.destroy(new Error("size"));
|
|
326
|
+
}
|
|
327
|
+
});
|
|
328
|
+
res.once("error", (error) => {
|
|
329
|
+
clearTimeout(timer);
|
|
330
|
+
reject(error);
|
|
331
|
+
});
|
|
332
|
+
res.once("end", () => {
|
|
333
|
+
clearTimeout(timer);
|
|
334
|
+
try {
|
|
335
|
+
if (res.statusCode !== 200) {
|
|
336
|
+
throw new Error("unsupported");
|
|
337
|
+
}
|
|
338
|
+
const value = processTelemetrySchema.parse(JSON.parse(body));
|
|
339
|
+
if (value.pid !== expectedPid ||
|
|
340
|
+
Math.abs(Date.now() - Date.parse(value.checkedAt)) > 5000) {
|
|
341
|
+
throw new Error("stale");
|
|
342
|
+
}
|
|
343
|
+
resolve(value);
|
|
344
|
+
}
|
|
345
|
+
catch (error) {
|
|
346
|
+
reject(error);
|
|
347
|
+
}
|
|
348
|
+
});
|
|
349
|
+
});
|
|
350
|
+
req.end();
|
|
351
|
+
});
|
|
352
|
+
return { status: "available", process: snapshot };
|
|
353
|
+
}
|
|
354
|
+
catch {
|
|
355
|
+
return { status: "unavailable", reason: "supervisor_telemetry_unverified" };
|
|
356
|
+
}
|
|
357
|
+
}
|
|
241
358
|
/** Perform one local control operation; never fall back to killing a process. */
|
|
242
359
|
export async function requestProxyRestart(identity, check) {
|
|
243
360
|
return new Promise((resolve, reject) => {
|