@juspay/neurolink 12.12.9 → 12.12.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +2 -2
- package/dist/browser/neurolink.min.js +394 -395
- package/dist/cli/commands/proxy.d.ts +4 -0
- package/dist/cli/commands/proxy.js +60 -18
- package/dist/cli/commands/proxyAnalyze.js +8 -1
- package/dist/proxy/bodyCaptureProcessing.d.ts +22 -0
- package/dist/proxy/bodyCaptureProcessing.js +219 -0
- package/dist/proxy/bodyCaptureWorker.d.ts +16 -0
- package/dist/proxy/bodyCaptureWorker.js +232 -0
- package/dist/proxy/bodyCaptureWorkerEntry.d.ts +1 -0
- package/dist/proxy/bodyCaptureWorkerEntry.js +34 -0
- package/dist/proxy/proxyAnalysis.js +159 -17
- package/dist/proxy/proxyLifecycle.d.ts +25 -0
- package/dist/proxy/proxyLifecycle.js +111 -5
- package/dist/proxy/proxyRequestKind.d.ts +2 -0
- package/dist/proxy/proxyRequestKind.js +6 -0
- package/dist/proxy/proxyRuntimeMetrics.d.ts +3 -0
- package/dist/proxy/proxyRuntimeMetrics.js +34 -0
- package/dist/proxy/requestLogger.d.ts +10 -6
- package/dist/proxy/requestLogger.js +99 -231
- package/dist/proxy/rollingProxyServer.js +15 -4
- package/dist/proxy/rollingWorkerProcess.d.ts +4 -0
- package/dist/proxy/rollingWorkerProcess.js +25 -8
- package/dist/proxy/rollingWorkerProtocol.d.ts +6 -0
- package/dist/proxy/rollingWorkerProtocol.js +12 -1
- package/dist/proxy/rollingWorkerSupervisor.d.ts +28 -0
- package/dist/proxy/rollingWorkerSupervisor.js +73 -25
- package/dist/proxy/socketWorkerRuntime.d.ts +5 -0
- package/dist/proxy/socketWorkerRuntime.js +19 -2
- package/dist/server/routes/codexProxyRoutes.js +39 -3
- package/dist/services/server/ai/observability/instrumentation.js +7 -1
- package/dist/types/cli.d.ts +1 -1
- package/dist/types/proxy.d.ts +69 -4
- package/package.json +1 -1
|
@@ -10,9 +10,9 @@ import { homedir } from "os";
|
|
|
10
10
|
import { logger } from "../utils/logger.js";
|
|
11
11
|
import { chmodSync, existsSync, mkdirSync, readdirSync, rmSync, statSync, unlinkSync, } from "fs";
|
|
12
12
|
import { writeFile } from "fs/promises";
|
|
13
|
-
import {
|
|
14
|
-
import {
|
|
15
|
-
import {
|
|
13
|
+
import { setImmediate as yieldToRequests } from "node:timers/promises";
|
|
14
|
+
import { captureProxyBody, getBodyCaptureWorkerSnapshot, PROXY_BODY_CAPTURE_DEADLINE_MS, } from "./bodyCaptureWorker.js";
|
|
15
|
+
import { prepareProxyBodyForLogging as prepareRedactedBody, redactProxyHeadersForLogging as redactHeaders, splitUtf8StringByBytes, } from "./bodyCaptureProcessing.js";
|
|
16
16
|
import { isBorrowedRequest } from "./shareContext.js";
|
|
17
17
|
import { OtelBridge } from "../observability/otelBridge.js";
|
|
18
18
|
import { SeverityNumber } from "@opentelemetry/api-logs";
|
|
@@ -40,15 +40,22 @@ const metadataSinks = {
|
|
|
40
40
|
attempts: createSinkSnapshot(),
|
|
41
41
|
debug: createSinkSnapshot(),
|
|
42
42
|
};
|
|
43
|
+
/**
|
|
44
|
+
* Expose independent metadata-sink and body-capture counters for incident reconciliation.
|
|
45
|
+
*/
|
|
43
46
|
export function getRequestLoggerSnapshot() {
|
|
44
47
|
return {
|
|
45
48
|
enabled: logEnabled,
|
|
46
49
|
requests: { ...metadataSinks.requests },
|
|
47
50
|
attempts: { ...metadataSinks.attempts },
|
|
48
51
|
debug: { ...metadataSinks.debug },
|
|
52
|
+
bodyCapture: getBodyCaptureWorkerSnapshot(),
|
|
49
53
|
};
|
|
50
54
|
}
|
|
51
|
-
|
|
55
|
+
/**
|
|
56
|
+
* Serialize append ownership; capture callers retain their memory lease until the write settles.
|
|
57
|
+
*/
|
|
58
|
+
async function appendMetadataRecord(file, line, kind, options = {}) {
|
|
52
59
|
const sink = metadataSinks[kind];
|
|
53
60
|
sink.attempted += 1;
|
|
54
61
|
if (sink.pending + sink.inFlight >= MAX_PENDING_METADATA_RECORDS) {
|
|
@@ -86,16 +93,25 @@ async function appendMetadataRecord(file, line, kind) {
|
|
|
86
93
|
appendChains.delete(file);
|
|
87
94
|
}
|
|
88
95
|
});
|
|
96
|
+
if (options.waitForPersistence) {
|
|
97
|
+
// Bulk capture owns a memory lease until publication settles. A caller
|
|
98
|
+
// timeout must not release that lease while the serialized index is queued.
|
|
99
|
+
return operation;
|
|
100
|
+
}
|
|
89
101
|
// Bound the caller's wait, not the lifetime/ownership of the underlying write.
|
|
90
102
|
await withTimeout(operation, REQUEST_LOG_IO_TIMEOUT_MS, "Proxy metadata write remains pending").catch(() => undefined);
|
|
91
103
|
}
|
|
104
|
+
/**
|
|
105
|
+
* Retain asynchronous log ownership until settlement so shutdown can await admitted publication.
|
|
106
|
+
*/
|
|
92
107
|
function trackLogOperation(operation) {
|
|
93
108
|
pendingLogOperations.add(operation);
|
|
94
109
|
void operation.then(() => pendingLogOperations.delete(operation), () => pendingLogOperations.delete(operation));
|
|
95
110
|
return operation;
|
|
96
111
|
}
|
|
97
112
|
/** Wait, up to a bounded deadline, for admitted request/body writes to settle. */
|
|
98
|
-
export async function flushRequestLogs(timeoutMs =
|
|
113
|
+
export async function flushRequestLogs(timeoutMs = PROXY_BODY_CAPTURE_DEADLINE_MS +
|
|
114
|
+
2 * REQUEST_LOG_IO_TIMEOUT_MS) {
|
|
99
115
|
const deadline = Date.now() + Math.max(1, timeoutMs);
|
|
100
116
|
while (pendingLogOperations.size > 0) {
|
|
101
117
|
const admitted = [...pendingLogOperations];
|
|
@@ -127,24 +143,10 @@ let otelLoggerProvider = null;
|
|
|
127
143
|
let otelResolveAttempts = 0;
|
|
128
144
|
/** Max number of resolve attempts before giving up. */
|
|
129
145
|
const MAX_RESOLVE_ATTEMPTS = 10;
|
|
130
|
-
/** Maximum body chunk size emitted to OTLP logs. */
|
|
131
146
|
const BODY_OTLP_CHUNK_SIZE = 16_000;
|
|
132
|
-
/**
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
const gzip = promisify(gzipCallback);
|
|
136
|
-
/** Headers whose values must always be redacted. */
|
|
137
|
-
const SENSITIVE_HEADER_NAMES = new Set([
|
|
138
|
-
"authorization",
|
|
139
|
-
"proxy-authorization",
|
|
140
|
-
"x-api-key",
|
|
141
|
-
"cookie",
|
|
142
|
-
"set-cookie",
|
|
143
|
-
]);
|
|
144
|
-
/** Pattern that matches header names likely to contain secrets. */
|
|
145
|
-
const SENSITIVE_HEADER_PATTERN = /token|secret|key|password|credential/i;
|
|
146
|
-
/** JSON keys whose values should be redacted in request/response bodies. */
|
|
147
|
-
const SENSITIVE_BODY_KEYS = /("(?:password|access_token|refresh_token|api_key|apiKey|secret|authorization|token|credential|x-api-key)"\s*:\s*)"(?:[^"\\]|\\.)*"/gi;
|
|
147
|
+
/**
|
|
148
|
+
* Initialize private request logs and preserve required lifecycle admission on startup failures.
|
|
149
|
+
*/
|
|
148
150
|
export function initRequestLogger(enabled = true, customLogsDir) {
|
|
149
151
|
// Lifecycle metadata deliberately shares the request logger's enablement,
|
|
150
152
|
// directory permissions, retention boundary, and operator privacy control.
|
|
@@ -164,7 +166,10 @@ export function initRequestLogger(enabled = true, customLogsDir) {
|
|
|
164
166
|
catch (err) {
|
|
165
167
|
logEnabled = false;
|
|
166
168
|
logDir = null;
|
|
167
|
-
configureProxyLifecycleLogger({
|
|
169
|
+
configureProxyLifecycleLogger({
|
|
170
|
+
enabled: true,
|
|
171
|
+
logDir: customLogsDir ?? join(homedir(), ".neurolink", "logs"),
|
|
172
|
+
});
|
|
168
173
|
logger.warn(`[proxy] Request logging disabled — failed to create log directory: ${err instanceof Error ? err.message : String(err)}`);
|
|
169
174
|
}
|
|
170
175
|
}
|
|
@@ -352,119 +357,17 @@ export function getLogDir() {
|
|
|
352
357
|
/**
|
|
353
358
|
* Redact sensitive header values in-place.
|
|
354
359
|
*/
|
|
355
|
-
function redactHeaders(headers) {
|
|
356
|
-
if (!headers) {
|
|
357
|
-
return headers;
|
|
358
|
-
}
|
|
359
|
-
const redacted = {};
|
|
360
|
-
for (const [key, value] of Object.entries(headers)) {
|
|
361
|
-
const lower = key.toLowerCase();
|
|
362
|
-
if (SENSITIVE_HEADER_NAMES.has(lower) ||
|
|
363
|
-
SENSITIVE_HEADER_PATTERN.test(lower)) {
|
|
364
|
-
redacted[key] = "[REDACTED]";
|
|
365
|
-
}
|
|
366
|
-
else {
|
|
367
|
-
redacted[key] = value;
|
|
368
|
-
}
|
|
369
|
-
}
|
|
370
|
-
return redacted;
|
|
371
|
-
}
|
|
372
|
-
function serializeBody(body) {
|
|
373
|
-
if (body === undefined || body === null) {
|
|
374
|
-
return undefined;
|
|
375
|
-
}
|
|
376
|
-
return typeof body === "string" ? body : JSON.stringify(body);
|
|
377
|
-
}
|
|
378
|
-
/**
|
|
379
|
-
* Redact sensitive keys from a JSON body string without truncation.
|
|
380
|
-
*/
|
|
381
|
-
function redactBody(body) {
|
|
382
|
-
const str = serializeBody(body);
|
|
383
|
-
if (str === undefined) {
|
|
384
|
-
return undefined;
|
|
385
|
-
}
|
|
386
|
-
return str.replace(SENSITIVE_BODY_KEYS, '$1"[REDACTED]"');
|
|
387
|
-
}
|
|
388
|
-
function sanitizePhase(phase) {
|
|
389
|
-
return phase.replace(/[^a-zA-Z0-9._-]+/g, "_");
|
|
390
|
-
}
|
|
391
|
-
function sha256(value) {
|
|
392
|
-
return createHash("sha256").update(value).digest("hex");
|
|
393
|
-
}
|
|
394
|
-
function utf8ByteLength(value) {
|
|
395
|
-
return Buffer.byteLength(value, "utf8");
|
|
396
|
-
}
|
|
397
|
-
function truncateUtf8String(input, maxBytes, marker = BODY_TRUNCATION_MARKER) {
|
|
398
|
-
const inputBytes = utf8ByteLength(input);
|
|
399
|
-
if (inputBytes <= maxBytes) {
|
|
400
|
-
return { value: input, bytes: inputBytes, truncated: false };
|
|
401
|
-
}
|
|
402
|
-
const markerBytes = utf8ByteLength(marker);
|
|
403
|
-
if (maxBytes <= markerBytes) {
|
|
404
|
-
return { value: marker, bytes: markerBytes, truncated: true };
|
|
405
|
-
}
|
|
406
|
-
let value = "";
|
|
407
|
-
let bytes = 0;
|
|
408
|
-
for (const char of input) {
|
|
409
|
-
const charBytes = utf8ByteLength(char);
|
|
410
|
-
if (bytes + charBytes + markerBytes > maxBytes) {
|
|
411
|
-
break;
|
|
412
|
-
}
|
|
413
|
-
value += char;
|
|
414
|
-
bytes += charBytes;
|
|
415
|
-
}
|
|
416
|
-
const truncatedValue = `${value}${marker}`;
|
|
417
|
-
return {
|
|
418
|
-
value: truncatedValue,
|
|
419
|
-
bytes: utf8ByteLength(truncatedValue),
|
|
420
|
-
truncated: true,
|
|
421
|
-
};
|
|
422
|
-
}
|
|
423
|
-
function splitUtf8StringByBytes(input, maxBytes) {
|
|
424
|
-
if (!input) {
|
|
425
|
-
return [""];
|
|
426
|
-
}
|
|
427
|
-
const chunks = [];
|
|
428
|
-
let currentChunk = "";
|
|
429
|
-
let currentBytes = 0;
|
|
430
|
-
for (const char of input) {
|
|
431
|
-
const charBytes = utf8ByteLength(char);
|
|
432
|
-
if (currentChunk && currentBytes + charBytes > maxBytes) {
|
|
433
|
-
chunks.push(currentChunk);
|
|
434
|
-
currentChunk = char;
|
|
435
|
-
currentBytes = charBytes;
|
|
436
|
-
continue;
|
|
437
|
-
}
|
|
438
|
-
currentChunk += char;
|
|
439
|
-
currentBytes += charBytes;
|
|
440
|
-
}
|
|
441
|
-
if (currentChunk) {
|
|
442
|
-
chunks.push(currentChunk);
|
|
443
|
-
}
|
|
444
|
-
return chunks;
|
|
445
|
-
}
|
|
446
|
-
function prepareRedactedBody(body) {
|
|
447
|
-
const redacted = redactBody(body);
|
|
448
|
-
if (redacted === undefined) {
|
|
449
|
-
return { truncated: false };
|
|
450
|
-
}
|
|
451
|
-
return truncateUtf8String(redacted, MAX_CAPTURED_BODY_BYTES);
|
|
452
|
-
}
|
|
453
|
-
/** Shared redaction used by offline replay exports and direct comparisons. */
|
|
454
360
|
export function redactProxyHeadersForLogging(headers) {
|
|
455
361
|
return redactHeaders(headers);
|
|
456
362
|
}
|
|
457
|
-
/**
|
|
458
|
-
* Apply the same bounded body redaction used by persisted proxy captures.
|
|
459
|
-
* `value` and `bytes` are omitted only when the input is null or undefined.
|
|
460
|
-
* This performs serialization immediately, so callers must keep it off proxy
|
|
461
|
-
* hot paths unless body processing has already been explicitly requested.
|
|
462
|
-
*/
|
|
363
|
+
/** Return a redacted body representation suitable for persisted request diagnostics. */
|
|
463
364
|
export function prepareProxyBodyForLogging(body) {
|
|
464
365
|
return prepareRedactedBody(body);
|
|
465
366
|
}
|
|
367
|
+
/** Enumerate recognized proxy journals and body artifacts for retention accounting. */
|
|
466
368
|
function collectManagedLogFiles(rootDir) {
|
|
467
369
|
const managedFiles = [];
|
|
370
|
+
/** Collect file sizes and modification times while descending the log directory. */
|
|
468
371
|
const walk = (directory) => {
|
|
469
372
|
for (const entry of readdirSync(directory, { withFileTypes: true })) {
|
|
470
373
|
const entryPath = join(directory, entry.name);
|
|
@@ -473,7 +376,7 @@ function collectManagedLogFiles(rootDir) {
|
|
|
473
376
|
continue;
|
|
474
377
|
}
|
|
475
378
|
const isTopLevelProxyLog = directory === rootDir &&
|
|
476
|
-
/^proxy(?:-attempts|-debug|-lifecycle)?-.*\.jsonl$/.test(entry.name);
|
|
379
|
+
/^proxy(?:-attempts|-debug|-lifecycle|-supervisor)?-.*\.jsonl$/.test(entry.name);
|
|
477
380
|
const isBodyArtifact = entry.name.endsWith(".json.gz") &&
|
|
478
381
|
entryPath.includes(`${join(rootDir, "bodies")}`);
|
|
479
382
|
if (!isTopLevelProxyLog && !isBodyArtifact) {
|
|
@@ -514,55 +417,10 @@ function pruneEmptyDirectories(directory, stopAt) {
|
|
|
514
417
|
// Non-fatal
|
|
515
418
|
}
|
|
516
419
|
}
|
|
517
|
-
|
|
518
|
-
if (!logDir || redactedBody === undefined) {
|
|
519
|
-
return {};
|
|
520
|
-
}
|
|
521
|
-
const dateStr = new Date(entry.timestamp).toISOString().split("T")[0];
|
|
522
|
-
const bodyDir = join(logDir, "bodies", dateStr, entry.requestId);
|
|
523
|
-
if (!existsSync(bodyDir)) {
|
|
524
|
-
mkdirSync(bodyDir, { recursive: true, mode: 0o700 });
|
|
525
|
-
}
|
|
526
|
-
chmodSync(bodyDir, 0o700);
|
|
527
|
-
const fileName = `${Date.now()}-${sanitizePhase(entry.phase)}` +
|
|
528
|
-
(entry.attempt !== undefined ? `-attempt-${entry.attempt}` : "") +
|
|
529
|
-
`.json.gz`;
|
|
530
|
-
const bodyPath = join(bodyDir, fileName);
|
|
531
|
-
const payload = JSON.stringify({
|
|
532
|
-
timestamp: entry.timestamp,
|
|
533
|
-
requestId: entry.requestId,
|
|
534
|
-
phase: entry.phase,
|
|
535
|
-
model: entry.model,
|
|
536
|
-
stream: entry.stream,
|
|
537
|
-
account: entry.account,
|
|
538
|
-
accountType: entry.accountType,
|
|
539
|
-
attempt: entry.attempt,
|
|
540
|
-
responseStatus: entry.responseStatus,
|
|
541
|
-
durationMs: entry.durationMs,
|
|
542
|
-
contentType: entry.contentType,
|
|
543
|
-
headers: redactedHeaders,
|
|
544
|
-
body: redactedBody,
|
|
545
|
-
traceId: entry.traceId,
|
|
546
|
-
spanId: entry.spanId,
|
|
547
|
-
metadata: entry.metadata,
|
|
548
|
-
});
|
|
549
|
-
const compressed = await gzip(payload);
|
|
550
|
-
await writeFile(bodyPath, compressed, {
|
|
551
|
-
mode: 0o600,
|
|
552
|
-
signal: AbortSignal.timeout(REQUEST_LOG_IO_TIMEOUT_MS),
|
|
553
|
-
});
|
|
554
|
-
return {
|
|
555
|
-
bodyPath,
|
|
556
|
-
bodySha256: sha256(redactedBody),
|
|
557
|
-
redactedBodyBytes: utf8ByteLength(redactedBody),
|
|
558
|
-
storedFileBytes: compressed.byteLength,
|
|
559
|
-
redactedBody,
|
|
560
|
-
bodyTruncated,
|
|
561
|
-
};
|
|
562
|
-
}
|
|
420
|
+
/** Publish redacted UTF-8 chunks, yielding between groups to keep requests responsive. */
|
|
563
421
|
function emitOtlpBodyLogRecord(entry, stored) {
|
|
564
|
-
resolveLoggerProvider()
|
|
565
|
-
.then((provider) => {
|
|
422
|
+
return resolveLoggerProvider()
|
|
423
|
+
.then(async (provider) => {
|
|
566
424
|
if (!provider || stored.redactedBody === undefined) {
|
|
567
425
|
return;
|
|
568
426
|
}
|
|
@@ -570,6 +428,9 @@ function emitOtlpBodyLogRecord(entry, stored) {
|
|
|
570
428
|
const chunks = splitUtf8StringByBytes(stored.redactedBody, BODY_OTLP_CHUNK_SIZE);
|
|
571
429
|
const totalChunks = Math.max(1, chunks.length);
|
|
572
430
|
for (let chunkIndex = 0; chunkIndex < totalChunks; chunkIndex++) {
|
|
431
|
+
if (chunkIndex > 0 && chunkIndex % 4 === 0) {
|
|
432
|
+
await yieldToRequests();
|
|
433
|
+
}
|
|
573
434
|
const chunk = chunks[chunkIndex] ?? "";
|
|
574
435
|
otelLogger.emit({
|
|
575
436
|
severityNumber: (entry.responseStatus ?? 0) >= 400
|
|
@@ -621,6 +482,7 @@ function emitOtlpBodyLogRecord(entry, stored) {
|
|
|
621
482
|
// Non-fatal — never crash proxy for OTLP log failures
|
|
622
483
|
});
|
|
623
484
|
}
|
|
485
|
+
/** Capture an owned request body with bounded processing and tracked index/export publication. */
|
|
624
486
|
export async function logBodyCapture(entry) {
|
|
625
487
|
if (!logEnabled || !logDir) {
|
|
626
488
|
return;
|
|
@@ -636,61 +498,61 @@ export async function logBodyCapture(entry) {
|
|
|
636
498
|
const traceCtx = entry.traceId && entry.spanId
|
|
637
499
|
? { traceId: entry.traceId, spanId: entry.spanId }
|
|
638
500
|
: bridge.getCurrentTraceContext();
|
|
639
|
-
const
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
501
|
+
const destination = logDir;
|
|
502
|
+
// Publication callbacks retain metadata and the bounded redacted result,
|
|
503
|
+
// never the original unbounded body while a sink is slow.
|
|
504
|
+
const metadata = { ...entry, body: undefined };
|
|
505
|
+
/** Persist the processed capture index and publish its redacted body before releasing capacity. */
|
|
506
|
+
const consume = async (processed) => {
|
|
507
|
+
const redactedHeaders = processed.headers;
|
|
508
|
+
const stored = processed.stored;
|
|
509
|
+
const dateStr = new Date(metadata.timestamp).toISOString().split("T")[0];
|
|
510
|
+
const logFile = join(destination, `proxy-debug-${dateStr}.jsonl`);
|
|
511
|
+
const indexEntry = {
|
|
512
|
+
timestamp: metadata.timestamp,
|
|
513
|
+
type: "body_capture",
|
|
514
|
+
requestId: metadata.requestId,
|
|
515
|
+
phase: metadata.phase,
|
|
516
|
+
model: metadata.model,
|
|
517
|
+
stream: metadata.stream,
|
|
518
|
+
headers: redactedHeaders,
|
|
519
|
+
contentType: metadata.contentType,
|
|
520
|
+
responseStatus: metadata.responseStatus,
|
|
521
|
+
durationMs: metadata.durationMs,
|
|
522
|
+
account: metadata.account,
|
|
523
|
+
accountType: metadata.accountType,
|
|
524
|
+
attempt: metadata.attempt,
|
|
525
|
+
bodyPath: stored.bodyPath,
|
|
526
|
+
bodySha256: stored.bodySha256,
|
|
527
|
+
observedBodyBytes: metadata.bodySize,
|
|
528
|
+
redactedBodyBytes: stored.redactedBodyBytes,
|
|
529
|
+
storedFileBytes: stored.storedFileBytes,
|
|
530
|
+
bodyTruncated: stored.bodyTruncated,
|
|
531
|
+
bodyWriteFailed: stored.bodyWriteFailed,
|
|
532
|
+
captureError: processed.error,
|
|
533
|
+
captureQueueWaitMs: processed.queueWaitMs,
|
|
534
|
+
captureProcessingMs: processed.processingMs,
|
|
535
|
+
metadata: processed.error ? undefined : metadata.metadata,
|
|
652
536
|
};
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
bodyPath: stored.bodyPath,
|
|
671
|
-
bodySha256: stored.bodySha256,
|
|
672
|
-
observedBodyBytes: entry.bodySize,
|
|
673
|
-
redactedBodyBytes: stored.redactedBodyBytes ?? preparedBody.bytes,
|
|
674
|
-
storedFileBytes: stored.storedFileBytes,
|
|
675
|
-
bodyTruncated: stored.bodyTruncated ?? preparedBody.truncated,
|
|
676
|
-
bodyWriteFailed: stored.bodyWriteFailed,
|
|
677
|
-
metadata: entry.metadata,
|
|
537
|
+
if (traceCtx) {
|
|
538
|
+
indexEntry.traceId = traceCtx.traceId;
|
|
539
|
+
indexEntry.spanId = traceCtx.spanId;
|
|
540
|
+
}
|
|
541
|
+
try {
|
|
542
|
+
await appendMetadataRecord(logFile, JSON.stringify(indexEntry) + "\n", "debug", { waitForPersistence: true });
|
|
543
|
+
}
|
|
544
|
+
catch {
|
|
545
|
+
// Non-fatal
|
|
546
|
+
}
|
|
547
|
+
// Emission yields between chunk groups. Keep it in the shutdown flush set
|
|
548
|
+
// so an exporter flush cannot race unfinished body-log publication.
|
|
549
|
+
await emitOtlpBodyLogRecord({
|
|
550
|
+
...metadata,
|
|
551
|
+
traceId: traceCtx?.traceId ?? metadata.traceId,
|
|
552
|
+
spanId: traceCtx?.spanId ?? metadata.spanId,
|
|
553
|
+
}, stored);
|
|
678
554
|
};
|
|
679
|
-
|
|
680
|
-
indexEntry.traceId = traceCtx.traceId;
|
|
681
|
-
indexEntry.spanId = traceCtx.spanId;
|
|
682
|
-
}
|
|
683
|
-
try {
|
|
684
|
-
await appendMetadataRecord(logFile, JSON.stringify(indexEntry) + "\n", "debug");
|
|
685
|
-
}
|
|
686
|
-
catch {
|
|
687
|
-
// Non-fatal
|
|
688
|
-
}
|
|
689
|
-
emitOtlpBodyLogRecord({
|
|
690
|
-
...entry,
|
|
691
|
-
traceId: traceCtx?.traceId ?? entry.traceId,
|
|
692
|
-
spanId: traceCtx?.spanId ?? entry.spanId,
|
|
693
|
-
}, stored);
|
|
555
|
+
return trackLogOperation(captureProxyBody(entry, destination, consume));
|
|
694
556
|
}
|
|
695
557
|
/**
|
|
696
558
|
* Log the FULL raw request and response for debugging.
|
|
@@ -787,7 +649,13 @@ export function cleanupLogsAt(activeLogDir, maxAgeDays = 7, maxSizeMb = 500) {
|
|
|
787
649
|
}
|
|
788
650
|
const files = collectManagedLogFiles(activeLogDir).sort((a, b) => a.mtime - b.mtime); // oldest first
|
|
789
651
|
const currentDate = new Date().toISOString().split("T")[0];
|
|
790
|
-
const currentMetadataLogs = new Set([
|
|
652
|
+
const currentMetadataLogs = new Set([
|
|
653
|
+
"proxy",
|
|
654
|
+
"proxy-attempts",
|
|
655
|
+
"proxy-debug",
|
|
656
|
+
"proxy-lifecycle",
|
|
657
|
+
"proxy-supervisor",
|
|
658
|
+
].map((prefix) => join(activeLogDir, `${prefix}-${currentDate}.jsonl`)));
|
|
791
659
|
const canDelete = (file) => !currentMetadataLogs.has(file.path);
|
|
792
660
|
const cutoff = Date.now() - maxAgeDays * 24 * 60 * 60 * 1000;
|
|
793
661
|
let deletedCount = 0;
|
|
@@ -115,10 +115,14 @@ export async function startRollingProxyServer(options) {
|
|
|
115
115
|
socketQueueTimeoutMs: options.socketQueueTimeoutMs,
|
|
116
116
|
shutdownTimeoutMs: options.shutdownTimeoutMs,
|
|
117
117
|
onStateChange: stateChanged,
|
|
118
|
+
onEvent: options.onEvent,
|
|
118
119
|
onReplacementRequested: scheduleRequestedReplacement,
|
|
119
120
|
log: options.log,
|
|
120
121
|
});
|
|
122
|
+
const ownedSockets = new Set();
|
|
121
123
|
const listener = createServer({ pauseOnConnect: true }, (socket) => {
|
|
124
|
+
ownedSockets.add(socket);
|
|
125
|
+
socket.once("close", () => ownedSockets.delete(socket));
|
|
122
126
|
// The parent keeps its descriptor until the worker commits the IPC
|
|
123
127
|
// transfer. Consume client resets during that interval so they cannot
|
|
124
128
|
// terminate the long-lived supervisor process.
|
|
@@ -216,10 +220,17 @@ export async function startRollingProxyServer(options) {
|
|
|
216
220
|
}
|
|
217
221
|
requestedReplacementSchedule += 1;
|
|
218
222
|
requestedReplacementPending = false;
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
+
// Stop accepting immediately, then await the ownership we actually hold.
|
|
224
|
+
// Node's net.Server can retain _usingWorkers=true with an empty worker
|
|
225
|
+
// list after an IPC recipient exits. Its close callback then never fires,
|
|
226
|
+
// even though the listener and every descriptor have closed. Avoid that
|
|
227
|
+
// private bookkeeping path; the supervisor owns remote worker draining.
|
|
228
|
+
listener.close();
|
|
229
|
+
await supervisor.close();
|
|
230
|
+
await Promise.all([...ownedSockets].map((socket) => new Promise((resolve) => {
|
|
231
|
+
socket.once("close", resolve);
|
|
232
|
+
socket.destroy();
|
|
233
|
+
})));
|
|
223
234
|
},
|
|
224
235
|
};
|
|
225
236
|
}
|
|
@@ -1,2 +1,6 @@
|
|
|
1
1
|
import type { RollingWorkerHandle, SpawnProxySocketWorkerOptions } from "../types/index.js";
|
|
2
|
+
/**
|
|
3
|
+
* Spawn a generation-specific IPC worker with separate offer and commit
|
|
4
|
+
* ownership deadlines.
|
|
5
|
+
*/
|
|
2
6
|
export declare function spawnProxySocketWorker(options: SpawnProxySocketWorkerOptions): RollingWorkerHandle;
|
|
@@ -1,6 +1,10 @@
|
|
|
1
1
|
import { spawn } from "node:child_process";
|
|
2
2
|
import { ErrorFactory } from "../utils/errorHandling.js";
|
|
3
|
-
import { isProxyWorkerStatusMessage, PROXY_SOCKET_WORKER_ENV, PROXY_SOCKET_OFFER_TIMEOUT, } from "./rollingWorkerProtocol.js";
|
|
3
|
+
import { isProxyWorkerStatusMessage, PROXY_SOCKET_WORKER_ENV, PROXY_SOCKET_OFFER_TIMEOUT, PROXY_SOCKET_COMMIT_TIMEOUT, } from "./rollingWorkerProtocol.js";
|
|
4
|
+
/**
|
|
5
|
+
* Spawn a generation-specific IPC worker with separate offer and commit
|
|
6
|
+
* ownership deadlines.
|
|
7
|
+
*/
|
|
4
8
|
export function spawnProxySocketWorker(options) {
|
|
5
9
|
const socketAckTimeoutMs = Math.max(1, options.socketAckTimeoutMs ?? 30_000);
|
|
6
10
|
let nextSocketId = 0;
|
|
@@ -8,7 +12,7 @@ export function spawnProxySocketWorker(options) {
|
|
|
8
12
|
const statusListeners = new Set();
|
|
9
13
|
const pendingStatusMessages = [];
|
|
10
14
|
let spawnError;
|
|
11
|
-
const child = spawn(options.command, options.args, {
|
|
15
|
+
const child = (options.spawn ?? spawn)(options.command, options.args, {
|
|
12
16
|
env: {
|
|
13
17
|
...process.env,
|
|
14
18
|
...options.env,
|
|
@@ -52,6 +56,10 @@ export function spawnProxySocketWorker(options) {
|
|
|
52
56
|
child.kill("SIGTERM");
|
|
53
57
|
throw ErrorFactory.proxyWorkerLifecycle("proxy worker spawn did not return a pid");
|
|
54
58
|
}
|
|
59
|
+
/**
|
|
60
|
+
* Finish a handoff once and cancel its worker-side copy on uncertain
|
|
61
|
+
* delivery.
|
|
62
|
+
*/
|
|
55
63
|
const settleSocket = (socketId, error) => {
|
|
56
64
|
const pending = pendingSockets.get(socketId);
|
|
57
65
|
if (!pending) {
|
|
@@ -59,7 +67,7 @@ export function spawnProxySocketWorker(options) {
|
|
|
59
67
|
}
|
|
60
68
|
pendingSockets.delete(socketId);
|
|
61
69
|
clearTimeout(pending.timeout);
|
|
62
|
-
if (error && child.connected
|
|
70
|
+
if (error && child.connected) {
|
|
63
71
|
try {
|
|
64
72
|
child.send({
|
|
65
73
|
type: "proxy-worker:socket-cancel",
|
|
@@ -76,6 +84,10 @@ export function spawnProxySocketWorker(options) {
|
|
|
76
84
|
}
|
|
77
85
|
pending.callback(error);
|
|
78
86
|
};
|
|
87
|
+
/**
|
|
88
|
+
* Accept messages only from this worker and start a fresh deadline for
|
|
89
|
+
* the commit phase.
|
|
90
|
+
*/
|
|
79
91
|
const onInternalMessage = (message) => {
|
|
80
92
|
if (isProxyWorkerStatusMessage(message) &&
|
|
81
93
|
message.type === "proxy-worker:socket-accepted" &&
|
|
@@ -86,6 +98,15 @@ export function spawnProxySocketWorker(options) {
|
|
|
86
98
|
return;
|
|
87
99
|
}
|
|
88
100
|
pending.accepted = true;
|
|
101
|
+
// Acceptance and commit are distinct phases. A late acceptance must not
|
|
102
|
+
// inherit an almost-expired offer timer and kill established streams.
|
|
103
|
+
clearTimeout(pending.timeout);
|
|
104
|
+
pending.timeout = setTimeout(() => {
|
|
105
|
+
const error = new Error(`proxy worker ${childPid} socket commit remained pending for ${socketAckTimeoutMs}ms`);
|
|
106
|
+
error.code = PROXY_SOCKET_COMMIT_TIMEOUT;
|
|
107
|
+
settleSocket(message.socketId, error);
|
|
108
|
+
}, socketAckTimeoutMs);
|
|
109
|
+
pending.timeout.unref?.();
|
|
89
110
|
try {
|
|
90
111
|
child.send({
|
|
91
112
|
type: "proxy-worker:socket-commit",
|
|
@@ -135,11 +156,7 @@ export function spawnProxySocketWorker(options) {
|
|
|
135
156
|
const socketId = `${generation}:${++nextSocketId}`;
|
|
136
157
|
const timeout = setTimeout(() => {
|
|
137
158
|
const error = new Error(`proxy worker ${childPid} did not accept socket within ${socketAckTimeoutMs}ms`);
|
|
138
|
-
|
|
139
|
-
// No commit was sent. The cancel message settles this offer without
|
|
140
|
-
// terminating unrelated requests already owned by the worker.
|
|
141
|
-
error.code = PROXY_SOCKET_OFFER_TIMEOUT;
|
|
142
|
-
}
|
|
159
|
+
error.code = PROXY_SOCKET_OFFER_TIMEOUT;
|
|
143
160
|
settleSocket(socketId, error);
|
|
144
161
|
}, socketAckTimeoutMs);
|
|
145
162
|
timeout.unref?.();
|
|
@@ -2,6 +2,12 @@ import type { ProxyWorkerControlMessage, ProxyWorkerStatusMessage } from "../typ
|
|
|
2
2
|
export declare const PROXY_SOCKET_WORKER_ENV = "NEUROLINK_PROXY_SOCKET_WORKER";
|
|
3
3
|
/** The worker has not been sent a commit and cannot have served this socket. */
|
|
4
4
|
export declare const PROXY_SOCKET_OFFER_TIMEOUT = "PROXY_SOCKET_OFFER_TIMEOUT";
|
|
5
|
+
/** Commit delivery is uncertain; close only this socket and never replay it. */
|
|
6
|
+
export declare const PROXY_SOCKET_COMMIT_TIMEOUT = "PROXY_SOCKET_COMMIT_TIMEOUT";
|
|
5
7
|
export declare const PROXY_ROLLING_SUPERVISOR_ENV = "NEUROLINK_PROXY_ROLLING_SUPERVISOR";
|
|
6
8
|
export declare function isProxyWorkerControlMessage(value: unknown): value is ProxyWorkerControlMessage;
|
|
9
|
+
/**
|
|
10
|
+
* Validate worker status and bounded process identity before the
|
|
11
|
+
* supervisor trusts IPC evidence.
|
|
12
|
+
*/
|
|
7
13
|
export declare function isProxyWorkerStatusMessage(value: unknown): value is ProxyWorkerStatusMessage;
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
export const PROXY_SOCKET_WORKER_ENV = "NEUROLINK_PROXY_SOCKET_WORKER";
|
|
2
2
|
/** The worker has not been sent a commit and cannot have served this socket. */
|
|
3
3
|
export const PROXY_SOCKET_OFFER_TIMEOUT = "PROXY_SOCKET_OFFER_TIMEOUT";
|
|
4
|
+
/** Commit delivery is uncertain; close only this socket and never replay it. */
|
|
5
|
+
export const PROXY_SOCKET_COMMIT_TIMEOUT = "PROXY_SOCKET_COMMIT_TIMEOUT";
|
|
4
6
|
export const PROXY_ROLLING_SUPERVISOR_ENV = "NEUROLINK_PROXY_ROLLING_SUPERVISOR";
|
|
5
7
|
export function isProxyWorkerControlMessage(value) {
|
|
6
8
|
if (!value || typeof value !== "object") {
|
|
@@ -19,6 +21,10 @@ export function isProxyWorkerControlMessage(value) {
|
|
|
19
21
|
message.type === "proxy-worker:activate" ||
|
|
20
22
|
message.type === "proxy-worker:shutdown");
|
|
21
23
|
}
|
|
24
|
+
/**
|
|
25
|
+
* Validate worker status and bounded process identity before the
|
|
26
|
+
* supervisor trusts IPC evidence.
|
|
27
|
+
*/
|
|
22
28
|
export function isProxyWorkerStatusMessage(value) {
|
|
23
29
|
if (!value || typeof value !== "object") {
|
|
24
30
|
return false;
|
|
@@ -31,7 +37,12 @@ export function isProxyWorkerStatusMessage(value) {
|
|
|
31
37
|
return false;
|
|
32
38
|
}
|
|
33
39
|
if (message.type === "proxy-worker:ready") {
|
|
34
|
-
return typeof message.version === "string" &&
|
|
40
|
+
return (typeof message.version === "string" &&
|
|
41
|
+
message.version.length > 0 &&
|
|
42
|
+
(message.processInstanceId === undefined ||
|
|
43
|
+
(typeof message.processInstanceId === "string" &&
|
|
44
|
+
message.processInstanceId.length > 0 &&
|
|
45
|
+
message.processInstanceId.length <= 256)));
|
|
35
46
|
}
|
|
36
47
|
if (message.type === "proxy-worker:activated" ||
|
|
37
48
|
message.type === "proxy-worker:drained") {
|
|
@@ -27,22 +27,50 @@ export declare class RollingWorkerSupervisor {
|
|
|
27
27
|
snapshot(): RollingWorkerSupervisorSnapshot;
|
|
28
28
|
start(expectedVersion: string): Promise<RollingWorkerSupervisorSnapshot>;
|
|
29
29
|
replace(expectedVersion: string): Promise<RollingWorkerSupervisorSnapshot>;
|
|
30
|
+
/**
|
|
31
|
+
* Admit a paused socket to the active generation or the bounded
|
|
32
|
+
* readiness queue.
|
|
33
|
+
*/
|
|
30
34
|
acceptSocket(socket: TransferableProxySocket): void;
|
|
35
|
+
/**
|
|
36
|
+
* Retain a socket only within the queue capacity and deadline, recording
|
|
37
|
+
* classified rejection.
|
|
38
|
+
*/
|
|
31
39
|
private queueSocket;
|
|
32
40
|
close(): Promise<void>;
|
|
33
41
|
private closeWorkers;
|
|
34
42
|
private requestWorkerShutdown;
|
|
35
43
|
private forceTerminateWorkers;
|
|
36
44
|
private notifyShutdownWaiters;
|
|
45
|
+
/**
|
|
46
|
+
* Validate candidate readiness and activation while preserving an
|
|
47
|
+
* independent listener for actual exit.
|
|
48
|
+
*/
|
|
37
49
|
private spawnCandidate;
|
|
38
50
|
private flushQueuedSockets;
|
|
39
51
|
private transferSocket;
|
|
40
52
|
private maybeDrainWorker;
|
|
53
|
+
/**
|
|
54
|
+
* Cancel the affected handoff and request bounded replacement without
|
|
55
|
+
* killing unrelated streams.
|
|
56
|
+
*/
|
|
41
57
|
private handleTransferFailure;
|
|
58
|
+
/**
|
|
59
|
+
* Record a classified admission rejection before releasing the
|
|
60
|
+
* parent-owned socket.
|
|
61
|
+
*/
|
|
42
62
|
private rejectSocket;
|
|
43
63
|
private describeTransferError;
|
|
44
64
|
private extractLifecycleFailureDetails;
|
|
65
|
+
/**
|
|
66
|
+
* Retain the latest failure and its observed process evidence in the
|
|
67
|
+
* incident journal.
|
|
68
|
+
*/
|
|
45
69
|
private recordFailure;
|
|
70
|
+
/**
|
|
71
|
+
* Publish an independent incident record and retain only a bounded
|
|
72
|
+
* recent summary.
|
|
73
|
+
*/
|
|
46
74
|
private recordEvent;
|
|
47
75
|
private scheduleTransferState;
|
|
48
76
|
private publishState;
|