@juspay/neurolink 12.12.8 → 12.12.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. package/CHANGELOG.md +2 -3
  2. package/dist/browser/neurolink.min.js +394 -395
  3. package/dist/cli/commands/proxy.d.ts +4 -0
  4. package/dist/cli/commands/proxy.js +60 -18
  5. package/dist/cli/commands/proxyAnalyze.js +8 -1
  6. package/dist/proxy/bodyCaptureProcessing.d.ts +22 -0
  7. package/dist/proxy/bodyCaptureProcessing.js +219 -0
  8. package/dist/proxy/bodyCaptureWorker.d.ts +16 -0
  9. package/dist/proxy/bodyCaptureWorker.js +232 -0
  10. package/dist/proxy/bodyCaptureWorkerEntry.d.ts +1 -0
  11. package/dist/proxy/bodyCaptureWorkerEntry.js +34 -0
  12. package/dist/proxy/proxyAnalysis.js +159 -17
  13. package/dist/proxy/proxyLifecycle.d.ts +25 -0
  14. package/dist/proxy/proxyLifecycle.js +111 -5
  15. package/dist/proxy/proxyRequestKind.d.ts +2 -0
  16. package/dist/proxy/proxyRequestKind.js +6 -0
  17. package/dist/proxy/proxyRuntimeMetrics.d.ts +3 -0
  18. package/dist/proxy/proxyRuntimeMetrics.js +34 -0
  19. package/dist/proxy/requestLogger.d.ts +10 -6
  20. package/dist/proxy/requestLogger.js +99 -231
  21. package/dist/proxy/rollingProxyServer.js +15 -4
  22. package/dist/proxy/rollingWorkerProcess.d.ts +4 -0
  23. package/dist/proxy/rollingWorkerProcess.js +25 -8
  24. package/dist/proxy/rollingWorkerProtocol.d.ts +6 -0
  25. package/dist/proxy/rollingWorkerProtocol.js +12 -1
  26. package/dist/proxy/rollingWorkerSupervisor.d.ts +28 -0
  27. package/dist/proxy/rollingWorkerSupervisor.js +73 -25
  28. package/dist/proxy/socketWorkerRuntime.d.ts +5 -0
  29. package/dist/proxy/socketWorkerRuntime.js +19 -2
  30. package/dist/server/routes/codexProxyRoutes.js +39 -3
  31. package/dist/services/server/ai/observability/instrumentation.js +7 -1
  32. package/dist/types/cli.d.ts +1 -1
  33. package/dist/types/proxy.d.ts +69 -4
  34. package/dist/utils/schemaConversion.d.ts +9 -0
  35. package/dist/utils/schemaConversion.js +12 -1
  36. package/package.json +2 -1
@@ -10,9 +10,9 @@ import { homedir } from "os";
10
10
  import { logger } from "../utils/logger.js";
11
11
  import { chmodSync, existsSync, mkdirSync, readdirSync, rmSync, statSync, unlinkSync, } from "fs";
12
12
  import { writeFile } from "fs/promises";
13
- import { createHash } from "crypto";
14
- import { promisify } from "util";
15
- import { gzip as gzipCallback } from "zlib";
13
+ import { setImmediate as yieldToRequests } from "node:timers/promises";
14
+ import { captureProxyBody, getBodyCaptureWorkerSnapshot, PROXY_BODY_CAPTURE_DEADLINE_MS, } from "./bodyCaptureWorker.js";
15
+ import { prepareProxyBodyForLogging as prepareRedactedBody, redactProxyHeadersForLogging as redactHeaders, splitUtf8StringByBytes, } from "./bodyCaptureProcessing.js";
16
16
  import { isBorrowedRequest } from "./shareContext.js";
17
17
  import { OtelBridge } from "../observability/otelBridge.js";
18
18
  import { SeverityNumber } from "@opentelemetry/api-logs";
@@ -40,15 +40,22 @@ const metadataSinks = {
40
40
  attempts: createSinkSnapshot(),
41
41
  debug: createSinkSnapshot(),
42
42
  };
43
+ /**
44
+ * Expose independent metadata-sink and body-capture counters for incident reconciliation.
45
+ */
43
46
  export function getRequestLoggerSnapshot() {
44
47
  return {
45
48
  enabled: logEnabled,
46
49
  requests: { ...metadataSinks.requests },
47
50
  attempts: { ...metadataSinks.attempts },
48
51
  debug: { ...metadataSinks.debug },
52
+ bodyCapture: getBodyCaptureWorkerSnapshot(),
49
53
  };
50
54
  }
51
- async function appendMetadataRecord(file, line, kind) {
55
+ /**
56
+ * Serialize append ownership; capture callers retain their memory lease until the write settles.
57
+ */
58
+ async function appendMetadataRecord(file, line, kind, options = {}) {
52
59
  const sink = metadataSinks[kind];
53
60
  sink.attempted += 1;
54
61
  if (sink.pending + sink.inFlight >= MAX_PENDING_METADATA_RECORDS) {
@@ -86,16 +93,25 @@ async function appendMetadataRecord(file, line, kind) {
86
93
  appendChains.delete(file);
87
94
  }
88
95
  });
96
+ if (options.waitForPersistence) {
97
+ // Bulk capture owns a memory lease until publication settles. A caller
98
+ // timeout must not release that lease while the serialized index is queued.
99
+ return operation;
100
+ }
89
101
  // Bound the caller's wait, not the lifetime/ownership of the underlying write.
90
102
  await withTimeout(operation, REQUEST_LOG_IO_TIMEOUT_MS, "Proxy metadata write remains pending").catch(() => undefined);
91
103
  }
104
+ /**
105
+ * Retain asynchronous log ownership until settlement so shutdown can await admitted publication.
106
+ */
92
107
  function trackLogOperation(operation) {
93
108
  pendingLogOperations.add(operation);
94
109
  void operation.then(() => pendingLogOperations.delete(operation), () => pendingLogOperations.delete(operation));
95
110
  return operation;
96
111
  }
97
112
  /** Wait, up to a bounded deadline, for admitted request/body writes to settle. */
98
- export async function flushRequestLogs(timeoutMs = REQUEST_LOG_IO_TIMEOUT_MS) {
113
+ export async function flushRequestLogs(timeoutMs = PROXY_BODY_CAPTURE_DEADLINE_MS +
114
+ 2 * REQUEST_LOG_IO_TIMEOUT_MS) {
99
115
  const deadline = Date.now() + Math.max(1, timeoutMs);
100
116
  while (pendingLogOperations.size > 0) {
101
117
  const admitted = [...pendingLogOperations];
@@ -127,24 +143,10 @@ let otelLoggerProvider = null;
127
143
  let otelResolveAttempts = 0;
128
144
  /** Max number of resolve attempts before giving up. */
129
145
  const MAX_RESOLVE_ATTEMPTS = 10;
130
- /** Maximum body chunk size emitted to OTLP logs. */
131
146
  const BODY_OTLP_CHUNK_SIZE = 16_000;
132
- /** Maximum redacted body bytes persisted per capture entry. */
133
- const MAX_CAPTURED_BODY_BYTES = 1024 * 1024;
134
- const BODY_TRUNCATION_MARKER = "\n...[TRUNCATED]";
135
- const gzip = promisify(gzipCallback);
136
- /** Headers whose values must always be redacted. */
137
- const SENSITIVE_HEADER_NAMES = new Set([
138
- "authorization",
139
- "proxy-authorization",
140
- "x-api-key",
141
- "cookie",
142
- "set-cookie",
143
- ]);
144
- /** Pattern that matches header names likely to contain secrets. */
145
- const SENSITIVE_HEADER_PATTERN = /token|secret|key|password|credential/i;
146
- /** JSON keys whose values should be redacted in request/response bodies. */
147
- const SENSITIVE_BODY_KEYS = /("(?:password|access_token|refresh_token|api_key|apiKey|secret|authorization|token|credential|x-api-key)"\s*:\s*)"(?:[^"\\]|\\.)*"/gi;
147
+ /**
148
+ * Initialize private request logs and preserve required lifecycle admission on startup failures.
149
+ */
148
150
  export function initRequestLogger(enabled = true, customLogsDir) {
149
151
  // Lifecycle metadata deliberately shares the request logger's enablement,
150
152
  // directory permissions, retention boundary, and operator privacy control.
@@ -164,7 +166,10 @@ export function initRequestLogger(enabled = true, customLogsDir) {
164
166
  catch (err) {
165
167
  logEnabled = false;
166
168
  logDir = null;
167
- configureProxyLifecycleLogger({ enabled: false });
169
+ configureProxyLifecycleLogger({
170
+ enabled: true,
171
+ logDir: customLogsDir ?? join(homedir(), ".neurolink", "logs"),
172
+ });
168
173
  logger.warn(`[proxy] Request logging disabled — failed to create log directory: ${err instanceof Error ? err.message : String(err)}`);
169
174
  }
170
175
  }
@@ -352,119 +357,17 @@ export function getLogDir() {
352
357
  /**
353
358
  * Redact sensitive header values in-place.
354
359
  */
355
- function redactHeaders(headers) {
356
- if (!headers) {
357
- return headers;
358
- }
359
- const redacted = {};
360
- for (const [key, value] of Object.entries(headers)) {
361
- const lower = key.toLowerCase();
362
- if (SENSITIVE_HEADER_NAMES.has(lower) ||
363
- SENSITIVE_HEADER_PATTERN.test(lower)) {
364
- redacted[key] = "[REDACTED]";
365
- }
366
- else {
367
- redacted[key] = value;
368
- }
369
- }
370
- return redacted;
371
- }
372
- function serializeBody(body) {
373
- if (body === undefined || body === null) {
374
- return undefined;
375
- }
376
- return typeof body === "string" ? body : JSON.stringify(body);
377
- }
378
- /**
379
- * Redact sensitive keys from a JSON body string without truncation.
380
- */
381
- function redactBody(body) {
382
- const str = serializeBody(body);
383
- if (str === undefined) {
384
- return undefined;
385
- }
386
- return str.replace(SENSITIVE_BODY_KEYS, '$1"[REDACTED]"');
387
- }
388
- function sanitizePhase(phase) {
389
- return phase.replace(/[^a-zA-Z0-9._-]+/g, "_");
390
- }
391
- function sha256(value) {
392
- return createHash("sha256").update(value).digest("hex");
393
- }
394
- function utf8ByteLength(value) {
395
- return Buffer.byteLength(value, "utf8");
396
- }
397
- function truncateUtf8String(input, maxBytes, marker = BODY_TRUNCATION_MARKER) {
398
- const inputBytes = utf8ByteLength(input);
399
- if (inputBytes <= maxBytes) {
400
- return { value: input, bytes: inputBytes, truncated: false };
401
- }
402
- const markerBytes = utf8ByteLength(marker);
403
- if (maxBytes <= markerBytes) {
404
- return { value: marker, bytes: markerBytes, truncated: true };
405
- }
406
- let value = "";
407
- let bytes = 0;
408
- for (const char of input) {
409
- const charBytes = utf8ByteLength(char);
410
- if (bytes + charBytes + markerBytes > maxBytes) {
411
- break;
412
- }
413
- value += char;
414
- bytes += charBytes;
415
- }
416
- const truncatedValue = `${value}${marker}`;
417
- return {
418
- value: truncatedValue,
419
- bytes: utf8ByteLength(truncatedValue),
420
- truncated: true,
421
- };
422
- }
423
- function splitUtf8StringByBytes(input, maxBytes) {
424
- if (!input) {
425
- return [""];
426
- }
427
- const chunks = [];
428
- let currentChunk = "";
429
- let currentBytes = 0;
430
- for (const char of input) {
431
- const charBytes = utf8ByteLength(char);
432
- if (currentChunk && currentBytes + charBytes > maxBytes) {
433
- chunks.push(currentChunk);
434
- currentChunk = char;
435
- currentBytes = charBytes;
436
- continue;
437
- }
438
- currentChunk += char;
439
- currentBytes += charBytes;
440
- }
441
- if (currentChunk) {
442
- chunks.push(currentChunk);
443
- }
444
- return chunks;
445
- }
446
- function prepareRedactedBody(body) {
447
- const redacted = redactBody(body);
448
- if (redacted === undefined) {
449
- return { truncated: false };
450
- }
451
- return truncateUtf8String(redacted, MAX_CAPTURED_BODY_BYTES);
452
- }
453
- /** Shared redaction used by offline replay exports and direct comparisons. */
454
360
  export function redactProxyHeadersForLogging(headers) {
455
361
  return redactHeaders(headers);
456
362
  }
457
- /**
458
- * Apply the same bounded body redaction used by persisted proxy captures.
459
- * `value` and `bytes` are omitted only when the input is null or undefined.
460
- * This performs serialization immediately, so callers must keep it off proxy
461
- * hot paths unless body processing has already been explicitly requested.
462
- */
363
+ /** Return a redacted body representation suitable for persisted request diagnostics. */
463
364
  export function prepareProxyBodyForLogging(body) {
464
365
  return prepareRedactedBody(body);
465
366
  }
367
+ /** Enumerate recognized proxy journals and body artifacts for retention accounting. */
466
368
  function collectManagedLogFiles(rootDir) {
467
369
  const managedFiles = [];
370
+ /** Collect file sizes and modification times while descending the log directory. */
468
371
  const walk = (directory) => {
469
372
  for (const entry of readdirSync(directory, { withFileTypes: true })) {
470
373
  const entryPath = join(directory, entry.name);
@@ -473,7 +376,7 @@ function collectManagedLogFiles(rootDir) {
473
376
  continue;
474
377
  }
475
378
  const isTopLevelProxyLog = directory === rootDir &&
476
- /^proxy(?:-attempts|-debug|-lifecycle)?-.*\.jsonl$/.test(entry.name);
379
+ /^proxy(?:-attempts|-debug|-lifecycle|-supervisor)?-.*\.jsonl$/.test(entry.name);
477
380
  const isBodyArtifact = entry.name.endsWith(".json.gz") &&
478
381
  entryPath.includes(`${join(rootDir, "bodies")}`);
479
382
  if (!isTopLevelProxyLog && !isBodyArtifact) {
@@ -514,55 +417,10 @@ function pruneEmptyDirectories(directory, stopAt) {
514
417
  // Non-fatal
515
418
  }
516
419
  }
517
- async function writeBodyArtifact(entry, redactedHeaders, redactedBody, bodyTruncated) {
518
- if (!logDir || redactedBody === undefined) {
519
- return {};
520
- }
521
- const dateStr = new Date(entry.timestamp).toISOString().split("T")[0];
522
- const bodyDir = join(logDir, "bodies", dateStr, entry.requestId);
523
- if (!existsSync(bodyDir)) {
524
- mkdirSync(bodyDir, { recursive: true, mode: 0o700 });
525
- }
526
- chmodSync(bodyDir, 0o700);
527
- const fileName = `${Date.now()}-${sanitizePhase(entry.phase)}` +
528
- (entry.attempt !== undefined ? `-attempt-${entry.attempt}` : "") +
529
- `.json.gz`;
530
- const bodyPath = join(bodyDir, fileName);
531
- const payload = JSON.stringify({
532
- timestamp: entry.timestamp,
533
- requestId: entry.requestId,
534
- phase: entry.phase,
535
- model: entry.model,
536
- stream: entry.stream,
537
- account: entry.account,
538
- accountType: entry.accountType,
539
- attempt: entry.attempt,
540
- responseStatus: entry.responseStatus,
541
- durationMs: entry.durationMs,
542
- contentType: entry.contentType,
543
- headers: redactedHeaders,
544
- body: redactedBody,
545
- traceId: entry.traceId,
546
- spanId: entry.spanId,
547
- metadata: entry.metadata,
548
- });
549
- const compressed = await gzip(payload);
550
- await writeFile(bodyPath, compressed, {
551
- mode: 0o600,
552
- signal: AbortSignal.timeout(REQUEST_LOG_IO_TIMEOUT_MS),
553
- });
554
- return {
555
- bodyPath,
556
- bodySha256: sha256(redactedBody),
557
- redactedBodyBytes: utf8ByteLength(redactedBody),
558
- storedFileBytes: compressed.byteLength,
559
- redactedBody,
560
- bodyTruncated,
561
- };
562
- }
420
+ /** Publish redacted UTF-8 chunks, yielding between groups to keep requests responsive. */
563
421
  function emitOtlpBodyLogRecord(entry, stored) {
564
- resolveLoggerProvider()
565
- .then((provider) => {
422
+ return resolveLoggerProvider()
423
+ .then(async (provider) => {
566
424
  if (!provider || stored.redactedBody === undefined) {
567
425
  return;
568
426
  }
@@ -570,6 +428,9 @@ function emitOtlpBodyLogRecord(entry, stored) {
570
428
  const chunks = splitUtf8StringByBytes(stored.redactedBody, BODY_OTLP_CHUNK_SIZE);
571
429
  const totalChunks = Math.max(1, chunks.length);
572
430
  for (let chunkIndex = 0; chunkIndex < totalChunks; chunkIndex++) {
431
+ if (chunkIndex > 0 && chunkIndex % 4 === 0) {
432
+ await yieldToRequests();
433
+ }
573
434
  const chunk = chunks[chunkIndex] ?? "";
574
435
  otelLogger.emit({
575
436
  severityNumber: (entry.responseStatus ?? 0) >= 400
@@ -621,6 +482,7 @@ function emitOtlpBodyLogRecord(entry, stored) {
621
482
  // Non-fatal — never crash proxy for OTLP log failures
622
483
  });
623
484
  }
485
+ /** Capture an owned request body with bounded processing and tracked index/export publication. */
624
486
  export async function logBodyCapture(entry) {
625
487
  if (!logEnabled || !logDir) {
626
488
  return;
@@ -636,61 +498,61 @@ export async function logBodyCapture(entry) {
636
498
  const traceCtx = entry.traceId && entry.spanId
637
499
  ? { traceId: entry.traceId, spanId: entry.spanId }
638
500
  : bridge.getCurrentTraceContext();
639
- const redactedHeaders = redactHeaders(entry.headers);
640
- const preparedBody = prepareRedactedBody(entry.body);
641
- let stored;
642
- try {
643
- stored = await trackLogOperation(writeBodyArtifact(entry, redactedHeaders, preparedBody.value, preparedBody.truncated));
644
- }
645
- catch (writeError) {
646
- logger.warn("[RequestLogger] writeBodyArtifact failed, falling back to in-memory body for OTLP", { error: writeError });
647
- stored = {
648
- redactedBody: preparedBody.value,
649
- redactedBodyBytes: preparedBody.bytes,
650
- bodyTruncated: preparedBody.truncated,
651
- bodyWriteFailed: true,
501
+ const destination = logDir;
502
+ // Publication callbacks retain metadata and the bounded redacted result,
503
+ // never the original unbounded body while a sink is slow.
504
+ const metadata = { ...entry, body: undefined };
505
+ /** Persist the processed capture index and publish its redacted body before releasing capacity. */
506
+ const consume = async (processed) => {
507
+ const redactedHeaders = processed.headers;
508
+ const stored = processed.stored;
509
+ const dateStr = new Date(metadata.timestamp).toISOString().split("T")[0];
510
+ const logFile = join(destination, `proxy-debug-${dateStr}.jsonl`);
511
+ const indexEntry = {
512
+ timestamp: metadata.timestamp,
513
+ type: "body_capture",
514
+ requestId: metadata.requestId,
515
+ phase: metadata.phase,
516
+ model: metadata.model,
517
+ stream: metadata.stream,
518
+ headers: redactedHeaders,
519
+ contentType: metadata.contentType,
520
+ responseStatus: metadata.responseStatus,
521
+ durationMs: metadata.durationMs,
522
+ account: metadata.account,
523
+ accountType: metadata.accountType,
524
+ attempt: metadata.attempt,
525
+ bodyPath: stored.bodyPath,
526
+ bodySha256: stored.bodySha256,
527
+ observedBodyBytes: metadata.bodySize,
528
+ redactedBodyBytes: stored.redactedBodyBytes,
529
+ storedFileBytes: stored.storedFileBytes,
530
+ bodyTruncated: stored.bodyTruncated,
531
+ bodyWriteFailed: stored.bodyWriteFailed,
532
+ captureError: processed.error,
533
+ captureQueueWaitMs: processed.queueWaitMs,
534
+ captureProcessingMs: processed.processingMs,
535
+ metadata: processed.error ? undefined : metadata.metadata,
652
536
  };
653
- }
654
- const dateStr = new Date(entry.timestamp).toISOString().split("T")[0];
655
- const logFile = join(logDir, `proxy-debug-${dateStr}.jsonl`);
656
- const indexEntry = {
657
- timestamp: entry.timestamp,
658
- type: "body_capture",
659
- requestId: entry.requestId,
660
- phase: entry.phase,
661
- model: entry.model,
662
- stream: entry.stream,
663
- headers: redactedHeaders,
664
- contentType: entry.contentType,
665
- responseStatus: entry.responseStatus,
666
- durationMs: entry.durationMs,
667
- account: entry.account,
668
- accountType: entry.accountType,
669
- attempt: entry.attempt,
670
- bodyPath: stored.bodyPath,
671
- bodySha256: stored.bodySha256,
672
- observedBodyBytes: entry.bodySize,
673
- redactedBodyBytes: stored.redactedBodyBytes ?? preparedBody.bytes,
674
- storedFileBytes: stored.storedFileBytes,
675
- bodyTruncated: stored.bodyTruncated ?? preparedBody.truncated,
676
- bodyWriteFailed: stored.bodyWriteFailed,
677
- metadata: entry.metadata,
537
+ if (traceCtx) {
538
+ indexEntry.traceId = traceCtx.traceId;
539
+ indexEntry.spanId = traceCtx.spanId;
540
+ }
541
+ try {
542
+ await appendMetadataRecord(logFile, JSON.stringify(indexEntry) + "\n", "debug", { waitForPersistence: true });
543
+ }
544
+ catch {
545
+ // Non-fatal
546
+ }
547
+ // Emission yields between chunk groups. Keep it in the shutdown flush set
548
+ // so an exporter flush cannot race unfinished body-log publication.
549
+ await emitOtlpBodyLogRecord({
550
+ ...metadata,
551
+ traceId: traceCtx?.traceId ?? metadata.traceId,
552
+ spanId: traceCtx?.spanId ?? metadata.spanId,
553
+ }, stored);
678
554
  };
679
- if (traceCtx) {
680
- indexEntry.traceId = traceCtx.traceId;
681
- indexEntry.spanId = traceCtx.spanId;
682
- }
683
- try {
684
- await appendMetadataRecord(logFile, JSON.stringify(indexEntry) + "\n", "debug");
685
- }
686
- catch {
687
- // Non-fatal
688
- }
689
- emitOtlpBodyLogRecord({
690
- ...entry,
691
- traceId: traceCtx?.traceId ?? entry.traceId,
692
- spanId: traceCtx?.spanId ?? entry.spanId,
693
- }, stored);
555
+ return trackLogOperation(captureProxyBody(entry, destination, consume));
694
556
  }
695
557
  /**
696
558
  * Log the FULL raw request and response for debugging.
@@ -787,7 +649,13 @@ export function cleanupLogsAt(activeLogDir, maxAgeDays = 7, maxSizeMb = 500) {
787
649
  }
788
650
  const files = collectManagedLogFiles(activeLogDir).sort((a, b) => a.mtime - b.mtime); // oldest first
789
651
  const currentDate = new Date().toISOString().split("T")[0];
790
- const currentMetadataLogs = new Set(["proxy", "proxy-attempts", "proxy-debug", "proxy-lifecycle"].map((prefix) => join(activeLogDir, `${prefix}-${currentDate}.jsonl`)));
652
+ const currentMetadataLogs = new Set([
653
+ "proxy",
654
+ "proxy-attempts",
655
+ "proxy-debug",
656
+ "proxy-lifecycle",
657
+ "proxy-supervisor",
658
+ ].map((prefix) => join(activeLogDir, `${prefix}-${currentDate}.jsonl`)));
791
659
  const canDelete = (file) => !currentMetadataLogs.has(file.path);
792
660
  const cutoff = Date.now() - maxAgeDays * 24 * 60 * 60 * 1000;
793
661
  let deletedCount = 0;
@@ -115,10 +115,14 @@ export async function startRollingProxyServer(options) {
115
115
  socketQueueTimeoutMs: options.socketQueueTimeoutMs,
116
116
  shutdownTimeoutMs: options.shutdownTimeoutMs,
117
117
  onStateChange: stateChanged,
118
+ onEvent: options.onEvent,
118
119
  onReplacementRequested: scheduleRequestedReplacement,
119
120
  log: options.log,
120
121
  });
122
+ const ownedSockets = new Set();
121
123
  const listener = createServer({ pauseOnConnect: true }, (socket) => {
124
+ ownedSockets.add(socket);
125
+ socket.once("close", () => ownedSockets.delete(socket));
122
126
  // The parent keeps its descriptor until the worker commits the IPC
123
127
  // transfer. Consume client resets during that interval so they cannot
124
128
  // terminate the long-lived supervisor process.
@@ -216,10 +220,17 @@ export async function startRollingProxyServer(options) {
216
220
  }
217
221
  requestedReplacementSchedule += 1;
218
222
  requestedReplacementPending = false;
219
- const listenerClosed = new Promise((resolve, reject) => {
220
- listener.close((error) => (error ? reject(error) : resolve()));
221
- });
222
- await Promise.all([listenerClosed, supervisor.close()]);
223
+ // Stop accepting immediately, then await the ownership we actually hold.
224
+ // Node's net.Server can retain _usingWorkers=true with an empty worker
225
+ // list after an IPC recipient exits. Its close callback then never fires,
226
+ // even though the listener and every descriptor have closed. Avoid that
227
+ // private bookkeeping path; the supervisor owns remote worker draining.
228
+ listener.close();
229
+ await supervisor.close();
230
+ await Promise.all([...ownedSockets].map((socket) => new Promise((resolve) => {
231
+ socket.once("close", resolve);
232
+ socket.destroy();
233
+ })));
223
234
  },
224
235
  };
225
236
  }
@@ -1,2 +1,6 @@
1
1
  import type { RollingWorkerHandle, SpawnProxySocketWorkerOptions } from "../types/index.js";
2
+ /**
3
+ * Spawn a generation-specific IPC worker with separate offer and commit
4
+ * ownership deadlines.
5
+ */
2
6
  export declare function spawnProxySocketWorker(options: SpawnProxySocketWorkerOptions): RollingWorkerHandle;
@@ -1,6 +1,10 @@
1
1
  import { spawn } from "node:child_process";
2
2
  import { ErrorFactory } from "../utils/errorHandling.js";
3
- import { isProxyWorkerStatusMessage, PROXY_SOCKET_WORKER_ENV, PROXY_SOCKET_OFFER_TIMEOUT, } from "./rollingWorkerProtocol.js";
3
+ import { isProxyWorkerStatusMessage, PROXY_SOCKET_WORKER_ENV, PROXY_SOCKET_OFFER_TIMEOUT, PROXY_SOCKET_COMMIT_TIMEOUT, } from "./rollingWorkerProtocol.js";
4
+ /**
5
+ * Spawn a generation-specific IPC worker with separate offer and commit
6
+ * ownership deadlines.
7
+ */
4
8
  export function spawnProxySocketWorker(options) {
5
9
  const socketAckTimeoutMs = Math.max(1, options.socketAckTimeoutMs ?? 30_000);
6
10
  let nextSocketId = 0;
@@ -8,7 +12,7 @@ export function spawnProxySocketWorker(options) {
8
12
  const statusListeners = new Set();
9
13
  const pendingStatusMessages = [];
10
14
  let spawnError;
11
- const child = spawn(options.command, options.args, {
15
+ const child = (options.spawn ?? spawn)(options.command, options.args, {
12
16
  env: {
13
17
  ...process.env,
14
18
  ...options.env,
@@ -52,6 +56,10 @@ export function spawnProxySocketWorker(options) {
52
56
  child.kill("SIGTERM");
53
57
  throw ErrorFactory.proxyWorkerLifecycle("proxy worker spawn did not return a pid");
54
58
  }
59
+ /**
60
+ * Finish a handoff once and cancel its worker-side copy on uncertain
61
+ * delivery.
62
+ */
55
63
  const settleSocket = (socketId, error) => {
56
64
  const pending = pendingSockets.get(socketId);
57
65
  if (!pending) {
@@ -59,7 +67,7 @@ export function spawnProxySocketWorker(options) {
59
67
  }
60
68
  pendingSockets.delete(socketId);
61
69
  clearTimeout(pending.timeout);
62
- if (error && child.connected && !pending.accepted) {
70
+ if (error && child.connected) {
63
71
  try {
64
72
  child.send({
65
73
  type: "proxy-worker:socket-cancel",
@@ -76,6 +84,10 @@ export function spawnProxySocketWorker(options) {
76
84
  }
77
85
  pending.callback(error);
78
86
  };
87
+ /**
88
+ * Accept messages only from this worker and start a fresh deadline for
89
+ * the commit phase.
90
+ */
79
91
  const onInternalMessage = (message) => {
80
92
  if (isProxyWorkerStatusMessage(message) &&
81
93
  message.type === "proxy-worker:socket-accepted" &&
@@ -86,6 +98,15 @@ export function spawnProxySocketWorker(options) {
86
98
  return;
87
99
  }
88
100
  pending.accepted = true;
101
+ // Acceptance and commit are distinct phases. A late acceptance must not
102
+ // inherit an almost-expired offer timer and kill established streams.
103
+ clearTimeout(pending.timeout);
104
+ pending.timeout = setTimeout(() => {
105
+ const error = new Error(`proxy worker ${childPid} socket commit remained pending for ${socketAckTimeoutMs}ms`);
106
+ error.code = PROXY_SOCKET_COMMIT_TIMEOUT;
107
+ settleSocket(message.socketId, error);
108
+ }, socketAckTimeoutMs);
109
+ pending.timeout.unref?.();
89
110
  try {
90
111
  child.send({
91
112
  type: "proxy-worker:socket-commit",
@@ -135,11 +156,7 @@ export function spawnProxySocketWorker(options) {
135
156
  const socketId = `${generation}:${++nextSocketId}`;
136
157
  const timeout = setTimeout(() => {
137
158
  const error = new Error(`proxy worker ${childPid} did not accept socket within ${socketAckTimeoutMs}ms`);
138
- if (!pendingSockets.get(socketId)?.accepted) {
139
- // No commit was sent. The cancel message settles this offer without
140
- // terminating unrelated requests already owned by the worker.
141
- error.code = PROXY_SOCKET_OFFER_TIMEOUT;
142
- }
159
+ error.code = PROXY_SOCKET_OFFER_TIMEOUT;
143
160
  settleSocket(socketId, error);
144
161
  }, socketAckTimeoutMs);
145
162
  timeout.unref?.();
@@ -2,6 +2,12 @@ import type { ProxyWorkerControlMessage, ProxyWorkerStatusMessage } from "../typ
2
2
  export declare const PROXY_SOCKET_WORKER_ENV = "NEUROLINK_PROXY_SOCKET_WORKER";
3
3
  /** The worker has not been sent a commit and cannot have served this socket. */
4
4
  export declare const PROXY_SOCKET_OFFER_TIMEOUT = "PROXY_SOCKET_OFFER_TIMEOUT";
5
+ /** Commit delivery is uncertain; close only this socket and never replay it. */
6
+ export declare const PROXY_SOCKET_COMMIT_TIMEOUT = "PROXY_SOCKET_COMMIT_TIMEOUT";
5
7
  export declare const PROXY_ROLLING_SUPERVISOR_ENV = "NEUROLINK_PROXY_ROLLING_SUPERVISOR";
6
8
  export declare function isProxyWorkerControlMessage(value: unknown): value is ProxyWorkerControlMessage;
9
+ /**
10
+ * Validate worker status and bounded process identity before the
11
+ * supervisor trusts IPC evidence.
12
+ */
7
13
  export declare function isProxyWorkerStatusMessage(value: unknown): value is ProxyWorkerStatusMessage;
@@ -1,6 +1,8 @@
1
1
  export const PROXY_SOCKET_WORKER_ENV = "NEUROLINK_PROXY_SOCKET_WORKER";
2
2
  /** The worker has not been sent a commit and cannot have served this socket. */
3
3
  export const PROXY_SOCKET_OFFER_TIMEOUT = "PROXY_SOCKET_OFFER_TIMEOUT";
4
+ /** Commit delivery is uncertain; close only this socket and never replay it. */
5
+ export const PROXY_SOCKET_COMMIT_TIMEOUT = "PROXY_SOCKET_COMMIT_TIMEOUT";
4
6
  export const PROXY_ROLLING_SUPERVISOR_ENV = "NEUROLINK_PROXY_ROLLING_SUPERVISOR";
5
7
  export function isProxyWorkerControlMessage(value) {
6
8
  if (!value || typeof value !== "object") {
@@ -19,6 +21,10 @@ export function isProxyWorkerControlMessage(value) {
19
21
  message.type === "proxy-worker:activate" ||
20
22
  message.type === "proxy-worker:shutdown");
21
23
  }
24
+ /**
25
+ * Validate worker status and bounded process identity before the
26
+ * supervisor trusts IPC evidence.
27
+ */
22
28
  export function isProxyWorkerStatusMessage(value) {
23
29
  if (!value || typeof value !== "object") {
24
30
  return false;
@@ -31,7 +37,12 @@ export function isProxyWorkerStatusMessage(value) {
31
37
  return false;
32
38
  }
33
39
  if (message.type === "proxy-worker:ready") {
34
- return typeof message.version === "string" && message.version.length > 0;
40
+ return (typeof message.version === "string" &&
41
+ message.version.length > 0 &&
42
+ (message.processInstanceId === undefined ||
43
+ (typeof message.processInstanceId === "string" &&
44
+ message.processInstanceId.length > 0 &&
45
+ message.processInstanceId.length <= 256)));
35
46
  }
36
47
  if (message.type === "proxy-worker:activated" ||
37
48
  message.type === "proxy-worker:drained") {
@@ -27,22 +27,50 @@ export declare class RollingWorkerSupervisor {
27
27
  snapshot(): RollingWorkerSupervisorSnapshot;
28
28
  start(expectedVersion: string): Promise<RollingWorkerSupervisorSnapshot>;
29
29
  replace(expectedVersion: string): Promise<RollingWorkerSupervisorSnapshot>;
30
+ /**
31
+ * Admit a paused socket to the active generation or the bounded
32
+ * readiness queue.
33
+ */
30
34
  acceptSocket(socket: TransferableProxySocket): void;
35
+ /**
36
+ * Retain a socket only within the queue capacity and deadline, recording
37
+ * classified rejection.
38
+ */
31
39
  private queueSocket;
32
40
  close(): Promise<void>;
33
41
  private closeWorkers;
34
42
  private requestWorkerShutdown;
35
43
  private forceTerminateWorkers;
36
44
  private notifyShutdownWaiters;
45
+ /**
46
+ * Validate candidate readiness and activation while preserving an
47
+ * independent listener for actual exit.
48
+ */
37
49
  private spawnCandidate;
38
50
  private flushQueuedSockets;
39
51
  private transferSocket;
40
52
  private maybeDrainWorker;
53
+ /**
54
+ * Cancel the affected handoff and request bounded replacement without
55
+ * killing unrelated streams.
56
+ */
41
57
  private handleTransferFailure;
58
+ /**
59
+ * Record a classified admission rejection before releasing the
60
+ * parent-owned socket.
61
+ */
42
62
  private rejectSocket;
43
63
  private describeTransferError;
44
64
  private extractLifecycleFailureDetails;
65
+ /**
66
+ * Retain the latest failure and its observed process evidence in the
67
+ * incident journal.
68
+ */
45
69
  private recordFailure;
70
+ /**
71
+ * Publish an independent incident record and retain only a bounded
72
+ * recent summary.
73
+ */
46
74
  private recordEvent;
47
75
  private scheduleTransferState;
48
76
  private publishState;