@juspay/neurolink 12.12.9 → 12.12.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +2 -2
- package/dist/browser/neurolink.min.js +394 -395
- package/dist/cli/commands/proxy.d.ts +4 -0
- package/dist/cli/commands/proxy.js +60 -18
- package/dist/cli/commands/proxyAnalyze.js +8 -1
- package/dist/proxy/bodyCaptureProcessing.d.ts +22 -0
- package/dist/proxy/bodyCaptureProcessing.js +219 -0
- package/dist/proxy/bodyCaptureWorker.d.ts +16 -0
- package/dist/proxy/bodyCaptureWorker.js +232 -0
- package/dist/proxy/bodyCaptureWorkerEntry.d.ts +1 -0
- package/dist/proxy/bodyCaptureWorkerEntry.js +34 -0
- package/dist/proxy/proxyAnalysis.js +159 -17
- package/dist/proxy/proxyLifecycle.d.ts +25 -0
- package/dist/proxy/proxyLifecycle.js +111 -5
- package/dist/proxy/proxyRequestKind.d.ts +2 -0
- package/dist/proxy/proxyRequestKind.js +6 -0
- package/dist/proxy/proxyRuntimeMetrics.d.ts +3 -0
- package/dist/proxy/proxyRuntimeMetrics.js +34 -0
- package/dist/proxy/requestLogger.d.ts +10 -6
- package/dist/proxy/requestLogger.js +99 -231
- package/dist/proxy/rollingProxyServer.js +15 -4
- package/dist/proxy/rollingWorkerProcess.d.ts +4 -0
- package/dist/proxy/rollingWorkerProcess.js +25 -8
- package/dist/proxy/rollingWorkerProtocol.d.ts +6 -0
- package/dist/proxy/rollingWorkerProtocol.js +12 -1
- package/dist/proxy/rollingWorkerSupervisor.d.ts +28 -0
- package/dist/proxy/rollingWorkerSupervisor.js +73 -25
- package/dist/proxy/socketWorkerRuntime.d.ts +5 -0
- package/dist/proxy/socketWorkerRuntime.js +19 -2
- package/dist/server/routes/codexProxyRoutes.js +39 -3
- package/dist/services/server/ai/observability/instrumentation.js +7 -1
- package/dist/types/cli.d.ts +1 -1
- package/dist/types/proxy.d.ts +69 -4
- package/package.json +1 -1
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { ErrorFactory } from "../utils/errorHandling.js";
|
|
2
|
-
import { PROXY_SOCKET_OFFER_TIMEOUT } from "./rollingWorkerProtocol.js";
|
|
3
|
-
const DEFAULT_READY_TIMEOUT_MS =
|
|
2
|
+
import { PROXY_SOCKET_OFFER_TIMEOUT, PROXY_SOCKET_COMMIT_TIMEOUT, } from "./rollingWorkerProtocol.js";
|
|
3
|
+
const DEFAULT_READY_TIMEOUT_MS = 120_000;
|
|
4
4
|
const DEFAULT_SOCKET_QUEUE_LIMIT = 1_024;
|
|
5
5
|
const DEFAULT_SOCKET_QUEUE_TIMEOUT_MS = 30_000;
|
|
6
6
|
const DEFAULT_SHUTDOWN_TIMEOUT_MS = 30_000;
|
|
@@ -95,10 +95,14 @@ export class RollingWorkerSupervisor {
|
|
|
95
95
|
});
|
|
96
96
|
return this.replacement;
|
|
97
97
|
}
|
|
98
|
+
/**
|
|
99
|
+
* Admit a paused socket to the active generation or the bounded
|
|
100
|
+
* readiness queue.
|
|
101
|
+
*/
|
|
98
102
|
acceptSocket(socket) {
|
|
99
103
|
socket.pause();
|
|
100
104
|
if (this.closed) {
|
|
101
|
-
this.rejectSocket(socket);
|
|
105
|
+
this.rejectSocket(socket, undefined, undefined, "supervisor_closed");
|
|
102
106
|
return;
|
|
103
107
|
}
|
|
104
108
|
if (this.active &&
|
|
@@ -110,9 +114,13 @@ export class RollingWorkerSupervisor {
|
|
|
110
114
|
this.queueSocket(socket);
|
|
111
115
|
this.flushQueuedSockets();
|
|
112
116
|
}
|
|
117
|
+
/**
|
|
118
|
+
* Retain a socket only within the queue capacity and deadline, recording
|
|
119
|
+
* classified rejection.
|
|
120
|
+
*/
|
|
113
121
|
queueSocket(socket) {
|
|
114
122
|
if (this.queuedSockets.length >= this.options.socketQueueLimit) {
|
|
115
|
-
this.rejectSocket(socket);
|
|
123
|
+
this.rejectSocket(socket, undefined, undefined, "queue_capacity");
|
|
116
124
|
return;
|
|
117
125
|
}
|
|
118
126
|
const queued = {
|
|
@@ -121,7 +129,7 @@ export class RollingWorkerSupervisor {
|
|
|
121
129
|
const index = this.queuedSockets.indexOf(queued);
|
|
122
130
|
if (index >= 0) {
|
|
123
131
|
this.queuedSockets.splice(index, 1);
|
|
124
|
-
this.rejectSocket(socket);
|
|
132
|
+
this.rejectSocket(socket, undefined, undefined, "queue_timeout");
|
|
125
133
|
}
|
|
126
134
|
}, this.options.socketQueueTimeoutMs),
|
|
127
135
|
};
|
|
@@ -201,8 +209,13 @@ export class RollingWorkerSupervisor {
|
|
|
201
209
|
finish();
|
|
202
210
|
}
|
|
203
211
|
}
|
|
212
|
+
/**
|
|
213
|
+
* Validate candidate readiness and activation while preserving an
|
|
214
|
+
* independent listener for actual exit.
|
|
215
|
+
*/
|
|
204
216
|
spawnCandidate(expectedVersion) {
|
|
205
217
|
const generation = ++this.generation;
|
|
218
|
+
let workerProcessInstanceId;
|
|
206
219
|
let handle;
|
|
207
220
|
try {
|
|
208
221
|
handle = this.options.spawnWorker(generation, expectedVersion);
|
|
@@ -212,6 +225,19 @@ export class RollingWorkerSupervisor {
|
|
|
212
225
|
this.publishState();
|
|
213
226
|
return Promise.reject(error instanceof Error ? error : new Error(String(error)));
|
|
214
227
|
}
|
|
228
|
+
// Keep this evidence listener until actual exit, even when candidate
|
|
229
|
+
// failure or a fatal message detaches the operational state listeners.
|
|
230
|
+
handle.onExit((code, signal) => {
|
|
231
|
+
this.recordEvent({
|
|
232
|
+
type: "worker_exit",
|
|
233
|
+
generation,
|
|
234
|
+
version: expectedVersion,
|
|
235
|
+
workerPid: handle.pid,
|
|
236
|
+
workerProcessInstanceId,
|
|
237
|
+
workerExitCode: code,
|
|
238
|
+
workerExitSignal: signal,
|
|
239
|
+
});
|
|
240
|
+
});
|
|
215
241
|
return new Promise((resolve, reject) => {
|
|
216
242
|
let settled = false;
|
|
217
243
|
const finish = (error, phase = "startup", details) => {
|
|
@@ -266,6 +292,7 @@ export class RollingWorkerSupervisor {
|
|
|
266
292
|
return;
|
|
267
293
|
}
|
|
268
294
|
if (message.type === "proxy-worker:ready") {
|
|
295
|
+
workerProcessInstanceId = message.processInstanceId;
|
|
269
296
|
if (message.version !== expectedVersion) {
|
|
270
297
|
finish(new Error(`worker ${handle.pid} reported v${message.version}; expected v${expectedVersion}`));
|
|
271
298
|
return;
|
|
@@ -441,6 +468,10 @@ export class RollingWorkerSupervisor {
|
|
|
441
468
|
worker.handle.terminate("SIGTERM");
|
|
442
469
|
}
|
|
443
470
|
}
|
|
471
|
+
/**
|
|
472
|
+
* Cancel the affected handoff and request bounded replacement without
|
|
473
|
+
* killing unrelated streams.
|
|
474
|
+
*/
|
|
444
475
|
handleTransferFailure(worker, socket, error) {
|
|
445
476
|
this.failedTransfers += 1;
|
|
446
477
|
const detail = this.describeTransferError(error);
|
|
@@ -454,12 +485,15 @@ export class RollingWorkerSupervisor {
|
|
|
454
485
|
const lifecycle = this.extractLifecycleFailureDetails(error, worker.handle.pid);
|
|
455
486
|
const cancelledOffer = error instanceof Error &&
|
|
456
487
|
error.code === PROXY_SOCKET_OFFER_TIMEOUT;
|
|
457
|
-
|
|
488
|
+
const commitTimeout = error instanceof Error &&
|
|
489
|
+
error.code === PROXY_SOCKET_COMMIT_TIMEOUT;
|
|
490
|
+
if (!lifecycle.observedExit &&
|
|
491
|
+
this.active?.generation === worker.generation) {
|
|
458
492
|
this.consecutiveOfferTimeouts += 1;
|
|
459
493
|
// Persistent stalls need recovery, but keep serving existing streams
|
|
460
494
|
// until a replacement activates. Avoid accumulating draining workers or
|
|
461
495
|
// spawning repeatedly when the whole host is under pressure.
|
|
462
|
-
if (this.consecutiveOfferTimeouts >= 3 &&
|
|
496
|
+
if ((this.consecutiveOfferTimeouts >= 3 || !cancelledOffer) &&
|
|
463
497
|
!this.candidate &&
|
|
464
498
|
this.draining.size === 0 &&
|
|
465
499
|
Date.now() - this.lastStallReplacementAt >= 60_000 &&
|
|
@@ -468,11 +502,15 @@ export class RollingWorkerSupervisor {
|
|
|
468
502
|
this.options.onReplacementRequested?.({
|
|
469
503
|
generation: worker.generation,
|
|
470
504
|
pid: worker.handle.pid,
|
|
471
|
-
reason:
|
|
505
|
+
reason: commitTimeout
|
|
506
|
+
? "socket_commit_timeout"
|
|
507
|
+
: cancelledOffer
|
|
508
|
+
? "socket_offer_timeout"
|
|
509
|
+
: "socket_transfer_failure",
|
|
472
510
|
});
|
|
473
511
|
}
|
|
474
512
|
}
|
|
475
|
-
this.recordFailure(worker.generation, worker.version, "transfer", `worker ${worker.handle.pid}
|
|
513
|
+
this.recordFailure(worker.generation, worker.version, "transfer", `worker ${worker.handle.pid} socket transfer failed: ${detail}`, {
|
|
476
514
|
...lifecycle.details,
|
|
477
515
|
// If the error already records an exit, the supervisor did not cause
|
|
478
516
|
// that exit. Otherwise this captures the deliberate cleanup following
|
|
@@ -481,24 +519,18 @@ export class RollingWorkerSupervisor {
|
|
|
481
519
|
? "cancel_uncommitted_socket"
|
|
482
520
|
: lifecycle.observedExit
|
|
483
521
|
? "none"
|
|
484
|
-
: "
|
|
522
|
+
: "cancel_socket_replace_before_drain",
|
|
485
523
|
});
|
|
486
524
|
this.options.log?.(`[proxy-supervisor] socket transfer failed generation=${worker.generation} pid=${worker.handle.pid}: ${detail}`);
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
this.active = null;
|
|
491
|
-
this.draining.set(worker.generation, worker);
|
|
492
|
-
if (!lifecycle.observedExit) {
|
|
493
|
-
// The child may own a duplicate of an incompletely transferred socket.
|
|
494
|
-
// SIGKILL closes its descriptor without worker-side shutdown(2), after
|
|
495
|
-
// which the parent rejects its copy rather than attempting unsafe replay.
|
|
496
|
-
worker.handle.terminate("SIGKILL");
|
|
497
|
-
}
|
|
498
|
-
this.publishState();
|
|
499
|
-
}
|
|
525
|
+
// A failed handoff is not evidence that every connection on this worker
|
|
526
|
+
// failed. Cancel only the affected socket and activate a replacement before
|
|
527
|
+
// draining existing streams. Actual exits are handled by onExit below.
|
|
500
528
|
this.rejectSocket(socket, worker.generation, worker.version, "transfer_failure");
|
|
501
529
|
}
|
|
530
|
+
/**
|
|
531
|
+
* Record a classified admission rejection before releasing the
|
|
532
|
+
* parent-owned socket.
|
|
533
|
+
*/
|
|
502
534
|
rejectSocket(socket, generation = this.active?.generation ?? null, version = this.active?.version ?? null, reason = "unavailable") {
|
|
503
535
|
this.rejectedSockets += 1;
|
|
504
536
|
this.recordEvent({
|
|
@@ -541,6 +573,10 @@ export class RollingWorkerSupervisor {
|
|
|
541
573
|
observedExit: hasExitCode || hasExitSignal,
|
|
542
574
|
};
|
|
543
575
|
}
|
|
576
|
+
/**
|
|
577
|
+
* Retain the latest failure and its observed process evidence in the
|
|
578
|
+
* incident journal.
|
|
579
|
+
*/
|
|
544
580
|
recordFailure(generation, version, phase, message, details = {}) {
|
|
545
581
|
this.lastFailure = {
|
|
546
582
|
at: new Date().toISOString(),
|
|
@@ -556,14 +592,26 @@ export class RollingWorkerSupervisor {
|
|
|
556
592
|
version,
|
|
557
593
|
phase,
|
|
558
594
|
reason: message,
|
|
595
|
+
...details,
|
|
559
596
|
});
|
|
560
597
|
}
|
|
598
|
+
/**
|
|
599
|
+
* Publish an independent incident record and retain only a bounded
|
|
600
|
+
* recent summary.
|
|
601
|
+
*/
|
|
561
602
|
recordEvent(event) {
|
|
562
|
-
|
|
603
|
+
const recorded = {
|
|
563
604
|
at: new Date().toISOString(),
|
|
564
605
|
...event,
|
|
565
606
|
...(event.reason ? { reason: event.reason.slice(0, 1_000) } : {}),
|
|
566
|
-
}
|
|
607
|
+
};
|
|
608
|
+
this.recentEvents.push(recorded);
|
|
609
|
+
try {
|
|
610
|
+
this.options.onEvent?.(recorded);
|
|
611
|
+
}
|
|
612
|
+
catch (error) {
|
|
613
|
+
this.options.log?.(`[proxy-supervisor] event journal failed: ${String(error)}`);
|
|
614
|
+
}
|
|
567
615
|
if (this.recentEvents.length > MAX_RECENT_SUPERVISOR_EVENTS) {
|
|
568
616
|
this.recentEvents.splice(0, this.recentEvents.length - MAX_RECENT_SUPERVISOR_EVENTS);
|
|
569
617
|
}
|
|
@@ -6,9 +6,14 @@ import type { SocketWorkerRuntime, SocketWorkerRuntimeOptions } from "../types/i
|
|
|
6
6
|
* soon as draining begins.
|
|
7
7
|
*/
|
|
8
8
|
export declare function createSocketWorkerRuntime(server: Server, options?: SocketWorkerRuntimeOptions): SocketWorkerRuntime;
|
|
9
|
+
/**
|
|
10
|
+
* Attach the IPC ownership protocol and keep committed sockets addressable
|
|
11
|
+
* by late cancellation.
|
|
12
|
+
*/
|
|
9
13
|
export declare function attachSocketWorkerProcess(server: Server, input: {
|
|
10
14
|
generation: number;
|
|
11
15
|
version: string;
|
|
16
|
+
processInstanceId?: string;
|
|
12
17
|
onActivated?: () => void;
|
|
13
18
|
onDrained?: () => void;
|
|
14
19
|
}): SocketWorkerRuntime;
|
|
@@ -120,9 +120,17 @@ export function createSocketWorkerRuntime(server, options) {
|
|
|
120
120
|
}),
|
|
121
121
|
};
|
|
122
122
|
}
|
|
123
|
+
/**
|
|
124
|
+
* Attach the IPC ownership protocol and keep committed sockets addressable
|
|
125
|
+
* by late cancellation.
|
|
126
|
+
*/
|
|
123
127
|
export function attachSocketWorkerProcess(server, input) {
|
|
124
128
|
let activated = false;
|
|
125
129
|
let gracefulDrain = false;
|
|
130
|
+
// A commit can reach the worker before its parent's send callback settles.
|
|
131
|
+
// Retain ownership until close so a late cancellation affects only that
|
|
132
|
+
// connection, never the worker's other requests. Never replay this socket.
|
|
133
|
+
const committedSockets = new Map();
|
|
126
134
|
const pendingSockets = new Map();
|
|
127
135
|
const send = (message) => {
|
|
128
136
|
if (!process.connected || !process.send) {
|
|
@@ -164,6 +172,7 @@ export function attachSocketWorkerProcess(server, input) {
|
|
|
164
172
|
pending.socket.off("close", pending.onClose);
|
|
165
173
|
return pending.socket;
|
|
166
174
|
};
|
|
175
|
+
/** Apply supervisor messages while retaining ownership of pending and committed sockets. */
|
|
167
176
|
const onMessage = (message, handle) => {
|
|
168
177
|
if (message &&
|
|
169
178
|
typeof message === "object" &&
|
|
@@ -183,7 +192,7 @@ export function attachSocketWorkerProcess(server, input) {
|
|
|
183
192
|
socket.destroy();
|
|
184
193
|
return;
|
|
185
194
|
}
|
|
186
|
-
if (pendingSockets.has(socketId)) {
|
|
195
|
+
if (pendingSockets.has(socketId) || committedSockets.has(socketId)) {
|
|
187
196
|
socket.destroy();
|
|
188
197
|
return;
|
|
189
198
|
}
|
|
@@ -260,14 +269,20 @@ export function attachSocketWorkerProcess(server, input) {
|
|
|
260
269
|
}
|
|
261
270
|
else if (message.type === "proxy-worker:socket-commit" ||
|
|
262
271
|
message.type === "proxy-worker:socket-cancel") {
|
|
263
|
-
const socket = takePendingSocket(message.socketId)
|
|
272
|
+
const socket = takePendingSocket(message.socketId) ??
|
|
273
|
+
(message.type === "proxy-worker:socket-cancel"
|
|
274
|
+
? committedSockets.get(message.socketId)
|
|
275
|
+
: undefined);
|
|
264
276
|
if (!socket) {
|
|
265
277
|
return;
|
|
266
278
|
}
|
|
267
279
|
if (message.type === "proxy-worker:socket-commit") {
|
|
280
|
+
committedSockets.set(message.socketId, socket);
|
|
281
|
+
socket.once("close", () => committedSockets.delete(message.socketId));
|
|
268
282
|
runtime.acceptSocket(socket);
|
|
269
283
|
}
|
|
270
284
|
else {
|
|
285
|
+
committedSockets.delete(message.socketId);
|
|
271
286
|
socket.destroy();
|
|
272
287
|
}
|
|
273
288
|
drainWhenPendingSettled();
|
|
@@ -304,6 +319,7 @@ export function attachSocketWorkerProcess(server, input) {
|
|
|
304
319
|
generation: input.generation,
|
|
305
320
|
pid: process.pid,
|
|
306
321
|
version: input.version,
|
|
322
|
+
processInstanceId: input.processInstanceId,
|
|
307
323
|
});
|
|
308
324
|
return {
|
|
309
325
|
...runtime,
|
|
@@ -316,6 +332,7 @@ export function attachSocketWorkerProcess(server, input) {
|
|
|
316
332
|
takePendingSocket(socketId)?.destroy();
|
|
317
333
|
}
|
|
318
334
|
runtime.close();
|
|
335
|
+
committedSockets.clear();
|
|
319
336
|
},
|
|
320
337
|
};
|
|
321
338
|
}
|
|
@@ -409,6 +409,7 @@ export async function handleCodexResponsesRequest(ctx) {
|
|
|
409
409
|
let attempt = 0;
|
|
410
410
|
let lastErrorMessage = "All Codex accounts failed";
|
|
411
411
|
let lastErrorStatus = 502;
|
|
412
|
+
let lastFailure = { errorType: "all_accounts_failed" };
|
|
412
413
|
let lastAttemptedAccount;
|
|
413
414
|
for (const account of eligible) {
|
|
414
415
|
let authRetried = false;
|
|
@@ -458,8 +459,33 @@ export async function handleCodexResponsesRequest(ctx) {
|
|
|
458
459
|
errorMessage,
|
|
459
460
|
...(errorCode ? { errorCode } : {}),
|
|
460
461
|
transportScope,
|
|
461
|
-
|
|
462
|
+
// These codes prove failure before HTTP dispatch. Socket resets,
|
|
463
|
+
// EPIPE and generic timeouts may follow dispatch and must not replay.
|
|
464
|
+
retryable: [
|
|
465
|
+
"UND_ERR_CONNECT_TIMEOUT",
|
|
466
|
+
"ECONNREFUSED",
|
|
467
|
+
"ENOTFOUND",
|
|
468
|
+
"EAI_AGAIN",
|
|
469
|
+
].includes(errorCode ?? ""),
|
|
462
470
|
});
|
|
471
|
+
lastFailure = {
|
|
472
|
+
errorType: "network_error",
|
|
473
|
+
errorMessage,
|
|
474
|
+
errorCode,
|
|
475
|
+
transportScope,
|
|
476
|
+
};
|
|
477
|
+
if (![
|
|
478
|
+
"UND_ERR_CONNECT_TIMEOUT",
|
|
479
|
+
"ECONNREFUSED",
|
|
480
|
+
"ENOTFOUND",
|
|
481
|
+
"EAI_AGAIN",
|
|
482
|
+
].includes(errorCode ?? "")) {
|
|
483
|
+
await recordFinalOutcome(account, 502, {
|
|
484
|
+
...lastFailure,
|
|
485
|
+
errorMessage,
|
|
486
|
+
});
|
|
487
|
+
return buildCodexErrorResponse(502, "Codex upstream request failed");
|
|
488
|
+
}
|
|
463
489
|
lastErrorMessage = "Codex upstream request failed";
|
|
464
490
|
lastErrorStatus = 502;
|
|
465
491
|
break; // rotate to next account
|
|
@@ -621,6 +647,10 @@ export async function handleCodexResponsesRequest(ctx) {
|
|
|
621
647
|
if (disabled) {
|
|
622
648
|
logger.always(`[proxy] codex account=${account.label} disabled until re-authentication. Run: neurolink auth login codex --label ${account.label}`);
|
|
623
649
|
}
|
|
650
|
+
lastFailure = {
|
|
651
|
+
errorType: "authentication_error",
|
|
652
|
+
errorCode: "refresh_invalid",
|
|
653
|
+
};
|
|
624
654
|
lastErrorStatus = 401;
|
|
625
655
|
lastErrorMessage = "Codex token refresh failed; re-login required";
|
|
626
656
|
break;
|
|
@@ -629,6 +659,10 @@ export async function handleCodexResponsesRequest(ctx) {
|
|
|
629
659
|
// account, so a 5xx or a timeout cannot cost the user a login.
|
|
630
660
|
await saveAccountCooldown(account.key, Date.now() + CODEX_AUTH_COOLDOWN_MS, "auth").catch(() => undefined);
|
|
631
661
|
logger.debug(`[proxy] codex account=${account.label} refresh failed transiently; cooling and rotating`);
|
|
662
|
+
lastFailure = {
|
|
663
|
+
errorType: "auth_refresh_unavailable",
|
|
664
|
+
errorCode: getCodexTransportErrorCode(error),
|
|
665
|
+
};
|
|
632
666
|
lastErrorStatus = 503;
|
|
633
667
|
lastErrorMessage = "Codex token refresh temporarily unavailable";
|
|
634
668
|
break;
|
|
@@ -653,6 +687,7 @@ export async function handleCodexResponsesRequest(ctx) {
|
|
|
653
687
|
rateLimitKind,
|
|
654
688
|
cooldownReason: plan.reason,
|
|
655
689
|
});
|
|
690
|
+
lastFailure = { errorType: "rate_limit_error" };
|
|
656
691
|
lastErrorStatus = 429;
|
|
657
692
|
lastErrorMessage = "Codex account rate-limited";
|
|
658
693
|
break; // rotate
|
|
@@ -674,14 +709,15 @@ export async function handleCodexResponsesRequest(ctx) {
|
|
|
674
709
|
errorMessage,
|
|
675
710
|
retryable: upstream.status >= 500,
|
|
676
711
|
});
|
|
712
|
+
lastFailure = { errorType };
|
|
677
713
|
lastErrorStatus = upstream.status >= 500 ? 502 : upstream.status;
|
|
678
714
|
lastErrorMessage = errorMessage;
|
|
679
715
|
break; // rotate
|
|
680
716
|
}
|
|
681
717
|
}
|
|
682
718
|
await recordFinalOutcome(lastAttemptedAccount, lastErrorStatus, {
|
|
683
|
-
|
|
684
|
-
errorMessage: lastErrorMessage,
|
|
719
|
+
...lastFailure,
|
|
720
|
+
errorMessage: lastFailure.errorMessage ?? lastErrorMessage,
|
|
685
721
|
});
|
|
686
722
|
return buildCodexErrorResponse(lastErrorStatus, lastErrorMessage);
|
|
687
723
|
}
|
|
@@ -30,6 +30,10 @@ function createOtelResource(config, serviceName) {
|
|
|
30
30
|
"deployment.environment": config.environment || "dev",
|
|
31
31
|
});
|
|
32
32
|
}
|
|
33
|
+
/**
|
|
34
|
+
* Configure OTLP metrics and logs with bounded serialization batches for
|
|
35
|
+
* request-serving processes.
|
|
36
|
+
*/
|
|
33
37
|
function initializeOtlpMetricsAndLogs(resource, otlpEndpoint, serviceName) {
|
|
34
38
|
if (!otlpEndpoint) {
|
|
35
39
|
return;
|
|
@@ -69,7 +73,9 @@ function initializeOtlpMetricsAndLogs(resource, otlpEndpoint, serviceName) {
|
|
|
69
73
|
});
|
|
70
74
|
const logProcessor = new BatchLogRecordProcessor(logExporter, {
|
|
71
75
|
maxQueueSize: 2048,
|
|
72
|
-
|
|
76
|
+
// Body logs contain up to 16 KB each. Bound exporter JSON work to
|
|
77
|
+
// roughly 1 MB per batch instead of an 8 MB main-thread serialization.
|
|
78
|
+
maxExportBatchSize: 64,
|
|
73
79
|
scheduledDelayMillis: 2000,
|
|
74
80
|
exportTimeoutMillis: 30000,
|
|
75
81
|
});
|
package/dist/types/cli.d.ts
CHANGED
|
@@ -872,7 +872,7 @@ export type ProxyRollingState = {
|
|
|
872
872
|
workerPid?: number;
|
|
873
873
|
workerExitCode?: number | null;
|
|
874
874
|
workerExitSignal?: string | null;
|
|
875
|
-
supervisorAction?: "none" | "sigkill_after_transfer_failure" | "cancel_uncommitted_socket";
|
|
875
|
+
supervisorAction?: "none" | "sigkill_after_transfer_failure" | "cancel_socket_replace_before_drain" | "cancel_uncommitted_socket";
|
|
876
876
|
} | null;
|
|
877
877
|
};
|
|
878
878
|
export type ProxySupervisorState = {
|
package/dist/types/proxy.d.ts
CHANGED
|
@@ -625,7 +625,26 @@ export type ProxyRequestLogSinkSnapshot = {
|
|
|
625
625
|
unconfirmedWrites: number;
|
|
626
626
|
lastErrorCode?: string;
|
|
627
627
|
};
|
|
628
|
+
export type ProcessedProxyBodyCapture = {
|
|
629
|
+
headers?: Record<string, string>;
|
|
630
|
+
stored: StoredBodyArtifact;
|
|
631
|
+
error?: string;
|
|
632
|
+
queueWaitMs?: number;
|
|
633
|
+
processingMs?: number;
|
|
634
|
+
};
|
|
635
|
+
export type ProxyBodyCaptureWorkerSnapshot = {
|
|
636
|
+
attempted: number;
|
|
637
|
+
completed: number;
|
|
638
|
+
rejected: number;
|
|
639
|
+
failed: number;
|
|
640
|
+
pending: number;
|
|
641
|
+
pendingBytes: number;
|
|
642
|
+
maxPending: number;
|
|
643
|
+
maxPendingBytes: number;
|
|
644
|
+
lastError?: string;
|
|
645
|
+
};
|
|
628
646
|
export type ProxyRequestLoggerSnapshot = {
|
|
647
|
+
bodyCapture?: ProxyBodyCaptureWorkerSnapshot;
|
|
629
648
|
enabled: boolean;
|
|
630
649
|
requests: ProxyRequestLogSinkSnapshot;
|
|
631
650
|
attempts: ProxyRequestLogSinkSnapshot;
|
|
@@ -1618,7 +1637,7 @@ export type ProxyResponseTrackingObserver = {
|
|
|
1618
1637
|
}) => unknown;
|
|
1619
1638
|
};
|
|
1620
1639
|
/** Versioned lifecycle event names persisted by the proxy adapter. */
|
|
1621
|
-
export type ProxyLifecycleEventName = "request_accepted" | "response_headers" | "response_first_chunk" | "request_terminal";
|
|
1640
|
+
export type ProxyLifecycleEventName = "runtime_sample" | "supervisor_event" | "request_accepted" | "response_headers" | "response_first_chunk" | "request_terminal";
|
|
1622
1641
|
/** Client-facing terminal classifications recorded by lifecycle metadata. */
|
|
1623
1642
|
export type ProxyLifecycleTerminalOutcome = ProxyResponseTerminalOutcome | "handler_error" | "unknown";
|
|
1624
1643
|
/** Content-free lifecycle event accepted by the bounded metadata logger. */
|
|
@@ -1649,6 +1668,20 @@ export type ProxyLifecycleEventInput = {
|
|
|
1649
1668
|
errorCode?: string;
|
|
1650
1669
|
timestampMs?: number;
|
|
1651
1670
|
monotonicMs?: number;
|
|
1671
|
+
/** Parent-owned evidence, independent of the serving worker's journal tail. */
|
|
1672
|
+
supervisorEvent?: RollingWorkerSupervisorEvent;
|
|
1673
|
+
runtimeSample?: ProxyRuntimeSample;
|
|
1674
|
+
};
|
|
1675
|
+
/** Process CPU and event-loop evidence; host load is not a request count. */
|
|
1676
|
+
export type ProxyRuntimeSample = {
|
|
1677
|
+
intervalMs: number;
|
|
1678
|
+
cpuPercentOneCore: number;
|
|
1679
|
+
rssBytes: number;
|
|
1680
|
+
heapUsedBytes: number;
|
|
1681
|
+
eventLoopDelayP99Ms: number;
|
|
1682
|
+
eventLoopDelayMaxMs: number;
|
|
1683
|
+
hostLoad1m: number;
|
|
1684
|
+
availableParallelism: number;
|
|
1652
1685
|
};
|
|
1653
1686
|
/** Data-quality counters for the bounded lifecycle metadata sink. */
|
|
1654
1687
|
export type ProxyLifecycleLoggerSnapshot = {
|
|
@@ -1676,6 +1709,7 @@ export type ProxyLifecycleLoggerSnapshot = {
|
|
|
1676
1709
|
};
|
|
1677
1710
|
/** Lifecycle logger configuration. Queue overrides are used by stress tests. */
|
|
1678
1711
|
export type ProxyLifecycleLoggerOptions = {
|
|
1712
|
+
filePrefix?: "proxy-lifecycle" | "proxy-supervisor";
|
|
1679
1713
|
enabled: boolean;
|
|
1680
1714
|
logDir?: string;
|
|
1681
1715
|
queueCapacity?: number;
|
|
@@ -1686,10 +1720,13 @@ export type ProxyLifecycleLoggerOptions = {
|
|
|
1686
1720
|
};
|
|
1687
1721
|
/** Serialized lifecycle line awaiting a bounded batch write. */
|
|
1688
1722
|
export type QueuedProxyLifecycleEvent = {
|
|
1723
|
+
filePrefix?: string;
|
|
1689
1724
|
logDir: string;
|
|
1690
1725
|
date: string;
|
|
1691
1726
|
record: Record<string, unknown>;
|
|
1692
1727
|
writeRetries: number;
|
|
1728
|
+
/** Resolve only after the original append settles; uncertain writes fail. */
|
|
1729
|
+
onPersisted?: (confirmed: boolean) => void;
|
|
1693
1730
|
};
|
|
1694
1731
|
/** Percentile summary used by offline proxy log analysis. */
|
|
1695
1732
|
export type ProxyLatencySummary = {
|
|
@@ -1714,6 +1751,13 @@ export type ProxyAnalysisAccount = {
|
|
|
1714
1751
|
/** Offline report generated from proxy request, attempt, and lifecycle logs. */
|
|
1715
1752
|
export type ProxyAnalysisStreamName = "lifecycle" | "requests" | "attempts" | "debug";
|
|
1716
1753
|
export type ProxyAnalysisReport = {
|
|
1754
|
+
runtime: {
|
|
1755
|
+
samples: number;
|
|
1756
|
+
maxEventLoopDelayMs: number | null;
|
|
1757
|
+
maxRssBytes: number | null;
|
|
1758
|
+
maxCpuPercentOneCore: number | null;
|
|
1759
|
+
maxHostLoad1m: number | null;
|
|
1760
|
+
};
|
|
1717
1761
|
generatedAt: string;
|
|
1718
1762
|
since: string;
|
|
1719
1763
|
until: string;
|
|
@@ -1770,7 +1814,18 @@ export type ProxyAnalysisReport = {
|
|
|
1770
1814
|
};
|
|
1771
1815
|
};
|
|
1772
1816
|
lifecycle: {
|
|
1817
|
+
/** Accepted requests lacking transport terminals when their worker exited. */
|
|
1818
|
+
unconfirmedAtWorkerExit: Array<{
|
|
1819
|
+
requestId: string;
|
|
1820
|
+
workerProcessInstanceId: string;
|
|
1821
|
+
at: string;
|
|
1822
|
+
workerExitCode: number | null;
|
|
1823
|
+
workerExitSignal: string | null;
|
|
1824
|
+
providerFinalRecorded: boolean;
|
|
1825
|
+
}>;
|
|
1773
1826
|
accepted: number;
|
|
1827
|
+
/** Accepted metadata requests that do not require model final records. */
|
|
1828
|
+
auxiliaryRequests: number;
|
|
1774
1829
|
headers: number;
|
|
1775
1830
|
firstChunks: number;
|
|
1776
1831
|
terminal: number;
|
|
@@ -2375,6 +2430,7 @@ export type ProxyWorkerStatusMessage = {
|
|
|
2375
2430
|
generation: number;
|
|
2376
2431
|
pid: number;
|
|
2377
2432
|
version: string;
|
|
2433
|
+
processInstanceId?: string;
|
|
2378
2434
|
} | {
|
|
2379
2435
|
type: "proxy-worker:drained";
|
|
2380
2436
|
generation: number;
|
|
@@ -2417,6 +2473,8 @@ export type RollingWorkerHandle = {
|
|
|
2417
2473
|
onExit: (listener: (code: number | null, signal: string | null) => void) => () => void;
|
|
2418
2474
|
};
|
|
2419
2475
|
export type SpawnProxySocketWorkerOptions = {
|
|
2476
|
+
/** Injectable process boundary for deterministic IPC fault tests. */
|
|
2477
|
+
spawn?: typeof import("node:child_process").spawn;
|
|
2420
2478
|
generation: number;
|
|
2421
2479
|
expectedVersion: string;
|
|
2422
2480
|
command: string;
|
|
@@ -2430,15 +2488,20 @@ export type RollingWorkerFailureDetails = {
|
|
|
2430
2488
|
workerPid?: number;
|
|
2431
2489
|
workerExitCode?: number | null;
|
|
2432
2490
|
workerExitSignal?: string | null;
|
|
2433
|
-
supervisorAction?: "none" | "sigkill_after_transfer_failure" | "cancel_uncommitted_socket";
|
|
2491
|
+
supervisorAction?: "none" | "sigkill_after_transfer_failure" | "cancel_socket_replace_before_drain" | "cancel_uncommitted_socket";
|
|
2434
2492
|
};
|
|
2435
2493
|
export type RollingWorkerSupervisorEvent = {
|
|
2436
2494
|
at: string;
|
|
2437
|
-
type: "activated" | "failure" | "failed_transfer" | "rejected_socket";
|
|
2495
|
+
type: "activated" | "failure" | "failed_transfer" | "rejected_socket" | "worker_exit";
|
|
2438
2496
|
generation: number | null;
|
|
2439
2497
|
version: string | null;
|
|
2440
2498
|
phase?: "startup" | "activation" | "runtime" | "transfer";
|
|
2441
2499
|
reason?: string;
|
|
2500
|
+
workerPid?: number;
|
|
2501
|
+
workerProcessInstanceId?: string;
|
|
2502
|
+
workerExitCode?: number | null;
|
|
2503
|
+
workerExitSignal?: string | null;
|
|
2504
|
+
supervisorAction?: RollingWorkerFailureDetails["supervisorAction"];
|
|
2442
2505
|
};
|
|
2443
2506
|
export type RollingWorkerSupervisorSnapshot = {
|
|
2444
2507
|
generation: number;
|
|
@@ -2481,10 +2544,11 @@ export type RollingWorkerSupervisorOptions = {
|
|
|
2481
2544
|
socketQueueTimeoutMs?: number;
|
|
2482
2545
|
shutdownTimeoutMs?: number;
|
|
2483
2546
|
onStateChange?: (snapshot: RollingWorkerSupervisorSnapshot) => void;
|
|
2547
|
+
onEvent?: (event: RollingWorkerSupervisorEvent) => void;
|
|
2484
2548
|
onReplacementRequested?: (request: {
|
|
2485
2549
|
generation: number;
|
|
2486
2550
|
pid: number;
|
|
2487
|
-
reason: "environment" | "socket_offer_timeout";
|
|
2551
|
+
reason: "environment" | "socket_offer_timeout" | "socket_commit_timeout" | "socket_transfer_failure";
|
|
2488
2552
|
}) => void;
|
|
2489
2553
|
log?: (message: string) => void;
|
|
2490
2554
|
};
|
|
@@ -2501,6 +2565,7 @@ export type RollingProxyServerOptions = {
|
|
|
2501
2565
|
recoveryDelayMs?: number;
|
|
2502
2566
|
maxRecoveryDelayMs?: number;
|
|
2503
2567
|
onStateChange?: (snapshot: RollingWorkerSupervisorSnapshot) => void;
|
|
2568
|
+
onEvent?: (event: RollingWorkerSupervisorEvent) => void;
|
|
2504
2569
|
log?: (message: string) => void;
|
|
2505
2570
|
};
|
|
2506
2571
|
export type RollingProxyServer = {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@juspay/neurolink",
|
|
3
|
-
"version": "12.12.
|
|
3
|
+
"version": "12.12.10",
|
|
4
4
|
"packageManager": "pnpm@10.15.1",
|
|
5
5
|
"description": "TypeScript AI SDK with 24+ LLM providers behind one consistent API. MCP-native (connect any MCP server), voice TTS/STT/realtime, RAG, agents, memory, context compaction. OpenAI · Anthropic · Gemini · Bedrock · Azure · Ollama · DeepSeek · NVIDIA NIM and more.",
|
|
6
6
|
"author": {
|