@juspay/neurolink 10.4.1 → 10.4.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/dist/browser/neurolink.min.js +386 -389
- package/dist/cli/commands/proxy.d.ts +1 -1
- package/dist/cli/commands/proxy.js +124 -56
- package/dist/cli/commands/proxyAnalyze.js +10 -1
- package/dist/lib/providers/anthropic.js +18 -1
- package/dist/lib/providers/googleAiStudio.js +18 -1
- package/dist/lib/providers/googleNativeGemini3.d.ts +19 -1
- package/dist/lib/providers/googleNativeGemini3.js +61 -4
- package/dist/lib/providers/googleVertex.d.ts +48 -0
- package/dist/lib/providers/googleVertex.js +204 -108
- package/dist/lib/providers/openaiChatCompletionsBase.js +34 -2
- package/dist/lib/providers/openaiChatCompletionsClient.d.ts +11 -5
- package/dist/lib/providers/openaiChatCompletionsClient.js +14 -8
- package/dist/lib/proxy/proxyAnalysis.js +136 -12
- package/dist/lib/proxy/requestLogger.d.ts +2 -0
- package/dist/lib/proxy/requestLogger.js +72 -13
- package/dist/lib/proxy/rollingProxyServer.js +79 -8
- package/dist/lib/proxy/rollingWorkerProcess.js +20 -15
- package/dist/lib/proxy/rollingWorkerProtocol.js +3 -0
- package/dist/lib/proxy/rollingWorkerSupervisor.d.ts +1 -0
- package/dist/lib/proxy/rollingWorkerSupervisor.js +31 -11
- package/dist/lib/proxy/runtimeConfig.d.ts +1 -0
- package/dist/lib/proxy/runtimeConfig.js +20 -5
- package/dist/lib/proxy/socketWorkerRuntime.js +41 -13
- package/dist/lib/server/routes/claudeProxyRoutes.js +63 -75
- package/dist/lib/tools/toolDiscovery.d.ts +25 -3
- package/dist/lib/tools/toolDiscovery.js +76 -4
- package/dist/lib/types/proxy.d.ts +31 -1
- package/dist/lib/types/toolResolution.d.ts +10 -0
- package/dist/providers/anthropic.js +18 -1
- package/dist/providers/googleAiStudio.js +18 -1
- package/dist/providers/googleNativeGemini3.d.ts +19 -1
- package/dist/providers/googleNativeGemini3.js +61 -4
- package/dist/providers/googleVertex.d.ts +48 -0
- package/dist/providers/googleVertex.js +204 -108
- package/dist/providers/openaiChatCompletionsBase.js +34 -2
- package/dist/providers/openaiChatCompletionsClient.d.ts +11 -5
- package/dist/providers/openaiChatCompletionsClient.js +14 -8
- package/dist/proxy/proxyAnalysis.js +136 -12
- package/dist/proxy/requestLogger.d.ts +2 -0
- package/dist/proxy/requestLogger.js +72 -13
- package/dist/proxy/rollingProxyServer.js +79 -8
- package/dist/proxy/rollingWorkerProcess.js +20 -15
- package/dist/proxy/rollingWorkerProtocol.js +3 -0
- package/dist/proxy/rollingWorkerSupervisor.d.ts +1 -0
- package/dist/proxy/rollingWorkerSupervisor.js +31 -11
- package/dist/proxy/runtimeConfig.d.ts +1 -0
- package/dist/proxy/runtimeConfig.js +20 -5
- package/dist/proxy/socketWorkerRuntime.js +41 -13
- package/dist/server/routes/claudeProxyRoutes.js +63 -75
- package/dist/tools/toolDiscovery.d.ts +25 -3
- package/dist/tools/toolDiscovery.js +76 -4
- package/dist/types/proxy.d.ts +31 -1
- package/dist/types/toolResolution.d.ts +10 -0
- package/package.json +1 -1
|
@@ -1,17 +1,41 @@
|
|
|
1
1
|
import { createReadStream } from "node:fs";
|
|
2
|
-
import { readdir } from "node:fs/promises";
|
|
2
|
+
import { lstat, readdir, realpath, stat } from "node:fs/promises";
|
|
3
3
|
import { homedir } from "node:os";
|
|
4
4
|
import { createInterface } from "node:readline";
|
|
5
|
-
import { join, resolve } from "node:path";
|
|
5
|
+
import { isAbsolute, join, relative, resolve, sep } from "node:path";
|
|
6
6
|
const LIFECYCLE_FILE_PATTERN = /^proxy-lifecycle-\d{4}-\d{2}-\d{2}\.jsonl$/;
|
|
7
7
|
const REQUEST_FILE_PATTERN = /^proxy-\d{4}-\d{2}-\d{2}\.jsonl$/;
|
|
8
8
|
const ATTEMPT_FILE_PATTERN = /^proxy-attempts-\d{4}-\d{2}-\d{2}\.jsonl$/;
|
|
9
|
+
const DEBUG_FILE_PATTERN = /^proxy-debug-\d{4}-\d{2}-\d{2}\.jsonl$/;
|
|
10
|
+
const ARTIFACT_STAT_CONCURRENCY = 64;
|
|
9
11
|
const LIFECYCLE_EVENTS = new Set([
|
|
10
12
|
"request_accepted",
|
|
11
13
|
"response_headers",
|
|
12
14
|
"response_first_chunk",
|
|
13
15
|
"request_terminal",
|
|
14
16
|
]);
|
|
17
|
+
function isContainedPath(root, candidate) {
|
|
18
|
+
const candidateRelative = relative(root, candidate);
|
|
19
|
+
return (candidateRelative.length > 0 &&
|
|
20
|
+
!isAbsolute(candidateRelative) &&
|
|
21
|
+
candidateRelative !== ".." &&
|
|
22
|
+
!candidateRelative.startsWith(`..${sep}`));
|
|
23
|
+
}
|
|
24
|
+
async function inspectBodyArtifact(artifactPath, canonicalBodiesRoot) {
|
|
25
|
+
if (!canonicalBodiesRoot) {
|
|
26
|
+
return "missing";
|
|
27
|
+
}
|
|
28
|
+
try {
|
|
29
|
+
const canonicalArtifactPath = await realpath(artifactPath);
|
|
30
|
+
if (!isContainedPath(canonicalBodiesRoot, canonicalArtifactPath)) {
|
|
31
|
+
return "invalid";
|
|
32
|
+
}
|
|
33
|
+
return (await stat(canonicalArtifactPath)).isFile() ? "present" : "missing";
|
|
34
|
+
}
|
|
35
|
+
catch {
|
|
36
|
+
return "missing";
|
|
37
|
+
}
|
|
38
|
+
}
|
|
15
39
|
function increment(counter, key) {
|
|
16
40
|
counter[key] = (counter[key] ?? 0) + 1;
|
|
17
41
|
}
|
|
@@ -192,6 +216,7 @@ async function discoverLogFiles(logsDir) {
|
|
|
192
216
|
lifecycleFiles: matching(LIFECYCLE_FILE_PATTERN),
|
|
193
217
|
requestFiles: matching(REQUEST_FILE_PATTERN),
|
|
194
218
|
attemptFiles: matching(ATTEMPT_FILE_PATTERN),
|
|
219
|
+
debugFiles: matching(DEBUG_FILE_PATTERN),
|
|
195
220
|
};
|
|
196
221
|
}
|
|
197
222
|
/** Analyze local proxy logs without reading request or response body artifacts. */
|
|
@@ -199,7 +224,24 @@ export async function analyzeProxyLogs(options) {
|
|
|
199
224
|
const nowMs = options?.nowMs ?? Date.now();
|
|
200
225
|
const sinceMs = parseSince(options?.since ?? "24h", nowMs);
|
|
201
226
|
const logsDir = resolve(options?.logsDir ?? join(homedir(), ".neurolink", "logs"));
|
|
202
|
-
const { lifecycleFiles, requestFiles, attemptFiles } = await discoverLogFiles(logsDir);
|
|
227
|
+
const { lifecycleFiles, requestFiles, attemptFiles, debugFiles } = await discoverLogFiles(logsDir);
|
|
228
|
+
const observedRanges = {
|
|
229
|
+
lifecycle: { from: null, to: null },
|
|
230
|
+
requests: { from: null, to: null },
|
|
231
|
+
attempts: { from: null, to: null },
|
|
232
|
+
debug: { from: null, to: null },
|
|
233
|
+
};
|
|
234
|
+
const observeTimestamp = (stream, record) => {
|
|
235
|
+
const timestamp = Date.parse(String(record.timestamp ?? ""));
|
|
236
|
+
if (!Number.isFinite(timestamp)) {
|
|
237
|
+
return null;
|
|
238
|
+
}
|
|
239
|
+
const range = observedRanges[stream];
|
|
240
|
+
range.from =
|
|
241
|
+
range.from === null ? timestamp : Math.min(range.from, timestamp);
|
|
242
|
+
range.to = range.to === null ? timestamp : Math.max(range.to, timestamp);
|
|
243
|
+
return timestamp;
|
|
244
|
+
};
|
|
203
245
|
let linesRead = 0;
|
|
204
246
|
let malformedLines = 0;
|
|
205
247
|
let unsupportedLifecycleLines = 0;
|
|
@@ -216,8 +258,8 @@ export async function analyzeProxyLogs(options) {
|
|
|
216
258
|
const sequences = new Map();
|
|
217
259
|
for (const filePath of lifecycleFiles) {
|
|
218
260
|
linesRead += await readJsonLines(filePath, (record) => {
|
|
219
|
-
const timestamp =
|
|
220
|
-
if (
|
|
261
|
+
const timestamp = observeTimestamp("lifecycle", record);
|
|
262
|
+
if (timestamp === null || timestamp < sinceMs) {
|
|
221
263
|
return;
|
|
222
264
|
}
|
|
223
265
|
const event = stringValue(record.event);
|
|
@@ -298,8 +340,8 @@ export async function analyzeProxyLogs(options) {
|
|
|
298
340
|
let unclassifiedRateLimits = 0;
|
|
299
341
|
for (const filePath of attemptFiles) {
|
|
300
342
|
linesRead += await readJsonLines(filePath, (record) => {
|
|
301
|
-
const timestamp =
|
|
302
|
-
if (
|
|
343
|
+
const timestamp = observeTimestamp("attempts", record);
|
|
344
|
+
if (timestamp === null || timestamp < sinceMs) {
|
|
303
345
|
return;
|
|
304
346
|
}
|
|
305
347
|
const requestId = stringValue(record.requestId);
|
|
@@ -360,8 +402,8 @@ export async function analyzeProxyLogs(options) {
|
|
|
360
402
|
const terminalStreamErrors = new Set();
|
|
361
403
|
for (const filePath of requestFiles) {
|
|
362
404
|
linesRead += await readJsonLines(filePath, (record) => {
|
|
363
|
-
const timestamp =
|
|
364
|
-
if (
|
|
405
|
+
const timestamp = observeTimestamp("requests", record);
|
|
406
|
+
if (timestamp === null || timestamp < sinceMs) {
|
|
365
407
|
return;
|
|
366
408
|
}
|
|
367
409
|
const requestId = stringValue(record.requestId);
|
|
@@ -391,6 +433,70 @@ export async function analyzeProxyLogs(options) {
|
|
|
391
433
|
malformedLines += 1;
|
|
392
434
|
});
|
|
393
435
|
}
|
|
436
|
+
let capturesIndexed = 0;
|
|
437
|
+
let truncatedCaptures = 0;
|
|
438
|
+
let writeFailures = 0;
|
|
439
|
+
let invalidPaths = 0;
|
|
440
|
+
const referencedArtifacts = new Set();
|
|
441
|
+
const bodiesRoot = resolve(logsDir, "bodies");
|
|
442
|
+
for (const filePath of debugFiles) {
|
|
443
|
+
linesRead += await readJsonLines(filePath, (record) => {
|
|
444
|
+
const timestamp = observeTimestamp("debug", record);
|
|
445
|
+
if (timestamp === null || timestamp < sinceMs) {
|
|
446
|
+
return;
|
|
447
|
+
}
|
|
448
|
+
if (record.type !== "body_capture") {
|
|
449
|
+
return;
|
|
450
|
+
}
|
|
451
|
+
capturesIndexed += 1;
|
|
452
|
+
truncatedCaptures += record.bodyTruncated === true ? 1 : 0;
|
|
453
|
+
writeFailures += record.bodyWriteFailed === true ? 1 : 0;
|
|
454
|
+
const bodyPath = stringValue(record.bodyPath);
|
|
455
|
+
if (!bodyPath) {
|
|
456
|
+
return;
|
|
457
|
+
}
|
|
458
|
+
if (bodyPath.includes("\0") || bodyPath.split(/[\\/]/).includes("..")) {
|
|
459
|
+
invalidPaths += 1;
|
|
460
|
+
return;
|
|
461
|
+
}
|
|
462
|
+
const resolvedBodyPath = resolve(logsDir, bodyPath);
|
|
463
|
+
if (!isContainedPath(bodiesRoot, resolvedBodyPath)) {
|
|
464
|
+
invalidPaths += 1;
|
|
465
|
+
return;
|
|
466
|
+
}
|
|
467
|
+
referencedArtifacts.add(resolvedBodyPath);
|
|
468
|
+
}, () => {
|
|
469
|
+
malformedLines += 1;
|
|
470
|
+
});
|
|
471
|
+
}
|
|
472
|
+
const artifactPaths = [...referencedArtifacts];
|
|
473
|
+
let canonicalBodiesRoot = null;
|
|
474
|
+
let bodiesRootUnsafe = false;
|
|
475
|
+
try {
|
|
476
|
+
const bodiesRootStat = await lstat(bodiesRoot);
|
|
477
|
+
if (bodiesRootStat.isDirectory() && !bodiesRootStat.isSymbolicLink()) {
|
|
478
|
+
canonicalBodiesRoot = await realpath(bodiesRoot);
|
|
479
|
+
}
|
|
480
|
+
else {
|
|
481
|
+
bodiesRootUnsafe = true;
|
|
482
|
+
}
|
|
483
|
+
}
|
|
484
|
+
catch {
|
|
485
|
+
// A missing body directory means every lexically valid reference is absent.
|
|
486
|
+
}
|
|
487
|
+
let artifactsPresent = 0;
|
|
488
|
+
let artifactsMissing = 0;
|
|
489
|
+
for (let offset = 0; offset < artifactPaths.length; offset += ARTIFACT_STAT_CONCURRENCY) {
|
|
490
|
+
const presence = await Promise.all(artifactPaths
|
|
491
|
+
.slice(offset, offset + ARTIFACT_STAT_CONCURRENCY)
|
|
492
|
+
.map((artifactPath) => bodiesRootUnsafe
|
|
493
|
+
? Promise.resolve("invalid")
|
|
494
|
+
: inspectBodyArtifact(artifactPath, canonicalBodiesRoot)));
|
|
495
|
+
artifactsPresent += presence.filter((value) => value === "present").length;
|
|
496
|
+
artifactsMissing += presence.filter((value) => value === "missing").length;
|
|
497
|
+
invalidPaths += presence.filter((value) => value === "invalid").length;
|
|
498
|
+
}
|
|
499
|
+
const artifactsReferenced = artifactsPresent + artifactsMissing;
|
|
394
500
|
const finalSummary = summarizeFinalRequests(finalRequests, terminalStreamErrors, attemptsByRequest, accounts);
|
|
395
501
|
return {
|
|
396
502
|
generatedAt: new Date(nowMs).toISOString(),
|
|
@@ -400,11 +506,12 @@ export async function analyzeProxyLogs(options) {
|
|
|
400
506
|
lifecycle: lifecycleFiles.length,
|
|
401
507
|
requests: requestFiles.length,
|
|
402
508
|
attempts: attemptFiles.length,
|
|
509
|
+
debug: debugFiles.length,
|
|
403
510
|
},
|
|
404
511
|
coverage: {
|
|
405
|
-
lifecycle:
|
|
406
|
-
finalRequests:
|
|
407
|
-
attempts:
|
|
512
|
+
lifecycle: accepted.size + headers.size + firstChunks.size + terminal.size > 0,
|
|
513
|
+
finalRequests: finalRequests.size > 0 || terminalStreamErrors.size > 0,
|
|
514
|
+
attempts: totalAttempts > 0,
|
|
408
515
|
attemptLatency: attemptLatency.length > 0,
|
|
409
516
|
cacheUsage: finalSummary.cache.requestsWithUsage > 0,
|
|
410
517
|
},
|
|
@@ -414,6 +521,23 @@ export async function analyzeProxyLogs(options) {
|
|
|
414
521
|
unsupportedLifecycleLines,
|
|
415
522
|
lifecycleSequenceGaps,
|
|
416
523
|
lifecycleSequenceDuplicates,
|
|
524
|
+
streams: Object.fromEntries(Object.entries(observedRanges).map(([stream, range]) => [
|
|
525
|
+
stream,
|
|
526
|
+
{
|
|
527
|
+
observedFrom: range.from === null ? null : new Date(range.from).toISOString(),
|
|
528
|
+
observedTo: range.to === null ? null : new Date(range.to).toISOString(),
|
|
529
|
+
startsAtOrBeforeRequestedWindow: range.from !== null && range.from <= sinceMs,
|
|
530
|
+
},
|
|
531
|
+
])),
|
|
532
|
+
bodyArtifacts: {
|
|
533
|
+
capturesIndexed,
|
|
534
|
+
artifactsReferenced,
|
|
535
|
+
artifactsPresent,
|
|
536
|
+
artifactsMissing,
|
|
537
|
+
invalidPaths,
|
|
538
|
+
writeFailures,
|
|
539
|
+
truncatedCaptures,
|
|
540
|
+
},
|
|
417
541
|
},
|
|
418
542
|
lifecycle: {
|
|
419
543
|
accepted: accepted.size,
|
|
@@ -6,6 +6,8 @@
|
|
|
6
6
|
* Useful for debugging and auditing proxy traffic.
|
|
7
7
|
*/
|
|
8
8
|
import type { ProxyBodyCaptureEntry, RequestAttemptLogEntry, RequestLogEntry } from "../types/index.js";
|
|
9
|
+
/** Wait, up to a bounded deadline, for admitted request/body writes to settle. */
|
|
10
|
+
export declare function flushRequestLogs(timeoutMs?: number): Promise<void>;
|
|
9
11
|
export declare function initRequestLogger(enabled?: boolean, customLogsDir?: string): void;
|
|
10
12
|
export declare function logRequest(entry: RequestLogEntry): Promise<void>;
|
|
11
13
|
/**
|
|
@@ -9,15 +9,49 @@ import { join } from "path";
|
|
|
9
9
|
import { homedir } from "os";
|
|
10
10
|
import { logger } from "../utils/logger.js";
|
|
11
11
|
import { chmodSync, existsSync, mkdirSync, readdirSync, rmSync, statSync, unlinkSync, } from "fs";
|
|
12
|
-
import {
|
|
12
|
+
import { writeFile } from "fs/promises";
|
|
13
13
|
import { createHash } from "crypto";
|
|
14
14
|
import { promisify } from "util";
|
|
15
15
|
import { gzip as gzipCallback } from "zlib";
|
|
16
16
|
import { OtelBridge } from "../observability/otelBridge.js";
|
|
17
17
|
import { SeverityNumber } from "@opentelemetry/api-logs";
|
|
18
18
|
import { configureProxyLifecycleLogger } from "./proxyLifecycle.js";
|
|
19
|
+
import { withTimeout } from "../utils/async/withTimeout.js";
|
|
19
20
|
let logDir = null;
|
|
20
21
|
let logEnabled = false;
|
|
22
|
+
const pendingLogOperations = new Set();
|
|
23
|
+
const REQUEST_LOG_IO_TIMEOUT_MS = 5_000;
|
|
24
|
+
function trackLogOperation(operation) {
|
|
25
|
+
pendingLogOperations.add(operation);
|
|
26
|
+
void operation.then(() => pendingLogOperations.delete(operation), () => pendingLogOperations.delete(operation));
|
|
27
|
+
return operation;
|
|
28
|
+
}
|
|
29
|
+
/** Wait, up to a bounded deadline, for admitted request/body writes to settle. */
|
|
30
|
+
export async function flushRequestLogs(timeoutMs = REQUEST_LOG_IO_TIMEOUT_MS) {
|
|
31
|
+
const deadline = Date.now() + Math.max(1, timeoutMs);
|
|
32
|
+
while (pendingLogOperations.size > 0) {
|
|
33
|
+
const admitted = [...pendingLogOperations];
|
|
34
|
+
const remainingMs = Math.max(1, deadline - Date.now());
|
|
35
|
+
try {
|
|
36
|
+
await withTimeout(Promise.allSettled(admitted), remainingMs, `Timed out flushing ${admitted.length} proxy request log operation(s)`);
|
|
37
|
+
}
|
|
38
|
+
catch (error) {
|
|
39
|
+
for (const operation of admitted) {
|
|
40
|
+
pendingLogOperations.delete(operation);
|
|
41
|
+
}
|
|
42
|
+
throw error;
|
|
43
|
+
}
|
|
44
|
+
if (Date.now() >= deadline && pendingLogOperations.size > 0) {
|
|
45
|
+
const remaining = pendingLogOperations.size;
|
|
46
|
+
throw new Error(`Timed out flushing ${remaining} proxy request log operation(s)`);
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
/** @internal Test-only hook for exercising shutdown behavior without real I/O. */
|
|
51
|
+
export const __requestLoggerTestHooks = {
|
|
52
|
+
pendingOperationCount: () => pendingLogOperations.size,
|
|
53
|
+
trackLogOperation,
|
|
54
|
+
};
|
|
21
55
|
/**
|
|
22
56
|
* Lazily-resolved LoggerProvider from OTel instrumentation.
|
|
23
57
|
* null = not resolved yet (will retry), LoggerProvider = resolved, false = permanently unavailable.
|
|
@@ -86,7 +120,11 @@ export async function logRequest(entry) {
|
|
|
86
120
|
const logFile = join(logDir, `proxy-${new Date().toISOString().split("T")[0]}.jsonl`);
|
|
87
121
|
const line = JSON.stringify(entry) + "\n";
|
|
88
122
|
try {
|
|
89
|
-
await
|
|
123
|
+
await trackLogOperation(writeFile(logFile, line, {
|
|
124
|
+
mode: 0o600,
|
|
125
|
+
flag: "a",
|
|
126
|
+
signal: AbortSignal.timeout(REQUEST_LOG_IO_TIMEOUT_MS),
|
|
127
|
+
}));
|
|
90
128
|
}
|
|
91
129
|
catch {
|
|
92
130
|
// Non-fatal — don't crash proxy for logging failures
|
|
@@ -114,7 +152,11 @@ export async function logRequestAttempt(entry) {
|
|
|
114
152
|
const logFile = join(logDir, `proxy-attempts-${new Date().toISOString().split("T")[0]}.jsonl`);
|
|
115
153
|
const line = JSON.stringify(entry) + "\n";
|
|
116
154
|
try {
|
|
117
|
-
await
|
|
155
|
+
await trackLogOperation(writeFile(logFile, line, {
|
|
156
|
+
mode: 0o600,
|
|
157
|
+
flag: "a",
|
|
158
|
+
signal: AbortSignal.timeout(REQUEST_LOG_IO_TIMEOUT_MS),
|
|
159
|
+
}));
|
|
118
160
|
}
|
|
119
161
|
catch {
|
|
120
162
|
// Non-fatal — don't crash proxy for logging failures
|
|
@@ -353,7 +395,7 @@ function collectManagedLogFiles(rootDir) {
|
|
|
353
395
|
continue;
|
|
354
396
|
}
|
|
355
397
|
const isTopLevelProxyLog = directory === rootDir &&
|
|
356
|
-
/^proxy(?:-attempts|-debug)?-.*\.jsonl$/.test(entry.name);
|
|
398
|
+
/^proxy(?:-attempts|-debug|-lifecycle)?-.*\.jsonl$/.test(entry.name);
|
|
357
399
|
const isBodyArtifact = entry.name.endsWith(".json.gz") &&
|
|
358
400
|
entryPath.includes(`${join(rootDir, "bodies")}`);
|
|
359
401
|
if (!isTopLevelProxyLog && !isBodyArtifact) {
|
|
@@ -427,7 +469,10 @@ async function writeBodyArtifact(entry, redactedHeaders, redactedBody, bodyTrunc
|
|
|
427
469
|
metadata: entry.metadata,
|
|
428
470
|
});
|
|
429
471
|
const compressed = await gzip(payload);
|
|
430
|
-
await writeFile(bodyPath, compressed, {
|
|
472
|
+
await writeFile(bodyPath, compressed, {
|
|
473
|
+
mode: 0o600,
|
|
474
|
+
signal: AbortSignal.timeout(REQUEST_LOG_IO_TIMEOUT_MS),
|
|
475
|
+
});
|
|
431
476
|
return {
|
|
432
477
|
bodyPath,
|
|
433
478
|
bodySha256: sha256(redactedBody),
|
|
@@ -510,7 +555,7 @@ export async function logBodyCapture(entry) {
|
|
|
510
555
|
const preparedBody = prepareRedactedBody(entry.body);
|
|
511
556
|
let stored;
|
|
512
557
|
try {
|
|
513
|
-
stored = await writeBodyArtifact(entry, redactedHeaders, preparedBody.value, preparedBody.truncated);
|
|
558
|
+
stored = await trackLogOperation(writeBodyArtifact(entry, redactedHeaders, preparedBody.value, preparedBody.truncated));
|
|
514
559
|
}
|
|
515
560
|
catch (writeError) {
|
|
516
561
|
logger.warn("[RequestLogger] writeBodyArtifact failed, falling back to in-memory body for OTLP", { error: writeError });
|
|
@@ -518,6 +563,7 @@ export async function logBodyCapture(entry) {
|
|
|
518
563
|
redactedBody: preparedBody.value,
|
|
519
564
|
redactedBodyBytes: preparedBody.bytes,
|
|
520
565
|
bodyTruncated: preparedBody.truncated,
|
|
566
|
+
bodyWriteFailed: true,
|
|
521
567
|
};
|
|
522
568
|
}
|
|
523
569
|
const dateStr = new Date(entry.timestamp).toISOString().split("T")[0];
|
|
@@ -542,6 +588,7 @@ export async function logBodyCapture(entry) {
|
|
|
542
588
|
redactedBodyBytes: stored.redactedBodyBytes ?? preparedBody.bytes,
|
|
543
589
|
storedFileBytes: stored.storedFileBytes,
|
|
544
590
|
bodyTruncated: stored.bodyTruncated ?? preparedBody.truncated,
|
|
591
|
+
bodyWriteFailed: stored.bodyWriteFailed,
|
|
545
592
|
metadata: entry.metadata,
|
|
546
593
|
};
|
|
547
594
|
if (traceCtx) {
|
|
@@ -549,9 +596,11 @@ export async function logBodyCapture(entry) {
|
|
|
549
596
|
indexEntry.spanId = traceCtx.spanId;
|
|
550
597
|
}
|
|
551
598
|
try {
|
|
552
|
-
await
|
|
599
|
+
await trackLogOperation(writeFile(logFile, JSON.stringify(indexEntry) + "\n", {
|
|
553
600
|
mode: 0o600,
|
|
554
|
-
|
|
601
|
+
flag: "a",
|
|
602
|
+
signal: AbortSignal.timeout(REQUEST_LOG_IO_TIMEOUT_MS),
|
|
603
|
+
}));
|
|
555
604
|
}
|
|
556
605
|
catch {
|
|
557
606
|
// Non-fatal
|
|
@@ -623,9 +672,11 @@ export async function logStreamError(entry) {
|
|
|
623
672
|
logEntry.spanId = traceCtx.spanId;
|
|
624
673
|
}
|
|
625
674
|
try {
|
|
626
|
-
await
|
|
675
|
+
await trackLogOperation(writeFile(logFile, JSON.stringify(logEntry) + "\n", {
|
|
627
676
|
mode: 0o600,
|
|
628
|
-
|
|
677
|
+
flag: "a",
|
|
678
|
+
signal: AbortSignal.timeout(REQUEST_LOG_IO_TIMEOUT_MS),
|
|
679
|
+
}));
|
|
629
680
|
}
|
|
630
681
|
catch {
|
|
631
682
|
// Non-fatal — don't crash proxy for logging failures
|
|
@@ -644,13 +695,16 @@ export function cleanupLogs(maxAgeDays = 7, maxSizeMb = 500) {
|
|
|
644
695
|
try {
|
|
645
696
|
const activeLogDir = logDir;
|
|
646
697
|
const files = collectManagedLogFiles(activeLogDir).sort((a, b) => a.mtime - b.mtime); // oldest first
|
|
698
|
+
const currentDate = new Date().toISOString().split("T")[0];
|
|
699
|
+
const currentMetadataLogs = new Set(["proxy", "proxy-attempts", "proxy-debug", "proxy-lifecycle"].map((prefix) => join(activeLogDir, `${prefix}-${currentDate}.jsonl`)));
|
|
700
|
+
const canDelete = (file) => !currentMetadataLogs.has(file.path);
|
|
647
701
|
const cutoff = Date.now() - maxAgeDays * 24 * 60 * 60 * 1000;
|
|
648
702
|
let deletedCount = 0;
|
|
649
703
|
let freedBytes = 0;
|
|
650
704
|
// Pass 1: delete files older than maxAgeDays
|
|
651
705
|
const remaining = [];
|
|
652
706
|
for (const file of files) {
|
|
653
|
-
if (file.mtime < cutoff) {
|
|
707
|
+
if (file.mtime < cutoff && canDelete(file)) {
|
|
654
708
|
unlinkSync(file.path);
|
|
655
709
|
deletedCount++;
|
|
656
710
|
freedBytes += file.size;
|
|
@@ -666,8 +720,13 @@ export function cleanupLogs(maxAgeDays = 7, maxSizeMb = 500) {
|
|
|
666
720
|
// Pass 2: if total size exceeds maxSizeMb, delete oldest until under limit
|
|
667
721
|
const maxBytes = maxSizeMb * 1024 * 1024;
|
|
668
722
|
let totalSize = remaining.reduce((sum, f) => sum + f.size, 0);
|
|
669
|
-
|
|
670
|
-
|
|
723
|
+
const deletionCandidates = remaining.filter(canDelete);
|
|
724
|
+
// Current-day metadata is the only reliable source for final-request,
|
|
725
|
+
// attempt, lifecycle, and body-index reconciliation. Keep those indexes
|
|
726
|
+
// intact during size cleanup; body artifacts and older indexes remain
|
|
727
|
+
// eligible for eviction.
|
|
728
|
+
while (totalSize > maxBytes && deletionCandidates.length > 0) {
|
|
729
|
+
const oldest = deletionCandidates.shift();
|
|
671
730
|
if (!oldest) {
|
|
672
731
|
break;
|
|
673
732
|
}
|
|
@@ -10,8 +10,28 @@ export async function startRollingProxyServer(options) {
|
|
|
10
10
|
let listening = false;
|
|
11
11
|
let recoveryFailures = 0;
|
|
12
12
|
let recoveryTimer;
|
|
13
|
+
let requestedReplacementTimer;
|
|
14
|
+
let requestedReplacementSchedule = 0;
|
|
15
|
+
let requestedReplacementPending = false;
|
|
16
|
+
let replacementQueueTail = null;
|
|
13
17
|
const recoveryDelayMs = Math.max(1, options.recoveryDelayMs ?? DEFAULT_RECOVERY_DELAY_MS);
|
|
14
18
|
const maxRecoveryDelayMs = Math.max(recoveryDelayMs, options.maxRecoveryDelayMs ?? DEFAULT_MAX_RECOVERY_DELAY_MS);
|
|
19
|
+
const queueReplacement = (operation) => {
|
|
20
|
+
const predecessor = replacementQueueTail;
|
|
21
|
+
const result = (predecessor ?? Promise.resolve()).then(operation);
|
|
22
|
+
const completion = result.then(() => undefined, () => undefined);
|
|
23
|
+
replacementQueueTail = completion;
|
|
24
|
+
void completion.finally(() => {
|
|
25
|
+
if (replacementQueueTail !== completion) {
|
|
26
|
+
return;
|
|
27
|
+
}
|
|
28
|
+
replacementQueueTail = null;
|
|
29
|
+
if (requestedReplacementPending) {
|
|
30
|
+
scheduleRequestedReplacement();
|
|
31
|
+
}
|
|
32
|
+
});
|
|
33
|
+
return result;
|
|
34
|
+
};
|
|
15
35
|
const scheduleRecovery = () => {
|
|
16
36
|
if (closing || !listening || recoveryTimer) {
|
|
17
37
|
return;
|
|
@@ -23,11 +43,13 @@ export async function startRollingProxyServer(options) {
|
|
|
23
43
|
// An explicit replace() may have started (or completed) a generation
|
|
24
44
|
// while this timer was pending. Re-validate before recovering so we never
|
|
25
45
|
// launch a duplicate generation that conflicts with the requested worker.
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
46
|
+
void queueReplacement(async () => {
|
|
47
|
+
const snapshot = supervisor.snapshot();
|
|
48
|
+
if (closing || snapshot.active || snapshot.candidate) {
|
|
49
|
+
return;
|
|
50
|
+
}
|
|
51
|
+
await supervisor.replace(desiredVersion);
|
|
52
|
+
}).then(() => {
|
|
31
53
|
recoveryFailures = 0;
|
|
32
54
|
}, (error) => {
|
|
33
55
|
recoveryFailures += 1;
|
|
@@ -48,6 +70,38 @@ export async function startRollingProxyServer(options) {
|
|
|
48
70
|
scheduleRecovery();
|
|
49
71
|
}
|
|
50
72
|
};
|
|
73
|
+
function scheduleRequestedReplacement() {
|
|
74
|
+
if (closing) {
|
|
75
|
+
return;
|
|
76
|
+
}
|
|
77
|
+
requestedReplacementPending = true;
|
|
78
|
+
if (requestedReplacementTimer || replacementQueueTail) {
|
|
79
|
+
return;
|
|
80
|
+
}
|
|
81
|
+
const schedule = ++requestedReplacementSchedule;
|
|
82
|
+
requestedReplacementTimer = setTimeout(() => {
|
|
83
|
+
requestedReplacementTimer = undefined;
|
|
84
|
+
if (schedule !== requestedReplacementSchedule) {
|
|
85
|
+
return;
|
|
86
|
+
}
|
|
87
|
+
if (closing || !supervisor.snapshot().active) {
|
|
88
|
+
return;
|
|
89
|
+
}
|
|
90
|
+
requestedReplacementPending = false;
|
|
91
|
+
const replacementVersion = desiredVersion;
|
|
92
|
+
void queueReplacement(async () => {
|
|
93
|
+
if (closing || !supervisor.snapshot().active) {
|
|
94
|
+
return;
|
|
95
|
+
}
|
|
96
|
+
options.log?.(`[proxy-supervisor] preparing same-version worker replacement version=${replacementVersion} reason=environment`);
|
|
97
|
+
await supervisor.replace(replacementVersion);
|
|
98
|
+
options.log?.(`[proxy-supervisor] same-version worker replacement complete version=${replacementVersion} reason=environment`);
|
|
99
|
+
}).catch((error) => {
|
|
100
|
+
options.log?.(`[proxy-supervisor] same-version worker replacement failed version=${replacementVersion} reason=environment: ${error instanceof Error ? error.message : String(error)}`);
|
|
101
|
+
});
|
|
102
|
+
}, 50);
|
|
103
|
+
requestedReplacementTimer.unref?.();
|
|
104
|
+
}
|
|
51
105
|
const supervisor = new RollingWorkerSupervisor({
|
|
52
106
|
spawnWorker: options.spawnWorker,
|
|
53
107
|
readyTimeoutMs: options.readyTimeoutMs,
|
|
@@ -55,9 +109,14 @@ export async function startRollingProxyServer(options) {
|
|
|
55
109
|
socketQueueTimeoutMs: options.socketQueueTimeoutMs,
|
|
56
110
|
shutdownTimeoutMs: options.shutdownTimeoutMs,
|
|
57
111
|
onStateChange: stateChanged,
|
|
112
|
+
onReplacementRequested: scheduleRequestedReplacement,
|
|
58
113
|
log: options.log,
|
|
59
114
|
});
|
|
60
115
|
const listener = createServer({ pauseOnConnect: true }, (socket) => {
|
|
116
|
+
// The parent keeps its descriptor until the worker commits the IPC
|
|
117
|
+
// transfer. Consume client resets during that interval so they cannot
|
|
118
|
+
// terminate the long-lived supervisor process.
|
|
119
|
+
socket.once("error", () => socket.destroy());
|
|
61
120
|
supervisor.acceptSocket(socket);
|
|
62
121
|
});
|
|
63
122
|
await new Promise((resolve, reject) => {
|
|
@@ -89,8 +148,8 @@ export async function startRollingProxyServer(options) {
|
|
|
89
148
|
}
|
|
90
149
|
const address = listener.address();
|
|
91
150
|
if (!address || typeof address === "string") {
|
|
92
|
-
listener.close();
|
|
93
|
-
|
|
151
|
+
await new Promise((resolve) => listener.close(() => resolve()));
|
|
152
|
+
await supervisor.close().catch(() => undefined);
|
|
94
153
|
throw ErrorFactory.proxyWorkerLifecycle("rolling proxy listener did not expose a TCP address");
|
|
95
154
|
}
|
|
96
155
|
return {
|
|
@@ -111,8 +170,14 @@ export async function startRollingProxyServer(options) {
|
|
|
111
170
|
clearTimeout(recoveryTimer);
|
|
112
171
|
recoveryTimer = undefined;
|
|
113
172
|
}
|
|
173
|
+
if (requestedReplacementTimer) {
|
|
174
|
+
clearTimeout(requestedReplacementTimer);
|
|
175
|
+
requestedReplacementTimer = undefined;
|
|
176
|
+
}
|
|
177
|
+
requestedReplacementSchedule += 1;
|
|
178
|
+
requestedReplacementPending = false;
|
|
114
179
|
try {
|
|
115
|
-
const snapshot = await supervisor.replace(expectedVersion);
|
|
180
|
+
const snapshot = await queueReplacement(() => supervisor.replace(expectedVersion));
|
|
116
181
|
recoveryFailures = 0;
|
|
117
182
|
return snapshot;
|
|
118
183
|
}
|
|
@@ -139,6 +204,12 @@ export async function startRollingProxyServer(options) {
|
|
|
139
204
|
clearTimeout(recoveryTimer);
|
|
140
205
|
recoveryTimer = undefined;
|
|
141
206
|
}
|
|
207
|
+
if (requestedReplacementTimer) {
|
|
208
|
+
clearTimeout(requestedReplacementTimer);
|
|
209
|
+
requestedReplacementTimer = undefined;
|
|
210
|
+
}
|
|
211
|
+
requestedReplacementSchedule += 1;
|
|
212
|
+
requestedReplacementPending = false;
|
|
142
213
|
const listenerClosed = new Promise((resolve, reject) => {
|
|
143
214
|
listener.close((error) => (error ? reject(error) : resolve()));
|
|
144
215
|
});
|
|
@@ -2,7 +2,7 @@ import { spawn } from "node:child_process";
|
|
|
2
2
|
import { ErrorFactory } from "../utils/errorHandling.js";
|
|
3
3
|
import { isProxyWorkerStatusMessage, PROXY_SOCKET_WORKER_ENV, } from "./rollingWorkerProtocol.js";
|
|
4
4
|
export function spawnProxySocketWorker(options) {
|
|
5
|
-
const socketAckTimeoutMs = Math.max(1, options.socketAckTimeoutMs ??
|
|
5
|
+
const socketAckTimeoutMs = Math.max(1, options.socketAckTimeoutMs ?? 30_000);
|
|
6
6
|
let nextSocketId = 0;
|
|
7
7
|
const pendingSockets = new Map();
|
|
8
8
|
const statusListeners = new Set();
|
|
@@ -59,6 +59,18 @@ export function spawnProxySocketWorker(options) {
|
|
|
59
59
|
}
|
|
60
60
|
pendingSockets.delete(socketId);
|
|
61
61
|
clearTimeout(pending.timeout);
|
|
62
|
+
if (error && child.connected && !pending.accepted) {
|
|
63
|
+
try {
|
|
64
|
+
child.send({
|
|
65
|
+
type: "proxy-worker:socket-cancel",
|
|
66
|
+
generation: options.generation,
|
|
67
|
+
socketId,
|
|
68
|
+
}, () => undefined);
|
|
69
|
+
}
|
|
70
|
+
catch {
|
|
71
|
+
// The supervisor will quarantine the worker after the failed transfer.
|
|
72
|
+
}
|
|
73
|
+
}
|
|
62
74
|
if (!error) {
|
|
63
75
|
pending.socket.destroy();
|
|
64
76
|
}
|
|
@@ -92,9 +104,14 @@ export function spawnProxySocketWorker(options) {
|
|
|
92
104
|
publishStatus(message);
|
|
93
105
|
}
|
|
94
106
|
});
|
|
95
|
-
child.once("exit", () => {
|
|
107
|
+
child.once("exit", (code, signal) => {
|
|
96
108
|
for (const socketId of [...pendingSockets.keys()]) {
|
|
97
|
-
settleSocket(socketId,
|
|
109
|
+
settleSocket(socketId, ErrorFactory.proxyWorkerLifecycle(`proxy worker ${childPid} exited before socket transfer committed (code=${code ?? "none"}, signal=${signal ?? "none"})`, {
|
|
110
|
+
workerPid: childPid,
|
|
111
|
+
generation: options.generation,
|
|
112
|
+
exitCode: code,
|
|
113
|
+
signal,
|
|
114
|
+
}));
|
|
98
115
|
}
|
|
99
116
|
});
|
|
100
117
|
const sendControl = (message) => {
|
|
@@ -117,18 +134,6 @@ export function spawnProxySocketWorker(options) {
|
|
|
117
134
|
}
|
|
118
135
|
const socketId = `${generation}:${++nextSocketId}`;
|
|
119
136
|
const timeout = setTimeout(() => {
|
|
120
|
-
if (child.connected) {
|
|
121
|
-
try {
|
|
122
|
-
child.send({
|
|
123
|
-
type: "proxy-worker:socket-cancel",
|
|
124
|
-
generation,
|
|
125
|
-
socketId,
|
|
126
|
-
});
|
|
127
|
-
}
|
|
128
|
-
catch {
|
|
129
|
-
// The worker is terminated after the failed transfer is reported.
|
|
130
|
-
}
|
|
131
|
-
}
|
|
132
137
|
settleSocket(socketId, new Error(`proxy worker ${childPid} did not accept socket within ${socketAckTimeoutMs}ms`));
|
|
133
138
|
}, socketAckTimeoutMs);
|
|
134
139
|
timeout.unref?.();
|
|
@@ -38,6 +38,9 @@ export function isProxyWorkerStatusMessage(value) {
|
|
|
38
38
|
if (message.type === "proxy-worker:socket-accepted") {
|
|
39
39
|
return typeof message.socketId === "string" && message.socketId.length > 0;
|
|
40
40
|
}
|
|
41
|
+
if (message.type === "proxy-worker:replacement-requested") {
|
|
42
|
+
return message.reason === "environment";
|
|
43
|
+
}
|
|
41
44
|
return (message.type === "proxy-worker:fatal" && typeof message.message === "string");
|
|
42
45
|
}
|
|
43
46
|
//# sourceMappingURL=rollingWorkerProtocol.js.map
|