@juspay/neurolink 12.12.9 → 12.12.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +2 -2
- package/dist/browser/neurolink.min.js +394 -396
- package/dist/cli/commands/proxy.d.ts +4 -0
- package/dist/cli/commands/proxy.js +60 -18
- package/dist/cli/commands/proxyAnalyze.js +8 -1
- package/dist/core/baseProvider.d.ts +0 -22
- package/dist/core/baseProvider.js +18 -81
- package/dist/providers/anthropic/client.js +4 -1
- package/dist/providers/openaiChatCompletionsBase.js +4 -1
- package/dist/proxy/bodyCaptureProcessing.d.ts +22 -0
- package/dist/proxy/bodyCaptureProcessing.js +219 -0
- package/dist/proxy/bodyCaptureWorker.d.ts +16 -0
- package/dist/proxy/bodyCaptureWorker.js +232 -0
- package/dist/proxy/bodyCaptureWorkerEntry.d.ts +1 -0
- package/dist/proxy/bodyCaptureWorkerEntry.js +34 -0
- package/dist/proxy/proxyAnalysis.js +159 -17
- package/dist/proxy/proxyLifecycle.d.ts +25 -0
- package/dist/proxy/proxyLifecycle.js +111 -5
- package/dist/proxy/proxyRequestKind.d.ts +2 -0
- package/dist/proxy/proxyRequestKind.js +6 -0
- package/dist/proxy/proxyRuntimeMetrics.d.ts +3 -0
- package/dist/proxy/proxyRuntimeMetrics.js +34 -0
- package/dist/proxy/requestLogger.d.ts +10 -6
- package/dist/proxy/requestLogger.js +99 -231
- package/dist/proxy/rollingProxyServer.js +15 -4
- package/dist/proxy/rollingWorkerProcess.d.ts +4 -0
- package/dist/proxy/rollingWorkerProcess.js +25 -8
- package/dist/proxy/rollingWorkerProtocol.d.ts +6 -0
- package/dist/proxy/rollingWorkerProtocol.js +12 -1
- package/dist/proxy/rollingWorkerSupervisor.d.ts +28 -0
- package/dist/proxy/rollingWorkerSupervisor.js +73 -25
- package/dist/proxy/socketWorkerRuntime.d.ts +5 -0
- package/dist/proxy/socketWorkerRuntime.js +19 -2
- package/dist/server/routes/codexProxyRoutes.js +39 -3
- package/dist/services/server/ai/observability/instrumentation.js +7 -1
- package/dist/types/cli.d.ts +1 -1
- package/dist/types/proxy.d.ts +69 -4
- package/package.json +1 -1
- package/dist/core/modules/GenerationHandler.d.ts +0 -145
- package/dist/core/modules/GenerationHandler.js +0 -754
|
@@ -1,17 +1,20 @@
|
|
|
1
1
|
import { createReadStream } from "node:fs";
|
|
2
2
|
import { isDeepStrictEqual } from "node:util";
|
|
3
|
+
import { isProxyAuxiliaryRequest } from "./proxyRequestKind.js";
|
|
3
4
|
import { lstat, readdir, realpath, stat } from "node:fs/promises";
|
|
4
5
|
import { homedir } from "node:os";
|
|
5
6
|
import { createInterface } from "node:readline";
|
|
6
7
|
import { isAbsolute, join, relative, resolve, sep } from "node:path";
|
|
7
8
|
import { ACCOUNT_COOLING_REASONS, PROXY_ACCOUNT_TYPES, PROXY_ACCOUNT_ROUTING_MODES, PROXY_ACCOUNT_ROUTING_REASONS, PROXY_ACCOUNT_ROUTING_STRATEGIES, } from "./routingEvidence.js";
|
|
8
9
|
import { calculateCost, hasPricing, isExactPricingMatch, } from "../utils/pricing.js";
|
|
9
|
-
const LIFECYCLE_FILE_PATTERN = /^proxy-lifecycle-\d{4}-\d{2}-\d{2}\.jsonl$/;
|
|
10
|
+
const LIFECYCLE_FILE_PATTERN = /^proxy-(?:lifecycle|supervisor)-\d{4}-\d{2}-\d{2}\.jsonl$/;
|
|
10
11
|
const REQUEST_FILE_PATTERN = /^proxy-\d{4}-\d{2}-\d{2}\.jsonl$/;
|
|
11
12
|
const ATTEMPT_FILE_PATTERN = /^proxy-attempts-\d{4}-\d{2}-\d{2}\.jsonl$/;
|
|
12
13
|
const DEBUG_FILE_PATTERN = /^proxy-debug-\d{4}-\d{2}-\d{2}\.jsonl$/;
|
|
13
14
|
const ARTIFACT_STAT_CONCURRENCY = 64;
|
|
14
15
|
const LIFECYCLE_EVENTS = new Set([
|
|
16
|
+
"runtime_sample",
|
|
17
|
+
"supervisor_event",
|
|
15
18
|
"request_accepted",
|
|
16
19
|
"response_headers",
|
|
17
20
|
"response_first_chunk",
|
|
@@ -585,6 +588,7 @@ export async function analyzeProxyLogs(options) {
|
|
|
585
588
|
let malformedLines = 0;
|
|
586
589
|
let unsupportedLifecycleLines = 0;
|
|
587
590
|
const accepted = new Set();
|
|
591
|
+
const auxiliaryRequests = new Set();
|
|
588
592
|
const headers = new Set();
|
|
589
593
|
const firstChunks = new Set();
|
|
590
594
|
const terminal = new Set();
|
|
@@ -595,18 +599,36 @@ export async function analyzeProxyLogs(options) {
|
|
|
595
599
|
const firstChunkLatencyByRequest = new Map();
|
|
596
600
|
const terminalLatencyByRequest = new Map();
|
|
597
601
|
const sequences = new Map();
|
|
602
|
+
const selectedSequenceRanges = new Map();
|
|
598
603
|
const seenLifecycleEvents = new Map();
|
|
599
604
|
let conflictingLifecycleDuplicates = 0;
|
|
600
605
|
const conflictedRequests = new Set();
|
|
601
606
|
const terminalRecords = new Map();
|
|
607
|
+
const acceptedWorkers = new Map();
|
|
608
|
+
const admittedWorkerIds = new Set();
|
|
609
|
+
const conflictedWorkerExits = new Set();
|
|
610
|
+
const conflictingIdentities = new Set();
|
|
611
|
+
const runtimeRecords = new Map();
|
|
612
|
+
const workerExits = new Map();
|
|
613
|
+
const runtime = {
|
|
614
|
+
samples: 0,
|
|
615
|
+
maxEventLoopDelayMs: null,
|
|
616
|
+
maxRssBytes: null,
|
|
617
|
+
maxCpuPercentOneCore: null,
|
|
618
|
+
maxHostLoad1m: null,
|
|
619
|
+
};
|
|
602
620
|
for (const filePath of lifecycleFiles) {
|
|
603
621
|
linesRead += await readJsonLines(filePath, (record) => {
|
|
604
|
-
const
|
|
622
|
+
const operational = record.event === "runtime_sample" ||
|
|
623
|
+
record.event === "supervisor_event";
|
|
624
|
+
const operationalTimestamp = Date.parse(String(record.timestamp));
|
|
625
|
+
const timestamp = operational
|
|
626
|
+
? Number.isFinite(operationalTimestamp)
|
|
627
|
+
? operationalTimestamp
|
|
628
|
+
: null
|
|
629
|
+
: observeTimestamp("lifecycle", record);
|
|
605
630
|
const requestId = stringValue(record.requestId);
|
|
606
|
-
if (timestamp === null ||
|
|
607
|
-
!requestId ||
|
|
608
|
-
(!accepted.has(requestId) &&
|
|
609
|
-
(timestamp < sinceMs || timestamp > untilMs))) {
|
|
631
|
+
if (timestamp === null || !requestId) {
|
|
610
632
|
return;
|
|
611
633
|
}
|
|
612
634
|
const event = stringValue(record.event);
|
|
@@ -614,7 +636,10 @@ export async function analyzeProxyLogs(options) {
|
|
|
614
636
|
!event ||
|
|
615
637
|
!LIFECYCLE_EVENTS.has(event) ||
|
|
616
638
|
!requestId) {
|
|
617
|
-
|
|
639
|
+
if (accepted.has(requestId) ||
|
|
640
|
+
(timestamp >= sinceMs && timestamp <= untilMs)) {
|
|
641
|
+
unsupportedLifecycleLines += 1;
|
|
642
|
+
}
|
|
618
643
|
return;
|
|
619
644
|
}
|
|
620
645
|
const processId = stringValue(record.processInstanceId);
|
|
@@ -623,11 +648,40 @@ export async function analyzeProxyLogs(options) {
|
|
|
623
648
|
const values = sequences.get(processId) ?? [];
|
|
624
649
|
values.push(sequence);
|
|
625
650
|
sequences.set(processId, values);
|
|
651
|
+
}
|
|
652
|
+
const details = record.supervisorEvent && typeof record.supervisorEvent === "object"
|
|
653
|
+
? record.supervisorEvent
|
|
654
|
+
: undefined;
|
|
655
|
+
const exitedWorkerId = stringValue(details?.workerProcessInstanceId);
|
|
656
|
+
const relatedExit = event === "supervisor_event" &&
|
|
657
|
+
details?.type === "worker_exit" &&
|
|
658
|
+
exitedWorkerId &&
|
|
659
|
+
admittedWorkerIds.has(exitedWorkerId);
|
|
660
|
+
// Audit intervening sequences before selecting the request cohort.
|
|
661
|
+
if (!accepted.has(requestId) &&
|
|
662
|
+
!relatedExit &&
|
|
663
|
+
(timestamp < sinceMs || timestamp > untilMs)) {
|
|
664
|
+
return;
|
|
665
|
+
}
|
|
666
|
+
if (processId && sequence !== null && Number.isInteger(sequence)) {
|
|
667
|
+
const range = selectedSequenceRanges.get(processId);
|
|
668
|
+
selectedSequenceRanges.set(processId, {
|
|
669
|
+
min: Math.min(range?.min ?? sequence, sequence),
|
|
670
|
+
max: Math.max(range?.max ?? sequence, sequence),
|
|
671
|
+
});
|
|
626
672
|
const identity = `${processId}:${sequence}`;
|
|
627
673
|
const previous = seenLifecycleEvents.get(identity);
|
|
628
674
|
if (previous) {
|
|
629
675
|
if (!isDeepStrictEqual(previous, record)) {
|
|
630
676
|
conflictingLifecycleDuplicates += 1;
|
|
677
|
+
conflictingIdentities.add(identity);
|
|
678
|
+
for (const copy of [previous, record]) {
|
|
679
|
+
const detail = copy.supervisorEvent;
|
|
680
|
+
const workerId = stringValue(detail?.workerProcessInstanceId);
|
|
681
|
+
if (workerId) {
|
|
682
|
+
conflictedWorkerExits.add(workerId);
|
|
683
|
+
}
|
|
684
|
+
}
|
|
631
685
|
conflictedRequests.add(requestId);
|
|
632
686
|
const previousRequestId = stringValue(previous.requestId);
|
|
633
687
|
if (previousRequestId) {
|
|
@@ -638,9 +692,36 @@ export async function analyzeProxyLogs(options) {
|
|
|
638
692
|
}
|
|
639
693
|
seenLifecycleEvents.set(identity, record);
|
|
640
694
|
}
|
|
695
|
+
if (event === "runtime_sample") {
|
|
696
|
+
if (record.runtimeSample &&
|
|
697
|
+
typeof record.runtimeSample === "object" &&
|
|
698
|
+
processId &&
|
|
699
|
+
sequence !== null) {
|
|
700
|
+
runtimeRecords.set(`${processId}:${sequence}`, record.runtimeSample);
|
|
701
|
+
}
|
|
702
|
+
return;
|
|
703
|
+
}
|
|
704
|
+
if (event === "supervisor_event") {
|
|
705
|
+
if (details?.type === "worker_exit" && exitedWorkerId) {
|
|
706
|
+
const exit = { ...details, at: record.timestamp };
|
|
707
|
+
const previous = workerExits.get(exitedWorkerId);
|
|
708
|
+
if (previous && !isDeepStrictEqual(previous, exit)) {
|
|
709
|
+
conflictedWorkerExits.add(exitedWorkerId);
|
|
710
|
+
}
|
|
711
|
+
workerExits.set(exitedWorkerId, exit);
|
|
712
|
+
}
|
|
713
|
+
return;
|
|
714
|
+
}
|
|
715
|
+
if (isProxyAuxiliaryRequest(stringValue(record.method) ?? "", stringValue(record.path) ?? "")) {
|
|
716
|
+
auxiliaryRequests.add(requestId);
|
|
717
|
+
}
|
|
641
718
|
const elapsed = finiteNumber(record.elapsedMs);
|
|
642
719
|
if (event === "request_accepted") {
|
|
643
720
|
accepted.add(requestId);
|
|
721
|
+
if (processId) {
|
|
722
|
+
acceptedWorkers.set(requestId, processId);
|
|
723
|
+
admittedWorkerIds.add(processId);
|
|
724
|
+
}
|
|
644
725
|
}
|
|
645
726
|
else if (event === "response_headers") {
|
|
646
727
|
if (headers.has(requestId)) {
|
|
@@ -683,6 +764,25 @@ export async function analyzeProxyLogs(options) {
|
|
|
683
764
|
malformedLines += 1;
|
|
684
765
|
});
|
|
685
766
|
}
|
|
767
|
+
for (const [identity, sample] of runtimeRecords) {
|
|
768
|
+
if (conflictingIdentities.has(identity)) {
|
|
769
|
+
continue;
|
|
770
|
+
}
|
|
771
|
+
runtime.samples += 1;
|
|
772
|
+
const mapping = {
|
|
773
|
+
maxEventLoopDelayMs: "eventLoopDelayMaxMs",
|
|
774
|
+
maxRssBytes: "rssBytes",
|
|
775
|
+
maxCpuPercentOneCore: "cpuPercentOneCore",
|
|
776
|
+
maxHostLoad1m: "hostLoad1m",
|
|
777
|
+
};
|
|
778
|
+
for (const [key, source] of Object.entries(mapping)) {
|
|
779
|
+
const value = finiteNumber(sample[source]);
|
|
780
|
+
if (value !== null && value >= 0) {
|
|
781
|
+
const target = key;
|
|
782
|
+
runtime[target] = Math.max(runtime[target] ?? value, value);
|
|
783
|
+
}
|
|
784
|
+
}
|
|
785
|
+
}
|
|
686
786
|
// Contradictory copies are not reliable latency samples. Keep their data
|
|
687
787
|
// quality count, but do not choose one timing arbitrarily.
|
|
688
788
|
const verifiedLatencies = (values) => [...values]
|
|
@@ -693,7 +793,12 @@ export async function analyzeProxyLogs(options) {
|
|
|
693
793
|
const terminalLatency = verifiedLatencies(terminalLatencyByRequest);
|
|
694
794
|
let lifecycleSequenceGaps = 0;
|
|
695
795
|
let lifecycleSequenceDuplicates = 0;
|
|
696
|
-
for (const
|
|
796
|
+
for (const [processId, allValues] of sequences) {
|
|
797
|
+
const range = selectedSequenceRanges.get(processId);
|
|
798
|
+
if (!range) {
|
|
799
|
+
continue;
|
|
800
|
+
}
|
|
801
|
+
const values = allValues.filter((value) => value >= range.min && value <= range.max);
|
|
697
802
|
values.sort((a, b) => a - b);
|
|
698
803
|
for (let index = 1; index < values.length; index += 1) {
|
|
699
804
|
const difference = values[index] - values[index - 1];
|
|
@@ -918,6 +1023,9 @@ export async function analyzeProxyLogs(options) {
|
|
|
918
1023
|
for (const [requestId, record] of terminalRecords) {
|
|
919
1024
|
const final = finalRequests.get(requestId);
|
|
920
1025
|
const recorded = stringValue(record.terminalOutcome) ?? "unknown";
|
|
1026
|
+
const auxiliaryTransport = auxiliaryRequests.has(requestId)
|
|
1027
|
+
? (stringValue(record.transportOutcome) ?? recorded)
|
|
1028
|
+
: null;
|
|
921
1029
|
const resolved = final
|
|
922
1030
|
? final.status === 499 || final.errorType === "client_cancelled"
|
|
923
1031
|
? "client_cancelled"
|
|
@@ -927,11 +1035,20 @@ export async function analyzeProxyLogs(options) {
|
|
|
927
1035
|
: final.status >= 400 || final.errorType
|
|
928
1036
|
? "handler_error"
|
|
929
1037
|
: "completed"
|
|
930
|
-
: conflictedRequests.has(requestId)
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
1038
|
+
: auxiliaryTransport && !conflictedRequests.has(requestId)
|
|
1039
|
+
? auxiliaryTransport === "completed" ||
|
|
1040
|
+
auxiliaryTransport === "bodyless"
|
|
1041
|
+
? finiteNumber(record.responseStatus) === null
|
|
1042
|
+
? "unknown"
|
|
1043
|
+
: Number(record.responseStatus) >= 400
|
|
1044
|
+
? "handler_error"
|
|
1045
|
+
: auxiliaryTransport
|
|
1046
|
+
: auxiliaryTransport
|
|
1047
|
+
: conflictedRequests.has(requestId) ||
|
|
1048
|
+
recorded === "completed" ||
|
|
1049
|
+
recorded === "bodyless"
|
|
1050
|
+
? "unknown"
|
|
1051
|
+
: recorded;
|
|
935
1052
|
if (final &&
|
|
936
1053
|
recorded !== resolved &&
|
|
937
1054
|
!(recorded === "bodyless" && resolved === "completed")) {
|
|
@@ -1038,10 +1155,8 @@ export async function analyzeProxyLogs(options) {
|
|
|
1038
1155
|
conflictingLifecycleDuplicates,
|
|
1039
1156
|
duplicateAttempts,
|
|
1040
1157
|
finalOutcomeConflicts,
|
|
1041
|
-
acceptedWithoutFinal: [...accepted].filter((id) => !finalRequests.has(id))
|
|
1042
|
-
|
|
1043
|
-
terminalWithoutFinal: [...terminal].filter((id) => !finalRequests.has(id))
|
|
1044
|
-
.length,
|
|
1158
|
+
acceptedWithoutFinal: [...accepted].filter((id) => !finalRequests.has(id) && !auxiliaryRequests.has(id)).length,
|
|
1159
|
+
terminalWithoutFinal: [...terminal].filter((id) => !finalRequests.has(id) && !auxiliaryRequests.has(id)).length,
|
|
1045
1160
|
streams: Object.fromEntries(Object.entries(observedRanges).map(([stream, range]) => [
|
|
1046
1161
|
stream,
|
|
1047
1162
|
{
|
|
@@ -1066,8 +1181,35 @@ export async function analyzeProxyLogs(options) {
|
|
|
1066
1181
|
absent: absentRoutingDecisions,
|
|
1067
1182
|
},
|
|
1068
1183
|
},
|
|
1184
|
+
runtime,
|
|
1069
1185
|
lifecycle: {
|
|
1186
|
+
unconfirmedAtWorkerExit: [...accepted].flatMap((requestId) => {
|
|
1187
|
+
const workerProcessInstanceId = acceptedWorkers.get(requestId);
|
|
1188
|
+
const exit = workerProcessInstanceId
|
|
1189
|
+
? workerExits.get(workerProcessInstanceId)
|
|
1190
|
+
: undefined;
|
|
1191
|
+
if (!exit ||
|
|
1192
|
+
!workerProcessInstanceId ||
|
|
1193
|
+
conflictedWorkerExits.has(workerProcessInstanceId) ||
|
|
1194
|
+
terminal.has(requestId) ||
|
|
1195
|
+
conflictedRequests.has(requestId)) {
|
|
1196
|
+
return [];
|
|
1197
|
+
}
|
|
1198
|
+
return [
|
|
1199
|
+
{
|
|
1200
|
+
requestId,
|
|
1201
|
+
workerProcessInstanceId,
|
|
1202
|
+
at: String(exit.at),
|
|
1203
|
+
workerExitCode: finiteNumber(exit.workerExitCode),
|
|
1204
|
+
workerExitSignal: stringValue(exit.workerExitSignal),
|
|
1205
|
+
// A provider final cannot prove the client received the entire body.
|
|
1206
|
+
providerFinalRecorded: finalRequests.has(requestId),
|
|
1207
|
+
},
|
|
1208
|
+
];
|
|
1209
|
+
}),
|
|
1070
1210
|
accepted: accepted.size,
|
|
1211
|
+
auxiliaryRequests: [...accepted].filter((id) => auxiliaryRequests.has(id))
|
|
1212
|
+
.length,
|
|
1071
1213
|
headers: headers.size,
|
|
1072
1214
|
firstChunks: firstChunks.size,
|
|
1073
1215
|
terminal: terminal.size,
|
|
@@ -1,13 +1,38 @@
|
|
|
1
|
+
import { chmodSync } from "node:fs";
|
|
1
2
|
import { appendFile } from "node:fs/promises";
|
|
2
3
|
import type { ProxyLifecycleEventInput, ProxyLifecycleLoggerOptions, ProxyLifecycleLoggerSnapshot } from "../types/index.js";
|
|
3
4
|
export declare function hashProxyLifecycleSessionId(sessionId: string | undefined): string | undefined;
|
|
5
|
+
/**
|
|
6
|
+
* Configure private journal storage while retaining required admission
|
|
7
|
+
* when initialization fails.
|
|
8
|
+
*/
|
|
4
9
|
export declare function configureProxyLifecycleLogger(options: ProxyLifecycleLoggerOptions): void;
|
|
5
10
|
/** Enqueue fixed-size lifecycle metadata without awaiting filesystem work. */
|
|
6
11
|
export declare function logProxyLifecycleEvent(input: ProxyLifecycleEventInput): void;
|
|
12
|
+
/** Confirm admission before upstream dispatch, without waiting for later traffic.
|
|
13
|
+
* Disabled logging is explicit; an enabled but unhealthy sink refuses dispatch.
|
|
14
|
+
* A timeout never retries an ambiguous provider request or the pending append.
|
|
15
|
+
*/
|
|
16
|
+
export declare function persistProxyLifecycleAcceptance(input: Omit<ProxyLifecycleEventInput, "event">, timeoutMs?: number): Promise<void>;
|
|
7
17
|
export declare function flushProxyLifecycleEvents(timeoutMs?: number): Promise<void>;
|
|
18
|
+
/**
|
|
19
|
+
* Expose journal accounting, process identity, and outstanding writes
|
|
20
|
+
* without changing them.
|
|
21
|
+
*/
|
|
8
22
|
export declare function getProxyLifecycleLoggerSnapshot(): ProxyLifecycleLoggerSnapshot;
|
|
23
|
+
/**
|
|
24
|
+
* Reset timers, accounting, and injected I/O after isolated tests have
|
|
25
|
+
* drained their work.
|
|
26
|
+
*/
|
|
9
27
|
export declare function resetProxyLifecycleLoggerForTests(): void;
|
|
10
28
|
/** Isolated failure injection for lifecycle durability tests. */
|
|
11
29
|
export declare const __proxyLifecycleTestHooks: {
|
|
30
|
+
/**
|
|
31
|
+
* Inject directory-hardening failures without altering real filesystem permissions.
|
|
32
|
+
*/
|
|
33
|
+
setChmodForTests(chmod: typeof chmodSync): void;
|
|
34
|
+
/**
|
|
35
|
+
* Inject controlled append outcomes while preserving the production admission and queue paths.
|
|
36
|
+
*/
|
|
12
37
|
setAppendFileForTests(append: typeof appendFile): void;
|
|
13
38
|
};
|
|
@@ -5,6 +5,7 @@ import { join } from "node:path";
|
|
|
5
5
|
import { performance } from "node:perf_hooks";
|
|
6
6
|
import { withTimeout } from "../utils/async/withTimeout.js";
|
|
7
7
|
import { logger } from "../utils/logger.js";
|
|
8
|
+
import { startProxyRuntimeMetrics } from "./proxyRuntimeMetrics.js";
|
|
8
9
|
const SCHEMA_VERSION = 1;
|
|
9
10
|
const DEFAULT_QUEUE_CAPACITY = 10_000;
|
|
10
11
|
const DEFAULT_BATCH_SIZE = 256;
|
|
@@ -14,8 +15,12 @@ const MAX_WRITE_RETRY_DELAY_MS = 1_000;
|
|
|
14
15
|
const LIFECYCLE_APPEND_TIMEOUT_MS = 2_000;
|
|
15
16
|
const MAX_SHORT_FIELD_LENGTH = 256;
|
|
16
17
|
const SESSION_KEY_FILE = ".proxy-lifecycle-session-key";
|
|
18
|
+
let chmodLifecycleDirectory = chmodSync;
|
|
17
19
|
let loggerEnabled = false;
|
|
20
|
+
let loggerRequired = false;
|
|
21
|
+
let stopRuntimeMetrics;
|
|
18
22
|
let lifecycleLogDir;
|
|
23
|
+
let filePrefix = "proxy-lifecycle";
|
|
19
24
|
let queueCapacity = DEFAULT_QUEUE_CAPACITY;
|
|
20
25
|
let batchSize = DEFAULT_BATCH_SIZE;
|
|
21
26
|
let flushIntervalMs = DEFAULT_FLUSH_INTERVAL_MS;
|
|
@@ -141,6 +146,10 @@ function scheduleFlush(delayMs = flushIntervalMs) {
|
|
|
141
146
|
}, delayMs);
|
|
142
147
|
flushTimer.unref?.();
|
|
143
148
|
}
|
|
149
|
+
/**
|
|
150
|
+
* Append a bounded metadata batch and confirm each admission without
|
|
151
|
+
* replaying ambiguous writes.
|
|
152
|
+
*/
|
|
144
153
|
async function flushBatch() {
|
|
145
154
|
if (queue.length === 0) {
|
|
146
155
|
return;
|
|
@@ -150,7 +159,7 @@ async function flushBatch() {
|
|
|
150
159
|
try {
|
|
151
160
|
const byPath = new Map();
|
|
152
161
|
for (const item of batch) {
|
|
153
|
-
const path = join(item.logDir,
|
|
162
|
+
const path = join(item.logDir, `${item.filePrefix ?? "proxy-lifecycle"}-${item.date}.jsonl`);
|
|
154
163
|
const items = byPath.get(path) ?? [];
|
|
155
164
|
items.push(item);
|
|
156
165
|
byPath.set(path, items);
|
|
@@ -167,11 +176,13 @@ async function flushBatch() {
|
|
|
167
176
|
}, LIFECYCLE_APPEND_TIMEOUT_MS);
|
|
168
177
|
timeout.unref?.();
|
|
169
178
|
try {
|
|
170
|
-
//
|
|
171
|
-
//
|
|
172
|
-
// writeDrops/writeFailures rather than delaying proxy responses.
|
|
179
|
+
// OS-acknowledged append survives serving-process death. This is not
|
|
180
|
+
// an fsync/power-loss guarantee. Admission waits on its own record.
|
|
173
181
|
await appendLifecycleFile(path, lines.join(""), { mode: 0o600 });
|
|
174
182
|
written += lines.length;
|
|
183
|
+
for (const item of items) {
|
|
184
|
+
item.onPersisted?.(true);
|
|
185
|
+
}
|
|
175
186
|
}
|
|
176
187
|
catch (error) {
|
|
177
188
|
writeFailures += 1;
|
|
@@ -189,6 +200,9 @@ async function flushBatch() {
|
|
|
189
200
|
]).has(code ?? "");
|
|
190
201
|
if (!definitelyNotWritten) {
|
|
191
202
|
unconfirmedWrites += items.length;
|
|
203
|
+
for (const item of items) {
|
|
204
|
+
item.onPersisted?.(false);
|
|
205
|
+
}
|
|
192
206
|
logger.warn("[proxy] lifecycle metadata append outcome is uncertain", {
|
|
193
207
|
path,
|
|
194
208
|
records: items.length,
|
|
@@ -211,6 +225,11 @@ async function flushBatch() {
|
|
|
211
225
|
if (exhausted > 0) {
|
|
212
226
|
dropped += exhausted;
|
|
213
227
|
writeDrops += exhausted;
|
|
228
|
+
for (const item of items) {
|
|
229
|
+
if (item.writeRetries >= maxWriteRetries) {
|
|
230
|
+
item.onPersisted?.(false);
|
|
231
|
+
}
|
|
232
|
+
}
|
|
214
233
|
}
|
|
215
234
|
logger.warn("[proxy] lifecycle metadata write failed", {
|
|
216
235
|
path,
|
|
@@ -234,6 +253,10 @@ async function flushBatch() {
|
|
|
234
253
|
inFlight = Math.max(0, inFlight - batch.length);
|
|
235
254
|
}
|
|
236
255
|
}
|
|
256
|
+
/**
|
|
257
|
+
* Serialize batch ownership so timeouts cannot create overlapping append
|
|
258
|
+
* retries.
|
|
259
|
+
*/
|
|
237
260
|
function startFlush() {
|
|
238
261
|
if (flushInFlight) {
|
|
239
262
|
return flushInFlight;
|
|
@@ -257,10 +280,18 @@ function startFlush() {
|
|
|
257
280
|
});
|
|
258
281
|
return currentFlush;
|
|
259
282
|
}
|
|
283
|
+
/**
|
|
284
|
+
* Configure private journal storage while retaining required admission
|
|
285
|
+
* when initialization fails.
|
|
286
|
+
*/
|
|
260
287
|
export function configureProxyLifecycleLogger(options) {
|
|
288
|
+
stopRuntimeMetrics?.();
|
|
289
|
+
stopRuntimeMetrics = undefined;
|
|
261
290
|
clearScheduledFlush();
|
|
262
291
|
nextFlushDelayMs = undefined;
|
|
263
292
|
loggerEnabled = false;
|
|
293
|
+
loggerRequired = options.enabled;
|
|
294
|
+
filePrefix = options.filePrefix ?? "proxy-lifecycle";
|
|
264
295
|
lifecycleLogDir = undefined;
|
|
265
296
|
queueCapacity = positiveInteger(options.queueCapacity, DEFAULT_QUEUE_CAPACITY);
|
|
266
297
|
batchSize = positiveInteger(options.batchSize, DEFAULT_BATCH_SIZE);
|
|
@@ -269,9 +300,21 @@ export function configureProxyLifecycleLogger(options) {
|
|
|
269
300
|
if (options.enabled && options.logDir) {
|
|
270
301
|
try {
|
|
271
302
|
mkdirSync(options.logDir, { recursive: true, mode: 0o700 });
|
|
303
|
+
// mkdir mode does not harden an existing directory. Keep admission
|
|
304
|
+
// required but the sink disabled if its privacy boundary cannot be set.
|
|
305
|
+
chmodLifecycleDirectory(options.logDir, 0o700);
|
|
272
306
|
sessionHashKey = resolveSessionHashKey(options.logDir);
|
|
273
307
|
lifecycleLogDir = options.logDir;
|
|
274
308
|
loggerEnabled = true;
|
|
309
|
+
stopRuntimeMetrics = startProxyRuntimeMetrics((runtimeSample) => {
|
|
310
|
+
logProxyLifecycleEvent({
|
|
311
|
+
event: "runtime_sample",
|
|
312
|
+
requestId: "-",
|
|
313
|
+
method: "-",
|
|
314
|
+
path: "-",
|
|
315
|
+
runtimeSample,
|
|
316
|
+
});
|
|
317
|
+
});
|
|
275
318
|
}
|
|
276
319
|
catch (error) {
|
|
277
320
|
logger.warn("[proxy] lifecycle metadata logging disabled", {
|
|
@@ -286,15 +329,24 @@ export function configureProxyLifecycleLogger(options) {
|
|
|
286
329
|
}
|
|
287
330
|
/** Enqueue fixed-size lifecycle metadata without awaiting filesystem work. */
|
|
288
331
|
export function logProxyLifecycleEvent(input) {
|
|
332
|
+
enqueueLifecycleEvent(input);
|
|
333
|
+
}
|
|
334
|
+
/**
|
|
335
|
+
* Enqueue bounded metadata and resolve admission failure immediately when
|
|
336
|
+
* capacity is unavailable.
|
|
337
|
+
*/
|
|
338
|
+
function enqueueLifecycleEvent(input, onPersisted) {
|
|
289
339
|
if (!loggerEnabled || !lifecycleLogDir) {
|
|
340
|
+
onPersisted?.(false);
|
|
290
341
|
return;
|
|
291
342
|
}
|
|
292
343
|
try {
|
|
293
344
|
attempted += 1;
|
|
294
345
|
const sequence = nextSequence++;
|
|
295
|
-
if (queue.length >= queueCapacity) {
|
|
346
|
+
if (queue.length + inFlight >= queueCapacity) {
|
|
296
347
|
dropped += 1;
|
|
297
348
|
queueDrops += 1;
|
|
349
|
+
onPersisted?.(false);
|
|
298
350
|
return;
|
|
299
351
|
}
|
|
300
352
|
const timestamp = formatTimestamp(input.timestampMs);
|
|
@@ -344,12 +396,23 @@ export function logProxyLifecycleEvent(input) {
|
|
|
344
396
|
...(terminalOutcome !== undefined ? { terminalOutcome } : {}),
|
|
345
397
|
...(errorType !== undefined ? { errorType } : {}),
|
|
346
398
|
...(errorCode !== undefined ? { errorCode } : {}),
|
|
399
|
+
...(input.supervisorEvent
|
|
400
|
+
? {
|
|
401
|
+
supervisorEvent: {
|
|
402
|
+
...input.supervisorEvent,
|
|
403
|
+
reason: clip(input.supervisorEvent.reason),
|
|
404
|
+
},
|
|
405
|
+
}
|
|
406
|
+
: {}),
|
|
407
|
+
...(input.runtimeSample ? { runtimeSample: input.runtimeSample } : {}),
|
|
347
408
|
};
|
|
348
409
|
queue.push({
|
|
410
|
+
filePrefix,
|
|
349
411
|
logDir: lifecycleLogDir,
|
|
350
412
|
date: String(record.timestamp).slice(0, 10),
|
|
351
413
|
record,
|
|
352
414
|
writeRetries: 0,
|
|
415
|
+
onPersisted,
|
|
353
416
|
});
|
|
354
417
|
enqueued += 1;
|
|
355
418
|
scheduleFlush();
|
|
@@ -357,6 +420,28 @@ export function logProxyLifecycleEvent(input) {
|
|
|
357
420
|
catch {
|
|
358
421
|
dropped += 1;
|
|
359
422
|
invalidDrops += 1;
|
|
423
|
+
onPersisted?.(false);
|
|
424
|
+
}
|
|
425
|
+
}
|
|
426
|
+
/** Confirm admission before upstream dispatch, without waiting for later traffic.
|
|
427
|
+
* Disabled logging is explicit; an enabled but unhealthy sink refuses dispatch.
|
|
428
|
+
* A timeout never retries an ambiguous provider request or the pending append.
|
|
429
|
+
*/
|
|
430
|
+
export async function persistProxyLifecycleAcceptance(input, timeoutMs = LIFECYCLE_APPEND_TIMEOUT_MS) {
|
|
431
|
+
if (!loggerRequired) {
|
|
432
|
+
return;
|
|
433
|
+
}
|
|
434
|
+
const confirmed = new Promise((resolve) => {
|
|
435
|
+
enqueueLifecycleEvent({ ...input, event: "request_accepted" }, resolve);
|
|
436
|
+
});
|
|
437
|
+
// Yield one turn for concurrent admissions to share a bounded batch.
|
|
438
|
+
clearScheduledFlush();
|
|
439
|
+
scheduleFlush(0);
|
|
440
|
+
const persisted = await withTimeout(confirmed, timeoutMs, "Proxy admission metadata append is still pending").catch(() => false);
|
|
441
|
+
if (!persisted) {
|
|
442
|
+
throw Object.assign(new Error("Proxy admission metadata could not be confirmed"), {
|
|
443
|
+
code: "PROXY_TELEMETRY_UNAVAILABLE",
|
|
444
|
+
});
|
|
360
445
|
}
|
|
361
446
|
}
|
|
362
447
|
export async function flushProxyLifecycleEvents(timeoutMs = 5_000) {
|
|
@@ -370,6 +455,10 @@ export async function flushProxyLifecycleEvents(timeoutMs = 5_000) {
|
|
|
370
455
|
}
|
|
371
456
|
}
|
|
372
457
|
}
|
|
458
|
+
/**
|
|
459
|
+
* Expose journal accounting, process identity, and outstanding writes
|
|
460
|
+
* without changing them.
|
|
461
|
+
*/
|
|
373
462
|
export function getProxyLifecycleLoggerSnapshot() {
|
|
374
463
|
return {
|
|
375
464
|
enabled: loggerEnabled,
|
|
@@ -392,9 +481,16 @@ export function getProxyLifecycleLoggerSnapshot() {
|
|
|
392
481
|
flushing: flushInFlight !== undefined,
|
|
393
482
|
};
|
|
394
483
|
}
|
|
484
|
+
/**
|
|
485
|
+
* Reset timers, accounting, and injected I/O after isolated tests have
|
|
486
|
+
* drained their work.
|
|
487
|
+
*/
|
|
395
488
|
export function resetProxyLifecycleLoggerForTests() {
|
|
489
|
+
stopRuntimeMetrics?.();
|
|
490
|
+
stopRuntimeMetrics = undefined;
|
|
396
491
|
clearScheduledFlush();
|
|
397
492
|
loggerEnabled = false;
|
|
493
|
+
loggerRequired = false;
|
|
398
494
|
lifecycleLogDir = undefined;
|
|
399
495
|
queueCapacity = DEFAULT_QUEUE_CAPACITY;
|
|
400
496
|
batchSize = DEFAULT_BATCH_SIZE;
|
|
@@ -419,9 +515,19 @@ export function resetProxyLifecycleLoggerForTests() {
|
|
|
419
515
|
flushInFlight = undefined;
|
|
420
516
|
nextFlushDelayMs = undefined;
|
|
421
517
|
appendLifecycleFile = appendFile;
|
|
518
|
+
chmodLifecycleDirectory = chmodSync;
|
|
422
519
|
}
|
|
423
520
|
/** Isolated failure injection for lifecycle durability tests. */
|
|
424
521
|
export const __proxyLifecycleTestHooks = {
|
|
522
|
+
/**
|
|
523
|
+
* Inject directory-hardening failures without altering real filesystem permissions.
|
|
524
|
+
*/
|
|
525
|
+
setChmodForTests(chmod) {
|
|
526
|
+
chmodLifecycleDirectory = chmod;
|
|
527
|
+
},
|
|
528
|
+
/**
|
|
529
|
+
* Inject controlled append outcomes while preserving the production admission and queue paths.
|
|
530
|
+
*/
|
|
425
531
|
setAppendFileForTests(append) {
|
|
426
532
|
appendLifecycleFile = append;
|
|
427
533
|
},
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
/** These metadata endpoints finish as HTTP responses, without model generation. */
|
|
2
|
+
export function isProxyAuxiliaryRequest(method, path) {
|
|
3
|
+
return ((method === "POST" && path === "/v1/messages/count_tokens") ||
|
|
4
|
+
(method === "GET" &&
|
|
5
|
+
(path === "/backend-api/codex/models" || path === "/v1/models")));
|
|
6
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { monitorEventLoopDelay, performance } from "node:perf_hooks";
|
|
2
|
+
import { availableParallelism, loadavg } from "node:os";
|
|
3
|
+
/** Samples use actual elapsed time, including delayed timers under host load. */
|
|
4
|
+
export function startProxyRuntimeMetrics(emit) {
|
|
5
|
+
const histogram = monitorEventLoopDelay({ resolution: 20 });
|
|
6
|
+
histogram.enable();
|
|
7
|
+
let previousCpu = process.cpuUsage();
|
|
8
|
+
let previousTime = performance.now();
|
|
9
|
+
const timer = setInterval(() => {
|
|
10
|
+
const now = performance.now();
|
|
11
|
+
const cpu = process.cpuUsage();
|
|
12
|
+
const intervalMs = now - previousTime;
|
|
13
|
+
emit({
|
|
14
|
+
intervalMs,
|
|
15
|
+
cpuPercentOneCore: ((cpu.user - previousCpu.user + cpu.system - previousCpu.system) /
|
|
16
|
+
(intervalMs * 1000)) *
|
|
17
|
+
100,
|
|
18
|
+
rssBytes: process.memoryUsage().rss,
|
|
19
|
+
heapUsedBytes: process.memoryUsage().heapUsed,
|
|
20
|
+
eventLoopDelayP99Ms: histogram.count ? histogram.percentile(99) / 1e6 : 0,
|
|
21
|
+
eventLoopDelayMaxMs: histogram.count ? histogram.max / 1e6 : 0,
|
|
22
|
+
hostLoad1m: loadavg()[0],
|
|
23
|
+
availableParallelism: availableParallelism(),
|
|
24
|
+
});
|
|
25
|
+
previousCpu = cpu;
|
|
26
|
+
previousTime = now;
|
|
27
|
+
histogram.reset();
|
|
28
|
+
}, 10_000);
|
|
29
|
+
timer.unref();
|
|
30
|
+
return () => {
|
|
31
|
+
clearInterval(timer);
|
|
32
|
+
histogram.disable();
|
|
33
|
+
};
|
|
34
|
+
}
|
|
@@ -6,9 +6,15 @@
|
|
|
6
6
|
* Useful for debugging and auditing proxy traffic.
|
|
7
7
|
*/
|
|
8
8
|
import type { ProxyBodyCaptureEntry, RequestAttemptLogEntry, RequestLogEntry, ProxyRequestLoggerSnapshot } from "../types/index.js";
|
|
9
|
+
/**
|
|
10
|
+
* Expose independent metadata-sink and body-capture counters for incident reconciliation.
|
|
11
|
+
*/
|
|
9
12
|
export declare function getRequestLoggerSnapshot(): ProxyRequestLoggerSnapshot;
|
|
10
13
|
/** Wait, up to a bounded deadline, for admitted request/body writes to settle. */
|
|
11
14
|
export declare function flushRequestLogs(timeoutMs?: number): Promise<void>;
|
|
15
|
+
/**
|
|
16
|
+
* Initialize private request logs and preserve required lifecycle admission on startup failures.
|
|
17
|
+
*/
|
|
12
18
|
export declare function initRequestLogger(enabled?: boolean, customLogsDir?: string): void;
|
|
13
19
|
export declare function logRequest(entry: RequestLogEntry): Promise<void>;
|
|
14
20
|
/**
|
|
@@ -18,19 +24,17 @@ export declare function logRequest(entry: RequestLogEntry): Promise<void>;
|
|
|
18
24
|
*/
|
|
19
25
|
export declare function logRequestAttempt(entry: RequestAttemptLogEntry): Promise<void>;
|
|
20
26
|
export declare function getLogDir(): string | null;
|
|
21
|
-
/** Shared redaction used by offline replay exports and direct comparisons. */
|
|
22
|
-
export declare function redactProxyHeadersForLogging(headers: Record<string, string> | undefined): Record<string, string> | undefined;
|
|
23
27
|
/**
|
|
24
|
-
*
|
|
25
|
-
* `value` and `bytes` are omitted only when the input is null or undefined.
|
|
26
|
-
* This performs serialization immediately, so callers must keep it off proxy
|
|
27
|
-
* hot paths unless body processing has already been explicitly requested.
|
|
28
|
+
* Redact sensitive header values in-place.
|
|
28
29
|
*/
|
|
30
|
+
export declare function redactProxyHeadersForLogging(headers: Record<string, string> | undefined): Record<string, string> | undefined;
|
|
31
|
+
/** Return a redacted body representation suitable for persisted request diagnostics. */
|
|
29
32
|
export declare function prepareProxyBodyForLogging(body: unknown): {
|
|
30
33
|
value?: string;
|
|
31
34
|
bytes?: number;
|
|
32
35
|
truncated: boolean;
|
|
33
36
|
};
|
|
37
|
+
/** Capture an owned request body with bounded processing and tracked index/export publication. */
|
|
34
38
|
export declare function logBodyCapture(entry: ProxyBodyCaptureEntry): Promise<void>;
|
|
35
39
|
/**
|
|
36
40
|
* Log the FULL raw request and response for debugging.
|