@juspay/neurolink 12.12.6 → 12.12.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +2 -2
- package/dist/browser/neurolink.min.js +398 -400
- package/dist/cli/commands/proxy.js +62 -24
- package/dist/cli/commands/proxyAnalyze.js +4 -1
- package/dist/proxy/codexUsage.d.ts +2 -1
- package/dist/proxy/codexUsage.js +82 -33
- package/dist/proxy/proxyActivity.d.ts +4 -1
- package/dist/proxy/proxyActivity.js +40 -14
- package/dist/proxy/proxyAnalysis.js +184 -59
- package/dist/proxy/proxyLifecycle.d.ts +1 -1
- package/dist/proxy/proxyLifecycle.js +54 -8
- package/dist/proxy/requestLogger.d.ts +2 -1
- package/dist/proxy/requestLogger.js +87 -29
- package/dist/proxy/sseInterceptor.js +36 -18
- package/dist/proxy/streamOutcome.d.ts +1 -1
- package/dist/proxy/streamOutcome.js +7 -1
- package/dist/server/routes/claudeProxyRoutes.js +70 -16
- package/dist/server/routes/codexProxyRoutes.js +73 -8
- package/dist/types/proxy.d.ts +69 -3
- package/package.json +2 -1
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { createReadStream } from "node:fs";
|
|
2
|
+
import { isDeepStrictEqual } from "node:util";
|
|
2
3
|
import { lstat, readdir, realpath, stat } from "node:fs/promises";
|
|
3
4
|
import { homedir } from "node:os";
|
|
4
5
|
import { createInterface } from "node:readline";
|
|
@@ -371,7 +372,9 @@ function summarizeFinalRequests(finalRequests, terminalStreamErrors, attemptsByR
|
|
|
371
372
|
const errorTypes = {};
|
|
372
373
|
const errorCodes = {};
|
|
373
374
|
for (const [requestId, request] of finalRequests) {
|
|
374
|
-
const failed = request.status >= 400 ||
|
|
375
|
+
const failed = request.status >= 400 ||
|
|
376
|
+
!!request.errorType ||
|
|
377
|
+
terminalStreamErrors.has(requestId);
|
|
375
378
|
if (failed) {
|
|
376
379
|
errors += 1;
|
|
377
380
|
}
|
|
@@ -588,18 +591,25 @@ export async function analyzeProxyLogs(options) {
|
|
|
588
591
|
const terminalOutcomes = {};
|
|
589
592
|
const lifecycleErrorTypes = {};
|
|
590
593
|
const lifecycleErrorCodes = {};
|
|
591
|
-
const
|
|
592
|
-
const
|
|
593
|
-
const
|
|
594
|
+
const headersLatencyByRequest = new Map();
|
|
595
|
+
const firstChunkLatencyByRequest = new Map();
|
|
596
|
+
const terminalLatencyByRequest = new Map();
|
|
594
597
|
const sequences = new Map();
|
|
598
|
+
const seenLifecycleEvents = new Map();
|
|
599
|
+
let conflictingLifecycleDuplicates = 0;
|
|
600
|
+
const conflictedRequests = new Set();
|
|
601
|
+
const terminalRecords = new Map();
|
|
595
602
|
for (const filePath of lifecycleFiles) {
|
|
596
603
|
linesRead += await readJsonLines(filePath, (record) => {
|
|
597
604
|
const timestamp = observeTimestamp("lifecycle", record);
|
|
598
|
-
|
|
605
|
+
const requestId = stringValue(record.requestId);
|
|
606
|
+
if (timestamp === null ||
|
|
607
|
+
!requestId ||
|
|
608
|
+
(!accepted.has(requestId) &&
|
|
609
|
+
(timestamp < sinceMs || timestamp > untilMs))) {
|
|
599
610
|
return;
|
|
600
611
|
}
|
|
601
612
|
const event = stringValue(record.event);
|
|
602
|
-
const requestId = stringValue(record.requestId);
|
|
603
613
|
if (record.schemaVersion !== 1 ||
|
|
604
614
|
!event ||
|
|
605
615
|
!LIFECYCLE_EVENTS.has(event) ||
|
|
@@ -613,27 +623,51 @@ export async function analyzeProxyLogs(options) {
|
|
|
613
623
|
const values = sequences.get(processId) ?? [];
|
|
614
624
|
values.push(sequence);
|
|
615
625
|
sequences.set(processId, values);
|
|
626
|
+
const identity = `${processId}:${sequence}`;
|
|
627
|
+
const previous = seenLifecycleEvents.get(identity);
|
|
628
|
+
if (previous) {
|
|
629
|
+
if (!isDeepStrictEqual(previous, record)) {
|
|
630
|
+
conflictingLifecycleDuplicates += 1;
|
|
631
|
+
conflictedRequests.add(requestId);
|
|
632
|
+
const previousRequestId = stringValue(previous.requestId);
|
|
633
|
+
if (previousRequestId) {
|
|
634
|
+
conflictedRequests.add(previousRequestId);
|
|
635
|
+
}
|
|
636
|
+
}
|
|
637
|
+
return;
|
|
638
|
+
}
|
|
639
|
+
seenLifecycleEvents.set(identity, record);
|
|
616
640
|
}
|
|
617
641
|
const elapsed = finiteNumber(record.elapsedMs);
|
|
618
642
|
if (event === "request_accepted") {
|
|
619
643
|
accepted.add(requestId);
|
|
620
644
|
}
|
|
621
645
|
else if (event === "response_headers") {
|
|
646
|
+
if (headers.has(requestId)) {
|
|
647
|
+
return;
|
|
648
|
+
}
|
|
622
649
|
headers.add(requestId);
|
|
623
650
|
if (elapsed !== null && elapsed >= 0) {
|
|
624
|
-
|
|
651
|
+
headersLatencyByRequest.set(requestId, elapsed);
|
|
625
652
|
}
|
|
626
653
|
}
|
|
627
654
|
else if (event === "response_first_chunk") {
|
|
655
|
+
if (firstChunks.has(requestId)) {
|
|
656
|
+
return;
|
|
657
|
+
}
|
|
628
658
|
firstChunks.add(requestId);
|
|
629
659
|
if (elapsed !== null && elapsed >= 0) {
|
|
630
|
-
|
|
660
|
+
firstChunkLatencyByRequest.set(requestId, elapsed);
|
|
631
661
|
}
|
|
632
662
|
}
|
|
633
663
|
else {
|
|
664
|
+
if (terminal.has(requestId)) {
|
|
665
|
+
return;
|
|
666
|
+
}
|
|
634
667
|
terminal.add(requestId);
|
|
668
|
+
terminalRecords.set(requestId, record);
|
|
635
669
|
if (elapsed !== null && elapsed >= 0) {
|
|
636
|
-
|
|
670
|
+
terminalLatencyByRequest.set(requestId, elapsed);
|
|
637
671
|
}
|
|
638
672
|
increment(terminalOutcomes, stringValue(record.terminalOutcome) ?? "unknown");
|
|
639
673
|
const errorType = stringValue(record.errorType);
|
|
@@ -649,6 +683,14 @@ export async function analyzeProxyLogs(options) {
|
|
|
649
683
|
malformedLines += 1;
|
|
650
684
|
});
|
|
651
685
|
}
|
|
686
|
+
// Contradictory copies are not reliable latency samples. Keep their data
|
|
687
|
+
// quality count, but do not choose one timing arbitrarily.
|
|
688
|
+
const verifiedLatencies = (values) => [...values]
|
|
689
|
+
.filter(([id]) => !conflictedRequests.has(id))
|
|
690
|
+
.map(([, ms]) => ms);
|
|
691
|
+
const headersLatency = verifiedLatencies(headersLatencyByRequest);
|
|
692
|
+
const firstChunkLatency = verifiedLatencies(firstChunkLatencyByRequest);
|
|
693
|
+
const terminalLatency = verifiedLatencies(terminalLatencyByRequest);
|
|
652
694
|
let lifecycleSequenceGaps = 0;
|
|
653
695
|
let lifecycleSequenceDuplicates = 0;
|
|
654
696
|
for (const values of sequences.values()) {
|
|
@@ -675,70 +717,106 @@ export async function analyzeProxyLogs(options) {
|
|
|
675
717
|
let transientRateLimits = 0;
|
|
676
718
|
let quotaRateLimits = 0;
|
|
677
719
|
let unclassifiedRateLimits = 0;
|
|
720
|
+
const uniqueAttempts = new Map();
|
|
721
|
+
let duplicateAttempts = 0;
|
|
678
722
|
for (const filePath of attemptFiles) {
|
|
679
723
|
linesRead += await readJsonLines(filePath, (record) => {
|
|
680
724
|
const timestamp = observeTimestamp("attempts", record);
|
|
681
|
-
|
|
725
|
+
const requestId = stringValue(record.requestId);
|
|
726
|
+
const parentId = stringValue(record.parentRequestId) ??
|
|
727
|
+
requestId?.replace(/:codex-fallback$/, "");
|
|
728
|
+
if (timestamp === null ||
|
|
729
|
+
!requestId ||
|
|
730
|
+
(!accepted.has(parentId ?? requestId) &&
|
|
731
|
+
(timestamp < sinceMs || timestamp > untilMs))) {
|
|
682
732
|
return;
|
|
683
733
|
}
|
|
684
|
-
const requestId = stringValue(record.requestId);
|
|
685
734
|
const status = finiteNumber(record.responseStatus);
|
|
686
|
-
const duration = finiteNumber(record.attemptDurationMs);
|
|
687
735
|
if (!requestId || status === null) {
|
|
688
736
|
return;
|
|
689
737
|
}
|
|
690
|
-
const
|
|
691
|
-
const
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
accountStats.attempts += 1;
|
|
695
|
-
const hadError = status >= 400 || !!stringValue(record.errorType);
|
|
696
|
-
if (hadError) {
|
|
697
|
-
totalAttemptErrors += 1;
|
|
698
|
-
accountStats.attemptErrors += 1;
|
|
699
|
-
increment(attemptErrorTypes, stringValue(record.errorType) ?? `http_${status}`);
|
|
700
|
-
const errorCode = stringValue(record.errorCode);
|
|
701
|
-
if (errorCode) {
|
|
702
|
-
increment(attemptErrorCodes, errorCode);
|
|
703
|
-
}
|
|
704
|
-
}
|
|
705
|
-
const transportScope = stringValue(record.transportScope);
|
|
706
|
-
if (transportScope) {
|
|
707
|
-
increment(attemptTransportScopes, transportScope);
|
|
738
|
+
const identity = `${requestId}:${String(record.attempt ?? record.timestamp)}`;
|
|
739
|
+
const previous = uniqueAttempts.get(identity);
|
|
740
|
+
if (previous) {
|
|
741
|
+
duplicateAttempts += 1;
|
|
708
742
|
}
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
if (record.rateLimitKind === "transient") {
|
|
726
|
-
transientRateLimits += 1;
|
|
727
|
-
accountStats.transientRateLimits += 1;
|
|
728
|
-
}
|
|
729
|
-
else if (record.rateLimitKind === "quota") {
|
|
730
|
-
quotaRateLimits += 1;
|
|
731
|
-
accountStats.quotaRateLimits += 1;
|
|
732
|
-
}
|
|
733
|
-
else {
|
|
734
|
-
unclassifiedRateLimits += 1;
|
|
735
|
-
accountStats.unclassifiedRateLimits += 1;
|
|
743
|
+
// A later terminal update enriches the same attempt, never creates a
|
|
744
|
+
// second upstream call. Preserve failure evidence across enrichment.
|
|
745
|
+
uniqueAttempts.set(identity, previous
|
|
746
|
+
? {
|
|
747
|
+
...previous,
|
|
748
|
+
...record,
|
|
749
|
+
...((previous.errorType ||
|
|
750
|
+
Number(previous.responseStatus) >= 400) &&
|
|
751
|
+
!record.errorType &&
|
|
752
|
+
status < 400
|
|
753
|
+
? {
|
|
754
|
+
errorType: previous.errorType,
|
|
755
|
+
responseStatus: previous.responseStatus,
|
|
756
|
+
errorCode: previous.errorCode,
|
|
757
|
+
}
|
|
758
|
+
: {}),
|
|
736
759
|
}
|
|
737
|
-
|
|
760
|
+
: record);
|
|
738
761
|
}, () => {
|
|
739
762
|
malformedLines += 1;
|
|
740
763
|
});
|
|
741
764
|
}
|
|
765
|
+
for (const record of uniqueAttempts.values()) {
|
|
766
|
+
const rawRequestId = String(record.requestId);
|
|
767
|
+
const requestId = stringValue(record.parentRequestId) ??
|
|
768
|
+
rawRequestId.replace(/:codex-fallback$/, "");
|
|
769
|
+
const status = Number(record.responseStatus);
|
|
770
|
+
const duration = finiteNumber(record.attemptDurationMs);
|
|
771
|
+
const account = stringValue(record.account) ?? "unknown";
|
|
772
|
+
const accountType = stringValue(record.accountType) ?? "unknown";
|
|
773
|
+
const accountStats = accountEntry(accounts, account, accountType);
|
|
774
|
+
totalAttempts += 1;
|
|
775
|
+
accountStats.attempts += 1;
|
|
776
|
+
const hadError = status >= 400 || !!stringValue(record.errorType);
|
|
777
|
+
if (hadError) {
|
|
778
|
+
totalAttemptErrors += 1;
|
|
779
|
+
accountStats.attemptErrors += 1;
|
|
780
|
+
increment(attemptErrorTypes, stringValue(record.errorType) ?? `http_${status}`);
|
|
781
|
+
const errorCode = stringValue(record.errorCode);
|
|
782
|
+
if (errorCode) {
|
|
783
|
+
increment(attemptErrorCodes, errorCode);
|
|
784
|
+
}
|
|
785
|
+
}
|
|
786
|
+
const transportScope = stringValue(record.transportScope);
|
|
787
|
+
if (transportScope) {
|
|
788
|
+
increment(attemptTransportScopes, transportScope);
|
|
789
|
+
}
|
|
790
|
+
const requestAttempts = attemptsByRequest.get(requestId) ?? {
|
|
791
|
+
count: 0,
|
|
792
|
+
hadError: false,
|
|
793
|
+
totalDurationMs: 0,
|
|
794
|
+
durationCount: 0,
|
|
795
|
+
};
|
|
796
|
+
requestAttempts.count += 1;
|
|
797
|
+
requestAttempts.hadError = requestAttempts.hadError || hadError;
|
|
798
|
+
if (duration !== null && duration >= 0) {
|
|
799
|
+
requestAttempts.totalDurationMs += duration;
|
|
800
|
+
requestAttempts.durationCount += 1;
|
|
801
|
+
attemptLatency.push(duration);
|
|
802
|
+
}
|
|
803
|
+
attemptsByRequest.set(requestId, requestAttempts);
|
|
804
|
+
if (status === 429) {
|
|
805
|
+
attemptRateLimits += 1;
|
|
806
|
+
if (record.rateLimitKind === "transient") {
|
|
807
|
+
transientRateLimits += 1;
|
|
808
|
+
accountStats.transientRateLimits += 1;
|
|
809
|
+
}
|
|
810
|
+
else if (record.rateLimitKind === "quota") {
|
|
811
|
+
quotaRateLimits += 1;
|
|
812
|
+
accountStats.quotaRateLimits += 1;
|
|
813
|
+
}
|
|
814
|
+
else {
|
|
815
|
+
unclassifiedRateLimits += 1;
|
|
816
|
+
accountStats.unclassifiedRateLimits += 1;
|
|
817
|
+
}
|
|
818
|
+
}
|
|
819
|
+
}
|
|
742
820
|
const finalRequests = new Map();
|
|
743
821
|
const terminalStreamErrors = new Set();
|
|
744
822
|
let validRoutingDecisions = 0;
|
|
@@ -763,7 +841,9 @@ export async function analyzeProxyLogs(options) {
|
|
|
763
841
|
// but contributing nothing to tokens or cost, with nothing in the
|
|
764
842
|
// report to say so. A request the window already admitted therefore
|
|
765
843
|
// keeps accepting its own later records.
|
|
766
|
-
const alreadyAdmitted =
|
|
844
|
+
const alreadyAdmitted = accepted.has(requestId) ||
|
|
845
|
+
finalRequests.has(requestId) ||
|
|
846
|
+
terminalStreamErrors.has(requestId);
|
|
767
847
|
if (!alreadyAdmitted && (timestamp < sinceMs || timestamp > untilMs)) {
|
|
768
848
|
return;
|
|
769
849
|
}
|
|
@@ -789,6 +869,7 @@ export async function analyzeProxyLogs(options) {
|
|
|
789
869
|
absentRoutingDecisions += 1;
|
|
790
870
|
}
|
|
791
871
|
const parsed = {
|
|
872
|
+
firstUsefulOutputMs: finiteNumber(record.firstUsefulOutputMs),
|
|
792
873
|
timestamp: new Date(timestamp).toISOString(),
|
|
793
874
|
status,
|
|
794
875
|
durationMs: finiteNumber(record.responseTimeMs),
|
|
@@ -814,6 +895,9 @@ export async function analyzeProxyLogs(options) {
|
|
|
814
895
|
? {
|
|
815
896
|
...previous,
|
|
816
897
|
...Object.fromEntries(Object.entries(parsed).filter(([, value]) => value !== null && value !== undefined)),
|
|
898
|
+
...(previous.status >= 400 && status < 400
|
|
899
|
+
? { status: previous.status }
|
|
900
|
+
: {}),
|
|
817
901
|
// Attribute the request to when it was first seen. A late
|
|
818
902
|
// completion record must not move it out of the window that
|
|
819
903
|
// admitted it.
|
|
@@ -824,6 +908,37 @@ export async function analyzeProxyLogs(options) {
|
|
|
824
908
|
malformedLines += 1;
|
|
825
909
|
});
|
|
826
910
|
}
|
|
911
|
+
// Old lifecycle records described transport EOF as success even when the
|
|
912
|
+
// final request recorded a semantic failure. Reconcile, and expose every
|
|
913
|
+
// disagreement instead of letting a choice of input file change the answer.
|
|
914
|
+
let finalOutcomeConflicts = 0;
|
|
915
|
+
for (const key of Object.keys(terminalOutcomes)) {
|
|
916
|
+
delete terminalOutcomes[key];
|
|
917
|
+
}
|
|
918
|
+
for (const [requestId, record] of terminalRecords) {
|
|
919
|
+
const final = finalRequests.get(requestId);
|
|
920
|
+
const recorded = stringValue(record.terminalOutcome) ?? "unknown";
|
|
921
|
+
const resolved = final
|
|
922
|
+
? final.status === 499 || final.errorType === "client_cancelled"
|
|
923
|
+
? "client_cancelled"
|
|
924
|
+
: final.errorType?.includes("stream") ||
|
|
925
|
+
terminalStreamErrors.has(requestId)
|
|
926
|
+
? "stream_error"
|
|
927
|
+
: final.status >= 400 || final.errorType
|
|
928
|
+
? "handler_error"
|
|
929
|
+
: "completed"
|
|
930
|
+
: conflictedRequests.has(requestId) ||
|
|
931
|
+
recorded === "completed" ||
|
|
932
|
+
recorded === "bodyless"
|
|
933
|
+
? "unknown"
|
|
934
|
+
: recorded;
|
|
935
|
+
if (final &&
|
|
936
|
+
recorded !== resolved &&
|
|
937
|
+
!(recorded === "bodyless" && resolved === "completed")) {
|
|
938
|
+
finalOutcomeConflicts += 1;
|
|
939
|
+
}
|
|
940
|
+
increment(terminalOutcomes, resolved);
|
|
941
|
+
}
|
|
827
942
|
let capturesIndexed = 0;
|
|
828
943
|
let truncatedCaptures = 0;
|
|
829
944
|
let writeFailures = 0;
|
|
@@ -920,6 +1035,13 @@ export async function analyzeProxyLogs(options) {
|
|
|
920
1035
|
unsupportedLifecycleLines,
|
|
921
1036
|
lifecycleSequenceGaps,
|
|
922
1037
|
lifecycleSequenceDuplicates,
|
|
1038
|
+
conflictingLifecycleDuplicates,
|
|
1039
|
+
duplicateAttempts,
|
|
1040
|
+
finalOutcomeConflicts,
|
|
1041
|
+
acceptedWithoutFinal: [...accepted].filter((id) => !finalRequests.has(id))
|
|
1042
|
+
.length,
|
|
1043
|
+
terminalWithoutFinal: [...terminal].filter((id) => !finalRequests.has(id))
|
|
1044
|
+
.length,
|
|
923
1045
|
streams: Object.fromEntries(Object.entries(observedRanges).map(([stream, range]) => [
|
|
924
1046
|
stream,
|
|
925
1047
|
{
|
|
@@ -972,6 +1094,9 @@ export async function analyzeProxyLogs(options) {
|
|
|
972
1094
|
latencyMs: {
|
|
973
1095
|
headers: summarizeLatency(headersLatency),
|
|
974
1096
|
firstChunk: summarizeLatency(firstChunkLatency),
|
|
1097
|
+
firstUsefulOutput: summarizeLatency([...finalRequests.values()].flatMap((record) => record.firstUsefulOutputMs === null
|
|
1098
|
+
? []
|
|
1099
|
+
: [record.firstUsefulOutputMs])),
|
|
975
1100
|
terminal: summarizeLatency(terminalLatency),
|
|
976
1101
|
finalRequest: summarizeLatency(finalSummary.finalRequestLatency),
|
|
977
1102
|
attempt: summarizeLatency(attemptLatency),
|
|
@@ -4,7 +4,7 @@ export declare function hashProxyLifecycleSessionId(sessionId: string | undefine
|
|
|
4
4
|
export declare function configureProxyLifecycleLogger(options: ProxyLifecycleLoggerOptions): void;
|
|
5
5
|
/** Enqueue fixed-size lifecycle metadata without awaiting filesystem work. */
|
|
6
6
|
export declare function logProxyLifecycleEvent(input: ProxyLifecycleEventInput): void;
|
|
7
|
-
export declare function flushProxyLifecycleEvents(): Promise<void>;
|
|
7
|
+
export declare function flushProxyLifecycleEvents(timeoutMs?: number): Promise<void>;
|
|
8
8
|
export declare function getProxyLifecycleLoggerSnapshot(): ProxyLifecycleLoggerSnapshot;
|
|
9
9
|
export declare function resetProxyLifecycleLoggerForTests(): void;
|
|
10
10
|
/** Isolated failure injection for lifecycle durability tests. */
|
|
@@ -32,6 +32,8 @@ let invalidDrops = 0;
|
|
|
32
32
|
let writeDrops = 0;
|
|
33
33
|
let writeFailures = 0;
|
|
34
34
|
let writeRetries = 0;
|
|
35
|
+
let writeTimeouts = 0;
|
|
36
|
+
let unconfirmedWrites = 0;
|
|
35
37
|
let inFlight = 0;
|
|
36
38
|
let queue = [];
|
|
37
39
|
let flushTimer;
|
|
@@ -157,15 +159,43 @@ async function flushBatch() {
|
|
|
157
159
|
let retryDelayMs = 0;
|
|
158
160
|
for (const [path, items] of byPath) {
|
|
159
161
|
const lines = items.map((item) => `${JSON.stringify(item.record)}\n`);
|
|
162
|
+
// A timeout does not cancel appendFile. Keep ownership of the original
|
|
163
|
+
// operation until it settles; retrying it while it is still running can
|
|
164
|
+
// append the same batch twice. The request path remains non-blocking.
|
|
165
|
+
const timeout = setTimeout(() => {
|
|
166
|
+
writeTimeouts += 1;
|
|
167
|
+
}, LIFECYCLE_APPEND_TIMEOUT_MS);
|
|
168
|
+
timeout.unref?.();
|
|
160
169
|
try {
|
|
161
170
|
// This best-effort telemetry sink intentionally avoids fsync so request
|
|
162
171
|
// throughput is not coupled to storage latency. Loss is surfaced by
|
|
163
172
|
// writeDrops/writeFailures rather than delaying proxy responses.
|
|
164
|
-
await
|
|
173
|
+
await appendLifecycleFile(path, lines.join(""), { mode: 0o600 });
|
|
165
174
|
written += lines.length;
|
|
166
175
|
}
|
|
167
176
|
catch (error) {
|
|
168
177
|
writeFailures += 1;
|
|
178
|
+
// These errors prevent opening the destination. Other failures (for
|
|
179
|
+
// example ENOSPC/EIO) can follow a partial append. Replaying those is
|
|
180
|
+
// unsafe; expose uncertainty instead of claiming either loss or success.
|
|
181
|
+
const code = error?.code;
|
|
182
|
+
const definitelyNotWritten = new Set([
|
|
183
|
+
"ENOENT",
|
|
184
|
+
"EACCES",
|
|
185
|
+
"EPERM",
|
|
186
|
+
"EROFS",
|
|
187
|
+
"EMFILE",
|
|
188
|
+
"ENFILE",
|
|
189
|
+
]).has(code ?? "");
|
|
190
|
+
if (!definitelyNotWritten) {
|
|
191
|
+
unconfirmedWrites += items.length;
|
|
192
|
+
logger.warn("[proxy] lifecycle metadata append outcome is uncertain", {
|
|
193
|
+
path,
|
|
194
|
+
records: items.length,
|
|
195
|
+
code,
|
|
196
|
+
});
|
|
197
|
+
continue;
|
|
198
|
+
}
|
|
169
199
|
const retryable = items.filter((item) => item.writeRetries < maxWriteRetries);
|
|
170
200
|
const exhausted = items.length - retryable.length;
|
|
171
201
|
if (retryable.length > 0) {
|
|
@@ -189,6 +219,9 @@ async function flushBatch() {
|
|
|
189
219
|
error: error instanceof Error ? error.message : String(error),
|
|
190
220
|
});
|
|
191
221
|
}
|
|
222
|
+
finally {
|
|
223
|
+
clearTimeout(timeout);
|
|
224
|
+
}
|
|
192
225
|
}
|
|
193
226
|
if (retries.length > 0) {
|
|
194
227
|
// Keep retried records ahead of newly admitted records. This preserves
|
|
@@ -293,6 +326,16 @@ export function logProxyLifecycleEvent(input) {
|
|
|
293
326
|
...(sessionHash !== undefined ? { sessionHash } : {}),
|
|
294
327
|
...(requestBytes !== undefined ? { requestBytes } : {}),
|
|
295
328
|
...(responseStatus !== undefined ? { responseStatus } : {}),
|
|
329
|
+
...(input.finalStatus !== undefined
|
|
330
|
+
? { finalStatus: nonNegativeInteger(input.finalStatus) }
|
|
331
|
+
: {}),
|
|
332
|
+
...(input.telemetryStatus
|
|
333
|
+
? { telemetryStatus: input.telemetryStatus }
|
|
334
|
+
: {}),
|
|
335
|
+
...(input.transportOutcome
|
|
336
|
+
? { transportOutcome: input.transportOutcome }
|
|
337
|
+
: {}),
|
|
338
|
+
...(input.outcomeSource ? { outcomeSource: input.outcomeSource } : {}),
|
|
296
339
|
...(observedBodyBytes !== undefined ? { observedBodyBytes } : {}),
|
|
297
340
|
...(responseChunks !== undefined ? { responseChunks } : {}),
|
|
298
341
|
...(elapsedMs !== undefined
|
|
@@ -316,16 +359,15 @@ export function logProxyLifecycleEvent(input) {
|
|
|
316
359
|
invalidDrops += 1;
|
|
317
360
|
}
|
|
318
361
|
}
|
|
319
|
-
export async function flushProxyLifecycleEvents() {
|
|
362
|
+
export async function flushProxyLifecycleEvents(timeoutMs = 5_000) {
|
|
320
363
|
clearScheduledFlush();
|
|
364
|
+
const deadline = performance.now() + timeoutMs;
|
|
321
365
|
while (queue.length > 0 || flushInFlight) {
|
|
322
|
-
|
|
323
|
-
await flushInFlight;
|
|
324
|
-
}
|
|
325
|
-
else {
|
|
326
|
-
await startFlush();
|
|
327
|
-
}
|
|
366
|
+
await withTimeout(flushInFlight ?? startFlush(), Math.max(1, deadline - performance.now()), "Timed out flushing proxy lifecycle metadata; writes remain pending");
|
|
328
367
|
clearScheduledFlush();
|
|
368
|
+
if (performance.now() >= deadline && (queue.length > 0 || flushInFlight)) {
|
|
369
|
+
throw new Error("Proxy lifecycle flush deadline exceeded; writes remain pending");
|
|
370
|
+
}
|
|
329
371
|
}
|
|
330
372
|
}
|
|
331
373
|
export function getProxyLifecycleLoggerSnapshot() {
|
|
@@ -343,6 +385,8 @@ export function getProxyLifecycleLoggerSnapshot() {
|
|
|
343
385
|
writeDrops,
|
|
344
386
|
writeFailures,
|
|
345
387
|
writeRetries,
|
|
388
|
+
writeTimeouts,
|
|
389
|
+
unconfirmedWrites,
|
|
346
390
|
pending: queue.length,
|
|
347
391
|
inFlight,
|
|
348
392
|
flushing: flushInFlight !== undefined,
|
|
@@ -368,6 +412,8 @@ export function resetProxyLifecycleLoggerForTests() {
|
|
|
368
412
|
writeDrops = 0;
|
|
369
413
|
writeFailures = 0;
|
|
370
414
|
writeRetries = 0;
|
|
415
|
+
writeTimeouts = 0;
|
|
416
|
+
unconfirmedWrites = 0;
|
|
371
417
|
inFlight = 0;
|
|
372
418
|
queue = [];
|
|
373
419
|
flushInFlight = undefined;
|
|
@@ -5,7 +5,8 @@
|
|
|
5
5
|
* when a LoggerProvider is configured via OpenTelemetry instrumentation.
|
|
6
6
|
* Useful for debugging and auditing proxy traffic.
|
|
7
7
|
*/
|
|
8
|
-
import type { ProxyBodyCaptureEntry, RequestAttemptLogEntry, RequestLogEntry } from "../types/index.js";
|
|
8
|
+
import type { ProxyBodyCaptureEntry, RequestAttemptLogEntry, RequestLogEntry, ProxyRequestLoggerSnapshot } from "../types/index.js";
|
|
9
|
+
export declare function getRequestLoggerSnapshot(): ProxyRequestLoggerSnapshot;
|
|
9
10
|
/** Wait, up to a bounded deadline, for admitted request/body writes to settle. */
|
|
10
11
|
export declare function flushRequestLogs(timeoutMs?: number): Promise<void>;
|
|
11
12
|
export declare function initRequestLogger(enabled?: boolean, customLogsDir?: string): void;
|
|
@@ -17,11 +17,78 @@ import { isBorrowedRequest } from "./shareContext.js";
|
|
|
17
17
|
import { OtelBridge } from "../observability/otelBridge.js";
|
|
18
18
|
import { SeverityNumber } from "@opentelemetry/api-logs";
|
|
19
19
|
import { configureProxyLifecycleLogger } from "./proxyLifecycle.js";
|
|
20
|
+
import { notifyProxyFinalLog } from "./proxyActivity.js";
|
|
20
21
|
import { withTimeout } from "../utils/async/withTimeout.js";
|
|
21
22
|
let logDir = null;
|
|
22
23
|
let logEnabled = false;
|
|
23
24
|
const pendingLogOperations = new Set();
|
|
24
25
|
const REQUEST_LOG_IO_TIMEOUT_MS = 5_000;
|
|
26
|
+
const MAX_PENDING_METADATA_RECORDS = 4_096;
|
|
27
|
+
const appendChains = new Map();
|
|
28
|
+
let appendMetadataFile = writeFile;
|
|
29
|
+
const createSinkSnapshot = () => ({
|
|
30
|
+
attempted: 0,
|
|
31
|
+
written: 0,
|
|
32
|
+
inFlight: 0,
|
|
33
|
+
pending: 0,
|
|
34
|
+
dropped: 0,
|
|
35
|
+
writeTimeouts: 0,
|
|
36
|
+
unconfirmedWrites: 0,
|
|
37
|
+
});
|
|
38
|
+
const metadataSinks = {
|
|
39
|
+
requests: createSinkSnapshot(),
|
|
40
|
+
attempts: createSinkSnapshot(),
|
|
41
|
+
debug: createSinkSnapshot(),
|
|
42
|
+
};
|
|
43
|
+
export function getRequestLoggerSnapshot() {
|
|
44
|
+
return {
|
|
45
|
+
enabled: logEnabled,
|
|
46
|
+
requests: { ...metadataSinks.requests },
|
|
47
|
+
attempts: { ...metadataSinks.attempts },
|
|
48
|
+
debug: { ...metadataSinks.debug },
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
async function appendMetadataRecord(file, line, kind) {
|
|
52
|
+
const sink = metadataSinks[kind];
|
|
53
|
+
sink.attempted += 1;
|
|
54
|
+
if (sink.pending + sink.inFlight >= MAX_PENDING_METADATA_RECORDS) {
|
|
55
|
+
sink.dropped += 1;
|
|
56
|
+
return;
|
|
57
|
+
}
|
|
58
|
+
sink.pending += 1;
|
|
59
|
+
// writeFile may perform multiple append syscalls for a large record. Order
|
|
60
|
+
// them per destination so concurrent records cannot interleave in this worker.
|
|
61
|
+
const operation = trackLogOperation((appendChains.get(file) ?? Promise.resolve()).then(async () => {
|
|
62
|
+
sink.pending -= 1;
|
|
63
|
+
sink.inFlight += 1;
|
|
64
|
+
const timer = setTimeout(() => {
|
|
65
|
+
sink.writeTimeouts += 1;
|
|
66
|
+
}, REQUEST_LOG_IO_TIMEOUT_MS);
|
|
67
|
+
timer.unref?.();
|
|
68
|
+
try {
|
|
69
|
+
await appendMetadataFile(file, line, { mode: 0o600, flag: "a" });
|
|
70
|
+
sink.written += 1;
|
|
71
|
+
}
|
|
72
|
+
catch (error) {
|
|
73
|
+
// A failed append may have written a prefix; never replay it.
|
|
74
|
+
sink.unconfirmedWrites += 1;
|
|
75
|
+
sink.lastErrorCode =
|
|
76
|
+
error?.code ?? "UNKNOWN";
|
|
77
|
+
}
|
|
78
|
+
finally {
|
|
79
|
+
clearTimeout(timer);
|
|
80
|
+
sink.inFlight -= 1;
|
|
81
|
+
}
|
|
82
|
+
}));
|
|
83
|
+
appendChains.set(file, operation);
|
|
84
|
+
void operation.then(() => {
|
|
85
|
+
if (appendChains.get(file) === operation) {
|
|
86
|
+
appendChains.delete(file);
|
|
87
|
+
}
|
|
88
|
+
});
|
|
89
|
+
// Bound the caller's wait, not the lifetime/ownership of the underlying write.
|
|
90
|
+
await withTimeout(operation, REQUEST_LOG_IO_TIMEOUT_MS, "Proxy metadata write remains pending").catch(() => undefined);
|
|
91
|
+
}
|
|
25
92
|
function trackLogOperation(operation) {
|
|
26
93
|
pendingLogOperations.add(operation);
|
|
27
94
|
void operation.then(() => pendingLogOperations.delete(operation), () => pendingLogOperations.delete(operation));
|
|
@@ -33,15 +100,7 @@ export async function flushRequestLogs(timeoutMs = REQUEST_LOG_IO_TIMEOUT_MS) {
|
|
|
33
100
|
while (pendingLogOperations.size > 0) {
|
|
34
101
|
const admitted = [...pendingLogOperations];
|
|
35
102
|
const remainingMs = Math.max(1, deadline - Date.now());
|
|
36
|
-
|
|
37
|
-
await withTimeout(Promise.allSettled(admitted), remainingMs, `Timed out flushing ${admitted.length} proxy request log operation(s)`);
|
|
38
|
-
}
|
|
39
|
-
catch (error) {
|
|
40
|
-
for (const operation of admitted) {
|
|
41
|
-
pendingLogOperations.delete(operation);
|
|
42
|
-
}
|
|
43
|
-
throw error;
|
|
44
|
-
}
|
|
103
|
+
await withTimeout(Promise.allSettled(admitted), remainingMs, `Timed out flushing ${admitted.length} proxy request log operation(s)`);
|
|
45
104
|
if (Date.now() >= deadline && pendingLogOperations.size > 0) {
|
|
46
105
|
const remaining = pendingLogOperations.size;
|
|
47
106
|
throw new Error(`Timed out flushing ${remaining} proxy request log operation(s)`);
|
|
@@ -52,6 +111,12 @@ export async function flushRequestLogs(timeoutMs = REQUEST_LOG_IO_TIMEOUT_MS) {
|
|
|
52
111
|
export const __requestLoggerTestHooks = {
|
|
53
112
|
pendingOperationCount: () => pendingLogOperations.size,
|
|
54
113
|
trackLogOperation,
|
|
114
|
+
setAppendFileForTests: (writer) => {
|
|
115
|
+
appendMetadataFile = writer;
|
|
116
|
+
},
|
|
117
|
+
restoreAppendFileForTests: () => {
|
|
118
|
+
appendMetadataFile = writeFile;
|
|
119
|
+
},
|
|
55
120
|
};
|
|
56
121
|
/**
|
|
57
122
|
* Lazily-resolved LoggerProvider from OTel instrumentation.
|
|
@@ -104,6 +169,15 @@ export function initRequestLogger(enabled = true, customLogsDir) {
|
|
|
104
169
|
}
|
|
105
170
|
}
|
|
106
171
|
export async function logRequest(entry) {
|
|
172
|
+
entry.terminalOutcome ??=
|
|
173
|
+
entry.errorType === "client_cancelled" || entry.responseStatus === 499
|
|
174
|
+
? "client_cancelled"
|
|
175
|
+
: entry.errorType?.includes("stream")
|
|
176
|
+
? "stream_error"
|
|
177
|
+
: entry.responseStatus >= 400 || entry.errorType
|
|
178
|
+
? "handler_error"
|
|
179
|
+
: "completed";
|
|
180
|
+
notifyProxyFinalLog(entry);
|
|
107
181
|
if (!logEnabled || !logDir) {
|
|
108
182
|
return;
|
|
109
183
|
}
|
|
@@ -121,11 +195,7 @@ export async function logRequest(entry) {
|
|
|
121
195
|
const logFile = join(logDir, `proxy-${new Date().toISOString().split("T")[0]}.jsonl`);
|
|
122
196
|
const line = JSON.stringify(entry) + "\n";
|
|
123
197
|
try {
|
|
124
|
-
await
|
|
125
|
-
mode: 0o600,
|
|
126
|
-
flag: "a",
|
|
127
|
-
signal: AbortSignal.timeout(REQUEST_LOG_IO_TIMEOUT_MS),
|
|
128
|
-
}));
|
|
198
|
+
await appendMetadataRecord(logFile, line, "requests");
|
|
129
199
|
}
|
|
130
200
|
catch {
|
|
131
201
|
// Non-fatal — don't crash proxy for logging failures
|
|
@@ -153,11 +223,7 @@ export async function logRequestAttempt(entry) {
|
|
|
153
223
|
const logFile = join(logDir, `proxy-attempts-${new Date().toISOString().split("T")[0]}.jsonl`);
|
|
154
224
|
const line = JSON.stringify(entry) + "\n";
|
|
155
225
|
try {
|
|
156
|
-
await
|
|
157
|
-
mode: 0o600,
|
|
158
|
-
flag: "a",
|
|
159
|
-
signal: AbortSignal.timeout(REQUEST_LOG_IO_TIMEOUT_MS),
|
|
160
|
-
}));
|
|
226
|
+
await appendMetadataRecord(logFile, line, "attempts");
|
|
161
227
|
}
|
|
162
228
|
catch {
|
|
163
229
|
// Non-fatal — don't crash proxy for logging failures
|
|
@@ -615,11 +681,7 @@ export async function logBodyCapture(entry) {
|
|
|
615
681
|
indexEntry.spanId = traceCtx.spanId;
|
|
616
682
|
}
|
|
617
683
|
try {
|
|
618
|
-
await
|
|
619
|
-
mode: 0o600,
|
|
620
|
-
flag: "a",
|
|
621
|
-
signal: AbortSignal.timeout(REQUEST_LOG_IO_TIMEOUT_MS),
|
|
622
|
-
}));
|
|
684
|
+
await appendMetadataRecord(logFile, JSON.stringify(indexEntry) + "\n", "debug");
|
|
623
685
|
}
|
|
624
686
|
catch {
|
|
625
687
|
// Non-fatal
|
|
@@ -691,11 +753,7 @@ export async function logStreamError(entry) {
|
|
|
691
753
|
logEntry.spanId = traceCtx.spanId;
|
|
692
754
|
}
|
|
693
755
|
try {
|
|
694
|
-
await
|
|
695
|
-
mode: 0o600,
|
|
696
|
-
flag: "a",
|
|
697
|
-
signal: AbortSignal.timeout(REQUEST_LOG_IO_TIMEOUT_MS),
|
|
698
|
-
}));
|
|
756
|
+
await appendMetadataRecord(logFile, JSON.stringify(logEntry) + "\n", "requests");
|
|
699
757
|
}
|
|
700
758
|
catch {
|
|
701
759
|
// Non-fatal — don't crash proxy for logging failures
|