@juspay/neurolink 9.41.0 → 9.42.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/README.md +7 -1
- package/dist/auth/anthropicOAuth.d.ts +18 -3
- package/dist/auth/anthropicOAuth.js +137 -4
- package/dist/auth/providers/firebase.js +5 -1
- package/dist/auth/providers/jwt.js +5 -1
- package/dist/auth/providers/workos.js +5 -1
- package/dist/auth/sessionManager.d.ts +1 -1
- package/dist/auth/sessionManager.js +58 -27
- package/dist/browser/neurolink.min.js +337 -318
- package/dist/cli/commands/mcp.js +3 -0
- package/dist/cli/commands/proxy.d.ts +2 -1
- package/dist/cli/commands/proxy.js +279 -16
- package/dist/cli/commands/task.js +3 -0
- package/dist/cli/factories/commandFactory.d.ts +2 -0
- package/dist/cli/factories/commandFactory.js +38 -0
- package/dist/cli/parser.js +4 -3
- package/dist/client/aiSdkAdapter.js +3 -0
- package/dist/client/streamingClient.js +30 -10
- package/dist/core/modules/GenerationHandler.js +3 -2
- package/dist/core/redisConversationMemoryManager.js +7 -3
- package/dist/evaluation/BatchEvaluator.js +4 -1
- package/dist/evaluation/hooks/observabilityHooks.js +5 -3
- package/dist/evaluation/pipeline/evaluationPipeline.d.ts +3 -2
- package/dist/evaluation/pipeline/evaluationPipeline.js +20 -8
- package/dist/evaluation/pipeline/strategies/batchStrategy.js +6 -3
- package/dist/evaluation/pipeline/strategies/samplingStrategy.js +18 -10
- package/dist/lib/auth/anthropicOAuth.d.ts +18 -3
- package/dist/lib/auth/anthropicOAuth.js +137 -4
- package/dist/lib/auth/providers/firebase.js +5 -1
- package/dist/lib/auth/providers/jwt.js +5 -1
- package/dist/lib/auth/providers/workos.js +5 -1
- package/dist/lib/auth/sessionManager.d.ts +1 -1
- package/dist/lib/auth/sessionManager.js +58 -27
- package/dist/lib/client/aiSdkAdapter.js +3 -0
- package/dist/lib/client/streamingClient.js +30 -10
- package/dist/lib/core/modules/GenerationHandler.js +3 -2
- package/dist/lib/core/redisConversationMemoryManager.js +7 -3
- package/dist/lib/evaluation/BatchEvaluator.js +4 -1
- package/dist/lib/evaluation/hooks/observabilityHooks.js +5 -3
- package/dist/lib/evaluation/pipeline/evaluationPipeline.d.ts +3 -2
- package/dist/lib/evaluation/pipeline/evaluationPipeline.js +20 -8
- package/dist/lib/evaluation/pipeline/strategies/batchStrategy.js +6 -3
- package/dist/lib/evaluation/pipeline/strategies/samplingStrategy.js +18 -10
- package/dist/lib/neurolink.d.ts +3 -2
- package/dist/lib/neurolink.js +260 -494
- package/dist/lib/observability/otelBridge.d.ts +2 -2
- package/dist/lib/observability/otelBridge.js +12 -3
- package/dist/lib/providers/amazonBedrock.js +2 -4
- package/dist/lib/providers/anthropic.d.ts +9 -5
- package/dist/lib/providers/anthropic.js +19 -14
- package/dist/lib/providers/anthropicBaseProvider.d.ts +3 -3
- package/dist/lib/providers/anthropicBaseProvider.js +5 -4
- package/dist/lib/providers/azureOpenai.d.ts +1 -1
- package/dist/lib/providers/azureOpenai.js +5 -4
- package/dist/lib/providers/googleAiStudio.js +30 -1
- package/dist/lib/providers/googleVertex.js +28 -6
- package/dist/lib/providers/huggingFace.d.ts +3 -3
- package/dist/lib/providers/huggingFace.js +6 -8
- package/dist/lib/providers/litellm.js +41 -29
- package/dist/lib/providers/mistral.js +2 -1
- package/dist/lib/providers/ollama.js +80 -23
- package/dist/lib/providers/openAI.js +3 -2
- package/dist/lib/providers/openRouter.js +2 -1
- package/dist/lib/providers/openaiCompatible.d.ts +4 -4
- package/dist/lib/providers/openaiCompatible.js +4 -4
- package/dist/lib/proxy/claudeFormat.d.ts +3 -2
- package/dist/lib/proxy/claudeFormat.js +25 -20
- package/dist/lib/proxy/cloaking/plugins/sessionIdentity.d.ts +2 -6
- package/dist/lib/proxy/cloaking/plugins/sessionIdentity.js +9 -33
- package/dist/lib/proxy/modelRouter.js +3 -0
- package/dist/lib/proxy/oauthFetch.d.ts +1 -1
- package/dist/lib/proxy/oauthFetch.js +65 -72
- package/dist/lib/proxy/proxyConfig.js +44 -24
- package/dist/lib/proxy/proxyEnv.d.ts +19 -0
- package/dist/lib/proxy/proxyEnv.js +73 -0
- package/dist/lib/proxy/proxyFetch.js +50 -4
- package/dist/lib/proxy/proxyTracer.d.ts +133 -0
- package/dist/lib/proxy/proxyTracer.js +645 -0
- package/dist/lib/proxy/rawStreamCapture.d.ts +10 -0
- package/dist/lib/proxy/rawStreamCapture.js +83 -0
- package/dist/lib/proxy/requestLogger.d.ts +32 -5
- package/dist/lib/proxy/requestLogger.js +406 -37
- package/dist/lib/proxy/sseInterceptor.d.ts +97 -0
- package/dist/lib/proxy/sseInterceptor.js +402 -0
- package/dist/lib/proxy/usageStats.d.ts +4 -3
- package/dist/lib/proxy/usageStats.js +25 -12
- package/dist/lib/rag/chunkers/MarkdownChunker.js +13 -5
- package/dist/lib/rag/chunking/markdownChunker.js +15 -6
- package/dist/lib/server/routes/claudeProxyRoutes.d.ts +7 -2
- package/dist/lib/server/routes/claudeProxyRoutes.js +1737 -508
- package/dist/lib/services/server/ai/observability/instrumentation.d.ts +7 -1
- package/dist/lib/services/server/ai/observability/instrumentation.js +240 -40
- package/dist/lib/tasks/backends/bullmqBackend.d.ts +1 -0
- package/dist/lib/tasks/backends/bullmqBackend.js +14 -7
- package/dist/lib/tasks/store/redisTaskStore.d.ts +1 -0
- package/dist/lib/tasks/store/redisTaskStore.js +34 -26
- package/dist/lib/tasks/taskManager.d.ts +3 -0
- package/dist/lib/tasks/taskManager.js +63 -30
- package/dist/lib/telemetry/index.d.ts +2 -1
- package/dist/lib/telemetry/index.js +2 -1
- package/dist/lib/telemetry/telemetryService.d.ts +3 -0
- package/dist/lib/telemetry/telemetryService.js +65 -5
- package/dist/lib/types/cli.d.ts +10 -0
- package/dist/lib/types/proxyTypes.d.ts +37 -5
- package/dist/lib/types/streamTypes.d.ts +25 -3
- package/dist/lib/utils/messageBuilder.js +3 -2
- package/dist/lib/utils/providerHealth.d.ts +18 -0
- package/dist/lib/utils/providerHealth.js +240 -9
- package/dist/lib/utils/providerUtils.js +14 -8
- package/dist/lib/utils/toolChoice.d.ts +4 -0
- package/dist/lib/utils/toolChoice.js +7 -0
- package/dist/neurolink.d.ts +3 -2
- package/dist/neurolink.js +260 -494
- package/dist/observability/otelBridge.d.ts +2 -2
- package/dist/observability/otelBridge.js +12 -3
- package/dist/providers/amazonBedrock.js +2 -4
- package/dist/providers/anthropic.d.ts +9 -5
- package/dist/providers/anthropic.js +19 -14
- package/dist/providers/anthropicBaseProvider.d.ts +3 -3
- package/dist/providers/anthropicBaseProvider.js +5 -4
- package/dist/providers/azureOpenai.d.ts +1 -1
- package/dist/providers/azureOpenai.js +5 -4
- package/dist/providers/googleAiStudio.js +30 -1
- package/dist/providers/googleVertex.js +28 -6
- package/dist/providers/huggingFace.d.ts +3 -3
- package/dist/providers/huggingFace.js +6 -7
- package/dist/providers/litellm.js +41 -29
- package/dist/providers/mistral.js +2 -1
- package/dist/providers/ollama.js +80 -23
- package/dist/providers/openAI.js +3 -2
- package/dist/providers/openRouter.js +2 -1
- package/dist/providers/openaiCompatible.d.ts +4 -4
- package/dist/providers/openaiCompatible.js +4 -3
- package/dist/proxy/claudeFormat.d.ts +3 -2
- package/dist/proxy/claudeFormat.js +25 -20
- package/dist/proxy/cloaking/plugins/sessionIdentity.d.ts +2 -6
- package/dist/proxy/cloaking/plugins/sessionIdentity.js +9 -33
- package/dist/proxy/modelRouter.js +3 -0
- package/dist/proxy/oauthFetch.d.ts +1 -1
- package/dist/proxy/oauthFetch.js +65 -72
- package/dist/proxy/proxyConfig.js +44 -24
- package/dist/proxy/proxyEnv.d.ts +19 -0
- package/dist/proxy/proxyEnv.js +72 -0
- package/dist/proxy/proxyFetch.js +50 -4
- package/dist/proxy/proxyTracer.d.ts +133 -0
- package/dist/proxy/proxyTracer.js +644 -0
- package/dist/proxy/rawStreamCapture.d.ts +10 -0
- package/dist/proxy/rawStreamCapture.js +82 -0
- package/dist/proxy/requestLogger.d.ts +32 -5
- package/dist/proxy/requestLogger.js +406 -37
- package/dist/proxy/sseInterceptor.d.ts +97 -0
- package/dist/proxy/sseInterceptor.js +401 -0
- package/dist/proxy/usageStats.d.ts +4 -3
- package/dist/proxy/usageStats.js +25 -12
- package/dist/rag/chunkers/MarkdownChunker.js +13 -5
- package/dist/rag/chunking/markdownChunker.js +15 -6
- package/dist/server/routes/claudeProxyRoutes.d.ts +7 -2
- package/dist/server/routes/claudeProxyRoutes.js +1737 -508
- package/dist/services/server/ai/observability/instrumentation.d.ts +7 -1
- package/dist/services/server/ai/observability/instrumentation.js +240 -40
- package/dist/tasks/backends/bullmqBackend.d.ts +1 -0
- package/dist/tasks/backends/bullmqBackend.js +14 -7
- package/dist/tasks/store/redisTaskStore.d.ts +1 -0
- package/dist/tasks/store/redisTaskStore.js +34 -26
- package/dist/tasks/taskManager.d.ts +3 -0
- package/dist/tasks/taskManager.js +63 -30
- package/dist/telemetry/index.d.ts +2 -1
- package/dist/telemetry/index.js +2 -1
- package/dist/telemetry/telemetryService.d.ts +3 -0
- package/dist/telemetry/telemetryService.js +65 -5
- package/dist/types/cli.d.ts +10 -0
- package/dist/types/proxyTypes.d.ts +37 -5
- package/dist/types/streamTypes.d.ts +25 -3
- package/dist/utils/messageBuilder.js +3 -2
- package/dist/utils/providerHealth.d.ts +18 -0
- package/dist/utils/providerHealth.js +240 -9
- package/dist/utils/providerUtils.js +14 -8
- package/dist/utils/toolChoice.d.ts +4 -0
- package/dist/utils/toolChoice.js +6 -0
- package/docs/assets/dashboards/neurolink-proxy-observability-dashboard.json +6609 -0
- package/docs/changelog.md +252 -0
- package/package.json +17 -1
- package/scripts/observability/check-proxy-telemetry.mjs +235 -0
- package/scripts/observability/docker-compose.proxy-observability.yaml +55 -0
- package/scripts/observability/import-openobserve-dashboard.mjs +240 -0
- package/scripts/observability/manage-local-openobserve.sh +184 -0
- package/scripts/observability/otel-collector.proxy-observability.yaml +78 -0
- package/scripts/observability/proxy-observability.env.example +23 -0
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
/** Maximum bytes to capture before stopping accumulation (1 MB). */
|
|
2
|
+
const MAX_CAPTURE_BYTES = 1024 * 1024;
|
|
3
|
+
const TRUNCATION_MARKER = "\n...[TRUNCATED]";
|
|
4
|
+
export function createRawStreamCapture() {
|
|
5
|
+
const decoder = new TextDecoder();
|
|
6
|
+
const chunks = [];
|
|
7
|
+
let totalBytes = 0;
|
|
8
|
+
let capturedBytes = 0;
|
|
9
|
+
let truncated = false;
|
|
10
|
+
let resolved = false;
|
|
11
|
+
let resolveCapture;
|
|
12
|
+
const capture = new Promise((resolve) => {
|
|
13
|
+
resolveCapture = resolve;
|
|
14
|
+
});
|
|
15
|
+
function settle() {
|
|
16
|
+
if (resolved) {
|
|
17
|
+
return;
|
|
18
|
+
}
|
|
19
|
+
resolved = true;
|
|
20
|
+
const finalChunk = decoder.decode();
|
|
21
|
+
if (finalChunk) {
|
|
22
|
+
if (capturedBytes < MAX_CAPTURE_BYTES) {
|
|
23
|
+
const remainingBytes = MAX_CAPTURE_BYTES - capturedBytes;
|
|
24
|
+
chunks.push(finalChunk.slice(0, remainingBytes));
|
|
25
|
+
capturedBytes += Math.min(finalChunk.length, remainingBytes);
|
|
26
|
+
}
|
|
27
|
+
else if (!truncated) {
|
|
28
|
+
chunks.push(TRUNCATION_MARKER);
|
|
29
|
+
truncated = true;
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
resolveCapture({
|
|
33
|
+
totalBytes,
|
|
34
|
+
text: chunks.join(""),
|
|
35
|
+
truncated,
|
|
36
|
+
});
|
|
37
|
+
}
|
|
38
|
+
const transform = new TransformStream({
|
|
39
|
+
transform(chunk, controller) {
|
|
40
|
+
controller.enqueue(chunk);
|
|
41
|
+
totalBytes += chunk.byteLength;
|
|
42
|
+
if (capturedBytes < MAX_CAPTURE_BYTES) {
|
|
43
|
+
const decoded = decoder.decode(chunk, { stream: true });
|
|
44
|
+
const remainingBytes = MAX_CAPTURE_BYTES - capturedBytes;
|
|
45
|
+
const slice = decoded.slice(0, remainingBytes);
|
|
46
|
+
chunks.push(slice);
|
|
47
|
+
capturedBytes += Math.min(decoded.length, remainingBytes);
|
|
48
|
+
if (decoded.length > remainingBytes && !truncated) {
|
|
49
|
+
chunks.push(TRUNCATION_MARKER);
|
|
50
|
+
truncated = true;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
else if (!truncated) {
|
|
54
|
+
chunks.push(TRUNCATION_MARKER);
|
|
55
|
+
truncated = true;
|
|
56
|
+
}
|
|
57
|
+
},
|
|
58
|
+
flush() {
|
|
59
|
+
settle();
|
|
60
|
+
},
|
|
61
|
+
});
|
|
62
|
+
const innerWriter = transform.writable.getWriter();
|
|
63
|
+
const writable = new WritableStream({
|
|
64
|
+
write(chunk) {
|
|
65
|
+
return innerWriter.write(chunk);
|
|
66
|
+
},
|
|
67
|
+
close() {
|
|
68
|
+
return innerWriter.close();
|
|
69
|
+
},
|
|
70
|
+
abort(reason) {
|
|
71
|
+
settle();
|
|
72
|
+
return innerWriter.abort(reason);
|
|
73
|
+
},
|
|
74
|
+
});
|
|
75
|
+
return {
|
|
76
|
+
stream: {
|
|
77
|
+
readable: transform.readable,
|
|
78
|
+
writable,
|
|
79
|
+
},
|
|
80
|
+
capture,
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
//# sourceMappingURL=rawStreamCapture.js.map
|
|
@@ -1,18 +1,44 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Proxy Request Logger
|
|
3
3
|
* Logs proxy request/response metadata to a rotating log file.
|
|
4
|
+
* Also emits OTLP log records to OpenObserve (or any OTLP-compatible backend)
|
|
5
|
+
* when a LoggerProvider is configured via OpenTelemetry instrumentation.
|
|
4
6
|
* Useful for debugging and auditing proxy traffic.
|
|
5
7
|
*/
|
|
6
|
-
import type { RequestLogEntry } from "../types/index.js";
|
|
8
|
+
import type { RequestAttemptLogEntry, RequestLogEntry } from "../types/index.js";
|
|
7
9
|
export declare function initRequestLogger(enabled?: boolean): void;
|
|
8
10
|
export declare function logRequest(entry: RequestLogEntry): Promise<void>;
|
|
11
|
+
/**
|
|
12
|
+
* Log an upstream attempt separately from the final request outcome.
|
|
13
|
+
* Attempt logs are local-only and must not pollute the final request summary
|
|
14
|
+
* or OTLP-derived dashboard panels.
|
|
15
|
+
*/
|
|
16
|
+
export declare function logRequestAttempt(entry: RequestAttemptLogEntry): Promise<void>;
|
|
9
17
|
export declare function getLogDir(): string | null;
|
|
18
|
+
type ProxyBodyCaptureEntry = {
|
|
19
|
+
timestamp: string;
|
|
20
|
+
requestId: string;
|
|
21
|
+
phase: string;
|
|
22
|
+
model: string;
|
|
23
|
+
stream: boolean;
|
|
24
|
+
headers?: Record<string, string>;
|
|
25
|
+
body?: unknown;
|
|
26
|
+
bodySize?: number;
|
|
27
|
+
contentType?: string;
|
|
28
|
+
responseStatus?: number;
|
|
29
|
+
durationMs?: number;
|
|
30
|
+
account?: string;
|
|
31
|
+
accountType?: string;
|
|
32
|
+
attempt?: number;
|
|
33
|
+
traceId?: string;
|
|
34
|
+
spanId?: string;
|
|
35
|
+
metadata?: Record<string, unknown>;
|
|
36
|
+
};
|
|
37
|
+
export declare function logBodyCapture(entry: ProxyBodyCaptureEntry): Promise<void>;
|
|
10
38
|
/**
|
|
11
39
|
* Log the FULL raw request and response for debugging.
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
* Sensitive headers and body fields are redacted before writing.
|
|
40
|
+
* Legacy helper kept for compatibility. New call sites should prefer
|
|
41
|
+
* logBodyCapture() so each phase can be indexed and persisted separately.
|
|
16
42
|
*/
|
|
17
43
|
export declare function logFullRequestResponse(entry: {
|
|
18
44
|
timestamp: string;
|
|
@@ -48,3 +74,4 @@ export declare function logStreamError(entry: {
|
|
|
48
74
|
* Non-fatal — proxy keeps working even if cleanup fails.
|
|
49
75
|
*/
|
|
50
76
|
export declare function cleanupLogs(maxAgeDays?: number, maxSizeMb?: number): void;
|
|
77
|
+
export {};
|
|
@@ -1,17 +1,34 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Proxy Request Logger
|
|
3
3
|
* Logs proxy request/response metadata to a rotating log file.
|
|
4
|
+
* Also emits OTLP log records to OpenObserve (or any OTLP-compatible backend)
|
|
5
|
+
* when a LoggerProvider is configured via OpenTelemetry instrumentation.
|
|
4
6
|
* Useful for debugging and auditing proxy traffic.
|
|
5
7
|
*/
|
|
6
8
|
import { join } from "path";
|
|
7
9
|
import { homedir } from "os";
|
|
8
10
|
import { logger } from "../utils/logger.js";
|
|
9
|
-
import { chmodSync, existsSync, mkdirSync, readdirSync, statSync, unlinkSync, } from "fs";
|
|
10
|
-
import { appendFile } from "fs/promises";
|
|
11
|
+
import { chmodSync, existsSync, mkdirSync, readdirSync, rmSync, statSync, unlinkSync, } from "fs";
|
|
12
|
+
import { appendFile, writeFile } from "fs/promises";
|
|
13
|
+
import { createHash } from "crypto";
|
|
14
|
+
import { promisify } from "util";
|
|
15
|
+
import { gzip as gzipCallback } from "zlib";
|
|
16
|
+
import { OtelBridge } from "../observability/otelBridge.js";
|
|
17
|
+
import { SeverityNumber } from "@opentelemetry/api-logs";
|
|
11
18
|
let logDir = null;
|
|
12
19
|
let logEnabled = false;
|
|
13
|
-
/**
|
|
14
|
-
|
|
20
|
+
/**
|
|
21
|
+
* Lazily-resolved LoggerProvider from OTel instrumentation.
|
|
22
|
+
* null = not resolved yet (will retry), LoggerProvider = resolved, false = permanently unavailable.
|
|
23
|
+
*/
|
|
24
|
+
let otelLoggerProvider = null;
|
|
25
|
+
/** Number of times we've tried to resolve the LoggerProvider. */
|
|
26
|
+
let otelResolveAttempts = 0;
|
|
27
|
+
/** Max number of resolve attempts before giving up. */
|
|
28
|
+
const MAX_RESOLVE_ATTEMPTS = 10;
|
|
29
|
+
/** Maximum body chunk size emitted to OTLP logs. */
|
|
30
|
+
const BODY_OTLP_CHUNK_SIZE = 16_000;
|
|
31
|
+
const gzip = promisify(gzipCallback);
|
|
15
32
|
/** Headers whose values must always be redacted. */
|
|
16
33
|
const SENSITIVE_HEADER_NAMES = new Set([
|
|
17
34
|
"authorization",
|
|
@@ -46,6 +63,17 @@ export async function logRequest(entry) {
|
|
|
46
63
|
if (!logEnabled || !logDir) {
|
|
47
64
|
return;
|
|
48
65
|
}
|
|
66
|
+
// Only use OtelBridge if traceId not already provided by caller.
|
|
67
|
+
// Deferred .then() callbacks lose async context, so OtelBridge would
|
|
68
|
+
// return undefined and overwrite the valid traceId the caller passed.
|
|
69
|
+
if (!entry.traceId) {
|
|
70
|
+
const bridge = new OtelBridge();
|
|
71
|
+
const traceCtx = bridge.getCurrentTraceContext();
|
|
72
|
+
if (traceCtx) {
|
|
73
|
+
entry.traceId = traceCtx.traceId;
|
|
74
|
+
entry.spanId = traceCtx.spanId;
|
|
75
|
+
}
|
|
76
|
+
}
|
|
49
77
|
const logFile = join(logDir, `proxy-${new Date().toISOString().split("T")[0]}.jsonl`);
|
|
50
78
|
const line = JSON.stringify(entry) + "\n";
|
|
51
79
|
try {
|
|
@@ -54,6 +82,140 @@ export async function logRequest(entry) {
|
|
|
54
82
|
catch {
|
|
55
83
|
// Non-fatal — don't crash proxy for logging failures
|
|
56
84
|
}
|
|
85
|
+
// Emit OTLP log record (additive — file logging is the primary sink)
|
|
86
|
+
emitOtlpLogRecord(entry);
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* Log an upstream attempt separately from the final request outcome.
|
|
90
|
+
* Attempt logs are local-only and must not pollute the final request summary
|
|
91
|
+
* or OTLP-derived dashboard panels.
|
|
92
|
+
*/
|
|
93
|
+
export async function logRequestAttempt(entry) {
|
|
94
|
+
if (!logEnabled || !logDir) {
|
|
95
|
+
return;
|
|
96
|
+
}
|
|
97
|
+
if (!entry.traceId) {
|
|
98
|
+
const bridge = new OtelBridge();
|
|
99
|
+
const traceCtx = bridge.getCurrentTraceContext();
|
|
100
|
+
if (traceCtx) {
|
|
101
|
+
entry.traceId = traceCtx.traceId;
|
|
102
|
+
entry.spanId = traceCtx.spanId;
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
const logFile = join(logDir, `proxy-attempts-${new Date().toISOString().split("T")[0]}.jsonl`);
|
|
106
|
+
const line = JSON.stringify(entry) + "\n";
|
|
107
|
+
try {
|
|
108
|
+
await appendFile(logFile, line, { mode: 0o600 });
|
|
109
|
+
}
|
|
110
|
+
catch {
|
|
111
|
+
// Non-fatal — don't crash proxy for logging failures
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* Lazily resolve the LoggerProvider from OTel instrumentation.
|
|
116
|
+
* Uses dynamic import to avoid hard dependency — if instrumentation.ts
|
|
117
|
+
* hasn't been loaded or OTLP is not configured, this is a no-op.
|
|
118
|
+
* Retries up to MAX_RESOLVE_ATTEMPTS times to handle race conditions
|
|
119
|
+
* where OTel initialization completes after the first log request.
|
|
120
|
+
*/
|
|
121
|
+
async function resolveLoggerProvider() {
|
|
122
|
+
if (otelLoggerProvider === false) {
|
|
123
|
+
return undefined;
|
|
124
|
+
} // permanently unavailable
|
|
125
|
+
if (otelLoggerProvider !== null) {
|
|
126
|
+
return otelLoggerProvider;
|
|
127
|
+
}
|
|
128
|
+
// Not resolved yet — try to resolve
|
|
129
|
+
otelResolveAttempts++;
|
|
130
|
+
try {
|
|
131
|
+
const { getLoggerProvider } = await import("../services/server/ai/observability/instrumentation.js");
|
|
132
|
+
const provider = getLoggerProvider();
|
|
133
|
+
if (provider) {
|
|
134
|
+
otelLoggerProvider = provider;
|
|
135
|
+
return provider;
|
|
136
|
+
}
|
|
137
|
+
// Provider not available yet — if we've exceeded max attempts, give up
|
|
138
|
+
if (otelResolveAttempts >= MAX_RESOLVE_ATTEMPTS) {
|
|
139
|
+
otelLoggerProvider = false; // permanently unavailable
|
|
140
|
+
}
|
|
141
|
+
// Otherwise leave as null so we retry next time
|
|
142
|
+
return undefined;
|
|
143
|
+
}
|
|
144
|
+
catch {
|
|
145
|
+
// instrumentation.ts not available (e.g. standalone mode) — disable permanently
|
|
146
|
+
otelLoggerProvider = false;
|
|
147
|
+
return undefined;
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
/**
|
|
151
|
+
* Emit a RequestLogEntry as an OTLP log record.
|
|
152
|
+
* Non-blocking, non-fatal — failures are silently swallowed.
|
|
153
|
+
*/
|
|
154
|
+
function emitOtlpLogRecord(entry) {
|
|
155
|
+
resolveLoggerProvider()
|
|
156
|
+
.then((provider) => {
|
|
157
|
+
if (!provider) {
|
|
158
|
+
return;
|
|
159
|
+
}
|
|
160
|
+
const otelLogger = provider.getLogger("neurolink-proxy", "1.0.0");
|
|
161
|
+
// Determine severity based on response status
|
|
162
|
+
const isError = (entry.responseStatus ?? 0) >= 400;
|
|
163
|
+
const isRateLimit = entry.responseStatus === 429;
|
|
164
|
+
const severityNumber = isError
|
|
165
|
+
? isRateLimit
|
|
166
|
+
? SeverityNumber.WARN
|
|
167
|
+
: SeverityNumber.ERROR
|
|
168
|
+
: SeverityNumber.INFO;
|
|
169
|
+
const severityText = isError ? (isRateLimit ? "WARN" : "ERROR") : "INFO";
|
|
170
|
+
otelLogger.emit({
|
|
171
|
+
severityNumber,
|
|
172
|
+
severityText,
|
|
173
|
+
body: `${entry.method} ${entry.path} → ${entry.responseStatus} (${entry.responseTimeMs}ms)`,
|
|
174
|
+
attributes: {
|
|
175
|
+
// Core request fields
|
|
176
|
+
"request.id": entry.requestId,
|
|
177
|
+
"http.method": entry.method,
|
|
178
|
+
"http.path": entry.path,
|
|
179
|
+
"http.status_code": entry.responseStatus,
|
|
180
|
+
"response.time_ms": entry.responseTimeMs,
|
|
181
|
+
// AI-specific fields
|
|
182
|
+
"ai.model": entry.model,
|
|
183
|
+
"ai.stream": entry.stream,
|
|
184
|
+
"ai.tool_count": entry.toolCount,
|
|
185
|
+
// Account info
|
|
186
|
+
"account.name": entry.account,
|
|
187
|
+
"account.type": entry.accountType,
|
|
188
|
+
// Token usage (when available)
|
|
189
|
+
...(entry.inputTokens !== undefined && {
|
|
190
|
+
"ai.input_tokens": entry.inputTokens,
|
|
191
|
+
}),
|
|
192
|
+
...(entry.outputTokens !== undefined && {
|
|
193
|
+
"ai.output_tokens": entry.outputTokens,
|
|
194
|
+
}),
|
|
195
|
+
...(entry.cacheCreationTokens !== undefined && {
|
|
196
|
+
"ai.cache_creation_tokens": entry.cacheCreationTokens,
|
|
197
|
+
}),
|
|
198
|
+
...(entry.cacheReadTokens !== undefined && {
|
|
199
|
+
"ai.cache_read_tokens": entry.cacheReadTokens,
|
|
200
|
+
}),
|
|
201
|
+
// Error info (when present)
|
|
202
|
+
...(entry.errorType && { "error.type": entry.errorType }),
|
|
203
|
+
...(entry.errorMessage && { "error.message": entry.errorMessage }),
|
|
204
|
+
// Trace correlation
|
|
205
|
+
...(entry.traceId && { "trace.id": entry.traceId }),
|
|
206
|
+
...(entry.spanId && { "span.id": entry.spanId }),
|
|
207
|
+
// Derived fields for dashboards (matches backfill script)
|
|
208
|
+
is_success: entry.responseStatus === 200,
|
|
209
|
+
is_rate_limited: entry.responseStatus === 429,
|
|
210
|
+
is_overloaded: entry.responseStatus === 529,
|
|
211
|
+
is_error: isError,
|
|
212
|
+
source: "otlp",
|
|
213
|
+
},
|
|
214
|
+
});
|
|
215
|
+
})
|
|
216
|
+
.catch(() => {
|
|
217
|
+
// Non-fatal — never crash proxy for OTLP log failures
|
|
218
|
+
});
|
|
57
219
|
}
|
|
58
220
|
export function getLogDir() {
|
|
59
221
|
return logDir;
|
|
@@ -78,51 +240,220 @@ function redactHeaders(headers) {
|
|
|
78
240
|
}
|
|
79
241
|
return redacted;
|
|
80
242
|
}
|
|
243
|
+
function serializeBody(body) {
|
|
244
|
+
if (body === undefined || body === null) {
|
|
245
|
+
return undefined;
|
|
246
|
+
}
|
|
247
|
+
return typeof body === "string" ? body : JSON.stringify(body);
|
|
248
|
+
}
|
|
81
249
|
/**
|
|
82
|
-
* Redact sensitive keys from a JSON body string
|
|
250
|
+
* Redact sensitive keys from a JSON body string without truncation.
|
|
83
251
|
*/
|
|
84
252
|
function redactBody(body) {
|
|
85
|
-
|
|
86
|
-
|
|
253
|
+
const str = serializeBody(body);
|
|
254
|
+
if (str === undefined) {
|
|
255
|
+
return undefined;
|
|
87
256
|
}
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
257
|
+
return str.replace(SENSITIVE_BODY_KEYS, '$1"[REDACTED]"');
|
|
258
|
+
}
|
|
259
|
+
function sanitizePhase(phase) {
|
|
260
|
+
return phase.replace(/[^a-zA-Z0-9._-]+/g, "_");
|
|
261
|
+
}
|
|
262
|
+
function sha256(value) {
|
|
263
|
+
return createHash("sha256").update(value).digest("hex");
|
|
264
|
+
}
|
|
265
|
+
async function writeBodyArtifact(entry, redactedHeaders, redactedBody) {
|
|
266
|
+
if (!logDir || redactedBody === undefined) {
|
|
267
|
+
return {};
|
|
97
268
|
}
|
|
98
|
-
|
|
269
|
+
const dateStr = new Date(entry.timestamp).toISOString().split("T")[0];
|
|
270
|
+
const bodyDir = join(logDir, "bodies", dateStr, entry.requestId);
|
|
271
|
+
if (!existsSync(bodyDir)) {
|
|
272
|
+
mkdirSync(bodyDir, { recursive: true, mode: 0o700 });
|
|
273
|
+
}
|
|
274
|
+
chmodSync(bodyDir, 0o700);
|
|
275
|
+
const fileName = `${Date.now()}-${sanitizePhase(entry.phase)}` +
|
|
276
|
+
(entry.attempt !== undefined ? `-attempt-${entry.attempt}` : "") +
|
|
277
|
+
`.json.gz`;
|
|
278
|
+
const bodyPath = join(bodyDir, fileName);
|
|
279
|
+
const payload = JSON.stringify({
|
|
280
|
+
timestamp: entry.timestamp,
|
|
281
|
+
requestId: entry.requestId,
|
|
282
|
+
phase: entry.phase,
|
|
283
|
+
model: entry.model,
|
|
284
|
+
stream: entry.stream,
|
|
285
|
+
account: entry.account,
|
|
286
|
+
accountType: entry.accountType,
|
|
287
|
+
attempt: entry.attempt,
|
|
288
|
+
responseStatus: entry.responseStatus,
|
|
289
|
+
durationMs: entry.durationMs,
|
|
290
|
+
contentType: entry.contentType,
|
|
291
|
+
headers: redactedHeaders,
|
|
292
|
+
body: redactedBody,
|
|
293
|
+
traceId: entry.traceId,
|
|
294
|
+
spanId: entry.spanId,
|
|
295
|
+
metadata: entry.metadata,
|
|
296
|
+
});
|
|
297
|
+
const compressed = await gzip(payload);
|
|
298
|
+
await writeFile(bodyPath, compressed, { mode: 0o600 });
|
|
299
|
+
return {
|
|
300
|
+
bodyPath,
|
|
301
|
+
bodySha256: sha256(redactedBody),
|
|
302
|
+
redactedBodyBytes: Buffer.byteLength(redactedBody, "utf8"),
|
|
303
|
+
storedFileBytes: compressed.byteLength,
|
|
304
|
+
redactedBody,
|
|
305
|
+
};
|
|
99
306
|
}
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
307
|
+
function emitOtlpBodyLogRecord(entry, stored) {
|
|
308
|
+
resolveLoggerProvider()
|
|
309
|
+
.then((provider) => {
|
|
310
|
+
if (!provider || stored.redactedBody === undefined) {
|
|
311
|
+
return;
|
|
312
|
+
}
|
|
313
|
+
const otelLogger = provider.getLogger("neurolink-proxy-bodies", "1.0.0");
|
|
314
|
+
const totalChunks = Math.max(1, Math.ceil(stored.redactedBody.length / BODY_OTLP_CHUNK_SIZE));
|
|
315
|
+
for (let chunkIndex = 0; chunkIndex < totalChunks; chunkIndex++) {
|
|
316
|
+
const chunk = stored.redactedBody.slice(chunkIndex * BODY_OTLP_CHUNK_SIZE, (chunkIndex + 1) * BODY_OTLP_CHUNK_SIZE);
|
|
317
|
+
otelLogger.emit({
|
|
318
|
+
severityNumber: (entry.responseStatus ?? 0) >= 400
|
|
319
|
+
? SeverityNumber.WARN
|
|
320
|
+
: SeverityNumber.INFO,
|
|
321
|
+
severityText: (entry.responseStatus ?? 0) >= 400 ? "WARN" : "INFO",
|
|
322
|
+
body: chunk,
|
|
323
|
+
attributes: {
|
|
324
|
+
"event.name": "proxy.body_capture",
|
|
325
|
+
"request.id": entry.requestId,
|
|
326
|
+
"body.phase": entry.phase,
|
|
327
|
+
"body.chunk_index": chunkIndex,
|
|
328
|
+
"body.chunk_count": totalChunks,
|
|
329
|
+
"body.content_type": entry.contentType ?? "application/json",
|
|
330
|
+
"ai.model": entry.model,
|
|
331
|
+
"ai.stream": entry.stream,
|
|
332
|
+
...(entry.account && { "account.name": entry.account }),
|
|
333
|
+
...(entry.accountType && { "account.type": entry.accountType }),
|
|
334
|
+
...(entry.attempt !== undefined && {
|
|
335
|
+
"proxy.attempt": entry.attempt,
|
|
336
|
+
}),
|
|
337
|
+
...(entry.responseStatus !== undefined && {
|
|
338
|
+
"http.status_code": entry.responseStatus,
|
|
339
|
+
}),
|
|
340
|
+
...(entry.durationMs !== undefined && {
|
|
341
|
+
"response.time_ms": entry.durationMs,
|
|
342
|
+
}),
|
|
343
|
+
...(stored.bodySha256 && { "body.sha256": stored.bodySha256 }),
|
|
344
|
+
...(stored.bodyPath && {
|
|
345
|
+
"body.path": stored.bodyPath.split("/").slice(-2).join("/"),
|
|
346
|
+
}),
|
|
347
|
+
...(stored.redactedBodyBytes !== undefined && {
|
|
348
|
+
"body.bytes": stored.redactedBodyBytes,
|
|
349
|
+
}),
|
|
350
|
+
...(entry.traceId && { "trace.id": entry.traceId }),
|
|
351
|
+
...(entry.spanId && { "span.id": entry.spanId }),
|
|
352
|
+
...(entry.metadata && {
|
|
353
|
+
"body.metadata_json": JSON.stringify(entry.metadata),
|
|
354
|
+
}),
|
|
355
|
+
source: "otlp",
|
|
356
|
+
},
|
|
357
|
+
});
|
|
358
|
+
}
|
|
359
|
+
})
|
|
360
|
+
.catch(() => {
|
|
361
|
+
// Non-fatal — never crash proxy for OTLP log failures
|
|
362
|
+
});
|
|
363
|
+
}
|
|
364
|
+
export async function logBodyCapture(entry) {
|
|
108
365
|
if (!logEnabled || !logDir) {
|
|
109
366
|
return;
|
|
110
367
|
}
|
|
111
|
-
const
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
368
|
+
const bridge = new OtelBridge();
|
|
369
|
+
const traceCtx = entry.traceId && entry.spanId
|
|
370
|
+
? { traceId: entry.traceId, spanId: entry.spanId }
|
|
371
|
+
: bridge.getCurrentTraceContext();
|
|
372
|
+
const redactedHeaders = redactHeaders(entry.headers);
|
|
373
|
+
let stored = {};
|
|
374
|
+
try {
|
|
375
|
+
stored = await writeBodyArtifact(entry, redactedHeaders, redactBody(entry.body));
|
|
376
|
+
}
|
|
377
|
+
catch {
|
|
378
|
+
// Best-effort artifact persistence; continue with in-memory metadata only.
|
|
379
|
+
}
|
|
380
|
+
const dateStr = new Date(entry.timestamp).toISOString().split("T")[0];
|
|
381
|
+
const logFile = join(logDir, `proxy-debug-${dateStr}.jsonl`);
|
|
382
|
+
const indexEntry = {
|
|
383
|
+
timestamp: entry.timestamp,
|
|
384
|
+
type: "body_capture",
|
|
385
|
+
requestId: entry.requestId,
|
|
386
|
+
phase: entry.phase,
|
|
387
|
+
model: entry.model,
|
|
388
|
+
stream: entry.stream,
|
|
389
|
+
headers: redactedHeaders,
|
|
390
|
+
contentType: entry.contentType,
|
|
391
|
+
responseStatus: entry.responseStatus,
|
|
392
|
+
durationMs: entry.durationMs,
|
|
393
|
+
account: entry.account,
|
|
394
|
+
accountType: entry.accountType,
|
|
395
|
+
attempt: entry.attempt,
|
|
396
|
+
bodyPath: stored.bodyPath,
|
|
397
|
+
bodySha256: stored.bodySha256,
|
|
398
|
+
observedBodyBytes: entry.bodySize,
|
|
399
|
+
redactedBodyBytes: stored.redactedBodyBytes,
|
|
400
|
+
storedFileBytes: stored.storedFileBytes,
|
|
401
|
+
metadata: entry.metadata,
|
|
117
402
|
};
|
|
118
|
-
|
|
119
|
-
|
|
403
|
+
if (traceCtx) {
|
|
404
|
+
indexEntry.traceId = traceCtx.traceId;
|
|
405
|
+
indexEntry.spanId = traceCtx.spanId;
|
|
406
|
+
}
|
|
120
407
|
try {
|
|
121
|
-
await appendFile(logFile,
|
|
408
|
+
await appendFile(logFile, JSON.stringify(indexEntry) + "\n", {
|
|
409
|
+
mode: 0o600,
|
|
410
|
+
});
|
|
122
411
|
}
|
|
123
412
|
catch {
|
|
124
413
|
// Non-fatal
|
|
125
414
|
}
|
|
415
|
+
emitOtlpBodyLogRecord({
|
|
416
|
+
...entry,
|
|
417
|
+
traceId: traceCtx?.traceId ?? entry.traceId,
|
|
418
|
+
spanId: traceCtx?.spanId ?? entry.spanId,
|
|
419
|
+
}, stored);
|
|
420
|
+
}
|
|
421
|
+
/**
|
|
422
|
+
* Log the FULL raw request and response for debugging.
|
|
423
|
+
* Legacy helper kept for compatibility. New call sites should prefer
|
|
424
|
+
* logBodyCapture() so each phase can be indexed and persisted separately.
|
|
425
|
+
*/
|
|
426
|
+
export async function logFullRequestResponse(entry) {
|
|
427
|
+
await Promise.all([
|
|
428
|
+
logBodyCapture({
|
|
429
|
+
timestamp: entry.timestamp,
|
|
430
|
+
requestId: entry.requestId,
|
|
431
|
+
phase: "legacy_upstream_request",
|
|
432
|
+
model: entry.model,
|
|
433
|
+
stream: entry.stream,
|
|
434
|
+
headers: entry.requestHeaders,
|
|
435
|
+
body: entry.requestBody,
|
|
436
|
+
bodySize: entry.requestBodySize,
|
|
437
|
+
contentType: entry.requestHeaders["content-type"] ?? "application/json",
|
|
438
|
+
account: entry.account,
|
|
439
|
+
responseStatus: entry.responseStatus,
|
|
440
|
+
durationMs: entry.durationMs,
|
|
441
|
+
}),
|
|
442
|
+
logBodyCapture({
|
|
443
|
+
timestamp: entry.timestamp,
|
|
444
|
+
requestId: entry.requestId,
|
|
445
|
+
phase: "legacy_upstream_response",
|
|
446
|
+
model: entry.model,
|
|
447
|
+
stream: entry.stream,
|
|
448
|
+
headers: entry.responseHeaders,
|
|
449
|
+
body: entry.responseBody,
|
|
450
|
+
bodySize: entry.responseBodySize,
|
|
451
|
+
contentType: entry.responseHeaders?.["content-type"] ?? "application/json",
|
|
452
|
+
account: entry.account,
|
|
453
|
+
responseStatus: entry.responseStatus,
|
|
454
|
+
durationMs: entry.durationMs,
|
|
455
|
+
}),
|
|
456
|
+
]);
|
|
126
457
|
}
|
|
127
458
|
/**
|
|
128
459
|
* Log a mid-stream error that occurs after the initial 200 was sent.
|
|
@@ -132,6 +463,8 @@ export async function logStreamError(entry) {
|
|
|
132
463
|
if (!logEnabled || !logDir) {
|
|
133
464
|
return;
|
|
134
465
|
}
|
|
466
|
+
const bridge = new OtelBridge();
|
|
467
|
+
const traceCtx = bridge.getCurrentTraceContext();
|
|
135
468
|
const logFile = join(logDir, `proxy-${new Date().toISOString().split("T")[0]}.jsonl`);
|
|
136
469
|
const logEntry = {
|
|
137
470
|
...entry,
|
|
@@ -139,6 +472,10 @@ export async function logStreamError(entry) {
|
|
|
139
472
|
errorType: "stream_error",
|
|
140
473
|
note: "mid-stream failure after initial 200",
|
|
141
474
|
};
|
|
475
|
+
if (traceCtx) {
|
|
476
|
+
logEntry.traceId = traceCtx.traceId;
|
|
477
|
+
logEntry.spanId = traceCtx.spanId;
|
|
478
|
+
}
|
|
142
479
|
try {
|
|
143
480
|
await appendFile(logFile, JSON.stringify(logEntry) + "\n", {
|
|
144
481
|
mode: 0o600,
|
|
@@ -159,10 +496,12 @@ export function cleanupLogs(maxAgeDays = 7, maxSizeMb = 500) {
|
|
|
159
496
|
return;
|
|
160
497
|
}
|
|
161
498
|
try {
|
|
499
|
+
const activeLogDir = logDir;
|
|
162
500
|
const files = readdirSync(logDir)
|
|
163
|
-
.filter((f) => f.startsWith("proxy-")
|
|
501
|
+
.filter((f) => (f.startsWith("proxy-") || f.startsWith("proxy-attempts-")) &&
|
|
502
|
+
f.endsWith(".jsonl"))
|
|
164
503
|
.map((f) => {
|
|
165
|
-
const filePath = join(
|
|
504
|
+
const filePath = join(activeLogDir, f);
|
|
166
505
|
const stat = statSync(filePath);
|
|
167
506
|
return {
|
|
168
507
|
name: f,
|
|
@@ -187,11 +526,41 @@ export function cleanupLogs(maxAgeDays = 7, maxSizeMb = 500) {
|
|
|
187
526
|
remaining.push(file);
|
|
188
527
|
}
|
|
189
528
|
}
|
|
529
|
+
const bodiesDir = join(logDir, "bodies");
|
|
530
|
+
if (existsSync(bodiesDir)) {
|
|
531
|
+
for (const entry of readdirSync(bodiesDir)) {
|
|
532
|
+
const bodyPath = join(bodiesDir, entry);
|
|
533
|
+
try {
|
|
534
|
+
if (statSync(bodyPath).mtimeMs < cutoff) {
|
|
535
|
+
rmSync(bodyPath, { recursive: true, force: true });
|
|
536
|
+
}
|
|
537
|
+
}
|
|
538
|
+
catch {
|
|
539
|
+
// Non-fatal
|
|
540
|
+
}
|
|
541
|
+
}
|
|
542
|
+
}
|
|
543
|
+
// Include body artifacts in total size calculation
|
|
544
|
+
const bodiesDirForSize = join(logDir, "bodies");
|
|
545
|
+
let bodiesSize = 0;
|
|
546
|
+
if (existsSync(bodiesDirForSize)) {
|
|
547
|
+
for (const entry of readdirSync(bodiesDirForSize)) {
|
|
548
|
+
try {
|
|
549
|
+
bodiesSize += statSync(join(bodiesDirForSize, entry)).size;
|
|
550
|
+
}
|
|
551
|
+
catch {
|
|
552
|
+
// Non-fatal
|
|
553
|
+
}
|
|
554
|
+
}
|
|
555
|
+
}
|
|
190
556
|
// Pass 2: if total size exceeds maxSizeMb, delete oldest until under limit
|
|
191
557
|
const maxBytes = maxSizeMb * 1024 * 1024;
|
|
192
|
-
let totalSize = remaining.reduce((sum, f) => sum + f.size, 0);
|
|
558
|
+
let totalSize = remaining.reduce((sum, f) => sum + f.size, 0) + bodiesSize;
|
|
193
559
|
while (totalSize > maxBytes && remaining.length > 0) {
|
|
194
560
|
const oldest = remaining.shift();
|
|
561
|
+
if (!oldest) {
|
|
562
|
+
break;
|
|
563
|
+
}
|
|
195
564
|
unlinkSync(oldest.path);
|
|
196
565
|
totalSize -= oldest.size;
|
|
197
566
|
deletedCount++;
|