@juspay/neurolink 9.41.0 → 9.42.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +8 -0
- package/README.md +7 -1
- package/dist/auth/anthropicOAuth.d.ts +18 -3
- package/dist/auth/anthropicOAuth.js +149 -4
- package/dist/auth/providers/firebase.js +5 -1
- package/dist/auth/providers/jwt.js +5 -1
- package/dist/auth/providers/workos.js +5 -1
- package/dist/auth/sessionManager.d.ts +1 -1
- package/dist/auth/sessionManager.js +58 -27
- package/dist/browser/neurolink.min.js +354 -334
- package/dist/cli/commands/mcp.d.ts +6 -0
- package/dist/cli/commands/mcp.js +188 -181
- package/dist/cli/commands/proxy.d.ts +2 -1
- package/dist/cli/commands/proxy.js +713 -431
- package/dist/cli/commands/task.js +3 -0
- package/dist/cli/factories/commandFactory.d.ts +2 -0
- package/dist/cli/factories/commandFactory.js +38 -0
- package/dist/cli/parser.js +4 -3
- package/dist/client/aiSdkAdapter.js +3 -0
- package/dist/client/streamingClient.js +30 -10
- package/dist/core/baseProvider.d.ts +6 -1
- package/dist/core/baseProvider.js +208 -230
- package/dist/core/factory.d.ts +3 -0
- package/dist/core/factory.js +138 -188
- package/dist/core/modules/GenerationHandler.js +3 -2
- package/dist/core/redisConversationMemoryManager.js +7 -3
- package/dist/evaluation/BatchEvaluator.js +4 -1
- package/dist/evaluation/hooks/observabilityHooks.js +5 -3
- package/dist/evaluation/pipeline/evaluationPipeline.d.ts +3 -2
- package/dist/evaluation/pipeline/evaluationPipeline.js +24 -9
- package/dist/evaluation/pipeline/strategies/batchStrategy.js +6 -3
- package/dist/evaluation/pipeline/strategies/samplingStrategy.js +18 -10
- package/dist/evaluation/scorers/scorerRegistry.d.ts +3 -0
- package/dist/evaluation/scorers/scorerRegistry.js +353 -282
- package/dist/lib/auth/anthropicOAuth.d.ts +18 -3
- package/dist/lib/auth/anthropicOAuth.js +149 -4
- package/dist/lib/auth/providers/firebase.js +5 -1
- package/dist/lib/auth/providers/jwt.js +5 -1
- package/dist/lib/auth/providers/workos.js +5 -1
- package/dist/lib/auth/sessionManager.d.ts +1 -1
- package/dist/lib/auth/sessionManager.js +58 -27
- package/dist/lib/client/aiSdkAdapter.js +3 -0
- package/dist/lib/client/streamingClient.js +30 -10
- package/dist/lib/core/baseProvider.d.ts +6 -1
- package/dist/lib/core/baseProvider.js +208 -230
- package/dist/lib/core/factory.d.ts +3 -0
- package/dist/lib/core/factory.js +138 -188
- package/dist/lib/core/modules/GenerationHandler.js +3 -2
- package/dist/lib/core/redisConversationMemoryManager.js +7 -3
- package/dist/lib/evaluation/BatchEvaluator.js +4 -1
- package/dist/lib/evaluation/hooks/observabilityHooks.js +5 -3
- package/dist/lib/evaluation/pipeline/evaluationPipeline.d.ts +3 -2
- package/dist/lib/evaluation/pipeline/evaluationPipeline.js +24 -9
- package/dist/lib/evaluation/pipeline/strategies/batchStrategy.js +6 -3
- package/dist/lib/evaluation/pipeline/strategies/samplingStrategy.js +18 -10
- package/dist/lib/evaluation/scorers/scorerRegistry.d.ts +3 -0
- package/dist/lib/evaluation/scorers/scorerRegistry.js +353 -282
- package/dist/lib/mcp/toolRegistry.d.ts +2 -0
- package/dist/lib/mcp/toolRegistry.js +32 -31
- package/dist/lib/neurolink.d.ts +41 -2
- package/dist/lib/neurolink.js +1616 -1681
- package/dist/lib/observability/otelBridge.d.ts +2 -2
- package/dist/lib/observability/otelBridge.js +12 -3
- package/dist/lib/providers/amazonBedrock.js +2 -4
- package/dist/lib/providers/anthropic.d.ts +9 -5
- package/dist/lib/providers/anthropic.js +19 -14
- package/dist/lib/providers/anthropicBaseProvider.d.ts +3 -3
- package/dist/lib/providers/anthropicBaseProvider.js +5 -4
- package/dist/lib/providers/azureOpenai.d.ts +1 -1
- package/dist/lib/providers/azureOpenai.js +5 -4
- package/dist/lib/providers/googleAiStudio.js +30 -6
- package/dist/lib/providers/googleVertex.d.ts +10 -0
- package/dist/lib/providers/googleVertex.js +437 -423
- package/dist/lib/providers/huggingFace.d.ts +3 -3
- package/dist/lib/providers/huggingFace.js +6 -8
- package/dist/lib/providers/litellm.d.ts +1 -0
- package/dist/lib/providers/litellm.js +76 -55
- package/dist/lib/providers/mistral.js +2 -1
- package/dist/lib/providers/ollama.js +93 -23
- package/dist/lib/providers/openAI.d.ts +2 -0
- package/dist/lib/providers/openAI.js +141 -141
- package/dist/lib/providers/openRouter.js +2 -1
- package/dist/lib/providers/openaiCompatible.d.ts +4 -4
- package/dist/lib/providers/openaiCompatible.js +4 -4
- package/dist/lib/proxy/claudeFormat.d.ts +3 -2
- package/dist/lib/proxy/claudeFormat.js +27 -14
- package/dist/lib/proxy/cloaking/plugins/sessionIdentity.d.ts +2 -6
- package/dist/lib/proxy/cloaking/plugins/sessionIdentity.js +9 -33
- package/dist/lib/proxy/modelRouter.js +3 -0
- package/dist/lib/proxy/oauthFetch.d.ts +1 -1
- package/dist/lib/proxy/oauthFetch.js +289 -316
- package/dist/lib/proxy/proxyConfig.js +46 -24
- package/dist/lib/proxy/proxyEnv.d.ts +19 -0
- package/dist/lib/proxy/proxyEnv.js +73 -0
- package/dist/lib/proxy/proxyFetch.js +291 -217
- package/dist/lib/proxy/proxyTracer.d.ts +133 -0
- package/dist/lib/proxy/proxyTracer.js +645 -0
- package/dist/lib/proxy/rawStreamCapture.d.ts +10 -0
- package/dist/lib/proxy/rawStreamCapture.js +83 -0
- package/dist/lib/proxy/requestLogger.d.ts +32 -5
- package/dist/lib/proxy/requestLogger.js +503 -47
- package/dist/lib/proxy/sseInterceptor.d.ts +97 -0
- package/dist/lib/proxy/sseInterceptor.js +427 -0
- package/dist/lib/proxy/usageStats.d.ts +4 -3
- package/dist/lib/proxy/usageStats.js +25 -12
- package/dist/lib/rag/chunkers/MarkdownChunker.js +13 -5
- package/dist/lib/rag/chunking/markdownChunker.js +15 -6
- package/dist/lib/server/routes/claudeProxyRoutes.d.ts +17 -3
- package/dist/lib/server/routes/claudeProxyRoutes.js +3032 -1349
- package/dist/lib/services/server/ai/observability/instrumentation.d.ts +7 -1
- package/dist/lib/services/server/ai/observability/instrumentation.js +337 -161
- package/dist/lib/tasks/backends/bullmqBackend.d.ts +1 -0
- package/dist/lib/tasks/backends/bullmqBackend.js +35 -22
- package/dist/lib/tasks/store/redisTaskStore.d.ts +1 -0
- package/dist/lib/tasks/store/redisTaskStore.js +54 -39
- package/dist/lib/tasks/taskManager.d.ts +5 -0
- package/dist/lib/tasks/taskManager.js +158 -30
- package/dist/lib/telemetry/index.d.ts +2 -1
- package/dist/lib/telemetry/index.js +2 -1
- package/dist/lib/telemetry/telemetryService.d.ts +3 -0
- package/dist/lib/telemetry/telemetryService.js +69 -5
- package/dist/lib/types/cli.d.ts +10 -0
- package/dist/lib/types/proxyTypes.d.ts +160 -5
- package/dist/lib/types/streamTypes.d.ts +25 -3
- package/dist/lib/utils/messageBuilder.js +3 -2
- package/dist/lib/utils/providerHealth.d.ts +19 -0
- package/dist/lib/utils/providerHealth.js +279 -33
- package/dist/lib/utils/providerUtils.js +17 -22
- package/dist/lib/utils/toolChoice.d.ts +4 -0
- package/dist/lib/utils/toolChoice.js +7 -0
- package/dist/mcp/toolRegistry.d.ts +2 -0
- package/dist/mcp/toolRegistry.js +32 -31
- package/dist/neurolink.d.ts +41 -2
- package/dist/neurolink.js +1616 -1681
- package/dist/observability/otelBridge.d.ts +2 -2
- package/dist/observability/otelBridge.js +12 -3
- package/dist/providers/amazonBedrock.js +2 -4
- package/dist/providers/anthropic.d.ts +9 -5
- package/dist/providers/anthropic.js +19 -14
- package/dist/providers/anthropicBaseProvider.d.ts +3 -3
- package/dist/providers/anthropicBaseProvider.js +5 -4
- package/dist/providers/azureOpenai.d.ts +1 -1
- package/dist/providers/azureOpenai.js +5 -4
- package/dist/providers/googleAiStudio.js +30 -6
- package/dist/providers/googleVertex.d.ts +10 -0
- package/dist/providers/googleVertex.js +437 -423
- package/dist/providers/huggingFace.d.ts +3 -3
- package/dist/providers/huggingFace.js +6 -7
- package/dist/providers/litellm.d.ts +1 -0
- package/dist/providers/litellm.js +76 -55
- package/dist/providers/mistral.js +2 -1
- package/dist/providers/ollama.js +93 -23
- package/dist/providers/openAI.d.ts +2 -0
- package/dist/providers/openAI.js +141 -141
- package/dist/providers/openRouter.js +2 -1
- package/dist/providers/openaiCompatible.d.ts +4 -4
- package/dist/providers/openaiCompatible.js +4 -3
- package/dist/proxy/claudeFormat.d.ts +3 -2
- package/dist/proxy/claudeFormat.js +27 -14
- package/dist/proxy/cloaking/plugins/sessionIdentity.d.ts +2 -6
- package/dist/proxy/cloaking/plugins/sessionIdentity.js +9 -33
- package/dist/proxy/modelRouter.js +3 -0
- package/dist/proxy/oauthFetch.d.ts +1 -1
- package/dist/proxy/oauthFetch.js +289 -316
- package/dist/proxy/proxyConfig.js +46 -24
- package/dist/proxy/proxyEnv.d.ts +19 -0
- package/dist/proxy/proxyEnv.js +72 -0
- package/dist/proxy/proxyFetch.js +291 -217
- package/dist/proxy/proxyTracer.d.ts +133 -0
- package/dist/proxy/proxyTracer.js +644 -0
- package/dist/proxy/rawStreamCapture.d.ts +10 -0
- package/dist/proxy/rawStreamCapture.js +82 -0
- package/dist/proxy/requestLogger.d.ts +32 -5
- package/dist/proxy/requestLogger.js +503 -47
- package/dist/proxy/sseInterceptor.d.ts +97 -0
- package/dist/proxy/sseInterceptor.js +426 -0
- package/dist/proxy/usageStats.d.ts +4 -3
- package/dist/proxy/usageStats.js +25 -12
- package/dist/rag/chunkers/MarkdownChunker.js +13 -5
- package/dist/rag/chunking/markdownChunker.js +15 -6
- package/dist/server/routes/claudeProxyRoutes.d.ts +17 -3
- package/dist/server/routes/claudeProxyRoutes.js +3032 -1349
- package/dist/services/server/ai/observability/instrumentation.d.ts +7 -1
- package/dist/services/server/ai/observability/instrumentation.js +337 -161
- package/dist/tasks/backends/bullmqBackend.d.ts +1 -0
- package/dist/tasks/backends/bullmqBackend.js +35 -22
- package/dist/tasks/store/redisTaskStore.d.ts +1 -0
- package/dist/tasks/store/redisTaskStore.js +54 -39
- package/dist/tasks/taskManager.d.ts +5 -0
- package/dist/tasks/taskManager.js +158 -30
- package/dist/telemetry/index.d.ts +2 -1
- package/dist/telemetry/index.js +2 -1
- package/dist/telemetry/telemetryService.d.ts +3 -0
- package/dist/telemetry/telemetryService.js +69 -5
- package/dist/types/cli.d.ts +10 -0
- package/dist/types/proxyTypes.d.ts +160 -5
- package/dist/types/streamTypes.d.ts +25 -3
- package/dist/utils/messageBuilder.js +3 -2
- package/dist/utils/providerHealth.d.ts +19 -0
- package/dist/utils/providerHealth.js +279 -33
- package/dist/utils/providerUtils.js +18 -22
- package/dist/utils/toolChoice.d.ts +4 -0
- package/dist/utils/toolChoice.js +6 -0
- package/docs/assets/dashboards/neurolink-proxy-observability-dashboard.json +6609 -0
- package/docs/changelog.md +252 -0
- package/package.json +19 -2
- package/scripts/observability/check-proxy-telemetry.mjs +235 -0
- package/scripts/observability/docker-compose.proxy-observability.yaml +55 -0
- package/scripts/observability/import-openobserve-dashboard.mjs +240 -0
- package/scripts/observability/manage-local-openobserve.sh +215 -0
- package/scripts/observability/otel-collector.proxy-observability.yaml +78 -0
- package/scripts/observability/proxy-observability.env.example +23 -0
|
@@ -1,17 +1,37 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Proxy Request Logger
|
|
3
3
|
* Logs proxy request/response metadata to a rotating log file.
|
|
4
|
+
* Also emits OTLP log records to OpenObserve (or any OTLP-compatible backend)
|
|
5
|
+
* when a LoggerProvider is configured via OpenTelemetry instrumentation.
|
|
4
6
|
* Useful for debugging and auditing proxy traffic.
|
|
5
7
|
*/
|
|
6
8
|
import { join } from "path";
|
|
7
9
|
import { homedir } from "os";
|
|
8
10
|
import { logger } from "../utils/logger.js";
|
|
9
|
-
import { chmodSync, existsSync, mkdirSync, readdirSync, statSync, unlinkSync, } from "fs";
|
|
10
|
-
import { appendFile } from "fs/promises";
|
|
11
|
+
import { chmodSync, existsSync, mkdirSync, readdirSync, rmSync, statSync, unlinkSync, } from "fs";
|
|
12
|
+
import { appendFile, writeFile } from "fs/promises";
|
|
13
|
+
import { createHash } from "crypto";
|
|
14
|
+
import { promisify } from "util";
|
|
15
|
+
import { gzip as gzipCallback } from "zlib";
|
|
16
|
+
import { OtelBridge } from "../observability/otelBridge.js";
|
|
17
|
+
import { SeverityNumber } from "@opentelemetry/api-logs";
|
|
11
18
|
let logDir = null;
|
|
12
19
|
let logEnabled = false;
|
|
13
|
-
/**
|
|
14
|
-
|
|
20
|
+
/**
|
|
21
|
+
* Lazily-resolved LoggerProvider from OTel instrumentation.
|
|
22
|
+
* null = not resolved yet (will retry), LoggerProvider = resolved, false = permanently unavailable.
|
|
23
|
+
*/
|
|
24
|
+
let otelLoggerProvider = null;
|
|
25
|
+
/** Number of times we've tried to resolve the LoggerProvider. */
|
|
26
|
+
let otelResolveAttempts = 0;
|
|
27
|
+
/** Max number of resolve attempts before giving up. */
|
|
28
|
+
const MAX_RESOLVE_ATTEMPTS = 10;
|
|
29
|
+
/** Maximum body chunk size emitted to OTLP logs. */
|
|
30
|
+
const BODY_OTLP_CHUNK_SIZE = 16_000;
|
|
31
|
+
/** Maximum redacted body bytes persisted per capture entry. */
|
|
32
|
+
const MAX_CAPTURED_BODY_BYTES = 1024 * 1024;
|
|
33
|
+
const BODY_TRUNCATION_MARKER = "\n...[TRUNCATED]";
|
|
34
|
+
const gzip = promisify(gzipCallback);
|
|
15
35
|
/** Headers whose values must always be redacted. */
|
|
16
36
|
const SENSITIVE_HEADER_NAMES = new Set([
|
|
17
37
|
"authorization",
|
|
@@ -46,6 +66,17 @@ export async function logRequest(entry) {
|
|
|
46
66
|
if (!logEnabled || !logDir) {
|
|
47
67
|
return;
|
|
48
68
|
}
|
|
69
|
+
// Only use OtelBridge if traceId not already provided by caller.
|
|
70
|
+
// Deferred .then() callbacks lose async context, so OtelBridge would
|
|
71
|
+
// return undefined and overwrite the valid traceId the caller passed.
|
|
72
|
+
if (!entry.traceId) {
|
|
73
|
+
const bridge = new OtelBridge();
|
|
74
|
+
const traceCtx = bridge.getCurrentTraceContext();
|
|
75
|
+
if (traceCtx) {
|
|
76
|
+
entry.traceId = traceCtx.traceId;
|
|
77
|
+
entry.spanId = traceCtx.spanId;
|
|
78
|
+
}
|
|
79
|
+
}
|
|
49
80
|
const logFile = join(logDir, `proxy-${new Date().toISOString().split("T")[0]}.jsonl`);
|
|
50
81
|
const line = JSON.stringify(entry) + "\n";
|
|
51
82
|
try {
|
|
@@ -54,6 +85,140 @@ export async function logRequest(entry) {
|
|
|
54
85
|
catch {
|
|
55
86
|
// Non-fatal — don't crash proxy for logging failures
|
|
56
87
|
}
|
|
88
|
+
// Emit OTLP log record (additive — file logging is the primary sink)
|
|
89
|
+
emitOtlpLogRecord(entry);
|
|
90
|
+
}
|
|
91
|
+
/**
|
|
92
|
+
* Log an upstream attempt separately from the final request outcome.
|
|
93
|
+
* Attempt logs are local-only and must not pollute the final request summary
|
|
94
|
+
* or OTLP-derived dashboard panels.
|
|
95
|
+
*/
|
|
96
|
+
export async function logRequestAttempt(entry) {
|
|
97
|
+
if (!logEnabled || !logDir) {
|
|
98
|
+
return;
|
|
99
|
+
}
|
|
100
|
+
if (!entry.traceId) {
|
|
101
|
+
const bridge = new OtelBridge();
|
|
102
|
+
const traceCtx = bridge.getCurrentTraceContext();
|
|
103
|
+
if (traceCtx) {
|
|
104
|
+
entry.traceId = traceCtx.traceId;
|
|
105
|
+
entry.spanId = traceCtx.spanId;
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
const logFile = join(logDir, `proxy-attempts-${new Date().toISOString().split("T")[0]}.jsonl`);
|
|
109
|
+
const line = JSON.stringify(entry) + "\n";
|
|
110
|
+
try {
|
|
111
|
+
await appendFile(logFile, line, { mode: 0o600 });
|
|
112
|
+
}
|
|
113
|
+
catch {
|
|
114
|
+
// Non-fatal — don't crash proxy for logging failures
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
/**
|
|
118
|
+
* Lazily resolve the LoggerProvider from OTel instrumentation.
|
|
119
|
+
* Uses dynamic import to avoid hard dependency — if instrumentation.ts
|
|
120
|
+
* hasn't been loaded or OTLP is not configured, this is a no-op.
|
|
121
|
+
* Retries up to MAX_RESOLVE_ATTEMPTS times to handle race conditions
|
|
122
|
+
* where OTel initialization completes after the first log request.
|
|
123
|
+
*/
|
|
124
|
+
async function resolveLoggerProvider() {
|
|
125
|
+
if (otelLoggerProvider === false) {
|
|
126
|
+
return undefined;
|
|
127
|
+
} // permanently unavailable
|
|
128
|
+
if (otelLoggerProvider !== null) {
|
|
129
|
+
return otelLoggerProvider;
|
|
130
|
+
}
|
|
131
|
+
// Not resolved yet — try to resolve
|
|
132
|
+
otelResolveAttempts++;
|
|
133
|
+
try {
|
|
134
|
+
const { getLoggerProvider } = await import("../services/server/ai/observability/instrumentation.js");
|
|
135
|
+
const provider = getLoggerProvider();
|
|
136
|
+
if (provider) {
|
|
137
|
+
otelLoggerProvider = provider;
|
|
138
|
+
return provider;
|
|
139
|
+
}
|
|
140
|
+
// Provider not available yet — if we've exceeded max attempts, give up
|
|
141
|
+
if (otelResolveAttempts >= MAX_RESOLVE_ATTEMPTS) {
|
|
142
|
+
otelLoggerProvider = false; // permanently unavailable
|
|
143
|
+
}
|
|
144
|
+
// Otherwise leave as null so we retry next time
|
|
145
|
+
return undefined;
|
|
146
|
+
}
|
|
147
|
+
catch {
|
|
148
|
+
// instrumentation.ts not available (e.g. standalone mode) — disable permanently
|
|
149
|
+
otelLoggerProvider = false;
|
|
150
|
+
return undefined;
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
/**
|
|
154
|
+
* Emit a RequestLogEntry as an OTLP log record.
|
|
155
|
+
* Non-blocking, non-fatal — failures are silently swallowed.
|
|
156
|
+
*/
|
|
157
|
+
function emitOtlpLogRecord(entry) {
|
|
158
|
+
resolveLoggerProvider()
|
|
159
|
+
.then((provider) => {
|
|
160
|
+
if (!provider) {
|
|
161
|
+
return;
|
|
162
|
+
}
|
|
163
|
+
const otelLogger = provider.getLogger("neurolink-proxy", "1.0.0");
|
|
164
|
+
// Determine severity based on response status
|
|
165
|
+
const isError = (entry.responseStatus ?? 0) >= 400;
|
|
166
|
+
const isRateLimit = entry.responseStatus === 429;
|
|
167
|
+
const severityNumber = isError
|
|
168
|
+
? isRateLimit
|
|
169
|
+
? SeverityNumber.WARN
|
|
170
|
+
: SeverityNumber.ERROR
|
|
171
|
+
: SeverityNumber.INFO;
|
|
172
|
+
const severityText = isError ? (isRateLimit ? "WARN" : "ERROR") : "INFO";
|
|
173
|
+
otelLogger.emit({
|
|
174
|
+
severityNumber,
|
|
175
|
+
severityText,
|
|
176
|
+
body: `${entry.method} ${entry.path} → ${entry.responseStatus} (${entry.responseTimeMs}ms)`,
|
|
177
|
+
attributes: {
|
|
178
|
+
// Core request fields
|
|
179
|
+
"request.id": entry.requestId,
|
|
180
|
+
"http.method": entry.method,
|
|
181
|
+
"http.path": entry.path,
|
|
182
|
+
"http.status_code": entry.responseStatus,
|
|
183
|
+
"response.time_ms": entry.responseTimeMs,
|
|
184
|
+
// AI-specific fields
|
|
185
|
+
"ai.model": entry.model,
|
|
186
|
+
"ai.stream": entry.stream,
|
|
187
|
+
"ai.tool_count": entry.toolCount,
|
|
188
|
+
// Account info
|
|
189
|
+
"account.name": entry.account,
|
|
190
|
+
"account.type": entry.accountType,
|
|
191
|
+
// Token usage (when available)
|
|
192
|
+
...(entry.inputTokens !== undefined && {
|
|
193
|
+
"ai.input_tokens": entry.inputTokens,
|
|
194
|
+
}),
|
|
195
|
+
...(entry.outputTokens !== undefined && {
|
|
196
|
+
"ai.output_tokens": entry.outputTokens,
|
|
197
|
+
}),
|
|
198
|
+
...(entry.cacheCreationTokens !== undefined && {
|
|
199
|
+
"ai.cache_creation_tokens": entry.cacheCreationTokens,
|
|
200
|
+
}),
|
|
201
|
+
...(entry.cacheReadTokens !== undefined && {
|
|
202
|
+
"ai.cache_read_tokens": entry.cacheReadTokens,
|
|
203
|
+
}),
|
|
204
|
+
// Error info (when present)
|
|
205
|
+
...(entry.errorType && { "error.type": entry.errorType }),
|
|
206
|
+
...(entry.errorMessage && { "error.message": entry.errorMessage }),
|
|
207
|
+
// Trace correlation
|
|
208
|
+
...(entry.traceId && { "trace.id": entry.traceId }),
|
|
209
|
+
...(entry.spanId && { "span.id": entry.spanId }),
|
|
210
|
+
// Derived fields for dashboards (matches backfill script)
|
|
211
|
+
is_success: entry.responseStatus === 200,
|
|
212
|
+
is_rate_limited: entry.responseStatus === 429,
|
|
213
|
+
is_overloaded: entry.responseStatus === 529,
|
|
214
|
+
is_error: isError,
|
|
215
|
+
source: "otlp",
|
|
216
|
+
},
|
|
217
|
+
});
|
|
218
|
+
})
|
|
219
|
+
.catch(() => {
|
|
220
|
+
// Non-fatal — never crash proxy for OTLP log failures
|
|
221
|
+
});
|
|
57
222
|
}
|
|
58
223
|
export function getLogDir() {
|
|
59
224
|
return logDir;
|
|
@@ -78,51 +243,337 @@ function redactHeaders(headers) {
|
|
|
78
243
|
}
|
|
79
244
|
return redacted;
|
|
80
245
|
}
|
|
246
|
+
function serializeBody(body) {
|
|
247
|
+
if (body === undefined || body === null) {
|
|
248
|
+
return undefined;
|
|
249
|
+
}
|
|
250
|
+
return typeof body === "string" ? body : JSON.stringify(body);
|
|
251
|
+
}
|
|
81
252
|
/**
|
|
82
|
-
* Redact sensitive keys from a JSON body string
|
|
253
|
+
* Redact sensitive keys from a JSON body string without truncation.
|
|
83
254
|
*/
|
|
84
255
|
function redactBody(body) {
|
|
85
|
-
|
|
86
|
-
|
|
256
|
+
const str = serializeBody(body);
|
|
257
|
+
if (str === undefined) {
|
|
258
|
+
return undefined;
|
|
87
259
|
}
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
260
|
+
return str.replace(SENSITIVE_BODY_KEYS, '$1"[REDACTED]"');
|
|
261
|
+
}
|
|
262
|
+
function sanitizePhase(phase) {
|
|
263
|
+
return phase.replace(/[^a-zA-Z0-9._-]+/g, "_");
|
|
264
|
+
}
|
|
265
|
+
function sha256(value) {
|
|
266
|
+
return createHash("sha256").update(value).digest("hex");
|
|
267
|
+
}
|
|
268
|
+
function utf8ByteLength(value) {
|
|
269
|
+
return Buffer.byteLength(value, "utf8");
|
|
270
|
+
}
|
|
271
|
+
function truncateUtf8String(input, maxBytes, marker = BODY_TRUNCATION_MARKER) {
|
|
272
|
+
const inputBytes = utf8ByteLength(input);
|
|
273
|
+
if (inputBytes <= maxBytes) {
|
|
274
|
+
return { value: input, bytes: inputBytes, truncated: false };
|
|
275
|
+
}
|
|
276
|
+
const markerBytes = utf8ByteLength(marker);
|
|
277
|
+
if (maxBytes <= markerBytes) {
|
|
278
|
+
return { value: marker, bytes: markerBytes, truncated: true };
|
|
279
|
+
}
|
|
280
|
+
let value = "";
|
|
281
|
+
let bytes = 0;
|
|
282
|
+
for (const char of input) {
|
|
283
|
+
const charBytes = utf8ByteLength(char);
|
|
284
|
+
if (bytes + charBytes + markerBytes > maxBytes) {
|
|
285
|
+
break;
|
|
286
|
+
}
|
|
287
|
+
value += char;
|
|
288
|
+
bytes += charBytes;
|
|
97
289
|
}
|
|
98
|
-
|
|
290
|
+
const truncatedValue = `${value}${marker}`;
|
|
291
|
+
return {
|
|
292
|
+
value: truncatedValue,
|
|
293
|
+
bytes: utf8ByteLength(truncatedValue),
|
|
294
|
+
truncated: true,
|
|
295
|
+
};
|
|
99
296
|
}
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
297
|
+
function splitUtf8StringByBytes(input, maxBytes) {
|
|
298
|
+
if (!input) {
|
|
299
|
+
return [""];
|
|
300
|
+
}
|
|
301
|
+
const chunks = [];
|
|
302
|
+
let currentChunk = "";
|
|
303
|
+
let currentBytes = 0;
|
|
304
|
+
for (const char of input) {
|
|
305
|
+
const charBytes = utf8ByteLength(char);
|
|
306
|
+
if (currentChunk && currentBytes + charBytes > maxBytes) {
|
|
307
|
+
chunks.push(currentChunk);
|
|
308
|
+
currentChunk = char;
|
|
309
|
+
currentBytes = charBytes;
|
|
310
|
+
continue;
|
|
311
|
+
}
|
|
312
|
+
currentChunk += char;
|
|
313
|
+
currentBytes += charBytes;
|
|
314
|
+
}
|
|
315
|
+
if (currentChunk) {
|
|
316
|
+
chunks.push(currentChunk);
|
|
317
|
+
}
|
|
318
|
+
return chunks;
|
|
319
|
+
}
|
|
320
|
+
function prepareRedactedBody(body) {
|
|
321
|
+
const redacted = redactBody(body);
|
|
322
|
+
if (redacted === undefined) {
|
|
323
|
+
return { truncated: false };
|
|
324
|
+
}
|
|
325
|
+
return truncateUtf8String(redacted, MAX_CAPTURED_BODY_BYTES);
|
|
326
|
+
}
|
|
327
|
+
function collectManagedLogFiles(rootDir) {
|
|
328
|
+
const managedFiles = [];
|
|
329
|
+
const walk = (directory) => {
|
|
330
|
+
for (const entry of readdirSync(directory, { withFileTypes: true })) {
|
|
331
|
+
const entryPath = join(directory, entry.name);
|
|
332
|
+
if (entry.isDirectory()) {
|
|
333
|
+
walk(entryPath);
|
|
334
|
+
continue;
|
|
335
|
+
}
|
|
336
|
+
const isTopLevelProxyLog = directory === rootDir &&
|
|
337
|
+
/^proxy(?:-attempts|-debug)?-.*\.jsonl$/.test(entry.name);
|
|
338
|
+
const isBodyArtifact = entry.name.endsWith(".json.gz") &&
|
|
339
|
+
entryPath.includes(`${join(rootDir, "bodies")}`);
|
|
340
|
+
if (!isTopLevelProxyLog && !isBodyArtifact) {
|
|
341
|
+
continue;
|
|
342
|
+
}
|
|
343
|
+
try {
|
|
344
|
+
const stat = statSync(entryPath);
|
|
345
|
+
managedFiles.push({
|
|
346
|
+
path: entryPath,
|
|
347
|
+
mtime: stat.mtimeMs,
|
|
348
|
+
size: stat.size,
|
|
349
|
+
});
|
|
350
|
+
}
|
|
351
|
+
catch {
|
|
352
|
+
// Non-fatal
|
|
353
|
+
}
|
|
354
|
+
}
|
|
355
|
+
};
|
|
356
|
+
walk(rootDir);
|
|
357
|
+
return managedFiles;
|
|
358
|
+
}
|
|
359
|
+
function pruneEmptyDirectories(directory, stopAt) {
|
|
360
|
+
if (!existsSync(directory)) {
|
|
361
|
+
return;
|
|
362
|
+
}
|
|
363
|
+
try {
|
|
364
|
+
const entries = readdirSync(directory, { withFileTypes: true });
|
|
365
|
+
for (const entry of entries) {
|
|
366
|
+
if (entry.isDirectory()) {
|
|
367
|
+
pruneEmptyDirectories(join(directory, entry.name), stopAt);
|
|
368
|
+
}
|
|
369
|
+
}
|
|
370
|
+
if (directory !== stopAt && readdirSync(directory).length === 0) {
|
|
371
|
+
rmSync(directory, { recursive: true, force: true });
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
catch {
|
|
375
|
+
// Non-fatal
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
async function writeBodyArtifact(entry, redactedHeaders, redactedBody, bodyTruncated) {
|
|
379
|
+
if (!logDir || redactedBody === undefined) {
|
|
380
|
+
return {};
|
|
381
|
+
}
|
|
382
|
+
const dateStr = new Date(entry.timestamp).toISOString().split("T")[0];
|
|
383
|
+
const bodyDir = join(logDir, "bodies", dateStr, entry.requestId);
|
|
384
|
+
if (!existsSync(bodyDir)) {
|
|
385
|
+
mkdirSync(bodyDir, { recursive: true, mode: 0o700 });
|
|
386
|
+
}
|
|
387
|
+
chmodSync(bodyDir, 0o700);
|
|
388
|
+
const fileName = `${Date.now()}-${sanitizePhase(entry.phase)}` +
|
|
389
|
+
(entry.attempt !== undefined ? `-attempt-${entry.attempt}` : "") +
|
|
390
|
+
`.json.gz`;
|
|
391
|
+
const bodyPath = join(bodyDir, fileName);
|
|
392
|
+
const payload = JSON.stringify({
|
|
393
|
+
timestamp: entry.timestamp,
|
|
394
|
+
requestId: entry.requestId,
|
|
395
|
+
phase: entry.phase,
|
|
396
|
+
model: entry.model,
|
|
397
|
+
stream: entry.stream,
|
|
398
|
+
account: entry.account,
|
|
399
|
+
accountType: entry.accountType,
|
|
400
|
+
attempt: entry.attempt,
|
|
401
|
+
responseStatus: entry.responseStatus,
|
|
402
|
+
durationMs: entry.durationMs,
|
|
403
|
+
contentType: entry.contentType,
|
|
404
|
+
headers: redactedHeaders,
|
|
405
|
+
body: redactedBody,
|
|
406
|
+
traceId: entry.traceId,
|
|
407
|
+
spanId: entry.spanId,
|
|
408
|
+
metadata: entry.metadata,
|
|
409
|
+
});
|
|
410
|
+
const compressed = await gzip(payload);
|
|
411
|
+
await writeFile(bodyPath, compressed, { mode: 0o600 });
|
|
412
|
+
return {
|
|
413
|
+
bodyPath,
|
|
414
|
+
bodySha256: sha256(redactedBody),
|
|
415
|
+
redactedBodyBytes: utf8ByteLength(redactedBody),
|
|
416
|
+
storedFileBytes: compressed.byteLength,
|
|
417
|
+
redactedBody,
|
|
418
|
+
bodyTruncated,
|
|
419
|
+
};
|
|
420
|
+
}
|
|
421
|
+
function emitOtlpBodyLogRecord(entry, stored) {
|
|
422
|
+
resolveLoggerProvider()
|
|
423
|
+
.then((provider) => {
|
|
424
|
+
if (!provider || stored.redactedBody === undefined) {
|
|
425
|
+
return;
|
|
426
|
+
}
|
|
427
|
+
const otelLogger = provider.getLogger("neurolink-proxy-bodies", "1.0.0");
|
|
428
|
+
const chunks = splitUtf8StringByBytes(stored.redactedBody, BODY_OTLP_CHUNK_SIZE);
|
|
429
|
+
const totalChunks = Math.max(1, chunks.length);
|
|
430
|
+
for (let chunkIndex = 0; chunkIndex < totalChunks; chunkIndex++) {
|
|
431
|
+
const chunk = chunks[chunkIndex] ?? "";
|
|
432
|
+
otelLogger.emit({
|
|
433
|
+
severityNumber: (entry.responseStatus ?? 0) >= 400
|
|
434
|
+
? SeverityNumber.WARN
|
|
435
|
+
: SeverityNumber.INFO,
|
|
436
|
+
severityText: (entry.responseStatus ?? 0) >= 400 ? "WARN" : "INFO",
|
|
437
|
+
body: chunk,
|
|
438
|
+
attributes: {
|
|
439
|
+
"event.name": "proxy.body_capture",
|
|
440
|
+
"request.id": entry.requestId,
|
|
441
|
+
"body.phase": entry.phase,
|
|
442
|
+
"body.chunk_index": chunkIndex,
|
|
443
|
+
"body.chunk_count": totalChunks,
|
|
444
|
+
"body.content_type": entry.contentType ?? "application/json",
|
|
445
|
+
"ai.model": entry.model,
|
|
446
|
+
"ai.stream": entry.stream,
|
|
447
|
+
...(entry.account && { "account.name": entry.account }),
|
|
448
|
+
...(entry.accountType && { "account.type": entry.accountType }),
|
|
449
|
+
...(entry.attempt !== undefined && {
|
|
450
|
+
"proxy.attempt": entry.attempt,
|
|
451
|
+
}),
|
|
452
|
+
...(entry.responseStatus !== undefined && {
|
|
453
|
+
"http.status_code": entry.responseStatus,
|
|
454
|
+
}),
|
|
455
|
+
...(entry.durationMs !== undefined && {
|
|
456
|
+
"response.time_ms": entry.durationMs,
|
|
457
|
+
}),
|
|
458
|
+
...(stored.bodySha256 && { "body.sha256": stored.bodySha256 }),
|
|
459
|
+
...(stored.bodyPath && {
|
|
460
|
+
"body.path": stored.bodyPath.split("/").slice(-2).join("/"),
|
|
461
|
+
}),
|
|
462
|
+
...(stored.redactedBodyBytes !== undefined && {
|
|
463
|
+
"body.bytes": stored.redactedBodyBytes,
|
|
464
|
+
}),
|
|
465
|
+
...(stored.bodyTruncated !== undefined && {
|
|
466
|
+
"body.truncated": stored.bodyTruncated,
|
|
467
|
+
}),
|
|
468
|
+
...(entry.traceId && { "trace.id": entry.traceId }),
|
|
469
|
+
...(entry.spanId && { "span.id": entry.spanId }),
|
|
470
|
+
...(entry.metadata && {
|
|
471
|
+
"body.metadata_json": JSON.stringify(entry.metadata),
|
|
472
|
+
}),
|
|
473
|
+
source: "otlp",
|
|
474
|
+
},
|
|
475
|
+
});
|
|
476
|
+
}
|
|
477
|
+
})
|
|
478
|
+
.catch(() => {
|
|
479
|
+
// Non-fatal — never crash proxy for OTLP log failures
|
|
480
|
+
});
|
|
481
|
+
}
|
|
482
|
+
export async function logBodyCapture(entry) {
|
|
108
483
|
if (!logEnabled || !logDir) {
|
|
109
484
|
return;
|
|
110
485
|
}
|
|
111
|
-
const
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
486
|
+
const bridge = new OtelBridge();
|
|
487
|
+
const traceCtx = entry.traceId && entry.spanId
|
|
488
|
+
? { traceId: entry.traceId, spanId: entry.spanId }
|
|
489
|
+
: bridge.getCurrentTraceContext();
|
|
490
|
+
const redactedHeaders = redactHeaders(entry.headers);
|
|
491
|
+
const preparedBody = prepareRedactedBody(entry.body);
|
|
492
|
+
let stored = {};
|
|
493
|
+
try {
|
|
494
|
+
stored = await writeBodyArtifact(entry, redactedHeaders, preparedBody.value, preparedBody.truncated);
|
|
495
|
+
}
|
|
496
|
+
catch {
|
|
497
|
+
// Best-effort artifact persistence; continue with in-memory metadata only.
|
|
498
|
+
}
|
|
499
|
+
const dateStr = new Date(entry.timestamp).toISOString().split("T")[0];
|
|
500
|
+
const logFile = join(logDir, `proxy-debug-${dateStr}.jsonl`);
|
|
501
|
+
const indexEntry = {
|
|
502
|
+
timestamp: entry.timestamp,
|
|
503
|
+
type: "body_capture",
|
|
504
|
+
requestId: entry.requestId,
|
|
505
|
+
phase: entry.phase,
|
|
506
|
+
model: entry.model,
|
|
507
|
+
stream: entry.stream,
|
|
508
|
+
headers: redactedHeaders,
|
|
509
|
+
contentType: entry.contentType,
|
|
510
|
+
responseStatus: entry.responseStatus,
|
|
511
|
+
durationMs: entry.durationMs,
|
|
512
|
+
account: entry.account,
|
|
513
|
+
accountType: entry.accountType,
|
|
514
|
+
attempt: entry.attempt,
|
|
515
|
+
bodyPath: stored.bodyPath,
|
|
516
|
+
bodySha256: stored.bodySha256,
|
|
517
|
+
observedBodyBytes: entry.bodySize,
|
|
518
|
+
redactedBodyBytes: stored.redactedBodyBytes ?? preparedBody.bytes,
|
|
519
|
+
storedFileBytes: stored.storedFileBytes,
|
|
520
|
+
bodyTruncated: stored.bodyTruncated ?? preparedBody.truncated,
|
|
521
|
+
metadata: entry.metadata,
|
|
117
522
|
};
|
|
118
|
-
|
|
119
|
-
|
|
523
|
+
if (traceCtx) {
|
|
524
|
+
indexEntry.traceId = traceCtx.traceId;
|
|
525
|
+
indexEntry.spanId = traceCtx.spanId;
|
|
526
|
+
}
|
|
120
527
|
try {
|
|
121
|
-
await appendFile(logFile,
|
|
528
|
+
await appendFile(logFile, JSON.stringify(indexEntry) + "\n", {
|
|
529
|
+
mode: 0o600,
|
|
530
|
+
});
|
|
122
531
|
}
|
|
123
532
|
catch {
|
|
124
533
|
// Non-fatal
|
|
125
534
|
}
|
|
535
|
+
emitOtlpBodyLogRecord({
|
|
536
|
+
...entry,
|
|
537
|
+
traceId: traceCtx?.traceId ?? entry.traceId,
|
|
538
|
+
spanId: traceCtx?.spanId ?? entry.spanId,
|
|
539
|
+
}, stored);
|
|
540
|
+
}
|
|
541
|
+
/**
|
|
542
|
+
* Log the FULL raw request and response for debugging.
|
|
543
|
+
* Legacy helper kept for compatibility. New call sites should prefer
|
|
544
|
+
* logBodyCapture() so each phase can be indexed and persisted separately.
|
|
545
|
+
*/
|
|
546
|
+
export async function logFullRequestResponse(entry) {
|
|
547
|
+
await Promise.all([
|
|
548
|
+
logBodyCapture({
|
|
549
|
+
timestamp: entry.timestamp,
|
|
550
|
+
requestId: entry.requestId,
|
|
551
|
+
phase: "legacy_upstream_request",
|
|
552
|
+
model: entry.model,
|
|
553
|
+
stream: entry.stream,
|
|
554
|
+
headers: entry.requestHeaders,
|
|
555
|
+
body: entry.requestBody,
|
|
556
|
+
bodySize: entry.requestBodySize,
|
|
557
|
+
contentType: entry.requestHeaders["content-type"] ?? "application/json",
|
|
558
|
+
account: entry.account,
|
|
559
|
+
responseStatus: entry.responseStatus,
|
|
560
|
+
durationMs: entry.durationMs,
|
|
561
|
+
}),
|
|
562
|
+
logBodyCapture({
|
|
563
|
+
timestamp: entry.timestamp,
|
|
564
|
+
requestId: entry.requestId,
|
|
565
|
+
phase: "legacy_upstream_response",
|
|
566
|
+
model: entry.model,
|
|
567
|
+
stream: entry.stream,
|
|
568
|
+
headers: entry.responseHeaders,
|
|
569
|
+
body: entry.responseBody,
|
|
570
|
+
bodySize: entry.responseBodySize,
|
|
571
|
+
contentType: entry.responseHeaders?.["content-type"] ?? "application/json",
|
|
572
|
+
account: entry.account,
|
|
573
|
+
responseStatus: entry.responseStatus,
|
|
574
|
+
durationMs: entry.durationMs,
|
|
575
|
+
}),
|
|
576
|
+
]);
|
|
126
577
|
}
|
|
127
578
|
/**
|
|
128
579
|
* Log a mid-stream error that occurs after the initial 200 was sent.
|
|
@@ -132,6 +583,8 @@ export async function logStreamError(entry) {
|
|
|
132
583
|
if (!logEnabled || !logDir) {
|
|
133
584
|
return;
|
|
134
585
|
}
|
|
586
|
+
const bridge = new OtelBridge();
|
|
587
|
+
const traceCtx = bridge.getCurrentTraceContext();
|
|
135
588
|
const logFile = join(logDir, `proxy-${new Date().toISOString().split("T")[0]}.jsonl`);
|
|
136
589
|
const logEntry = {
|
|
137
590
|
...entry,
|
|
@@ -139,6 +592,10 @@ export async function logStreamError(entry) {
|
|
|
139
592
|
errorType: "stream_error",
|
|
140
593
|
note: "mid-stream failure after initial 200",
|
|
141
594
|
};
|
|
595
|
+
if (traceCtx) {
|
|
596
|
+
logEntry.traceId = traceCtx.traceId;
|
|
597
|
+
logEntry.spanId = traceCtx.spanId;
|
|
598
|
+
}
|
|
142
599
|
try {
|
|
143
600
|
await appendFile(logFile, JSON.stringify(logEntry) + "\n", {
|
|
144
601
|
mode: 0o600,
|
|
@@ -159,19 +616,8 @@ export function cleanupLogs(maxAgeDays = 7, maxSizeMb = 500) {
|
|
|
159
616
|
return;
|
|
160
617
|
}
|
|
161
618
|
try {
|
|
162
|
-
const
|
|
163
|
-
|
|
164
|
-
.map((f) => {
|
|
165
|
-
const filePath = join(logDir, f);
|
|
166
|
-
const stat = statSync(filePath);
|
|
167
|
-
return {
|
|
168
|
-
name: f,
|
|
169
|
-
path: filePath,
|
|
170
|
-
mtime: stat.mtimeMs,
|
|
171
|
-
size: stat.size,
|
|
172
|
-
};
|
|
173
|
-
})
|
|
174
|
-
.sort((a, b) => a.mtime - b.mtime); // oldest first
|
|
619
|
+
const activeLogDir = logDir;
|
|
620
|
+
const files = collectManagedLogFiles(activeLogDir).sort((a, b) => a.mtime - b.mtime); // oldest first
|
|
175
621
|
const cutoff = Date.now() - maxAgeDays * 24 * 60 * 60 * 1000;
|
|
176
622
|
let deletedCount = 0;
|
|
177
623
|
let freedBytes = 0;
|
|
@@ -187,16 +633,26 @@ export function cleanupLogs(maxAgeDays = 7, maxSizeMb = 500) {
|
|
|
187
633
|
remaining.push(file);
|
|
188
634
|
}
|
|
189
635
|
}
|
|
636
|
+
const bodiesDir = join(logDir, "bodies");
|
|
637
|
+
if (existsSync(bodiesDir)) {
|
|
638
|
+
pruneEmptyDirectories(bodiesDir, bodiesDir);
|
|
639
|
+
}
|
|
190
640
|
// Pass 2: if total size exceeds maxSizeMb, delete oldest until under limit
|
|
191
641
|
const maxBytes = maxSizeMb * 1024 * 1024;
|
|
192
642
|
let totalSize = remaining.reduce((sum, f) => sum + f.size, 0);
|
|
193
643
|
while (totalSize > maxBytes && remaining.length > 0) {
|
|
194
644
|
const oldest = remaining.shift();
|
|
645
|
+
if (!oldest) {
|
|
646
|
+
break;
|
|
647
|
+
}
|
|
195
648
|
unlinkSync(oldest.path);
|
|
196
649
|
totalSize -= oldest.size;
|
|
197
650
|
deletedCount++;
|
|
198
651
|
freedBytes += oldest.size;
|
|
199
652
|
}
|
|
653
|
+
if (existsSync(bodiesDir)) {
|
|
654
|
+
pruneEmptyDirectories(bodiesDir, bodiesDir);
|
|
655
|
+
}
|
|
200
656
|
if (deletedCount > 0) {
|
|
201
657
|
logger.info(`[proxy] log cleanup: deleted ${deletedCount} file(s), freed ${(freedBytes / 1024 / 1024).toFixed(1)} MB`);
|
|
202
658
|
}
|