@juspay/neurolink 12.14.3 → 12.14.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +2 -2
- package/dist/browser/neurolink.min.js +370 -370
- package/dist/cli/commands/proxy.js +11 -3
- package/dist/mcp/caching/toolCache.d.ts +30 -0
- package/dist/mcp/caching/toolCache.js +52 -3
- package/dist/proxy/bodyCaptureProcessing.d.ts +1 -0
- package/dist/proxy/bodyCaptureProcessing.js +19 -4
- package/dist/proxy/bodyCaptureWorker.js +35 -13
- package/dist/proxy/otelLogSink.d.ts +22 -1
- package/dist/proxy/otelLogSink.js +247 -8
- package/dist/proxy/proxyActivity.d.ts +4 -2
- package/dist/proxy/proxyActivity.js +20 -1
- package/dist/proxy/requestLogger.d.ts +1 -0
- package/dist/proxy/requestLogger.js +53 -15
- package/dist/types/proxy.d.ts +29 -0
- package/docs-site/static/search-index.json +1 -1
- package/package.json +1 -1
- package/scripts/observability/check-proxy-telemetry.mjs +20 -7
- package/scripts/observability/query-proxy-history.mjs +193 -0
|
@@ -1,20 +1,40 @@
|
|
|
1
1
|
/* eslint-disable no-console -- This proxy-only sink replaces console methods with OTLP emission. */
|
|
2
2
|
import { inspect } from "node:util";
|
|
3
|
+
import { setImmediate as yieldToRequests } from "node:timers/promises";
|
|
3
4
|
import { SeverityNumber } from "@opentelemetry/api-logs";
|
|
4
5
|
import { ExportResultCode } from "@opentelemetry/core";
|
|
5
6
|
import { OTLPLogExporter } from "@opentelemetry/exporter-logs-otlp-http";
|
|
6
7
|
import { resourceFromAttributes } from "@opentelemetry/resources";
|
|
7
8
|
import { BatchLogRecordProcessor, LoggerProvider, } from "@opentelemetry/sdk-logs";
|
|
8
9
|
import { sanitizeForLog } from "../utils/logSanitize.js";
|
|
10
|
+
import { splitUtf8StringByBytes } from "./bodyCaptureProcessing.js";
|
|
9
11
|
let provider;
|
|
10
12
|
let restoreConsole;
|
|
11
13
|
const queues = [];
|
|
14
|
+
const bodyPublications = new Map();
|
|
15
|
+
let bodyPublicationChain = Promise.resolve();
|
|
16
|
+
let shuttingDown = false;
|
|
17
|
+
const bodyDelivery = {
|
|
18
|
+
attempted: 0,
|
|
19
|
+
transportAcknowledged: 0,
|
|
20
|
+
exportUnconfirmed: 0,
|
|
21
|
+
rejected: 0,
|
|
22
|
+
partial: 0,
|
|
23
|
+
pending: 0,
|
|
24
|
+
pendingBytes: 0,
|
|
25
|
+
highWaterPending: 0,
|
|
26
|
+
highWaterBytes: 0,
|
|
27
|
+
maxPending: 16,
|
|
28
|
+
maxPendingBytes: 32 * 1024 * 1024,
|
|
29
|
+
};
|
|
12
30
|
/** Explicit opt-in; configuration never silently falls back to file logging. */
|
|
13
31
|
export function isProxyOtelOnly() {
|
|
14
32
|
return process.env.NEUROLINK_PROXY_LOG_SINK === "otel";
|
|
15
33
|
}
|
|
16
34
|
/** Reserve capacity including exports in flight, independently for metadata and bodies. */
|
|
17
|
-
function createTrackedProcessor(url, capacity) {
|
|
35
|
+
function createTrackedProcessor(url, capacity, kind) {
|
|
36
|
+
const unsettled = new Set();
|
|
37
|
+
const flushWaiters = new Set();
|
|
18
38
|
const state = {
|
|
19
39
|
attempted: 0,
|
|
20
40
|
submitted: 0,
|
|
@@ -24,16 +44,23 @@ function createTrackedProcessor(url, capacity) {
|
|
|
24
44
|
outstanding: 0,
|
|
25
45
|
lastAcknowledgedAt: undefined,
|
|
26
46
|
lastFailureAt: undefined,
|
|
47
|
+
highWaterOutstanding: 0,
|
|
27
48
|
};
|
|
28
49
|
const transport = new OTLPLogExporter({ url, timeoutMillis: 5000 });
|
|
29
50
|
const exporter = {
|
|
30
51
|
export(records, callback) {
|
|
31
52
|
let settled = false;
|
|
53
|
+
const deadline = setTimeout(() => settle({
|
|
54
|
+
code: ExportResultCode.FAILED,
|
|
55
|
+
error: new Error("OTLP export callback deadline exceeded"),
|
|
56
|
+
}), 6_000);
|
|
57
|
+
deadline.unref();
|
|
32
58
|
const settle = (result) => {
|
|
33
59
|
if (settled) {
|
|
34
60
|
return;
|
|
35
61
|
}
|
|
36
62
|
settled = true;
|
|
63
|
+
clearTimeout(deadline);
|
|
37
64
|
state.outstanding -= records.length;
|
|
38
65
|
if (result.code === ExportResultCode.SUCCESS) {
|
|
39
66
|
state.transportAcknowledged += records.length;
|
|
@@ -43,6 +70,23 @@ function createTrackedProcessor(url, capacity) {
|
|
|
43
70
|
state.exportUnconfirmed += records.length;
|
|
44
71
|
state.lastFailureAt = new Date().toISOString();
|
|
45
72
|
}
|
|
73
|
+
for (const record of records) {
|
|
74
|
+
unsettled.delete(record);
|
|
75
|
+
const id = record.attributes?.["body.capture_id"];
|
|
76
|
+
const publication = typeof id === "string" ? bodyPublications.get(id) : undefined;
|
|
77
|
+
if (publication) {
|
|
78
|
+
if (result.code === ExportResultCode.SUCCESS) {
|
|
79
|
+
publication.acknowledged++;
|
|
80
|
+
}
|
|
81
|
+
else {
|
|
82
|
+
publication.unconfirmed++;
|
|
83
|
+
}
|
|
84
|
+
publication.notify?.();
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
for (const notify of flushWaiters) {
|
|
88
|
+
notify();
|
|
89
|
+
}
|
|
46
90
|
callback(result);
|
|
47
91
|
};
|
|
48
92
|
try {
|
|
@@ -66,18 +110,193 @@ function createTrackedProcessor(url, capacity) {
|
|
|
66
110
|
const processor = {
|
|
67
111
|
onEmit(record) {
|
|
68
112
|
state.attempted++;
|
|
113
|
+
const id = record.attributes?.["body.capture_id"];
|
|
114
|
+
const publication = typeof id === "string" ? bodyPublications.get(id) : undefined;
|
|
115
|
+
if (publication) {
|
|
116
|
+
publication.emitted++;
|
|
117
|
+
}
|
|
69
118
|
if (state.outstanding >= capacity) {
|
|
70
119
|
state.dropped++;
|
|
120
|
+
if (publication) {
|
|
121
|
+
publication.dropped++;
|
|
122
|
+
publication.notify?.();
|
|
123
|
+
}
|
|
71
124
|
return;
|
|
72
125
|
}
|
|
73
126
|
state.submitted++;
|
|
74
127
|
state.outstanding++;
|
|
128
|
+
unsettled.add(record);
|
|
129
|
+
state.highWaterOutstanding = Math.max(state.highWaterOutstanding, state.outstanding);
|
|
75
130
|
batch.onEmit(record);
|
|
76
131
|
},
|
|
77
|
-
forceFlush: () =>
|
|
78
|
-
|
|
132
|
+
forceFlush: async () => {
|
|
133
|
+
const boundary = new Set(unsettled);
|
|
134
|
+
try {
|
|
135
|
+
await batch.forceFlush();
|
|
136
|
+
}
|
|
137
|
+
finally {
|
|
138
|
+
await new Promise((resolve) => {
|
|
139
|
+
const check = () => {
|
|
140
|
+
if (![...boundary].some((record) => unsettled.has(record))) {
|
|
141
|
+
flushWaiters.delete(check);
|
|
142
|
+
resolve();
|
|
143
|
+
}
|
|
144
|
+
};
|
|
145
|
+
flushWaiters.add(check);
|
|
146
|
+
check();
|
|
147
|
+
});
|
|
148
|
+
}
|
|
149
|
+
},
|
|
150
|
+
shutdown: async () => {
|
|
151
|
+
try {
|
|
152
|
+
await processor.forceFlush();
|
|
153
|
+
}
|
|
154
|
+
finally {
|
|
155
|
+
await batch.shutdown();
|
|
156
|
+
}
|
|
157
|
+
},
|
|
79
158
|
};
|
|
80
|
-
return { state, processor, capacity };
|
|
159
|
+
return { state, processor, capacity, kind };
|
|
160
|
+
}
|
|
161
|
+
/**
|
|
162
|
+
* Own a whole capture within byte/count bounds, then pace its chunks by actual
|
|
163
|
+
* export callbacks. SDK forceFlush alone does not await an automatic export
|
|
164
|
+
* already in flight. Serial publication prevents bursts from dropping tails.
|
|
165
|
+
*/
|
|
166
|
+
export async function publishProxyOtelBody(captureId, body, emit) {
|
|
167
|
+
bodyDelivery.attempted++;
|
|
168
|
+
const bytes = Buffer.byteLength(body, "utf8");
|
|
169
|
+
if (!provider ||
|
|
170
|
+
shuttingDown ||
|
|
171
|
+
bodyPublications.has(captureId) ||
|
|
172
|
+
bodyDelivery.pending >= bodyDelivery.maxPending ||
|
|
173
|
+
bodyDelivery.pendingBytes + bytes > bodyDelivery.maxPendingBytes) {
|
|
174
|
+
bodyDelivery.rejected++;
|
|
175
|
+
return {
|
|
176
|
+
status: "rejected",
|
|
177
|
+
acknowledgedChunks: 0,
|
|
178
|
+
unconfirmedChunks: 0,
|
|
179
|
+
droppedChunks: 0,
|
|
180
|
+
reason: bodyPublications.has(captureId)
|
|
181
|
+
? "body_capture_id_in_use"
|
|
182
|
+
: !provider || shuttingDown
|
|
183
|
+
? "body_exporter_unavailable"
|
|
184
|
+
: "body_publication_queue_full",
|
|
185
|
+
};
|
|
186
|
+
}
|
|
187
|
+
bodyDelivery.pending++;
|
|
188
|
+
bodyDelivery.pendingBytes += bytes;
|
|
189
|
+
bodyDelivery.highWaterPending = Math.max(bodyDelivery.highWaterPending, bodyDelivery.pending);
|
|
190
|
+
bodyDelivery.highWaterBytes = Math.max(bodyDelivery.highWaterBytes, bodyDelivery.pendingBytes);
|
|
191
|
+
const progress = {
|
|
192
|
+
acknowledged: 0,
|
|
193
|
+
unconfirmed: 0,
|
|
194
|
+
dropped: 0,
|
|
195
|
+
emitted: 0,
|
|
196
|
+
};
|
|
197
|
+
bodyPublications.set(captureId, progress);
|
|
198
|
+
const deadline = Date.now() + 20_000;
|
|
199
|
+
const operation = bodyPublicationChain.then(async () => {
|
|
200
|
+
const queue = queues.find((candidate) => candidate.kind === "bodies");
|
|
201
|
+
if (!queue || Date.now() >= deadline) {
|
|
202
|
+
return {
|
|
203
|
+
status: "rejected",
|
|
204
|
+
acknowledgedChunks: 0,
|
|
205
|
+
unconfirmedChunks: 0,
|
|
206
|
+
droppedChunks: 0,
|
|
207
|
+
reason: !queue
|
|
208
|
+
? "body_exporter_unavailable"
|
|
209
|
+
: "body_publication_deadline",
|
|
210
|
+
};
|
|
211
|
+
}
|
|
212
|
+
const chunks = splitUtf8StringByBytes(body, 16_000);
|
|
213
|
+
const awaitSettlement = () => new Promise((resolve) => {
|
|
214
|
+
const check = () => {
|
|
215
|
+
if (progress.acknowledged + progress.unconfirmed + progress.dropped >=
|
|
216
|
+
progress.emitted) {
|
|
217
|
+
progress.notify = undefined;
|
|
218
|
+
resolve();
|
|
219
|
+
}
|
|
220
|
+
};
|
|
221
|
+
progress.notify = check;
|
|
222
|
+
check();
|
|
223
|
+
});
|
|
224
|
+
let reason;
|
|
225
|
+
try {
|
|
226
|
+
for (let offset = 0; offset < chunks.length; offset += 64) {
|
|
227
|
+
if (Date.now() >= deadline) {
|
|
228
|
+
reason = "body_publication_deadline";
|
|
229
|
+
break;
|
|
230
|
+
}
|
|
231
|
+
// Ordinary body records from an external logger may share this queue.
|
|
232
|
+
// Wait for them before admitting any part of this batch.
|
|
233
|
+
while (queue.state.outstanding > queue.capacity - 64) {
|
|
234
|
+
await queue.processor.forceFlush();
|
|
235
|
+
if (Date.now() >= deadline) {
|
|
236
|
+
throw new Error("body_publication_deadline");
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
for (let i = offset; i < Math.min(offset + 64, chunks.length); i++) {
|
|
240
|
+
emit(chunks[i], i, chunks.length);
|
|
241
|
+
}
|
|
242
|
+
await queue.processor.forceFlush();
|
|
243
|
+
await awaitSettlement();
|
|
244
|
+
if (progress.unconfirmed || progress.dropped) {
|
|
245
|
+
reason = "body_export_unconfirmed";
|
|
246
|
+
break;
|
|
247
|
+
}
|
|
248
|
+
await yieldToRequests();
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
catch (error) {
|
|
252
|
+
reason =
|
|
253
|
+
error instanceof Error &&
|
|
254
|
+
error.message === "body_publication_deadline"
|
|
255
|
+
? "body_publication_deadline"
|
|
256
|
+
: "body_publication_failed";
|
|
257
|
+
// Retain ownership of already submitted chunks until their callbacks settle.
|
|
258
|
+
await queue.processor.forceFlush().catch(() => undefined);
|
|
259
|
+
await awaitSettlement();
|
|
260
|
+
}
|
|
261
|
+
const status = progress.emitted === 0 && chunks.length > 0
|
|
262
|
+
? "rejected"
|
|
263
|
+
: progress.dropped || progress.emitted !== chunks.length
|
|
264
|
+
? "partial"
|
|
265
|
+
: progress.unconfirmed
|
|
266
|
+
? "export_unconfirmed"
|
|
267
|
+
: "transport_acknowledged";
|
|
268
|
+
return {
|
|
269
|
+
status,
|
|
270
|
+
expectedChunks: chunks.length,
|
|
271
|
+
acknowledgedChunks: progress.acknowledged,
|
|
272
|
+
unconfirmedChunks: progress.unconfirmed,
|
|
273
|
+
droppedChunks: progress.dropped,
|
|
274
|
+
notSubmittedChunks: chunks.length - progress.emitted,
|
|
275
|
+
...(reason ? { reason } : {}),
|
|
276
|
+
};
|
|
277
|
+
});
|
|
278
|
+
bodyPublicationChain = operation.then(() => undefined, () => undefined);
|
|
279
|
+
try {
|
|
280
|
+
const result = await operation;
|
|
281
|
+
if (result.status === "transport_acknowledged") {
|
|
282
|
+
bodyDelivery.transportAcknowledged++;
|
|
283
|
+
}
|
|
284
|
+
else if (result.status === "export_unconfirmed") {
|
|
285
|
+
bodyDelivery.exportUnconfirmed++;
|
|
286
|
+
}
|
|
287
|
+
else if (result.status === "rejected") {
|
|
288
|
+
bodyDelivery.rejected++;
|
|
289
|
+
}
|
|
290
|
+
else {
|
|
291
|
+
bodyDelivery.partial++;
|
|
292
|
+
}
|
|
293
|
+
return result;
|
|
294
|
+
}
|
|
295
|
+
finally {
|
|
296
|
+
bodyPublications.delete(captureId);
|
|
297
|
+
bodyDelivery.pending--;
|
|
298
|
+
bodyDelivery.pendingBytes -= bytes;
|
|
299
|
+
}
|
|
81
300
|
}
|
|
82
301
|
/** Initialize a log-only provider in every proxy process, including the supervisor. */
|
|
83
302
|
export function initializeProxyOtelLogs(role = "worker") {
|
|
@@ -102,8 +321,8 @@ export function initializeProxyOtelLogs(role = "worker") {
|
|
|
102
321
|
if (url.protocol === "http:" && !loopback) {
|
|
103
322
|
throw new Error("Proxy OTLP logs require HTTPS for non-loopback collectors");
|
|
104
323
|
}
|
|
105
|
-
const metadata = createTrackedProcessor(endpoint, 2048);
|
|
106
|
-
const bodies = createTrackedProcessor(endpoint, 256);
|
|
324
|
+
const metadata = createTrackedProcessor(endpoint, 2048, "metadata");
|
|
325
|
+
const bodies = createTrackedProcessor(endpoint, 256, "bodies");
|
|
107
326
|
queues.push(metadata, bodies);
|
|
108
327
|
provider = new LoggerProvider({
|
|
109
328
|
resource: resourceFromAttributes({
|
|
@@ -220,8 +439,9 @@ export function getProxyOtelLogSnapshot() {
|
|
|
220
439
|
initialized: provider !== undefined,
|
|
221
440
|
deliveryGuarantee: "best-effort; HTTP success is not per-record acceptance or backend persistence",
|
|
222
441
|
invalidRecords,
|
|
223
|
-
|
|
224
|
-
|
|
442
|
+
bodyDelivery: { ...bodyDelivery },
|
|
443
|
+
queues: queues.map((q) => ({
|
|
444
|
+
kind: q.kind,
|
|
225
445
|
capacity: q.capacity,
|
|
226
446
|
...q.state,
|
|
227
447
|
})),
|
|
@@ -229,16 +449,35 @@ export function getProxyOtelLogSnapshot() {
|
|
|
229
449
|
}
|
|
230
450
|
/** Bounded provider flush belongs after final request and lifecycle publication. */
|
|
231
451
|
export async function flushProxyOtelLogs() {
|
|
452
|
+
await bodyPublicationChain;
|
|
232
453
|
await provider?.forceFlush();
|
|
233
454
|
}
|
|
234
455
|
/** Release this process's exporter and restore console ownership. */
|
|
235
456
|
export async function shutdownProxyOtelLogs() {
|
|
457
|
+
shuttingDown = true;
|
|
458
|
+
await bodyPublicationChain;
|
|
236
459
|
restoreConsole?.();
|
|
237
460
|
restoreConsole = undefined;
|
|
238
461
|
await provider?.shutdown();
|
|
239
462
|
provider = undefined;
|
|
240
463
|
queues.length = 0;
|
|
241
464
|
invalidRecords = 0;
|
|
465
|
+
bodyPublications.clear();
|
|
466
|
+
bodyPublicationChain = Promise.resolve();
|
|
467
|
+
shuttingDown = false;
|
|
468
|
+
for (const key of [
|
|
469
|
+
"attempted",
|
|
470
|
+
"transportAcknowledged",
|
|
471
|
+
"exportUnconfirmed",
|
|
472
|
+
"rejected",
|
|
473
|
+
"partial",
|
|
474
|
+
"pending",
|
|
475
|
+
"pendingBytes",
|
|
476
|
+
"highWaterPending",
|
|
477
|
+
"highWaterBytes",
|
|
478
|
+
]) {
|
|
479
|
+
bodyDelivery[key] = 0;
|
|
480
|
+
}
|
|
242
481
|
}
|
|
243
482
|
/** Flush short-lived proxy command diagnostics on every normal return or exception. */
|
|
244
483
|
export function withProxyOtelLogShutdown(handler) {
|
|
@@ -1,7 +1,9 @@
|
|
|
1
|
-
import type { ProxyActivitySnapshot, ProxyResponseTrackingObserver, RequestLogEntry } from "../types/index.js";
|
|
1
|
+
import type { ProxyActivitySnapshot, ProxyResponseTrackingObserver, RequestLogEntry, RequestAttemptLogEntry } from "../types/index.js";
|
|
2
2
|
/** Join route accounting to the HTTP lifecycle without relying on write order. */
|
|
3
|
-
export declare function observeProxyFinalLog(requestId: string, observer: (entry: RequestLogEntry) => void): () => void;
|
|
3
|
+
export declare function observeProxyFinalLog(requestId: string, observer: (entry: RequestLogEntry) => void, attemptObserver?: (entry: RequestAttemptLogEntry) => void): () => void;
|
|
4
4
|
export declare function notifyProxyFinalLog(entry: RequestLogEntry): void;
|
|
5
|
+
/** Retain only the last attempt for a currently observed HTTP request. */
|
|
6
|
+
export declare function notifyProxyAttemptLog(entry: RequestAttemptLogEntry): void;
|
|
5
7
|
export declare function registerProxyResponseObserver(metadata: object, observer: ProxyResponseTrackingObserver): void;
|
|
6
8
|
export declare function takeProxyResponseObservers(metadata: object): ProxyResponseTrackingObserver[];
|
|
7
9
|
/** Track one client-facing proxy request until its response body settles. */
|
|
@@ -9,18 +9,37 @@ let lastActivityAtMs = null;
|
|
|
9
9
|
// where bytes actually leave the proxy.
|
|
10
10
|
const responseObserversByMetadata = new WeakMap();
|
|
11
11
|
const finalLogObservers = new Map();
|
|
12
|
+
const attemptLogObservers = new Map();
|
|
12
13
|
/** Join route accounting to the HTTP lifecycle without relying on write order. */
|
|
13
|
-
export function observeProxyFinalLog(requestId, observer) {
|
|
14
|
+
export function observeProxyFinalLog(requestId, observer, attemptObserver) {
|
|
14
15
|
finalLogObservers.set(requestId, observer);
|
|
16
|
+
if (attemptObserver) {
|
|
17
|
+
attemptLogObservers.set(requestId, attemptObserver);
|
|
18
|
+
}
|
|
19
|
+
else {
|
|
20
|
+
attemptLogObservers.delete(requestId);
|
|
21
|
+
}
|
|
15
22
|
return () => {
|
|
16
23
|
if (finalLogObservers.get(requestId) === observer) {
|
|
17
24
|
finalLogObservers.delete(requestId);
|
|
25
|
+
attemptLogObservers.delete(requestId);
|
|
18
26
|
}
|
|
19
27
|
};
|
|
20
28
|
}
|
|
21
29
|
export function notifyProxyFinalLog(entry) {
|
|
22
30
|
finalLogObservers.get(entry.requestId)?.(entry);
|
|
23
31
|
}
|
|
32
|
+
/** Retain only the last attempt for a currently observed HTTP request. */
|
|
33
|
+
export function notifyProxyAttemptLog(entry) {
|
|
34
|
+
const observer = attemptLogObservers.get(entry.requestId);
|
|
35
|
+
observer?.(entry);
|
|
36
|
+
if (entry.parentRequestId && entry.parentRequestId !== entry.requestId) {
|
|
37
|
+
const parentObserver = attemptLogObservers.get(entry.parentRequestId);
|
|
38
|
+
if (parentObserver !== observer) {
|
|
39
|
+
parentObserver?.(entry);
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
}
|
|
24
43
|
export function registerProxyResponseObserver(metadata, observer) {
|
|
25
44
|
const existing = responseObserversByMetadata.get(metadata);
|
|
26
45
|
if (existing) {
|
|
@@ -33,6 +33,7 @@ export declare function prepareProxyBodyForLogging(body: unknown): {
|
|
|
33
33
|
value?: string;
|
|
34
34
|
bytes?: number;
|
|
35
35
|
truncated: boolean;
|
|
36
|
+
originalBytes?: number;
|
|
36
37
|
};
|
|
37
38
|
/** Capture an owned request body with bounded processing and tracked index/export publication. */
|
|
38
39
|
export declare function logBodyCapture(entry: ProxyBodyCaptureEntry): Promise<void>;
|
|
@@ -5,7 +5,8 @@
|
|
|
5
5
|
* when a LoggerProvider is configured via OpenTelemetry instrumentation.
|
|
6
6
|
* Useful for debugging and auditing proxy traffic.
|
|
7
7
|
*/
|
|
8
|
-
import { emitProxyOtelEvent, getProxyOtelLogSnapshot, initializeProxyOtelLogs, isProxyOtelOnly, } from "./otelLogSink.js";
|
|
8
|
+
import { emitProxyOtelEvent, getProxyOtelLogSnapshot, initializeProxyOtelLogs, isProxyOtelOnly, publishProxyOtelBody, } from "./otelLogSink.js";
|
|
9
|
+
import { randomUUID } from "node:crypto";
|
|
9
10
|
import { join } from "path";
|
|
10
11
|
import { homedir } from "os";
|
|
11
12
|
import { logger } from "../utils/logger.js";
|
|
@@ -18,7 +19,7 @@ import { isBorrowedRequest } from "./shareContext.js";
|
|
|
18
19
|
import { OtelBridge } from "../observability/otelBridge.js";
|
|
19
20
|
import { SeverityNumber } from "@opentelemetry/api-logs";
|
|
20
21
|
import { configureProxyLifecycleLogger } from "./proxyLifecycle.js";
|
|
21
|
-
import { notifyProxyFinalLog } from "./proxyActivity.js";
|
|
22
|
+
import { notifyProxyFinalLog, notifyProxyAttemptLog } from "./proxyActivity.js";
|
|
22
23
|
import { withTimeout } from "../utils/async/withTimeout.js";
|
|
23
24
|
let logDir = null;
|
|
24
25
|
let logEnabled = false;
|
|
@@ -227,9 +228,6 @@ export async function logRequest(entry) {
|
|
|
227
228
|
* or OTLP-derived dashboard panels.
|
|
228
229
|
*/
|
|
229
230
|
export async function logRequestAttempt(entry) {
|
|
230
|
-
if (!logEnabled || (!logDir && !isProxyOtelOnly())) {
|
|
231
|
-
return;
|
|
232
|
-
}
|
|
233
231
|
if (!entry.traceId) {
|
|
234
232
|
const bridge = new OtelBridge();
|
|
235
233
|
const traceCtx = bridge.getCurrentTraceContext();
|
|
@@ -238,6 +236,10 @@ export async function logRequestAttempt(entry) {
|
|
|
238
236
|
entry.spanId = traceCtx.spanId;
|
|
239
237
|
}
|
|
240
238
|
}
|
|
239
|
+
notifyProxyAttemptLog(entry);
|
|
240
|
+
if (!logEnabled || (!logDir && !isProxyOtelOnly())) {
|
|
241
|
+
return;
|
|
242
|
+
}
|
|
241
243
|
if (isProxyOtelOnly()) {
|
|
242
244
|
emitProxyOtelEvent("attempt", entry);
|
|
243
245
|
return;
|
|
@@ -445,16 +447,11 @@ function emitOtlpBodyLogRecord(entry, stored) {
|
|
|
445
447
|
return resolveLoggerProvider()
|
|
446
448
|
.then(async (provider) => {
|
|
447
449
|
if (!provider || stored.redactedBody === undefined) {
|
|
448
|
-
return;
|
|
450
|
+
return undefined;
|
|
449
451
|
}
|
|
450
452
|
const otelLogger = provider.getLogger("neurolink-proxy-bodies", "1.0.0");
|
|
451
|
-
const
|
|
452
|
-
const
|
|
453
|
-
for (let chunkIndex = 0; chunkIndex < totalChunks; chunkIndex++) {
|
|
454
|
-
if (chunkIndex > 0 && chunkIndex % 4 === 0) {
|
|
455
|
-
await yieldToRequests();
|
|
456
|
-
}
|
|
457
|
-
const chunk = chunks[chunkIndex] ?? "";
|
|
453
|
+
const captureId = entry.captureId ?? randomUUID();
|
|
454
|
+
const emit = (chunk, chunkIndex, totalChunks) => {
|
|
458
455
|
otelLogger.emit({
|
|
459
456
|
severityNumber: (entry.responseStatus ?? 0) >= 400
|
|
460
457
|
? SeverityNumber.WARN
|
|
@@ -466,6 +463,7 @@ function emitOtlpBodyLogRecord(entry, stored) {
|
|
|
466
463
|
"proxy.record_kind": "body",
|
|
467
464
|
"request.id": entry.requestId,
|
|
468
465
|
"body.phase": entry.phase,
|
|
466
|
+
"body.capture_id": captureId,
|
|
469
467
|
"body.chunk_index": chunkIndex,
|
|
470
468
|
"body.chunk_count": totalChunks,
|
|
471
469
|
"body.content_type": entry.contentType ?? "application/json",
|
|
@@ -500,10 +498,22 @@ function emitOtlpBodyLogRecord(entry, stored) {
|
|
|
500
498
|
source: "otlp",
|
|
501
499
|
},
|
|
502
500
|
});
|
|
501
|
+
};
|
|
502
|
+
if (isProxyOtelOnly()) {
|
|
503
|
+
return publishProxyOtelBody(captureId, stored.redactedBody, emit);
|
|
503
504
|
}
|
|
505
|
+
const chunks = splitUtf8StringByBytes(stored.redactedBody, BODY_OTLP_CHUNK_SIZE);
|
|
506
|
+
for (let i = 0; i < chunks.length; i++) {
|
|
507
|
+
if (i > 0 && i % 4 === 0) {
|
|
508
|
+
await yieldToRequests();
|
|
509
|
+
}
|
|
510
|
+
emit(chunks[i], i, chunks.length);
|
|
511
|
+
}
|
|
512
|
+
return undefined;
|
|
504
513
|
})
|
|
505
514
|
.catch(() => {
|
|
506
515
|
// Non-fatal — never crash proxy for OTLP log failures
|
|
516
|
+
return undefined;
|
|
507
517
|
});
|
|
508
518
|
}
|
|
509
519
|
/** Capture an owned request body with bounded processing and tracked index/export publication. */
|
|
@@ -525,7 +535,11 @@ export async function logBodyCapture(entry) {
|
|
|
525
535
|
const destination = logDir;
|
|
526
536
|
// Publication callbacks retain metadata and the bounded redacted result,
|
|
527
537
|
// never the original unbounded body while a sink is slow.
|
|
528
|
-
const metadata = {
|
|
538
|
+
const metadata = {
|
|
539
|
+
...entry,
|
|
540
|
+
captureId: entry.captureId ?? randomUUID(),
|
|
541
|
+
body: undefined,
|
|
542
|
+
};
|
|
529
543
|
/** Persist the processed capture index and publish its redacted body before releasing capacity. */
|
|
530
544
|
const consume = async (processed) => {
|
|
531
545
|
const redactedHeaders = processed.headers;
|
|
@@ -538,6 +552,7 @@ export async function logBodyCapture(entry) {
|
|
|
538
552
|
timestamp: metadata.timestamp,
|
|
539
553
|
type: "body_capture",
|
|
540
554
|
requestId: metadata.requestId,
|
|
555
|
+
captureId: metadata.captureId,
|
|
541
556
|
phase: metadata.phase,
|
|
542
557
|
model: metadata.model,
|
|
543
558
|
stream: metadata.stream,
|
|
@@ -554,6 +569,8 @@ export async function logBodyCapture(entry) {
|
|
|
554
569
|
redactedBodyBytes: stored.redactedBodyBytes,
|
|
555
570
|
storedFileBytes: stored.storedFileBytes,
|
|
556
571
|
bodyTruncated: stored.bodyTruncated,
|
|
572
|
+
bodyCaptureLimitBytes: stored.bodyCaptureLimitBytes,
|
|
573
|
+
originalRedactedBodyBytes: stored.originalRedactedBodyBytes,
|
|
557
574
|
bodyWriteFailed: stored.bodyWriteFailed,
|
|
558
575
|
captureError: processed.error,
|
|
559
576
|
captureQueueWaitMs: processed.queueWaitMs,
|
|
@@ -565,7 +582,21 @@ export async function logBodyCapture(entry) {
|
|
|
565
582
|
indexEntry.spanId = traceCtx.spanId;
|
|
566
583
|
}
|
|
567
584
|
if (isProxyOtelOnly()) {
|
|
585
|
+
const delivery = await emitOtlpBodyLogRecord({
|
|
586
|
+
...metadata,
|
|
587
|
+
traceId: traceCtx?.traceId ?? metadata.traceId,
|
|
588
|
+
spanId: traceCtx?.spanId ?? metadata.spanId,
|
|
589
|
+
}, stored);
|
|
590
|
+
indexEntry.bodyDelivery = delivery ?? {
|
|
591
|
+
status: processed.error
|
|
592
|
+
? "capture_rejected"
|
|
593
|
+
: stored.redactedBody === undefined
|
|
594
|
+
? "no_body"
|
|
595
|
+
: "export_unconfirmed",
|
|
596
|
+
...(processed.error ? { reason: processed.error } : {}),
|
|
597
|
+
};
|
|
568
598
|
emitProxyOtelEvent("body_capture_index", indexEntry);
|
|
599
|
+
return;
|
|
569
600
|
}
|
|
570
601
|
try {
|
|
571
602
|
if (logFile) {
|
|
@@ -583,7 +614,14 @@ export async function logBodyCapture(entry) {
|
|
|
583
614
|
spanId: traceCtx?.spanId ?? metadata.spanId,
|
|
584
615
|
}, stored);
|
|
585
616
|
};
|
|
586
|
-
|
|
617
|
+
const operation = trackLogOperation(captureProxyBody(entry, destination, consume));
|
|
618
|
+
// HTTP handlers may await this function. Collector latency must never hold
|
|
619
|
+
// their response open; shutdown uses flushRequestLogs as the completion fence.
|
|
620
|
+
if (isProxyOtelOnly()) {
|
|
621
|
+
void operation;
|
|
622
|
+
return;
|
|
623
|
+
}
|
|
624
|
+
return operation;
|
|
587
625
|
}
|
|
588
626
|
/**
|
|
589
627
|
* Log the FULL raw request and response for debugging.
|
package/dist/types/proxy.d.ts
CHANGED
|
@@ -641,8 +641,31 @@ export type ProxyBodyCaptureWorkerSnapshot = {
|
|
|
641
641
|
pendingBytes: number;
|
|
642
642
|
maxPending: number;
|
|
643
643
|
maxPendingBytes: number;
|
|
644
|
+
/** Admission failures by exact guard, independent of processing failures. */
|
|
645
|
+
rejectionReasons: Record<string, number>;
|
|
644
646
|
lastError?: string;
|
|
645
647
|
};
|
|
648
|
+
/** Collector transport evidence; acknowledgement does not prove backend storage. */
|
|
649
|
+
export type ProxyBodyDeliveryResult = {
|
|
650
|
+
status: "transport_acknowledged" | "export_unconfirmed" | "rejected" | "partial";
|
|
651
|
+
/** Absent when publication was rejected before chunking. */
|
|
652
|
+
expectedChunks?: number;
|
|
653
|
+
acknowledgedChunks: number;
|
|
654
|
+
unconfirmedChunks: number;
|
|
655
|
+
droppedChunks: number;
|
|
656
|
+
notSubmittedChunks?: number;
|
|
657
|
+
reason?: string;
|
|
658
|
+
};
|
|
659
|
+
/** One bounded body publication, tracked across exporter callbacks. */
|
|
660
|
+
export type ProxyBodyPublicationProgress = {
|
|
661
|
+
acknowledged: number;
|
|
662
|
+
unconfirmed: number;
|
|
663
|
+
dropped: number;
|
|
664
|
+
emitted: number;
|
|
665
|
+
notify?: () => void;
|
|
666
|
+
};
|
|
667
|
+
/** Chunk emission stays in the request logger, which owns request attributes. */
|
|
668
|
+
export type ProxyBodyChunkEmitter = (chunk: string, index: number, count: number) => void;
|
|
646
669
|
export type ProxyRequestLoggerSnapshot = {
|
|
647
670
|
diskEnabled?: boolean;
|
|
648
671
|
otel?: ReturnType<typeof import("../proxy/otelLogSink.js").getProxyOtelLogSnapshot>;
|
|
@@ -1991,6 +2014,8 @@ export type ProxyAnalysisRoutingRecord = {
|
|
|
1991
2014
|
};
|
|
1992
2015
|
/** Request metadata retained by the HTTP adapter for terminal error logging. */
|
|
1993
2016
|
export type RuntimeRequestMetadata = {
|
|
2017
|
+
/** Last dispatched attempt, retained until this HTTP request terminates. */
|
|
2018
|
+
lastUpstreamAttempt?: RequestAttemptLogEntry;
|
|
1994
2019
|
requestId: string;
|
|
1995
2020
|
method: string;
|
|
1996
2021
|
path: string;
|
|
@@ -2022,6 +2047,8 @@ export type RawStreamCaptureResult = {
|
|
|
2022
2047
|
};
|
|
2023
2048
|
/** Single captured body/headers entry written to disk by the proxy logger. */
|
|
2024
2049
|
export type ProxyBodyCaptureEntry = {
|
|
2050
|
+
/** Unique capture identity shared by its index and every exported chunk. */
|
|
2051
|
+
captureId?: string;
|
|
2025
2052
|
timestamp: string;
|
|
2026
2053
|
requestId: string;
|
|
2027
2054
|
phase: string;
|
|
@@ -2169,6 +2196,8 @@ export type StoredBodyArtifact = {
|
|
|
2169
2196
|
storedFileBytes?: number;
|
|
2170
2197
|
redactedBody?: string;
|
|
2171
2198
|
bodyTruncated?: boolean;
|
|
2199
|
+
bodyCaptureLimitBytes?: number;
|
|
2200
|
+
originalRedactedBodyBytes?: number;
|
|
2172
2201
|
bodyWriteFailed?: boolean;
|
|
2173
2202
|
};
|
|
2174
2203
|
/** File the proxy logger tracks for rotation and cleanup. */
|