@juspay/neurolink 12.14.4 → 12.14.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +2 -2
- package/dist/browser/neurolink.min.js +370 -370
- package/dist/cli/commands/proxy.js +11 -3
- package/dist/proxy/bodyCaptureProcessing.d.ts +1 -0
- package/dist/proxy/bodyCaptureProcessing.js +19 -4
- package/dist/proxy/bodyCaptureWorker.js +35 -13
- package/dist/proxy/otelLogSink.d.ts +22 -1
- package/dist/proxy/otelLogSink.js +247 -8
- package/dist/proxy/proxyActivity.d.ts +4 -2
- package/dist/proxy/proxyActivity.js +20 -1
- package/dist/proxy/requestLogger.d.ts +1 -0
- package/dist/proxy/requestLogger.js +53 -15
- package/dist/types/proxy.d.ts +29 -0
- package/docs-site/static/search-index.json +1 -1
- package/package.json +1 -1
- package/scripts/observability/check-proxy-telemetry.mjs +20 -7
- package/scripts/observability/query-proxy-history.mjs +193 -0
|
@@ -1079,6 +1079,8 @@ function registerProxyRequestTracking(app, requestMetadata, readiness) {
|
|
|
1079
1079
|
requestMetadata.set(c.req.raw, metadata);
|
|
1080
1080
|
const stopObservingFinalLog = observeProxyFinalLog(metadata.requestId, (entry) => {
|
|
1081
1081
|
metadata.terminalResult = entry;
|
|
1082
|
+
}, (entry) => {
|
|
1083
|
+
metadata.lastUpstreamAttempt = { ...entry };
|
|
1082
1084
|
});
|
|
1083
1085
|
const finishActivity = metadata.rejectForUpdate
|
|
1084
1086
|
? () => undefined
|
|
@@ -1306,7 +1308,8 @@ export async function createProxyStartApp(params) {
|
|
|
1306
1308
|
const recordRuntimeError = async (metadata, status, errorType, errorMessage, options) => {
|
|
1307
1309
|
const clientMessage = options?.clientMessage ?? errorMessage;
|
|
1308
1310
|
const clientErrorType = options?.clientErrorType ?? errorType;
|
|
1309
|
-
|
|
1311
|
+
const attempt = metadata.lastUpstreamAttempt;
|
|
1312
|
+
recordFinalError(status, attempt?.account ?? PROXY_INTERNAL_ACCOUNT_LABEL, attempt?.accountType ?? PROXY_INTERNAL_ACCOUNT_TYPE, {
|
|
1310
1313
|
requestId: metadata.requestId,
|
|
1311
1314
|
errorType,
|
|
1312
1315
|
errorCode: options?.errorCode,
|
|
@@ -1322,8 +1325,13 @@ export async function createProxyStartApp(params) {
|
|
|
1322
1325
|
model: metadata.model,
|
|
1323
1326
|
stream: metadata.stream,
|
|
1324
1327
|
toolCount: metadata.toolCount,
|
|
1325
|
-
account: "",
|
|
1326
|
-
accountType: "proxy-runtime",
|
|
1328
|
+
account: attempt?.account ?? "",
|
|
1329
|
+
accountType: attempt?.accountType ?? "proxy-runtime",
|
|
1330
|
+
accountKey: attempt?.accountKey,
|
|
1331
|
+
provider: attempt?.provider,
|
|
1332
|
+
transportScope: attempt?.transportScope,
|
|
1333
|
+
traceId: attempt?.traceId,
|
|
1334
|
+
spanId: attempt?.spanId,
|
|
1327
1335
|
responseStatus: status,
|
|
1328
1336
|
responseTimeMs: Date.now() - metadata.startedAt,
|
|
1329
1337
|
errorType,
|
|
@@ -6,6 +6,7 @@ import { gzip as gzipCallback } from "node:zlib";
|
|
|
6
6
|
const REQUEST_LOG_IO_TIMEOUT_MS = 5_000;
|
|
7
7
|
/** Maximum redacted body bytes persisted per capture entry. */
|
|
8
8
|
const MAX_CAPTURED_BODY_BYTES = 1024 * 1024;
|
|
9
|
+
const MAX_OTEL_CAPTURED_BODY_BYTES = 8 * 1024 * 1024;
|
|
9
10
|
const BODY_TRUNCATION_MARKER = "\n...[TRUNCATED]";
|
|
10
11
|
const gzip = promisify(gzipCallback);
|
|
11
12
|
/** Headers whose values must always be redacted. */
|
|
@@ -130,12 +131,15 @@ export function splitUtf8StringByBytes(input, maxBytes) {
|
|
|
130
131
|
* Apply structural redaction before enforcing the per-artifact byte
|
|
131
132
|
* ceiling.
|
|
132
133
|
*/
|
|
133
|
-
function prepareRedactedBody(body) {
|
|
134
|
+
function prepareRedactedBody(body, maxBytes = MAX_CAPTURED_BODY_BYTES) {
|
|
134
135
|
const redacted = redactBody(body);
|
|
135
136
|
if (redacted === undefined) {
|
|
136
137
|
return { truncated: false };
|
|
137
138
|
}
|
|
138
|
-
return
|
|
139
|
+
return {
|
|
140
|
+
...truncateUtf8String(redacted, maxBytes),
|
|
141
|
+
originalBytes: utf8ByteLength(redacted),
|
|
142
|
+
};
|
|
139
143
|
}
|
|
140
144
|
/**
|
|
141
145
|
* Write a private gzip artifact with a unique name and return its
|
|
@@ -202,7 +206,8 @@ export function redactProxyHeadersForLogging(headers) {
|
|
|
202
206
|
*/
|
|
203
207
|
export async function processProxyBodyCapture(entry, logDir) {
|
|
204
208
|
const headers = redactHeaders(entry.headers);
|
|
205
|
-
const
|
|
209
|
+
const limit = logDir === null ? MAX_OTEL_CAPTURED_BODY_BYTES : MAX_CAPTURED_BODY_BYTES;
|
|
210
|
+
const prepared = prepareRedactedBody(entry.body, limit);
|
|
206
211
|
if (logDir === null) {
|
|
207
212
|
return {
|
|
208
213
|
headers,
|
|
@@ -210,6 +215,9 @@ export async function processProxyBodyCapture(entry, logDir) {
|
|
|
210
215
|
redactedBody: prepared.value,
|
|
211
216
|
redactedBodyBytes: prepared.bytes,
|
|
212
217
|
bodyTruncated: prepared.truncated,
|
|
218
|
+
bodyCaptureLimitBytes: limit,
|
|
219
|
+
originalRedactedBodyBytes: prepared.originalBytes,
|
|
220
|
+
bodySha256: prepared.value === undefined ? undefined : sha256(prepared.value),
|
|
213
221
|
},
|
|
214
222
|
};
|
|
215
223
|
}
|
|
@@ -225,5 +233,12 @@ export async function processProxyBodyCapture(entry, logDir) {
|
|
|
225
233
|
bodyWriteFailed: true,
|
|
226
234
|
};
|
|
227
235
|
}
|
|
228
|
-
return {
|
|
236
|
+
return {
|
|
237
|
+
headers,
|
|
238
|
+
stored: {
|
|
239
|
+
...stored,
|
|
240
|
+
bodyCaptureLimitBytes: limit,
|
|
241
|
+
originalRedactedBodyBytes: prepared.originalBytes,
|
|
242
|
+
},
|
|
243
|
+
};
|
|
229
244
|
}
|
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import { Worker } from "node:worker_threads";
|
|
2
2
|
const MAX_PENDING = 16;
|
|
3
3
|
const MAX_PENDING_BYTES = 32 * 1024 * 1024;
|
|
4
|
-
|
|
4
|
+
// Bound retained UTF-16 strings rather than a 3x UTF-8 guess. This accommodates
|
|
5
|
+
// the observed 7.3 MB JSON requests while retaining a 32 MiB aggregate pool.
|
|
6
|
+
const MAX_ENTRY_BYTES = 16 * 1024 * 1024;
|
|
5
7
|
export const PROXY_BODY_CAPTURE_DEADLINE_MS = 20_000;
|
|
6
8
|
let worker;
|
|
7
9
|
let workerUrl;
|
|
@@ -16,6 +18,7 @@ const snapshot = {
|
|
|
16
18
|
pendingBytes: 0,
|
|
17
19
|
maxPending: MAX_PENDING,
|
|
18
20
|
maxPendingBytes: MAX_PENDING_BYTES,
|
|
21
|
+
rejectionReasons: {},
|
|
19
22
|
};
|
|
20
23
|
const pending = new Map();
|
|
21
24
|
// Bound traversal as well as the structured clone sent to the worker. Never
|
|
@@ -29,12 +32,15 @@ function estimateCloneBytes(value) {
|
|
|
29
32
|
const seen = new Set();
|
|
30
33
|
let bytes = 0, nodes = 0;
|
|
31
34
|
while (stack.length) {
|
|
32
|
-
if (++nodes > 100_000
|
|
33
|
-
|
|
35
|
+
if (++nodes > 100_000) {
|
|
36
|
+
throw new Error("body_capture_traversal_limit");
|
|
37
|
+
}
|
|
38
|
+
if (bytes > MAX_ENTRY_BYTES) {
|
|
39
|
+
throw new Error("body_capture_entry_too_large");
|
|
34
40
|
}
|
|
35
41
|
const item = stack.pop();
|
|
36
42
|
if (typeof item === "string") {
|
|
37
|
-
bytes += item.length *
|
|
43
|
+
bytes += 16 + item.length * 2;
|
|
38
44
|
continue;
|
|
39
45
|
}
|
|
40
46
|
bytes += 16;
|
|
@@ -42,23 +48,26 @@ function estimateCloneBytes(value) {
|
|
|
42
48
|
continue;
|
|
43
49
|
}
|
|
44
50
|
if (seen.has(item)) {
|
|
45
|
-
|
|
51
|
+
throw new Error("body_capture_unsupported_value");
|
|
46
52
|
}
|
|
47
53
|
seen.add(item);
|
|
48
54
|
if (!Array.isArray(item) &&
|
|
49
55
|
Object.getPrototypeOf(item) !== Object.prototype &&
|
|
50
56
|
Object.getPrototypeOf(item) !== null) {
|
|
51
|
-
|
|
57
|
+
throw new Error("body_capture_unsupported_value");
|
|
52
58
|
}
|
|
53
59
|
for (const key of Object.keys(item)) {
|
|
54
|
-
bytes += key.length *
|
|
60
|
+
bytes += 16 + key.length * 2;
|
|
55
61
|
const descriptor = Object.getOwnPropertyDescriptor(item, key);
|
|
56
62
|
if (!descriptor || descriptor.get || descriptor.set) {
|
|
57
|
-
|
|
63
|
+
throw new Error("body_capture_unsupported_value");
|
|
58
64
|
}
|
|
59
65
|
stack.push(descriptor.value);
|
|
60
|
-
if (stack.length > 100_000
|
|
61
|
-
|
|
66
|
+
if (stack.length > 100_000) {
|
|
67
|
+
throw new Error("body_capture_traversal_limit");
|
|
68
|
+
}
|
|
69
|
+
if (bytes > MAX_ENTRY_BYTES) {
|
|
70
|
+
throw new Error("body_capture_entry_too_large");
|
|
62
71
|
}
|
|
63
72
|
}
|
|
64
73
|
}
|
|
@@ -125,11 +134,21 @@ function getWorker() {
|
|
|
125
134
|
export async function captureProxyBody(entry, logDir, consume) {
|
|
126
135
|
snapshot.attempted += 1;
|
|
127
136
|
let bytes;
|
|
137
|
+
let admissionError;
|
|
128
138
|
try {
|
|
129
139
|
bytes = estimateCloneBytes(entry);
|
|
130
140
|
}
|
|
131
|
-
catch {
|
|
141
|
+
catch (error) {
|
|
132
142
|
bytes = Infinity;
|
|
143
|
+
admissionError =
|
|
144
|
+
error instanceof Error &&
|
|
145
|
+
[
|
|
146
|
+
"body_capture_entry_too_large",
|
|
147
|
+
"body_capture_traversal_limit",
|
|
148
|
+
"body_capture_unsupported_value",
|
|
149
|
+
].includes(error.message)
|
|
150
|
+
? error.message
|
|
151
|
+
: "body_capture_unsupported_value";
|
|
133
152
|
}
|
|
134
153
|
if (bytes > MAX_ENTRY_BYTES ||
|
|
135
154
|
snapshot.pending >= MAX_PENDING ||
|
|
@@ -137,11 +156,13 @@ export async function captureProxyBody(entry, logDir, consume) {
|
|
|
137
156
|
Date.now() < retryAfter) {
|
|
138
157
|
snapshot.rejected += 1;
|
|
139
158
|
const error = bytes > MAX_ENTRY_BYTES
|
|
140
|
-
? "
|
|
159
|
+
? (admissionError ?? "body_capture_entry_too_large")
|
|
141
160
|
: Date.now() < retryAfter
|
|
142
161
|
? "body_worker_backoff"
|
|
143
162
|
: "body_capture_queue_full";
|
|
144
163
|
snapshot.lastError = error;
|
|
164
|
+
snapshot.rejectionReasons[error] =
|
|
165
|
+
(snapshot.rejectionReasons[error] ?? 0) + 1;
|
|
145
166
|
return consume({ error, stored: { bodyWriteFailed: true } });
|
|
146
167
|
}
|
|
147
168
|
let current;
|
|
@@ -204,7 +225,7 @@ export async function captureProxyBody(entry, logDir, consume) {
|
|
|
204
225
|
* retained publication work.
|
|
205
226
|
*/
|
|
206
227
|
export function getBodyCaptureWorkerSnapshot() {
|
|
207
|
-
return { ...snapshot };
|
|
228
|
+
return { ...snapshot, rejectionReasons: { ...snapshot.rejectionReasons } };
|
|
208
229
|
}
|
|
209
230
|
/** Isolated tests point at a separately executed built worker. */
|
|
210
231
|
export const __bodyCaptureWorkerTestHooks = {
|
|
@@ -227,6 +248,7 @@ export const __bodyCaptureWorkerTestHooks = {
|
|
|
227
248
|
pending: 0,
|
|
228
249
|
pendingBytes: 0,
|
|
229
250
|
lastError: undefined,
|
|
251
|
+
rejectionReasons: {},
|
|
230
252
|
});
|
|
231
253
|
},
|
|
232
254
|
};
|
|
@@ -1,6 +1,13 @@
|
|
|
1
1
|
import { LoggerProvider } from "@opentelemetry/sdk-logs";
|
|
2
|
+
import type { ProxyBodyChunkEmitter, ProxyBodyDeliveryResult } from "../types/index.js";
|
|
2
3
|
/** Explicit opt-in; configuration never silently falls back to file logging. */
|
|
3
4
|
export declare function isProxyOtelOnly(): boolean;
|
|
5
|
+
/**
|
|
6
|
+
* Own a whole capture within byte/count bounds, then pace its chunks by actual
|
|
7
|
+
* export callbacks. SDK forceFlush alone does not await an automatic export
|
|
8
|
+
* already in flight. Serial publication prevents bursts from dropping tails.
|
|
9
|
+
*/
|
|
10
|
+
export declare function publishProxyOtelBody(captureId: string, body: string, emit: ProxyBodyChunkEmitter): Promise<ProxyBodyDeliveryResult>;
|
|
4
11
|
/** Initialize a log-only provider in every proxy process, including the supervisor. */
|
|
5
12
|
export declare function initializeProxyOtelLogs(role?: string): LoggerProvider | undefined;
|
|
6
13
|
/** Structured evidence without final-request dashboard fields on auxiliary events. */
|
|
@@ -13,6 +20,19 @@ export declare function getProxyOtelLogSnapshot(): {
|
|
|
13
20
|
initialized: boolean;
|
|
14
21
|
deliveryGuarantee: string;
|
|
15
22
|
invalidRecords: number;
|
|
23
|
+
bodyDelivery: {
|
|
24
|
+
attempted: number;
|
|
25
|
+
transportAcknowledged: number;
|
|
26
|
+
exportUnconfirmed: number;
|
|
27
|
+
rejected: number;
|
|
28
|
+
partial: number;
|
|
29
|
+
pending: number;
|
|
30
|
+
pendingBytes: number;
|
|
31
|
+
highWaterPending: number;
|
|
32
|
+
highWaterBytes: number;
|
|
33
|
+
maxPending: number;
|
|
34
|
+
maxPendingBytes: number;
|
|
35
|
+
};
|
|
16
36
|
queues: {
|
|
17
37
|
attempted: number;
|
|
18
38
|
submitted: number;
|
|
@@ -22,7 +42,8 @@ export declare function getProxyOtelLogSnapshot(): {
|
|
|
22
42
|
outstanding: number;
|
|
23
43
|
lastAcknowledgedAt: string | undefined;
|
|
24
44
|
lastFailureAt: string | undefined;
|
|
25
|
-
|
|
45
|
+
highWaterOutstanding: number;
|
|
46
|
+
kind: "metadata" | "bodies";
|
|
26
47
|
capacity: number;
|
|
27
48
|
}[];
|
|
28
49
|
};
|
|
@@ -1,20 +1,40 @@
|
|
|
1
1
|
/* eslint-disable no-console -- This proxy-only sink replaces console methods with OTLP emission. */
|
|
2
2
|
import { inspect } from "node:util";
|
|
3
|
+
import { setImmediate as yieldToRequests } from "node:timers/promises";
|
|
3
4
|
import { SeverityNumber } from "@opentelemetry/api-logs";
|
|
4
5
|
import { ExportResultCode } from "@opentelemetry/core";
|
|
5
6
|
import { OTLPLogExporter } from "@opentelemetry/exporter-logs-otlp-http";
|
|
6
7
|
import { resourceFromAttributes } from "@opentelemetry/resources";
|
|
7
8
|
import { BatchLogRecordProcessor, LoggerProvider, } from "@opentelemetry/sdk-logs";
|
|
8
9
|
import { sanitizeForLog } from "../utils/logSanitize.js";
|
|
10
|
+
import { splitUtf8StringByBytes } from "./bodyCaptureProcessing.js";
|
|
9
11
|
let provider;
|
|
10
12
|
let restoreConsole;
|
|
11
13
|
const queues = [];
|
|
14
|
+
const bodyPublications = new Map();
|
|
15
|
+
let bodyPublicationChain = Promise.resolve();
|
|
16
|
+
let shuttingDown = false;
|
|
17
|
+
const bodyDelivery = {
|
|
18
|
+
attempted: 0,
|
|
19
|
+
transportAcknowledged: 0,
|
|
20
|
+
exportUnconfirmed: 0,
|
|
21
|
+
rejected: 0,
|
|
22
|
+
partial: 0,
|
|
23
|
+
pending: 0,
|
|
24
|
+
pendingBytes: 0,
|
|
25
|
+
highWaterPending: 0,
|
|
26
|
+
highWaterBytes: 0,
|
|
27
|
+
maxPending: 16,
|
|
28
|
+
maxPendingBytes: 32 * 1024 * 1024,
|
|
29
|
+
};
|
|
12
30
|
/** Explicit opt-in; configuration never silently falls back to file logging. */
|
|
13
31
|
export function isProxyOtelOnly() {
|
|
14
32
|
return process.env.NEUROLINK_PROXY_LOG_SINK === "otel";
|
|
15
33
|
}
|
|
16
34
|
/** Reserve capacity including exports in flight, independently for metadata and bodies. */
|
|
17
|
-
function createTrackedProcessor(url, capacity) {
|
|
35
|
+
function createTrackedProcessor(url, capacity, kind) {
|
|
36
|
+
const unsettled = new Set();
|
|
37
|
+
const flushWaiters = new Set();
|
|
18
38
|
const state = {
|
|
19
39
|
attempted: 0,
|
|
20
40
|
submitted: 0,
|
|
@@ -24,16 +44,23 @@ function createTrackedProcessor(url, capacity) {
|
|
|
24
44
|
outstanding: 0,
|
|
25
45
|
lastAcknowledgedAt: undefined,
|
|
26
46
|
lastFailureAt: undefined,
|
|
47
|
+
highWaterOutstanding: 0,
|
|
27
48
|
};
|
|
28
49
|
const transport = new OTLPLogExporter({ url, timeoutMillis: 5000 });
|
|
29
50
|
const exporter = {
|
|
30
51
|
export(records, callback) {
|
|
31
52
|
let settled = false;
|
|
53
|
+
const deadline = setTimeout(() => settle({
|
|
54
|
+
code: ExportResultCode.FAILED,
|
|
55
|
+
error: new Error("OTLP export callback deadline exceeded"),
|
|
56
|
+
}), 6_000);
|
|
57
|
+
deadline.unref();
|
|
32
58
|
const settle = (result) => {
|
|
33
59
|
if (settled) {
|
|
34
60
|
return;
|
|
35
61
|
}
|
|
36
62
|
settled = true;
|
|
63
|
+
clearTimeout(deadline);
|
|
37
64
|
state.outstanding -= records.length;
|
|
38
65
|
if (result.code === ExportResultCode.SUCCESS) {
|
|
39
66
|
state.transportAcknowledged += records.length;
|
|
@@ -43,6 +70,23 @@ function createTrackedProcessor(url, capacity) {
|
|
|
43
70
|
state.exportUnconfirmed += records.length;
|
|
44
71
|
state.lastFailureAt = new Date().toISOString();
|
|
45
72
|
}
|
|
73
|
+
for (const record of records) {
|
|
74
|
+
unsettled.delete(record);
|
|
75
|
+
const id = record.attributes?.["body.capture_id"];
|
|
76
|
+
const publication = typeof id === "string" ? bodyPublications.get(id) : undefined;
|
|
77
|
+
if (publication) {
|
|
78
|
+
if (result.code === ExportResultCode.SUCCESS) {
|
|
79
|
+
publication.acknowledged++;
|
|
80
|
+
}
|
|
81
|
+
else {
|
|
82
|
+
publication.unconfirmed++;
|
|
83
|
+
}
|
|
84
|
+
publication.notify?.();
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
for (const notify of flushWaiters) {
|
|
88
|
+
notify();
|
|
89
|
+
}
|
|
46
90
|
callback(result);
|
|
47
91
|
};
|
|
48
92
|
try {
|
|
@@ -66,18 +110,193 @@ function createTrackedProcessor(url, capacity) {
|
|
|
66
110
|
const processor = {
|
|
67
111
|
onEmit(record) {
|
|
68
112
|
state.attempted++;
|
|
113
|
+
const id = record.attributes?.["body.capture_id"];
|
|
114
|
+
const publication = typeof id === "string" ? bodyPublications.get(id) : undefined;
|
|
115
|
+
if (publication) {
|
|
116
|
+
publication.emitted++;
|
|
117
|
+
}
|
|
69
118
|
if (state.outstanding >= capacity) {
|
|
70
119
|
state.dropped++;
|
|
120
|
+
if (publication) {
|
|
121
|
+
publication.dropped++;
|
|
122
|
+
publication.notify?.();
|
|
123
|
+
}
|
|
71
124
|
return;
|
|
72
125
|
}
|
|
73
126
|
state.submitted++;
|
|
74
127
|
state.outstanding++;
|
|
128
|
+
unsettled.add(record);
|
|
129
|
+
state.highWaterOutstanding = Math.max(state.highWaterOutstanding, state.outstanding);
|
|
75
130
|
batch.onEmit(record);
|
|
76
131
|
},
|
|
77
|
-
forceFlush: () =>
|
|
78
|
-
|
|
132
|
+
forceFlush: async () => {
|
|
133
|
+
const boundary = new Set(unsettled);
|
|
134
|
+
try {
|
|
135
|
+
await batch.forceFlush();
|
|
136
|
+
}
|
|
137
|
+
finally {
|
|
138
|
+
await new Promise((resolve) => {
|
|
139
|
+
const check = () => {
|
|
140
|
+
if (![...boundary].some((record) => unsettled.has(record))) {
|
|
141
|
+
flushWaiters.delete(check);
|
|
142
|
+
resolve();
|
|
143
|
+
}
|
|
144
|
+
};
|
|
145
|
+
flushWaiters.add(check);
|
|
146
|
+
check();
|
|
147
|
+
});
|
|
148
|
+
}
|
|
149
|
+
},
|
|
150
|
+
shutdown: async () => {
|
|
151
|
+
try {
|
|
152
|
+
await processor.forceFlush();
|
|
153
|
+
}
|
|
154
|
+
finally {
|
|
155
|
+
await batch.shutdown();
|
|
156
|
+
}
|
|
157
|
+
},
|
|
79
158
|
};
|
|
80
|
-
return { state, processor, capacity };
|
|
159
|
+
return { state, processor, capacity, kind };
|
|
160
|
+
}
|
|
161
|
+
/**
|
|
162
|
+
* Own a whole capture within byte/count bounds, then pace its chunks by actual
|
|
163
|
+
* export callbacks. SDK forceFlush alone does not await an automatic export
|
|
164
|
+
* already in flight. Serial publication prevents bursts from dropping tails.
|
|
165
|
+
*/
|
|
166
|
+
export async function publishProxyOtelBody(captureId, body, emit) {
|
|
167
|
+
bodyDelivery.attempted++;
|
|
168
|
+
const bytes = Buffer.byteLength(body, "utf8");
|
|
169
|
+
if (!provider ||
|
|
170
|
+
shuttingDown ||
|
|
171
|
+
bodyPublications.has(captureId) ||
|
|
172
|
+
bodyDelivery.pending >= bodyDelivery.maxPending ||
|
|
173
|
+
bodyDelivery.pendingBytes + bytes > bodyDelivery.maxPendingBytes) {
|
|
174
|
+
bodyDelivery.rejected++;
|
|
175
|
+
return {
|
|
176
|
+
status: "rejected",
|
|
177
|
+
acknowledgedChunks: 0,
|
|
178
|
+
unconfirmedChunks: 0,
|
|
179
|
+
droppedChunks: 0,
|
|
180
|
+
reason: bodyPublications.has(captureId)
|
|
181
|
+
? "body_capture_id_in_use"
|
|
182
|
+
: !provider || shuttingDown
|
|
183
|
+
? "body_exporter_unavailable"
|
|
184
|
+
: "body_publication_queue_full",
|
|
185
|
+
};
|
|
186
|
+
}
|
|
187
|
+
bodyDelivery.pending++;
|
|
188
|
+
bodyDelivery.pendingBytes += bytes;
|
|
189
|
+
bodyDelivery.highWaterPending = Math.max(bodyDelivery.highWaterPending, bodyDelivery.pending);
|
|
190
|
+
bodyDelivery.highWaterBytes = Math.max(bodyDelivery.highWaterBytes, bodyDelivery.pendingBytes);
|
|
191
|
+
const progress = {
|
|
192
|
+
acknowledged: 0,
|
|
193
|
+
unconfirmed: 0,
|
|
194
|
+
dropped: 0,
|
|
195
|
+
emitted: 0,
|
|
196
|
+
};
|
|
197
|
+
bodyPublications.set(captureId, progress);
|
|
198
|
+
const deadline = Date.now() + 20_000;
|
|
199
|
+
const operation = bodyPublicationChain.then(async () => {
|
|
200
|
+
const queue = queues.find((candidate) => candidate.kind === "bodies");
|
|
201
|
+
if (!queue || Date.now() >= deadline) {
|
|
202
|
+
return {
|
|
203
|
+
status: "rejected",
|
|
204
|
+
acknowledgedChunks: 0,
|
|
205
|
+
unconfirmedChunks: 0,
|
|
206
|
+
droppedChunks: 0,
|
|
207
|
+
reason: !queue
|
|
208
|
+
? "body_exporter_unavailable"
|
|
209
|
+
: "body_publication_deadline",
|
|
210
|
+
};
|
|
211
|
+
}
|
|
212
|
+
const chunks = splitUtf8StringByBytes(body, 16_000);
|
|
213
|
+
const awaitSettlement = () => new Promise((resolve) => {
|
|
214
|
+
const check = () => {
|
|
215
|
+
if (progress.acknowledged + progress.unconfirmed + progress.dropped >=
|
|
216
|
+
progress.emitted) {
|
|
217
|
+
progress.notify = undefined;
|
|
218
|
+
resolve();
|
|
219
|
+
}
|
|
220
|
+
};
|
|
221
|
+
progress.notify = check;
|
|
222
|
+
check();
|
|
223
|
+
});
|
|
224
|
+
let reason;
|
|
225
|
+
try {
|
|
226
|
+
for (let offset = 0; offset < chunks.length; offset += 64) {
|
|
227
|
+
if (Date.now() >= deadline) {
|
|
228
|
+
reason = "body_publication_deadline";
|
|
229
|
+
break;
|
|
230
|
+
}
|
|
231
|
+
// Ordinary body records from an external logger may share this queue.
|
|
232
|
+
// Wait for them before admitting any part of this batch.
|
|
233
|
+
while (queue.state.outstanding > queue.capacity - 64) {
|
|
234
|
+
await queue.processor.forceFlush();
|
|
235
|
+
if (Date.now() >= deadline) {
|
|
236
|
+
throw new Error("body_publication_deadline");
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
for (let i = offset; i < Math.min(offset + 64, chunks.length); i++) {
|
|
240
|
+
emit(chunks[i], i, chunks.length);
|
|
241
|
+
}
|
|
242
|
+
await queue.processor.forceFlush();
|
|
243
|
+
await awaitSettlement();
|
|
244
|
+
if (progress.unconfirmed || progress.dropped) {
|
|
245
|
+
reason = "body_export_unconfirmed";
|
|
246
|
+
break;
|
|
247
|
+
}
|
|
248
|
+
await yieldToRequests();
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
catch (error) {
|
|
252
|
+
reason =
|
|
253
|
+
error instanceof Error &&
|
|
254
|
+
error.message === "body_publication_deadline"
|
|
255
|
+
? "body_publication_deadline"
|
|
256
|
+
: "body_publication_failed";
|
|
257
|
+
// Retain ownership of already submitted chunks until their callbacks settle.
|
|
258
|
+
await queue.processor.forceFlush().catch(() => undefined);
|
|
259
|
+
await awaitSettlement();
|
|
260
|
+
}
|
|
261
|
+
const status = progress.emitted === 0 && chunks.length > 0
|
|
262
|
+
? "rejected"
|
|
263
|
+
: progress.dropped || progress.emitted !== chunks.length
|
|
264
|
+
? "partial"
|
|
265
|
+
: progress.unconfirmed
|
|
266
|
+
? "export_unconfirmed"
|
|
267
|
+
: "transport_acknowledged";
|
|
268
|
+
return {
|
|
269
|
+
status,
|
|
270
|
+
expectedChunks: chunks.length,
|
|
271
|
+
acknowledgedChunks: progress.acknowledged,
|
|
272
|
+
unconfirmedChunks: progress.unconfirmed,
|
|
273
|
+
droppedChunks: progress.dropped,
|
|
274
|
+
notSubmittedChunks: chunks.length - progress.emitted,
|
|
275
|
+
...(reason ? { reason } : {}),
|
|
276
|
+
};
|
|
277
|
+
});
|
|
278
|
+
bodyPublicationChain = operation.then(() => undefined, () => undefined);
|
|
279
|
+
try {
|
|
280
|
+
const result = await operation;
|
|
281
|
+
if (result.status === "transport_acknowledged") {
|
|
282
|
+
bodyDelivery.transportAcknowledged++;
|
|
283
|
+
}
|
|
284
|
+
else if (result.status === "export_unconfirmed") {
|
|
285
|
+
bodyDelivery.exportUnconfirmed++;
|
|
286
|
+
}
|
|
287
|
+
else if (result.status === "rejected") {
|
|
288
|
+
bodyDelivery.rejected++;
|
|
289
|
+
}
|
|
290
|
+
else {
|
|
291
|
+
bodyDelivery.partial++;
|
|
292
|
+
}
|
|
293
|
+
return result;
|
|
294
|
+
}
|
|
295
|
+
finally {
|
|
296
|
+
bodyPublications.delete(captureId);
|
|
297
|
+
bodyDelivery.pending--;
|
|
298
|
+
bodyDelivery.pendingBytes -= bytes;
|
|
299
|
+
}
|
|
81
300
|
}
|
|
82
301
|
/** Initialize a log-only provider in every proxy process, including the supervisor. */
|
|
83
302
|
export function initializeProxyOtelLogs(role = "worker") {
|
|
@@ -102,8 +321,8 @@ export function initializeProxyOtelLogs(role = "worker") {
|
|
|
102
321
|
if (url.protocol === "http:" && !loopback) {
|
|
103
322
|
throw new Error("Proxy OTLP logs require HTTPS for non-loopback collectors");
|
|
104
323
|
}
|
|
105
|
-
const metadata = createTrackedProcessor(endpoint, 2048);
|
|
106
|
-
const bodies = createTrackedProcessor(endpoint, 256);
|
|
324
|
+
const metadata = createTrackedProcessor(endpoint, 2048, "metadata");
|
|
325
|
+
const bodies = createTrackedProcessor(endpoint, 256, "bodies");
|
|
107
326
|
queues.push(metadata, bodies);
|
|
108
327
|
provider = new LoggerProvider({
|
|
109
328
|
resource: resourceFromAttributes({
|
|
@@ -220,8 +439,9 @@ export function getProxyOtelLogSnapshot() {
|
|
|
220
439
|
initialized: provider !== undefined,
|
|
221
440
|
deliveryGuarantee: "best-effort; HTTP success is not per-record acceptance or backend persistence",
|
|
222
441
|
invalidRecords,
|
|
223
|
-
|
|
224
|
-
|
|
442
|
+
bodyDelivery: { ...bodyDelivery },
|
|
443
|
+
queues: queues.map((q) => ({
|
|
444
|
+
kind: q.kind,
|
|
225
445
|
capacity: q.capacity,
|
|
226
446
|
...q.state,
|
|
227
447
|
})),
|
|
@@ -229,16 +449,35 @@ export function getProxyOtelLogSnapshot() {
|
|
|
229
449
|
}
|
|
230
450
|
/** Bounded provider flush belongs after final request and lifecycle publication. */
|
|
231
451
|
export async function flushProxyOtelLogs() {
|
|
452
|
+
await bodyPublicationChain;
|
|
232
453
|
await provider?.forceFlush();
|
|
233
454
|
}
|
|
234
455
|
/** Release this process's exporter and restore console ownership. */
|
|
235
456
|
export async function shutdownProxyOtelLogs() {
|
|
457
|
+
shuttingDown = true;
|
|
458
|
+
await bodyPublicationChain;
|
|
236
459
|
restoreConsole?.();
|
|
237
460
|
restoreConsole = undefined;
|
|
238
461
|
await provider?.shutdown();
|
|
239
462
|
provider = undefined;
|
|
240
463
|
queues.length = 0;
|
|
241
464
|
invalidRecords = 0;
|
|
465
|
+
bodyPublications.clear();
|
|
466
|
+
bodyPublicationChain = Promise.resolve();
|
|
467
|
+
shuttingDown = false;
|
|
468
|
+
for (const key of [
|
|
469
|
+
"attempted",
|
|
470
|
+
"transportAcknowledged",
|
|
471
|
+
"exportUnconfirmed",
|
|
472
|
+
"rejected",
|
|
473
|
+
"partial",
|
|
474
|
+
"pending",
|
|
475
|
+
"pendingBytes",
|
|
476
|
+
"highWaterPending",
|
|
477
|
+
"highWaterBytes",
|
|
478
|
+
]) {
|
|
479
|
+
bodyDelivery[key] = 0;
|
|
480
|
+
}
|
|
242
481
|
}
|
|
243
482
|
/** Flush short-lived proxy command diagnostics on every normal return or exception. */
|
|
244
483
|
export function withProxyOtelLogShutdown(handler) {
|
|
@@ -1,7 +1,9 @@
|
|
|
1
|
-
import type { ProxyActivitySnapshot, ProxyResponseTrackingObserver, RequestLogEntry } from "../types/index.js";
|
|
1
|
+
import type { ProxyActivitySnapshot, ProxyResponseTrackingObserver, RequestLogEntry, RequestAttemptLogEntry } from "../types/index.js";
|
|
2
2
|
/** Join route accounting to the HTTP lifecycle without relying on write order. */
|
|
3
|
-
export declare function observeProxyFinalLog(requestId: string, observer: (entry: RequestLogEntry) => void): () => void;
|
|
3
|
+
export declare function observeProxyFinalLog(requestId: string, observer: (entry: RequestLogEntry) => void, attemptObserver?: (entry: RequestAttemptLogEntry) => void): () => void;
|
|
4
4
|
export declare function notifyProxyFinalLog(entry: RequestLogEntry): void;
|
|
5
|
+
/** Retain only the last attempt for a currently observed HTTP request. */
|
|
6
|
+
export declare function notifyProxyAttemptLog(entry: RequestAttemptLogEntry): void;
|
|
5
7
|
export declare function registerProxyResponseObserver(metadata: object, observer: ProxyResponseTrackingObserver): void;
|
|
6
8
|
export declare function takeProxyResponseObservers(metadata: object): ProxyResponseTrackingObserver[];
|
|
7
9
|
/** Track one client-facing proxy request until its response body settles. */
|
|
@@ -9,18 +9,37 @@ let lastActivityAtMs = null;
|
|
|
9
9
|
// where bytes actually leave the proxy.
|
|
10
10
|
const responseObserversByMetadata = new WeakMap();
|
|
11
11
|
const finalLogObservers = new Map();
|
|
12
|
+
const attemptLogObservers = new Map();
|
|
12
13
|
/** Join route accounting to the HTTP lifecycle without relying on write order. */
|
|
13
|
-
export function observeProxyFinalLog(requestId, observer) {
|
|
14
|
+
export function observeProxyFinalLog(requestId, observer, attemptObserver) {
|
|
14
15
|
finalLogObservers.set(requestId, observer);
|
|
16
|
+
if (attemptObserver) {
|
|
17
|
+
attemptLogObservers.set(requestId, attemptObserver);
|
|
18
|
+
}
|
|
19
|
+
else {
|
|
20
|
+
attemptLogObservers.delete(requestId);
|
|
21
|
+
}
|
|
15
22
|
return () => {
|
|
16
23
|
if (finalLogObservers.get(requestId) === observer) {
|
|
17
24
|
finalLogObservers.delete(requestId);
|
|
25
|
+
attemptLogObservers.delete(requestId);
|
|
18
26
|
}
|
|
19
27
|
};
|
|
20
28
|
}
|
|
21
29
|
export function notifyProxyFinalLog(entry) {
|
|
22
30
|
finalLogObservers.get(entry.requestId)?.(entry);
|
|
23
31
|
}
|
|
32
|
+
/** Retain only the last attempt for a currently observed HTTP request. */
|
|
33
|
+
export function notifyProxyAttemptLog(entry) {
|
|
34
|
+
const observer = attemptLogObservers.get(entry.requestId);
|
|
35
|
+
observer?.(entry);
|
|
36
|
+
if (entry.parentRequestId && entry.parentRequestId !== entry.requestId) {
|
|
37
|
+
const parentObserver = attemptLogObservers.get(entry.parentRequestId);
|
|
38
|
+
if (parentObserver !== observer) {
|
|
39
|
+
parentObserver?.(entry);
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
}
|
|
24
43
|
export function registerProxyResponseObserver(metadata, observer) {
|
|
25
44
|
const existing = responseObserversByMetadata.get(metadata);
|
|
26
45
|
if (existing) {
|
|
@@ -33,6 +33,7 @@ export declare function prepareProxyBodyForLogging(body: unknown): {
|
|
|
33
33
|
value?: string;
|
|
34
34
|
bytes?: number;
|
|
35
35
|
truncated: boolean;
|
|
36
|
+
originalBytes?: number;
|
|
36
37
|
};
|
|
37
38
|
/** Capture an owned request body with bounded processing and tracked index/export publication. */
|
|
38
39
|
export declare function logBodyCapture(entry: ProxyBodyCaptureEntry): Promise<void>;
|