@juspay/neurolink 12.17.1 → 12.17.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -2
- package/dist/browser/neurolink.min.js +430 -421
- package/dist/cli/commands/proxy.js +455 -259
- package/dist/constants/contextWindows.d.ts +2 -0
- package/dist/constants/contextWindows.js +4 -0
- package/dist/neurolink.js +8 -3
- package/dist/providers/openaiChatCompletionsBase.js +60 -51
- package/dist/proxy/anthropicBudgetResponse.d.ts +3 -0
- package/dist/proxy/anthropicBudgetResponse.js +153 -0
- package/dist/proxy/bodyCaptureProcessing.d.ts +1 -0
- package/dist/proxy/bodyCaptureProcessing.js +23 -4
- package/dist/proxy/bodyCaptureWorker.js +20 -2
- package/dist/proxy/codexFallback.d.ts +10 -0
- package/dist/proxy/codexFallback.js +108 -24
- package/dist/proxy/codexUsage.js +2 -0
- package/dist/proxy/globalInstaller.d.ts +32 -1
- package/dist/proxy/globalInstaller.js +386 -3
- package/dist/proxy/otelLogSink.d.ts +3 -3
- package/dist/proxy/otelLogSink.js +137 -144
- package/dist/proxy/otlpLogTransport.d.ts +3 -0
- package/dist/proxy/otlpLogTransport.js +334 -0
- package/dist/proxy/proxyActivity.d.ts +22 -2
- package/dist/proxy/proxyActivity.js +114 -9
- package/dist/proxy/proxyAnalysis.js +82 -52
- package/dist/proxy/proxyClientCancellation.d.ts +7 -0
- package/dist/proxy/proxyClientCancellation.js +23 -0
- package/dist/proxy/proxyContextPreflight.d.ts +25 -0
- package/dist/proxy/proxyContextPreflight.js +337 -0
- package/dist/proxy/proxyFailureDetails.d.ts +19 -0
- package/dist/proxy/proxyFailureDetails.js +96 -0
- package/dist/proxy/proxyLifecycle.js +7 -0
- package/dist/proxy/proxyRouteBodyCapture.d.ts +13 -0
- package/dist/proxy/proxyRouteBodyCapture.js +112 -0
- package/dist/proxy/proxyTokenBudget.d.ts +40 -0
- package/dist/proxy/proxyTokenBudget.js +523 -0
- package/dist/proxy/proxyTracer.d.ts +3 -2
- package/dist/proxy/proxyTracer.js +46 -20
- package/dist/proxy/proxyTranslationEngine.d.ts +8 -10
- package/dist/proxy/proxyTranslationEngine.js +736 -376
- package/dist/proxy/requestLogger.d.ts +1 -0
- package/dist/proxy/requestLogger.js +107 -5
- package/dist/proxy/restartControl.js +9 -4
- package/dist/proxy/rollingWorkerProcess.js +65 -0
- package/dist/proxy/rollingWorkerSupervisor.js +7 -0
- package/dist/proxy/updateCoordinator.d.ts +14 -1
- package/dist/proxy/updateCoordinator.js +57 -2
- package/dist/server/routes/claudeProxyRoutes.d.ts +4 -5
- package/dist/server/routes/claudeProxyRoutes.js +809 -577
- package/dist/server/routes/codexProxyRoutes.js +191 -9
- package/dist/server/routes/openaiProxyRoutes.d.ts +1 -1
- package/dist/server/routes/openaiProxyRoutes.js +261 -69
- package/dist/types/cli.d.ts +3 -0
- package/dist/types/codex.d.ts +5 -1
- package/dist/types/index.d.ts +2 -0
- package/dist/types/index.js +2 -0
- package/dist/types/proxy.d.ts +148 -3
- package/dist/types/proxy.js +0 -15
- package/dist/types/proxyBudget.d.ts +74 -0
- package/dist/types/proxyBudget.js +1 -0
- package/dist/types/proxyContext.d.ts +35 -0
- package/dist/types/proxyContext.js +1 -0
- package/dist/utils/fileDetector.js +28 -7
- package/dist/utils/messageBuilder.js +51 -26
- package/dist/utils/pdfProcessor.js +26 -4
- package/docs/assets/dashboards/README.md +37 -0
- package/docs/assets/dashboards/neurolink-proxy-observability-dashboard.json +197 -89
- package/docs-site/static/search-index.json +1 -1
- package/package.json +7 -2
- package/scripts/observability/NATIVE-DURABLE.md +63 -0
- package/scripts/observability/otel-collector.proxy-native-durable.yaml +93 -0
- package/scripts/observability/proxy-telemetry-backend.mjs +32 -5
- package/scripts/observability/proxy-telemetry-check.mjs +396 -61
- package/scripts/observability/query-proxy-history.mjs +34 -5
|
@@ -69,7 +69,6 @@ function normalizeMetadataRows(kind, rows) {
|
|
|
69
69
|
? row.proxy_event_id
|
|
70
70
|
: undefined;
|
|
71
71
|
const physicalIdentity = JSON.stringify([
|
|
72
|
-
row._timestamp,
|
|
73
72
|
row.service_instance_id,
|
|
74
73
|
row.request_id,
|
|
75
74
|
row.body,
|
|
@@ -195,7 +194,10 @@ function normalizeBodyChunkRows(rows) {
|
|
|
195
194
|
for (const item of parsed) {
|
|
196
195
|
const index = Number(item.row.body_chunk_index);
|
|
197
196
|
if (!Number.isSafeInteger(index) || index < 0) {
|
|
198
|
-
conflicts.push({
|
|
197
|
+
conflicts.push({
|
|
198
|
+
identity: String(index),
|
|
199
|
+
reason: "invalid_chunk_index",
|
|
200
|
+
});
|
|
199
201
|
continue;
|
|
200
202
|
}
|
|
201
203
|
const prior = byChunkIndex.get(index);
|
|
@@ -230,6 +232,221 @@ function normalizeBodyChunkRows(rows) {
|
|
|
230
232
|
};
|
|
231
233
|
}
|
|
232
234
|
|
|
235
|
+
/**
|
|
236
|
+
* Reconcile admissions against endings in an explicit bounded history window.
|
|
237
|
+
* A missing ending is a telemetry/lifecycle gap, not proof of an upstream failure.
|
|
238
|
+
* @param {import("../../src/lib/types/index.js").ProxyTelemetryStoredRecord[]} lifecycle
|
|
239
|
+
* @param {{asOfMicroseconds: number, requestTimeoutMs: number, ingestionGraceMs: number, effectiveLookbackMs?: number}} options
|
|
240
|
+
*/
|
|
241
|
+
export function reconcileProxyAdmissions(
|
|
242
|
+
lifecycle,
|
|
243
|
+
{
|
|
244
|
+
asOfMicroseconds,
|
|
245
|
+
requestTimeoutMs,
|
|
246
|
+
ingestionGraceMs,
|
|
247
|
+
effectiveLookbackMs = Infinity,
|
|
248
|
+
},
|
|
249
|
+
) {
|
|
250
|
+
const terminals = new Set(
|
|
251
|
+
lifecycle
|
|
252
|
+
.filter(
|
|
253
|
+
(row) =>
|
|
254
|
+
row.event === "request_terminal" &&
|
|
255
|
+
!(Date.parse(row.timestamp ?? row.at ?? "") > asOfMicroseconds / 1000),
|
|
256
|
+
)
|
|
257
|
+
.map((row) => row.requestId),
|
|
258
|
+
);
|
|
259
|
+
const admissions = lifecycle.filter(
|
|
260
|
+
(row) =>
|
|
261
|
+
row.event === "request_accepted" &&
|
|
262
|
+
!(Date.parse(row.timestamp ?? row.at ?? "") > asOfMicroseconds / 1000),
|
|
263
|
+
);
|
|
264
|
+
const overdue = [];
|
|
265
|
+
const pending = [];
|
|
266
|
+
const invalid = [];
|
|
267
|
+
const insufficientWindow = [];
|
|
268
|
+
for (const row of admissions) {
|
|
269
|
+
const timeout = row.requestTimeoutMs ?? requestTimeoutMs;
|
|
270
|
+
if (
|
|
271
|
+
Number.isFinite(timeout) &&
|
|
272
|
+
timeout > 0 &&
|
|
273
|
+
effectiveLookbackMs <= timeout + ingestionGraceMs
|
|
274
|
+
) {
|
|
275
|
+
insufficientWindow.push({
|
|
276
|
+
requestId: row.requestId,
|
|
277
|
+
requestTimeoutMs: timeout,
|
|
278
|
+
ingestionGraceMs,
|
|
279
|
+
effectiveLookbackMs,
|
|
280
|
+
});
|
|
281
|
+
}
|
|
282
|
+
if (terminals.has(row.requestId)) {continue;}
|
|
283
|
+
const acceptedAt = Date.parse(row.timestamp ?? row.at ?? "");
|
|
284
|
+
if (
|
|
285
|
+
!row.requestId ||
|
|
286
|
+
!Number.isFinite(acceptedAt) ||
|
|
287
|
+
!Number.isFinite(timeout) ||
|
|
288
|
+
timeout <= 0
|
|
289
|
+
) {
|
|
290
|
+
invalid.push(row.requestId);
|
|
291
|
+
continue;
|
|
292
|
+
}
|
|
293
|
+
const ageMs = asOfMicroseconds / 1000 - acceptedAt;
|
|
294
|
+
const evidence = {
|
|
295
|
+
requestId: row.requestId,
|
|
296
|
+
acceptedAt: new Date(acceptedAt).toISOString(),
|
|
297
|
+
ageMs,
|
|
298
|
+
requestTimeoutMs: timeout,
|
|
299
|
+
};
|
|
300
|
+
if (ageMs > timeout + ingestionGraceMs) {overdue.push(evidence);}
|
|
301
|
+
else {pending.push(evidence);}
|
|
302
|
+
}
|
|
303
|
+
return {
|
|
304
|
+
admissions: admissions.length,
|
|
305
|
+
clientAdmissions: admissions.filter(
|
|
306
|
+
(row) => row.accountingScope !== "internal",
|
|
307
|
+
).length,
|
|
308
|
+
internalAdmissions: admissions.filter(
|
|
309
|
+
(row) => row.accountingScope === "internal",
|
|
310
|
+
).length,
|
|
311
|
+
ended: admissions.length - overdue.length - pending.length - invalid.length,
|
|
312
|
+
overdue,
|
|
313
|
+
pending,
|
|
314
|
+
invalid,
|
|
315
|
+
insufficientWindow,
|
|
316
|
+
};
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
/** Verify both ends of bridge accounting inside the selected final-record window.
|
|
320
|
+
* A missing linked final may be outside the window or not yet ingested; it is
|
|
321
|
+
* inconclusive, never proof that the designated usage owner was recorded.
|
|
322
|
+
* @param {import("../../src/lib/types/index.js").ProxyTelemetryStoredRecord[]} finals
|
|
323
|
+
*/
|
|
324
|
+
export function reconcileProxyUsageOwnership(finals) {
|
|
325
|
+
const byId = new Map(finals.map((row) => [row.requestId, row]));
|
|
326
|
+
const duplicateOwnerUsage = finals.filter(
|
|
327
|
+
(row) =>
|
|
328
|
+
row.usageOwnerRequestId &&
|
|
329
|
+
row.usageOwnerRequestId !== row.requestId &&
|
|
330
|
+
[
|
|
331
|
+
row.inputTokens,
|
|
332
|
+
row.outputTokens,
|
|
333
|
+
row.cacheReadTokens,
|
|
334
|
+
row.cacheCreationTokens,
|
|
335
|
+
row.reasoningTokens,
|
|
336
|
+
].some((value) => value !== undefined),
|
|
337
|
+
);
|
|
338
|
+
const missingOwners = [];
|
|
339
|
+
const missingParents = [];
|
|
340
|
+
const mismatches = [];
|
|
341
|
+
for (const row of finals) {
|
|
342
|
+
if (row.usageOwnerRequestId && row.usageOwnerRequestId !== row.requestId) {
|
|
343
|
+
const owner = byId.get(row.usageOwnerRequestId);
|
|
344
|
+
if (!owner) {
|
|
345
|
+
missingOwners.push({
|
|
346
|
+
requestId: row.requestId,
|
|
347
|
+
usageOwnerRequestId: row.usageOwnerRequestId,
|
|
348
|
+
});
|
|
349
|
+
} else if (
|
|
350
|
+
owner.accountingScope !== "internal" ||
|
|
351
|
+
owner.parentRequestId !== row.requestId ||
|
|
352
|
+
(owner.usageOwnerRequestId &&
|
|
353
|
+
owner.usageOwnerRequestId !== owner.requestId)
|
|
354
|
+
) {
|
|
355
|
+
mismatches.push({
|
|
356
|
+
requestId: row.requestId,
|
|
357
|
+
linkedRequestId: owner.requestId,
|
|
358
|
+
reason: "invalid_usage_owner_link",
|
|
359
|
+
});
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
if (row.accountingScope === "internal") {
|
|
363
|
+
if (!row.parentRequestId) {
|
|
364
|
+
mismatches.push({
|
|
365
|
+
requestId: row.requestId,
|
|
366
|
+
reason: "internal_request_has_no_parent",
|
|
367
|
+
});
|
|
368
|
+
continue;
|
|
369
|
+
}
|
|
370
|
+
const parent = byId.get(row.parentRequestId);
|
|
371
|
+
if (!parent) {
|
|
372
|
+
missingParents.push({
|
|
373
|
+
requestId: row.requestId,
|
|
374
|
+
parentRequestId: row.parentRequestId,
|
|
375
|
+
});
|
|
376
|
+
} else if (parent.usageOwnerRequestId !== row.requestId) {
|
|
377
|
+
mismatches.push({
|
|
378
|
+
requestId: row.requestId,
|
|
379
|
+
linkedRequestId: parent.requestId,
|
|
380
|
+
reason: "parent_does_not_designate_child",
|
|
381
|
+
});
|
|
382
|
+
}
|
|
383
|
+
}
|
|
384
|
+
}
|
|
385
|
+
/** @type {import("../../src/lib/types/index.js").ProxyTelemetryCheck["status"]} */
|
|
386
|
+
const status =
|
|
387
|
+
duplicateOwnerUsage.length || mismatches.length
|
|
388
|
+
? "fail"
|
|
389
|
+
: missingOwners.length || missingParents.length || !finals.length
|
|
390
|
+
? "unverified"
|
|
391
|
+
: "pass";
|
|
392
|
+
return {
|
|
393
|
+
status,
|
|
394
|
+
clientFinals: finals.filter((row) => row.accountingScope !== "internal")
|
|
395
|
+
.length,
|
|
396
|
+
internalFinals: finals.filter((row) => row.accountingScope === "internal")
|
|
397
|
+
.length,
|
|
398
|
+
duplicateOwnerUsage: duplicateOwnerUsage.length,
|
|
399
|
+
requestIds: duplicateOwnerUsage.slice(0, 50).map((row) => row.requestId),
|
|
400
|
+
missingOwnerCount: missingOwners.length,
|
|
401
|
+
missingParentCount: missingParents.length,
|
|
402
|
+
mismatchCount: mismatches.length,
|
|
403
|
+
missingOwners: missingOwners.slice(0, 50),
|
|
404
|
+
missingParents: missingParents.slice(0, 50),
|
|
405
|
+
mismatches: mismatches.slice(0, 50),
|
|
406
|
+
boundary:
|
|
407
|
+
"Both linked finals must be present and reciprocal in this bounded selected window. Missing owners/parents may be outside it or not ingested; widen the interval or rerun before claiming complete usage accounting.",
|
|
408
|
+
};
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
/** All shipped generation routes require a client-response capture outcome.
|
|
412
|
+
* @param {import("../../src/lib/types/index.js").ProxyTelemetryStoredRecord[]} finals
|
|
413
|
+
* @param {import("../../src/lib/types/index.js").ProxyTelemetryStoredRecord[]} indexes
|
|
414
|
+
*/
|
|
415
|
+
export function reconcileProxyCaptureCoverage(finals, indexes) {
|
|
416
|
+
const responseCaptures = new Set(
|
|
417
|
+
indexes
|
|
418
|
+
.filter((row) => row.phase === "client_response")
|
|
419
|
+
.map((row) => row.requestId),
|
|
420
|
+
);
|
|
421
|
+
const eligible = finals.filter(
|
|
422
|
+
(row) =>
|
|
423
|
+
row.path &&
|
|
424
|
+
([
|
|
425
|
+
"/v1/messages",
|
|
426
|
+
"/backend-api/codex/responses",
|
|
427
|
+
"/v1/responses",
|
|
428
|
+
"/v1/chat/completions",
|
|
429
|
+
].includes(row.path) ||
|
|
430
|
+
/^\/v1beta\/models\/[^/]+:(?:generateContent|streamGenerateContent)$/.test(
|
|
431
|
+
row.path,
|
|
432
|
+
)),
|
|
433
|
+
);
|
|
434
|
+
const missing = eligible
|
|
435
|
+
.filter((row) => !responseCaptures.has(row.requestId))
|
|
436
|
+
.map((row) => ({
|
|
437
|
+
requestId: row.requestId,
|
|
438
|
+
path: row.path,
|
|
439
|
+
missingPhase: "client_response",
|
|
440
|
+
}));
|
|
441
|
+
/** @type {import("../../src/lib/types/index.js").ProxyTelemetryCheck["status"]} */
|
|
442
|
+
const status = !eligible.length
|
|
443
|
+
? "unverified"
|
|
444
|
+
: missing.length
|
|
445
|
+
? "fail"
|
|
446
|
+
: "pass";
|
|
447
|
+
return { status, eligibleFinals: eligible.length, missing };
|
|
448
|
+
}
|
|
449
|
+
|
|
233
450
|
/** Empty traffic, unavailable evidence, and partial queries cannot become a green check.
|
|
234
451
|
* @param {import("../../src/lib/types/index.js").ProxyTelemetryDoctorOptions} options
|
|
235
452
|
*/
|
|
@@ -239,6 +456,9 @@ export async function checkProxyTelemetry({
|
|
|
239
456
|
endTime,
|
|
240
457
|
proxyUrl = "http://127.0.0.1:55669",
|
|
241
458
|
maxRows = 10000,
|
|
459
|
+
admissionLookbackMs = 24 * 60 * 60 * 1000,
|
|
460
|
+
requestTimeoutMs = 15 * 60 * 1000,
|
|
461
|
+
ingestionGraceMs = 120000,
|
|
242
462
|
fetchImpl = fetch,
|
|
243
463
|
}) {
|
|
244
464
|
validateProxyTelemetryBackend(backend);
|
|
@@ -249,12 +469,25 @@ export async function checkProxyTelemetry({
|
|
|
249
469
|
startTime >= endTime ||
|
|
250
470
|
!Number.isSafeInteger(maxRows) ||
|
|
251
471
|
maxRows < 1 ||
|
|
252
|
-
maxRows > 100000
|
|
472
|
+
maxRows > 100000 ||
|
|
473
|
+
!Number.isSafeInteger(admissionLookbackMs) ||
|
|
474
|
+
admissionLookbackMs < 1 ||
|
|
475
|
+
admissionLookbackMs > 7 * 24 * 60 * 60 * 1000 ||
|
|
476
|
+
!Number.isSafeInteger(requestTimeoutMs) ||
|
|
477
|
+
requestTimeoutMs < 1 ||
|
|
478
|
+
!Number.isSafeInteger(ingestionGraceMs) ||
|
|
479
|
+
ingestionGraceMs < 0
|
|
253
480
|
) {
|
|
254
481
|
throw new Error(
|
|
255
482
|
"Provide an increasing microsecond time range and maxRows between 1 and 100000",
|
|
256
483
|
);
|
|
257
484
|
}
|
|
485
|
+
if (admissionLookbackMs <= requestTimeoutMs + ingestionGraceMs) {
|
|
486
|
+
throw new Error(
|
|
487
|
+
"The admission lookback must exceed the request timeout plus ingestion grace",
|
|
488
|
+
);
|
|
489
|
+
}
|
|
490
|
+
const checkedAtMicroseconds = Date.now() * 1000;
|
|
258
491
|
const budget = { used: 0, limit: 512 };
|
|
259
492
|
/** @type {import("../../src/lib/types/index.js").ProxyTelemetryCheck[]} */
|
|
260
493
|
const checks = [];
|
|
@@ -312,15 +545,37 @@ export async function checkProxyTelemetry({
|
|
|
312
545
|
const supervisorProcess = supervisor?.process;
|
|
313
546
|
const workerProcess = runtime.observability?.process;
|
|
314
547
|
/** @param {import("../../src/lib/types/index.js").ProxyProcessTelemetrySnapshot | undefined} process */
|
|
315
|
-
const otelProcess = (process) =>
|
|
316
|
-
process
|
|
317
|
-
process.
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
548
|
+
const otelProcess = (process) =>
|
|
549
|
+
process?.configuredSink === "otel" &&
|
|
550
|
+
process.lifecycleSink === "otel" &&
|
|
551
|
+
process.otelInitialized &&
|
|
552
|
+
process.stdio?.stdout === "non_file" &&
|
|
553
|
+
process.stdio?.stderr === "non_file";
|
|
554
|
+
add(
|
|
555
|
+
"worker_process_logging",
|
|
556
|
+
!workerProcess
|
|
557
|
+
? "unverified"
|
|
558
|
+
: otelProcess(workerProcess)
|
|
559
|
+
? "pass"
|
|
560
|
+
: "fail",
|
|
561
|
+
workerProcess ?? { reason: "process_telemetry_missing" },
|
|
562
|
+
);
|
|
563
|
+
add(
|
|
564
|
+
"supervisor_logging",
|
|
565
|
+
supervisor?.status === "not_applicable" &&
|
|
566
|
+
!runtime.autoUpdate?.supervisorPid
|
|
567
|
+
? "pass"
|
|
568
|
+
: supervisor?.status !== "available"
|
|
569
|
+
? "unverified"
|
|
570
|
+
: supervisorProcess?.pid === runtime.autoUpdate?.supervisorPid &&
|
|
571
|
+
otelProcess(supervisorProcess)
|
|
572
|
+
? "pass"
|
|
573
|
+
: "fail",
|
|
574
|
+
supervisor ?? {
|
|
575
|
+
status: "unavailable",
|
|
576
|
+
reason: "supervisor_telemetry_missing",
|
|
577
|
+
},
|
|
578
|
+
);
|
|
324
579
|
/** @type {Array<{kind: string, dropped: number, exportUnconfirmed: number, outstanding: number, capacity: number, recentFailures?: unknown, failureHistoryEvicted?: number}>} */
|
|
325
580
|
const failures = (logs?.otel?.queues ?? []).map(
|
|
326
581
|
(
|
|
@@ -392,6 +647,8 @@ export async function checkProxyTelemetry({
|
|
|
392
647
|
queries.push(...result.queries.map((q) => ({ ...q, kind })));
|
|
393
648
|
const normalized = normalizeMetadataRows(kind, result.records);
|
|
394
649
|
history[kind] = normalized.records;
|
|
650
|
+
normalized.diagnostics.rawRecords = result.rawRecordCount;
|
|
651
|
+
normalized.diagnostics.exactRetryDuplicates += result.exactRetryDuplicates;
|
|
395
652
|
historyDiagnostics[kind] = normalized.diagnostics;
|
|
396
653
|
}
|
|
397
654
|
const identityConflicts = Object.values(historyDiagnostics).reduce(
|
|
@@ -405,11 +662,7 @@ export async function checkProxyTelemetry({
|
|
|
405
662
|
);
|
|
406
663
|
add(
|
|
407
664
|
"backend_row_identity",
|
|
408
|
-
identityConflicts
|
|
409
|
-
? "fail"
|
|
410
|
-
: missingEventIds
|
|
411
|
-
? "unverified"
|
|
412
|
-
: "pass",
|
|
665
|
+
identityConflicts ? "fail" : missingEventIds ? "unverified" : "pass",
|
|
413
666
|
{
|
|
414
667
|
scope: "selected metadata and bounded capture-settling windows",
|
|
415
668
|
exactRetryDuplicates: Object.values(historyDiagnostics).reduce(
|
|
@@ -424,15 +677,10 @@ export async function checkProxyTelemetry({
|
|
|
424
677
|
/** @param {{at?: unknown}} row */
|
|
425
678
|
const failedInsideInterval = (row) => {
|
|
426
679
|
const at = Date.parse(typeof row.at === "string" ? row.at : "");
|
|
427
|
-
return (
|
|
428
|
-
Number.isFinite(at) &&
|
|
429
|
-
at * 1000 >= startTime &&
|
|
430
|
-
at * 1000 < endTime
|
|
431
|
-
);
|
|
680
|
+
return Number.isFinite(at) && at * 1000 >= startTime && at * 1000 < endTime;
|
|
432
681
|
};
|
|
433
|
-
const storedDeliveryFailures =
|
|
434
|
-
failedInsideInterval
|
|
435
|
-
);
|
|
682
|
+
const storedDeliveryFailures =
|
|
683
|
+
history.telemetry_delivery.filter(failedInsideInterval);
|
|
436
684
|
const runtimeDeliveryFailures = failures.flatMap((queue) =>
|
|
437
685
|
Array.isArray(queue.recentFailures)
|
|
438
686
|
? queue.recentFailures.filter(
|
|
@@ -447,9 +695,7 @@ export async function checkProxyTelemetry({
|
|
|
447
695
|
...new Map(
|
|
448
696
|
[...storedDeliveryFailures, ...runtimeDeliveryFailures].map(
|
|
449
697
|
(failure, index) => [
|
|
450
|
-
typeof failure.id === "string"
|
|
451
|
-
? failure.id
|
|
452
|
-
: `unidentified-${index}`,
|
|
698
|
+
typeof failure.id === "string" ? failure.id : `unidentified-${index}`,
|
|
453
699
|
failure,
|
|
454
700
|
],
|
|
455
701
|
),
|
|
@@ -521,11 +767,9 @@ export async function checkProxyTelemetry({
|
|
|
521
767
|
const unhealthyCaptures = captureIndexes.filter(
|
|
522
768
|
(row) =>
|
|
523
769
|
row.captureError !== undefined ||
|
|
524
|
-
![
|
|
525
|
-
"
|
|
526
|
-
|
|
527
|
-
"policy_excluded",
|
|
528
|
-
].includes(row.bodyDelivery?.status ?? ""),
|
|
770
|
+
!["transport_acknowledged", "no_body", "policy_excluded"].includes(
|
|
771
|
+
row.bodyDelivery?.status ?? "",
|
|
772
|
+
),
|
|
529
773
|
);
|
|
530
774
|
add(
|
|
531
775
|
"capture_admission",
|
|
@@ -538,9 +782,7 @@ export async function checkProxyTelemetry({
|
|
|
538
782
|
scope: "selected_interval",
|
|
539
783
|
captures: captureIndexes.length,
|
|
540
784
|
unhealthy: unhealthyCaptures.length,
|
|
541
|
-
requestIds: unhealthyCaptures
|
|
542
|
-
.slice(0, 50)
|
|
543
|
-
.map((row) => row.requestId),
|
|
785
|
+
requestIds: unhealthyCaptures.slice(0, 50).map((row) => row.requestId),
|
|
544
786
|
workerLifetime: {
|
|
545
787
|
scope: "worker_lifetime",
|
|
546
788
|
counters: logs?.bodyCapture ?? { status: "unavailable" },
|
|
@@ -561,12 +803,18 @@ export async function checkProxyTelemetry({
|
|
|
561
803
|
: "fail",
|
|
562
804
|
{
|
|
563
805
|
records: finals.length,
|
|
806
|
+
clientFinals: finals.filter((row) => row.accountingScope !== "internal")
|
|
807
|
+
.length,
|
|
808
|
+
internalFinals: finals.filter((row) => row.accountingScope === "internal")
|
|
809
|
+
.length,
|
|
564
810
|
rawRecords: finalDiagnostics.rawRecords,
|
|
565
811
|
exactRetryDuplicates: finalDiagnostics.exactRetryDuplicates,
|
|
566
812
|
uniqueRequestIds: unique.size,
|
|
567
813
|
producerOrConflictingDuplicates: finalDiagnostics.conflicts.length,
|
|
568
814
|
},
|
|
569
815
|
);
|
|
816
|
+
const ownership = reconcileProxyUsageOwnership(finals);
|
|
817
|
+
add("bridge_usage_ownership", ownership.status, ownership);
|
|
570
818
|
const terminalEvents = history.lifecycle.filter(
|
|
571
819
|
(row) => row.event === "request_terminal",
|
|
572
820
|
);
|
|
@@ -593,6 +841,80 @@ export async function checkProxyTelemetry({
|
|
|
593
841
|
"Terminals inside the selected interval; admissions still in flight are not failures",
|
|
594
842
|
},
|
|
595
843
|
);
|
|
844
|
+
add(
|
|
845
|
+
"ingestion_boundary",
|
|
846
|
+
checkedAtMicroseconds - endTime >= ingestionGraceMs * 1000
|
|
847
|
+
? "pass"
|
|
848
|
+
: "unverified",
|
|
849
|
+
{
|
|
850
|
+
endTimeExclusive: endTime,
|
|
851
|
+
queriedAtMicroseconds: checkedAtMicroseconds,
|
|
852
|
+
ingestionGraceMs,
|
|
853
|
+
boundary:
|
|
854
|
+
"A bounded event-time query cannot prove that no late records will arrive; rerun after the ingestion grace period",
|
|
855
|
+
},
|
|
856
|
+
);
|
|
857
|
+
const admissionStart = Math.max(0, endTime - admissionLookbackMs * 1000);
|
|
858
|
+
const effectiveLookbackMs = (endTime - admissionStart) / 1000;
|
|
859
|
+
const admissionHistory = await queryProxyHistory({
|
|
860
|
+
...backend,
|
|
861
|
+
startTime: admissionStart,
|
|
862
|
+
endTime: Math.min(checkedAtMicroseconds, endTime + ingestionGraceMs * 1000),
|
|
863
|
+
kind: "lifecycle",
|
|
864
|
+
lifecycleEvents: ["request_accepted", "request_terminal"],
|
|
865
|
+
maxRows,
|
|
866
|
+
fetchImpl,
|
|
867
|
+
budget,
|
|
868
|
+
});
|
|
869
|
+
queries.push(
|
|
870
|
+
...admissionHistory.queries.map((query) => ({
|
|
871
|
+
...query,
|
|
872
|
+
kind: "admission_reconciliation",
|
|
873
|
+
})),
|
|
874
|
+
);
|
|
875
|
+
const normalizedAdmissions = normalizeMetadataRows(
|
|
876
|
+
"lifecycle",
|
|
877
|
+
admissionHistory.records,
|
|
878
|
+
);
|
|
879
|
+
const admissionResult = reconcileProxyAdmissions(
|
|
880
|
+
normalizedAdmissions.records,
|
|
881
|
+
{
|
|
882
|
+
asOfMicroseconds: endTime,
|
|
883
|
+
requestTimeoutMs,
|
|
884
|
+
ingestionGraceMs,
|
|
885
|
+
effectiveLookbackMs,
|
|
886
|
+
},
|
|
887
|
+
);
|
|
888
|
+
const windowSufficient =
|
|
889
|
+
effectiveLookbackMs > requestTimeoutMs + ingestionGraceMs &&
|
|
890
|
+
!admissionResult.insufficientWindow.length;
|
|
891
|
+
add(
|
|
892
|
+
"admission_reconciliation",
|
|
893
|
+
admissionResult.overdue.length ||
|
|
894
|
+
admissionResult.invalid.length ||
|
|
895
|
+
normalizedAdmissions.diagnostics.conflicts.length
|
|
896
|
+
? "fail"
|
|
897
|
+
: admissionResult.admissions && windowSufficient
|
|
898
|
+
? "pass"
|
|
899
|
+
: "unverified",
|
|
900
|
+
{
|
|
901
|
+
...admissionResult,
|
|
902
|
+
overdue: admissionResult.overdue.slice(0, 50),
|
|
903
|
+
pending: admissionResult.pending.slice(0, 50),
|
|
904
|
+
overdueCount: admissionResult.overdue.length,
|
|
905
|
+
pendingCount: admissionResult.pending.length,
|
|
906
|
+
insufficientWindow: admissionResult.insufficientWindow.slice(0, 50),
|
|
907
|
+
insufficientWindowCount: admissionResult.insufficientWindow.length,
|
|
908
|
+
windowSufficient,
|
|
909
|
+
effectiveLookbackMs,
|
|
910
|
+
requestTimeoutMs,
|
|
911
|
+
ingestionGraceMs,
|
|
912
|
+
startTime: admissionStart,
|
|
913
|
+
endTimeExclusive: admissionHistory.endTimeExclusive,
|
|
914
|
+
boundary:
|
|
915
|
+
"Missing endings after the request deadline plus ingestion grace are lifecycle/telemetry gaps, not inferred provider failures. Admissions before this explicit lookback are unverified. A lookback that cannot exceed the default or an observed request deadline plus grace is insufficient for a pass.",
|
|
916
|
+
},
|
|
917
|
+
);
|
|
596
918
|
/** @type {Record<string, import("../../src/lib/types/index.js").ProxyTelemetryFieldCoverage>} */
|
|
597
919
|
const coverage = Object.create(null);
|
|
598
920
|
for (const row of finals) {
|
|
@@ -669,25 +991,18 @@ export async function checkProxyTelemetry({
|
|
|
669
991
|
: "fail",
|
|
670
992
|
coverage,
|
|
671
993
|
);
|
|
672
|
-
const indexes =
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
// A request may have started outside the selected interval. Its response
|
|
685
|
-
// capture still has to be present when this final is inside the interval.
|
|
686
|
-
return phasesByRequest.get(row.requestId)?.has("client_response") ? [] : [{ requestId: row.requestId, missingPhase: "client_response" }];
|
|
687
|
-
});
|
|
688
|
-
add("request_capture_coverage", !captureFinals.length ? "unverified" : missingCapturePhases.length ? "fail" : "pass", {
|
|
689
|
-
finals: finals.length, eligibleFinals: captureFinals.length, missing: missingCapturePhases.length,
|
|
690
|
-
examples: missingCapturePhases.slice(0, 50), omitted: Math.max(0, missingCapturePhases.length - 50),
|
|
994
|
+
const indexes = captureIndexes;
|
|
995
|
+
const captureCoverage = reconcileProxyCaptureCoverage(
|
|
996
|
+
finals,
|
|
997
|
+
history.body_capture_index,
|
|
998
|
+
);
|
|
999
|
+
const missingCapturePhases = captureCoverage.missing;
|
|
1000
|
+
add("request_capture_coverage", captureCoverage.status, {
|
|
1001
|
+
finals: finals.length,
|
|
1002
|
+
eligibleFinals: captureCoverage.eligibleFinals,
|
|
1003
|
+
missing: missingCapturePhases.length,
|
|
1004
|
+
examples: missingCapturePhases.slice(0, 50),
|
|
1005
|
+
omitted: Math.max(0, missingCapturePhases.length - 50),
|
|
691
1006
|
});
|
|
692
1007
|
/** @type {Record<string, number>} */
|
|
693
1008
|
const byDelivery = Object.create(null);
|
|
@@ -721,6 +1036,20 @@ export async function checkProxyTelemetry({
|
|
|
721
1036
|
failuresOmitted: Math.max(0, captureFailures.length - 50),
|
|
722
1037
|
},
|
|
723
1038
|
);
|
|
1039
|
+
const lossyCaptures = indexes.filter(
|
|
1040
|
+
(row) => row.bodyTruncated || row.redactionLossy,
|
|
1041
|
+
);
|
|
1042
|
+
add(
|
|
1043
|
+
"capture_content_completeness",
|
|
1044
|
+
lossyCaptures.length ? "warn" : indexes.length ? "pass" : "unverified",
|
|
1045
|
+
{
|
|
1046
|
+
captures: indexes.length,
|
|
1047
|
+
truncatedOrUnparseable: lossyCaptures.length,
|
|
1048
|
+
captureIds: lossyCaptures.slice(0, 50).map((row) => row.captureId),
|
|
1049
|
+
boundary:
|
|
1050
|
+
"Transport acknowledgement does not restore source truncation, processing limits, or invalid frames removed by redaction",
|
|
1051
|
+
},
|
|
1052
|
+
);
|
|
724
1053
|
const bodyChecks = [];
|
|
725
1054
|
for (const index of indexes
|
|
726
1055
|
.filter(
|
|
@@ -740,7 +1069,7 @@ export async function checkProxyTelemetry({
|
|
|
740
1069
|
throw new Error("Capture verification exceeds the 8 MiB per-body bound");
|
|
741
1070
|
}
|
|
742
1071
|
const rawChunks = await query(
|
|
743
|
-
`SELECT _timestamp, proxy_event_id, body_capture_id, body_chunk_index, body_chunk_count, body FROM "${backend.stream}" WHERE proxy_record_kind='body' AND body_capture_id='${index.captureId}' ORDER BY body_chunk_index ASC, proxy_event_id ASC`,
|
|
1072
|
+
`SELECT _timestamp, proxy_event_id, body_capture_id, body_chunk_index, body_chunk_count, body FROM "${backend.bodyStream ?? backend.stream}" WHERE proxy_record_kind='body' AND body_capture_id='${index.captureId}' ORDER BY body_chunk_index ASC, proxy_event_id ASC`,
|
|
744
1073
|
"logs",
|
|
745
1074
|
{
|
|
746
1075
|
startTime: startTime - 120e6,
|
|
@@ -775,8 +1104,7 @@ export async function checkProxyTelemetry({
|
|
|
775
1104
|
expectedChunks: expected,
|
|
776
1105
|
storedChunks: chunks.length,
|
|
777
1106
|
rawStoredChunks: rawChunks.length,
|
|
778
|
-
exactRetryDuplicates:
|
|
779
|
-
normalizedChunks.diagnostics.exactRetryDuplicates,
|
|
1107
|
+
exactRetryDuplicates: normalizedChunks.diagnostics.exactRetryDuplicates,
|
|
780
1108
|
missingEventIds: normalizedChunks.diagnostics.missingEventIds,
|
|
781
1109
|
identityConflicts: normalizedChunks.diagnostics.conflicts,
|
|
782
1110
|
verified:
|
|
@@ -897,7 +1225,10 @@ export async function checkProxyTelemetry({
|
|
|
897
1225
|
key,
|
|
898
1226
|
value,
|
|
899
1227
|
})),
|
|
900
|
-
invalidCounters: invalidCounters.map(([key, value]) => ({
|
|
1228
|
+
invalidCounters: invalidCounters.map(([key, value]) => ({
|
|
1229
|
+
key,
|
|
1230
|
+
value,
|
|
1231
|
+
})),
|
|
901
1232
|
counters,
|
|
902
1233
|
},
|
|
903
1234
|
);
|
|
@@ -935,13 +1266,13 @@ export async function runProxyTelemetryDoctor() {
|
|
|
935
1266
|
};
|
|
936
1267
|
if (args.includes("--help")) {
|
|
937
1268
|
console.log(
|
|
938
|
-
"Usage: neurolink proxy telemetry doctor [--since ISO_DATE] [--until ISO_DATE] [--format json|text] [--proxy-url URL] [--max-rows 10000]",
|
|
1269
|
+
"Usage: neurolink proxy telemetry doctor [--since ISO_DATE] [--until ISO_DATE] [--format json|text] [--proxy-url URL] [--max-rows 10000] [--admission-lookback-minutes 1440] [--request-timeout-ms 900000] [--ingestion-grace-ms 120000]",
|
|
939
1270
|
);
|
|
940
1271
|
return;
|
|
941
1272
|
}
|
|
942
1273
|
const until = value("--until")
|
|
943
1274
|
? Date.parse(value("--until") ?? "")
|
|
944
|
-
: Date.now() -
|
|
1275
|
+
: Date.now() - 120000;
|
|
945
1276
|
const since = value("--since")
|
|
946
1277
|
? Date.parse(value("--since") ?? "")
|
|
947
1278
|
: until - 15 * 60000;
|
|
@@ -951,6 +1282,10 @@ export async function runProxyTelemetryDoctor() {
|
|
|
951
1282
|
endTime: until * 1000,
|
|
952
1283
|
proxyUrl: value("--proxy-url") ?? process.env.NEUROLINK_PROXY_URL,
|
|
953
1284
|
maxRows: Number(value("--max-rows") ?? 10000),
|
|
1285
|
+
admissionLookbackMs:
|
|
1286
|
+
Number(value("--admission-lookback-minutes") ?? 1440) * 60000,
|
|
1287
|
+
requestTimeoutMs: Number(value("--request-timeout-ms") ?? 900000),
|
|
1288
|
+
ingestionGraceMs: Number(value("--ingestion-grace-ms") ?? 120000),
|
|
954
1289
|
});
|
|
955
1290
|
if (value("--format") === "json") {
|
|
956
1291
|
console.log(JSON.stringify(report, null, 2));
|