@juspay/neurolink 12.17.0 → 12.17.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. package/CHANGELOG.md +2 -7
  2. package/dist/browser/neurolink.min.js +422 -413
  3. package/dist/cli/commands/proxy.js +455 -259
  4. package/dist/constants/contextWindows.d.ts +2 -0
  5. package/dist/constants/contextWindows.js +4 -0
  6. package/dist/providers/anthropic/client.js +15 -5
  7. package/dist/proxy/anthropicBudgetResponse.d.ts +3 -0
  8. package/dist/proxy/anthropicBudgetResponse.js +153 -0
  9. package/dist/proxy/bodyCaptureProcessing.d.ts +1 -0
  10. package/dist/proxy/bodyCaptureProcessing.js +23 -4
  11. package/dist/proxy/bodyCaptureWorker.js +20 -2
  12. package/dist/proxy/codexFallback.d.ts +10 -0
  13. package/dist/proxy/codexFallback.js +108 -24
  14. package/dist/proxy/codexUsage.js +2 -0
  15. package/dist/proxy/globalInstaller.d.ts +32 -1
  16. package/dist/proxy/globalInstaller.js +386 -3
  17. package/dist/proxy/otelLogSink.d.ts +3 -3
  18. package/dist/proxy/otelLogSink.js +137 -144
  19. package/dist/proxy/otlpLogTransport.d.ts +3 -0
  20. package/dist/proxy/otlpLogTransport.js +334 -0
  21. package/dist/proxy/proxyActivity.d.ts +22 -2
  22. package/dist/proxy/proxyActivity.js +114 -9
  23. package/dist/proxy/proxyAnalysis.js +82 -52
  24. package/dist/proxy/proxyClientCancellation.d.ts +7 -0
  25. package/dist/proxy/proxyClientCancellation.js +23 -0
  26. package/dist/proxy/proxyContextPreflight.d.ts +25 -0
  27. package/dist/proxy/proxyContextPreflight.js +337 -0
  28. package/dist/proxy/proxyFailureDetails.d.ts +19 -0
  29. package/dist/proxy/proxyFailureDetails.js +96 -0
  30. package/dist/proxy/proxyLifecycle.js +7 -0
  31. package/dist/proxy/proxyRouteBodyCapture.d.ts +13 -0
  32. package/dist/proxy/proxyRouteBodyCapture.js +112 -0
  33. package/dist/proxy/proxyTokenBudget.d.ts +40 -0
  34. package/dist/proxy/proxyTokenBudget.js +523 -0
  35. package/dist/proxy/proxyTracer.d.ts +3 -2
  36. package/dist/proxy/proxyTracer.js +46 -20
  37. package/dist/proxy/proxyTranslationEngine.d.ts +8 -10
  38. package/dist/proxy/proxyTranslationEngine.js +736 -376
  39. package/dist/proxy/requestLogger.d.ts +1 -0
  40. package/dist/proxy/requestLogger.js +107 -5
  41. package/dist/proxy/restartControl.js +9 -4
  42. package/dist/proxy/rollingWorkerProcess.js +65 -0
  43. package/dist/proxy/rollingWorkerSupervisor.js +7 -0
  44. package/dist/proxy/updateCoordinator.d.ts +14 -1
  45. package/dist/proxy/updateCoordinator.js +57 -2
  46. package/dist/server/routes/claudeProxyRoutes.d.ts +4 -5
  47. package/dist/server/routes/claudeProxyRoutes.js +809 -577
  48. package/dist/server/routes/codexProxyRoutes.js +191 -9
  49. package/dist/server/routes/openaiProxyRoutes.d.ts +1 -1
  50. package/dist/server/routes/openaiProxyRoutes.js +261 -69
  51. package/dist/types/cli.d.ts +3 -0
  52. package/dist/types/codex.d.ts +5 -1
  53. package/dist/types/index.d.ts +2 -0
  54. package/dist/types/index.js +2 -0
  55. package/dist/types/proxy.d.ts +148 -3
  56. package/dist/types/proxy.js +0 -15
  57. package/dist/types/proxyBudget.d.ts +74 -0
  58. package/dist/types/proxyBudget.js +1 -0
  59. package/dist/types/proxyContext.d.ts +35 -0
  60. package/dist/types/proxyContext.js +1 -0
  61. package/dist/utils/toolChoice.d.ts +19 -0
  62. package/dist/utils/toolChoice.js +39 -0
  63. package/docs/assets/dashboards/README.md +37 -0
  64. package/docs/assets/dashboards/neurolink-proxy-observability-dashboard.json +197 -89
  65. package/docs-site/static/search-index.json +1 -1
  66. package/package.json +4 -1
  67. package/scripts/observability/NATIVE-DURABLE.md +63 -0
  68. package/scripts/observability/otel-collector.proxy-native-durable.yaml +93 -0
  69. package/scripts/observability/proxy-telemetry-backend.mjs +32 -5
  70. package/scripts/observability/proxy-telemetry-check.mjs +396 -61
  71. package/scripts/observability/query-proxy-history.mjs +34 -5
@@ -69,7 +69,6 @@ function normalizeMetadataRows(kind, rows) {
69
69
  ? row.proxy_event_id
70
70
  : undefined;
71
71
  const physicalIdentity = JSON.stringify([
72
- row._timestamp,
73
72
  row.service_instance_id,
74
73
  row.request_id,
75
74
  row.body,
@@ -195,7 +194,10 @@ function normalizeBodyChunkRows(rows) {
195
194
  for (const item of parsed) {
196
195
  const index = Number(item.row.body_chunk_index);
197
196
  if (!Number.isSafeInteger(index) || index < 0) {
198
- conflicts.push({ identity: String(index), reason: "invalid_chunk_index" });
197
+ conflicts.push({
198
+ identity: String(index),
199
+ reason: "invalid_chunk_index",
200
+ });
199
201
  continue;
200
202
  }
201
203
  const prior = byChunkIndex.get(index);
@@ -230,6 +232,221 @@ function normalizeBodyChunkRows(rows) {
230
232
  };
231
233
  }
232
234
 
235
+ /**
236
+ * Reconcile admissions against endings in an explicit bounded history window.
237
+ * A missing ending is a telemetry/lifecycle gap, not proof of an upstream failure.
238
+ * @param {import("../../src/lib/types/index.js").ProxyTelemetryStoredRecord[]} lifecycle
239
+ * @param {{asOfMicroseconds: number, requestTimeoutMs: number, ingestionGraceMs: number, effectiveLookbackMs?: number}} options
240
+ */
241
+ export function reconcileProxyAdmissions(
242
+ lifecycle,
243
+ {
244
+ asOfMicroseconds,
245
+ requestTimeoutMs,
246
+ ingestionGraceMs,
247
+ effectiveLookbackMs = Infinity,
248
+ },
249
+ ) {
250
+ const terminals = new Set(
251
+ lifecycle
252
+ .filter(
253
+ (row) =>
254
+ row.event === "request_terminal" &&
255
+ !(Date.parse(row.timestamp ?? row.at ?? "") > asOfMicroseconds / 1000),
256
+ )
257
+ .map((row) => row.requestId),
258
+ );
259
+ const admissions = lifecycle.filter(
260
+ (row) =>
261
+ row.event === "request_accepted" &&
262
+ !(Date.parse(row.timestamp ?? row.at ?? "") > asOfMicroseconds / 1000),
263
+ );
264
+ const overdue = [];
265
+ const pending = [];
266
+ const invalid = [];
267
+ const insufficientWindow = [];
268
+ for (const row of admissions) {
269
+ const timeout = row.requestTimeoutMs ?? requestTimeoutMs;
270
+ if (
271
+ Number.isFinite(timeout) &&
272
+ timeout > 0 &&
273
+ effectiveLookbackMs <= timeout + ingestionGraceMs
274
+ ) {
275
+ insufficientWindow.push({
276
+ requestId: row.requestId,
277
+ requestTimeoutMs: timeout,
278
+ ingestionGraceMs,
279
+ effectiveLookbackMs,
280
+ });
281
+ }
282
+ if (terminals.has(row.requestId)) {continue;}
283
+ const acceptedAt = Date.parse(row.timestamp ?? row.at ?? "");
284
+ if (
285
+ !row.requestId ||
286
+ !Number.isFinite(acceptedAt) ||
287
+ !Number.isFinite(timeout) ||
288
+ timeout <= 0
289
+ ) {
290
+ invalid.push(row.requestId);
291
+ continue;
292
+ }
293
+ const ageMs = asOfMicroseconds / 1000 - acceptedAt;
294
+ const evidence = {
295
+ requestId: row.requestId,
296
+ acceptedAt: new Date(acceptedAt).toISOString(),
297
+ ageMs,
298
+ requestTimeoutMs: timeout,
299
+ };
300
+ if (ageMs > timeout + ingestionGraceMs) {overdue.push(evidence);}
301
+ else {pending.push(evidence);}
302
+ }
303
+ return {
304
+ admissions: admissions.length,
305
+ clientAdmissions: admissions.filter(
306
+ (row) => row.accountingScope !== "internal",
307
+ ).length,
308
+ internalAdmissions: admissions.filter(
309
+ (row) => row.accountingScope === "internal",
310
+ ).length,
311
+ ended: admissions.length - overdue.length - pending.length - invalid.length,
312
+ overdue,
313
+ pending,
314
+ invalid,
315
+ insufficientWindow,
316
+ };
317
+ }
318
+
319
+ /** Verify both ends of bridge accounting inside the selected final-record window.
320
+ * A missing linked final may be outside the window or not yet ingested; it is
321
+ * inconclusive, never proof that the designated usage owner was recorded.
322
+ * @param {import("../../src/lib/types/index.js").ProxyTelemetryStoredRecord[]} finals
323
+ */
324
+ export function reconcileProxyUsageOwnership(finals) {
325
+ const byId = new Map(finals.map((row) => [row.requestId, row]));
326
+ const duplicateOwnerUsage = finals.filter(
327
+ (row) =>
328
+ row.usageOwnerRequestId &&
329
+ row.usageOwnerRequestId !== row.requestId &&
330
+ [
331
+ row.inputTokens,
332
+ row.outputTokens,
333
+ row.cacheReadTokens,
334
+ row.cacheCreationTokens,
335
+ row.reasoningTokens,
336
+ ].some((value) => value !== undefined),
337
+ );
338
+ const missingOwners = [];
339
+ const missingParents = [];
340
+ const mismatches = [];
341
+ for (const row of finals) {
342
+ if (row.usageOwnerRequestId && row.usageOwnerRequestId !== row.requestId) {
343
+ const owner = byId.get(row.usageOwnerRequestId);
344
+ if (!owner) {
345
+ missingOwners.push({
346
+ requestId: row.requestId,
347
+ usageOwnerRequestId: row.usageOwnerRequestId,
348
+ });
349
+ } else if (
350
+ owner.accountingScope !== "internal" ||
351
+ owner.parentRequestId !== row.requestId ||
352
+ (owner.usageOwnerRequestId &&
353
+ owner.usageOwnerRequestId !== owner.requestId)
354
+ ) {
355
+ mismatches.push({
356
+ requestId: row.requestId,
357
+ linkedRequestId: owner.requestId,
358
+ reason: "invalid_usage_owner_link",
359
+ });
360
+ }
361
+ }
362
+ if (row.accountingScope === "internal") {
363
+ if (!row.parentRequestId) {
364
+ mismatches.push({
365
+ requestId: row.requestId,
366
+ reason: "internal_request_has_no_parent",
367
+ });
368
+ continue;
369
+ }
370
+ const parent = byId.get(row.parentRequestId);
371
+ if (!parent) {
372
+ missingParents.push({
373
+ requestId: row.requestId,
374
+ parentRequestId: row.parentRequestId,
375
+ });
376
+ } else if (parent.usageOwnerRequestId !== row.requestId) {
377
+ mismatches.push({
378
+ requestId: row.requestId,
379
+ linkedRequestId: parent.requestId,
380
+ reason: "parent_does_not_designate_child",
381
+ });
382
+ }
383
+ }
384
+ }
385
+ /** @type {import("../../src/lib/types/index.js").ProxyTelemetryCheck["status"]} */
386
+ const status =
387
+ duplicateOwnerUsage.length || mismatches.length
388
+ ? "fail"
389
+ : missingOwners.length || missingParents.length || !finals.length
390
+ ? "unverified"
391
+ : "pass";
392
+ return {
393
+ status,
394
+ clientFinals: finals.filter((row) => row.accountingScope !== "internal")
395
+ .length,
396
+ internalFinals: finals.filter((row) => row.accountingScope === "internal")
397
+ .length,
398
+ duplicateOwnerUsage: duplicateOwnerUsage.length,
399
+ requestIds: duplicateOwnerUsage.slice(0, 50).map((row) => row.requestId),
400
+ missingOwnerCount: missingOwners.length,
401
+ missingParentCount: missingParents.length,
402
+ mismatchCount: mismatches.length,
403
+ missingOwners: missingOwners.slice(0, 50),
404
+ missingParents: missingParents.slice(0, 50),
405
+ mismatches: mismatches.slice(0, 50),
406
+ boundary:
407
+ "Both linked finals must be present and reciprocal in this bounded selected window. Missing owners/parents may be outside it or not ingested; widen the interval or rerun before claiming complete usage accounting.",
408
+ };
409
+ }
410
+
411
+ /** All shipped generation routes require a client-response capture outcome.
412
+ * @param {import("../../src/lib/types/index.js").ProxyTelemetryStoredRecord[]} finals
413
+ * @param {import("../../src/lib/types/index.js").ProxyTelemetryStoredRecord[]} indexes
414
+ */
415
+ export function reconcileProxyCaptureCoverage(finals, indexes) {
416
+ const responseCaptures = new Set(
417
+ indexes
418
+ .filter((row) => row.phase === "client_response")
419
+ .map((row) => row.requestId),
420
+ );
421
+ const eligible = finals.filter(
422
+ (row) =>
423
+ row.path &&
424
+ ([
425
+ "/v1/messages",
426
+ "/backend-api/codex/responses",
427
+ "/v1/responses",
428
+ "/v1/chat/completions",
429
+ ].includes(row.path) ||
430
+ /^\/v1beta\/models\/[^/]+:(?:generateContent|streamGenerateContent)$/.test(
431
+ row.path,
432
+ )),
433
+ );
434
+ const missing = eligible
435
+ .filter((row) => !responseCaptures.has(row.requestId))
436
+ .map((row) => ({
437
+ requestId: row.requestId,
438
+ path: row.path,
439
+ missingPhase: "client_response",
440
+ }));
441
+ /** @type {import("../../src/lib/types/index.js").ProxyTelemetryCheck["status"]} */
442
+ const status = !eligible.length
443
+ ? "unverified"
444
+ : missing.length
445
+ ? "fail"
446
+ : "pass";
447
+ return { status, eligibleFinals: eligible.length, missing };
448
+ }
449
+
233
450
  /** Empty traffic, unavailable evidence, and partial queries cannot become a green check.
234
451
  * @param {import("../../src/lib/types/index.js").ProxyTelemetryDoctorOptions} options
235
452
  */
@@ -239,6 +456,9 @@ export async function checkProxyTelemetry({
239
456
  endTime,
240
457
  proxyUrl = "http://127.0.0.1:55669",
241
458
  maxRows = 10000,
459
+ admissionLookbackMs = 24 * 60 * 60 * 1000,
460
+ requestTimeoutMs = 15 * 60 * 1000,
461
+ ingestionGraceMs = 120000,
242
462
  fetchImpl = fetch,
243
463
  }) {
244
464
  validateProxyTelemetryBackend(backend);
@@ -249,12 +469,25 @@ export async function checkProxyTelemetry({
249
469
  startTime >= endTime ||
250
470
  !Number.isSafeInteger(maxRows) ||
251
471
  maxRows < 1 ||
252
- maxRows > 100000
472
+ maxRows > 100000 ||
473
+ !Number.isSafeInteger(admissionLookbackMs) ||
474
+ admissionLookbackMs < 1 ||
475
+ admissionLookbackMs > 7 * 24 * 60 * 60 * 1000 ||
476
+ !Number.isSafeInteger(requestTimeoutMs) ||
477
+ requestTimeoutMs < 1 ||
478
+ !Number.isSafeInteger(ingestionGraceMs) ||
479
+ ingestionGraceMs < 0
253
480
  ) {
254
481
  throw new Error(
255
482
  "Provide an increasing microsecond time range and maxRows between 1 and 100000",
256
483
  );
257
484
  }
485
+ if (admissionLookbackMs <= requestTimeoutMs + ingestionGraceMs) {
486
+ throw new Error(
487
+ "The admission lookback must exceed the request timeout plus ingestion grace",
488
+ );
489
+ }
490
+ const checkedAtMicroseconds = Date.now() * 1000;
258
491
  const budget = { used: 0, limit: 512 };
259
492
  /** @type {import("../../src/lib/types/index.js").ProxyTelemetryCheck[]} */
260
493
  const checks = [];
@@ -312,15 +545,37 @@ export async function checkProxyTelemetry({
312
545
  const supervisorProcess = supervisor?.process;
313
546
  const workerProcess = runtime.observability?.process;
314
547
  /** @param {import("../../src/lib/types/index.js").ProxyProcessTelemetrySnapshot | undefined} process */
315
- const otelProcess = (process) => process?.configuredSink === "otel" &&
316
- process.lifecycleSink === "otel" && process.otelInitialized &&
317
- process.stdio?.stdout === "non_file" && process.stdio?.stderr === "non_file";
318
- add("worker_process_logging", !workerProcess ? "unverified" : otelProcess(workerProcess) ? "pass" : "fail", workerProcess ?? { reason: "process_telemetry_missing" });
319
- add("supervisor_logging",
320
- supervisor?.status === "not_applicable" && !runtime.autoUpdate?.supervisorPid ? "pass" :
321
- supervisor?.status !== "available" ? "unverified" :
322
- supervisorProcess?.pid === runtime.autoUpdate?.supervisorPid && otelProcess(supervisorProcess) ? "pass" : "fail",
323
- supervisor ?? { status: "unavailable", reason: "supervisor_telemetry_missing" });
548
+ const otelProcess = (process) =>
549
+ process?.configuredSink === "otel" &&
550
+ process.lifecycleSink === "otel" &&
551
+ process.otelInitialized &&
552
+ process.stdio?.stdout === "non_file" &&
553
+ process.stdio?.stderr === "non_file";
554
+ add(
555
+ "worker_process_logging",
556
+ !workerProcess
557
+ ? "unverified"
558
+ : otelProcess(workerProcess)
559
+ ? "pass"
560
+ : "fail",
561
+ workerProcess ?? { reason: "process_telemetry_missing" },
562
+ );
563
+ add(
564
+ "supervisor_logging",
565
+ supervisor?.status === "not_applicable" &&
566
+ !runtime.autoUpdate?.supervisorPid
567
+ ? "pass"
568
+ : supervisor?.status !== "available"
569
+ ? "unverified"
570
+ : supervisorProcess?.pid === runtime.autoUpdate?.supervisorPid &&
571
+ otelProcess(supervisorProcess)
572
+ ? "pass"
573
+ : "fail",
574
+ supervisor ?? {
575
+ status: "unavailable",
576
+ reason: "supervisor_telemetry_missing",
577
+ },
578
+ );
324
579
  /** @type {Array<{kind: string, dropped: number, exportUnconfirmed: number, outstanding: number, capacity: number, recentFailures?: unknown, failureHistoryEvicted?: number}>} */
325
580
  const failures = (logs?.otel?.queues ?? []).map(
326
581
  (
@@ -392,6 +647,8 @@ export async function checkProxyTelemetry({
392
647
  queries.push(...result.queries.map((q) => ({ ...q, kind })));
393
648
  const normalized = normalizeMetadataRows(kind, result.records);
394
649
  history[kind] = normalized.records;
650
+ normalized.diagnostics.rawRecords = result.rawRecordCount;
651
+ normalized.diagnostics.exactRetryDuplicates += result.exactRetryDuplicates;
395
652
  historyDiagnostics[kind] = normalized.diagnostics;
396
653
  }
397
654
  const identityConflicts = Object.values(historyDiagnostics).reduce(
@@ -405,11 +662,7 @@ export async function checkProxyTelemetry({
405
662
  );
406
663
  add(
407
664
  "backend_row_identity",
408
- identityConflicts
409
- ? "fail"
410
- : missingEventIds
411
- ? "unverified"
412
- : "pass",
665
+ identityConflicts ? "fail" : missingEventIds ? "unverified" : "pass",
413
666
  {
414
667
  scope: "selected metadata and bounded capture-settling windows",
415
668
  exactRetryDuplicates: Object.values(historyDiagnostics).reduce(
@@ -424,15 +677,10 @@ export async function checkProxyTelemetry({
424
677
  /** @param {{at?: unknown}} row */
425
678
  const failedInsideInterval = (row) => {
426
679
  const at = Date.parse(typeof row.at === "string" ? row.at : "");
427
- return (
428
- Number.isFinite(at) &&
429
- at * 1000 >= startTime &&
430
- at * 1000 < endTime
431
- );
680
+ return Number.isFinite(at) && at * 1000 >= startTime && at * 1000 < endTime;
432
681
  };
433
- const storedDeliveryFailures = history.telemetry_delivery.filter(
434
- failedInsideInterval,
435
- );
682
+ const storedDeliveryFailures =
683
+ history.telemetry_delivery.filter(failedInsideInterval);
436
684
  const runtimeDeliveryFailures = failures.flatMap((queue) =>
437
685
  Array.isArray(queue.recentFailures)
438
686
  ? queue.recentFailures.filter(
@@ -447,9 +695,7 @@ export async function checkProxyTelemetry({
447
695
  ...new Map(
448
696
  [...storedDeliveryFailures, ...runtimeDeliveryFailures].map(
449
697
  (failure, index) => [
450
- typeof failure.id === "string"
451
- ? failure.id
452
- : `unidentified-${index}`,
698
+ typeof failure.id === "string" ? failure.id : `unidentified-${index}`,
453
699
  failure,
454
700
  ],
455
701
  ),
@@ -521,11 +767,9 @@ export async function checkProxyTelemetry({
521
767
  const unhealthyCaptures = captureIndexes.filter(
522
768
  (row) =>
523
769
  row.captureError !== undefined ||
524
- ![
525
- "transport_acknowledged",
526
- "no_body",
527
- "policy_excluded",
528
- ].includes(row.bodyDelivery?.status ?? ""),
770
+ !["transport_acknowledged", "no_body", "policy_excluded"].includes(
771
+ row.bodyDelivery?.status ?? "",
772
+ ),
529
773
  );
530
774
  add(
531
775
  "capture_admission",
@@ -538,9 +782,7 @@ export async function checkProxyTelemetry({
538
782
  scope: "selected_interval",
539
783
  captures: captureIndexes.length,
540
784
  unhealthy: unhealthyCaptures.length,
541
- requestIds: unhealthyCaptures
542
- .slice(0, 50)
543
- .map((row) => row.requestId),
785
+ requestIds: unhealthyCaptures.slice(0, 50).map((row) => row.requestId),
544
786
  workerLifetime: {
545
787
  scope: "worker_lifetime",
546
788
  counters: logs?.bodyCapture ?? { status: "unavailable" },
@@ -561,12 +803,18 @@ export async function checkProxyTelemetry({
561
803
  : "fail",
562
804
  {
563
805
  records: finals.length,
806
+ clientFinals: finals.filter((row) => row.accountingScope !== "internal")
807
+ .length,
808
+ internalFinals: finals.filter((row) => row.accountingScope === "internal")
809
+ .length,
564
810
  rawRecords: finalDiagnostics.rawRecords,
565
811
  exactRetryDuplicates: finalDiagnostics.exactRetryDuplicates,
566
812
  uniqueRequestIds: unique.size,
567
813
  producerOrConflictingDuplicates: finalDiagnostics.conflicts.length,
568
814
  },
569
815
  );
816
+ const ownership = reconcileProxyUsageOwnership(finals);
817
+ add("bridge_usage_ownership", ownership.status, ownership);
570
818
  const terminalEvents = history.lifecycle.filter(
571
819
  (row) => row.event === "request_terminal",
572
820
  );
@@ -593,6 +841,80 @@ export async function checkProxyTelemetry({
593
841
  "Terminals inside the selected interval; admissions still in flight are not failures",
594
842
  },
595
843
  );
844
+ add(
845
+ "ingestion_boundary",
846
+ checkedAtMicroseconds - endTime >= ingestionGraceMs * 1000
847
+ ? "pass"
848
+ : "unverified",
849
+ {
850
+ endTimeExclusive: endTime,
851
+ queriedAtMicroseconds: checkedAtMicroseconds,
852
+ ingestionGraceMs,
853
+ boundary:
854
+ "A bounded event-time query cannot prove that no late records will arrive; rerun after the ingestion grace period",
855
+ },
856
+ );
857
+ const admissionStart = Math.max(0, endTime - admissionLookbackMs * 1000);
858
+ const effectiveLookbackMs = (endTime - admissionStart) / 1000;
859
+ const admissionHistory = await queryProxyHistory({
860
+ ...backend,
861
+ startTime: admissionStart,
862
+ endTime: Math.min(checkedAtMicroseconds, endTime + ingestionGraceMs * 1000),
863
+ kind: "lifecycle",
864
+ lifecycleEvents: ["request_accepted", "request_terminal"],
865
+ maxRows,
866
+ fetchImpl,
867
+ budget,
868
+ });
869
+ queries.push(
870
+ ...admissionHistory.queries.map((query) => ({
871
+ ...query,
872
+ kind: "admission_reconciliation",
873
+ })),
874
+ );
875
+ const normalizedAdmissions = normalizeMetadataRows(
876
+ "lifecycle",
877
+ admissionHistory.records,
878
+ );
879
+ const admissionResult = reconcileProxyAdmissions(
880
+ normalizedAdmissions.records,
881
+ {
882
+ asOfMicroseconds: endTime,
883
+ requestTimeoutMs,
884
+ ingestionGraceMs,
885
+ effectiveLookbackMs,
886
+ },
887
+ );
888
+ const windowSufficient =
889
+ effectiveLookbackMs > requestTimeoutMs + ingestionGraceMs &&
890
+ !admissionResult.insufficientWindow.length;
891
+ add(
892
+ "admission_reconciliation",
893
+ admissionResult.overdue.length ||
894
+ admissionResult.invalid.length ||
895
+ normalizedAdmissions.diagnostics.conflicts.length
896
+ ? "fail"
897
+ : admissionResult.admissions && windowSufficient
898
+ ? "pass"
899
+ : "unverified",
900
+ {
901
+ ...admissionResult,
902
+ overdue: admissionResult.overdue.slice(0, 50),
903
+ pending: admissionResult.pending.slice(0, 50),
904
+ overdueCount: admissionResult.overdue.length,
905
+ pendingCount: admissionResult.pending.length,
906
+ insufficientWindow: admissionResult.insufficientWindow.slice(0, 50),
907
+ insufficientWindowCount: admissionResult.insufficientWindow.length,
908
+ windowSufficient,
909
+ effectiveLookbackMs,
910
+ requestTimeoutMs,
911
+ ingestionGraceMs,
912
+ startTime: admissionStart,
913
+ endTimeExclusive: admissionHistory.endTimeExclusive,
914
+ boundary:
915
+ "Missing endings after the request deadline plus ingestion grace are lifecycle/telemetry gaps, not inferred provider failures. Admissions before this explicit lookback are unverified. A lookback that cannot exceed the default or an observed request deadline plus grace is insufficient for a pass.",
916
+ },
917
+ );
596
918
  /** @type {Record<string, import("../../src/lib/types/index.js").ProxyTelemetryFieldCoverage>} */
597
919
  const coverage = Object.create(null);
598
920
  for (const row of finals) {
@@ -669,25 +991,18 @@ export async function checkProxyTelemetry({
669
991
  : "fail",
670
992
  coverage,
671
993
  );
672
- const indexes = history.body_capture_index.filter((row) =>
673
- (row.recordedAtMicroseconds ?? -1) >= startTime &&
674
- (row.recordedAtMicroseconds ?? Infinity) < endTime);
675
- /** @type {Map<string | undefined, Set<string | undefined>>} */
676
- const phasesByRequest = new Map();
677
- for (const index of history.body_capture_index) {
678
- let phases = phasesByRequest.get(index.requestId);
679
- if (!phases) { phases = new Set(); phasesByRequest.set(index.requestId, phases); }
680
- phases.add(index.phase);
681
- }
682
- const captureFinals = finals.filter((row) => row.path && ["/v1/messages", "/backend-api/codex/responses", "/v1/responses"].includes(row.path));
683
- const missingCapturePhases = captureFinals.flatMap((row) => {
684
- // A request may have started outside the selected interval. Its response
685
- // capture still has to be present when this final is inside the interval.
686
- return phasesByRequest.get(row.requestId)?.has("client_response") ? [] : [{ requestId: row.requestId, missingPhase: "client_response" }];
687
- });
688
- add("request_capture_coverage", !captureFinals.length ? "unverified" : missingCapturePhases.length ? "fail" : "pass", {
689
- finals: finals.length, eligibleFinals: captureFinals.length, missing: missingCapturePhases.length,
690
- examples: missingCapturePhases.slice(0, 50), omitted: Math.max(0, missingCapturePhases.length - 50),
994
+ const indexes = captureIndexes;
995
+ const captureCoverage = reconcileProxyCaptureCoverage(
996
+ finals,
997
+ history.body_capture_index,
998
+ );
999
+ const missingCapturePhases = captureCoverage.missing;
1000
+ add("request_capture_coverage", captureCoverage.status, {
1001
+ finals: finals.length,
1002
+ eligibleFinals: captureCoverage.eligibleFinals,
1003
+ missing: missingCapturePhases.length,
1004
+ examples: missingCapturePhases.slice(0, 50),
1005
+ omitted: Math.max(0, missingCapturePhases.length - 50),
691
1006
  });
692
1007
  /** @type {Record<string, number>} */
693
1008
  const byDelivery = Object.create(null);
@@ -721,6 +1036,20 @@ export async function checkProxyTelemetry({
721
1036
  failuresOmitted: Math.max(0, captureFailures.length - 50),
722
1037
  },
723
1038
  );
1039
+ const lossyCaptures = indexes.filter(
1040
+ (row) => row.bodyTruncated || row.redactionLossy,
1041
+ );
1042
+ add(
1043
+ "capture_content_completeness",
1044
+ lossyCaptures.length ? "warn" : indexes.length ? "pass" : "unverified",
1045
+ {
1046
+ captures: indexes.length,
1047
+ truncatedOrUnparseable: lossyCaptures.length,
1048
+ captureIds: lossyCaptures.slice(0, 50).map((row) => row.captureId),
1049
+ boundary:
1050
+ "Transport acknowledgement does not restore source truncation, processing limits, or invalid frames removed by redaction",
1051
+ },
1052
+ );
724
1053
  const bodyChecks = [];
725
1054
  for (const index of indexes
726
1055
  .filter(
@@ -740,7 +1069,7 @@ export async function checkProxyTelemetry({
740
1069
  throw new Error("Capture verification exceeds the 8 MiB per-body bound");
741
1070
  }
742
1071
  const rawChunks = await query(
743
- `SELECT _timestamp, proxy_event_id, body_capture_id, body_chunk_index, body_chunk_count, body FROM "${backend.stream}" WHERE proxy_record_kind='body' AND body_capture_id='${index.captureId}' ORDER BY body_chunk_index ASC, proxy_event_id ASC`,
1072
+ `SELECT _timestamp, proxy_event_id, body_capture_id, body_chunk_index, body_chunk_count, body FROM "${backend.bodyStream ?? backend.stream}" WHERE proxy_record_kind='body' AND body_capture_id='${index.captureId}' ORDER BY body_chunk_index ASC, proxy_event_id ASC`,
744
1073
  "logs",
745
1074
  {
746
1075
  startTime: startTime - 120e6,
@@ -775,8 +1104,7 @@ export async function checkProxyTelemetry({
775
1104
  expectedChunks: expected,
776
1105
  storedChunks: chunks.length,
777
1106
  rawStoredChunks: rawChunks.length,
778
- exactRetryDuplicates:
779
- normalizedChunks.diagnostics.exactRetryDuplicates,
1107
+ exactRetryDuplicates: normalizedChunks.diagnostics.exactRetryDuplicates,
780
1108
  missingEventIds: normalizedChunks.diagnostics.missingEventIds,
781
1109
  identityConflicts: normalizedChunks.diagnostics.conflicts,
782
1110
  verified:
@@ -897,7 +1225,10 @@ export async function checkProxyTelemetry({
897
1225
  key,
898
1226
  value,
899
1227
  })),
900
- invalidCounters: invalidCounters.map(([key, value]) => ({ key, value })),
1228
+ invalidCounters: invalidCounters.map(([key, value]) => ({
1229
+ key,
1230
+ value,
1231
+ })),
901
1232
  counters,
902
1233
  },
903
1234
  );
@@ -935,13 +1266,13 @@ export async function runProxyTelemetryDoctor() {
935
1266
  };
936
1267
  if (args.includes("--help")) {
937
1268
  console.log(
938
- "Usage: neurolink proxy telemetry doctor [--since ISO_DATE] [--until ISO_DATE] [--format json|text] [--proxy-url URL] [--max-rows 10000]",
1269
+ "Usage: neurolink proxy telemetry doctor [--since ISO_DATE] [--until ISO_DATE] [--format json|text] [--proxy-url URL] [--max-rows 10000] [--admission-lookback-minutes 1440] [--request-timeout-ms 900000] [--ingestion-grace-ms 120000]",
939
1270
  );
940
1271
  return;
941
1272
  }
942
1273
  const until = value("--until")
943
1274
  ? Date.parse(value("--until") ?? "")
944
- : Date.now() - 30000;
1275
+ : Date.now() - 120000;
945
1276
  const since = value("--since")
946
1277
  ? Date.parse(value("--since") ?? "")
947
1278
  : until - 15 * 60000;
@@ -951,6 +1282,10 @@ export async function runProxyTelemetryDoctor() {
951
1282
  endTime: until * 1000,
952
1283
  proxyUrl: value("--proxy-url") ?? process.env.NEUROLINK_PROXY_URL,
953
1284
  maxRows: Number(value("--max-rows") ?? 10000),
1285
+ admissionLookbackMs:
1286
+ Number(value("--admission-lookback-minutes") ?? 1440) * 60000,
1287
+ requestTimeoutMs: Number(value("--request-timeout-ms") ?? 900000),
1288
+ ingestionGraceMs: Number(value("--ingestion-grace-ms") ?? 120000),
954
1289
  });
955
1290
  if (value("--format") === "json") {
956
1291
  console.log(JSON.stringify(report, null, 2));