@juspay/neurolink 12.14.5 → 12.14.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -11,7 +11,7 @@
11
11
  */
12
12
  import type { CommandModule } from "yargs";
13
13
  import type { Hono } from "hono";
14
- import type { AccountAllowlist, LoadedProxyConfig, ModelRouterInterface, ProxyGuardArgs, ProxyHealthProbe, ProxyNeurolinkRuntime, ProxyStartArgs, ProxyStartStrategy, ProxySupervisorState, ProxyStatusArgs, ProxyTelemetryArgs, ProxyReadinessState } from "../../types/index.js";
14
+ import type { AccountAllowlist, LoadedProxyConfig, ModelRouterInterface, ProxyGuardArgs, ProxyHealthProbe, ProxyNeurolinkRuntime, ProxyStartArgs, ProxyStartStrategy, ProxyRestartSupervisorState as ProxySupervisorState, ProxyStatusArgs, ProxyTelemetryArgs, ProxyReadinessState } from "../../types/index.js";
15
15
  import { ProxyRuntimeConfigStore } from "../../proxy/runtimeConfig.js";
16
16
  /**
17
17
  * Drop a supervisor `version` that is not a string.
@@ -12,6 +12,7 @@ import { applyAllClients, restoreAllClients, } from "../proxy-clients/registry.j
12
12
  import { resolveProxyConfigPath } from "../../proxy/proxyConfig.js";
13
13
  import { redactUrlsInText, sanitizeForLog, } from "../../utils/logSanitize.js";
14
14
  import { withTimeout } from "../../utils/async/withTimeout.js";
15
+ import { startProxyHttpTrace } from "../../proxy/proxyTracer.js";
15
16
  import { formatUptime, isProcessRunning, StateFileManager, } from "../utils/serverUtils.js";
16
17
  import { configureProxyKeepAliveDispatcher } from "../../proxy/proxyDispatcher.js";
17
18
  import { ProxyRuntimeConfigStore } from "../../proxy/runtimeConfig.js";
@@ -24,6 +25,7 @@ import { describeInstallFailure, getGlobalInstallArgs, isTransientInstallFailure
24
25
  import { startUpdaterWorkerSupervisor } from "../../proxy/updaterSupervisor.js";
25
26
  import { openProxyWorkerLog } from "../../proxy/workerLog.js";
26
27
  import { startRollingProxyServer } from "../../proxy/rollingProxyServer.js";
28
+ import { startProxyRestartControl } from "../../proxy/restartControl.js";
27
29
  import { isProxyAuxiliaryRequest } from "../../proxy/proxyRequestKind.js";
28
30
  import { spawnProxySocketWorker } from "../../proxy/rollingWorkerProcess.js";
29
31
  import { PROXY_ROLLING_SUPERVISOR_ENV, PROXY_SOCKET_WORKER_ENV, } from "../../proxy/rollingWorkerProtocol.js";
@@ -836,13 +838,36 @@ function spawnProxyUpdater(host, port, parentPid, rollingSupervisor = false) {
836
838
  workerLog.close();
837
839
  }
838
840
  }
839
- async function runProxyTelemetryManager(command) {
841
+ async function runProxyTelemetryManager(command, argv) {
840
842
  const { existsSync } = await import("fs");
841
843
  if (!existsSync(PROXY_TELEMETRY_SCRIPT_PATH)) {
842
844
  throw new Error("Proxy telemetry helper files were not found in this installation. Reinstall NeuroLink with observability assets included.");
843
845
  }
844
846
  await new Promise((resolve, reject) => {
845
- const child = spawn("bash", [PROXY_TELEMETRY_SCRIPT_PATH, command], {
847
+ const queryCommand = command === "doctor" || command === "query";
848
+ const queryArgs = [];
849
+ if (queryCommand && argv) {
850
+ for (const [key, option] of [
851
+ ["since", "--since"],
852
+ ["until", "--until"],
853
+ ["kind", "--kind"],
854
+ ["format", "--format"],
855
+ ["maxRows", "--max-rows"],
856
+ ["proxyUrl", "--proxy-url"],
857
+ ]) {
858
+ if (argv[key] !== undefined) {
859
+ queryArgs.push(option, String(argv[key]));
860
+ }
861
+ }
862
+ }
863
+ const child = spawn(queryCommand ? process.execPath : "bash", queryCommand
864
+ ? [
865
+ join(dirname(PROXY_TELEMETRY_SCRIPT_PATH), command === "doctor"
866
+ ? "check-proxy-telemetry.mjs"
867
+ : "query-proxy-history.mjs"),
868
+ ...queryArgs,
869
+ ]
870
+ : [PROXY_TELEMETRY_SCRIPT_PATH, command], {
846
871
  stdio: "inherit",
847
872
  env: process.env,
848
873
  });
@@ -1077,6 +1102,7 @@ function registerProxyRequestTracking(app, requestMetadata, readiness) {
1077
1102
  rejectForUpdate: readiness.drainingForUpdate,
1078
1103
  };
1079
1104
  requestMetadata.set(c.req.raw, metadata);
1105
+ const httpTrace = startProxyHttpTrace(metadata, Object.fromEntries(c.req.raw.headers));
1080
1106
  const stopObservingFinalLog = observeProxyFinalLog(metadata.requestId, (entry) => {
1081
1107
  metadata.terminalResult = entry;
1082
1108
  }, (entry) => {
@@ -1086,6 +1112,7 @@ function registerProxyRequestTracking(app, requestMetadata, readiness) {
1086
1112
  ? () => undefined
1087
1113
  : beginProxyRequest();
1088
1114
  const finish = () => {
1115
+ httpTrace.end(metadata.terminalResult?.responseStatus ?? c.res.status, metadata.terminalResult?.terminalOutcome ?? "unknown", metadata.terminalErrorType);
1089
1116
  stopObservingFinalLog();
1090
1117
  finishActivity();
1091
1118
  // Borrowed traffic holds a concurrency slot for the lifetime of the
@@ -1098,7 +1125,7 @@ function registerProxyRequestTracking(app, requestMetadata, readiness) {
1098
1125
  // them at acceptance instead of publishing misleading placeholder values;
1099
1126
  // subsequent events carry the parsed metadata under the same request ID.
1100
1127
  try {
1101
- await persistProxyLifecycleAcceptance({
1128
+ await httpTrace.run(() => persistProxyLifecycleAcceptance({
1102
1129
  requestId: metadata.requestId,
1103
1130
  method: metadata.method,
1104
1131
  path: metadata.path,
@@ -1106,8 +1133,8 @@ function registerProxyRequestTracking(app, requestMetadata, readiness) {
1106
1133
  requestBytes,
1107
1134
  elapsedMs: 0,
1108
1135
  monotonicMs: startedMonotonicMs,
1109
- });
1110
- await next();
1136
+ }));
1137
+ await httpTrace.run(next);
1111
1138
  const responseStatus = c.res.status;
1112
1139
  logProxyLifecycleEvent({
1113
1140
  event: "response_headers",
@@ -1168,12 +1195,12 @@ function registerProxyRequestTracking(app, requestMetadata, readiness) {
1168
1195
  let accountingTimedOut = false;
1169
1196
  let accountingFailed = false;
1170
1197
  try {
1171
- accountingFailed = await notifyRouteTerminal({
1198
+ accountingFailed = await httpTrace.run(() => notifyRouteTerminal({
1172
1199
  outcome,
1173
1200
  error,
1174
1201
  observedBodyBytes,
1175
1202
  responseChunks,
1176
- });
1203
+ }));
1177
1204
  }
1178
1205
  catch {
1179
1206
  accountingTimedOut = true;
@@ -1230,6 +1257,7 @@ function registerProxyRequestTracking(app, requestMetadata, readiness) {
1230
1257
  errorCode: final?.errorCode ?? metadata.terminalErrorCode,
1231
1258
  });
1232
1259
  stopObservingFinalLog();
1260
+ httpTrace.end(final?.responseStatus ?? responseStatus, terminalOutcome, final?.errorType ?? metadata.terminalErrorType);
1233
1261
  },
1234
1262
  });
1235
1263
  }
@@ -1258,6 +1286,9 @@ function registerProxyRequestTracking(app, requestMetadata, readiness) {
1258
1286
  errorType: error instanceof Error ? error.name : "unknown_error",
1259
1287
  errorCode: getProxyRuntimeErrorCode(error),
1260
1288
  });
1289
+ httpTrace.end(getProxyRuntimeErrorCode(error) === "PROXY_TELEMETRY_UNAVAILABLE"
1290
+ ? 503
1291
+ : 502, "handler_error", error instanceof Error ? error.name : "unknown_error");
1261
1292
  throw error;
1262
1293
  }
1263
1294
  };
@@ -1330,8 +1361,9 @@ export async function createProxyStartApp(params) {
1330
1361
  accountKey: attempt?.accountKey,
1331
1362
  provider: attempt?.provider,
1332
1363
  transportScope: attempt?.transportScope,
1333
- traceId: attempt?.traceId,
1334
- spanId: attempt?.spanId,
1364
+ traceId: attempt?.traceId ?? metadata.traceId,
1365
+ spanId: attempt?.spanId ?? metadata.spanId,
1366
+ traceFlags: attempt?.traceFlags ?? metadata.traceFlags,
1335
1367
  responseStatus: status,
1336
1368
  responseTimeMs: Date.now() - metadata.startedAt,
1337
1369
  errorType,
@@ -1341,6 +1373,9 @@ export async function createProxyStartApp(params) {
1341
1373
  logBodyCapture({
1342
1374
  timestamp: new Date().toISOString(),
1343
1375
  requestId: metadata.requestId,
1376
+ traceId: attempt?.traceId ?? metadata.traceId,
1377
+ spanId: attempt?.spanId ?? metadata.spanId,
1378
+ traceFlags: attempt?.traceFlags ?? metadata.traceFlags,
1344
1379
  model: metadata.model,
1345
1380
  stream: metadata.stream,
1346
1381
  phase: "client_response",
@@ -1400,6 +1435,15 @@ export async function createProxyStartApp(params) {
1400
1435
  .json()
1401
1436
  .catch(() => ({ action: undefined }));
1402
1437
  if (payload.action === "drain") {
1438
+ // A rolling worker must keep admitting until the supervisor has a ready
1439
+ // replacement. The legacy global drain can otherwise strand the listener
1440
+ // behind maintenance responses when its caller stalls or disappears.
1441
+ if (isProxySocketWorkerProcess()) {
1442
+ return c.json({
1443
+ error: "rolling_restart_required",
1444
+ message: "Use neurolink proxy restart; global update drain is disabled for rolling workers.",
1445
+ }, 409);
1446
+ }
1403
1447
  if (!markProxyDrainingForUpdate(readiness)) {
1404
1448
  return c.json({ error: "proxy_not_ready" }, 409);
1405
1449
  }
@@ -2777,6 +2821,7 @@ async function runLaunchdProxySupervisor(argv, spinner) {
2777
2821
  filePrefix: "proxy-supervisor",
2778
2822
  });
2779
2823
  let currentUpdaterPid;
2824
+ let restartControl;
2780
2825
  const rollingServer = await startRollingProxyServer({
2781
2826
  onEvent: (event) => logProxyLifecycleEvent({
2782
2827
  event: "supervisor_event",
@@ -2807,6 +2852,7 @@ async function runLaunchdProxySupervisor(argv, spinner) {
2807
2852
  version: PROXY_VERSION,
2808
2853
  updaterPid: currentUpdaterPid,
2809
2854
  rolling: snapshot,
2855
+ restartControl: restartControl?.identity,
2810
2856
  });
2811
2857
  },
2812
2858
  log: (message) => logger.always(message),
@@ -2843,13 +2889,46 @@ async function runLaunchdProxySupervisor(argv, spinner) {
2843
2889
  };
2844
2890
  process.on("SIGUSR2", activatePendingUpdate);
2845
2891
  const readinessHost = host === "0.0.0.0" ? "127.0.0.1" : host;
2892
+ try {
2893
+ restartControl = await startProxyRestartControl({
2894
+ stateDir: join(homedir(), ".neurolink"),
2895
+ server: rollingServer,
2896
+ log: (message) => logger.warn(message),
2897
+ isUpdatePending: () => !!rollingReplacement || !!loadUpdateState()?.pendingRestartVersion,
2898
+ getInstalledVersion: async () => {
2899
+ const { execFile } = await import("node:child_process");
2900
+ const { promisify } = await import("node:util");
2901
+ const { stdout } = await promisify(execFile)(TRAMPOLINE_PATH, ["--version"], { timeout: 5_000, maxBuffer: 65_536 });
2902
+ return stdout.trim();
2903
+ },
2904
+ getStatus: async () => {
2905
+ const probeHost = readinessHost === "::" ? "::1" : readinessHost;
2906
+ const response = await fetch(`http://${probeHost.includes(":") ? `[${probeHost}]` : probeHost}:${rollingServer.address.port}/status`, {
2907
+ headers: { connection: "close" },
2908
+ signal: AbortSignal.timeout(3_000),
2909
+ });
2910
+ if (!response.ok) {
2911
+ throw new Error("Serving status is unavailable.");
2912
+ }
2913
+ return response.json();
2914
+ },
2915
+ });
2916
+ }
2917
+ catch (error) {
2918
+ // A control-plane setup failure must not stop an otherwise serving proxy.
2919
+ logger.warn(`[proxy-supervisor] safe restart control unavailable: ${error instanceof Error ? error.message : String(error)}`);
2920
+ }
2846
2921
  const updatePersistedUpdaterPid = (updaterPid) => {
2847
2922
  currentUpdaterPid = updaterPid;
2848
2923
  const state = loadProxySupervisorState();
2849
2924
  if (!state || state.pid !== process.pid) {
2850
2925
  return;
2851
2926
  }
2852
- saveProxySupervisorState({ ...state, updaterPid });
2927
+ saveProxySupervisorState({
2928
+ ...state,
2929
+ updaterPid,
2930
+ restartControl: restartControl?.identity,
2931
+ });
2853
2932
  };
2854
2933
  const updaterSupervisor = startUpdaterWorkerSupervisor({
2855
2934
  spawnWorker: () => spawnProxyUpdater(readinessHost, rollingServer.address.port, process.pid, true),
@@ -2872,6 +2951,7 @@ async function runLaunchdProxySupervisor(argv, spinner) {
2872
2951
  stopping = true;
2873
2952
  logger.always(`[proxy-supervisor] shutting down (${signal})`);
2874
2953
  process.off("SIGUSR2", activatePendingUpdate);
2954
+ await restartControl?.close();
2875
2955
  updaterSupervisor.stop();
2876
2956
  await rollingServer.close();
2877
2957
  await flushProxyLifecycleEvents().catch((error) => logger.warn(String(error)));
@@ -3507,6 +3587,8 @@ export const proxyStatusCommand = {
3507
3587
  // PROXY TELEMETRY COMMAND
3508
3588
  // =============================================================================
3509
3589
  const PROXY_TELEMETRY_ACTIONS = [
3590
+ "doctor",
3591
+ "query",
3510
3592
  "setup",
3511
3593
  "start",
3512
3594
  "stop",
@@ -3516,12 +3598,37 @@ const PROXY_TELEMETRY_ACTIONS = [
3516
3598
  ];
3517
3599
  export const proxyTelemetryCommand = {
3518
3600
  command: "telemetry <action>",
3519
- describe: "Manage the local OpenObserve stack and dashboard for proxy observability",
3601
+ describe: "Query and verify stored OTel telemetry or manage the local observability stack",
3520
3602
  builder: (yargs) => yargs
3521
3603
  .positional("action", {
3522
3604
  type: "string",
3523
3605
  choices: [...PROXY_TELEMETRY_ACTIONS],
3524
- describe: "Telemetry action: setup, start, stop, status, logs, or import-dashboard",
3606
+ describe: "Telemetry action: doctor, query, setup, start, stop, status, logs, or import-dashboard",
3607
+ })
3608
+ .option("since", {
3609
+ type: "string",
3610
+ description: "Inclusive ISO timestamp for stored telemetry",
3611
+ })
3612
+ .option("until", {
3613
+ type: "string",
3614
+ description: "Exclusive ISO timestamp for stored telemetry",
3615
+ })
3616
+ .option("kind", {
3617
+ type: "string",
3618
+ description: "Metadata kind for query, such as request_final or telemetry_delivery",
3619
+ })
3620
+ .option("format", {
3621
+ type: "string",
3622
+ choices: ["json", "text"],
3623
+ description: "Doctor report format; query always returns JSON",
3624
+ })
3625
+ .option("max-rows", {
3626
+ type: "number",
3627
+ description: "Explicit bounded history record limit",
3628
+ })
3629
+ .option("proxy-url", {
3630
+ type: "string",
3631
+ description: "Proxy endpoint used by the read-only doctor",
3525
3632
  })
3526
3633
  .option("quiet", {
3527
3634
  type: "boolean",
@@ -3529,19 +3636,21 @@ export const proxyTelemetryCommand = {
3529
3636
  default: false,
3530
3637
  description: "Suppress the local CLI spinner and delegate directly",
3531
3638
  })
3639
+ .example("neurolink proxy telemetry doctor --format json", "Verify field coverage, stored captures, trace correlation, and delivery evidence")
3640
+ .example("neurolink proxy telemetry query --since 2026-09-15T00:00:00Z --kind request_final", "Read stored OTLP metadata without scanning proxy log files")
3532
3641
  .example("neurolink proxy telemetry setup", "Start OpenObserve, start the OTEL collector, and import the dashboard")
3533
3642
  .example("neurolink proxy telemetry start", "Start the local proxy telemetry stack without re-importing the dashboard")
3534
3643
  .example("neurolink proxy telemetry stop", "Stop the local OpenObserve and OTEL collector containers"),
3535
3644
  handler: async (argv) => {
3536
3645
  const action = argv.action;
3537
- const spinner = argv.quiet
3646
+ const spinner = argv.quiet || action === "doctor" || action === "query"
3538
3647
  ? null
3539
3648
  : ora(`Running proxy telemetry ${action}...`).start();
3540
3649
  try {
3541
3650
  if (spinner) {
3542
3651
  spinner.stop();
3543
3652
  }
3544
- await runProxyTelemetryManager(action);
3653
+ await runProxyTelemetryManager(action, argv);
3545
3654
  if (spinner) {
3546
3655
  spinner.succeed(`proxy telemetry ${action} completed`);
3547
3656
  }
@@ -0,0 +1,3 @@
1
+ import type { CommandModule } from "yargs";
2
+ import type { ProxyRestartArgs } from "../../types/index.js";
3
+ export declare const proxyRestartCommand: CommandModule<object, ProxyRestartArgs>;
@@ -0,0 +1,88 @@
1
+ import { readFile, lstat } from "node:fs/promises";
2
+ import { join } from "node:path";
3
+ import { z } from "zod";
4
+ import { resolveProxyPaths } from "../../proxy/proxyPaths.js";
5
+ import { requestProxyRestart } from "../../proxy/restartControl.js";
6
+ const supervisorSchema = z.object({
7
+ pid: z.number().int().positive(),
8
+ restartControl: z.object({
9
+ protocol: z.literal(1),
10
+ socketPath: z.string(),
11
+ instanceId: z.string().uuid(),
12
+ }),
13
+ });
14
+ export const proxyRestartCommand = {
15
+ command: "restart",
16
+ describe: "Replace the serving worker without stopping the proxy listener",
17
+ builder: (yargs) => yargs
18
+ .option("check", {
19
+ type: "boolean",
20
+ default: false,
21
+ description: "Check restart readiness without changing the running proxy",
22
+ })
23
+ .option("dev", {
24
+ type: "boolean",
25
+ default: false,
26
+ description: "Use the isolated proxy state in .neurolink-dev/",
27
+ })
28
+ .option("format", {
29
+ type: "string",
30
+ choices: ["text", "json"],
31
+ default: "text",
32
+ })
33
+ .example("neurolink proxy restart --check", "Verify the running supervisor supports safe worker replacement")
34
+ .example("neurolink proxy restart", "Start a replacement, verify it, then let existing streams finish"),
35
+ handler: async (argv) => {
36
+ try {
37
+ const { stateDir } = resolveProxyPaths(argv.dev);
38
+ const saved = supervisorSchema.safeParse(JSON.parse(await readFile(join(stateDir, "proxy-supervisor-state.json"), "utf8")));
39
+ if (!saved.success) {
40
+ throw new Error("This running supervisor does not advertise safe restart support. It must first be upgraded through a separately planned service activation.");
41
+ }
42
+ const { pid, restartControl } = saved.data;
43
+ if (restartControl.socketPath !==
44
+ join(stateDir, `restart-${pid}-${restartControl.instanceId}.sock`)) {
45
+ throw new Error("Restart control path does not match the recorded supervisor.");
46
+ }
47
+ const socket = await lstat(restartControl.socketPath);
48
+ if (!socket.isSocket() ||
49
+ (socket.mode & 0o077) !== 0 ||
50
+ (process.getuid && socket.uid !== process.getuid())) {
51
+ throw new Error("Restart control socket ownership or permissions are invalid.");
52
+ }
53
+ const result = await requestProxyRestart(restartControl, argv.check);
54
+ if (result.supervisorPid !== pid) {
55
+ throw new Error("Restart control identity changed; outcome is unverified.");
56
+ }
57
+ if (argv.format === "json") {
58
+ console.info(JSON.stringify(result, null, 2));
59
+ }
60
+ else {
61
+ console.info(result.message);
62
+ console.info(`Supervisor ${result.supervisorPid}; worker ${result.workerPid ?? "unknown"}; version ${result.version ?? "unknown"}; previous workers finishing requests: ${result.drainingWorkers}.`);
63
+ console.info("This operation preserves the supervisor and launchd settings. It does not reinstall the service.");
64
+ }
65
+ if (!result.ok) {
66
+ process.exitCode = 1;
67
+ }
68
+ }
69
+ catch (error) {
70
+ const message = error instanceof Error
71
+ ? error.message
72
+ : "Restart control is unavailable.";
73
+ if (argv.format === "json") {
74
+ const result = {
75
+ ok: false,
76
+ phase: "unverified",
77
+ message,
78
+ };
79
+ console.info(JSON.stringify(result));
80
+ }
81
+ else {
82
+ console.error(message);
83
+ }
84
+ process.exitCode = 1;
85
+ }
86
+ },
87
+ };
88
+ //# sourceMappingURL=proxyRestart.js.map
@@ -23,6 +23,7 @@ import { ObservabilityCommandFactory } from "./commands/observability.js";
23
23
  import { TelemetryCommandFactory } from "./commands/telemetry.js";
24
24
  import { proxyStartCommand, proxyStatusCommand, proxyTelemetryCommand, proxySetupCommand, proxyGuardCommand, proxyInstallCommand, proxyUninstallCommand, } from "./commands/proxy.js";
25
25
  import { proxyAnalyzeCommand } from "./commands/proxyAnalyze.js";
26
+ import { proxyRestartCommand } from "./commands/proxyRestart.js";
26
27
  import { proxyShareCommand } from "./commands/proxyShare.js";
27
28
  import { proxyPeerCommand } from "./commands/proxyPeer.js";
28
29
  import { proxyExposeCommand } from "./commands/proxyExpose.js";
@@ -218,6 +219,7 @@ export function initializeCliParser() {
218
219
  builder: (yargs) => yargs
219
220
  .command(proxyStartCommand)
220
221
  .command(proxyStatusCommand)
222
+ .command(proxyRestartCommand)
221
223
  .command(proxyShareCommand)
222
224
  .command(proxyPeerCommand)
223
225
  .command(proxyExposeCommand)
@@ -228,7 +230,7 @@ export function initializeCliParser() {
228
230
  .command(proxyGuardCommand)
229
231
  .command(proxyInstallCommand)
230
232
  .command(proxyUninstallCommand)
231
- .demandCommand(1, "Please specify a proxy subcommand: start, status, share <create|provision|url|list|status|pause|resume|revoke|topup|set|link|rotate|level|note|notes|receipts|delete>, peer <add|request|sync|receipts|net|redeem|list|status|test|remove|pause|resume|set>, expose, analyze, replay <export|compare>, telemetry <setup|start|stop|status|logs|import-dashboard>, setup, guard, install, or uninstall"),
233
+ .demandCommand(1, "Please specify a proxy subcommand: start, status, restart, share <create|provision|url|list|status|pause|resume|revoke|topup|set|link|rotate|level|note|notes|receipts|delete>, peer <add|request|sync|receipts|net|redeem|list|status|test|remove|pause|resume|set>, expose, analyze, replay <export|compare>, telemetry <setup|start|stop|status|logs|import-dashboard>, setup, guard, install, or uninstall"),
232
234
  handler: () => { },
233
235
  })
234
236
  // Evaluate Command Group - Using EvaluateCommandFactory
@@ -18,6 +18,8 @@ const snapshot = {
18
18
  pendingBytes: 0,
19
19
  maxPending: MAX_PENDING,
20
20
  maxPendingBytes: MAX_PENDING_BYTES,
21
+ highWaterPending: 0,
22
+ highWaterBytes: 0,
21
23
  rejectionReasons: {},
22
24
  };
23
25
  const pending = new Map();
@@ -161,9 +163,27 @@ export async function captureProxyBody(entry, logDir, consume) {
161
163
  ? "body_worker_backoff"
162
164
  : "body_capture_queue_full";
163
165
  snapshot.lastError = error;
166
+ snapshot.lastRejectedAt = new Date().toISOString();
164
167
  snapshot.rejectionReasons[error] =
165
168
  (snapshot.rejectionReasons[error] ?? 0) + 1;
166
- return consume({ error, stored: { bodyWriteFailed: true } });
169
+ return consume({
170
+ error,
171
+ stored: { bodyWriteFailed: true },
172
+ admission: {
173
+ limitingResource: bytes > MAX_ENTRY_BYTES
174
+ ? "entry"
175
+ : Date.now() < retryAfter
176
+ ? "worker"
177
+ : snapshot.pending >= MAX_PENDING
178
+ ? "captures"
179
+ : "bytes",
180
+ estimatedBytes: Number.isFinite(bytes) ? bytes : undefined,
181
+ pending: snapshot.pending,
182
+ pendingBytes: snapshot.pendingBytes,
183
+ maxPending: MAX_PENDING,
184
+ maxPendingBytes: MAX_PENDING_BYTES,
185
+ },
186
+ });
167
187
  }
168
188
  let current;
169
189
  try {
@@ -207,7 +227,9 @@ export async function captureProxyBody(entry, logDir, consume) {
207
227
  },
208
228
  });
209
229
  snapshot.pending += 1;
230
+ snapshot.highWaterPending = Math.max(snapshot.highWaterPending, snapshot.pending);
210
231
  snapshot.pendingBytes += bytes;
232
+ snapshot.highWaterBytes = Math.max(snapshot.highWaterBytes, snapshot.pendingBytes);
211
233
  current.ref();
212
234
  try {
213
235
  current.postMessage({ id, entry, logDir, queuedAt: Date.now() });
@@ -248,6 +270,9 @@ export const __bodyCaptureWorkerTestHooks = {
248
270
  pending: 0,
249
271
  pendingBytes: 0,
250
272
  lastError: undefined,
273
+ lastRejectedAt: undefined,
274
+ highWaterPending: 0,
275
+ highWaterBytes: 0,
251
276
  rejectionReasons: {},
252
277
  });
253
278
  },
@@ -45,6 +45,34 @@ import { sanitizeForLog } from "../utils/logSanitize.js";
45
45
  const nonNegativeInt = (value) => typeof value === "number" && Number.isFinite(value) && value > 0
46
46
  ? Math.floor(value)
47
47
  : 0;
48
+ function usefulOutputItem(item) {
49
+ if (!item || typeof item !== "object") {
50
+ return false;
51
+ }
52
+ if ("type" in item) {
53
+ if (item.type === "function_call") {
54
+ return ("arguments" in item &&
55
+ typeof item.arguments === "string" &&
56
+ item.arguments.length > 0);
57
+ }
58
+ if (item.type === "custom_tool_call") {
59
+ return ("input" in item &&
60
+ typeof item.input === "string" &&
61
+ item.input.length > 0);
62
+ }
63
+ if (item.type === "output_text") {
64
+ return ("text" in item && typeof item.text === "string" && item.text.length > 0);
65
+ }
66
+ if (item.type === "refusal") {
67
+ return ("refusal" in item &&
68
+ typeof item.refusal === "string" &&
69
+ item.refusal.length > 0);
70
+ }
71
+ }
72
+ return ("content" in item &&
73
+ Array.isArray(item.content) &&
74
+ item.content.some(usefulOutputItem));
75
+ }
48
76
  /**
49
77
  * Pull usage out of one parsed SSE `data:` payload.
50
78
  *
@@ -125,7 +153,7 @@ const CAPTURE_LIMIT_BYTES = 256 * 1024;
125
153
  */
126
154
  function createCaptureSink() {
127
155
  const target = process.env.NEUROLINK_PROXY_CODEX_CAPTURE;
128
- if (!target) {
156
+ if (!target || process.env.NEUROLINK_PROXY_LOG_SINK === "otel") {
129
157
  return null;
130
158
  }
131
159
  let written = 0;
@@ -167,9 +195,13 @@ export function createCodexUsageTap() {
167
195
  let totalBytes = 0;
168
196
  const inspectEvidence = (events) => {
169
197
  for (const frame of events) {
198
+ if (frame.data.trim() === "[DONE]") {
199
+ continue;
200
+ }
170
201
  try {
171
202
  const event = JSON.parse(frame.data);
172
203
  if (!event || typeof event !== "object") {
204
+ evidence.observationIncomplete = true;
173
205
  continue;
174
206
  }
175
207
  const seen = extractCodexUsage(event);
@@ -178,10 +210,53 @@ export function createCodexUsageTap() {
178
210
  }
179
211
  const type = event.type ?? frame.event;
180
212
  if ((type === "response.output_text.delta" ||
213
+ type === "response.refusal.delta" ||
181
214
  type === "response.function_call_arguments.delta") &&
182
215
  typeof event.delta === "string" &&
183
216
  event.delta.length > 0) {
184
- evidence.firstUsefulOutputAt ??= Date.now();
217
+ if (evidence.firstUsefulOutputAt === undefined) {
218
+ evidence.firstUsefulOutputAt = Date.now();
219
+ evidence.firstUsefulOutputEvent = String(type);
220
+ }
221
+ }
222
+ // Some clients/providers emit complete output items without deltas.
223
+ // Reasoning/control events are not useful client output. Recognize
224
+ // actual tool calls and text only, without inventing an earlier time.
225
+ const usefulItem = usefulOutputItem(event.item);
226
+ const completedResponse = event.response;
227
+ const usefulCompletion = type === "response.completed" &&
228
+ completedResponse &&
229
+ typeof completedResponse === "object" &&
230
+ "output" in completedResponse &&
231
+ Array.isArray(completedResponse.output) &&
232
+ completedResponse.output.some(usefulOutputItem);
233
+ const usefulDone = (type === "response.output_text.done" &&
234
+ typeof event.text === "string" &&
235
+ event.text.length > 0) ||
236
+ (type === "response.refusal.done" &&
237
+ typeof event.refusal === "string" &&
238
+ event.refusal.length > 0) ||
239
+ (type === "response.function_call_arguments.done" &&
240
+ typeof event.arguments === "string" &&
241
+ event.arguments.length > 0) ||
242
+ (type === "response.custom_tool_call_input.done" &&
243
+ typeof event.input === "string" &&
244
+ event.input.length > 0) ||
245
+ ((type === "response.content_part.added" ||
246
+ type === "response.content_part.done") &&
247
+ usefulOutputItem(event.part));
248
+ const usefulToolDelta = type === "response.custom_tool_call_input.delta" &&
249
+ typeof event.delta === "string" &&
250
+ event.delta.length > 0;
251
+ if (evidence.firstUsefulOutputAt === undefined &&
252
+ (usefulDone ||
253
+ usefulCompletion ||
254
+ usefulToolDelta ||
255
+ ((type === "response.output_item.added" ||
256
+ type === "response.output_item.done") &&
257
+ usefulItem))) {
258
+ evidence.firstUsefulOutputAt = Date.now();
259
+ evidence.firstUsefulOutputEvent = String(type);
185
260
  }
186
261
  if (type === "response.completed") {
187
262
  evidence.completed = true;
@@ -222,6 +297,7 @@ export function createCodexUsageTap() {
222
297
  }
223
298
  catch {
224
299
  // Unknown frames cannot establish successful completion.
300
+ evidence.observationIncomplete = true;
225
301
  }
226
302
  }
227
303
  };
@@ -268,16 +344,21 @@ export function createCodexUsageTap() {
268
344
  carry = remainder;
269
345
  inspectEvidence(events);
270
346
  if (carry.length > CARRY_LIMIT_CHARS) {
347
+ evidence.observationIncomplete = true;
271
348
  carry = carry.slice(-2);
272
349
  discardingEvent = true;
273
350
  }
274
351
  }
275
352
  catch {
276
353
  // Telemetry must never break the relay.
354
+ evidence.observationIncomplete = true;
277
355
  }
278
356
  },
279
357
  flush() {
280
358
  // An event without its dispatch delimiter is incomplete on the wire.
359
+ if (discardingEvent || carry.trim()) {
360
+ evidence.observationIncomplete = true;
361
+ }
281
362
  settle(latest);
282
363
  },
283
364
  /**
@@ -34,6 +34,18 @@ export declare function getProxyOtelLogSnapshot(): {
34
34
  maxPendingBytes: number;
35
35
  };
36
36
  queues: {
37
+ recentFailures: {
38
+ records: {
39
+ eventId: string;
40
+ kind?: string;
41
+ requestId?: string;
42
+ captureId?: string;
43
+ }[];
44
+ id: string;
45
+ at: string;
46
+ reason: "export_unconfirmed" | "queue_full";
47
+ error?: string;
48
+ }[];
37
49
  attempted: number;
38
50
  submitted: number;
39
51
  transportAcknowledged: number;
@@ -43,6 +55,7 @@ export declare function getProxyOtelLogSnapshot(): {
43
55
  lastAcknowledgedAt: string | undefined;
44
56
  lastFailureAt: string | undefined;
45
57
  highWaterOutstanding: number;
58
+ failureHistoryEvicted: number;
46
59
  kind: "metadata" | "bodies";
47
60
  capacity: number;
48
61
  }[];