@juspay/neurolink 12.17.0 → 12.17.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. package/CHANGELOG.md +2 -7
  2. package/dist/browser/neurolink.min.js +422 -413
  3. package/dist/cli/commands/proxy.js +455 -259
  4. package/dist/constants/contextWindows.d.ts +2 -0
  5. package/dist/constants/contextWindows.js +4 -0
  6. package/dist/providers/anthropic/client.js +15 -5
  7. package/dist/proxy/anthropicBudgetResponse.d.ts +3 -0
  8. package/dist/proxy/anthropicBudgetResponse.js +153 -0
  9. package/dist/proxy/bodyCaptureProcessing.d.ts +1 -0
  10. package/dist/proxy/bodyCaptureProcessing.js +23 -4
  11. package/dist/proxy/bodyCaptureWorker.js +20 -2
  12. package/dist/proxy/codexFallback.d.ts +10 -0
  13. package/dist/proxy/codexFallback.js +108 -24
  14. package/dist/proxy/codexUsage.js +2 -0
  15. package/dist/proxy/globalInstaller.d.ts +32 -1
  16. package/dist/proxy/globalInstaller.js +386 -3
  17. package/dist/proxy/otelLogSink.d.ts +3 -3
  18. package/dist/proxy/otelLogSink.js +137 -144
  19. package/dist/proxy/otlpLogTransport.d.ts +3 -0
  20. package/dist/proxy/otlpLogTransport.js +334 -0
  21. package/dist/proxy/proxyActivity.d.ts +22 -2
  22. package/dist/proxy/proxyActivity.js +114 -9
  23. package/dist/proxy/proxyAnalysis.js +82 -52
  24. package/dist/proxy/proxyClientCancellation.d.ts +7 -0
  25. package/dist/proxy/proxyClientCancellation.js +23 -0
  26. package/dist/proxy/proxyContextPreflight.d.ts +25 -0
  27. package/dist/proxy/proxyContextPreflight.js +337 -0
  28. package/dist/proxy/proxyFailureDetails.d.ts +19 -0
  29. package/dist/proxy/proxyFailureDetails.js +96 -0
  30. package/dist/proxy/proxyLifecycle.js +7 -0
  31. package/dist/proxy/proxyRouteBodyCapture.d.ts +13 -0
  32. package/dist/proxy/proxyRouteBodyCapture.js +112 -0
  33. package/dist/proxy/proxyTokenBudget.d.ts +40 -0
  34. package/dist/proxy/proxyTokenBudget.js +523 -0
  35. package/dist/proxy/proxyTracer.d.ts +3 -2
  36. package/dist/proxy/proxyTracer.js +46 -20
  37. package/dist/proxy/proxyTranslationEngine.d.ts +8 -10
  38. package/dist/proxy/proxyTranslationEngine.js +736 -376
  39. package/dist/proxy/requestLogger.d.ts +1 -0
  40. package/dist/proxy/requestLogger.js +107 -5
  41. package/dist/proxy/restartControl.js +9 -4
  42. package/dist/proxy/rollingWorkerProcess.js +65 -0
  43. package/dist/proxy/rollingWorkerSupervisor.js +7 -0
  44. package/dist/proxy/updateCoordinator.d.ts +14 -1
  45. package/dist/proxy/updateCoordinator.js +57 -2
  46. package/dist/server/routes/claudeProxyRoutes.d.ts +4 -5
  47. package/dist/server/routes/claudeProxyRoutes.js +809 -577
  48. package/dist/server/routes/codexProxyRoutes.js +191 -9
  49. package/dist/server/routes/openaiProxyRoutes.d.ts +1 -1
  50. package/dist/server/routes/openaiProxyRoutes.js +261 -69
  51. package/dist/types/cli.d.ts +3 -0
  52. package/dist/types/codex.d.ts +5 -1
  53. package/dist/types/index.d.ts +2 -0
  54. package/dist/types/index.js +2 -0
  55. package/dist/types/proxy.d.ts +148 -3
  56. package/dist/types/proxy.js +0 -15
  57. package/dist/types/proxyBudget.d.ts +74 -0
  58. package/dist/types/proxyBudget.js +1 -0
  59. package/dist/types/proxyContext.d.ts +35 -0
  60. package/dist/types/proxyContext.js +1 -0
  61. package/dist/utils/toolChoice.d.ts +19 -0
  62. package/dist/utils/toolChoice.js +39 -0
  63. package/docs/assets/dashboards/README.md +37 -0
  64. package/docs/assets/dashboards/neurolink-proxy-observability-dashboard.json +197 -89
  65. package/docs-site/static/search-index.json +1 -1
  66. package/package.json +4 -1
  67. package/scripts/observability/NATIVE-DURABLE.md +63 -0
  68. package/scripts/observability/otel-collector.proxy-native-durable.yaml +93 -0
  69. package/scripts/observability/proxy-telemetry-backend.mjs +32 -5
  70. package/scripts/observability/proxy-telemetry-check.mjs +396 -61
  71. package/scripts/observability/query-proxy-history.mjs +34 -5
@@ -12,7 +12,7 @@ import { logger } from "../../utils/logger.js";
12
12
  import { applyAllClients, restoreAllClients, } from "../proxy-clients/registry.js";
13
13
  import { resolveProxyConfigPath } from "../../proxy/proxyConfig.js";
14
14
  import { redactUrlsInText, sanitizeForLog, } from "../../utils/logSanitize.js";
15
- import { withTimeout } from "../../utils/async/withTimeout.js";
15
+ import { raceWithAbort, withTimeout, } from "../../utils/async/withTimeout.js";
16
16
  import { startProxyHttpTrace } from "../../proxy/proxyTracer.js";
17
17
  import { formatUptime, isProcessRunning, StateFileManager, } from "../utils/serverUtils.js";
18
18
  import { configureProxyKeepAliveDispatcher } from "../../proxy/proxyDispatcher.js";
@@ -20,19 +20,22 @@ import { ProxyRuntimeConfigStore } from "../../proxy/runtimeConfig.js";
20
20
  import { startProxyLogCleanupScheduler } from "../../proxy/logCleanupScheduler.js";
21
21
  import { anthropicAccountKeysEqual, createAccountAllowlist, isAccountAllowed, LEGACY_ANTHROPIC_ACCOUNT_KEY, normalizeAnthropicAccountKey, shouldLoadFallbackCredential, } from "../../proxy/accountSelection.js";
22
22
  import { resolveProxyStatusAccountIdentity } from "../../proxy/codexAccountUsage.js";
23
- import { beginProxyRequest, getProxyActivitySnapshot, observeProxyFinalLog, takeProxyResponseObservers, trackProxyResponse, } from "../../proxy/proxyActivity.js";
23
+ import { beginProxyRequest, consumeInternalProxyRequest, releaseProxyRequestAccounting, withProxyFinalLogOwnership, getProxyActivitySnapshot, observeProxyFinalLog, takeProxyResponseObservers, trackProxyResponse, } from "../../proxy/proxyActivity.js";
24
24
  import { configureProxyLifecycleLogger, flushProxyLifecycleEvents, persistProxyLifecycleAcceptance, getProxyLifecycleLoggerSnapshot, hashProxyLifecycleSessionId, logProxyLifecycleEvent, } from "../../proxy/proxyLifecycle.js";
25
- import { describeInstallFailure, getGlobalInstallArgs, isTransientInstallFailure, resolveGlobalInstaller, validateInstalledVersion, } from "../../proxy/globalInstaller.js";
25
+ import { describeInstallFailure, assertProxyServiceInstallIdle, probeProxyLaunchdPresence, inspectProxyPackage, isProxyServiceEnvironmentKey, parseProxyServiceInstallSettings, installStagedProxyPackage, readProxyPackageSelection, resolveProxyWorkerPackage, selectProxyPackage, writeProxyPackageLauncher, isTransientInstallFailure, resolveGlobalInstaller, validateInstalledVersion, } from "../../proxy/globalInstaller.js";
26
26
  import { startUpdaterWorkerSupervisor } from "../../proxy/updaterSupervisor.js";
27
27
  import { openProxyWorkerLog } from "../../proxy/workerLog.js";
28
28
  import { startRollingProxyServer } from "../../proxy/rollingProxyServer.js";
29
29
  import { startProxyRestartControl, requestProxySupervisorTelemetry, } from "../../proxy/restartControl.js";
30
30
  import { getProxyProcessTelemetry } from "../../proxy/processTelemetry.js";
31
+ import { recordFinalError as recordProxyRuntimeFinalError } from "../../proxy/usageStats.js";
32
+ import { logRequest as logProxyRequestFinal } from "../../proxy/requestLogger.js";
33
+ import { bindProxyClientCancellation } from "../../proxy/proxyClientCancellation.js";
31
34
  import { isProxyAuxiliaryRequest } from "../../proxy/proxyRequestKind.js";
32
35
  import { spawnProxySocketWorker } from "../../proxy/rollingWorkerProcess.js";
33
36
  import { PROXY_ROLLING_SUPERVISOR_ENV, PROXY_SOCKET_WORKER_ENV, } from "../../proxy/rollingWorkerProtocol.js";
34
37
  import { attachSocketWorkerProcess } from "../../proxy/socketWorkerRuntime.js";
35
- import { shouldRefreshStaleSupervisor, waitForProxyUpdateWindow, } from "../../proxy/updateCoordinator.js";
38
+ import { parseProxyRuntimeActivity, isProxyUpdateOwnerCurrent, shouldRefreshStaleSupervisor, waitForProxyUpdateWindow, } from "../../proxy/updateCoordinator.js";
36
39
  import { abandonPendingUpdate, clearUpdateDeferral, isVersionSuppressed, loadUpdateState, recordCheck, recordSuccessfulUpdate, recordUpdateDeferred, recordUpdateFailure, recordUpdateInstalled, suppressVersion, } from "../../proxy/updateState.js";
37
40
  import { loadProxyEnvFile, resolveProxyEnvFile, } from "../../proxy/proxyEnv.js";
38
41
  import { createRequire } from "node:module";
@@ -479,17 +482,7 @@ async function getProxyRuntimeActivity(host, port, timeoutMs = 3_000) {
479
482
  if (!response.ok) {
480
483
  return null;
481
484
  }
482
- const payload = (await response.json());
483
- const activeRequests = Number(payload.activity?.activeRequests);
484
- if (!Number.isFinite(activeRequests) || activeRequests < 0) {
485
- return null;
486
- }
487
- return {
488
- activeRequests,
489
- lastActivityAt: typeof payload.activity?.lastActivityAt === "string"
490
- ? payload.activity.lastActivityAt
491
- : null,
492
- };
485
+ return parseProxyRuntimeActivity(await response.json());
493
486
  }
494
487
  catch {
495
488
  return null;
@@ -665,11 +658,12 @@ async function resumeProxyUpdateDrain(host, port, attempts = 3) {
665
658
  // ---------------------------------------------------------------------------
666
659
  /**
667
660
  * Path to a small trampoline script that the plist invokes.
668
- * The trampoline re-resolves `neurolink` via PATH on every launch,
669
- * so launchd never gets pinned to a version-specific store path.
661
+ * The trampoline selects a validated immutable package. The stable launchd
662
+ * path does not require reinstalling the service to activate another version.
670
663
  */
671
664
  const TRAMPOLINE_DIR = join(homedir(), ".neurolink", "bin");
672
665
  const TRAMPOLINE_PATH = join(TRAMPOLINE_DIR, "neurolink-proxy");
666
+ const PROXY_PACKAGES_DIR = join(homedir(), ".neurolink", "proxy-packages");
673
667
  /**
674
668
  * Verify a candidate bin path actually runs by invoking `--version` on it.
675
669
  * Returns the version string on success, or undefined on any failure.
@@ -691,96 +685,26 @@ function probeBinVersion(binPath) {
691
685
  /**
692
686
  * Write (or overwrite) the trampoline shell script.
693
687
  *
694
- * Defensive design: the trampoline tries multiple candidates in order and
695
- * only `exec`s one whose `--version` check succeeds. If every PATH-based
696
- * candidate is broken (stale shims, missing packages), it falls back to the
697
- * baked-in `node + script` path that was verified to work at install time.
688
+ * Selection is explicit: package validation happens before rewriting the
689
+ * launcher, and launch itself never imports the CLI merely to probe a version.
698
690
  */
699
- function writeTrampoline() {
700
- const { writeFileSync, mkdirSync, existsSync, chmodSync } = _require("fs");
701
- if (!existsSync(TRAMPOLINE_DIR)) {
702
- mkdirSync(TRAMPOLINE_DIR, { recursive: true });
703
- }
704
- // Baked-in fallback: the specific node + JS script currently running
705
- // (guaranteed to work, since we ARE running). Used only if all PATH-based
706
- // candidates fail their --version probe.
707
- const bakedNode = process.execPath;
708
- const bakedScript = process.argv[1] ?? join(__dirname, "..", "index.js");
709
- // Shell-escape the baked paths (they shouldn't contain quotes in practice,
710
- // but be safe for paths with spaces).
711
- const shEscape = (s) => `'${s.replace(/'/g, "'\\''")}'`;
712
- const script = `#!/bin/sh
713
- # Auto-generated by \`neurolink proxy install\` — do not edit.
714
- # Resolves a working neurolink binary on every launchd invocation so the
715
- # plist never gets pinned to a broken/stale shim.
716
-
717
- BAKED_NODE=${shEscape(bakedNode)}
718
- BAKED_SCRIPT=${shEscape(bakedScript)}
719
-
720
- # IPC workers must preserve the inherited Node channel until the final exec.
721
- # Do not launch a --version probe subprocess on that descriptor; the parent
722
- # validates the worker's exact package version before activation instead.
723
- if [ "\${NEUROLINK_PROXY_TRAMPOLINE_EXEC_ONLY:-0}" = "1" ]; then
724
- if [ -n "\${NEUROLINK_BIN:-}" ] && [ -x "$NEUROLINK_BIN" ]; then
725
- exec "$NEUROLINK_BIN" "$@"
726
- fi
727
- for cand in \
728
- "$(command -v neurolink 2>/dev/null || true)" \
729
- "\${PNPM_HOME:-}/neurolink" \
730
- "$HOME/.local/share/pnpm/neurolink" \
731
- "$HOME/Library/pnpm/neurolink" \
732
- "/usr/local/bin/neurolink" \
733
- "/opt/homebrew/bin/neurolink"; do
734
- [ -n "$cand" ] && [ -x "$cand" ] && exec "$cand" "$@"
735
- done
736
- if [ -x "$BAKED_NODE" ] && [ -f "$BAKED_SCRIPT" ]; then
737
- exec "$BAKED_NODE" "$BAKED_SCRIPT" "$@"
738
- fi
739
- exit 127
740
- fi
741
-
742
- # Probe a candidate: must be executable and respond to --version cleanly.
743
- _try() {
744
- [ -n "$1" ] && [ -x "$1" ] || return 1
745
- "$1" --version >/dev/null 2>&1 || return 1
746
- return 0
691
+ function writeTrampoline(selection, isCurrentOwner) {
692
+ // Reinstall preserves an already validated proxy-owned selection. A manual
693
+ // global npm upgrade cannot silently retarget existing worker generations.
694
+ const selected = selection ??
695
+ readProxyPackageSelection(PROXY_PACKAGES_DIR) ??
696
+ inspectProxyPackage(process.argv[1] ?? join(__dirname, "..", "index.js"));
697
+ writeProxyPackageLauncher(TRAMPOLINE_PATH, selected, isCurrentOwner);
698
+ selectProxyPackage(PROXY_PACKAGES_DIR, selected, isCurrentOwner);
747
699
  }
748
-
749
- # 1. Explicit user override (escape hatch for broken environments).
750
- if [ -n "\${NEUROLINK_BIN:-}" ]; then
751
- if _try "$NEUROLINK_BIN"; then
752
- exec "$NEUROLINK_BIN" "$@"
753
- fi
754
- echo "[neurolink-proxy] WARN: NEUROLINK_BIN=$NEUROLINK_BIN is not runnable, trying defaults" >&2
755
- fi
756
-
757
- # 2. PATH-based and common install locations. First working one wins.
758
- for cand in \\
759
- "$(command -v neurolink 2>/dev/null || true)" \\
760
- "\${PNPM_HOME:-}/neurolink" \\
761
- "$HOME/.local/share/pnpm/neurolink" \\
762
- "$HOME/Library/pnpm/neurolink" \\
763
- "/usr/local/bin/neurolink" \\
764
- "/opt/homebrew/bin/neurolink"; do
765
- if _try "$cand"; then
766
- exec "$cand" "$@"
767
- fi
768
- done
769
-
770
- # 3. Baked-in fallback: the exact node + script that worked at install time.
771
- # Always valid at install time; may become stale after package updates
772
- # (but at that point the PATH candidates above should work).
773
- if [ -x "$BAKED_NODE" ] && [ -f "$BAKED_SCRIPT" ]; then
774
- exec "$BAKED_NODE" "$BAKED_SCRIPT" "$@"
775
- fi
776
-
777
- echo "[neurolink-proxy] FATAL: no working neurolink binary found." >&2
778
- echo "[neurolink-proxy] Tried: PATH, \\$PNPM_HOME, \\$HOME/.local/share/pnpm, \\$HOME/Library/pnpm, /usr/local/bin, /opt/homebrew/bin, baked-in install path." >&2
779
- echo "[neurolink-proxy] Fix: reinstall with 'pnpm add -g @juspay/neurolink' or set NEUROLINK_BIN=/path/to/working/neurolink." >&2
780
- exit 127
781
- `;
782
- writeFileSync(TRAMPOLINE_PATH, script, { mode: 0o755 });
783
- chmodSync(TRAMPOLINE_PATH, 0o755);
700
+ /** Status must remain available when an external installer damages its entrypoint. */
701
+ function readProxyDiskVersion() {
702
+ try {
703
+ return inspectProxyPackage(process.argv[1]).version;
704
+ }
705
+ catch {
706
+ return null;
707
+ }
784
708
  }
785
709
  function spawnFailOpenGuard(host, port, parentPid) {
786
710
  // The guard runs the same version as this process, so process.argv[1]
@@ -892,6 +816,9 @@ async function runProxyTelemetryManager(command, argv) {
892
816
  ["format", "--format"],
893
817
  ["maxRows", "--max-rows"],
894
818
  ["proxyUrl", "--proxy-url"],
819
+ ["admissionLookbackMinutes", "--admission-lookback-minutes"],
820
+ ["requestTimeoutMs", "--request-timeout-ms"],
821
+ ["ingestionGraceMs", "--ingestion-grace-ms"],
895
822
  ]) {
896
823
  if (argv[key] !== undefined) {
897
824
  queryArgs.push(option, String(argv[key]));
@@ -1129,15 +1056,22 @@ function registerProxyRequestTracking(app, requestMetadata, readiness) {
1129
1056
  c.req.raw.headers.get("x-claude-code-session-id") ??
1130
1057
  undefined;
1131
1058
  const sessionHash = hashProxyLifecycleSessionId(sessionId);
1059
+ const internal = consumeInternalProxyRequest(c.req.raw.headers.get("x-neurolink-internal-request"));
1060
+ const abortController = new AbortController();
1132
1061
  const metadata = {
1133
- requestId: crypto.randomUUID(),
1062
+ requestId: internal?.requestId ?? crypto.randomUUID(),
1063
+ parentRequestId: internal?.parentRequestId,
1064
+ accountingScope: internal ? "internal" : "client",
1065
+ usageOwnerRequestId: internal?.requestId,
1066
+ abortController,
1134
1067
  method: c.req.method,
1135
1068
  path: c.req.path,
1136
1069
  startedAt: Date.now(),
1137
1070
  model: "-",
1138
1071
  stream: false,
1139
1072
  toolCount: 0,
1140
- rejectForUpdate: readiness.drainingForUpdate,
1073
+ // Trusted child work belongs to an already admitted parent request.
1074
+ rejectForUpdate: !internal && readiness.drainingForUpdate,
1141
1075
  };
1142
1076
  requestMetadata.set(c.req.raw, metadata);
1143
1077
  const httpTrace = startProxyHttpTrace(metadata, Object.fromEntries(c.req.raw.headers));
@@ -1149,7 +1083,16 @@ function registerProxyRequestTracking(app, requestMetadata, readiness) {
1149
1083
  const finishActivity = metadata.rejectForUpdate
1150
1084
  ? () => undefined
1151
1085
  : beginProxyRequest();
1086
+ let responseTracking = false;
1087
+ let cancelledBeforeHeaders = false;
1088
+ const deadline = setTimeout(() => abortController.abort(new DOMException("Proxy request deadline exceeded", "TimeoutError")), 900_000);
1089
+ deadline.unref?.();
1090
+ let unbindClient = () => { };
1152
1091
  const finish = () => {
1092
+ clearTimeout(deadline);
1093
+ unbindClient();
1094
+ abortController.signal.removeEventListener("abort", onPreheaderAbort);
1095
+ releaseProxyRequestAccounting(metadata.requestId);
1153
1096
  httpTrace.end(metadata.terminalResult?.responseStatus ?? c.res.status, metadata.terminalResult?.terminalOutcome ?? "unknown", metadata.terminalErrorType);
1154
1097
  stopObservingFinalLog();
1155
1098
  finishActivity();
@@ -1159,12 +1102,77 @@ function registerProxyRequestTracking(app, requestMetadata, readiness) {
1159
1102
  metadata.shareRelease?.();
1160
1103
  requestMetadata.delete(c.req.raw);
1161
1104
  };
1105
+ const onPreheaderAbort = () => {
1106
+ if (responseTracking || cancelledBeforeHeaders) {
1107
+ return;
1108
+ }
1109
+ cancelledBeforeHeaders = true;
1110
+ const timeout = abortController.signal.reason?.name === "TimeoutError";
1111
+ const status = timeout ? 504 : 499;
1112
+ const errorType = timeout ? "request_timeout" : "client_cancelled";
1113
+ if (!metadata.terminalResult && metadata.accountingScope !== "internal") {
1114
+ recordProxyRuntimeFinalError(status, undefined, undefined, {
1115
+ requestId: metadata.requestId,
1116
+ errorType,
1117
+ terminalOutcome: timeout ? "handler_error" : "client_cancelled",
1118
+ message: timeout
1119
+ ? "Request deadline exceeded"
1120
+ : "Client disconnected before response headers",
1121
+ });
1122
+ }
1123
+ void logProxyRequestFinal({
1124
+ timestamp: new Date().toISOString(),
1125
+ requestId: metadata.requestId,
1126
+ method: metadata.method,
1127
+ path: metadata.path,
1128
+ model: metadata.model,
1129
+ stream: metadata.stream,
1130
+ toolCount: metadata.toolCount,
1131
+ account: "",
1132
+ accountType: "proxy-runtime",
1133
+ responseStatus: status,
1134
+ responseTimeMs: Date.now() - metadata.startedAt,
1135
+ errorType,
1136
+ errorMessage: timeout
1137
+ ? "Request deadline exceeded"
1138
+ : "Client disconnected before response headers",
1139
+ });
1140
+ logProxyLifecycleEvent({
1141
+ event: "request_terminal",
1142
+ requestId: metadata.requestId,
1143
+ parentRequestId: metadata.parentRequestId,
1144
+ accountingScope: metadata.accountingScope,
1145
+ method: metadata.method,
1146
+ path: metadata.path,
1147
+ model: metadata.model,
1148
+ stream: metadata.stream,
1149
+ sessionHash,
1150
+ elapsedMs: performance.now() - startedMonotonicMs,
1151
+ terminalOutcome: timeout ? "handler_error" : "client_cancelled",
1152
+ transportOutcome: timeout ? "stream_error" : "client_cancelled",
1153
+ finalStatus: status,
1154
+ errorType,
1155
+ telemetryStatus: "complete",
1156
+ outcomeSource: "transport_error",
1157
+ });
1158
+ finishActivity();
1159
+ metadata.shareRelease?.();
1160
+ clearTimeout(deadline);
1161
+ unbindClient();
1162
+ };
1163
+ abortController.signal.addEventListener("abort", onPreheaderAbort, {
1164
+ once: true,
1165
+ });
1166
+ unbindClient = bindProxyClientCancellation(c.req.raw, c.env, abortController);
1162
1167
  // The route adapter populates model/stream/toolCount after parsing. Omit
1163
1168
  // them at acceptance instead of publishing misleading placeholder values;
1164
1169
  // subsequent events carry the parsed metadata under the same request ID.
1165
1170
  try {
1166
1171
  await httpTrace.run(() => persistProxyLifecycleAcceptance({
1167
1172
  requestId: metadata.requestId,
1173
+ parentRequestId: metadata.parentRequestId,
1174
+ accountingScope: metadata.accountingScope,
1175
+ requestTimeoutMs: 900_000,
1168
1176
  method: metadata.method,
1169
1177
  path: metadata.path,
1170
1178
  sessionHash,
@@ -1172,7 +1180,19 @@ function registerProxyRequestTracking(app, requestMetadata, readiness) {
1172
1180
  elapsedMs: 0,
1173
1181
  monotonicMs: startedMonotonicMs,
1174
1182
  }));
1175
- await httpTrace.run(next);
1183
+ if (abortController.signal.aborted) {
1184
+ c.res = new Response(null, { status: 499 });
1185
+ finish();
1186
+ return;
1187
+ }
1188
+ await httpTrace.run(() => withProxyFinalLogOwnership(metadata.requestId, next));
1189
+ if (cancelledBeforeHeaders) {
1190
+ void c.res.body
1191
+ ?.cancel(abortController.signal.reason)
1192
+ .catch(() => undefined);
1193
+ finish();
1194
+ return;
1195
+ }
1176
1196
  const responseStatus = c.res.status;
1177
1197
  logProxyLifecycleEvent({
1178
1198
  event: "response_headers",
@@ -1199,9 +1219,10 @@ function registerProxyRequestTracking(app, requestMetadata, readiness) {
1199
1219
  }
1200
1220
  };
1201
1221
  const notifyRouteTerminal = async (details) => {
1202
- const results = await withTimeout(Promise.allSettled(routeResponseObservers.map(async (observer) => observer.onTerminal?.(details))), 2_000, "Timed out joining proxy response accounting");
1222
+ const results = await withTimeout(Promise.allSettled(routeResponseObservers.map(async (observer) => observer.onTerminal?.(details))), 3_000, "Timed out joining proxy response accounting");
1203
1223
  return results.some((result) => result.status === "rejected");
1204
1224
  };
1225
+ responseTracking = true;
1205
1226
  c.res = trackProxyResponse(c.res, finish, {
1206
1227
  onFirstChunk: ({ observedBodyBytes, responseChunks }) => {
1207
1228
  logProxyLifecycleEvent({
@@ -1243,6 +1264,34 @@ function registerProxyRequestTracking(app, requestMetadata, readiness) {
1243
1264
  catch {
1244
1265
  accountingTimedOut = true;
1245
1266
  }
1267
+ if (!metadata.terminalResult && responseStatus >= 400) {
1268
+ if (metadata.accountingScope !== "internal") {
1269
+ recordProxyRuntimeFinalError(responseStatus, undefined, undefined, {
1270
+ requestId: metadata.requestId,
1271
+ errorType: responseStatus === 404
1272
+ ? "route_not_found"
1273
+ : "request_rejected",
1274
+ terminalOutcome: "handler_error",
1275
+ });
1276
+ }
1277
+ await logProxyRequestFinal({
1278
+ timestamp: new Date().toISOString(),
1279
+ requestId: metadata.requestId,
1280
+ method: metadata.method,
1281
+ path: metadata.path,
1282
+ model: metadata.model,
1283
+ stream: metadata.stream,
1284
+ toolCount: metadata.toolCount,
1285
+ account: "",
1286
+ accountType: "proxy-runtime",
1287
+ responseStatus,
1288
+ responseTimeMs: Date.now() - metadata.startedAt,
1289
+ errorType: responseStatus === 404
1290
+ ? "route_not_found"
1291
+ : "request_rejected",
1292
+ terminalOutcome: "handler_error",
1293
+ });
1294
+ }
1246
1295
  const final = metadata.terminalResult;
1247
1296
  const auxiliary = isProxyAuxiliaryRequest(metadata.method, metadata.path);
1248
1297
  const terminalOutcome = final?.terminalOutcome ??
@@ -1297,9 +1346,16 @@ function registerProxyRequestTracking(app, requestMetadata, readiness) {
1297
1346
  stopObservingFinalLog();
1298
1347
  httpTrace.end(final?.responseStatus ?? responseStatus, terminalOutcome, final?.errorType ?? metadata.terminalErrorType);
1299
1348
  },
1300
- });
1349
+ }, abortController.signal);
1301
1350
  }
1302
1351
  catch (error) {
1352
+ clearTimeout(deadline);
1353
+ unbindClient();
1354
+ abortController.signal.removeEventListener("abort", onPreheaderAbort);
1355
+ if (cancelledBeforeHeaders) {
1356
+ finish();
1357
+ return;
1358
+ }
1303
1359
  stopObservingFinalLog();
1304
1360
  // Keep metadata available to app.onError, which records the client-facing
1305
1361
  // failure with the same request ID before deleting the WeakMap entry.
@@ -1352,7 +1408,7 @@ export async function createProxyStartApp(params) {
1352
1408
  const { createOpenAIProxyRoutes } = await import("../../server/routes/openaiProxyRoutes.js");
1353
1409
  const { createCodexProxyRoutes } = await import("../../server/routes/codexProxyRoutes.js");
1354
1410
  const { createGeminiProxyRoutes } = await import("../../server/routes/geminiProxyRoutes.js");
1355
- const { logBodyCapture, logRequest, getRequestLoggerSnapshot } = await import("../../proxy/requestLogger.js");
1411
+ const { logBodyCapture, getRequestLoggerSnapshot } = await import("../../proxy/requestLogger.js");
1356
1412
  const { recordFinalError } = await import("../../proxy/usageStats.js");
1357
1413
  const { admitInboundShareRequest, isGrantRequiredByEnv } = await import("../../proxy/shareGate.js");
1358
1414
  const { runWithShareContext } = await import("../../proxy/shareContext.js");
@@ -1378,15 +1434,17 @@ export async function createProxyStartApp(params) {
1378
1434
  const clientMessage = options?.clientMessage ?? errorMessage;
1379
1435
  const clientErrorType = options?.clientErrorType ?? errorType;
1380
1436
  const attempt = metadata.lastUpstreamAttempt;
1381
- recordFinalError(status, attempt?.account ?? PROXY_INTERNAL_ACCOUNT_LABEL, attempt?.accountType ?? PROXY_INTERNAL_ACCOUNT_TYPE, {
1382
- requestId: metadata.requestId,
1383
- errorType,
1384
- errorCode: options?.errorCode,
1385
- terminalOutcome: "handler_error",
1386
- message: errorMessage,
1387
- });
1437
+ if (metadata.accountingScope !== "internal" && !metadata.terminalResult) {
1438
+ recordFinalError(status, attempt?.account ?? PROXY_INTERNAL_ACCOUNT_LABEL, attempt?.accountType ?? PROXY_INTERNAL_ACCOUNT_TYPE, {
1439
+ requestId: metadata.requestId,
1440
+ errorType,
1441
+ errorCode: options?.errorCode,
1442
+ terminalOutcome: "handler_error",
1443
+ message: errorMessage,
1444
+ });
1445
+ }
1388
1446
  await Promise.all([
1389
- logRequest({
1447
+ logProxyRequestFinal({
1390
1448
  timestamp: new Date().toISOString(),
1391
1449
  requestId: metadata.requestId,
1392
1450
  method: metadata.method,
@@ -1449,12 +1507,20 @@ export async function createProxyStartApp(params) {
1449
1507
  metadata.terminalErrorType = telemetryUnavailable
1450
1508
  ? "telemetry_unavailable"
1451
1509
  : "unhandled_proxy_error";
1452
- await recordRuntimeError(metadata, status, metadata.terminalErrorType, errMsg, {
1453
- clientMessage: "Proxy internal error",
1454
- clientErrorType: "api_error",
1455
- errorCode: metadata.terminalErrorCode,
1456
- });
1457
- requestMetadata.delete(c.req.raw);
1510
+ try {
1511
+ await recordRuntimeError(metadata, status, metadata.terminalErrorType, errMsg, {
1512
+ clientMessage: "Proxy internal error",
1513
+ clientErrorType: "api_error",
1514
+ errorCode: metadata.terminalErrorCode,
1515
+ });
1516
+ }
1517
+ finally {
1518
+ // Admission/handler failures may never reach the response tracker. Keep
1519
+ // bridge ownership through final publication, then release this request
1520
+ // only; a child failure must retain its parent's result until delivery.
1521
+ releaseProxyRequestAccounting(metadata.requestId);
1522
+ requestMetadata.delete(c.req.raw);
1523
+ }
1458
1524
  return c.json({
1459
1525
  type: "error",
1460
1526
  error: {
@@ -1506,7 +1572,7 @@ export async function createProxyStartApp(params) {
1506
1572
  if (payload.action === "drain_for_supervisor_refresh") {
1507
1573
  // A crashed updater must not strand the current rolling worker in
1508
1574
  // maintenance mode. A successful launchd refresh replaces this process.
1509
- scheduleSupervisorRefreshResume(35 * 60 * 1000);
1575
+ scheduleSupervisorRefreshResume(90 * 1000);
1510
1576
  }
1511
1577
  }
1512
1578
  else if (payload.action === "resume") {
@@ -1544,7 +1610,10 @@ export async function createProxyStartApp(params) {
1544
1610
  runtimeConfigProvider,
1545
1611
  }
1546
1612
  : params.accountAllowlist);
1547
- const openaiRouteGroup = createOpenAIProxyRoutes(params.modelRouter, "", params.port, runtimeConfigProvider);
1613
+ const openaiRouteGroup = createOpenAIProxyRoutes(params.modelRouter, "", params.port, runtimeConfigProvider,
1614
+ // Keep bridge capabilities, accounting and cancellation in this worker
1615
+ // across supervisor generation switches; never re-enter the public listener.
1616
+ (request) => app.fetch(request));
1548
1617
  const codexRouteGroup = createCodexProxyRoutes("");
1549
1618
  const geminiRouteGroup = createGeminiProxyRoutes(params.modelRouter, "", params.port, runtimeConfigProvider);
1550
1619
  const allProxyRoutes = [
@@ -1631,7 +1700,8 @@ export async function createProxyStartApp(params) {
1631
1700
  // secret. Running it through the request gate as well would spend the
1632
1701
  // grant's rate allowance and open a coin hold for a call that consumes no
1633
1702
  // capacity at all.
1634
- const shareOutcome = c.req.path.startsWith("/peer/")
1703
+ const shareOutcome = metadata?.accountingScope === "internal" ||
1704
+ c.req.path.startsWith("/peer/")
1635
1705
  ? { kind: "local" }
1636
1706
  : await admitInboundShareRequest({
1637
1707
  headers: Object.fromEntries(c.req.raw.headers.entries()),
@@ -1665,7 +1735,7 @@ export async function createProxyStartApp(params) {
1665
1735
  shareOutcome.release();
1666
1736
  }
1667
1737
  }
1668
- const requestAbortController = new AbortController();
1738
+ const requestAbortController = metadata?.abortController ?? new AbortController();
1669
1739
  const ctx = {
1670
1740
  requestId: metadata?.requestId ?? crypto.randomUUID(),
1671
1741
  method: c.req.method,
@@ -1700,9 +1770,9 @@ export async function createProxyStartApp(params) {
1700
1770
  c.header(key, value);
1701
1771
  }
1702
1772
  };
1703
- const result = shareContext
1704
- ? await runWithShareContext(shareContext, () => route.handler(ctx))
1705
- : await route.handler(ctx);
1773
+ const result = await raceWithAbort(Promise.resolve(shareContext
1774
+ ? runWithShareContext(shareContext, () => route.handler(ctx))
1775
+ : route.handler(ctx)), ctx.abortSignal);
1706
1776
  if (result instanceof Response) {
1707
1777
  // Streaming responses own their headers; merge in anything the
1708
1778
  // handler published on the context that the Response lacks. A Response
@@ -2161,6 +2231,10 @@ export async function createProxyStartApp(params) {
2161
2231
  installedVersion: updateState?.installedVersion ??
2162
2232
  updateState?.lastUpdateVersion ??
2163
2233
  null,
2234
+ validatedVersion: updateState?.installedVersion ?? null,
2235
+ selectedPackageVersion: readProxyPackageSelection(PROXY_PACKAGES_DIR)?.version ?? null,
2236
+ diskPackageVersion: readProxyDiskVersion(),
2237
+ candidateVersion: supervisorState?.rolling?.candidate?.expectedVersion ?? null,
2164
2238
  activatedVersion: PROXY_VERSION,
2165
2239
  pendingActivationVersion: updateState?.pendingRestartVersion ?? null,
2166
2240
  pendingRestartVersion: updateState?.pendingRestartVersion ?? null,
@@ -2968,16 +3042,23 @@ async function runLaunchdProxySupervisor(argv, spinner) {
2968
3042
  host,
2969
3043
  port,
2970
3044
  initialVersion: PROXY_VERSION,
2971
- spawnWorker: (generation, expectedVersion) => spawnProxySocketWorker({
2972
- generation,
2973
- expectedVersion,
2974
- command: TRAMPOLINE_PATH,
2975
- args: workerArgs,
2976
- env: {
2977
- NEUROLINK_PROXY_UPDATE_CONTROL_TOKEN: PROXY_UPDATE_CONTROL_TOKEN,
2978
- NEUROLINK_PROXY_TRAMPOLINE_EXEC_ONLY: "1",
2979
- },
2980
- }),
3045
+ spawnWorker: (generation, expectedVersion) => {
3046
+ const selected = resolveProxyWorkerPackage({
3047
+ packagesDir: PROXY_PACKAGES_DIR,
3048
+ entryScript,
3049
+ expectedVersion,
3050
+ });
3051
+ return spawnProxySocketWorker({
3052
+ generation,
3053
+ expectedVersion,
3054
+ command: selected.nodePath,
3055
+ args: [selected.entryScript, ...workerArgs],
3056
+ env: {
3057
+ NEUROLINK_PROXY_UPDATE_CONTROL_TOKEN: PROXY_UPDATE_CONTROL_TOKEN,
3058
+ NEUROLINK_PROXY_TRAMPOLINE_EXEC_ONLY: "1",
3059
+ },
3060
+ });
3061
+ },
2981
3062
  onStateChange: (snapshot) => {
2982
3063
  saveProxySupervisorState({
2983
3064
  pid: process.pid,
@@ -3507,7 +3588,11 @@ export const proxyStatusCommand = {
3507
3588
  installedVersion: updateState?.installedVersion ??
3508
3589
  updateState?.lastUpdateVersion ??
3509
3590
  null,
3510
- activatedVersion: supervisorState?.rolling.active?.version ?? null,
3591
+ validatedVersion: updateState?.installedVersion ?? null,
3592
+ selectedPackageVersion: readProxyPackageSelection(PROXY_PACKAGES_DIR)?.version ?? null,
3593
+ diskPackageVersion: readProxyDiskVersion(),
3594
+ candidateVersion: supervisorState?.rolling?.candidate?.expectedVersion ?? null,
3595
+ activatedVersion: supervisorState?.rolling?.active?.version ?? null,
3511
3596
  pendingActivationVersion: updateState?.pendingRestartVersion ?? null,
3512
3597
  pendingRestartVersion: updateState?.pendingRestartVersion ?? null,
3513
3598
  deferredUpdate: updateState?.deferredUpdate ?? null,
@@ -3521,7 +3606,7 @@ export const proxyStatusCommand = {
3521
3606
  const servingWorker = workerRunning &&
3522
3607
  (supervisorPid
3523
3608
  ? supervisorRunning &&
3524
- (!supervisorState ||
3609
+ (!supervisorState?.rolling ||
3525
3610
  supervisorState.rolling.active?.pid === state?.pid)
3526
3611
  : true);
3527
3612
  if ((state || supervisorState) && (servingWorker || supervisorRunning)) {
@@ -3667,7 +3752,7 @@ export const proxyStatusCommand = {
3667
3752
  logger.always(` ${chalk.bold("Last update check:")} ${chalk.cyan(status.lastCheckAt)}`);
3668
3753
  }
3669
3754
  if (status.installedVersion) {
3670
- logger.always(` ${chalk.bold("Installed:")} ${chalk.cyan(`v${status.installedVersion}`)}`);
3755
+ logger.always(` ${chalk.bold("Validated:")} ${chalk.cyan(`v${status.installedVersion}`)}`);
3671
3756
  }
3672
3757
  if (status.activatedVersion) {
3673
3758
  logger.always(` ${chalk.bold("Activated:")} ${chalk.cyan(`v${status.activatedVersion}`)}`);
@@ -3773,6 +3858,18 @@ export const proxyTelemetryCommand = {
3773
3858
  .option("proxy-url", {
3774
3859
  type: "string",
3775
3860
  description: "Proxy endpoint used by the read-only doctor",
3861
+ })
3862
+ .option("admission-lookback-minutes", {
3863
+ type: "number",
3864
+ description: "Doctor admission history horizon; must exceed request timeout plus ingestion grace",
3865
+ })
3866
+ .option("request-timeout-ms", {
3867
+ type: "number",
3868
+ description: "Default request deadline for doctor admission reconciliation",
3869
+ })
3870
+ .option("ingestion-grace-ms", {
3871
+ type: "number",
3872
+ description: "Time allowed for telemetry ingestion before reconciliation",
3776
3873
  })
3777
3874
  .option("quiet", {
3778
3875
  type: "boolean",
@@ -3875,12 +3972,12 @@ export const proxyGuardCommand = {
3875
3972
  const UPDATE_CHECK_INTERVAL_MS = 2 * 60 * 60 * 1000; // 2 hours
3876
3973
  const QUIET_THRESHOLD_MS = 120 * 1000; // 2 minutes of silence
3877
3974
  const NATURAL_WINDOW_WAIT_MS = 10 * 60 * 1000; // prefer no admission pause
3878
- const UPDATE_DRAIN_TIMEOUT_MS = 30 * 60 * 1000; // preserve long streams
3975
+ const UPDATE_DRAIN_TIMEOUT_MS = 30 * 1000; // defer and reopen admission instead of waiting through long streams
3879
3976
  const UPDATE_RETRY_DELAY_MS = 5 * 60 * 1000;
3880
3977
  const UPDATE_RETRY_MAX_DELAY_MS = 30 * 60 * 1000;
3881
3978
  const UPDATE_RETRY_MAX_ATTEMPTS = 4;
3882
3979
  const UPDATE_ACTIVITY_POLL_MS = 10 * 1000;
3883
- const UPDATE_TIMEOUT_MS = 30 * 1000; // 30 seconds to come healthy
3980
+ const UPDATE_TIMEOUT_MS = 150 * 1000; // include the supervisor candidate readiness deadline
3884
3981
  // Get running version from /health endpoint (with timeout to avoid hanging)
3885
3982
  let runningVersion = PROXY_VERSION; // fallback
3886
3983
  try {
@@ -3929,15 +4026,33 @@ export const proxyGuardCommand = {
3929
4026
  logger.always(`[updater] WARNING: failed to ${operation}: ${error instanceof Error ? error.message : String(error)}`);
3930
4027
  }
3931
4028
  };
4029
+ const isCurrentUpdateOwner = () => {
4030
+ const runtime = rollingSupervisor
4031
+ ? loadProxySupervisorState()
4032
+ : loadProxyState();
4033
+ return isProxyUpdateOwnerCurrent({
4034
+ stopping: guardStopping,
4035
+ parentPid,
4036
+ updaterPid: process.pid,
4037
+ parentStatus: getProcessStatus(parentPid),
4038
+ runtimePid: runtime?.pid,
4039
+ runtimeUpdaterPid: runtime?.updaterPid,
4040
+ });
4041
+ };
3932
4042
  let updateInProgress = false;
3933
4043
  let updateRestartInProgress = false;
3934
4044
  const runUpdateCheck = async () => {
3935
- if (guardStopping || updateInProgress) {
4045
+ if (updateInProgress || !isCurrentUpdateOwner()) {
3936
4046
  return;
3937
4047
  }
3938
4048
  updateInProgress = true;
3939
4049
  let updateVersion = runningVersion;
3940
4050
  let drainActive = false;
4051
+ let previousPackage = readProxyPackageSelection(PROXY_PACKAGES_DIR);
4052
+ if (previousPackage?.version !== runningVersion) {
4053
+ previousPackage = readProxyPackageSelection(PROXY_PACKAGES_DIR, true);
4054
+ }
4055
+ let candidatePackage;
3941
4056
  try {
3942
4057
  // Lazy-load update modules so they're only imported at check time
3943
4058
  const { checkForUpdate } = await import("../../proxy/updateChecker.js");
@@ -3979,65 +4094,10 @@ export const proxyGuardCommand = {
3979
4094
  return;
3980
4095
  }
3981
4096
  logger.always(`[updater] update available: ${runningVersion} → ${result.latestVersion}`);
3982
- if (!rollingSupervisor) {
3983
- // Legacy launchd services must become idle before their listener is
3984
- // restarted. Rolling services keep the current worker serving while
3985
- // the candidate package is installed and validated.
3986
- let lastDeferralSignature = "";
3987
- const waitForWindow = async (quietWaitMs) => {
3988
- const window = await waitForProxyUpdateWindow({
3989
- quietThresholdMs: QUIET_THRESHOLD_MS,
3990
- quietWaitMs,
3991
- drainWaitMs: UPDATE_DRAIN_TIMEOUT_MS,
3992
- pollIntervalMs: UPDATE_ACTIVITY_POLL_MS,
3993
- getActivity: () => getProxyRuntimeActivity(host, port),
3994
- setDraining: async (draining) => {
3995
- const changed = await setProxyUpdateDrain(host, port, draining);
3996
- if (changed) {
3997
- logger.always(`[updater] ${draining ? "draining new inference requests" : "inference admission resumed"}`);
3998
- }
3999
- return changed;
4000
- },
4001
- isStopping: () => guardStopping,
4002
- isParentAlive: () => getProcessStatus(parentPid) !== "not_running",
4003
- onPhase: (phase, activity) => {
4004
- const reason = phase === "waiting_for_quiet" && !activity
4005
- ? "activity_unavailable"
4006
- : phase;
4007
- const signature = `${reason}:${activity?.activeRequests ?? "unknown"}`;
4008
- if (signature !== lastDeferralSignature) {
4009
- lastDeferralSignature = signature;
4010
- persistUpdaterState("record update deferral", () => recordUpdateDeferred(result.latestVersion, reason, activity?.activeRequests ?? null));
4011
- }
4012
- logger.debug(`[updater] ${reason} (${activity?.activeRequests ?? "unknown"} active requests)`);
4013
- },
4014
- });
4015
- drainActive = drainActive || window.draining;
4016
- return window;
4017
- };
4018
- let updateWindow = await waitForWindow(NATURAL_WINDOW_WAIT_MS);
4019
- if (!updateWindow.ready) {
4020
- if (updateWindow.reason === "drain_failed") {
4021
- persistUpdaterState("record unavailable update drain", () => recordUpdateDeferred(result.latestVersion, "drain_unavailable", null));
4022
- }
4023
- scheduleUpdateRetry("update window unavailable", result.latestVersion);
4024
- return;
4025
- }
4026
- // Hold admission closed through install and restart after a quiet
4027
- // observation, closing the race with a newly admitted request.
4028
- if (!drainActive) {
4029
- updateWindow = await waitForWindow(0);
4030
- if (!updateWindow.ready) {
4031
- scheduleUpdateRetry("update drain unavailable", result.latestVersion);
4032
- return;
4033
- }
4034
- }
4035
- }
4036
- else if (!supervisorRefreshOnly) {
4037
- logger.always(`[updater] rolling supervisor will keep v${runningVersion} serving during installation`);
4097
+ if (rollingSupervisor && !supervisorRefreshOnly) {
4098
+ logger.always(`[updater] keeping v${runningVersion} serving while staging the candidate`);
4038
4099
  }
4039
- // Refresh after waiting so a long deferral can never install a stale
4040
- // target while a newer release is already available.
4100
+ // Recheck the target before staging because discovery can be delayed.
4041
4101
  let pendingRestart = initiallyPendingRestart;
4042
4102
  if (!supervisorRefreshOnly) {
4043
4103
  const refreshedResult = await checkForUpdate(runningVersion);
@@ -4056,7 +4116,7 @@ export const proxyGuardCommand = {
4056
4116
  }
4057
4117
  }
4058
4118
  persistUpdaterState("clear update deferral", () => clearUpdateDeferral());
4059
- logger.always(`[updater] ${supervisorRefreshOnly ? "supervisor reconciliation prepared" : rollingSupervisor ? "rolling activation prepared" : "safe update window acquired"} for v${result.latestVersion}`);
4119
+ logger.always(`[updater] ${supervisorRefreshOnly ? "supervisor reconciliation prepared" : rollingSupervisor ? "rolling activation prepared" : "candidate preparation started"} for v${result.latestVersion}`);
4060
4120
  // 3. Install update (validate version string before passing to shell)
4061
4121
  if (!/^\d+\.\d+\.\d+$/.test(result.latestVersion)) {
4062
4122
  const message = `invalid version format: ${result.latestVersion}`;
@@ -4072,26 +4132,44 @@ export const proxyGuardCommand = {
4072
4132
  logger.always(`[updater] package-manager candidates: ${installerResolution.tried
4073
4133
  .map((candidate) => `${candidate.kind}:${candidate.bin}(${candidate.installable ? `v${candidate.version}` : (candidate.reason ?? "unusable")})`)
4074
4134
  .join(", ")}`);
4075
- const installer = installerResolution.installer;
4135
+ // Staging never writes a global package root, including when this
4136
+ // updater itself already runs from a private immutable package.
4137
+ const installer = installerResolution.installer ??
4138
+ installerResolution.tried.find((candidate) => candidate.working);
4076
4139
  if (!installer) {
4077
- const message = "no package manager has a writable global root and executable directory";
4140
+ const message = "no working package manager is available for a private staged install";
4078
4141
  logger.always(`[updater] WARNING: ${message}; skipping this cycle`);
4079
4142
  persistUpdaterState("record update failure", () => recordUpdateFailure(result.latestVersion, "install", message));
4080
4143
  return;
4081
4144
  }
4082
4145
  logger.always(`[updater] installing @juspay/neurolink@${result.latestVersion} via ${installer.kind} ${installer.bin} (v${installer.version})`);
4083
- if (guardStopping || getProcessStatus(parentPid) === "not_running") {
4146
+ if (!isCurrentUpdateOwner()) {
4084
4147
  return;
4085
4148
  }
4086
4149
  try {
4087
- execFileSync(installer.bin, getGlobalInstallArgs(installer.kind, `@juspay/neurolink@${result.latestVersion}`), {
4088
- timeout: 120_000,
4089
- stdio: "pipe",
4150
+ // Retain a validated rollback even when migrating from a mutable
4151
+ // global package whose on-disk version already differs from live.
4152
+ previousPackage = await installStagedProxyPackage({
4153
+ version: runningVersion,
4154
+ packagesDir: PROXY_PACKAGES_DIR,
4155
+ installer,
4156
+ });
4157
+ if (!isCurrentUpdateOwner()) {
4158
+ return;
4159
+ }
4160
+ candidatePackage = await installStagedProxyPackage({
4161
+ version: result.latestVersion,
4162
+ packagesDir: PROXY_PACKAGES_DIR,
4163
+ installer,
4164
+ onProgress: ({ elapsedMs, outputBytes }) => logger.debug(`[updater] staged install progress elapsedMs=${elapsedMs} outputBytes=${outputBytes}`),
4090
4165
  });
4091
4166
  }
4092
4167
  catch (installErr) {
4168
+ if (!isCurrentUpdateOwner()) {
4169
+ return;
4170
+ }
4093
4171
  const detail = describeInstallFailure(installErr);
4094
- logger.always(`[updater] WARNING: global install failed:\n${detail}`);
4172
+ logger.always(`[updater] WARNING: staged install failed:\n${detail}`);
4095
4173
  persistUpdaterState("record update failure", () => recordUpdateFailure(result.latestVersion, "install", detail));
4096
4174
  if (isTransientInstallFailure(installErr)) {
4097
4175
  scheduleUpdateRetry("transient install failure", result.latestVersion);
@@ -4102,36 +4180,121 @@ export const proxyGuardCommand = {
4102
4180
  else if (!supervisorRefreshOnly) {
4103
4181
  logger.always(`[updater] resuming pending restart for already-installed v${result.latestVersion}`);
4104
4182
  }
4183
+ // Staging can outlive an operator reinstall or owner replacement.
4184
+ // Never publish or roll back another service's package selection.
4185
+ if (!isCurrentUpdateOwner()) {
4186
+ return;
4187
+ }
4105
4188
  // 4. Refresh and validate the stable trampoline. The plist already
4106
4189
  // points at this path, so it must not be unloaded or rewritten here.
4107
4190
  if (!supervisorRefreshOnly) {
4108
4191
  try {
4109
- writeTrampoline();
4192
+ previousPackage ??= inspectProxyPackage(process.argv[1]);
4193
+ candidatePackage ??=
4194
+ readProxyPackageSelection(PROXY_PACKAGES_DIR) ?? undefined;
4195
+ if (!candidatePackage ||
4196
+ candidatePackage.version !== result.latestVersion) {
4197
+ // A legacy pending marker may refer to an in-place global install.
4198
+ // Never guess that it is safe or point a launcher at unverified files.
4199
+ throw new Error("Pending update has no validated staged package; restaging required");
4200
+ }
4201
+ if (previousPackage?.version === runningVersion &&
4202
+ !pendingRestart) {
4203
+ selectProxyPackage(PROXY_PACKAGES_DIR, previousPackage, isCurrentUpdateOwner);
4204
+ }
4205
+ writeTrampoline(candidatePackage, isCurrentUpdateOwner);
4110
4206
  const validation = await validateInstalledVersion({
4111
4207
  binPath: TRAMPOLINE_PATH,
4112
4208
  expectedVersion: result.latestVersion,
4113
4209
  });
4210
+ if (!isCurrentUpdateOwner()) {
4211
+ return;
4212
+ }
4114
4213
  if (validation.version !== result.latestVersion) {
4115
4214
  const message = `trampoline validation failed after ${validation.attempts} attempts: ${validation.failure ?? "unknown failure"}`;
4116
4215
  logger.always(`[updater] WARNING: ${message}; restart deferred`);
4117
4216
  persistUpdaterState("record update failure", () => recordUpdateFailure(result.latestVersion, "validation", message));
4118
4217
  persistUpdaterState("abandon invalid pending update", () => abandonPendingUpdate(result.latestVersion));
4218
+ if (previousPackage) {
4219
+ writeTrampoline(previousPackage, isCurrentUpdateOwner);
4220
+ }
4221
+ scheduleUpdateRetry("transient candidate validation failure", result.latestVersion);
4119
4222
  return;
4120
4223
  }
4121
4224
  persistUpdaterState("record installed update", () => recordUpdateInstalled(result.latestVersion));
4122
4225
  logger.always(`[updater] trampoline validated at v${validation.version} after ${validation.attempts} attempt(s)`);
4123
4226
  }
4124
4227
  catch (trampolineError) {
4228
+ if (!isCurrentUpdateOwner()) {
4229
+ return;
4230
+ }
4125
4231
  const message = trampolineError instanceof Error
4126
4232
  ? trampolineError.message
4127
4233
  : String(trampolineError);
4128
4234
  logger.always(`[updater] WARNING: failed to refresh trampoline; refusing restart: ${message}`);
4129
4235
  persistUpdaterState("record update failure", () => recordUpdateFailure(result.latestVersion, "validation", message));
4130
4236
  persistUpdaterState("abandon invalid pending update", () => abandonPendingUpdate(result.latestVersion));
4237
+ if (previousPackage) {
4238
+ writeTrampoline(previousPackage, isCurrentUpdateOwner);
4239
+ }
4240
+ scheduleUpdateRetry("candidate validation failed; restaging permitted", result.latestVersion);
4131
4241
  return;
4132
4242
  }
4133
4243
  }
4134
- if (guardStopping || getProcessStatus(parentPid) === "not_running") {
4244
+ if (!rollingSupervisor) {
4245
+ // Staging and validation finish before closing legacy admission.
4246
+ // A busy service gets a bounded drain attempt and resumes on failure.
4247
+ let lastDeferralSignature = "";
4248
+ const waitForWindow = async (quietWaitMs) => {
4249
+ const window = await waitForProxyUpdateWindow({
4250
+ quietThresholdMs: QUIET_THRESHOLD_MS,
4251
+ quietWaitMs,
4252
+ drainWaitMs: UPDATE_DRAIN_TIMEOUT_MS,
4253
+ pollIntervalMs: UPDATE_ACTIVITY_POLL_MS,
4254
+ getActivity: () => getProxyRuntimeActivity(host, port),
4255
+ setDraining: async (draining) => {
4256
+ const changed = await setProxyUpdateDrain(host, port, draining);
4257
+ if (changed) {
4258
+ logger.always(`[updater] ${draining ? "draining new inference requests" : "inference admission resumed"}`);
4259
+ }
4260
+ return changed;
4261
+ },
4262
+ isStopping: () => guardStopping,
4263
+ isParentAlive: () => getProcessStatus(parentPid) !== "not_running",
4264
+ onPhase: (phase, activity) => {
4265
+ const reason = phase === "waiting_for_quiet" && !activity
4266
+ ? "activity_unavailable"
4267
+ : phase;
4268
+ const signature = `${reason}:${activity?.activeRequests ?? "unknown"}`;
4269
+ if (signature !== lastDeferralSignature) {
4270
+ lastDeferralSignature = signature;
4271
+ persistUpdaterState("record update deferral", () => recordUpdateDeferred(result.latestVersion, reason, activity?.activeRequests ?? null));
4272
+ }
4273
+ logger.debug(`[updater] ${reason} (${activity?.activeRequests ?? "unknown"} active requests)`);
4274
+ },
4275
+ });
4276
+ drainActive = drainActive || window.draining;
4277
+ return window;
4278
+ };
4279
+ let updateWindow = await waitForWindow(NATURAL_WINDOW_WAIT_MS);
4280
+ if (!updateWindow.ready) {
4281
+ if (updateWindow.reason === "drain_failed") {
4282
+ persistUpdaterState("record unavailable update drain", () => recordUpdateDeferred(result.latestVersion, "drain_unavailable", null));
4283
+ }
4284
+ scheduleUpdateRetry("update window unavailable", result.latestVersion);
4285
+ return;
4286
+ }
4287
+ // Hold admission closed through restart after a quiet
4288
+ // observation, closing the race with a newly admitted request.
4289
+ if (!drainActive) {
4290
+ updateWindow = await waitForWindow(0);
4291
+ if (!updateWindow.ready) {
4292
+ scheduleUpdateRetry("update drain unavailable", result.latestVersion);
4293
+ return;
4294
+ }
4295
+ }
4296
+ }
4297
+ if (!isCurrentUpdateOwner()) {
4135
4298
  return;
4136
4299
  }
4137
4300
  updateRestartInProgress = true;
@@ -4143,6 +4306,10 @@ export const proxyGuardCommand = {
4143
4306
  else {
4144
4307
  logger.always(`[updater] requesting rolling activation of v${result.latestVersion}`);
4145
4308
  try {
4309
+ if (!isCurrentUpdateOwner()) {
4310
+ updateRestartInProgress = false;
4311
+ return;
4312
+ }
4146
4313
  process.kill(parentPid, "SIGUSR2");
4147
4314
  }
4148
4315
  catch (activationError) {
@@ -4254,6 +4421,10 @@ export const proxyGuardCommand = {
4254
4421
  scheduleUpdateRetry("supervisor refresh window unavailable", result.latestVersion);
4255
4422
  return;
4256
4423
  }
4424
+ if (!isCurrentUpdateOwner()) {
4425
+ updateRestartInProgress = false;
4426
+ return;
4427
+ }
4257
4428
  logger.always(`[updater] refreshing launchd supervisor at v${result.latestVersion}`);
4258
4429
  const uid = process.getuid?.() ?? 501;
4259
4430
  try {
@@ -4312,30 +4483,34 @@ export const proxyGuardCommand = {
4312
4483
  ? `rolling candidate failed: ${rollingFailure}`
4313
4484
  : `proxy did not report v${result.latestVersion} healthy within ${UPDATE_TIMEOUT_MS}ms`;
4314
4485
  if (rollingSupervisor && !supervisorRefreshOnly) {
4486
+ if (!isCurrentUpdateOwner()) {
4487
+ updateRestartInProgress = false;
4488
+ return;
4489
+ }
4315
4490
  try {
4316
- const rollbackResolution = resolveGlobalInstaller({
4317
- entryScript: process.argv[1],
4318
- });
4319
- const rollbackInstaller = rollbackResolution.installer;
4320
- if (!rollbackInstaller) {
4321
- throw new Error("no usable package manager is available for rollback");
4491
+ if (!previousPackage ||
4492
+ previousPackage.version !== runningVersion) {
4493
+ throw new Error("Previous validated package selection is unavailable");
4322
4494
  }
4323
- logger.always(`[updater] rolling activation failed; restoring @juspay/neurolink@${runningVersion}`);
4324
- execFileSync(rollbackInstaller.bin, getGlobalInstallArgs(rollbackInstaller.kind, `@juspay/neurolink@${runningVersion}`), { timeout: 120_000, stdio: "pipe" });
4325
- writeTrampoline();
4495
+ logger.always(`[updater] restoring immutable package selection v${runningVersion}`);
4496
+ writeTrampoline(previousPackage, isCurrentUpdateOwner);
4326
4497
  const rollbackValidation = await validateInstalledVersion({
4327
4498
  binPath: TRAMPOLINE_PATH,
4328
4499
  expectedVersion: runningVersion,
4329
4500
  });
4330
4501
  if (rollbackValidation.version !== runningVersion) {
4331
- throw new Error(`rollback trampoline reported v${rollbackValidation.version ?? "unknown"}; expected v${runningVersion}`);
4502
+ throw new Error(`Rollback selection did not validate as v${runningVersion}`);
4332
4503
  }
4333
- // Reinstalling the old package is not enough: the supervisor was
4504
+ // Restoring the old package selection is not enough: the supervisor was
4334
4505
  // told to activate result.latestVersion, so its recovery loop
4335
4506
  // keeps demanding the new version against the now-downgraded
4336
4507
  // installation and can never serve a worker again. Re-point the
4337
4508
  // supervisor at runningVersion and report success only once it
4338
4509
  // actually serves that version again.
4510
+ if (!isCurrentUpdateOwner()) {
4511
+ updateRestartInProgress = false;
4512
+ return;
4513
+ }
4339
4514
  const rollbackActivated = await activateRollbackVersion(host, port, parentPid, runningVersion, UPDATE_TIMEOUT_MS);
4340
4515
  if (!rollbackActivated) {
4341
4516
  throw new Error(`supervisor did not report v${runningVersion} healthy within ${UPDATE_TIMEOUT_MS}ms after rollback`);
@@ -4367,7 +4542,7 @@ export const proxyGuardCommand = {
4367
4542
  persistUpdaterState("record update failure", () => recordUpdateFailure(updateVersion, "check", message));
4368
4543
  }
4369
4544
  finally {
4370
- if (drainActive && !updateRestartInProgress) {
4545
+ if (drainActive && !updateRestartInProgress && isCurrentUpdateOwner()) {
4371
4546
  const resumed = await resumeProxyUpdateDrain(host, port);
4372
4547
  if (!resumed && getProcessStatus(parentPid) !== "not_running") {
4373
4548
  logger.always(`[updater] WARNING: failed to resume inference admission after deferred update`);
@@ -4651,10 +4826,8 @@ function buildLaunchdPath() {
4651
4826
  return [...segments].join(":");
4652
4827
  }
4653
4828
  export function buildProxyLaunchdPlist(port, host, envFile, configFile) {
4654
- // The plist invokes the trampoline script (a tiny shell wrapper at
4655
- // ~/.neurolink/bin/neurolink-proxy) which re-resolves the real
4656
- // `neurolink` binary via PATH on every launch. This way, launchd
4657
- // is never pinned to a version-specific pnpm store path.
4829
+ // The stable launcher selects an already validated package without
4830
+ // rewriting the launchd service or its inherited runtime environment.
4658
4831
  const trampolinePath = escapeXml(TRAMPOLINE_PATH);
4659
4832
  const envFileArgs = envFile
4660
4833
  ? `
@@ -4666,20 +4839,11 @@ export function buildProxyLaunchdPlist(port, host, envFile, configFile) {
4666
4839
  <string>--config</string>
4667
4840
  <string>${escapeXml(configFile)}</string>`
4668
4841
  : "";
4669
- const otelEnvironment = isProxyOtelOnly()
4670
- ? [
4671
- "NEUROLINK_PROXY_LOG_SINK",
4672
- "OTEL_EXPORTER_OTLP_ENDPOINT",
4673
- "OTEL_EXPORTER_OTLP_LOGS_ENDPOINT",
4674
- "OTEL_EXPORTER_OTLP_HEADERS",
4675
- "OTEL_EXPORTER_OTLP_LOGS_HEADERS",
4676
- "OTEL_SERVICE_NAME",
4677
- "NEUROLINK_PROXY_SESSION_SECRET",
4678
- ]
4679
- .filter((name) => process.env[name] !== undefined)
4680
- .map((name) => ` <key>${name}</key>\n <string>${escapeXml(process.env[name])}</string>`)
4681
- .join("\n")
4682
- : "";
4842
+ const otelEnvironment = Object.keys(process.env)
4843
+ .filter((name) => isProxyServiceEnvironmentKey(name) && process.env[name] !== undefined)
4844
+ .sort()
4845
+ .map((name) => ` <key>${escapeXml(name)}</key>\n <string>${escapeXml(process.env[name])}</string>`)
4846
+ .join("\n");
4683
4847
  return `<?xml version="1.0" encoding="UTF-8"?>
4684
4848
  <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN"
4685
4849
  "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
@@ -4742,12 +4906,10 @@ export const proxyInstallCommand = {
4742
4906
  .option("port", {
4743
4907
  type: "number",
4744
4908
  alias: "p",
4745
- default: 55669,
4746
4909
  description: "Proxy port",
4747
4910
  })
4748
4911
  .option("host", {
4749
4912
  type: "string",
4750
- default: "127.0.0.1",
4751
4913
  description: "Proxy host",
4752
4914
  })
4753
4915
  .option("env-file", {
@@ -4763,19 +4925,56 @@ export const proxyInstallCommand = {
4763
4925
  .example("neurolink proxy install -p 9000", "Install on custom port");
4764
4926
  },
4765
4927
  handler: async (argv) => {
4766
- const port = argv.port ?? 55669;
4767
- const host = argv.host ?? "127.0.0.1";
4768
4928
  if (process.platform !== "darwin") {
4769
4929
  console.info(chalk.red("proxy install is currently macOS-only (uses launchd)."));
4770
4930
  console.info(chalk.yellow("On Linux, use systemd. On Windows, use Task Scheduler."));
4771
4931
  process.exit(1);
4772
4932
  }
4933
+ const existingSupervisor = loadProxySupervisorState();
4934
+ const existingWorker = loadProxyState();
4935
+ assertProxyServiceInstallIdle({
4936
+ pids: [
4937
+ existingSupervisor?.pid,
4938
+ existingWorker?.pid,
4939
+ existingSupervisor?.rolling?.active?.pid,
4940
+ existingSupervisor?.rolling?.candidate?.pid,
4941
+ ...(existingSupervisor?.rolling?.draining?.map((worker) => worker.pid) ?? []),
4942
+ ].filter((pid) => typeof pid === "number"),
4943
+ getProcessStatus,
4944
+ launchdPresence: probeProxyLaunchdPresence({
4945
+ label: PLIST_LABEL,
4946
+ uid: process.getuid?.() ?? 501,
4947
+ }),
4948
+ });
4773
4949
  const { mkdirSync, existsSync, chmodSync } = await import("fs");
4950
+ let savedSettings = parseProxyServiceInstallSettings(null);
4951
+ if (existsSync(PLIST_PATH)) {
4952
+ // Existing runtime values are defaults, never printed. Refuse to rewrite
4953
+ // an unreadable service rather than silently dropping its OTel settings.
4954
+ const { execFileSync } = await import("node:child_process");
4955
+ try {
4956
+ savedSettings = parseProxyServiceInstallSettings(JSON.parse(execFileSync("plutil", ["-convert", "json", "-o", "-", PLIST_PATH], {
4957
+ encoding: "utf8",
4958
+ timeout: 5_000,
4959
+ stdio: ["ignore", "pipe", "pipe"],
4960
+ })));
4961
+ }
4962
+ catch {
4963
+ // Neither captured plutil output nor JSON parse excerpts may expose
4964
+ // stored exporter authorization headers in a reinstall error.
4965
+ throw new Error("Existing proxy service settings could not be read; refusing reinstall before service changes.");
4966
+ }
4967
+ for (const [name, value] of Object.entries(savedSettings.environment)) {
4968
+ process.env[name] ??= value;
4969
+ }
4970
+ }
4971
+ const port = argv.port ?? savedSettings.port ?? 55669;
4972
+ const host = argv.host ?? savedSettings.host ?? "127.0.0.1";
4774
4973
  const envResolution = resolveProxyEnvFile({
4775
- explicitEnvFile: argv.envFile,
4974
+ explicitEnvFile: argv.envFile ?? savedSettings.envFile,
4776
4975
  });
4777
4976
  const envFile = envResolution.path;
4778
- const explicitConfig = argv.config;
4977
+ const explicitConfig = argv.config ?? savedSettings.configFile;
4779
4978
  const configPath = resolveProxyConfigPath(explicitConfig);
4780
4979
  if (explicitConfig && !existsSync(configPath)) {
4781
4980
  console.info(chalk.red(`Proxy config file not found: ${configPath}`));
@@ -4796,20 +4995,17 @@ export const proxyInstallCommand = {
4796
4995
  }
4797
4996
  writeTrampoline();
4798
4997
  console.info(chalk.green(`✓ Trampoline written to ${TRAMPOLINE_PATH}`));
4799
- // Sanity-check: run the trampoline itself and confirm it resolves to
4800
- // a working neurolink binary. This catches environments where every
4801
- // PATH-based candidate is broken AND the baked-in path is unreachable.
4998
+ // Validate selected package metadata without initializing the CLI graph.
4999
+ // Worker startup separately proves readiness before accepting traffic.
4802
5000
  const trampolineVersion = probeBinVersion(TRAMPOLINE_PATH);
4803
5001
  if (!trampolineVersion) {
4804
5002
  console.info(chalk.red(`✗ Trampoline validation failed: ${TRAMPOLINE_PATH} --version did not run cleanly.`));
4805
5003
  console.info(chalk.yellow(` The launchd service would not be able to start neurolink. Fix your install first.`));
4806
- console.info(chalk.yellow(` Try: 'pnpm add -g @juspay/neurolink' or set NEUROLINK_BIN=/path/to/working/neurolink.`));
5004
+ console.info(chalk.yellow(` Inspect the selected package and reinstall the missing version before retrying.`));
4807
5005
  process.exit(1);
4808
5006
  }
4809
5007
  if (trampolineVersion !== PROXY_VERSION) {
4810
- console.info(chalk.red(`✗ Trampoline resolves to v${trampolineVersion} but this installer is v${PROXY_VERSION}.`));
4811
- console.info(chalk.yellow(` PATH may shadow this installation with an older version. Fix your PATH or set NEUROLINK_BIN.`));
4812
- process.exit(1);
5008
+ console.info(chalk.yellow(`Retaining selected proxy package v${trampolineVersion}; this service installer is v${PROXY_VERSION}.`));
4813
5009
  }
4814
5010
  console.info(chalk.green(`✓ Trampoline validated (resolves to neurolink v${trampolineVersion})`));
4815
5011
  const plist = buildProxyLaunchdPlist(port, host, envFile, configFile);