@juspay/neurolink 12.17.0 → 12.17.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +2 -7
- package/dist/browser/neurolink.min.js +422 -413
- package/dist/cli/commands/proxy.js +455 -259
- package/dist/constants/contextWindows.d.ts +2 -0
- package/dist/constants/contextWindows.js +4 -0
- package/dist/providers/anthropic/client.js +15 -5
- package/dist/proxy/anthropicBudgetResponse.d.ts +3 -0
- package/dist/proxy/anthropicBudgetResponse.js +153 -0
- package/dist/proxy/bodyCaptureProcessing.d.ts +1 -0
- package/dist/proxy/bodyCaptureProcessing.js +23 -4
- package/dist/proxy/bodyCaptureWorker.js +20 -2
- package/dist/proxy/codexFallback.d.ts +10 -0
- package/dist/proxy/codexFallback.js +108 -24
- package/dist/proxy/codexUsage.js +2 -0
- package/dist/proxy/globalInstaller.d.ts +32 -1
- package/dist/proxy/globalInstaller.js +386 -3
- package/dist/proxy/otelLogSink.d.ts +3 -3
- package/dist/proxy/otelLogSink.js +137 -144
- package/dist/proxy/otlpLogTransport.d.ts +3 -0
- package/dist/proxy/otlpLogTransport.js +334 -0
- package/dist/proxy/proxyActivity.d.ts +22 -2
- package/dist/proxy/proxyActivity.js +114 -9
- package/dist/proxy/proxyAnalysis.js +82 -52
- package/dist/proxy/proxyClientCancellation.d.ts +7 -0
- package/dist/proxy/proxyClientCancellation.js +23 -0
- package/dist/proxy/proxyContextPreflight.d.ts +25 -0
- package/dist/proxy/proxyContextPreflight.js +337 -0
- package/dist/proxy/proxyFailureDetails.d.ts +19 -0
- package/dist/proxy/proxyFailureDetails.js +96 -0
- package/dist/proxy/proxyLifecycle.js +7 -0
- package/dist/proxy/proxyRouteBodyCapture.d.ts +13 -0
- package/dist/proxy/proxyRouteBodyCapture.js +112 -0
- package/dist/proxy/proxyTokenBudget.d.ts +40 -0
- package/dist/proxy/proxyTokenBudget.js +523 -0
- package/dist/proxy/proxyTracer.d.ts +3 -2
- package/dist/proxy/proxyTracer.js +46 -20
- package/dist/proxy/proxyTranslationEngine.d.ts +8 -10
- package/dist/proxy/proxyTranslationEngine.js +736 -376
- package/dist/proxy/requestLogger.d.ts +1 -0
- package/dist/proxy/requestLogger.js +107 -5
- package/dist/proxy/restartControl.js +9 -4
- package/dist/proxy/rollingWorkerProcess.js +65 -0
- package/dist/proxy/rollingWorkerSupervisor.js +7 -0
- package/dist/proxy/updateCoordinator.d.ts +14 -1
- package/dist/proxy/updateCoordinator.js +57 -2
- package/dist/server/routes/claudeProxyRoutes.d.ts +4 -5
- package/dist/server/routes/claudeProxyRoutes.js +809 -577
- package/dist/server/routes/codexProxyRoutes.js +191 -9
- package/dist/server/routes/openaiProxyRoutes.d.ts +1 -1
- package/dist/server/routes/openaiProxyRoutes.js +261 -69
- package/dist/types/cli.d.ts +3 -0
- package/dist/types/codex.d.ts +5 -1
- package/dist/types/index.d.ts +2 -0
- package/dist/types/index.js +2 -0
- package/dist/types/proxy.d.ts +148 -3
- package/dist/types/proxy.js +0 -15
- package/dist/types/proxyBudget.d.ts +74 -0
- package/dist/types/proxyBudget.js +1 -0
- package/dist/types/proxyContext.d.ts +35 -0
- package/dist/types/proxyContext.js +1 -0
- package/dist/utils/toolChoice.d.ts +19 -0
- package/dist/utils/toolChoice.js +39 -0
- package/docs/assets/dashboards/README.md +37 -0
- package/docs/assets/dashboards/neurolink-proxy-observability-dashboard.json +197 -89
- package/docs-site/static/search-index.json +1 -1
- package/package.json +4 -1
- package/scripts/observability/NATIVE-DURABLE.md +63 -0
- package/scripts/observability/otel-collector.proxy-native-durable.yaml +93 -0
- package/scripts/observability/proxy-telemetry-backend.mjs +32 -5
- package/scripts/observability/proxy-telemetry-check.mjs +396 -61
- package/scripts/observability/query-proxy-history.mjs +34 -5
|
@@ -12,7 +12,7 @@ import { logger } from "../../utils/logger.js";
|
|
|
12
12
|
import { applyAllClients, restoreAllClients, } from "../proxy-clients/registry.js";
|
|
13
13
|
import { resolveProxyConfigPath } from "../../proxy/proxyConfig.js";
|
|
14
14
|
import { redactUrlsInText, sanitizeForLog, } from "../../utils/logSanitize.js";
|
|
15
|
-
import { withTimeout } from "../../utils/async/withTimeout.js";
|
|
15
|
+
import { raceWithAbort, withTimeout, } from "../../utils/async/withTimeout.js";
|
|
16
16
|
import { startProxyHttpTrace } from "../../proxy/proxyTracer.js";
|
|
17
17
|
import { formatUptime, isProcessRunning, StateFileManager, } from "../utils/serverUtils.js";
|
|
18
18
|
import { configureProxyKeepAliveDispatcher } from "../../proxy/proxyDispatcher.js";
|
|
@@ -20,19 +20,22 @@ import { ProxyRuntimeConfigStore } from "../../proxy/runtimeConfig.js";
|
|
|
20
20
|
import { startProxyLogCleanupScheduler } from "../../proxy/logCleanupScheduler.js";
|
|
21
21
|
import { anthropicAccountKeysEqual, createAccountAllowlist, isAccountAllowed, LEGACY_ANTHROPIC_ACCOUNT_KEY, normalizeAnthropicAccountKey, shouldLoadFallbackCredential, } from "../../proxy/accountSelection.js";
|
|
22
22
|
import { resolveProxyStatusAccountIdentity } from "../../proxy/codexAccountUsage.js";
|
|
23
|
-
import { beginProxyRequest, getProxyActivitySnapshot, observeProxyFinalLog, takeProxyResponseObservers, trackProxyResponse, } from "../../proxy/proxyActivity.js";
|
|
23
|
+
import { beginProxyRequest, consumeInternalProxyRequest, releaseProxyRequestAccounting, withProxyFinalLogOwnership, getProxyActivitySnapshot, observeProxyFinalLog, takeProxyResponseObservers, trackProxyResponse, } from "../../proxy/proxyActivity.js";
|
|
24
24
|
import { configureProxyLifecycleLogger, flushProxyLifecycleEvents, persistProxyLifecycleAcceptance, getProxyLifecycleLoggerSnapshot, hashProxyLifecycleSessionId, logProxyLifecycleEvent, } from "../../proxy/proxyLifecycle.js";
|
|
25
|
-
import { describeInstallFailure,
|
|
25
|
+
import { describeInstallFailure, assertProxyServiceInstallIdle, probeProxyLaunchdPresence, inspectProxyPackage, isProxyServiceEnvironmentKey, parseProxyServiceInstallSettings, installStagedProxyPackage, readProxyPackageSelection, resolveProxyWorkerPackage, selectProxyPackage, writeProxyPackageLauncher, isTransientInstallFailure, resolveGlobalInstaller, validateInstalledVersion, } from "../../proxy/globalInstaller.js";
|
|
26
26
|
import { startUpdaterWorkerSupervisor } from "../../proxy/updaterSupervisor.js";
|
|
27
27
|
import { openProxyWorkerLog } from "../../proxy/workerLog.js";
|
|
28
28
|
import { startRollingProxyServer } from "../../proxy/rollingProxyServer.js";
|
|
29
29
|
import { startProxyRestartControl, requestProxySupervisorTelemetry, } from "../../proxy/restartControl.js";
|
|
30
30
|
import { getProxyProcessTelemetry } from "../../proxy/processTelemetry.js";
|
|
31
|
+
import { recordFinalError as recordProxyRuntimeFinalError } from "../../proxy/usageStats.js";
|
|
32
|
+
import { logRequest as logProxyRequestFinal } from "../../proxy/requestLogger.js";
|
|
33
|
+
import { bindProxyClientCancellation } from "../../proxy/proxyClientCancellation.js";
|
|
31
34
|
import { isProxyAuxiliaryRequest } from "../../proxy/proxyRequestKind.js";
|
|
32
35
|
import { spawnProxySocketWorker } from "../../proxy/rollingWorkerProcess.js";
|
|
33
36
|
import { PROXY_ROLLING_SUPERVISOR_ENV, PROXY_SOCKET_WORKER_ENV, } from "../../proxy/rollingWorkerProtocol.js";
|
|
34
37
|
import { attachSocketWorkerProcess } from "../../proxy/socketWorkerRuntime.js";
|
|
35
|
-
import { shouldRefreshStaleSupervisor, waitForProxyUpdateWindow, } from "../../proxy/updateCoordinator.js";
|
|
38
|
+
import { parseProxyRuntimeActivity, isProxyUpdateOwnerCurrent, shouldRefreshStaleSupervisor, waitForProxyUpdateWindow, } from "../../proxy/updateCoordinator.js";
|
|
36
39
|
import { abandonPendingUpdate, clearUpdateDeferral, isVersionSuppressed, loadUpdateState, recordCheck, recordSuccessfulUpdate, recordUpdateDeferred, recordUpdateFailure, recordUpdateInstalled, suppressVersion, } from "../../proxy/updateState.js";
|
|
37
40
|
import { loadProxyEnvFile, resolveProxyEnvFile, } from "../../proxy/proxyEnv.js";
|
|
38
41
|
import { createRequire } from "node:module";
|
|
@@ -479,17 +482,7 @@ async function getProxyRuntimeActivity(host, port, timeoutMs = 3_000) {
|
|
|
479
482
|
if (!response.ok) {
|
|
480
483
|
return null;
|
|
481
484
|
}
|
|
482
|
-
|
|
483
|
-
const activeRequests = Number(payload.activity?.activeRequests);
|
|
484
|
-
if (!Number.isFinite(activeRequests) || activeRequests < 0) {
|
|
485
|
-
return null;
|
|
486
|
-
}
|
|
487
|
-
return {
|
|
488
|
-
activeRequests,
|
|
489
|
-
lastActivityAt: typeof payload.activity?.lastActivityAt === "string"
|
|
490
|
-
? payload.activity.lastActivityAt
|
|
491
|
-
: null,
|
|
492
|
-
};
|
|
485
|
+
return parseProxyRuntimeActivity(await response.json());
|
|
493
486
|
}
|
|
494
487
|
catch {
|
|
495
488
|
return null;
|
|
@@ -665,11 +658,12 @@ async function resumeProxyUpdateDrain(host, port, attempts = 3) {
|
|
|
665
658
|
// ---------------------------------------------------------------------------
|
|
666
659
|
/**
|
|
667
660
|
* Path to a small trampoline script that the plist invokes.
|
|
668
|
-
* The trampoline
|
|
669
|
-
*
|
|
661
|
+
* The trampoline selects a validated immutable package. The stable launchd
|
|
662
|
+
* path does not require reinstalling the service to activate another version.
|
|
670
663
|
*/
|
|
671
664
|
const TRAMPOLINE_DIR = join(homedir(), ".neurolink", "bin");
|
|
672
665
|
const TRAMPOLINE_PATH = join(TRAMPOLINE_DIR, "neurolink-proxy");
|
|
666
|
+
const PROXY_PACKAGES_DIR = join(homedir(), ".neurolink", "proxy-packages");
|
|
673
667
|
/**
|
|
674
668
|
* Verify a candidate bin path actually runs by invoking `--version` on it.
|
|
675
669
|
* Returns the version string on success, or undefined on any failure.
|
|
@@ -691,96 +685,26 @@ function probeBinVersion(binPath) {
|
|
|
691
685
|
/**
|
|
692
686
|
* Write (or overwrite) the trampoline shell script.
|
|
693
687
|
*
|
|
694
|
-
*
|
|
695
|
-
*
|
|
696
|
-
* candidate is broken (stale shims, missing packages), it falls back to the
|
|
697
|
-
* baked-in `node + script` path that was verified to work at install time.
|
|
688
|
+
* Selection is explicit: package validation happens before rewriting the
|
|
689
|
+
* launcher, and launch itself never imports the CLI merely to probe a version.
|
|
698
690
|
*/
|
|
699
|
-
function writeTrampoline() {
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
const bakedNode = process.execPath;
|
|
708
|
-
const bakedScript = process.argv[1] ?? join(__dirname, "..", "index.js");
|
|
709
|
-
// Shell-escape the baked paths (they shouldn't contain quotes in practice,
|
|
710
|
-
// but be safe for paths with spaces).
|
|
711
|
-
const shEscape = (s) => `'${s.replace(/'/g, "'\\''")}'`;
|
|
712
|
-
const script = `#!/bin/sh
|
|
713
|
-
# Auto-generated by \`neurolink proxy install\` — do not edit.
|
|
714
|
-
# Resolves a working neurolink binary on every launchd invocation so the
|
|
715
|
-
# plist never gets pinned to a broken/stale shim.
|
|
716
|
-
|
|
717
|
-
BAKED_NODE=${shEscape(bakedNode)}
|
|
718
|
-
BAKED_SCRIPT=${shEscape(bakedScript)}
|
|
719
|
-
|
|
720
|
-
# IPC workers must preserve the inherited Node channel until the final exec.
|
|
721
|
-
# Do not launch a --version probe subprocess on that descriptor; the parent
|
|
722
|
-
# validates the worker's exact package version before activation instead.
|
|
723
|
-
if [ "\${NEUROLINK_PROXY_TRAMPOLINE_EXEC_ONLY:-0}" = "1" ]; then
|
|
724
|
-
if [ -n "\${NEUROLINK_BIN:-}" ] && [ -x "$NEUROLINK_BIN" ]; then
|
|
725
|
-
exec "$NEUROLINK_BIN" "$@"
|
|
726
|
-
fi
|
|
727
|
-
for cand in \
|
|
728
|
-
"$(command -v neurolink 2>/dev/null || true)" \
|
|
729
|
-
"\${PNPM_HOME:-}/neurolink" \
|
|
730
|
-
"$HOME/.local/share/pnpm/neurolink" \
|
|
731
|
-
"$HOME/Library/pnpm/neurolink" \
|
|
732
|
-
"/usr/local/bin/neurolink" \
|
|
733
|
-
"/opt/homebrew/bin/neurolink"; do
|
|
734
|
-
[ -n "$cand" ] && [ -x "$cand" ] && exec "$cand" "$@"
|
|
735
|
-
done
|
|
736
|
-
if [ -x "$BAKED_NODE" ] && [ -f "$BAKED_SCRIPT" ]; then
|
|
737
|
-
exec "$BAKED_NODE" "$BAKED_SCRIPT" "$@"
|
|
738
|
-
fi
|
|
739
|
-
exit 127
|
|
740
|
-
fi
|
|
741
|
-
|
|
742
|
-
# Probe a candidate: must be executable and respond to --version cleanly.
|
|
743
|
-
_try() {
|
|
744
|
-
[ -n "$1" ] && [ -x "$1" ] || return 1
|
|
745
|
-
"$1" --version >/dev/null 2>&1 || return 1
|
|
746
|
-
return 0
|
|
691
|
+
function writeTrampoline(selection, isCurrentOwner) {
|
|
692
|
+
// Reinstall preserves an already validated proxy-owned selection. A manual
|
|
693
|
+
// global npm upgrade cannot silently retarget existing worker generations.
|
|
694
|
+
const selected = selection ??
|
|
695
|
+
readProxyPackageSelection(PROXY_PACKAGES_DIR) ??
|
|
696
|
+
inspectProxyPackage(process.argv[1] ?? join(__dirname, "..", "index.js"));
|
|
697
|
+
writeProxyPackageLauncher(TRAMPOLINE_PATH, selected, isCurrentOwner);
|
|
698
|
+
selectProxyPackage(PROXY_PACKAGES_DIR, selected, isCurrentOwner);
|
|
747
699
|
}
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
# 2. PATH-based and common install locations. First working one wins.
|
|
758
|
-
for cand in \\
|
|
759
|
-
"$(command -v neurolink 2>/dev/null || true)" \\
|
|
760
|
-
"\${PNPM_HOME:-}/neurolink" \\
|
|
761
|
-
"$HOME/.local/share/pnpm/neurolink" \\
|
|
762
|
-
"$HOME/Library/pnpm/neurolink" \\
|
|
763
|
-
"/usr/local/bin/neurolink" \\
|
|
764
|
-
"/opt/homebrew/bin/neurolink"; do
|
|
765
|
-
if _try "$cand"; then
|
|
766
|
-
exec "$cand" "$@"
|
|
767
|
-
fi
|
|
768
|
-
done
|
|
769
|
-
|
|
770
|
-
# 3. Baked-in fallback: the exact node + script that worked at install time.
|
|
771
|
-
# Always valid at install time; may become stale after package updates
|
|
772
|
-
# (but at that point the PATH candidates above should work).
|
|
773
|
-
if [ -x "$BAKED_NODE" ] && [ -f "$BAKED_SCRIPT" ]; then
|
|
774
|
-
exec "$BAKED_NODE" "$BAKED_SCRIPT" "$@"
|
|
775
|
-
fi
|
|
776
|
-
|
|
777
|
-
echo "[neurolink-proxy] FATAL: no working neurolink binary found." >&2
|
|
778
|
-
echo "[neurolink-proxy] Tried: PATH, \\$PNPM_HOME, \\$HOME/.local/share/pnpm, \\$HOME/Library/pnpm, /usr/local/bin, /opt/homebrew/bin, baked-in install path." >&2
|
|
779
|
-
echo "[neurolink-proxy] Fix: reinstall with 'pnpm add -g @juspay/neurolink' or set NEUROLINK_BIN=/path/to/working/neurolink." >&2
|
|
780
|
-
exit 127
|
|
781
|
-
`;
|
|
782
|
-
writeFileSync(TRAMPOLINE_PATH, script, { mode: 0o755 });
|
|
783
|
-
chmodSync(TRAMPOLINE_PATH, 0o755);
|
|
700
|
+
/** Status must remain available when an external installer damages its entrypoint. */
|
|
701
|
+
function readProxyDiskVersion() {
|
|
702
|
+
try {
|
|
703
|
+
return inspectProxyPackage(process.argv[1]).version;
|
|
704
|
+
}
|
|
705
|
+
catch {
|
|
706
|
+
return null;
|
|
707
|
+
}
|
|
784
708
|
}
|
|
785
709
|
function spawnFailOpenGuard(host, port, parentPid) {
|
|
786
710
|
// The guard runs the same version as this process, so process.argv[1]
|
|
@@ -892,6 +816,9 @@ async function runProxyTelemetryManager(command, argv) {
|
|
|
892
816
|
["format", "--format"],
|
|
893
817
|
["maxRows", "--max-rows"],
|
|
894
818
|
["proxyUrl", "--proxy-url"],
|
|
819
|
+
["admissionLookbackMinutes", "--admission-lookback-minutes"],
|
|
820
|
+
["requestTimeoutMs", "--request-timeout-ms"],
|
|
821
|
+
["ingestionGraceMs", "--ingestion-grace-ms"],
|
|
895
822
|
]) {
|
|
896
823
|
if (argv[key] !== undefined) {
|
|
897
824
|
queryArgs.push(option, String(argv[key]));
|
|
@@ -1129,15 +1056,22 @@ function registerProxyRequestTracking(app, requestMetadata, readiness) {
|
|
|
1129
1056
|
c.req.raw.headers.get("x-claude-code-session-id") ??
|
|
1130
1057
|
undefined;
|
|
1131
1058
|
const sessionHash = hashProxyLifecycleSessionId(sessionId);
|
|
1059
|
+
const internal = consumeInternalProxyRequest(c.req.raw.headers.get("x-neurolink-internal-request"));
|
|
1060
|
+
const abortController = new AbortController();
|
|
1132
1061
|
const metadata = {
|
|
1133
|
-
requestId: crypto.randomUUID(),
|
|
1062
|
+
requestId: internal?.requestId ?? crypto.randomUUID(),
|
|
1063
|
+
parentRequestId: internal?.parentRequestId,
|
|
1064
|
+
accountingScope: internal ? "internal" : "client",
|
|
1065
|
+
usageOwnerRequestId: internal?.requestId,
|
|
1066
|
+
abortController,
|
|
1134
1067
|
method: c.req.method,
|
|
1135
1068
|
path: c.req.path,
|
|
1136
1069
|
startedAt: Date.now(),
|
|
1137
1070
|
model: "-",
|
|
1138
1071
|
stream: false,
|
|
1139
1072
|
toolCount: 0,
|
|
1140
|
-
|
|
1073
|
+
// Trusted child work belongs to an already admitted parent request.
|
|
1074
|
+
rejectForUpdate: !internal && readiness.drainingForUpdate,
|
|
1141
1075
|
};
|
|
1142
1076
|
requestMetadata.set(c.req.raw, metadata);
|
|
1143
1077
|
const httpTrace = startProxyHttpTrace(metadata, Object.fromEntries(c.req.raw.headers));
|
|
@@ -1149,7 +1083,16 @@ function registerProxyRequestTracking(app, requestMetadata, readiness) {
|
|
|
1149
1083
|
const finishActivity = metadata.rejectForUpdate
|
|
1150
1084
|
? () => undefined
|
|
1151
1085
|
: beginProxyRequest();
|
|
1086
|
+
let responseTracking = false;
|
|
1087
|
+
let cancelledBeforeHeaders = false;
|
|
1088
|
+
const deadline = setTimeout(() => abortController.abort(new DOMException("Proxy request deadline exceeded", "TimeoutError")), 900_000);
|
|
1089
|
+
deadline.unref?.();
|
|
1090
|
+
let unbindClient = () => { };
|
|
1152
1091
|
const finish = () => {
|
|
1092
|
+
clearTimeout(deadline);
|
|
1093
|
+
unbindClient();
|
|
1094
|
+
abortController.signal.removeEventListener("abort", onPreheaderAbort);
|
|
1095
|
+
releaseProxyRequestAccounting(metadata.requestId);
|
|
1153
1096
|
httpTrace.end(metadata.terminalResult?.responseStatus ?? c.res.status, metadata.terminalResult?.terminalOutcome ?? "unknown", metadata.terminalErrorType);
|
|
1154
1097
|
stopObservingFinalLog();
|
|
1155
1098
|
finishActivity();
|
|
@@ -1159,12 +1102,77 @@ function registerProxyRequestTracking(app, requestMetadata, readiness) {
|
|
|
1159
1102
|
metadata.shareRelease?.();
|
|
1160
1103
|
requestMetadata.delete(c.req.raw);
|
|
1161
1104
|
};
|
|
1105
|
+
const onPreheaderAbort = () => {
|
|
1106
|
+
if (responseTracking || cancelledBeforeHeaders) {
|
|
1107
|
+
return;
|
|
1108
|
+
}
|
|
1109
|
+
cancelledBeforeHeaders = true;
|
|
1110
|
+
const timeout = abortController.signal.reason?.name === "TimeoutError";
|
|
1111
|
+
const status = timeout ? 504 : 499;
|
|
1112
|
+
const errorType = timeout ? "request_timeout" : "client_cancelled";
|
|
1113
|
+
if (!metadata.terminalResult && metadata.accountingScope !== "internal") {
|
|
1114
|
+
recordProxyRuntimeFinalError(status, undefined, undefined, {
|
|
1115
|
+
requestId: metadata.requestId,
|
|
1116
|
+
errorType,
|
|
1117
|
+
terminalOutcome: timeout ? "handler_error" : "client_cancelled",
|
|
1118
|
+
message: timeout
|
|
1119
|
+
? "Request deadline exceeded"
|
|
1120
|
+
: "Client disconnected before response headers",
|
|
1121
|
+
});
|
|
1122
|
+
}
|
|
1123
|
+
void logProxyRequestFinal({
|
|
1124
|
+
timestamp: new Date().toISOString(),
|
|
1125
|
+
requestId: metadata.requestId,
|
|
1126
|
+
method: metadata.method,
|
|
1127
|
+
path: metadata.path,
|
|
1128
|
+
model: metadata.model,
|
|
1129
|
+
stream: metadata.stream,
|
|
1130
|
+
toolCount: metadata.toolCount,
|
|
1131
|
+
account: "",
|
|
1132
|
+
accountType: "proxy-runtime",
|
|
1133
|
+
responseStatus: status,
|
|
1134
|
+
responseTimeMs: Date.now() - metadata.startedAt,
|
|
1135
|
+
errorType,
|
|
1136
|
+
errorMessage: timeout
|
|
1137
|
+
? "Request deadline exceeded"
|
|
1138
|
+
: "Client disconnected before response headers",
|
|
1139
|
+
});
|
|
1140
|
+
logProxyLifecycleEvent({
|
|
1141
|
+
event: "request_terminal",
|
|
1142
|
+
requestId: metadata.requestId,
|
|
1143
|
+
parentRequestId: metadata.parentRequestId,
|
|
1144
|
+
accountingScope: metadata.accountingScope,
|
|
1145
|
+
method: metadata.method,
|
|
1146
|
+
path: metadata.path,
|
|
1147
|
+
model: metadata.model,
|
|
1148
|
+
stream: metadata.stream,
|
|
1149
|
+
sessionHash,
|
|
1150
|
+
elapsedMs: performance.now() - startedMonotonicMs,
|
|
1151
|
+
terminalOutcome: timeout ? "handler_error" : "client_cancelled",
|
|
1152
|
+
transportOutcome: timeout ? "stream_error" : "client_cancelled",
|
|
1153
|
+
finalStatus: status,
|
|
1154
|
+
errorType,
|
|
1155
|
+
telemetryStatus: "complete",
|
|
1156
|
+
outcomeSource: "transport_error",
|
|
1157
|
+
});
|
|
1158
|
+
finishActivity();
|
|
1159
|
+
metadata.shareRelease?.();
|
|
1160
|
+
clearTimeout(deadline);
|
|
1161
|
+
unbindClient();
|
|
1162
|
+
};
|
|
1163
|
+
abortController.signal.addEventListener("abort", onPreheaderAbort, {
|
|
1164
|
+
once: true,
|
|
1165
|
+
});
|
|
1166
|
+
unbindClient = bindProxyClientCancellation(c.req.raw, c.env, abortController);
|
|
1162
1167
|
// The route adapter populates model/stream/toolCount after parsing. Omit
|
|
1163
1168
|
// them at acceptance instead of publishing misleading placeholder values;
|
|
1164
1169
|
// subsequent events carry the parsed metadata under the same request ID.
|
|
1165
1170
|
try {
|
|
1166
1171
|
await httpTrace.run(() => persistProxyLifecycleAcceptance({
|
|
1167
1172
|
requestId: metadata.requestId,
|
|
1173
|
+
parentRequestId: metadata.parentRequestId,
|
|
1174
|
+
accountingScope: metadata.accountingScope,
|
|
1175
|
+
requestTimeoutMs: 900_000,
|
|
1168
1176
|
method: metadata.method,
|
|
1169
1177
|
path: metadata.path,
|
|
1170
1178
|
sessionHash,
|
|
@@ -1172,7 +1180,19 @@ function registerProxyRequestTracking(app, requestMetadata, readiness) {
|
|
|
1172
1180
|
elapsedMs: 0,
|
|
1173
1181
|
monotonicMs: startedMonotonicMs,
|
|
1174
1182
|
}));
|
|
1175
|
-
|
|
1183
|
+
if (abortController.signal.aborted) {
|
|
1184
|
+
c.res = new Response(null, { status: 499 });
|
|
1185
|
+
finish();
|
|
1186
|
+
return;
|
|
1187
|
+
}
|
|
1188
|
+
await httpTrace.run(() => withProxyFinalLogOwnership(metadata.requestId, next));
|
|
1189
|
+
if (cancelledBeforeHeaders) {
|
|
1190
|
+
void c.res.body
|
|
1191
|
+
?.cancel(abortController.signal.reason)
|
|
1192
|
+
.catch(() => undefined);
|
|
1193
|
+
finish();
|
|
1194
|
+
return;
|
|
1195
|
+
}
|
|
1176
1196
|
const responseStatus = c.res.status;
|
|
1177
1197
|
logProxyLifecycleEvent({
|
|
1178
1198
|
event: "response_headers",
|
|
@@ -1199,9 +1219,10 @@ function registerProxyRequestTracking(app, requestMetadata, readiness) {
|
|
|
1199
1219
|
}
|
|
1200
1220
|
};
|
|
1201
1221
|
const notifyRouteTerminal = async (details) => {
|
|
1202
|
-
const results = await withTimeout(Promise.allSettled(routeResponseObservers.map(async (observer) => observer.onTerminal?.(details))),
|
|
1222
|
+
const results = await withTimeout(Promise.allSettled(routeResponseObservers.map(async (observer) => observer.onTerminal?.(details))), 3_000, "Timed out joining proxy response accounting");
|
|
1203
1223
|
return results.some((result) => result.status === "rejected");
|
|
1204
1224
|
};
|
|
1225
|
+
responseTracking = true;
|
|
1205
1226
|
c.res = trackProxyResponse(c.res, finish, {
|
|
1206
1227
|
onFirstChunk: ({ observedBodyBytes, responseChunks }) => {
|
|
1207
1228
|
logProxyLifecycleEvent({
|
|
@@ -1243,6 +1264,34 @@ function registerProxyRequestTracking(app, requestMetadata, readiness) {
|
|
|
1243
1264
|
catch {
|
|
1244
1265
|
accountingTimedOut = true;
|
|
1245
1266
|
}
|
|
1267
|
+
if (!metadata.terminalResult && responseStatus >= 400) {
|
|
1268
|
+
if (metadata.accountingScope !== "internal") {
|
|
1269
|
+
recordProxyRuntimeFinalError(responseStatus, undefined, undefined, {
|
|
1270
|
+
requestId: metadata.requestId,
|
|
1271
|
+
errorType: responseStatus === 404
|
|
1272
|
+
? "route_not_found"
|
|
1273
|
+
: "request_rejected",
|
|
1274
|
+
terminalOutcome: "handler_error",
|
|
1275
|
+
});
|
|
1276
|
+
}
|
|
1277
|
+
await logProxyRequestFinal({
|
|
1278
|
+
timestamp: new Date().toISOString(),
|
|
1279
|
+
requestId: metadata.requestId,
|
|
1280
|
+
method: metadata.method,
|
|
1281
|
+
path: metadata.path,
|
|
1282
|
+
model: metadata.model,
|
|
1283
|
+
stream: metadata.stream,
|
|
1284
|
+
toolCount: metadata.toolCount,
|
|
1285
|
+
account: "",
|
|
1286
|
+
accountType: "proxy-runtime",
|
|
1287
|
+
responseStatus,
|
|
1288
|
+
responseTimeMs: Date.now() - metadata.startedAt,
|
|
1289
|
+
errorType: responseStatus === 404
|
|
1290
|
+
? "route_not_found"
|
|
1291
|
+
: "request_rejected",
|
|
1292
|
+
terminalOutcome: "handler_error",
|
|
1293
|
+
});
|
|
1294
|
+
}
|
|
1246
1295
|
const final = metadata.terminalResult;
|
|
1247
1296
|
const auxiliary = isProxyAuxiliaryRequest(metadata.method, metadata.path);
|
|
1248
1297
|
const terminalOutcome = final?.terminalOutcome ??
|
|
@@ -1297,9 +1346,16 @@ function registerProxyRequestTracking(app, requestMetadata, readiness) {
|
|
|
1297
1346
|
stopObservingFinalLog();
|
|
1298
1347
|
httpTrace.end(final?.responseStatus ?? responseStatus, terminalOutcome, final?.errorType ?? metadata.terminalErrorType);
|
|
1299
1348
|
},
|
|
1300
|
-
});
|
|
1349
|
+
}, abortController.signal);
|
|
1301
1350
|
}
|
|
1302
1351
|
catch (error) {
|
|
1352
|
+
clearTimeout(deadline);
|
|
1353
|
+
unbindClient();
|
|
1354
|
+
abortController.signal.removeEventListener("abort", onPreheaderAbort);
|
|
1355
|
+
if (cancelledBeforeHeaders) {
|
|
1356
|
+
finish();
|
|
1357
|
+
return;
|
|
1358
|
+
}
|
|
1303
1359
|
stopObservingFinalLog();
|
|
1304
1360
|
// Keep metadata available to app.onError, which records the client-facing
|
|
1305
1361
|
// failure with the same request ID before deleting the WeakMap entry.
|
|
@@ -1352,7 +1408,7 @@ export async function createProxyStartApp(params) {
|
|
|
1352
1408
|
const { createOpenAIProxyRoutes } = await import("../../server/routes/openaiProxyRoutes.js");
|
|
1353
1409
|
const { createCodexProxyRoutes } = await import("../../server/routes/codexProxyRoutes.js");
|
|
1354
1410
|
const { createGeminiProxyRoutes } = await import("../../server/routes/geminiProxyRoutes.js");
|
|
1355
|
-
const { logBodyCapture,
|
|
1411
|
+
const { logBodyCapture, getRequestLoggerSnapshot } = await import("../../proxy/requestLogger.js");
|
|
1356
1412
|
const { recordFinalError } = await import("../../proxy/usageStats.js");
|
|
1357
1413
|
const { admitInboundShareRequest, isGrantRequiredByEnv } = await import("../../proxy/shareGate.js");
|
|
1358
1414
|
const { runWithShareContext } = await import("../../proxy/shareContext.js");
|
|
@@ -1378,15 +1434,17 @@ export async function createProxyStartApp(params) {
|
|
|
1378
1434
|
const clientMessage = options?.clientMessage ?? errorMessage;
|
|
1379
1435
|
const clientErrorType = options?.clientErrorType ?? errorType;
|
|
1380
1436
|
const attempt = metadata.lastUpstreamAttempt;
|
|
1381
|
-
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
1437
|
+
if (metadata.accountingScope !== "internal" && !metadata.terminalResult) {
|
|
1438
|
+
recordFinalError(status, attempt?.account ?? PROXY_INTERNAL_ACCOUNT_LABEL, attempt?.accountType ?? PROXY_INTERNAL_ACCOUNT_TYPE, {
|
|
1439
|
+
requestId: metadata.requestId,
|
|
1440
|
+
errorType,
|
|
1441
|
+
errorCode: options?.errorCode,
|
|
1442
|
+
terminalOutcome: "handler_error",
|
|
1443
|
+
message: errorMessage,
|
|
1444
|
+
});
|
|
1445
|
+
}
|
|
1388
1446
|
await Promise.all([
|
|
1389
|
-
|
|
1447
|
+
logProxyRequestFinal({
|
|
1390
1448
|
timestamp: new Date().toISOString(),
|
|
1391
1449
|
requestId: metadata.requestId,
|
|
1392
1450
|
method: metadata.method,
|
|
@@ -1449,12 +1507,20 @@ export async function createProxyStartApp(params) {
|
|
|
1449
1507
|
metadata.terminalErrorType = telemetryUnavailable
|
|
1450
1508
|
? "telemetry_unavailable"
|
|
1451
1509
|
: "unhandled_proxy_error";
|
|
1452
|
-
|
|
1453
|
-
|
|
1454
|
-
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
|
|
1510
|
+
try {
|
|
1511
|
+
await recordRuntimeError(metadata, status, metadata.terminalErrorType, errMsg, {
|
|
1512
|
+
clientMessage: "Proxy internal error",
|
|
1513
|
+
clientErrorType: "api_error",
|
|
1514
|
+
errorCode: metadata.terminalErrorCode,
|
|
1515
|
+
});
|
|
1516
|
+
}
|
|
1517
|
+
finally {
|
|
1518
|
+
// Admission/handler failures may never reach the response tracker. Keep
|
|
1519
|
+
// bridge ownership through final publication, then release this request
|
|
1520
|
+
// only; a child failure must retain its parent's result until delivery.
|
|
1521
|
+
releaseProxyRequestAccounting(metadata.requestId);
|
|
1522
|
+
requestMetadata.delete(c.req.raw);
|
|
1523
|
+
}
|
|
1458
1524
|
return c.json({
|
|
1459
1525
|
type: "error",
|
|
1460
1526
|
error: {
|
|
@@ -1506,7 +1572,7 @@ export async function createProxyStartApp(params) {
|
|
|
1506
1572
|
if (payload.action === "drain_for_supervisor_refresh") {
|
|
1507
1573
|
// A crashed updater must not strand the current rolling worker in
|
|
1508
1574
|
// maintenance mode. A successful launchd refresh replaces this process.
|
|
1509
|
-
scheduleSupervisorRefreshResume(
|
|
1575
|
+
scheduleSupervisorRefreshResume(90 * 1000);
|
|
1510
1576
|
}
|
|
1511
1577
|
}
|
|
1512
1578
|
else if (payload.action === "resume") {
|
|
@@ -1544,7 +1610,10 @@ export async function createProxyStartApp(params) {
|
|
|
1544
1610
|
runtimeConfigProvider,
|
|
1545
1611
|
}
|
|
1546
1612
|
: params.accountAllowlist);
|
|
1547
|
-
const openaiRouteGroup = createOpenAIProxyRoutes(params.modelRouter, "", params.port, runtimeConfigProvider
|
|
1613
|
+
const openaiRouteGroup = createOpenAIProxyRoutes(params.modelRouter, "", params.port, runtimeConfigProvider,
|
|
1614
|
+
// Keep bridge capabilities, accounting and cancellation in this worker
|
|
1615
|
+
// across supervisor generation switches; never re-enter the public listener.
|
|
1616
|
+
(request) => app.fetch(request));
|
|
1548
1617
|
const codexRouteGroup = createCodexProxyRoutes("");
|
|
1549
1618
|
const geminiRouteGroup = createGeminiProxyRoutes(params.modelRouter, "", params.port, runtimeConfigProvider);
|
|
1550
1619
|
const allProxyRoutes = [
|
|
@@ -1631,7 +1700,8 @@ export async function createProxyStartApp(params) {
|
|
|
1631
1700
|
// secret. Running it through the request gate as well would spend the
|
|
1632
1701
|
// grant's rate allowance and open a coin hold for a call that consumes no
|
|
1633
1702
|
// capacity at all.
|
|
1634
|
-
const shareOutcome =
|
|
1703
|
+
const shareOutcome = metadata?.accountingScope === "internal" ||
|
|
1704
|
+
c.req.path.startsWith("/peer/")
|
|
1635
1705
|
? { kind: "local" }
|
|
1636
1706
|
: await admitInboundShareRequest({
|
|
1637
1707
|
headers: Object.fromEntries(c.req.raw.headers.entries()),
|
|
@@ -1665,7 +1735,7 @@ export async function createProxyStartApp(params) {
|
|
|
1665
1735
|
shareOutcome.release();
|
|
1666
1736
|
}
|
|
1667
1737
|
}
|
|
1668
|
-
const requestAbortController = new AbortController();
|
|
1738
|
+
const requestAbortController = metadata?.abortController ?? new AbortController();
|
|
1669
1739
|
const ctx = {
|
|
1670
1740
|
requestId: metadata?.requestId ?? crypto.randomUUID(),
|
|
1671
1741
|
method: c.req.method,
|
|
@@ -1700,9 +1770,9 @@ export async function createProxyStartApp(params) {
|
|
|
1700
1770
|
c.header(key, value);
|
|
1701
1771
|
}
|
|
1702
1772
|
};
|
|
1703
|
-
const result = shareContext
|
|
1704
|
-
?
|
|
1705
|
-
:
|
|
1773
|
+
const result = await raceWithAbort(Promise.resolve(shareContext
|
|
1774
|
+
? runWithShareContext(shareContext, () => route.handler(ctx))
|
|
1775
|
+
: route.handler(ctx)), ctx.abortSignal);
|
|
1706
1776
|
if (result instanceof Response) {
|
|
1707
1777
|
// Streaming responses own their headers; merge in anything the
|
|
1708
1778
|
// handler published on the context that the Response lacks. A Response
|
|
@@ -2161,6 +2231,10 @@ export async function createProxyStartApp(params) {
|
|
|
2161
2231
|
installedVersion: updateState?.installedVersion ??
|
|
2162
2232
|
updateState?.lastUpdateVersion ??
|
|
2163
2233
|
null,
|
|
2234
|
+
validatedVersion: updateState?.installedVersion ?? null,
|
|
2235
|
+
selectedPackageVersion: readProxyPackageSelection(PROXY_PACKAGES_DIR)?.version ?? null,
|
|
2236
|
+
diskPackageVersion: readProxyDiskVersion(),
|
|
2237
|
+
candidateVersion: supervisorState?.rolling?.candidate?.expectedVersion ?? null,
|
|
2164
2238
|
activatedVersion: PROXY_VERSION,
|
|
2165
2239
|
pendingActivationVersion: updateState?.pendingRestartVersion ?? null,
|
|
2166
2240
|
pendingRestartVersion: updateState?.pendingRestartVersion ?? null,
|
|
@@ -2968,16 +3042,23 @@ async function runLaunchdProxySupervisor(argv, spinner) {
|
|
|
2968
3042
|
host,
|
|
2969
3043
|
port,
|
|
2970
3044
|
initialVersion: PROXY_VERSION,
|
|
2971
|
-
spawnWorker: (generation, expectedVersion) =>
|
|
2972
|
-
|
|
2973
|
-
|
|
2974
|
-
|
|
2975
|
-
|
|
2976
|
-
|
|
2977
|
-
|
|
2978
|
-
|
|
2979
|
-
|
|
2980
|
-
|
|
3045
|
+
spawnWorker: (generation, expectedVersion) => {
|
|
3046
|
+
const selected = resolveProxyWorkerPackage({
|
|
3047
|
+
packagesDir: PROXY_PACKAGES_DIR,
|
|
3048
|
+
entryScript,
|
|
3049
|
+
expectedVersion,
|
|
3050
|
+
});
|
|
3051
|
+
return spawnProxySocketWorker({
|
|
3052
|
+
generation,
|
|
3053
|
+
expectedVersion,
|
|
3054
|
+
command: selected.nodePath,
|
|
3055
|
+
args: [selected.entryScript, ...workerArgs],
|
|
3056
|
+
env: {
|
|
3057
|
+
NEUROLINK_PROXY_UPDATE_CONTROL_TOKEN: PROXY_UPDATE_CONTROL_TOKEN,
|
|
3058
|
+
NEUROLINK_PROXY_TRAMPOLINE_EXEC_ONLY: "1",
|
|
3059
|
+
},
|
|
3060
|
+
});
|
|
3061
|
+
},
|
|
2981
3062
|
onStateChange: (snapshot) => {
|
|
2982
3063
|
saveProxySupervisorState({
|
|
2983
3064
|
pid: process.pid,
|
|
@@ -3507,7 +3588,11 @@ export const proxyStatusCommand = {
|
|
|
3507
3588
|
installedVersion: updateState?.installedVersion ??
|
|
3508
3589
|
updateState?.lastUpdateVersion ??
|
|
3509
3590
|
null,
|
|
3510
|
-
|
|
3591
|
+
validatedVersion: updateState?.installedVersion ?? null,
|
|
3592
|
+
selectedPackageVersion: readProxyPackageSelection(PROXY_PACKAGES_DIR)?.version ?? null,
|
|
3593
|
+
diskPackageVersion: readProxyDiskVersion(),
|
|
3594
|
+
candidateVersion: supervisorState?.rolling?.candidate?.expectedVersion ?? null,
|
|
3595
|
+
activatedVersion: supervisorState?.rolling?.active?.version ?? null,
|
|
3511
3596
|
pendingActivationVersion: updateState?.pendingRestartVersion ?? null,
|
|
3512
3597
|
pendingRestartVersion: updateState?.pendingRestartVersion ?? null,
|
|
3513
3598
|
deferredUpdate: updateState?.deferredUpdate ?? null,
|
|
@@ -3521,7 +3606,7 @@ export const proxyStatusCommand = {
|
|
|
3521
3606
|
const servingWorker = workerRunning &&
|
|
3522
3607
|
(supervisorPid
|
|
3523
3608
|
? supervisorRunning &&
|
|
3524
|
-
(!supervisorState ||
|
|
3609
|
+
(!supervisorState?.rolling ||
|
|
3525
3610
|
supervisorState.rolling.active?.pid === state?.pid)
|
|
3526
3611
|
: true);
|
|
3527
3612
|
if ((state || supervisorState) && (servingWorker || supervisorRunning)) {
|
|
@@ -3667,7 +3752,7 @@ export const proxyStatusCommand = {
|
|
|
3667
3752
|
logger.always(` ${chalk.bold("Last update check:")} ${chalk.cyan(status.lastCheckAt)}`);
|
|
3668
3753
|
}
|
|
3669
3754
|
if (status.installedVersion) {
|
|
3670
|
-
logger.always(` ${chalk.bold("
|
|
3755
|
+
logger.always(` ${chalk.bold("Validated:")} ${chalk.cyan(`v${status.installedVersion}`)}`);
|
|
3671
3756
|
}
|
|
3672
3757
|
if (status.activatedVersion) {
|
|
3673
3758
|
logger.always(` ${chalk.bold("Activated:")} ${chalk.cyan(`v${status.activatedVersion}`)}`);
|
|
@@ -3773,6 +3858,18 @@ export const proxyTelemetryCommand = {
|
|
|
3773
3858
|
.option("proxy-url", {
|
|
3774
3859
|
type: "string",
|
|
3775
3860
|
description: "Proxy endpoint used by the read-only doctor",
|
|
3861
|
+
})
|
|
3862
|
+
.option("admission-lookback-minutes", {
|
|
3863
|
+
type: "number",
|
|
3864
|
+
description: "Doctor admission history horizon; must exceed request timeout plus ingestion grace",
|
|
3865
|
+
})
|
|
3866
|
+
.option("request-timeout-ms", {
|
|
3867
|
+
type: "number",
|
|
3868
|
+
description: "Default request deadline for doctor admission reconciliation",
|
|
3869
|
+
})
|
|
3870
|
+
.option("ingestion-grace-ms", {
|
|
3871
|
+
type: "number",
|
|
3872
|
+
description: "Time allowed for telemetry ingestion before reconciliation",
|
|
3776
3873
|
})
|
|
3777
3874
|
.option("quiet", {
|
|
3778
3875
|
type: "boolean",
|
|
@@ -3875,12 +3972,12 @@ export const proxyGuardCommand = {
|
|
|
3875
3972
|
const UPDATE_CHECK_INTERVAL_MS = 2 * 60 * 60 * 1000; // 2 hours
|
|
3876
3973
|
const QUIET_THRESHOLD_MS = 120 * 1000; // 2 minutes of silence
|
|
3877
3974
|
const NATURAL_WINDOW_WAIT_MS = 10 * 60 * 1000; // prefer no admission pause
|
|
3878
|
-
const UPDATE_DRAIN_TIMEOUT_MS = 30 *
|
|
3975
|
+
const UPDATE_DRAIN_TIMEOUT_MS = 30 * 1000; // defer and reopen admission instead of waiting through long streams
|
|
3879
3976
|
const UPDATE_RETRY_DELAY_MS = 5 * 60 * 1000;
|
|
3880
3977
|
const UPDATE_RETRY_MAX_DELAY_MS = 30 * 60 * 1000;
|
|
3881
3978
|
const UPDATE_RETRY_MAX_ATTEMPTS = 4;
|
|
3882
3979
|
const UPDATE_ACTIVITY_POLL_MS = 10 * 1000;
|
|
3883
|
-
const UPDATE_TIMEOUT_MS =
|
|
3980
|
+
const UPDATE_TIMEOUT_MS = 150 * 1000; // include the supervisor candidate readiness deadline
|
|
3884
3981
|
// Get running version from /health endpoint (with timeout to avoid hanging)
|
|
3885
3982
|
let runningVersion = PROXY_VERSION; // fallback
|
|
3886
3983
|
try {
|
|
@@ -3929,15 +4026,33 @@ export const proxyGuardCommand = {
|
|
|
3929
4026
|
logger.always(`[updater] WARNING: failed to ${operation}: ${error instanceof Error ? error.message : String(error)}`);
|
|
3930
4027
|
}
|
|
3931
4028
|
};
|
|
4029
|
+
const isCurrentUpdateOwner = () => {
|
|
4030
|
+
const runtime = rollingSupervisor
|
|
4031
|
+
? loadProxySupervisorState()
|
|
4032
|
+
: loadProxyState();
|
|
4033
|
+
return isProxyUpdateOwnerCurrent({
|
|
4034
|
+
stopping: guardStopping,
|
|
4035
|
+
parentPid,
|
|
4036
|
+
updaterPid: process.pid,
|
|
4037
|
+
parentStatus: getProcessStatus(parentPid),
|
|
4038
|
+
runtimePid: runtime?.pid,
|
|
4039
|
+
runtimeUpdaterPid: runtime?.updaterPid,
|
|
4040
|
+
});
|
|
4041
|
+
};
|
|
3932
4042
|
let updateInProgress = false;
|
|
3933
4043
|
let updateRestartInProgress = false;
|
|
3934
4044
|
const runUpdateCheck = async () => {
|
|
3935
|
-
if (
|
|
4045
|
+
if (updateInProgress || !isCurrentUpdateOwner()) {
|
|
3936
4046
|
return;
|
|
3937
4047
|
}
|
|
3938
4048
|
updateInProgress = true;
|
|
3939
4049
|
let updateVersion = runningVersion;
|
|
3940
4050
|
let drainActive = false;
|
|
4051
|
+
let previousPackage = readProxyPackageSelection(PROXY_PACKAGES_DIR);
|
|
4052
|
+
if (previousPackage?.version !== runningVersion) {
|
|
4053
|
+
previousPackage = readProxyPackageSelection(PROXY_PACKAGES_DIR, true);
|
|
4054
|
+
}
|
|
4055
|
+
let candidatePackage;
|
|
3941
4056
|
try {
|
|
3942
4057
|
// Lazy-load update modules so they're only imported at check time
|
|
3943
4058
|
const { checkForUpdate } = await import("../../proxy/updateChecker.js");
|
|
@@ -3979,65 +4094,10 @@ export const proxyGuardCommand = {
|
|
|
3979
4094
|
return;
|
|
3980
4095
|
}
|
|
3981
4096
|
logger.always(`[updater] update available: ${runningVersion} → ${result.latestVersion}`);
|
|
3982
|
-
if (!
|
|
3983
|
-
|
|
3984
|
-
// restarted. Rolling services keep the current worker serving while
|
|
3985
|
-
// the candidate package is installed and validated.
|
|
3986
|
-
let lastDeferralSignature = "";
|
|
3987
|
-
const waitForWindow = async (quietWaitMs) => {
|
|
3988
|
-
const window = await waitForProxyUpdateWindow({
|
|
3989
|
-
quietThresholdMs: QUIET_THRESHOLD_MS,
|
|
3990
|
-
quietWaitMs,
|
|
3991
|
-
drainWaitMs: UPDATE_DRAIN_TIMEOUT_MS,
|
|
3992
|
-
pollIntervalMs: UPDATE_ACTIVITY_POLL_MS,
|
|
3993
|
-
getActivity: () => getProxyRuntimeActivity(host, port),
|
|
3994
|
-
setDraining: async (draining) => {
|
|
3995
|
-
const changed = await setProxyUpdateDrain(host, port, draining);
|
|
3996
|
-
if (changed) {
|
|
3997
|
-
logger.always(`[updater] ${draining ? "draining new inference requests" : "inference admission resumed"}`);
|
|
3998
|
-
}
|
|
3999
|
-
return changed;
|
|
4000
|
-
},
|
|
4001
|
-
isStopping: () => guardStopping,
|
|
4002
|
-
isParentAlive: () => getProcessStatus(parentPid) !== "not_running",
|
|
4003
|
-
onPhase: (phase, activity) => {
|
|
4004
|
-
const reason = phase === "waiting_for_quiet" && !activity
|
|
4005
|
-
? "activity_unavailable"
|
|
4006
|
-
: phase;
|
|
4007
|
-
const signature = `${reason}:${activity?.activeRequests ?? "unknown"}`;
|
|
4008
|
-
if (signature !== lastDeferralSignature) {
|
|
4009
|
-
lastDeferralSignature = signature;
|
|
4010
|
-
persistUpdaterState("record update deferral", () => recordUpdateDeferred(result.latestVersion, reason, activity?.activeRequests ?? null));
|
|
4011
|
-
}
|
|
4012
|
-
logger.debug(`[updater] ${reason} (${activity?.activeRequests ?? "unknown"} active requests)`);
|
|
4013
|
-
},
|
|
4014
|
-
});
|
|
4015
|
-
drainActive = drainActive || window.draining;
|
|
4016
|
-
return window;
|
|
4017
|
-
};
|
|
4018
|
-
let updateWindow = await waitForWindow(NATURAL_WINDOW_WAIT_MS);
|
|
4019
|
-
if (!updateWindow.ready) {
|
|
4020
|
-
if (updateWindow.reason === "drain_failed") {
|
|
4021
|
-
persistUpdaterState("record unavailable update drain", () => recordUpdateDeferred(result.latestVersion, "drain_unavailable", null));
|
|
4022
|
-
}
|
|
4023
|
-
scheduleUpdateRetry("update window unavailable", result.latestVersion);
|
|
4024
|
-
return;
|
|
4025
|
-
}
|
|
4026
|
-
// Hold admission closed through install and restart after a quiet
|
|
4027
|
-
// observation, closing the race with a newly admitted request.
|
|
4028
|
-
if (!drainActive) {
|
|
4029
|
-
updateWindow = await waitForWindow(0);
|
|
4030
|
-
if (!updateWindow.ready) {
|
|
4031
|
-
scheduleUpdateRetry("update drain unavailable", result.latestVersion);
|
|
4032
|
-
return;
|
|
4033
|
-
}
|
|
4034
|
-
}
|
|
4035
|
-
}
|
|
4036
|
-
else if (!supervisorRefreshOnly) {
|
|
4037
|
-
logger.always(`[updater] rolling supervisor will keep v${runningVersion} serving during installation`);
|
|
4097
|
+
if (rollingSupervisor && !supervisorRefreshOnly) {
|
|
4098
|
+
logger.always(`[updater] keeping v${runningVersion} serving while staging the candidate`);
|
|
4038
4099
|
}
|
|
4039
|
-
//
|
|
4040
|
-
// target while a newer release is already available.
|
|
4100
|
+
// Recheck the target before staging because discovery can be delayed.
|
|
4041
4101
|
let pendingRestart = initiallyPendingRestart;
|
|
4042
4102
|
if (!supervisorRefreshOnly) {
|
|
4043
4103
|
const refreshedResult = await checkForUpdate(runningVersion);
|
|
@@ -4056,7 +4116,7 @@ export const proxyGuardCommand = {
|
|
|
4056
4116
|
}
|
|
4057
4117
|
}
|
|
4058
4118
|
persistUpdaterState("clear update deferral", () => clearUpdateDeferral());
|
|
4059
|
-
logger.always(`[updater] ${supervisorRefreshOnly ? "supervisor reconciliation prepared" : rollingSupervisor ? "rolling activation prepared" : "
|
|
4119
|
+
logger.always(`[updater] ${supervisorRefreshOnly ? "supervisor reconciliation prepared" : rollingSupervisor ? "rolling activation prepared" : "candidate preparation started"} for v${result.latestVersion}`);
|
|
4060
4120
|
// 3. Install update (validate version string before passing to shell)
|
|
4061
4121
|
if (!/^\d+\.\d+\.\d+$/.test(result.latestVersion)) {
|
|
4062
4122
|
const message = `invalid version format: ${result.latestVersion}`;
|
|
@@ -4072,26 +4132,44 @@ export const proxyGuardCommand = {
|
|
|
4072
4132
|
logger.always(`[updater] package-manager candidates: ${installerResolution.tried
|
|
4073
4133
|
.map((candidate) => `${candidate.kind}:${candidate.bin}(${candidate.installable ? `v${candidate.version}` : (candidate.reason ?? "unusable")})`)
|
|
4074
4134
|
.join(", ")}`);
|
|
4075
|
-
|
|
4135
|
+
// Staging never writes a global package root, including when this
|
|
4136
|
+
// updater itself already runs from a private immutable package.
|
|
4137
|
+
const installer = installerResolution.installer ??
|
|
4138
|
+
installerResolution.tried.find((candidate) => candidate.working);
|
|
4076
4139
|
if (!installer) {
|
|
4077
|
-
const message = "no package manager
|
|
4140
|
+
const message = "no working package manager is available for a private staged install";
|
|
4078
4141
|
logger.always(`[updater] WARNING: ${message}; skipping this cycle`);
|
|
4079
4142
|
persistUpdaterState("record update failure", () => recordUpdateFailure(result.latestVersion, "install", message));
|
|
4080
4143
|
return;
|
|
4081
4144
|
}
|
|
4082
4145
|
logger.always(`[updater] installing @juspay/neurolink@${result.latestVersion} via ${installer.kind} ${installer.bin} (v${installer.version})`);
|
|
4083
|
-
if (
|
|
4146
|
+
if (!isCurrentUpdateOwner()) {
|
|
4084
4147
|
return;
|
|
4085
4148
|
}
|
|
4086
4149
|
try {
|
|
4087
|
-
|
|
4088
|
-
|
|
4089
|
-
|
|
4150
|
+
// Retain a validated rollback even when migrating from a mutable
|
|
4151
|
+
// global package whose on-disk version already differs from live.
|
|
4152
|
+
previousPackage = await installStagedProxyPackage({
|
|
4153
|
+
version: runningVersion,
|
|
4154
|
+
packagesDir: PROXY_PACKAGES_DIR,
|
|
4155
|
+
installer,
|
|
4156
|
+
});
|
|
4157
|
+
if (!isCurrentUpdateOwner()) {
|
|
4158
|
+
return;
|
|
4159
|
+
}
|
|
4160
|
+
candidatePackage = await installStagedProxyPackage({
|
|
4161
|
+
version: result.latestVersion,
|
|
4162
|
+
packagesDir: PROXY_PACKAGES_DIR,
|
|
4163
|
+
installer,
|
|
4164
|
+
onProgress: ({ elapsedMs, outputBytes }) => logger.debug(`[updater] staged install progress elapsedMs=${elapsedMs} outputBytes=${outputBytes}`),
|
|
4090
4165
|
});
|
|
4091
4166
|
}
|
|
4092
4167
|
catch (installErr) {
|
|
4168
|
+
if (!isCurrentUpdateOwner()) {
|
|
4169
|
+
return;
|
|
4170
|
+
}
|
|
4093
4171
|
const detail = describeInstallFailure(installErr);
|
|
4094
|
-
logger.always(`[updater] WARNING:
|
|
4172
|
+
logger.always(`[updater] WARNING: staged install failed:\n${detail}`);
|
|
4095
4173
|
persistUpdaterState("record update failure", () => recordUpdateFailure(result.latestVersion, "install", detail));
|
|
4096
4174
|
if (isTransientInstallFailure(installErr)) {
|
|
4097
4175
|
scheduleUpdateRetry("transient install failure", result.latestVersion);
|
|
@@ -4102,36 +4180,121 @@ export const proxyGuardCommand = {
|
|
|
4102
4180
|
else if (!supervisorRefreshOnly) {
|
|
4103
4181
|
logger.always(`[updater] resuming pending restart for already-installed v${result.latestVersion}`);
|
|
4104
4182
|
}
|
|
4183
|
+
// Staging can outlive an operator reinstall or owner replacement.
|
|
4184
|
+
// Never publish or roll back another service's package selection.
|
|
4185
|
+
if (!isCurrentUpdateOwner()) {
|
|
4186
|
+
return;
|
|
4187
|
+
}
|
|
4105
4188
|
// 4. Refresh and validate the stable trampoline. The plist already
|
|
4106
4189
|
// points at this path, so it must not be unloaded or rewritten here.
|
|
4107
4190
|
if (!supervisorRefreshOnly) {
|
|
4108
4191
|
try {
|
|
4109
|
-
|
|
4192
|
+
previousPackage ??= inspectProxyPackage(process.argv[1]);
|
|
4193
|
+
candidatePackage ??=
|
|
4194
|
+
readProxyPackageSelection(PROXY_PACKAGES_DIR) ?? undefined;
|
|
4195
|
+
if (!candidatePackage ||
|
|
4196
|
+
candidatePackage.version !== result.latestVersion) {
|
|
4197
|
+
// A legacy pending marker may refer to an in-place global install.
|
|
4198
|
+
// Never guess that it is safe or point a launcher at unverified files.
|
|
4199
|
+
throw new Error("Pending update has no validated staged package; restaging required");
|
|
4200
|
+
}
|
|
4201
|
+
if (previousPackage?.version === runningVersion &&
|
|
4202
|
+
!pendingRestart) {
|
|
4203
|
+
selectProxyPackage(PROXY_PACKAGES_DIR, previousPackage, isCurrentUpdateOwner);
|
|
4204
|
+
}
|
|
4205
|
+
writeTrampoline(candidatePackage, isCurrentUpdateOwner);
|
|
4110
4206
|
const validation = await validateInstalledVersion({
|
|
4111
4207
|
binPath: TRAMPOLINE_PATH,
|
|
4112
4208
|
expectedVersion: result.latestVersion,
|
|
4113
4209
|
});
|
|
4210
|
+
if (!isCurrentUpdateOwner()) {
|
|
4211
|
+
return;
|
|
4212
|
+
}
|
|
4114
4213
|
if (validation.version !== result.latestVersion) {
|
|
4115
4214
|
const message = `trampoline validation failed after ${validation.attempts} attempts: ${validation.failure ?? "unknown failure"}`;
|
|
4116
4215
|
logger.always(`[updater] WARNING: ${message}; restart deferred`);
|
|
4117
4216
|
persistUpdaterState("record update failure", () => recordUpdateFailure(result.latestVersion, "validation", message));
|
|
4118
4217
|
persistUpdaterState("abandon invalid pending update", () => abandonPendingUpdate(result.latestVersion));
|
|
4218
|
+
if (previousPackage) {
|
|
4219
|
+
writeTrampoline(previousPackage, isCurrentUpdateOwner);
|
|
4220
|
+
}
|
|
4221
|
+
scheduleUpdateRetry("transient candidate validation failure", result.latestVersion);
|
|
4119
4222
|
return;
|
|
4120
4223
|
}
|
|
4121
4224
|
persistUpdaterState("record installed update", () => recordUpdateInstalled(result.latestVersion));
|
|
4122
4225
|
logger.always(`[updater] trampoline validated at v${validation.version} after ${validation.attempts} attempt(s)`);
|
|
4123
4226
|
}
|
|
4124
4227
|
catch (trampolineError) {
|
|
4228
|
+
if (!isCurrentUpdateOwner()) {
|
|
4229
|
+
return;
|
|
4230
|
+
}
|
|
4125
4231
|
const message = trampolineError instanceof Error
|
|
4126
4232
|
? trampolineError.message
|
|
4127
4233
|
: String(trampolineError);
|
|
4128
4234
|
logger.always(`[updater] WARNING: failed to refresh trampoline; refusing restart: ${message}`);
|
|
4129
4235
|
persistUpdaterState("record update failure", () => recordUpdateFailure(result.latestVersion, "validation", message));
|
|
4130
4236
|
persistUpdaterState("abandon invalid pending update", () => abandonPendingUpdate(result.latestVersion));
|
|
4237
|
+
if (previousPackage) {
|
|
4238
|
+
writeTrampoline(previousPackage, isCurrentUpdateOwner);
|
|
4239
|
+
}
|
|
4240
|
+
scheduleUpdateRetry("candidate validation failed; restaging permitted", result.latestVersion);
|
|
4131
4241
|
return;
|
|
4132
4242
|
}
|
|
4133
4243
|
}
|
|
4134
|
-
if (
|
|
4244
|
+
if (!rollingSupervisor) {
|
|
4245
|
+
// Staging and validation finish before closing legacy admission.
|
|
4246
|
+
// A busy service gets a bounded drain attempt and resumes on failure.
|
|
4247
|
+
let lastDeferralSignature = "";
|
|
4248
|
+
const waitForWindow = async (quietWaitMs) => {
|
|
4249
|
+
const window = await waitForProxyUpdateWindow({
|
|
4250
|
+
quietThresholdMs: QUIET_THRESHOLD_MS,
|
|
4251
|
+
quietWaitMs,
|
|
4252
|
+
drainWaitMs: UPDATE_DRAIN_TIMEOUT_MS,
|
|
4253
|
+
pollIntervalMs: UPDATE_ACTIVITY_POLL_MS,
|
|
4254
|
+
getActivity: () => getProxyRuntimeActivity(host, port),
|
|
4255
|
+
setDraining: async (draining) => {
|
|
4256
|
+
const changed = await setProxyUpdateDrain(host, port, draining);
|
|
4257
|
+
if (changed) {
|
|
4258
|
+
logger.always(`[updater] ${draining ? "draining new inference requests" : "inference admission resumed"}`);
|
|
4259
|
+
}
|
|
4260
|
+
return changed;
|
|
4261
|
+
},
|
|
4262
|
+
isStopping: () => guardStopping,
|
|
4263
|
+
isParentAlive: () => getProcessStatus(parentPid) !== "not_running",
|
|
4264
|
+
onPhase: (phase, activity) => {
|
|
4265
|
+
const reason = phase === "waiting_for_quiet" && !activity
|
|
4266
|
+
? "activity_unavailable"
|
|
4267
|
+
: phase;
|
|
4268
|
+
const signature = `${reason}:${activity?.activeRequests ?? "unknown"}`;
|
|
4269
|
+
if (signature !== lastDeferralSignature) {
|
|
4270
|
+
lastDeferralSignature = signature;
|
|
4271
|
+
persistUpdaterState("record update deferral", () => recordUpdateDeferred(result.latestVersion, reason, activity?.activeRequests ?? null));
|
|
4272
|
+
}
|
|
4273
|
+
logger.debug(`[updater] ${reason} (${activity?.activeRequests ?? "unknown"} active requests)`);
|
|
4274
|
+
},
|
|
4275
|
+
});
|
|
4276
|
+
drainActive = drainActive || window.draining;
|
|
4277
|
+
return window;
|
|
4278
|
+
};
|
|
4279
|
+
let updateWindow = await waitForWindow(NATURAL_WINDOW_WAIT_MS);
|
|
4280
|
+
if (!updateWindow.ready) {
|
|
4281
|
+
if (updateWindow.reason === "drain_failed") {
|
|
4282
|
+
persistUpdaterState("record unavailable update drain", () => recordUpdateDeferred(result.latestVersion, "drain_unavailable", null));
|
|
4283
|
+
}
|
|
4284
|
+
scheduleUpdateRetry("update window unavailable", result.latestVersion);
|
|
4285
|
+
return;
|
|
4286
|
+
}
|
|
4287
|
+
// Hold admission closed through restart after a quiet
|
|
4288
|
+
// observation, closing the race with a newly admitted request.
|
|
4289
|
+
if (!drainActive) {
|
|
4290
|
+
updateWindow = await waitForWindow(0);
|
|
4291
|
+
if (!updateWindow.ready) {
|
|
4292
|
+
scheduleUpdateRetry("update drain unavailable", result.latestVersion);
|
|
4293
|
+
return;
|
|
4294
|
+
}
|
|
4295
|
+
}
|
|
4296
|
+
}
|
|
4297
|
+
if (!isCurrentUpdateOwner()) {
|
|
4135
4298
|
return;
|
|
4136
4299
|
}
|
|
4137
4300
|
updateRestartInProgress = true;
|
|
@@ -4143,6 +4306,10 @@ export const proxyGuardCommand = {
|
|
|
4143
4306
|
else {
|
|
4144
4307
|
logger.always(`[updater] requesting rolling activation of v${result.latestVersion}`);
|
|
4145
4308
|
try {
|
|
4309
|
+
if (!isCurrentUpdateOwner()) {
|
|
4310
|
+
updateRestartInProgress = false;
|
|
4311
|
+
return;
|
|
4312
|
+
}
|
|
4146
4313
|
process.kill(parentPid, "SIGUSR2");
|
|
4147
4314
|
}
|
|
4148
4315
|
catch (activationError) {
|
|
@@ -4254,6 +4421,10 @@ export const proxyGuardCommand = {
|
|
|
4254
4421
|
scheduleUpdateRetry("supervisor refresh window unavailable", result.latestVersion);
|
|
4255
4422
|
return;
|
|
4256
4423
|
}
|
|
4424
|
+
if (!isCurrentUpdateOwner()) {
|
|
4425
|
+
updateRestartInProgress = false;
|
|
4426
|
+
return;
|
|
4427
|
+
}
|
|
4257
4428
|
logger.always(`[updater] refreshing launchd supervisor at v${result.latestVersion}`);
|
|
4258
4429
|
const uid = process.getuid?.() ?? 501;
|
|
4259
4430
|
try {
|
|
@@ -4312,30 +4483,34 @@ export const proxyGuardCommand = {
|
|
|
4312
4483
|
? `rolling candidate failed: ${rollingFailure}`
|
|
4313
4484
|
: `proxy did not report v${result.latestVersion} healthy within ${UPDATE_TIMEOUT_MS}ms`;
|
|
4314
4485
|
if (rollingSupervisor && !supervisorRefreshOnly) {
|
|
4486
|
+
if (!isCurrentUpdateOwner()) {
|
|
4487
|
+
updateRestartInProgress = false;
|
|
4488
|
+
return;
|
|
4489
|
+
}
|
|
4315
4490
|
try {
|
|
4316
|
-
|
|
4317
|
-
|
|
4318
|
-
|
|
4319
|
-
const rollbackInstaller = rollbackResolution.installer;
|
|
4320
|
-
if (!rollbackInstaller) {
|
|
4321
|
-
throw new Error("no usable package manager is available for rollback");
|
|
4491
|
+
if (!previousPackage ||
|
|
4492
|
+
previousPackage.version !== runningVersion) {
|
|
4493
|
+
throw new Error("Previous validated package selection is unavailable");
|
|
4322
4494
|
}
|
|
4323
|
-
logger.always(`[updater]
|
|
4324
|
-
|
|
4325
|
-
writeTrampoline();
|
|
4495
|
+
logger.always(`[updater] restoring immutable package selection v${runningVersion}`);
|
|
4496
|
+
writeTrampoline(previousPackage, isCurrentUpdateOwner);
|
|
4326
4497
|
const rollbackValidation = await validateInstalledVersion({
|
|
4327
4498
|
binPath: TRAMPOLINE_PATH,
|
|
4328
4499
|
expectedVersion: runningVersion,
|
|
4329
4500
|
});
|
|
4330
4501
|
if (rollbackValidation.version !== runningVersion) {
|
|
4331
|
-
throw new Error(`
|
|
4502
|
+
throw new Error(`Rollback selection did not validate as v${runningVersion}`);
|
|
4332
4503
|
}
|
|
4333
|
-
//
|
|
4504
|
+
// Restoring the old package selection is not enough: the supervisor was
|
|
4334
4505
|
// told to activate result.latestVersion, so its recovery loop
|
|
4335
4506
|
// keeps demanding the new version against the now-downgraded
|
|
4336
4507
|
// installation and can never serve a worker again. Re-point the
|
|
4337
4508
|
// supervisor at runningVersion and report success only once it
|
|
4338
4509
|
// actually serves that version again.
|
|
4510
|
+
if (!isCurrentUpdateOwner()) {
|
|
4511
|
+
updateRestartInProgress = false;
|
|
4512
|
+
return;
|
|
4513
|
+
}
|
|
4339
4514
|
const rollbackActivated = await activateRollbackVersion(host, port, parentPid, runningVersion, UPDATE_TIMEOUT_MS);
|
|
4340
4515
|
if (!rollbackActivated) {
|
|
4341
4516
|
throw new Error(`supervisor did not report v${runningVersion} healthy within ${UPDATE_TIMEOUT_MS}ms after rollback`);
|
|
@@ -4367,7 +4542,7 @@ export const proxyGuardCommand = {
|
|
|
4367
4542
|
persistUpdaterState("record update failure", () => recordUpdateFailure(updateVersion, "check", message));
|
|
4368
4543
|
}
|
|
4369
4544
|
finally {
|
|
4370
|
-
if (drainActive && !updateRestartInProgress) {
|
|
4545
|
+
if (drainActive && !updateRestartInProgress && isCurrentUpdateOwner()) {
|
|
4371
4546
|
const resumed = await resumeProxyUpdateDrain(host, port);
|
|
4372
4547
|
if (!resumed && getProcessStatus(parentPid) !== "not_running") {
|
|
4373
4548
|
logger.always(`[updater] WARNING: failed to resume inference admission after deferred update`);
|
|
@@ -4651,10 +4826,8 @@ function buildLaunchdPath() {
|
|
|
4651
4826
|
return [...segments].join(":");
|
|
4652
4827
|
}
|
|
4653
4828
|
export function buildProxyLaunchdPlist(port, host, envFile, configFile) {
|
|
4654
|
-
// The
|
|
4655
|
-
//
|
|
4656
|
-
// `neurolink` binary via PATH on every launch. This way, launchd
|
|
4657
|
-
// is never pinned to a version-specific pnpm store path.
|
|
4829
|
+
// The stable launcher selects an already validated package without
|
|
4830
|
+
// rewriting the launchd service or its inherited runtime environment.
|
|
4658
4831
|
const trampolinePath = escapeXml(TRAMPOLINE_PATH);
|
|
4659
4832
|
const envFileArgs = envFile
|
|
4660
4833
|
? `
|
|
@@ -4666,20 +4839,11 @@ export function buildProxyLaunchdPlist(port, host, envFile, configFile) {
|
|
|
4666
4839
|
<string>--config</string>
|
|
4667
4840
|
<string>${escapeXml(configFile)}</string>`
|
|
4668
4841
|
: "";
|
|
4669
|
-
const otelEnvironment =
|
|
4670
|
-
|
|
4671
|
-
|
|
4672
|
-
|
|
4673
|
-
|
|
4674
|
-
"OTEL_EXPORTER_OTLP_HEADERS",
|
|
4675
|
-
"OTEL_EXPORTER_OTLP_LOGS_HEADERS",
|
|
4676
|
-
"OTEL_SERVICE_NAME",
|
|
4677
|
-
"NEUROLINK_PROXY_SESSION_SECRET",
|
|
4678
|
-
]
|
|
4679
|
-
.filter((name) => process.env[name] !== undefined)
|
|
4680
|
-
.map((name) => ` <key>${name}</key>\n <string>${escapeXml(process.env[name])}</string>`)
|
|
4681
|
-
.join("\n")
|
|
4682
|
-
: "";
|
|
4842
|
+
const otelEnvironment = Object.keys(process.env)
|
|
4843
|
+
.filter((name) => isProxyServiceEnvironmentKey(name) && process.env[name] !== undefined)
|
|
4844
|
+
.sort()
|
|
4845
|
+
.map((name) => ` <key>${escapeXml(name)}</key>\n <string>${escapeXml(process.env[name])}</string>`)
|
|
4846
|
+
.join("\n");
|
|
4683
4847
|
return `<?xml version="1.0" encoding="UTF-8"?>
|
|
4684
4848
|
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN"
|
|
4685
4849
|
"http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
|
@@ -4742,12 +4906,10 @@ export const proxyInstallCommand = {
|
|
|
4742
4906
|
.option("port", {
|
|
4743
4907
|
type: "number",
|
|
4744
4908
|
alias: "p",
|
|
4745
|
-
default: 55669,
|
|
4746
4909
|
description: "Proxy port",
|
|
4747
4910
|
})
|
|
4748
4911
|
.option("host", {
|
|
4749
4912
|
type: "string",
|
|
4750
|
-
default: "127.0.0.1",
|
|
4751
4913
|
description: "Proxy host",
|
|
4752
4914
|
})
|
|
4753
4915
|
.option("env-file", {
|
|
@@ -4763,19 +4925,56 @@ export const proxyInstallCommand = {
|
|
|
4763
4925
|
.example("neurolink proxy install -p 9000", "Install on custom port");
|
|
4764
4926
|
},
|
|
4765
4927
|
handler: async (argv) => {
|
|
4766
|
-
const port = argv.port ?? 55669;
|
|
4767
|
-
const host = argv.host ?? "127.0.0.1";
|
|
4768
4928
|
if (process.platform !== "darwin") {
|
|
4769
4929
|
console.info(chalk.red("proxy install is currently macOS-only (uses launchd)."));
|
|
4770
4930
|
console.info(chalk.yellow("On Linux, use systemd. On Windows, use Task Scheduler."));
|
|
4771
4931
|
process.exit(1);
|
|
4772
4932
|
}
|
|
4933
|
+
const existingSupervisor = loadProxySupervisorState();
|
|
4934
|
+
const existingWorker = loadProxyState();
|
|
4935
|
+
assertProxyServiceInstallIdle({
|
|
4936
|
+
pids: [
|
|
4937
|
+
existingSupervisor?.pid,
|
|
4938
|
+
existingWorker?.pid,
|
|
4939
|
+
existingSupervisor?.rolling?.active?.pid,
|
|
4940
|
+
existingSupervisor?.rolling?.candidate?.pid,
|
|
4941
|
+
...(existingSupervisor?.rolling?.draining?.map((worker) => worker.pid) ?? []),
|
|
4942
|
+
].filter((pid) => typeof pid === "number"),
|
|
4943
|
+
getProcessStatus,
|
|
4944
|
+
launchdPresence: probeProxyLaunchdPresence({
|
|
4945
|
+
label: PLIST_LABEL,
|
|
4946
|
+
uid: process.getuid?.() ?? 501,
|
|
4947
|
+
}),
|
|
4948
|
+
});
|
|
4773
4949
|
const { mkdirSync, existsSync, chmodSync } = await import("fs");
|
|
4950
|
+
let savedSettings = parseProxyServiceInstallSettings(null);
|
|
4951
|
+
if (existsSync(PLIST_PATH)) {
|
|
4952
|
+
// Existing runtime values are defaults, never printed. Refuse to rewrite
|
|
4953
|
+
// an unreadable service rather than silently dropping its OTel settings.
|
|
4954
|
+
const { execFileSync } = await import("node:child_process");
|
|
4955
|
+
try {
|
|
4956
|
+
savedSettings = parseProxyServiceInstallSettings(JSON.parse(execFileSync("plutil", ["-convert", "json", "-o", "-", PLIST_PATH], {
|
|
4957
|
+
encoding: "utf8",
|
|
4958
|
+
timeout: 5_000,
|
|
4959
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
4960
|
+
})));
|
|
4961
|
+
}
|
|
4962
|
+
catch {
|
|
4963
|
+
// Neither captured plutil output nor JSON parse excerpts may expose
|
|
4964
|
+
// stored exporter authorization headers in a reinstall error.
|
|
4965
|
+
throw new Error("Existing proxy service settings could not be read; refusing reinstall before service changes.");
|
|
4966
|
+
}
|
|
4967
|
+
for (const [name, value] of Object.entries(savedSettings.environment)) {
|
|
4968
|
+
process.env[name] ??= value;
|
|
4969
|
+
}
|
|
4970
|
+
}
|
|
4971
|
+
const port = argv.port ?? savedSettings.port ?? 55669;
|
|
4972
|
+
const host = argv.host ?? savedSettings.host ?? "127.0.0.1";
|
|
4774
4973
|
const envResolution = resolveProxyEnvFile({
|
|
4775
|
-
explicitEnvFile: argv.envFile,
|
|
4974
|
+
explicitEnvFile: argv.envFile ?? savedSettings.envFile,
|
|
4776
4975
|
});
|
|
4777
4976
|
const envFile = envResolution.path;
|
|
4778
|
-
const explicitConfig = argv.config;
|
|
4977
|
+
const explicitConfig = argv.config ?? savedSettings.configFile;
|
|
4779
4978
|
const configPath = resolveProxyConfigPath(explicitConfig);
|
|
4780
4979
|
if (explicitConfig && !existsSync(configPath)) {
|
|
4781
4980
|
console.info(chalk.red(`Proxy config file not found: ${configPath}`));
|
|
@@ -4796,20 +4995,17 @@ export const proxyInstallCommand = {
|
|
|
4796
4995
|
}
|
|
4797
4996
|
writeTrampoline();
|
|
4798
4997
|
console.info(chalk.green(`✓ Trampoline written to ${TRAMPOLINE_PATH}`));
|
|
4799
|
-
//
|
|
4800
|
-
//
|
|
4801
|
-
// PATH-based candidate is broken AND the baked-in path is unreachable.
|
|
4998
|
+
// Validate selected package metadata without initializing the CLI graph.
|
|
4999
|
+
// Worker startup separately proves readiness before accepting traffic.
|
|
4802
5000
|
const trampolineVersion = probeBinVersion(TRAMPOLINE_PATH);
|
|
4803
5001
|
if (!trampolineVersion) {
|
|
4804
5002
|
console.info(chalk.red(`✗ Trampoline validation failed: ${TRAMPOLINE_PATH} --version did not run cleanly.`));
|
|
4805
5003
|
console.info(chalk.yellow(` The launchd service would not be able to start neurolink. Fix your install first.`));
|
|
4806
|
-
console.info(chalk.yellow(`
|
|
5004
|
+
console.info(chalk.yellow(` Inspect the selected package and reinstall the missing version before retrying.`));
|
|
4807
5005
|
process.exit(1);
|
|
4808
5006
|
}
|
|
4809
5007
|
if (trampolineVersion !== PROXY_VERSION) {
|
|
4810
|
-
console.info(chalk.
|
|
4811
|
-
console.info(chalk.yellow(` PATH may shadow this installation with an older version. Fix your PATH or set NEUROLINK_BIN.`));
|
|
4812
|
-
process.exit(1);
|
|
5008
|
+
console.info(chalk.yellow(`Retaining selected proxy package v${trampolineVersion}; this service installer is v${PROXY_VERSION}.`));
|
|
4813
5009
|
}
|
|
4814
5010
|
console.info(chalk.green(`✓ Trampoline validated (resolves to neurolink v${trampolineVersion})`));
|
|
4815
5011
|
const plist = buildProxyLaunchdPlist(port, host, envFile, configFile);
|