@juspay/neurolink 9.93.1 → 9.94.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/dist/browser/neurolink.min.js +306 -306
- package/dist/cli/commands/proxy.js +687 -200
- package/dist/cli/utils/serverUtils.js +4 -1
- package/dist/lib/proxy/globalInstaller.js +1 -1
- package/dist/lib/proxy/openaiFormat.js +1 -0
- package/dist/lib/proxy/rollingProxyServer.d.ts +3 -0
- package/dist/lib/proxy/rollingProxyServer.js +149 -0
- package/dist/lib/proxy/rollingWorkerProcess.d.ts +2 -0
- package/dist/lib/proxy/rollingWorkerProcess.js +194 -0
- package/dist/lib/proxy/rollingWorkerProtocol.d.ts +5 -0
- package/dist/lib/proxy/rollingWorkerProtocol.js +43 -0
- package/dist/lib/proxy/rollingWorkerSupervisor.d.ts +39 -0
- package/dist/lib/proxy/rollingWorkerSupervisor.js +402 -0
- package/dist/lib/proxy/socketWorkerRuntime.d.ts +14 -0
- package/dist/lib/proxy/socketWorkerRuntime.js +294 -0
- package/dist/lib/proxy/sseInterceptor.js +1 -1
- package/dist/lib/proxy/workerLog.d.ts +6 -0
- package/dist/lib/proxy/workerLog.js +42 -0
- package/dist/lib/server/routes/openaiProxyRoutes.d.ts +9 -0
- package/dist/lib/server/routes/openaiProxyRoutes.js +16 -1
- package/dist/lib/types/cli.d.ts +38 -0
- package/dist/lib/types/proxy.d.ts +156 -0
- package/dist/lib/utils/errorHandling.d.ts +8 -0
- package/dist/lib/utils/errorHandling.js +20 -0
- package/dist/proxy/globalInstaller.js +1 -1
- package/dist/proxy/openaiFormat.js +1 -0
- package/dist/proxy/rollingProxyServer.d.ts +3 -0
- package/dist/proxy/rollingProxyServer.js +148 -0
- package/dist/proxy/rollingWorkerProcess.d.ts +2 -0
- package/dist/proxy/rollingWorkerProcess.js +193 -0
- package/dist/proxy/rollingWorkerProtocol.d.ts +5 -0
- package/dist/proxy/rollingWorkerProtocol.js +42 -0
- package/dist/proxy/rollingWorkerSupervisor.d.ts +39 -0
- package/dist/proxy/rollingWorkerSupervisor.js +401 -0
- package/dist/proxy/socketWorkerRuntime.d.ts +14 -0
- package/dist/proxy/socketWorkerRuntime.js +293 -0
- package/dist/proxy/sseInterceptor.js +1 -1
- package/dist/proxy/workerLog.d.ts +6 -0
- package/dist/proxy/workerLog.js +41 -0
- package/dist/server/routes/openaiProxyRoutes.d.ts +9 -0
- package/dist/server/routes/openaiProxyRoutes.js +16 -1
- package/dist/types/cli.d.ts +38 -0
- package/dist/types/proxy.d.ts +156 -0
- package/dist/utils/errorHandling.d.ts +8 -0
- package/dist/utils/errorHandling.js +20 -0
- package/package.json +3 -2
|
@@ -16,6 +16,7 @@ import chalk from "chalk";
|
|
|
16
16
|
import ora from "ora";
|
|
17
17
|
import { buildProxyHealthResponse, createProxyReadinessState, markProxyDrainingForUpdate, markProxyReady, resumeProxyConnections, waitForProxyReadiness, } from "../../lib/proxy/proxyHealth.js";
|
|
18
18
|
import { logger } from "../../lib/utils/logger.js";
|
|
19
|
+
import { sanitizeForLog } from "../../lib/utils/logSanitize.js";
|
|
19
20
|
import { withTimeout } from "../../lib/utils/async/withTimeout.js";
|
|
20
21
|
import { formatUptime, isProcessRunning, StateFileManager, } from "../utils/serverUtils.js";
|
|
21
22
|
import { configureProxyKeepAliveDispatcher } from "../../lib/proxy/proxyDispatcher.js";
|
|
@@ -25,6 +26,11 @@ import { beginProxyRequest, getProxyActivitySnapshot, trackProxyResponse, } from
|
|
|
25
26
|
import { flushProxyLifecycleEvents, getProxyLifecycleLoggerSnapshot, hashProxyLifecycleSessionId, logProxyLifecycleEvent, } from "../../lib/proxy/proxyLifecycle.js";
|
|
26
27
|
import { describeInstallFailure, getGlobalInstallArgs, resolveGlobalInstaller, validateInstalledVersion, } from "../../lib/proxy/globalInstaller.js";
|
|
27
28
|
import { startUpdaterWorkerSupervisor } from "../../lib/proxy/updaterSupervisor.js";
|
|
29
|
+
import { openProxyWorkerLog } from "../../lib/proxy/workerLog.js";
|
|
30
|
+
import { startRollingProxyServer } from "../../lib/proxy/rollingProxyServer.js";
|
|
31
|
+
import { spawnProxySocketWorker } from "../../lib/proxy/rollingWorkerProcess.js";
|
|
32
|
+
import { PROXY_ROLLING_SUPERVISOR_ENV, PROXY_SOCKET_WORKER_ENV, } from "../../lib/proxy/rollingWorkerProtocol.js";
|
|
33
|
+
import { attachSocketWorkerProcess } from "../../lib/proxy/socketWorkerRuntime.js";
|
|
28
34
|
import { waitForProxyUpdateWindow } from "../../lib/proxy/updateCoordinator.js";
|
|
29
35
|
import { abandonPendingUpdate, clearUpdateDeferral, isVersionSuppressed, loadUpdateState, recordCheck, recordSuccessfulUpdate, recordUpdateDeferred, recordUpdateFailure, recordUpdateInstalled, suppressVersion, } from "../../lib/proxy/updateState.js";
|
|
30
36
|
import { loadProxyEnvFile, resolveProxyEnvFile, } from "../../lib/proxy/proxyEnv.js";
|
|
@@ -41,12 +47,14 @@ const PROXY_UPDATE_CONTROL_TOKEN = process.env.NEUROLINK_PROXY_UPDATE_CONTROL_TO
|
|
|
41
47
|
// STATE MANAGEMENT
|
|
42
48
|
// =============================================================================
|
|
43
49
|
let proxyStateManager = new StateFileManager("proxy-state.json");
|
|
50
|
+
let proxySupervisorStateManager = new StateFileManager("proxy-supervisor-state.json");
|
|
44
51
|
/**
|
|
45
52
|
* Reinitialise the state manager with a custom base directory.
|
|
46
53
|
* Called when --dev redirects writable paths to .neurolink-dev/.
|
|
47
54
|
*/
|
|
48
55
|
function setProxyStateDir(baseDir) {
|
|
49
56
|
proxyStateManager = new StateFileManager("proxy-state.json", baseDir);
|
|
57
|
+
proxySupervisorStateManager = new StateFileManager("proxy-supervisor-state.json", baseDir);
|
|
50
58
|
}
|
|
51
59
|
function saveProxyState(state) {
|
|
52
60
|
proxyStateManager.save(state);
|
|
@@ -57,6 +65,15 @@ function loadProxyState() {
|
|
|
57
65
|
function clearProxyState() {
|
|
58
66
|
proxyStateManager.clear();
|
|
59
67
|
}
|
|
68
|
+
function saveProxySupervisorState(state) {
|
|
69
|
+
proxySupervisorStateManager.save(state);
|
|
70
|
+
}
|
|
71
|
+
function loadProxySupervisorState() {
|
|
72
|
+
return proxySupervisorStateManager.load();
|
|
73
|
+
}
|
|
74
|
+
function clearProxySupervisorState() {
|
|
75
|
+
proxySupervisorStateManager.clear();
|
|
76
|
+
}
|
|
60
77
|
const CLAUDE_SETTINGS_PATH = join(homedir(), ".claude", "settings.json");
|
|
61
78
|
const PLIST_LABEL = "com.neurolink.proxy";
|
|
62
79
|
const PLIST_DIR = join(homedir(), "Library", "LaunchAgents");
|
|
@@ -80,6 +97,67 @@ function getProcessStatus(pid) {
|
|
|
80
97
|
return "not_running";
|
|
81
98
|
}
|
|
82
99
|
}
|
|
100
|
+
/**
|
|
101
|
+
* Best-effort check that a pid actually belongs to a neurolink proxy process,
|
|
102
|
+
* so a stale/recycled supervisor pid is never mistaken for a live supervisor
|
|
103
|
+
* and signalled. Returns false when the process cannot be confirmed as ours.
|
|
104
|
+
*/
|
|
105
|
+
async function processLooksLikeProxySupervisor(pid) {
|
|
106
|
+
try {
|
|
107
|
+
const { execFileSync } = await import("node:child_process");
|
|
108
|
+
const args = execFileSync("ps", ["-p", String(pid), "-o", "args="], {
|
|
109
|
+
encoding: "utf-8",
|
|
110
|
+
stdio: ["ignore", "pipe", "ignore"],
|
|
111
|
+
});
|
|
112
|
+
return /neurolink/i.test(args);
|
|
113
|
+
}
|
|
114
|
+
catch {
|
|
115
|
+
// `ps` unavailable or the pid vanished — do not signal an unverified pid.
|
|
116
|
+
return false;
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
/**
|
|
120
|
+
* Ensure any supervisor recorded in state is actually stopped before its state
|
|
121
|
+
* record is cleared. A running supervisor still owns the listening socket;
|
|
122
|
+
* deleting its record would orphan it with no way to find it again. Returns
|
|
123
|
+
* false when a running supervisor could not be stopped, so callers can abort
|
|
124
|
+
* the uninstall instead of silently discarding the record.
|
|
125
|
+
*/
|
|
126
|
+
async function ensureSupervisorStoppedBeforeClear() {
|
|
127
|
+
const pid = loadProxySupervisorState()?.pid;
|
|
128
|
+
if (!pid || getProcessStatus(pid) === "not_running") {
|
|
129
|
+
return true;
|
|
130
|
+
}
|
|
131
|
+
// Guard against a stale/recycled pid: only signal a process that still looks
|
|
132
|
+
// like a neurolink proxy supervisor, so uninstall never terminates an
|
|
133
|
+
// unrelated, same-user process that inherited the recorded pid.
|
|
134
|
+
if (!(await processLooksLikeProxySupervisor(pid))) {
|
|
135
|
+
return true;
|
|
136
|
+
}
|
|
137
|
+
try {
|
|
138
|
+
process.kill(pid, "SIGTERM");
|
|
139
|
+
}
|
|
140
|
+
catch (error) {
|
|
141
|
+
if (error.code === "ESRCH") {
|
|
142
|
+
return true;
|
|
143
|
+
}
|
|
144
|
+
return false;
|
|
145
|
+
}
|
|
146
|
+
for (let attempt = 0; attempt < 25; attempt += 1) {
|
|
147
|
+
if (getProcessStatus(pid) === "not_running") {
|
|
148
|
+
return true;
|
|
149
|
+
}
|
|
150
|
+
await sleep(200);
|
|
151
|
+
}
|
|
152
|
+
try {
|
|
153
|
+
process.kill(pid, "SIGKILL");
|
|
154
|
+
}
|
|
155
|
+
catch {
|
|
156
|
+
/* already gone or not permitted */
|
|
157
|
+
}
|
|
158
|
+
await sleep(200);
|
|
159
|
+
return getProcessStatus(pid) === "not_running";
|
|
160
|
+
}
|
|
83
161
|
/** Resolve the primary-account info shown in /status. Reads the operator's
|
|
84
162
|
* configured email from proxy config and cross-checks it against the token
|
|
85
163
|
* store; falls back to the first enabled anthropic account when not set or
|
|
@@ -147,6 +225,16 @@ async function isLaunchdManaging() {
|
|
|
147
225
|
function isLaunchdManagedProcess() {
|
|
148
226
|
return process.platform === "darwin" && process.ppid === 1;
|
|
149
227
|
}
|
|
228
|
+
function isProxySocketWorkerProcess() {
|
|
229
|
+
return process.env[PROXY_SOCKET_WORKER_ENV] === "1";
|
|
230
|
+
}
|
|
231
|
+
function getProxyWorkerGeneration() {
|
|
232
|
+
const generation = Number(process.env.NEUROLINK_PROXY_WORKER_GENERATION);
|
|
233
|
+
if (!Number.isSafeInteger(generation) || generation <= 0) {
|
|
234
|
+
throw new Error("proxy socket worker generation is missing or invalid");
|
|
235
|
+
}
|
|
236
|
+
return generation;
|
|
237
|
+
}
|
|
150
238
|
function isProxyAutoUpdateEnabled(value = process.env.NEUROLINK_PROXY_AUTO_UPDATE) {
|
|
151
239
|
return !["0", "off", "false"].includes((value ?? "").trim().toLowerCase());
|
|
152
240
|
}
|
|
@@ -373,6 +461,75 @@ async function getProxyRuntimeActivity(host, port, timeoutMs = 3_000) {
|
|
|
373
461
|
return null;
|
|
374
462
|
}
|
|
375
463
|
}
|
|
464
|
+
async function getRollingActivationFailure(host, port, expectedVersion) {
|
|
465
|
+
try {
|
|
466
|
+
const response = await fetch(`http://${host}:${port}/status`, {
|
|
467
|
+
signal: AbortSignal.timeout(3_000),
|
|
468
|
+
});
|
|
469
|
+
if (!response.ok) {
|
|
470
|
+
return null;
|
|
471
|
+
}
|
|
472
|
+
const payload = (await response.json());
|
|
473
|
+
const failure = payload.autoUpdate?.rolling?.lastFailure;
|
|
474
|
+
if (failure?.version !== expectedVersion ||
|
|
475
|
+
typeof failure.phase !== "string" ||
|
|
476
|
+
typeof failure.message !== "string") {
|
|
477
|
+
return null;
|
|
478
|
+
}
|
|
479
|
+
return `${failure.phase}: ${failure.message}`;
|
|
480
|
+
}
|
|
481
|
+
catch {
|
|
482
|
+
return null;
|
|
483
|
+
}
|
|
484
|
+
}
|
|
485
|
+
/** Read the version the proxy currently reports healthy, or undefined. */
|
|
486
|
+
async function fetchProxyHealthVersion(host, port) {
|
|
487
|
+
try {
|
|
488
|
+
const response = await fetch(`http://${host}:${port}/health`, {
|
|
489
|
+
signal: AbortSignal.timeout(3000),
|
|
490
|
+
});
|
|
491
|
+
if (!response.ok) {
|
|
492
|
+
return undefined;
|
|
493
|
+
}
|
|
494
|
+
const data = (await response.json());
|
|
495
|
+
return data.version;
|
|
496
|
+
}
|
|
497
|
+
catch {
|
|
498
|
+
return undefined;
|
|
499
|
+
}
|
|
500
|
+
}
|
|
501
|
+
/**
|
|
502
|
+
* After a failed rolling update, re-point the live supervisor at the reinstalled
|
|
503
|
+
* running version (mirroring the forward path: record the pending target, then
|
|
504
|
+
* signal SIGUSR2) and wait until the supervisor actually serves that version
|
|
505
|
+
* again. Returns false if it never reports the running version healthy in time.
|
|
506
|
+
*/
|
|
507
|
+
async function activateRollbackVersion(host, port, parentPid, runningVersion, timeoutMs) {
|
|
508
|
+
recordUpdateInstalled(runningVersion);
|
|
509
|
+
try {
|
|
510
|
+
const start = Date.now();
|
|
511
|
+
while (Date.now() - start < timeoutMs) {
|
|
512
|
+
try {
|
|
513
|
+
process.kill(parentPid, "SIGUSR2");
|
|
514
|
+
}
|
|
515
|
+
catch {
|
|
516
|
+
// Supervisor may be mid-restart; its recovery loop also re-targets
|
|
517
|
+
// runningVersion once no worker is active.
|
|
518
|
+
}
|
|
519
|
+
await new Promise((resolve) => setTimeout(resolve, 2000));
|
|
520
|
+
if ((await fetchProxyHealthVersion(host, port)) === runningVersion) {
|
|
521
|
+
return true;
|
|
522
|
+
}
|
|
523
|
+
if (getProcessStatus(parentPid) === "not_running") {
|
|
524
|
+
return false;
|
|
525
|
+
}
|
|
526
|
+
}
|
|
527
|
+
return false;
|
|
528
|
+
}
|
|
529
|
+
finally {
|
|
530
|
+
abandonPendingUpdate(runningVersion);
|
|
531
|
+
}
|
|
532
|
+
}
|
|
376
533
|
async function setProxyUpdateDrain(host, port, draining) {
|
|
377
534
|
const confirmState = async () => {
|
|
378
535
|
try {
|
|
@@ -474,6 +631,31 @@ function writeTrampoline() {
|
|
|
474
631
|
# Resolves a working neurolink binary on every launchd invocation so the
|
|
475
632
|
# plist never gets pinned to a broken/stale shim.
|
|
476
633
|
|
|
634
|
+
BAKED_NODE=${shEscape(bakedNode)}
|
|
635
|
+
BAKED_SCRIPT=${shEscape(bakedScript)}
|
|
636
|
+
|
|
637
|
+
# IPC workers must preserve the inherited Node channel until the final exec.
|
|
638
|
+
# Do not launch a --version probe subprocess on that descriptor; the parent
|
|
639
|
+
# validates the worker's exact package version before activation instead.
|
|
640
|
+
if [ "\${NEUROLINK_PROXY_TRAMPOLINE_EXEC_ONLY:-0}" = "1" ]; then
|
|
641
|
+
if [ -n "\${NEUROLINK_BIN:-}" ] && [ -x "$NEUROLINK_BIN" ]; then
|
|
642
|
+
exec "$NEUROLINK_BIN" "$@"
|
|
643
|
+
fi
|
|
644
|
+
for cand in \
|
|
645
|
+
"$(command -v neurolink 2>/dev/null || true)" \
|
|
646
|
+
"\${PNPM_HOME:-}/neurolink" \
|
|
647
|
+
"$HOME/.local/share/pnpm/neurolink" \
|
|
648
|
+
"$HOME/Library/pnpm/neurolink" \
|
|
649
|
+
"/usr/local/bin/neurolink" \
|
|
650
|
+
"/opt/homebrew/bin/neurolink"; do
|
|
651
|
+
[ -n "$cand" ] && [ -x "$cand" ] && exec "$cand" "$@"
|
|
652
|
+
done
|
|
653
|
+
if [ -x "$BAKED_NODE" ] && [ -f "$BAKED_SCRIPT" ]; then
|
|
654
|
+
exec "$BAKED_NODE" "$BAKED_SCRIPT" "$@"
|
|
655
|
+
fi
|
|
656
|
+
exit 127
|
|
657
|
+
fi
|
|
658
|
+
|
|
477
659
|
# Probe a candidate: must be executable and respond to --version cleanly.
|
|
478
660
|
_try() {
|
|
479
661
|
[ -n "$1" ] && [ -x "$1" ] || return 1
|
|
@@ -505,8 +687,6 @@ done
|
|
|
505
687
|
# 3. Baked-in fallback: the exact node + script that worked at install time.
|
|
506
688
|
# Always valid at install time; may become stale after package updates
|
|
507
689
|
# (but at that point the PATH candidates above should work).
|
|
508
|
-
BAKED_NODE=${shEscape(bakedNode)}
|
|
509
|
-
BAKED_SCRIPT=${shEscape(bakedScript)}
|
|
510
690
|
if [ -x "$BAKED_NODE" ] && [ -f "$BAKED_SCRIPT" ]; then
|
|
511
691
|
exec "$BAKED_NODE" "$BAKED_SCRIPT" "$@"
|
|
512
692
|
fi
|
|
@@ -538,18 +718,14 @@ function spawnFailOpenGuard(host, port, parentPid) {
|
|
|
538
718
|
String(parentPid),
|
|
539
719
|
"--quiet",
|
|
540
720
|
];
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
mkdirSync(guardLogDir, { recursive: true });
|
|
546
|
-
}
|
|
547
|
-
const guardLogPath = join(guardLogDir, "proxy-guard.log");
|
|
548
|
-
const logFd = openSync(guardLogPath, "a");
|
|
721
|
+
const workerLog = openProxyWorkerLog("proxy-guard.log");
|
|
722
|
+
if (workerLog.error) {
|
|
723
|
+
logger.debug(`[proxy] guard logging disabled: ${workerLog.error}`);
|
|
724
|
+
}
|
|
549
725
|
try {
|
|
550
726
|
const child = spawn(process.execPath, args, {
|
|
551
727
|
detached: true,
|
|
552
|
-
stdio: ["ignore",
|
|
728
|
+
stdio: ["ignore", workerLog.stdio, workerLog.stdio],
|
|
553
729
|
});
|
|
554
730
|
child.unref();
|
|
555
731
|
return child.pid;
|
|
@@ -559,10 +735,10 @@ function spawnFailOpenGuard(host, port, parentPid) {
|
|
|
559
735
|
return undefined;
|
|
560
736
|
}
|
|
561
737
|
finally {
|
|
562
|
-
|
|
738
|
+
workerLog.close();
|
|
563
739
|
}
|
|
564
740
|
}
|
|
565
|
-
function spawnProxyUpdater(host, port, parentPid) {
|
|
741
|
+
function spawnProxyUpdater(host, port, parentPid, rollingSupervisor = false) {
|
|
566
742
|
if (!isProxyAutoUpdateEnabled()) {
|
|
567
743
|
logger.always("[proxy] automatic updates disabled by environment");
|
|
568
744
|
return undefined;
|
|
@@ -572,8 +748,9 @@ function spawnProxyUpdater(host, port, parentPid) {
|
|
|
572
748
|
logger.always("[proxy] updater disabled: CLI entry script is unavailable");
|
|
573
749
|
return undefined;
|
|
574
750
|
}
|
|
751
|
+
const command = rollingSupervisor ? TRAMPOLINE_PATH : process.execPath;
|
|
575
752
|
const args = [
|
|
576
|
-
entryScript,
|
|
753
|
+
...(rollingSupervisor ? [] : [entryScript]),
|
|
577
754
|
"proxy",
|
|
578
755
|
"guard",
|
|
579
756
|
"--host",
|
|
@@ -585,19 +762,18 @@ function spawnProxyUpdater(host, port, parentPid) {
|
|
|
585
762
|
"--updater-only",
|
|
586
763
|
"--quiet",
|
|
587
764
|
];
|
|
588
|
-
const
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
mkdirSync(logsDir, { recursive: true });
|
|
765
|
+
const workerLog = openProxyWorkerLog("proxy-updater.log");
|
|
766
|
+
if (workerLog.error) {
|
|
767
|
+
logger.always(`[proxy] updater logging disabled: ${workerLog.error}`);
|
|
592
768
|
}
|
|
593
|
-
const logFd = openSync(join(logsDir, "proxy-updater.log"), "a");
|
|
594
769
|
try {
|
|
595
|
-
const child = spawn(
|
|
770
|
+
const child = spawn(command, args, {
|
|
596
771
|
detached: true,
|
|
597
|
-
stdio: ["ignore",
|
|
772
|
+
stdio: ["ignore", workerLog.stdio, workerLog.stdio],
|
|
598
773
|
env: {
|
|
599
774
|
...process.env,
|
|
600
775
|
NEUROLINK_PROXY_UPDATE_CONTROL_TOKEN: PROXY_UPDATE_CONTROL_TOKEN,
|
|
776
|
+
[PROXY_ROLLING_SUPERVISOR_ENV]: rollingSupervisor ? "1" : "0",
|
|
601
777
|
},
|
|
602
778
|
});
|
|
603
779
|
child.once("error", (error) => {
|
|
@@ -614,7 +790,7 @@ function spawnProxyUpdater(host, port, parentPid) {
|
|
|
614
790
|
return undefined;
|
|
615
791
|
}
|
|
616
792
|
finally {
|
|
617
|
-
|
|
793
|
+
workerLog.close();
|
|
618
794
|
}
|
|
619
795
|
}
|
|
620
796
|
async function runProxyTelemetryManager(command) {
|
|
@@ -705,9 +881,29 @@ function getProxyRuntimeErrorCode(error) {
|
|
|
705
881
|
async function ensureProxyStartAllowed(spinner) {
|
|
706
882
|
const ignoreLaunchd = process.env.NEUROLINK_PROXY_IGNORE_LAUNCHD === "1" ||
|
|
707
883
|
process.env.NEUROLINK_PROXY_IGNORE_LAUNCHD === "true";
|
|
884
|
+
const existingSupervisor = loadProxySupervisorState();
|
|
885
|
+
if (existingSupervisor) {
|
|
886
|
+
if (existingSupervisor.pid !== process.pid &&
|
|
887
|
+
isProcessRunning(existingSupervisor.pid) &&
|
|
888
|
+
!ignoreLaunchd) {
|
|
889
|
+
if (spinner) {
|
|
890
|
+
spinner.fail(chalk.red(`Proxy supervisor already running on port ${existingSupervisor.port} (PID: ${existingSupervisor.pid})`));
|
|
891
|
+
}
|
|
892
|
+
process.exit(process.ppid === 1 ? 0 : 1);
|
|
893
|
+
}
|
|
894
|
+
if (!isProcessRunning(existingSupervisor.pid)) {
|
|
895
|
+
clearProxySupervisorState();
|
|
896
|
+
}
|
|
897
|
+
}
|
|
708
898
|
const existingState = loadProxyState();
|
|
709
899
|
if (existingState) {
|
|
710
900
|
if (isProcessRunning(existingState.pid)) {
|
|
901
|
+
// A newly relaunched stable supervisor may inherit state from an orphaned
|
|
902
|
+
// worker that is draining old connections but no longer owns a listener.
|
|
903
|
+
// launchd must be allowed to restore listener ownership in that case.
|
|
904
|
+
if (process.ppid === 1) {
|
|
905
|
+
return;
|
|
906
|
+
}
|
|
711
907
|
// Test / dev escape hatch: when NEUROLINK_PROXY_IGNORE_LAUNCHD is set,
|
|
712
908
|
// allow starting a second proxy on the test's requested port even if
|
|
713
909
|
// a launchd-managed instance is using a different port (its state
|
|
@@ -1065,7 +1261,8 @@ export async function createProxyStartApp(params) {
|
|
|
1065
1261
|
metadata.stream = stream === "stream";
|
|
1066
1262
|
metadata.toolCount = toolCount;
|
|
1067
1263
|
}
|
|
1068
|
-
|
|
1264
|
+
const logModel = sanitizeForLog(String(model));
|
|
1265
|
+
logger.always(`[proxy] ${c.req.method} ${c.req.path} → model=${logModel} ${stream} tools=${toolCount}`);
|
|
1069
1266
|
const ctx = {
|
|
1070
1267
|
requestId: metadata?.requestId ?? crypto.randomUUID(),
|
|
1071
1268
|
method: c.req.method,
|
|
@@ -1180,6 +1377,7 @@ export async function createProxyStartApp(params) {
|
|
|
1180
1377
|
const { loadAccountCooldowns } = await import("../../lib/proxy/accountCooldown.js");
|
|
1181
1378
|
const stats = getStats();
|
|
1182
1379
|
const runtimeState = loadProxyState();
|
|
1380
|
+
const supervisorState = loadProxySupervisorState();
|
|
1183
1381
|
const updateState = loadUpdateState();
|
|
1184
1382
|
const cooldowns = await loadAccountCooldowns();
|
|
1185
1383
|
const storedAccountKeys = new Set();
|
|
@@ -1199,6 +1397,7 @@ export async function createProxyStartApp(params) {
|
|
|
1199
1397
|
version: PROXY_VERSION,
|
|
1200
1398
|
});
|
|
1201
1399
|
const primaryAccount = await resolveStatusPrimaryAccount(activeProxyConfig);
|
|
1400
|
+
const activeUpdaterPid = supervisorState?.updaterPid ?? runtimeState?.updaterPid;
|
|
1202
1401
|
return c.json({
|
|
1203
1402
|
status: "running",
|
|
1204
1403
|
ready: health.ready,
|
|
@@ -1257,9 +1456,11 @@ export async function createProxyStartApp(params) {
|
|
|
1257
1456
|
},
|
|
1258
1457
|
autoUpdate: {
|
|
1259
1458
|
enabled: isProxyAutoUpdateEnabled(),
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1459
|
+
supervisorPid: supervisorState?.pid ?? null,
|
|
1460
|
+
rolling: supervisorState?.rolling ?? null,
|
|
1461
|
+
updaterPid: activeUpdaterPid ?? null,
|
|
1462
|
+
updaterRunning: activeUpdaterPid
|
|
1463
|
+
? isProcessRunning(activeUpdaterPid)
|
|
1263
1464
|
: false,
|
|
1264
1465
|
liveVersion: PROXY_VERSION,
|
|
1265
1466
|
latestVersion: updateState?.lastCheckVersion || null,
|
|
@@ -1535,7 +1736,7 @@ function registerProxyShutdownHandlers(params) {
|
|
|
1535
1736
|
}
|
|
1536
1737
|
});
|
|
1537
1738
|
};
|
|
1538
|
-
const shutdown = async (signal) => {
|
|
1739
|
+
const shutdown = async (signal, options) => {
|
|
1539
1740
|
if (shutdownStarted) {
|
|
1540
1741
|
return;
|
|
1541
1742
|
}
|
|
@@ -1545,13 +1746,15 @@ function registerProxyShutdownHandlers(params) {
|
|
|
1545
1746
|
params.updaterSupervisor?.stop();
|
|
1546
1747
|
params.stopRuntimeConfig?.();
|
|
1547
1748
|
logger.always(`\nShutting down proxy (${signal})...`);
|
|
1548
|
-
let exitCode = signal === "SIGINT" ? 0 : 1;
|
|
1549
|
-
|
|
1550
|
-
|
|
1551
|
-
|
|
1552
|
-
|
|
1553
|
-
|
|
1554
|
-
|
|
1749
|
+
let exitCode = signal === "SIGINT" || signal === "ROLLING_DRAIN" ? 0 : 1;
|
|
1750
|
+
if (!options?.skipServerClose) {
|
|
1751
|
+
try {
|
|
1752
|
+
await closeServer();
|
|
1753
|
+
}
|
|
1754
|
+
catch (error) {
|
|
1755
|
+
exitCode = 1;
|
|
1756
|
+
logger.error(`[proxy] failed to drain server during shutdown: ${error instanceof Error ? error.message : String(error)}`);
|
|
1757
|
+
}
|
|
1555
1758
|
}
|
|
1556
1759
|
try {
|
|
1557
1760
|
await withTimeout(flushProxyLifecycleEvents(), PROXY_LIFECYCLE_SHUTDOWN_TIMEOUT_MS, "Timed out flushing proxy lifecycle metadata during shutdown");
|
|
@@ -1584,7 +1787,10 @@ function registerProxyShutdownHandlers(params) {
|
|
|
1584
1787
|
}
|
|
1585
1788
|
}
|
|
1586
1789
|
try {
|
|
1587
|
-
|
|
1790
|
+
const state = loadProxyState();
|
|
1791
|
+
if (state?.pid === process.pid) {
|
|
1792
|
+
clearProxyState();
|
|
1793
|
+
}
|
|
1588
1794
|
}
|
|
1589
1795
|
catch (error) {
|
|
1590
1796
|
exitCode = 1;
|
|
@@ -1596,21 +1802,30 @@ function registerProxyShutdownHandlers(params) {
|
|
|
1596
1802
|
logger.error(`[proxy] unexpected shutdown failure: ${error instanceof Error ? error.message : String(error)}`);
|
|
1597
1803
|
process.exit(1);
|
|
1598
1804
|
};
|
|
1599
|
-
|
|
1600
|
-
|
|
1601
|
-
|
|
1602
|
-
|
|
1603
|
-
|
|
1604
|
-
|
|
1805
|
+
if (params.registerSignals ?? true) {
|
|
1806
|
+
process.on("SIGTERM", () => {
|
|
1807
|
+
void shutdown("SIGTERM").catch(forceExitAfterShutdownFailure);
|
|
1808
|
+
});
|
|
1809
|
+
process.on("SIGINT", () => {
|
|
1810
|
+
void shutdown("SIGINT").catch(forceExitAfterShutdownFailure);
|
|
1811
|
+
});
|
|
1812
|
+
}
|
|
1813
|
+
return shutdown;
|
|
1605
1814
|
}
|
|
1606
1815
|
async function startProxyRuntime(params) {
|
|
1607
|
-
const
|
|
1608
|
-
const
|
|
1609
|
-
|
|
1610
|
-
|
|
1611
|
-
|
|
1612
|
-
|
|
1613
|
-
|
|
1816
|
+
const socketWorker = isProxySocketWorkerProcess();
|
|
1817
|
+
const { createAdaptorServer, serve } = await import("@hono/node-server");
|
|
1818
|
+
const server = socketWorker
|
|
1819
|
+
? createAdaptorServer({
|
|
1820
|
+
fetch: params.app.fetch,
|
|
1821
|
+
hostname: params.host,
|
|
1822
|
+
})
|
|
1823
|
+
: serve({
|
|
1824
|
+
fetch: params.app.fetch,
|
|
1825
|
+
port: params.port,
|
|
1826
|
+
hostname: params.host,
|
|
1827
|
+
});
|
|
1828
|
+
const managedByLaunchd = isLaunchdManagedProcess() || socketWorker;
|
|
1614
1829
|
// launchd already owns restart supervision. A second detached supervisor can
|
|
1615
1830
|
// outlive its parent and terminate a healthy replacement, so the guard is
|
|
1616
1831
|
// reserved for foreground mode where it only cleans stale client settings.
|
|
@@ -1618,19 +1833,31 @@ async function startProxyRuntime(params) {
|
|
|
1618
1833
|
? undefined
|
|
1619
1834
|
: spawnFailOpenGuard(params.host, params.port, process.pid);
|
|
1620
1835
|
const readinessHost = params.host === "0.0.0.0" ? "127.0.0.1" : params.host;
|
|
1621
|
-
|
|
1622
|
-
|
|
1623
|
-
|
|
1624
|
-
|
|
1836
|
+
if (!socketWorker) {
|
|
1837
|
+
await waitForProxyReadiness({
|
|
1838
|
+
host: readinessHost,
|
|
1839
|
+
port: params.port,
|
|
1840
|
+
});
|
|
1841
|
+
}
|
|
1625
1842
|
markProxyReady(params.readiness);
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
if (
|
|
1629
|
-
|
|
1843
|
+
let updateReconciled = false;
|
|
1844
|
+
const reconcileActivatedUpdate = async () => {
|
|
1845
|
+
if (updateReconciled) {
|
|
1846
|
+
return;
|
|
1630
1847
|
}
|
|
1631
|
-
|
|
1632
|
-
|
|
1633
|
-
|
|
1848
|
+
updateReconciled = true;
|
|
1849
|
+
try {
|
|
1850
|
+
const { reconcileRunningUpdate } = await import("../../lib/proxy/updateState.js");
|
|
1851
|
+
if (reconcileRunningUpdate(PROXY_VERSION)) {
|
|
1852
|
+
logger.always(`[proxy] confirmed pending update is now running at v${PROXY_VERSION}`);
|
|
1853
|
+
}
|
|
1854
|
+
}
|
|
1855
|
+
catch (error) {
|
|
1856
|
+
logger.always(`[proxy] WARNING: failed to reconcile update state: ${error instanceof Error ? error.message : String(error)}`);
|
|
1857
|
+
}
|
|
1858
|
+
};
|
|
1859
|
+
if (!socketWorker) {
|
|
1860
|
+
await reconcileActivatedUpdate();
|
|
1634
1861
|
}
|
|
1635
1862
|
/** Mirror the supervised worker PID into the proxy state used by status. */
|
|
1636
1863
|
const updatePersistedUpdaterPid = (updaterPid) => {
|
|
@@ -1640,7 +1867,7 @@ async function startProxyRuntime(params) {
|
|
|
1640
1867
|
}
|
|
1641
1868
|
saveProxyState({ ...state, updaterPid });
|
|
1642
1869
|
};
|
|
1643
|
-
const updaterSupervisor = managedByLaunchd && !params.argv.dev
|
|
1870
|
+
const updaterSupervisor = managedByLaunchd && !params.argv.dev && !socketWorker
|
|
1644
1871
|
? startUpdaterWorkerSupervisor({
|
|
1645
1872
|
spawnWorker: () => spawnProxyUpdater(readinessHost, params.port, process.pid),
|
|
1646
1873
|
isProcessRunning,
|
|
@@ -1668,32 +1895,41 @@ async function startProxyRuntime(params) {
|
|
|
1668
1895
|
model: entry.model,
|
|
1669
1896
|
}));
|
|
1670
1897
|
const initialConfigStatus = params.runtimeConfigStore?.getStatus();
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
:
|
|
1681
|
-
|
|
1682
|
-
|
|
1683
|
-
|
|
1684
|
-
|
|
1685
|
-
|
|
1686
|
-
|
|
1687
|
-
:
|
|
1688
|
-
|
|
1689
|
-
|
|
1690
|
-
|
|
1691
|
-
|
|
1692
|
-
|
|
1693
|
-
|
|
1694
|
-
|
|
1695
|
-
|
|
1696
|
-
|
|
1898
|
+
const persistInitialProxyState = () => {
|
|
1899
|
+
const supervisorState = socketWorker ? loadProxySupervisorState() : null;
|
|
1900
|
+
saveProxyState({
|
|
1901
|
+
pid: process.pid,
|
|
1902
|
+
port: params.port,
|
|
1903
|
+
host: params.host,
|
|
1904
|
+
strategy: activeStrategy,
|
|
1905
|
+
startTime: new Date().toISOString(),
|
|
1906
|
+
ready: true,
|
|
1907
|
+
readyAt: params.readiness.readyAtMs
|
|
1908
|
+
? new Date(params.readiness.readyAtMs).toISOString()
|
|
1909
|
+
: undefined,
|
|
1910
|
+
healthPath: "/health",
|
|
1911
|
+
statusPath: "/status",
|
|
1912
|
+
envFile: params.loadedEnvFile,
|
|
1913
|
+
fallbackChain,
|
|
1914
|
+
accountAllowlist: activeAccountAllowlist
|
|
1915
|
+
? [...activeAccountAllowlist]
|
|
1916
|
+
: undefined,
|
|
1917
|
+
guardPid,
|
|
1918
|
+
updaterPid: socketWorker ? supervisorState?.updaterPid : updaterPid,
|
|
1919
|
+
supervisorPid: socketWorker
|
|
1920
|
+
? (supervisorState?.pid ?? process.ppid)
|
|
1921
|
+
: undefined,
|
|
1922
|
+
managedBy: managedByLaunchd ? "launchd" : "manual",
|
|
1923
|
+
passthrough: activePassthrough,
|
|
1924
|
+
configGeneration: initialRuntimeConfig?.generation,
|
|
1925
|
+
configLoadedAt: initialRuntimeConfig?.loadedAt,
|
|
1926
|
+
lastConfigReloadError: initialConfigStatus?.lastReloadError,
|
|
1927
|
+
configFile: initialConfigStatus?.configPath,
|
|
1928
|
+
});
|
|
1929
|
+
};
|
|
1930
|
+
if (!socketWorker) {
|
|
1931
|
+
persistInitialProxyState();
|
|
1932
|
+
}
|
|
1697
1933
|
const persistRuntimeConfig = (snapshot) => {
|
|
1698
1934
|
const state = loadProxyState();
|
|
1699
1935
|
if (!state || state.pid !== process.pid) {
|
|
@@ -1781,20 +2017,152 @@ async function startProxyRuntime(params) {
|
|
|
1781
2017
|
const maintenance = startProxyBackgroundMaintenance(params.cleanupLogs, () => params.runtimeConfigStore
|
|
1782
2018
|
? params.runtimeConfigStore.getSnapshot().accountAllowlist
|
|
1783
2019
|
: params.accountAllowlist);
|
|
1784
|
-
registerProxyShutdownHandlers({
|
|
2020
|
+
const shutdown = registerProxyShutdownHandlers({
|
|
1785
2021
|
server,
|
|
1786
2022
|
host: params.host,
|
|
1787
2023
|
port: params.port,
|
|
1788
2024
|
isDev,
|
|
1789
2025
|
updaterSupervisor,
|
|
1790
2026
|
stopRuntimeConfig,
|
|
2027
|
+
registerSignals: !socketWorker,
|
|
1791
2028
|
...maintenance,
|
|
1792
2029
|
});
|
|
2030
|
+
if (socketWorker) {
|
|
2031
|
+
attachSocketWorkerProcess(server, {
|
|
2032
|
+
generation: getProxyWorkerGeneration(),
|
|
2033
|
+
version: PROXY_VERSION,
|
|
2034
|
+
onActivated: () => {
|
|
2035
|
+
persistInitialProxyState();
|
|
2036
|
+
void reconcileActivatedUpdate();
|
|
2037
|
+
},
|
|
2038
|
+
onDrained: () => {
|
|
2039
|
+
void shutdown("ROLLING_DRAIN", { skipServerClose: true });
|
|
2040
|
+
},
|
|
2041
|
+
});
|
|
2042
|
+
}
|
|
2043
|
+
}
|
|
2044
|
+
async function runLaunchdProxySupervisor(argv, spinner) {
|
|
2045
|
+
await ensureProxyStartAllowed(spinner);
|
|
2046
|
+
const entryScript = process.argv[1];
|
|
2047
|
+
if (!entryScript) {
|
|
2048
|
+
throw new Error("proxy supervisor cannot resolve the CLI entry script");
|
|
2049
|
+
}
|
|
2050
|
+
const host = argv.host ?? "127.0.0.1";
|
|
2051
|
+
const port = argv.port ?? 55669;
|
|
2052
|
+
const workerArgs = process.argv.slice(2);
|
|
2053
|
+
const supervisorStartedAt = new Date().toISOString();
|
|
2054
|
+
let currentUpdaterPid;
|
|
2055
|
+
const rollingServer = await startRollingProxyServer({
|
|
2056
|
+
host,
|
|
2057
|
+
port,
|
|
2058
|
+
initialVersion: PROXY_VERSION,
|
|
2059
|
+
spawnWorker: (generation, expectedVersion) => spawnProxySocketWorker({
|
|
2060
|
+
generation,
|
|
2061
|
+
expectedVersion,
|
|
2062
|
+
command: TRAMPOLINE_PATH,
|
|
2063
|
+
args: workerArgs,
|
|
2064
|
+
env: {
|
|
2065
|
+
NEUROLINK_PROXY_UPDATE_CONTROL_TOKEN: PROXY_UPDATE_CONTROL_TOKEN,
|
|
2066
|
+
NEUROLINK_PROXY_TRAMPOLINE_EXEC_ONLY: "1",
|
|
2067
|
+
},
|
|
2068
|
+
}),
|
|
2069
|
+
onStateChange: (snapshot) => {
|
|
2070
|
+
saveProxySupervisorState({
|
|
2071
|
+
pid: process.pid,
|
|
2072
|
+
host,
|
|
2073
|
+
port,
|
|
2074
|
+
startTime: supervisorStartedAt,
|
|
2075
|
+
updaterPid: currentUpdaterPid,
|
|
2076
|
+
rolling: snapshot,
|
|
2077
|
+
});
|
|
2078
|
+
},
|
|
2079
|
+
log: (message) => logger.always(message),
|
|
2080
|
+
});
|
|
2081
|
+
let rollingReplacement = null;
|
|
2082
|
+
const activatePendingUpdate = () => {
|
|
2083
|
+
if (rollingReplacement) {
|
|
2084
|
+
return;
|
|
2085
|
+
}
|
|
2086
|
+
let pendingVersion;
|
|
2087
|
+
try {
|
|
2088
|
+
pendingVersion = loadUpdateState()?.pendingRestartVersion ?? undefined;
|
|
2089
|
+
}
|
|
2090
|
+
catch (error) {
|
|
2091
|
+
logger.always(`[proxy-supervisor] failed to read pending update state: ${error instanceof Error ? error.message : String(error)}`);
|
|
2092
|
+
return;
|
|
2093
|
+
}
|
|
2094
|
+
if (!pendingVersion || !/^\d+\.\d+\.\d+$/.test(pendingVersion)) {
|
|
2095
|
+
logger.always("[proxy-supervisor] ignored update activation without a valid pending version");
|
|
2096
|
+
return;
|
|
2097
|
+
}
|
|
2098
|
+
logger.always(`[proxy-supervisor] preparing rolling activation for v${pendingVersion}`);
|
|
2099
|
+
rollingReplacement = rollingServer
|
|
2100
|
+
.replace(pendingVersion)
|
|
2101
|
+
.then(() => {
|
|
2102
|
+
logger.always(`[proxy-supervisor] rolling activation complete version=${pendingVersion}`);
|
|
2103
|
+
})
|
|
2104
|
+
.catch((error) => {
|
|
2105
|
+
logger.always(`[proxy-supervisor] rolling activation failed version=${pendingVersion}: ${error instanceof Error ? error.message : String(error)}`);
|
|
2106
|
+
})
|
|
2107
|
+
.finally(() => {
|
|
2108
|
+
rollingReplacement = null;
|
|
2109
|
+
});
|
|
2110
|
+
};
|
|
2111
|
+
process.on("SIGUSR2", activatePendingUpdate);
|
|
2112
|
+
const readinessHost = host === "0.0.0.0" ? "127.0.0.1" : host;
|
|
2113
|
+
const updatePersistedUpdaterPid = (updaterPid) => {
|
|
2114
|
+
currentUpdaterPid = updaterPid;
|
|
2115
|
+
const state = loadProxySupervisorState();
|
|
2116
|
+
if (!state || state.pid !== process.pid) {
|
|
2117
|
+
return;
|
|
2118
|
+
}
|
|
2119
|
+
saveProxySupervisorState({ ...state, updaterPid });
|
|
2120
|
+
};
|
|
2121
|
+
const updaterSupervisor = startUpdaterWorkerSupervisor({
|
|
2122
|
+
spawnWorker: () => spawnProxyUpdater(readinessHost, rollingServer.address.port, process.pid, true),
|
|
2123
|
+
isProcessRunning,
|
|
2124
|
+
stopWorker: (pid) => process.kill(pid, "SIGTERM"),
|
|
2125
|
+
onPidChange: updatePersistedUpdaterPid,
|
|
2126
|
+
log: (message) => logger.always(message),
|
|
2127
|
+
});
|
|
2128
|
+
updatePersistedUpdaterPid(updaterSupervisor.currentPid());
|
|
2129
|
+
if (spinner) {
|
|
2130
|
+
spinner.succeed(chalk.green("Claude proxy supervisor started successfully"));
|
|
2131
|
+
}
|
|
2132
|
+
logger.always(`[proxy-supervisor] listening on ${host}:${rollingServer.address.port} workerPid=${rollingServer.snapshot().active?.pid ?? "unknown"} version=${PROXY_VERSION}`);
|
|
2133
|
+
let stopping = false;
|
|
2134
|
+
const shutdown = async (signal) => {
|
|
2135
|
+
if (stopping) {
|
|
2136
|
+
return;
|
|
2137
|
+
}
|
|
2138
|
+
stopping = true;
|
|
2139
|
+
logger.always(`[proxy-supervisor] shutting down (${signal})`);
|
|
2140
|
+
process.off("SIGUSR2", activatePendingUpdate);
|
|
2141
|
+
updaterSupervisor.stop();
|
|
2142
|
+
await rollingServer.close();
|
|
2143
|
+
const supervisorState = loadProxySupervisorState();
|
|
2144
|
+
if (supervisorState?.pid === process.pid) {
|
|
2145
|
+
clearProxySupervisorState();
|
|
2146
|
+
}
|
|
2147
|
+
process.exit(0);
|
|
2148
|
+
};
|
|
2149
|
+
process.once("SIGTERM", () => {
|
|
2150
|
+
void shutdown("SIGTERM");
|
|
2151
|
+
});
|
|
2152
|
+
process.once("SIGINT", () => {
|
|
2153
|
+
void shutdown("SIGINT");
|
|
2154
|
+
});
|
|
2155
|
+
await new Promise(() => undefined);
|
|
1793
2156
|
}
|
|
1794
2157
|
async function startProxyCommandHandler(argv) {
|
|
1795
2158
|
const spinner = argv.quiet ? null : ora("Starting Claude proxy...").start();
|
|
1796
2159
|
const isDev = argv.dev ?? false;
|
|
2160
|
+
const socketWorker = isProxySocketWorkerProcess();
|
|
1797
2161
|
try {
|
|
2162
|
+
if (!isDev && isLaunchdManagedProcess() && !socketWorker) {
|
|
2163
|
+
await runLaunchdProxySupervisor(argv, spinner);
|
|
2164
|
+
return;
|
|
2165
|
+
}
|
|
1798
2166
|
// In dev mode: redirect writable state to .neurolink-dev/ and skip singleton check
|
|
1799
2167
|
let devPaths;
|
|
1800
2168
|
if (isDev) {
|
|
@@ -1811,7 +2179,7 @@ async function startProxyCommandHandler(argv) {
|
|
|
1811
2179
|
mkdirSync(devPaths.stateDir, { recursive: true, mode: 0o700 });
|
|
1812
2180
|
}
|
|
1813
2181
|
}
|
|
1814
|
-
if (!isDev) {
|
|
2182
|
+
if (!isDev && !socketWorker) {
|
|
1815
2183
|
await ensureProxyStartAllowed(spinner);
|
|
1816
2184
|
}
|
|
1817
2185
|
const baseEnv = { ...process.env };
|
|
@@ -2046,6 +2414,7 @@ export const proxyStatusCommand = {
|
|
|
2046
2414
|
handler: async (argv) => {
|
|
2047
2415
|
try {
|
|
2048
2416
|
const state = loadProxyState();
|
|
2417
|
+
const supervisorState = loadProxySupervisorState();
|
|
2049
2418
|
const updateState = loadUpdateState();
|
|
2050
2419
|
const status = {
|
|
2051
2420
|
running: false,
|
|
@@ -2064,6 +2433,10 @@ export const proxyStatusCommand = {
|
|
|
2064
2433
|
configLoadedAt: null,
|
|
2065
2434
|
lastConfigReloadError: null,
|
|
2066
2435
|
autoUpdateEnabled: isProxyAutoUpdateEnabled(),
|
|
2436
|
+
workerVersion: null,
|
|
2437
|
+
supervisorPid: null,
|
|
2438
|
+
supervisorRunning: false,
|
|
2439
|
+
rolling: null,
|
|
2067
2440
|
updaterPid: null,
|
|
2068
2441
|
updaterRunning: false,
|
|
2069
2442
|
latestVersion: updateState?.lastCheckVersion || null,
|
|
@@ -2071,25 +2444,52 @@ export const proxyStatusCommand = {
|
|
|
2071
2444
|
deferredUpdate: updateState?.deferredUpdate ?? null,
|
|
2072
2445
|
lastUpdateFailure: updateState?.lastFailure ?? null,
|
|
2073
2446
|
};
|
|
2074
|
-
|
|
2447
|
+
const workerRunning = state ? isProcessRunning(state.pid) : false;
|
|
2448
|
+
const supervisorPid = supervisorState?.pid ?? state?.supervisorPid;
|
|
2449
|
+
const supervisorRunning = supervisorPid
|
|
2450
|
+
? isProcessRunning(supervisorPid)
|
|
2451
|
+
: false;
|
|
2452
|
+
const servingWorker = workerRunning &&
|
|
2453
|
+
(supervisorPid
|
|
2454
|
+
? supervisorRunning &&
|
|
2455
|
+
(!supervisorState ||
|
|
2456
|
+
supervisorState.rolling.active?.pid === state?.pid)
|
|
2457
|
+
: true);
|
|
2458
|
+
if ((state || supervisorState) && (servingWorker || supervisorRunning)) {
|
|
2459
|
+
const activeHost = state?.host ?? supervisorState?.host ?? null;
|
|
2460
|
+
const activePort = state?.port ?? supervisorState?.port ?? null;
|
|
2075
2461
|
status.running = true;
|
|
2076
|
-
status.pid = state.pid;
|
|
2077
|
-
status.port =
|
|
2078
|
-
status.host =
|
|
2079
|
-
status.mode = state
|
|
2080
|
-
|
|
2081
|
-
|
|
2082
|
-
|
|
2083
|
-
|
|
2084
|
-
status.
|
|
2085
|
-
status.
|
|
2086
|
-
|
|
2087
|
-
status.
|
|
2088
|
-
|
|
2089
|
-
|
|
2090
|
-
status.
|
|
2091
|
-
|
|
2092
|
-
|
|
2462
|
+
status.pid = servingWorker && state ? state.pid : null;
|
|
2463
|
+
status.port = activePort;
|
|
2464
|
+
status.host = activeHost;
|
|
2465
|
+
status.mode = state
|
|
2466
|
+
? state.passthrough
|
|
2467
|
+
? "passthrough"
|
|
2468
|
+
: "full"
|
|
2469
|
+
: null;
|
|
2470
|
+
status.strategy = state?.strategy ?? null;
|
|
2471
|
+
status.startTime =
|
|
2472
|
+
state?.startTime ?? supervisorState?.startTime ?? null;
|
|
2473
|
+
status.uptime = status.startTime
|
|
2474
|
+
? Date.now() - new Date(status.startTime).getTime()
|
|
2475
|
+
: null;
|
|
2476
|
+
status.url =
|
|
2477
|
+
activeHost && activePort
|
|
2478
|
+
? `http://${activeHost === "0.0.0.0" ? "localhost" : activeHost}:${activePort}`
|
|
2479
|
+
: null;
|
|
2480
|
+
status.envFile = state?.envFile ?? null;
|
|
2481
|
+
status.fallbackChain = state?.fallbackChain ?? null;
|
|
2482
|
+
status.accountAllowlist = state?.accountAllowlist ?? null;
|
|
2483
|
+
status.configGeneration = state?.configGeneration ?? null;
|
|
2484
|
+
status.configLoadedAt = state?.configLoadedAt ?? null;
|
|
2485
|
+
status.lastConfigReloadError = state?.lastConfigReloadError ?? null;
|
|
2486
|
+
status.supervisorPid = supervisorPid ?? null;
|
|
2487
|
+
status.supervisorRunning = supervisorRunning;
|
|
2488
|
+
status.rolling = supervisorState?.rolling ?? null;
|
|
2489
|
+
status.updaterPid =
|
|
2490
|
+
supervisorState?.updaterPid ?? state?.updaterPid ?? null;
|
|
2491
|
+
status.updaterRunning = status.updaterPid
|
|
2492
|
+
? isProcessRunning(status.updaterPid)
|
|
2093
2493
|
: false;
|
|
2094
2494
|
}
|
|
2095
2495
|
// Fetch live stats before rendering (JSON or text)
|
|
@@ -2097,11 +2497,17 @@ export const proxyStatusCommand = {
|
|
|
2097
2497
|
let liveConfig = null;
|
|
2098
2498
|
if (status.running && status.url) {
|
|
2099
2499
|
try {
|
|
2100
|
-
const statusResp = await fetch(`${status.url}/status
|
|
2500
|
+
const statusResp = await fetch(`${status.url}/status`, {
|
|
2501
|
+
signal: AbortSignal.timeout(2_000),
|
|
2502
|
+
});
|
|
2101
2503
|
if (statusResp.ok) {
|
|
2102
2504
|
const statusData = (await statusResp.json());
|
|
2103
2505
|
liveStats = statusData.stats;
|
|
2104
2506
|
liveConfig = statusData.config;
|
|
2507
|
+
status.workerVersion =
|
|
2508
|
+
typeof statusData.version === "string"
|
|
2509
|
+
? statusData.version
|
|
2510
|
+
: null;
|
|
2105
2511
|
if (typeof liveConfig?.generation === "number") {
|
|
2106
2512
|
status.configGeneration = liveConfig.generation;
|
|
2107
2513
|
}
|
|
@@ -2128,8 +2534,15 @@ export const proxyStatusCommand = {
|
|
|
2128
2534
|
logger.always(chalk.gray("=".repeat(50)));
|
|
2129
2535
|
logger.always("");
|
|
2130
2536
|
if (status.running) {
|
|
2131
|
-
|
|
2132
|
-
logger.always(` ${chalk.bold("
|
|
2537
|
+
const serving = status.pid !== null;
|
|
2538
|
+
logger.always(` ${chalk.bold("Status:")} ${serving ? chalk.green("RUNNING") : chalk.yellow("RECOVERING")}`);
|
|
2539
|
+
logger.always(` ${chalk.bold("Worker PID:")} ${serving ? chalk.cyan(status.pid) : chalk.yellow("unavailable")}`);
|
|
2540
|
+
if (status.supervisorPid) {
|
|
2541
|
+
logger.always(` ${chalk.bold("Supervisor:")} ${status.supervisorRunning ? chalk.cyan(status.supervisorPid) : chalk.red(`${status.supervisorPid} (not running)`)}`);
|
|
2542
|
+
}
|
|
2543
|
+
if (status.workerVersion) {
|
|
2544
|
+
logger.always(` ${chalk.bold("Version:")} ${chalk.cyan(`v${status.workerVersion}`)}`);
|
|
2545
|
+
}
|
|
2133
2546
|
logger.always(` ${chalk.bold("URL:")} ${chalk.cyan(status.url)}`);
|
|
2134
2547
|
logger.always(` ${chalk.bold("Strategy:")} ${chalk.cyan(status.strategy)}`);
|
|
2135
2548
|
logger.always(` ${chalk.bold("Mode:")} ${chalk.cyan(status.mode ?? "full")}`);
|
|
@@ -2148,6 +2561,12 @@ export const proxyStatusCommand = {
|
|
|
2148
2561
|
if (status.pendingRestartVersion) {
|
|
2149
2562
|
logger.always(` ${chalk.bold("Pending:")} ${chalk.yellow(`v${status.pendingRestartVersion} installed; restart pending`)}`);
|
|
2150
2563
|
}
|
|
2564
|
+
if (status.rolling?.candidate) {
|
|
2565
|
+
logger.always(` ${chalk.bold("Handoff:")} ${chalk.yellow(`preparing v${status.rolling.candidate.expectedVersion} (PID ${status.rolling.candidate.pid})`)}`);
|
|
2566
|
+
}
|
|
2567
|
+
else if (status.rolling?.draining.length) {
|
|
2568
|
+
logger.always(` ${chalk.bold("Handoff:")} ${chalk.cyan(`${status.rolling.draining.length} previous worker(s) draining`)}`);
|
|
2569
|
+
}
|
|
2151
2570
|
if (status.deferredUpdate) {
|
|
2152
2571
|
const active = status.deferredUpdate.activeRequests === null
|
|
2153
2572
|
? "unknown activity"
|
|
@@ -2181,7 +2600,9 @@ export const proxyStatusCommand = {
|
|
|
2181
2600
|
}
|
|
2182
2601
|
// Try to fetch live status from the running proxy
|
|
2183
2602
|
try {
|
|
2184
|
-
const response = await fetch(`${status.url}/health
|
|
2603
|
+
const response = await fetch(`${status.url}/health`, {
|
|
2604
|
+
signal: AbortSignal.timeout(2_000),
|
|
2605
|
+
});
|
|
2185
2606
|
if (response.ok) {
|
|
2186
2607
|
const liveStatus = (await response.json());
|
|
2187
2608
|
logger.always("");
|
|
@@ -2196,7 +2617,9 @@ export const proxyStatusCommand = {
|
|
|
2196
2617
|
// Try to get detailed stats
|
|
2197
2618
|
try {
|
|
2198
2619
|
const liveUrl = status.url;
|
|
2199
|
-
const statusResp = await fetch(`${liveUrl}/status
|
|
2620
|
+
const statusResp = await fetch(`${liveUrl}/status`, {
|
|
2621
|
+
signal: AbortSignal.timeout(2_000),
|
|
2622
|
+
});
|
|
2200
2623
|
if (statusResp.ok) {
|
|
2201
2624
|
const statusData = (await statusResp.json());
|
|
2202
2625
|
if (statusData.stats) {
|
|
@@ -2329,6 +2752,7 @@ export const proxyGuardCommand = {
|
|
|
2329
2752
|
const failureThreshold = Math.max(1, Number(argv.failureThreshold ?? 5));
|
|
2330
2753
|
const pollIntervalMs = Math.max(250, Number(argv.pollIntervalMs ?? 1_000));
|
|
2331
2754
|
const updaterOnly = argv.updaterOnly === true;
|
|
2755
|
+
const rollingSupervisor = process.env[PROXY_ROLLING_SUPERVISOR_ENV] === "1";
|
|
2332
2756
|
if (!Number.isFinite(parentPid) || parentPid <= 0) {
|
|
2333
2757
|
return;
|
|
2334
2758
|
}
|
|
@@ -2414,56 +2838,62 @@ export const proxyGuardCommand = {
|
|
|
2414
2838
|
return;
|
|
2415
2839
|
}
|
|
2416
2840
|
logger.always(`[updater] update available: ${runningVersion} → ${result.latestVersion}`);
|
|
2417
|
-
|
|
2418
|
-
|
|
2419
|
-
|
|
2420
|
-
|
|
2421
|
-
|
|
2422
|
-
|
|
2423
|
-
|
|
2424
|
-
|
|
2425
|
-
|
|
2426
|
-
|
|
2427
|
-
|
|
2428
|
-
|
|
2429
|
-
|
|
2430
|
-
|
|
2431
|
-
|
|
2432
|
-
|
|
2433
|
-
|
|
2434
|
-
|
|
2435
|
-
|
|
2436
|
-
|
|
2437
|
-
|
|
2438
|
-
|
|
2439
|
-
|
|
2440
|
-
|
|
2441
|
-
|
|
2442
|
-
|
|
2443
|
-
|
|
2444
|
-
|
|
2445
|
-
|
|
2446
|
-
|
|
2447
|
-
|
|
2448
|
-
|
|
2449
|
-
|
|
2450
|
-
|
|
2451
|
-
|
|
2452
|
-
|
|
2453
|
-
|
|
2454
|
-
persistUpdaterState("record unavailable update drain", () => recordUpdateDeferred(result.latestVersion, "drain_unavailable", null));
|
|
2455
|
-
}
|
|
2456
|
-
scheduleUpdateRetry();
|
|
2457
|
-
return;
|
|
2458
|
-
}
|
|
2459
|
-
// Close the small race between observing zero activity and package
|
|
2460
|
-
// mutation by holding admission closed through install and restart.
|
|
2461
|
-
if (!drainActive) {
|
|
2462
|
-
updateWindow = await waitForWindow(0);
|
|
2841
|
+
if (!rollingSupervisor) {
|
|
2842
|
+
// Legacy launchd services must become idle before their listener is
|
|
2843
|
+
// restarted. Rolling services keep the current worker serving while
|
|
2844
|
+
// the candidate package is installed and validated.
|
|
2845
|
+
let lastDeferralSignature = "";
|
|
2846
|
+
const waitForWindow = async (quietWaitMs) => {
|
|
2847
|
+
const window = await waitForProxyUpdateWindow({
|
|
2848
|
+
quietThresholdMs: QUIET_THRESHOLD_MS,
|
|
2849
|
+
quietWaitMs,
|
|
2850
|
+
drainWaitMs: UPDATE_DRAIN_TIMEOUT_MS,
|
|
2851
|
+
pollIntervalMs: UPDATE_ACTIVITY_POLL_MS,
|
|
2852
|
+
getActivity: () => getProxyRuntimeActivity(host, port),
|
|
2853
|
+
setDraining: async (draining) => {
|
|
2854
|
+
const changed = await setProxyUpdateDrain(host, port, draining);
|
|
2855
|
+
if (changed) {
|
|
2856
|
+
logger.always(`[updater] ${draining ? "draining new inference requests" : "inference admission resumed"}`);
|
|
2857
|
+
}
|
|
2858
|
+
return changed;
|
|
2859
|
+
},
|
|
2860
|
+
isStopping: () => guardStopping,
|
|
2861
|
+
isParentAlive: () => getProcessStatus(parentPid) !== "not_running",
|
|
2862
|
+
onPhase: (phase, activity) => {
|
|
2863
|
+
const reason = phase === "waiting_for_quiet" && !activity
|
|
2864
|
+
? "activity_unavailable"
|
|
2865
|
+
: phase;
|
|
2866
|
+
const signature = `${reason}:${activity?.activeRequests ?? "unknown"}`;
|
|
2867
|
+
if (signature !== lastDeferralSignature) {
|
|
2868
|
+
lastDeferralSignature = signature;
|
|
2869
|
+
persistUpdaterState("record update deferral", () => recordUpdateDeferred(result.latestVersion, reason, activity?.activeRequests ?? null));
|
|
2870
|
+
}
|
|
2871
|
+
logger.debug(`[updater] ${reason} (${activity?.activeRequests ?? "unknown"} active requests)`);
|
|
2872
|
+
},
|
|
2873
|
+
});
|
|
2874
|
+
drainActive = drainActive || window.draining;
|
|
2875
|
+
return window;
|
|
2876
|
+
};
|
|
2877
|
+
let updateWindow = await waitForWindow(NATURAL_WINDOW_WAIT_MS);
|
|
2463
2878
|
if (!updateWindow.ready) {
|
|
2879
|
+
if (updateWindow.reason === "drain_failed") {
|
|
2880
|
+
persistUpdaterState("record unavailable update drain", () => recordUpdateDeferred(result.latestVersion, "drain_unavailable", null));
|
|
2881
|
+
}
|
|
2464
2882
|
scheduleUpdateRetry();
|
|
2465
2883
|
return;
|
|
2466
2884
|
}
|
|
2885
|
+
// Hold admission closed through install and restart after a quiet
|
|
2886
|
+
// observation, closing the race with a newly admitted request.
|
|
2887
|
+
if (!drainActive) {
|
|
2888
|
+
updateWindow = await waitForWindow(0);
|
|
2889
|
+
if (!updateWindow.ready) {
|
|
2890
|
+
scheduleUpdateRetry();
|
|
2891
|
+
return;
|
|
2892
|
+
}
|
|
2893
|
+
}
|
|
2894
|
+
}
|
|
2895
|
+
else {
|
|
2896
|
+
logger.always(`[updater] rolling supervisor will keep v${runningVersion} serving during installation`);
|
|
2467
2897
|
}
|
|
2468
2898
|
// Refresh after waiting so a long deferral can never install a stale
|
|
2469
2899
|
// target while a newer release is already available.
|
|
@@ -2481,7 +2911,7 @@ export const proxyGuardCommand = {
|
|
|
2481
2911
|
return;
|
|
2482
2912
|
}
|
|
2483
2913
|
persistUpdaterState("clear update deferral", () => clearUpdateDeferral());
|
|
2484
|
-
logger.always(`[updater] safe update window acquired for v${result.latestVersion}`);
|
|
2914
|
+
logger.always(`[updater] ${rollingSupervisor ? "rolling activation prepared" : "safe update window acquired"} for v${result.latestVersion}`);
|
|
2485
2915
|
// 3. Install update (validate version string before passing to shell)
|
|
2486
2916
|
if (!/^\d+\.\d+\.\d+$/.test(result.latestVersion)) {
|
|
2487
2917
|
const message = `invalid version format: ${result.latestVersion}`;
|
|
@@ -2554,31 +2984,47 @@ export const proxyGuardCommand = {
|
|
|
2554
2984
|
if (guardStopping || getProcessStatus(parentPid) === "not_running") {
|
|
2555
2985
|
return;
|
|
2556
2986
|
}
|
|
2557
|
-
// Admission has remained closed since active requests reached zero, so
|
|
2558
|
-
// restart cannot interrupt an accepted request or long-lived stream.
|
|
2559
|
-
// Signal the health loop to not exit when it detects
|
|
2560
|
-
// the parent PID is gone — we're intentionally restarting.
|
|
2561
2987
|
updateRestartInProgress = true;
|
|
2562
|
-
|
|
2563
|
-
|
|
2564
|
-
|
|
2565
|
-
|
|
2566
|
-
|
|
2567
|
-
|
|
2568
|
-
|
|
2988
|
+
if (rollingSupervisor) {
|
|
2989
|
+
logger.always(`[updater] requesting rolling activation of v${result.latestVersion}`);
|
|
2990
|
+
try {
|
|
2991
|
+
process.kill(parentPid, "SIGUSR2");
|
|
2992
|
+
}
|
|
2993
|
+
catch (activationError) {
|
|
2994
|
+
updateRestartInProgress = false;
|
|
2995
|
+
const message = activationError instanceof Error
|
|
2996
|
+
? activationError.message
|
|
2997
|
+
: String(activationError);
|
|
2998
|
+
logger.always(`[updater] WARNING: rolling activation request failed: ${message}`);
|
|
2999
|
+
persistUpdaterState("record update failure", () => recordUpdateFailure(result.latestVersion, "restart", message));
|
|
3000
|
+
return;
|
|
3001
|
+
}
|
|
2569
3002
|
}
|
|
2570
|
-
|
|
2571
|
-
|
|
2572
|
-
|
|
2573
|
-
|
|
2574
|
-
|
|
2575
|
-
|
|
2576
|
-
|
|
2577
|
-
|
|
3003
|
+
else {
|
|
3004
|
+
// Compatibility fallback for services installed before rolling
|
|
3005
|
+
// supervision was enabled.
|
|
3006
|
+
logger.always(`[updater] restarting proxy via launchctl kickstart`);
|
|
3007
|
+
const uid = process.getuid?.() ?? 501;
|
|
3008
|
+
try {
|
|
3009
|
+
execFileSync("launchctl", ["kickstart", "-k", `gui/${uid}/${PLIST_LABEL}`], {
|
|
3010
|
+
timeout: 10_000,
|
|
3011
|
+
stdio: "pipe",
|
|
3012
|
+
});
|
|
3013
|
+
}
|
|
3014
|
+
catch (restartErr) {
|
|
3015
|
+
updateRestartInProgress = false;
|
|
3016
|
+
const message = restartErr instanceof Error
|
|
3017
|
+
? restartErr.message
|
|
3018
|
+
: String(restartErr);
|
|
3019
|
+
logger.always(`[updater] WARNING: launchctl kickstart failed: ${message}`);
|
|
3020
|
+
persistUpdaterState("record update failure", () => recordUpdateFailure(result.latestVersion, "restart", message));
|
|
3021
|
+
return;
|
|
3022
|
+
}
|
|
2578
3023
|
}
|
|
2579
3024
|
drainActive = false;
|
|
2580
3025
|
// 5. Wait for healthy restart
|
|
2581
3026
|
let healthy = false;
|
|
3027
|
+
let rollingFailure = null;
|
|
2582
3028
|
const restartStart = Date.now();
|
|
2583
3029
|
while (Date.now() - restartStart < UPDATE_TIMEOUT_MS) {
|
|
2584
3030
|
await new Promise((r) => setTimeout(r, 2000));
|
|
@@ -2597,6 +3043,12 @@ export const proxyGuardCommand = {
|
|
|
2597
3043
|
catch {
|
|
2598
3044
|
/* retry */
|
|
2599
3045
|
}
|
|
3046
|
+
if (rollingSupervisor) {
|
|
3047
|
+
rollingFailure = await getRollingActivationFailure(host, port, result.latestVersion);
|
|
3048
|
+
if (rollingFailure) {
|
|
3049
|
+
break;
|
|
3050
|
+
}
|
|
3051
|
+
}
|
|
2600
3052
|
}
|
|
2601
3053
|
if (healthy) {
|
|
2602
3054
|
logger.always(`[updater] update successful: now running ${result.latestVersion}`);
|
|
@@ -2606,7 +3058,49 @@ export const proxyGuardCommand = {
|
|
|
2606
3058
|
}
|
|
2607
3059
|
else {
|
|
2608
3060
|
logger.always(`[updater] WARNING: proxy unhealthy after update to ${result.latestVersion}`);
|
|
2609
|
-
|
|
3061
|
+
let failureMessage = rollingFailure
|
|
3062
|
+
? `rolling candidate failed: ${rollingFailure}`
|
|
3063
|
+
: `proxy did not report v${result.latestVersion} healthy within ${UPDATE_TIMEOUT_MS}ms`;
|
|
3064
|
+
if (rollingSupervisor) {
|
|
3065
|
+
try {
|
|
3066
|
+
const rollbackResolution = resolveGlobalInstaller({
|
|
3067
|
+
entryScript: process.argv[1],
|
|
3068
|
+
});
|
|
3069
|
+
const rollbackInstaller = rollbackResolution.installer;
|
|
3070
|
+
if (!rollbackInstaller) {
|
|
3071
|
+
throw new Error("no usable package manager is available for rollback");
|
|
3072
|
+
}
|
|
3073
|
+
logger.always(`[updater] rolling activation failed; restoring @juspay/neurolink@${runningVersion}`);
|
|
3074
|
+
execFileSync(rollbackInstaller.bin, getGlobalInstallArgs(rollbackInstaller.kind, `@juspay/neurolink@${runningVersion}`), { timeout: 120_000, stdio: "pipe" });
|
|
3075
|
+
writeTrampoline();
|
|
3076
|
+
const rollbackValidation = await validateInstalledVersion({
|
|
3077
|
+
binPath: TRAMPOLINE_PATH,
|
|
3078
|
+
expectedVersion: runningVersion,
|
|
3079
|
+
});
|
|
3080
|
+
if (rollbackValidation.version !== runningVersion) {
|
|
3081
|
+
throw new Error(`rollback trampoline reported v${rollbackValidation.version ?? "unknown"}; expected v${runningVersion}`);
|
|
3082
|
+
}
|
|
3083
|
+
// Reinstalling the old package is not enough: the supervisor was
|
|
3084
|
+
// told to activate result.latestVersion, so its recovery loop
|
|
3085
|
+
// keeps demanding the new version against the now-downgraded
|
|
3086
|
+
// installation and can never serve a worker again. Re-point the
|
|
3087
|
+
// supervisor at runningVersion and report success only once it
|
|
3088
|
+
// actually serves that version again.
|
|
3089
|
+
const rollbackActivated = await activateRollbackVersion(host, port, parentPid, runningVersion, UPDATE_TIMEOUT_MS);
|
|
3090
|
+
if (!rollbackActivated) {
|
|
3091
|
+
throw new Error(`supervisor did not report v${runningVersion} healthy within ${UPDATE_TIMEOUT_MS}ms after rollback`);
|
|
3092
|
+
}
|
|
3093
|
+
logger.always(`[updater] package rollback complete; v${runningVersion} active again`);
|
|
3094
|
+
}
|
|
3095
|
+
catch (rollbackError) {
|
|
3096
|
+
const rollbackMessage = rollbackError instanceof Error
|
|
3097
|
+
? rollbackError.message
|
|
3098
|
+
: String(rollbackError);
|
|
3099
|
+
failureMessage += `; package rollback failed: ${rollbackMessage}`;
|
|
3100
|
+
logger.always(`[updater] WARNING: ${failureMessage}`);
|
|
3101
|
+
}
|
|
3102
|
+
}
|
|
3103
|
+
persistUpdaterState("record update failure", () => recordUpdateFailure(result.latestVersion, "health", failureMessage));
|
|
2610
3104
|
persistUpdaterState("abandon unhealthy pending update", () => abandonPendingUpdate(result.latestVersion));
|
|
2611
3105
|
persistUpdaterState("suppress unhealthy update", () => suppressVersion(result.latestVersion, "unhealthy_after_restart"));
|
|
2612
3106
|
updateRestartInProgress = false;
|
|
@@ -3058,28 +3552,6 @@ export const proxyInstallCommand = {
|
|
|
3058
3552
|
console.info(chalk.red(`Failed to load service: ${e}`));
|
|
3059
3553
|
process.exit(1);
|
|
3060
3554
|
}
|
|
3061
|
-
// Wait briefly for launchd to start the process, then persist state
|
|
3062
|
-
await new Promise((resolve) => setTimeout(resolve, 2_000));
|
|
3063
|
-
try {
|
|
3064
|
-
const { execFileSync } = await import("node:child_process");
|
|
3065
|
-
const uid = process.getuid?.() ?? 501;
|
|
3066
|
-
const output = execFileSync("launchctl", ["print", `gui/${uid}/${PLIST_LABEL}`], { encoding: "utf-8", stdio: ["pipe", "pipe", "pipe"] });
|
|
3067
|
-
const pidMatch = output.match(/pid\s*=\s*(\d+)/);
|
|
3068
|
-
if (pidMatch) {
|
|
3069
|
-
saveProxyState({
|
|
3070
|
-
pid: Number(pidMatch[1]),
|
|
3071
|
-
port,
|
|
3072
|
-
host,
|
|
3073
|
-
strategy: "fill-first",
|
|
3074
|
-
startTime: new Date().toISOString(),
|
|
3075
|
-
envFile,
|
|
3076
|
-
managedBy: "launchd",
|
|
3077
|
-
});
|
|
3078
|
-
}
|
|
3079
|
-
}
|
|
3080
|
-
catch {
|
|
3081
|
-
/* non-fatal — state will be written by the proxy process itself */
|
|
3082
|
-
}
|
|
3083
3555
|
console.info("");
|
|
3084
3556
|
console.info(chalk.bold("Proxy is now a persistent service:"));
|
|
3085
3557
|
console.info(` • Auto-starts on login`);
|
|
@@ -3102,6 +3574,12 @@ export const proxyUninstallCommand = {
|
|
|
3102
3574
|
}
|
|
3103
3575
|
const { existsSync, unlinkSync } = await import("fs");
|
|
3104
3576
|
if (!existsSync(PLIST_PATH)) {
|
|
3577
|
+
if (!(await ensureSupervisorStoppedBeforeClear())) {
|
|
3578
|
+
console.info(chalk.red("A proxy supervisor is still running and could not be stopped; leaving its state intact. Stop it manually and retry."));
|
|
3579
|
+
process.exit(1);
|
|
3580
|
+
}
|
|
3581
|
+
clearProxyState();
|
|
3582
|
+
clearProxySupervisorState();
|
|
3105
3583
|
console.info(chalk.yellow("No proxy service installed."));
|
|
3106
3584
|
return;
|
|
3107
3585
|
}
|
|
@@ -3113,7 +3591,16 @@ export const proxyUninstallCommand = {
|
|
|
3113
3591
|
catch {
|
|
3114
3592
|
/* may not be loaded */
|
|
3115
3593
|
}
|
|
3594
|
+
// Do not delete supervisor state while a supervisor still owns the
|
|
3595
|
+
// listener: a swallowed unload failure (or a supervisor started outside
|
|
3596
|
+
// launchd) would otherwise be orphaned.
|
|
3597
|
+
if (!(await ensureSupervisorStoppedBeforeClear())) {
|
|
3598
|
+
console.info(chalk.red("launchctl could not stop the running supervisor; aborting uninstall so its state is preserved. Stop it manually and retry."));
|
|
3599
|
+
process.exit(1);
|
|
3600
|
+
}
|
|
3116
3601
|
unlinkSync(PLIST_PATH);
|
|
3602
|
+
clearProxyState();
|
|
3603
|
+
clearProxySupervisorState();
|
|
3117
3604
|
console.info(chalk.green(`✓ Plist removed from ${PLIST_PATH}`));
|
|
3118
3605
|
console.info(chalk.green(`✓ Proxy service uninstalled`));
|
|
3119
3606
|
},
|