@juspay/neurolink 12.14.5 → 12.14.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +2 -2
- package/dist/browser/neurolink.min.js +225 -225
- package/dist/cli/commands/proxy.d.ts +1 -1
- package/dist/cli/commands/proxy.js +47 -1
- package/dist/cli/commands/proxyRestart.d.ts +3 -0
- package/dist/cli/commands/proxyRestart.js +88 -0
- package/dist/cli/parser.js +3 -1
- package/dist/proxy/restartControl.d.ts +12 -0
- package/dist/proxy/restartControl.js +283 -0
- package/dist/proxy/rollingWorkerSupervisor.js +8 -0
- package/dist/types/index.d.ts +1 -0
- package/dist/types/index.js +1 -0
- package/dist/types/proxyRestart.d.ts +49 -0
- package/dist/types/proxyRestart.js +1 -0
- package/docs-site/static/search-index.json +1 -1
- package/package.json +2 -1
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
*/
|
|
12
12
|
import type { CommandModule } from "yargs";
|
|
13
13
|
import type { Hono } from "hono";
|
|
14
|
-
import type { AccountAllowlist, LoadedProxyConfig, ModelRouterInterface, ProxyGuardArgs, ProxyHealthProbe, ProxyNeurolinkRuntime, ProxyStartArgs, ProxyStartStrategy, ProxySupervisorState, ProxyStatusArgs, ProxyTelemetryArgs, ProxyReadinessState } from "../../types/index.js";
|
|
14
|
+
import type { AccountAllowlist, LoadedProxyConfig, ModelRouterInterface, ProxyGuardArgs, ProxyHealthProbe, ProxyNeurolinkRuntime, ProxyStartArgs, ProxyStartStrategy, ProxyRestartSupervisorState as ProxySupervisorState, ProxyStatusArgs, ProxyTelemetryArgs, ProxyReadinessState } from "../../types/index.js";
|
|
15
15
|
import { ProxyRuntimeConfigStore } from "../../proxy/runtimeConfig.js";
|
|
16
16
|
/**
|
|
17
17
|
* Drop a supervisor `version` that is not a string.
|
|
@@ -24,6 +24,7 @@ import { describeInstallFailure, getGlobalInstallArgs, isTransientInstallFailure
|
|
|
24
24
|
import { startUpdaterWorkerSupervisor } from "../../proxy/updaterSupervisor.js";
|
|
25
25
|
import { openProxyWorkerLog } from "../../proxy/workerLog.js";
|
|
26
26
|
import { startRollingProxyServer } from "../../proxy/rollingProxyServer.js";
|
|
27
|
+
import { startProxyRestartControl } from "../../proxy/restartControl.js";
|
|
27
28
|
import { isProxyAuxiliaryRequest } from "../../proxy/proxyRequestKind.js";
|
|
28
29
|
import { spawnProxySocketWorker } from "../../proxy/rollingWorkerProcess.js";
|
|
29
30
|
import { PROXY_ROLLING_SUPERVISOR_ENV, PROXY_SOCKET_WORKER_ENV, } from "../../proxy/rollingWorkerProtocol.js";
|
|
@@ -1400,6 +1401,15 @@ export async function createProxyStartApp(params) {
|
|
|
1400
1401
|
.json()
|
|
1401
1402
|
.catch(() => ({ action: undefined }));
|
|
1402
1403
|
if (payload.action === "drain") {
|
|
1404
|
+
// A rolling worker must keep admitting until the supervisor has a ready
|
|
1405
|
+
// replacement. The legacy global drain can otherwise strand the listener
|
|
1406
|
+
// behind maintenance responses when its caller stalls or disappears.
|
|
1407
|
+
if (isProxySocketWorkerProcess()) {
|
|
1408
|
+
return c.json({
|
|
1409
|
+
error: "rolling_restart_required",
|
|
1410
|
+
message: "Use neurolink proxy restart; global update drain is disabled for rolling workers.",
|
|
1411
|
+
}, 409);
|
|
1412
|
+
}
|
|
1403
1413
|
if (!markProxyDrainingForUpdate(readiness)) {
|
|
1404
1414
|
return c.json({ error: "proxy_not_ready" }, 409);
|
|
1405
1415
|
}
|
|
@@ -2777,6 +2787,7 @@ async function runLaunchdProxySupervisor(argv, spinner) {
|
|
|
2777
2787
|
filePrefix: "proxy-supervisor",
|
|
2778
2788
|
});
|
|
2779
2789
|
let currentUpdaterPid;
|
|
2790
|
+
let restartControl;
|
|
2780
2791
|
const rollingServer = await startRollingProxyServer({
|
|
2781
2792
|
onEvent: (event) => logProxyLifecycleEvent({
|
|
2782
2793
|
event: "supervisor_event",
|
|
@@ -2807,6 +2818,7 @@ async function runLaunchdProxySupervisor(argv, spinner) {
|
|
|
2807
2818
|
version: PROXY_VERSION,
|
|
2808
2819
|
updaterPid: currentUpdaterPid,
|
|
2809
2820
|
rolling: snapshot,
|
|
2821
|
+
restartControl: restartControl?.identity,
|
|
2810
2822
|
});
|
|
2811
2823
|
},
|
|
2812
2824
|
log: (message) => logger.always(message),
|
|
@@ -2843,13 +2855,46 @@ async function runLaunchdProxySupervisor(argv, spinner) {
|
|
|
2843
2855
|
};
|
|
2844
2856
|
process.on("SIGUSR2", activatePendingUpdate);
|
|
2845
2857
|
const readinessHost = host === "0.0.0.0" ? "127.0.0.1" : host;
|
|
2858
|
+
try {
|
|
2859
|
+
restartControl = await startProxyRestartControl({
|
|
2860
|
+
stateDir: join(homedir(), ".neurolink"),
|
|
2861
|
+
server: rollingServer,
|
|
2862
|
+
log: (message) => logger.warn(message),
|
|
2863
|
+
isUpdatePending: () => !!rollingReplacement || !!loadUpdateState()?.pendingRestartVersion,
|
|
2864
|
+
getInstalledVersion: async () => {
|
|
2865
|
+
const { execFile } = await import("node:child_process");
|
|
2866
|
+
const { promisify } = await import("node:util");
|
|
2867
|
+
const { stdout } = await promisify(execFile)(TRAMPOLINE_PATH, ["--version"], { timeout: 5_000, maxBuffer: 65_536 });
|
|
2868
|
+
return stdout.trim();
|
|
2869
|
+
},
|
|
2870
|
+
getStatus: async () => {
|
|
2871
|
+
const probeHost = readinessHost === "::" ? "::1" : readinessHost;
|
|
2872
|
+
const response = await fetch(`http://${probeHost.includes(":") ? `[${probeHost}]` : probeHost}:${rollingServer.address.port}/status`, {
|
|
2873
|
+
headers: { connection: "close" },
|
|
2874
|
+
signal: AbortSignal.timeout(3_000),
|
|
2875
|
+
});
|
|
2876
|
+
if (!response.ok) {
|
|
2877
|
+
throw new Error("Serving status is unavailable.");
|
|
2878
|
+
}
|
|
2879
|
+
return response.json();
|
|
2880
|
+
},
|
|
2881
|
+
});
|
|
2882
|
+
}
|
|
2883
|
+
catch (error) {
|
|
2884
|
+
// A control-plane setup failure must not stop an otherwise serving proxy.
|
|
2885
|
+
logger.warn(`[proxy-supervisor] safe restart control unavailable: ${error instanceof Error ? error.message : String(error)}`);
|
|
2886
|
+
}
|
|
2846
2887
|
const updatePersistedUpdaterPid = (updaterPid) => {
|
|
2847
2888
|
currentUpdaterPid = updaterPid;
|
|
2848
2889
|
const state = loadProxySupervisorState();
|
|
2849
2890
|
if (!state || state.pid !== process.pid) {
|
|
2850
2891
|
return;
|
|
2851
2892
|
}
|
|
2852
|
-
saveProxySupervisorState({
|
|
2893
|
+
saveProxySupervisorState({
|
|
2894
|
+
...state,
|
|
2895
|
+
updaterPid,
|
|
2896
|
+
restartControl: restartControl?.identity,
|
|
2897
|
+
});
|
|
2853
2898
|
};
|
|
2854
2899
|
const updaterSupervisor = startUpdaterWorkerSupervisor({
|
|
2855
2900
|
spawnWorker: () => spawnProxyUpdater(readinessHost, rollingServer.address.port, process.pid, true),
|
|
@@ -2872,6 +2917,7 @@ async function runLaunchdProxySupervisor(argv, spinner) {
|
|
|
2872
2917
|
stopping = true;
|
|
2873
2918
|
logger.always(`[proxy-supervisor] shutting down (${signal})`);
|
|
2874
2919
|
process.off("SIGUSR2", activatePendingUpdate);
|
|
2920
|
+
await restartControl?.close();
|
|
2875
2921
|
updaterSupervisor.stop();
|
|
2876
2922
|
await rollingServer.close();
|
|
2877
2923
|
await flushProxyLifecycleEvents().catch((error) => logger.warn(String(error)));
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
import { readFile, lstat } from "node:fs/promises";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
import { z } from "zod";
|
|
4
|
+
import { resolveProxyPaths } from "../../proxy/proxyPaths.js";
|
|
5
|
+
import { requestProxyRestart } from "../../proxy/restartControl.js";
|
|
6
|
+
const supervisorSchema = z.object({
|
|
7
|
+
pid: z.number().int().positive(),
|
|
8
|
+
restartControl: z.object({
|
|
9
|
+
protocol: z.literal(1),
|
|
10
|
+
socketPath: z.string(),
|
|
11
|
+
instanceId: z.string().uuid(),
|
|
12
|
+
}),
|
|
13
|
+
});
|
|
14
|
+
export const proxyRestartCommand = {
|
|
15
|
+
command: "restart",
|
|
16
|
+
describe: "Replace the serving worker without stopping the proxy listener",
|
|
17
|
+
builder: (yargs) => yargs
|
|
18
|
+
.option("check", {
|
|
19
|
+
type: "boolean",
|
|
20
|
+
default: false,
|
|
21
|
+
description: "Check restart readiness without changing the running proxy",
|
|
22
|
+
})
|
|
23
|
+
.option("dev", {
|
|
24
|
+
type: "boolean",
|
|
25
|
+
default: false,
|
|
26
|
+
description: "Use the isolated proxy state in .neurolink-dev/",
|
|
27
|
+
})
|
|
28
|
+
.option("format", {
|
|
29
|
+
type: "string",
|
|
30
|
+
choices: ["text", "json"],
|
|
31
|
+
default: "text",
|
|
32
|
+
})
|
|
33
|
+
.example("neurolink proxy restart --check", "Verify the running supervisor supports safe worker replacement")
|
|
34
|
+
.example("neurolink proxy restart", "Start a replacement, verify it, then let existing streams finish"),
|
|
35
|
+
handler: async (argv) => {
|
|
36
|
+
try {
|
|
37
|
+
const { stateDir } = resolveProxyPaths(argv.dev);
|
|
38
|
+
const saved = supervisorSchema.safeParse(JSON.parse(await readFile(join(stateDir, "proxy-supervisor-state.json"), "utf8")));
|
|
39
|
+
if (!saved.success) {
|
|
40
|
+
throw new Error("This running supervisor does not advertise safe restart support. It must first be upgraded through a separately planned service activation.");
|
|
41
|
+
}
|
|
42
|
+
const { pid, restartControl } = saved.data;
|
|
43
|
+
if (restartControl.socketPath !==
|
|
44
|
+
join(stateDir, `restart-${pid}-${restartControl.instanceId}.sock`)) {
|
|
45
|
+
throw new Error("Restart control path does not match the recorded supervisor.");
|
|
46
|
+
}
|
|
47
|
+
const socket = await lstat(restartControl.socketPath);
|
|
48
|
+
if (!socket.isSocket() ||
|
|
49
|
+
(socket.mode & 0o077) !== 0 ||
|
|
50
|
+
(process.getuid && socket.uid !== process.getuid())) {
|
|
51
|
+
throw new Error("Restart control socket ownership or permissions are invalid.");
|
|
52
|
+
}
|
|
53
|
+
const result = await requestProxyRestart(restartControl, argv.check);
|
|
54
|
+
if (result.supervisorPid !== pid) {
|
|
55
|
+
throw new Error("Restart control identity changed; outcome is unverified.");
|
|
56
|
+
}
|
|
57
|
+
if (argv.format === "json") {
|
|
58
|
+
console.info(JSON.stringify(result, null, 2));
|
|
59
|
+
}
|
|
60
|
+
else {
|
|
61
|
+
console.info(result.message);
|
|
62
|
+
console.info(`Supervisor ${result.supervisorPid}; worker ${result.workerPid ?? "unknown"}; version ${result.version ?? "unknown"}; previous workers finishing requests: ${result.drainingWorkers}.`);
|
|
63
|
+
console.info("This operation preserves the supervisor and launchd settings. It does not reinstall the service.");
|
|
64
|
+
}
|
|
65
|
+
if (!result.ok) {
|
|
66
|
+
process.exitCode = 1;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
catch (error) {
|
|
70
|
+
const message = error instanceof Error
|
|
71
|
+
? error.message
|
|
72
|
+
: "Restart control is unavailable.";
|
|
73
|
+
if (argv.format === "json") {
|
|
74
|
+
const result = {
|
|
75
|
+
ok: false,
|
|
76
|
+
phase: "unverified",
|
|
77
|
+
message,
|
|
78
|
+
};
|
|
79
|
+
console.info(JSON.stringify(result));
|
|
80
|
+
}
|
|
81
|
+
else {
|
|
82
|
+
console.error(message);
|
|
83
|
+
}
|
|
84
|
+
process.exitCode = 1;
|
|
85
|
+
}
|
|
86
|
+
},
|
|
87
|
+
};
|
|
88
|
+
//# sourceMappingURL=proxyRestart.js.map
|
package/dist/cli/parser.js
CHANGED
|
@@ -23,6 +23,7 @@ import { ObservabilityCommandFactory } from "./commands/observability.js";
|
|
|
23
23
|
import { TelemetryCommandFactory } from "./commands/telemetry.js";
|
|
24
24
|
import { proxyStartCommand, proxyStatusCommand, proxyTelemetryCommand, proxySetupCommand, proxyGuardCommand, proxyInstallCommand, proxyUninstallCommand, } from "./commands/proxy.js";
|
|
25
25
|
import { proxyAnalyzeCommand } from "./commands/proxyAnalyze.js";
|
|
26
|
+
import { proxyRestartCommand } from "./commands/proxyRestart.js";
|
|
26
27
|
import { proxyShareCommand } from "./commands/proxyShare.js";
|
|
27
28
|
import { proxyPeerCommand } from "./commands/proxyPeer.js";
|
|
28
29
|
import { proxyExposeCommand } from "./commands/proxyExpose.js";
|
|
@@ -218,6 +219,7 @@ export function initializeCliParser() {
|
|
|
218
219
|
builder: (yargs) => yargs
|
|
219
220
|
.command(proxyStartCommand)
|
|
220
221
|
.command(proxyStatusCommand)
|
|
222
|
+
.command(proxyRestartCommand)
|
|
221
223
|
.command(proxyShareCommand)
|
|
222
224
|
.command(proxyPeerCommand)
|
|
223
225
|
.command(proxyExposeCommand)
|
|
@@ -228,7 +230,7 @@ export function initializeCliParser() {
|
|
|
228
230
|
.command(proxyGuardCommand)
|
|
229
231
|
.command(proxyInstallCommand)
|
|
230
232
|
.command(proxyUninstallCommand)
|
|
231
|
-
.demandCommand(1, "Please specify a proxy subcommand: start, status, share <create|provision|url|list|status|pause|resume|revoke|topup|set|link|rotate|level|note|notes|receipts|delete>, peer <add|request|sync|receipts|net|redeem|list|status|test|remove|pause|resume|set>, expose, analyze, replay <export|compare>, telemetry <setup|start|stop|status|logs|import-dashboard>, setup, guard, install, or uninstall"),
|
|
233
|
+
.demandCommand(1, "Please specify a proxy subcommand: start, status, restart, share <create|provision|url|list|status|pause|resume|revoke|topup|set|link|rotate|level|note|notes|receipts|delete>, peer <add|request|sync|receipts|net|redeem|list|status|test|remove|pause|resume|set>, expose, analyze, replay <export|compare>, telemetry <setup|start|stop|status|logs|import-dashboard>, setup, guard, install, or uninstall"),
|
|
232
234
|
handler: () => { },
|
|
233
235
|
})
|
|
234
236
|
// Evaluate Command Group - Using EvaluateCommandFactory
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { ProxyRestartControlIdentity, ProxyRestartControlOptions, ProxyRestartResult } from "../types/index.js";
|
|
2
|
+
/**
|
|
3
|
+
* Own restart completion in the supervisor, so a disconnected CLI cannot leave
|
|
4
|
+
* admission closed. The control socket is private to the service's OS user.
|
|
5
|
+
* No update history, launcher, environment file or launchd unit is rewritten.
|
|
6
|
+
*/
|
|
7
|
+
export declare function startProxyRestartControl(options: ProxyRestartControlOptions): Promise<{
|
|
8
|
+
identity: ProxyRestartControlIdentity;
|
|
9
|
+
close: () => Promise<void>;
|
|
10
|
+
}>;
|
|
11
|
+
/** Perform one local control operation; never fall back to killing a process. */
|
|
12
|
+
export declare function requestProxyRestart(identity: ProxyRestartControlIdentity, check: boolean): Promise<ProxyRestartResult>;
|
|
@@ -0,0 +1,283 @@
|
|
|
1
|
+
import { createServer, request } from "node:http";
|
|
2
|
+
import { chmod, mkdir, rm } from "node:fs/promises";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
import { randomUUID } from "node:crypto";
|
|
5
|
+
import { z } from "zod";
|
|
6
|
+
const statusSchema = z.object({
|
|
7
|
+
pid: z.number().int().positive(),
|
|
8
|
+
version: z.string(),
|
|
9
|
+
health: z.object({
|
|
10
|
+
ready: z.literal(true),
|
|
11
|
+
acceptingConnections: z.literal(true),
|
|
12
|
+
drainingForUpdate: z.literal(false),
|
|
13
|
+
}),
|
|
14
|
+
observability: z.object({
|
|
15
|
+
requestLogs: z.object({
|
|
16
|
+
diskEnabled: z.boolean(),
|
|
17
|
+
otel: z.object({ initialized: z.boolean() }),
|
|
18
|
+
}),
|
|
19
|
+
}),
|
|
20
|
+
});
|
|
21
|
+
const resultSchema = z.object({
|
|
22
|
+
ok: z.boolean(),
|
|
23
|
+
phase: z.enum([
|
|
24
|
+
"checked",
|
|
25
|
+
"refused",
|
|
26
|
+
"failed",
|
|
27
|
+
"activated",
|
|
28
|
+
"activated_unverified",
|
|
29
|
+
]),
|
|
30
|
+
message: z.string(),
|
|
31
|
+
supervisorPid: z.number().int().positive(),
|
|
32
|
+
previousWorkerPid: z.number().int().positive().optional(),
|
|
33
|
+
workerPid: z.number().int().positive().optional(),
|
|
34
|
+
version: z.string().optional(),
|
|
35
|
+
drainingWorkers: z.number().int().nonnegative(),
|
|
36
|
+
rejectedSocketsDelta: z.number().int().optional(),
|
|
37
|
+
failedTransfersDelta: z.number().int().optional(),
|
|
38
|
+
});
|
|
39
|
+
/**
|
|
40
|
+
* Own restart completion in the supervisor, so a disconnected CLI cannot leave
|
|
41
|
+
* admission closed. The control socket is private to the service's OS user.
|
|
42
|
+
* No update history, launcher, environment file or launchd unit is rewritten.
|
|
43
|
+
*/
|
|
44
|
+
export async function startProxyRestartControl(options) {
|
|
45
|
+
const instanceId = randomUUID();
|
|
46
|
+
const socketPath = join(options.stateDir, `restart-${process.pid}-${instanceId}.sock`);
|
|
47
|
+
await mkdir(options.stateDir, { recursive: true, mode: 0o700 });
|
|
48
|
+
// Never unlink an existing path: an unexpected owner must make setup fail.
|
|
49
|
+
let busy = false;
|
|
50
|
+
let closing = false;
|
|
51
|
+
const result = (phase, message) => {
|
|
52
|
+
const snapshot = options.server.snapshot();
|
|
53
|
+
return {
|
|
54
|
+
ok: phase === "checked" || phase === "activated",
|
|
55
|
+
phase,
|
|
56
|
+
message,
|
|
57
|
+
supervisorPid: process.pid,
|
|
58
|
+
workerPid: snapshot.active?.pid,
|
|
59
|
+
version: snapshot.active?.version,
|
|
60
|
+
drainingWorkers: snapshot.draining.length,
|
|
61
|
+
};
|
|
62
|
+
};
|
|
63
|
+
const assertIdle = () => {
|
|
64
|
+
const snapshot = options.server.snapshot();
|
|
65
|
+
if (closing || !snapshot.active) {
|
|
66
|
+
throw new Error("No serving worker is available for a rolling restart.");
|
|
67
|
+
}
|
|
68
|
+
if (snapshot.candidate || options.isUpdatePending()) {
|
|
69
|
+
throw new Error("An update or worker replacement is already in progress.");
|
|
70
|
+
}
|
|
71
|
+
if (snapshot.draining.length) {
|
|
72
|
+
throw new Error("A previous worker is still finishing requests; another restart is deferred.");
|
|
73
|
+
}
|
|
74
|
+
};
|
|
75
|
+
const operate = async (restart) => {
|
|
76
|
+
if (busy || closing) {
|
|
77
|
+
return result("refused", "A restart check or activation is already in progress.");
|
|
78
|
+
}
|
|
79
|
+
busy = true;
|
|
80
|
+
let activated = false;
|
|
81
|
+
let previousWorkerPid;
|
|
82
|
+
let handoffBaseline;
|
|
83
|
+
const transferDeltas = () => {
|
|
84
|
+
if (!handoffBaseline) {
|
|
85
|
+
return {};
|
|
86
|
+
}
|
|
87
|
+
const latest = options.server.snapshot();
|
|
88
|
+
return {
|
|
89
|
+
rejectedSocketsDelta: latest.rejectedSockets - handoffBaseline.rejectedSockets,
|
|
90
|
+
failedTransfersDelta: latest.failedTransfers - handoffBaseline.failedTransfers,
|
|
91
|
+
};
|
|
92
|
+
};
|
|
93
|
+
try {
|
|
94
|
+
assertIdle();
|
|
95
|
+
const before = options.server.snapshot();
|
|
96
|
+
const active = before.active;
|
|
97
|
+
if (!active) {
|
|
98
|
+
throw new Error("The serving worker exited during preflight.");
|
|
99
|
+
}
|
|
100
|
+
previousWorkerPid = active.pid;
|
|
101
|
+
const status = statusSchema.parse(await options.getStatus());
|
|
102
|
+
if (status.pid !== previousWorkerPid ||
|
|
103
|
+
status.version !== active.version) {
|
|
104
|
+
throw new Error("Serving worker identity changed during preflight; retry the check.");
|
|
105
|
+
}
|
|
106
|
+
const version = await options.getInstalledVersion();
|
|
107
|
+
if (!version || !/^\d+\.\d+\.\d+$/.test(version)) {
|
|
108
|
+
throw new Error("The configured worker executable did not report a valid installed version.");
|
|
109
|
+
}
|
|
110
|
+
assertIdle();
|
|
111
|
+
if (options.server.snapshot().active?.pid !== previousWorkerPid) {
|
|
112
|
+
throw new Error("Serving worker changed during preflight; retry the check.");
|
|
113
|
+
}
|
|
114
|
+
if (!restart) {
|
|
115
|
+
return {
|
|
116
|
+
...result("checked", "Rolling restart is available. Service settings and existing streams will be preserved."),
|
|
117
|
+
version,
|
|
118
|
+
};
|
|
119
|
+
}
|
|
120
|
+
// The existing supervisor imposes a 120-second candidate readiness
|
|
121
|
+
// deadline and retains the serving worker when startup/activation fails.
|
|
122
|
+
// It switches new sockets before draining the previous generation.
|
|
123
|
+
handoffBaseline = options.server.snapshot();
|
|
124
|
+
const after = await options.server.replace(version);
|
|
125
|
+
activated = true;
|
|
126
|
+
const fresh = statusSchema.parse(await options.getStatus());
|
|
127
|
+
if (fresh.pid !== after.active?.pid ||
|
|
128
|
+
fresh.pid === previousWorkerPid ||
|
|
129
|
+
fresh.version !== version) {
|
|
130
|
+
throw new Error("The replacement's serving identity could not be verified.");
|
|
131
|
+
}
|
|
132
|
+
const beforeLogs = status.observability.requestLogs;
|
|
133
|
+
const afterLogs = fresh.observability.requestLogs;
|
|
134
|
+
// Preserve the selected sink in both directions. Unexpected disk writes
|
|
135
|
+
// regress OTel-only deployments; disabling an existing disk sink loses
|
|
136
|
+
// requested logs. Restart must not silently make either policy change.
|
|
137
|
+
if (beforeLogs.diskEnabled !== afterLogs.diskEnabled ||
|
|
138
|
+
(beforeLogs.otel.initialized && !afterLogs.otel.initialized)) {
|
|
139
|
+
throw new Error("The replacement's logging state regressed; inspect telemetry before further changes.");
|
|
140
|
+
}
|
|
141
|
+
const latest = options.server.snapshot();
|
|
142
|
+
const deltas = transferDeltas();
|
|
143
|
+
// These are observed failures, not attribution to the restart. A ready
|
|
144
|
+
// worker can coexist with a handoff whose no-loss claim is unverified.
|
|
145
|
+
if (latest.active?.pid !== fresh.pid ||
|
|
146
|
+
deltas.rejectedSocketsDelta !== 0 ||
|
|
147
|
+
deltas.failedTransfersDelta !== 0) {
|
|
148
|
+
throw new Error("Worker identity or socket transfer counters changed during verification.");
|
|
149
|
+
}
|
|
150
|
+
return {
|
|
151
|
+
...result("activated", "Replacement is serving and accepting requests. Previous streams finish on their original worker."),
|
|
152
|
+
previousWorkerPid,
|
|
153
|
+
...deltas,
|
|
154
|
+
};
|
|
155
|
+
}
|
|
156
|
+
catch (error) {
|
|
157
|
+
// Schema errors contain paths only; never echo the status payload or env.
|
|
158
|
+
const message = error instanceof z.ZodError
|
|
159
|
+
? "Readiness, admission or logging status could not be verified."
|
|
160
|
+
: error instanceof Error
|
|
161
|
+
? error.message
|
|
162
|
+
: "Restart verification failed.";
|
|
163
|
+
return {
|
|
164
|
+
...result(activated ? "activated_unverified" : "failed", message),
|
|
165
|
+
previousWorkerPid,
|
|
166
|
+
...transferDeltas(),
|
|
167
|
+
};
|
|
168
|
+
}
|
|
169
|
+
finally {
|
|
170
|
+
busy = false;
|
|
171
|
+
}
|
|
172
|
+
};
|
|
173
|
+
const control = createServer((req, res) => {
|
|
174
|
+
// The instance nonce prevents a stale client from acting on a reused path.
|
|
175
|
+
if (closing || req.headers["x-neurolink-instance"] !== instanceId) {
|
|
176
|
+
res.writeHead(409).end();
|
|
177
|
+
return;
|
|
178
|
+
}
|
|
179
|
+
if (req.headers["transfer-encoding"] ||
|
|
180
|
+
(req.headers["content-length"] && req.headers["content-length"] !== "0")) {
|
|
181
|
+
res.writeHead(400, { connection: "close" }).end();
|
|
182
|
+
return;
|
|
183
|
+
}
|
|
184
|
+
if (!((req.method === "GET" && req.url === "/check") ||
|
|
185
|
+
(req.method === "POST" && req.url === "/restart"))) {
|
|
186
|
+
res.writeHead(404).end();
|
|
187
|
+
return;
|
|
188
|
+
}
|
|
189
|
+
req.resume();
|
|
190
|
+
void operate(req.method === "POST")
|
|
191
|
+
.then((outcome) => {
|
|
192
|
+
if (res.destroyed || res.writableEnded) {
|
|
193
|
+
return;
|
|
194
|
+
}
|
|
195
|
+
res.writeHead(outcome.ok ? 200 : 409, {
|
|
196
|
+
"content-type": "application/json",
|
|
197
|
+
connection: "close",
|
|
198
|
+
});
|
|
199
|
+
res.end(JSON.stringify(outcome));
|
|
200
|
+
})
|
|
201
|
+
.catch(() => {
|
|
202
|
+
if (res.destroyed || res.writableEnded) {
|
|
203
|
+
return;
|
|
204
|
+
}
|
|
205
|
+
res.writeHead(500).end();
|
|
206
|
+
});
|
|
207
|
+
});
|
|
208
|
+
control.headersTimeout = 5_000;
|
|
209
|
+
control.requestTimeout = 5_000;
|
|
210
|
+
control.maxHeadersCount = 10;
|
|
211
|
+
control.maxConnections = 16;
|
|
212
|
+
await new Promise((resolve, reject) => {
|
|
213
|
+
control.once("error", reject);
|
|
214
|
+
control.listen(socketPath, () => {
|
|
215
|
+
control.off("error", reject);
|
|
216
|
+
// Keep handling errors after binding: a control-plane accept failure
|
|
217
|
+
// must not become an unhandled event that stops the serving listener.
|
|
218
|
+
control.on("error", (error) => {
|
|
219
|
+
options.log?.(`[proxy-supervisor] safe restart control error: ${error.message}`);
|
|
220
|
+
});
|
|
221
|
+
resolve();
|
|
222
|
+
});
|
|
223
|
+
});
|
|
224
|
+
try {
|
|
225
|
+
await chmod(socketPath, 0o600);
|
|
226
|
+
}
|
|
227
|
+
catch (error) {
|
|
228
|
+
control.close();
|
|
229
|
+
throw error;
|
|
230
|
+
}
|
|
231
|
+
return {
|
|
232
|
+
identity: { protocol: 1, socketPath, instanceId },
|
|
233
|
+
close: async () => {
|
|
234
|
+
closing = true;
|
|
235
|
+
control.closeAllConnections();
|
|
236
|
+
await new Promise((resolve) => control.close(() => resolve()));
|
|
237
|
+
await rm(socketPath, { force: true });
|
|
238
|
+
},
|
|
239
|
+
};
|
|
240
|
+
}
|
|
241
|
+
/** Perform one local control operation; never fall back to killing a process. */
|
|
242
|
+
export async function requestProxyRestart(identity, check) {
|
|
243
|
+
return new Promise((resolve, reject) => {
|
|
244
|
+
const req = request({
|
|
245
|
+
socketPath: identity.socketPath,
|
|
246
|
+
path: check ? "/check" : "/restart",
|
|
247
|
+
method: check ? "GET" : "POST",
|
|
248
|
+
headers: {
|
|
249
|
+
"x-neurolink-instance": identity.instanceId,
|
|
250
|
+
connection: "close",
|
|
251
|
+
},
|
|
252
|
+
});
|
|
253
|
+
const timeout = setTimeout(() => req.destroy(new Error("Restart result timed out; outcome is unknown. Inspect proxy status before retrying.")), check ? 15_000 : 150_000);
|
|
254
|
+
req.once("error", (error) => {
|
|
255
|
+
clearTimeout(timeout);
|
|
256
|
+
reject(error);
|
|
257
|
+
});
|
|
258
|
+
req.once("response", (res) => {
|
|
259
|
+
let body = "";
|
|
260
|
+
res.setEncoding("utf8");
|
|
261
|
+
res.on("data", (chunk) => {
|
|
262
|
+
body += chunk;
|
|
263
|
+
if (body.length > 65_536) {
|
|
264
|
+
req.destroy(new Error("Invalid restart control response."));
|
|
265
|
+
}
|
|
266
|
+
});
|
|
267
|
+
res.once("error", (error) => {
|
|
268
|
+
clearTimeout(timeout);
|
|
269
|
+
reject(error);
|
|
270
|
+
});
|
|
271
|
+
res.once("end", () => {
|
|
272
|
+
clearTimeout(timeout);
|
|
273
|
+
try {
|
|
274
|
+
resolve(resultSchema.parse(JSON.parse(body)));
|
|
275
|
+
}
|
|
276
|
+
catch {
|
|
277
|
+
reject(new Error("Restart control did not return a verified result; no service restart fallback was attempted."));
|
|
278
|
+
}
|
|
279
|
+
});
|
|
280
|
+
});
|
|
281
|
+
req.end();
|
|
282
|
+
});
|
|
283
|
+
}
|
|
@@ -253,6 +253,14 @@ export class RollingWorkerSupervisor {
|
|
|
253
253
|
if (this.candidate?.generation === generation) {
|
|
254
254
|
this.candidate = null;
|
|
255
255
|
}
|
|
256
|
+
// A candidate that never became ready may also ignore SIGTERM.
|
|
257
|
+
// Bound its cleanup independently of the still-serving generation.
|
|
258
|
+
const killTimeout = setTimeout(() => handle.terminate("SIGKILL"), 1_000);
|
|
259
|
+
killTimeout.unref?.();
|
|
260
|
+
const offCandidateExit = handle.onExit(() => {
|
|
261
|
+
clearTimeout(killTimeout);
|
|
262
|
+
offCandidateExit();
|
|
263
|
+
});
|
|
256
264
|
handle.terminate("SIGTERM");
|
|
257
265
|
dispose();
|
|
258
266
|
this.publishState();
|
package/dist/types/index.d.ts
CHANGED
package/dist/types/index.js
CHANGED
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import type { RollingProxyServer } from "./proxy.js";
|
|
2
|
+
import type { ProxySupervisorState } from "./cli.js";
|
|
3
|
+
/** Supervisor state with the optional local restart capability. */
|
|
4
|
+
export type ProxyRestartSupervisorState = ProxySupervisorState & {
|
|
5
|
+
restartControl?: ProxyRestartControlIdentity;
|
|
6
|
+
};
|
|
7
|
+
export type ProxyRestartArgs = {
|
|
8
|
+
check: boolean;
|
|
9
|
+
dev: boolean;
|
|
10
|
+
format: "text" | "json";
|
|
11
|
+
};
|
|
12
|
+
/** Local supervisor control identity; contains no account credentials. */
|
|
13
|
+
export type ProxyRestartControlIdentity = {
|
|
14
|
+
protocol: 1;
|
|
15
|
+
socketPath: string;
|
|
16
|
+
instanceId: string;
|
|
17
|
+
};
|
|
18
|
+
/** Terminal result of a local restart check or worker activation. */
|
|
19
|
+
export type ProxyRestartResult = {
|
|
20
|
+
ok: boolean;
|
|
21
|
+
phase: "checked" | "refused" | "failed" | "activated" | "activated_unverified";
|
|
22
|
+
message: string;
|
|
23
|
+
supervisorPid: number;
|
|
24
|
+
previousWorkerPid?: number;
|
|
25
|
+
workerPid?: number;
|
|
26
|
+
version?: string;
|
|
27
|
+
drainingWorkers: number;
|
|
28
|
+
/** Observed during handoff/verification; does not attribute the cause. */
|
|
29
|
+
rejectedSocketsDelta?: number;
|
|
30
|
+
failedTransfersDelta?: number;
|
|
31
|
+
};
|
|
32
|
+
/** CLI failure before a supervisor result can be authenticated or received. */
|
|
33
|
+
export type CliProxyRestartError = {
|
|
34
|
+
ok: false;
|
|
35
|
+
phase: "unverified";
|
|
36
|
+
message: string;
|
|
37
|
+
};
|
|
38
|
+
/** JSON emitted by the restart CLI, including an unknown control outcome. */
|
|
39
|
+
export type CliProxyRestartOutput = ProxyRestartResult | CliProxyRestartError;
|
|
40
|
+
/** Supervisor-owned restart dependencies, injectable for isolated process tests. */
|
|
41
|
+
export type ProxyRestartControlOptions = {
|
|
42
|
+
stateDir: string;
|
|
43
|
+
server: RollingProxyServer;
|
|
44
|
+
getInstalledVersion: () => Promise<string | undefined>;
|
|
45
|
+
isUpdatePending: () => boolean;
|
|
46
|
+
getStatus: () => Promise<unknown>;
|
|
47
|
+
/** Report control-server errors without stopping the serving listener. */
|
|
48
|
+
log?: (message: string) => void;
|
|
49
|
+
};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|