@velum-labs/routekit-daemon 0.18.0 → 0.18.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -6
- package/dist/host-protocol.d.ts +88 -0
- package/dist/host-protocol.js +10 -0
- package/dist/host.d.ts +11 -0
- package/dist/host.js +556 -0
- package/dist/index.d.ts +41 -0
- package/dist/index.js +213 -71
- package/dist/worker.d.ts +2 -0
- package/dist/worker.js +168 -0
- package/package.json +9 -9
package/dist/host.js
ADDED
|
@@ -0,0 +1,556 @@
|
|
|
1
|
+
import { spawnSync } from "node:child_process";
|
|
2
|
+
import cluster from "node:cluster";
|
|
3
|
+
import { createServer } from "node:net";
|
|
4
|
+
import { isAbsolute, join } from "node:path";
|
|
5
|
+
import { routekitHome } from "@velum-labs/routekit-config";
|
|
6
|
+
import { acquireLifecycleLock, CONTROL_PROTOCOL_VERSION, createPortlessSession, createServiceRecordStore, extendCleanupGrace, generateControlToken, nextServiceGeneration, processIdentity, registerCleanup, supervisorFromEnv } from "@velum-labs/routekit-runtime";
|
|
7
|
+
import { createCliproxySidecar } from "./cliproxy-sidecar.js";
|
|
8
|
+
import { DAEMON_HOST_PROTOCOL_VERSION, ROUTEKIT_DAEMON_CONTROL_TOKEN_ENV, ROUTEKIT_DAEMON_CONTROL_PORT_ENV, ROUTEKIT_DAEMON_DATA_PORT_ENV, ROUTEKIT_DAEMON_DATA_URL_ENV, ROUTEKIT_DAEMON_GENERATION_ENV, ROUTEKIT_DAEMON_HOST_PID_ENV, ROUTEKIT_DAEMON_HOST_STARTED_AT_ENV, ROUTEKIT_DAEMON_INITIAL_PAUSED_ENV, ROUTEKIT_DAEMON_WORKER_ENV } from "./host-protocol.js";
|
|
9
|
+
import { readDaemonRevisions, removeDaemonPublicRecord, ROUTEKIT_DAEMON_KIND, ROUTEKIT_PRODUCT, writeDaemonPublicRecord, writeDaemonRevisions } from "./index.js";
|
|
10
|
+
const WORKER_READY_TIMEOUT_MS = 90_000;
|
|
11
|
+
const WORKER_REQUEST_TIMEOUT_MS = 120_000;
|
|
12
|
+
const RETIRE_FORCE_EXTRA_MS = 10_000;
|
|
13
|
+
async function reservePort(host) {
|
|
14
|
+
const server = createServer();
|
|
15
|
+
await new Promise((resolve, reject) => {
|
|
16
|
+
server.once("error", reject);
|
|
17
|
+
server.listen(0, host, () => resolve());
|
|
18
|
+
});
|
|
19
|
+
const address = server.address();
|
|
20
|
+
const port = typeof address === "object" && address !== null ? address.port : 0;
|
|
21
|
+
await new Promise((resolve) => server.close(() => resolve()));
|
|
22
|
+
if (port <= 0)
|
|
23
|
+
throw new Error("failed to reserve daemon port");
|
|
24
|
+
return port;
|
|
25
|
+
}
|
|
26
|
+
function loopbackUrl(host, port) {
|
|
27
|
+
return `http://${host.includes(":") ? `[${host}]` : host}:${port}`;
|
|
28
|
+
}
|
|
29
|
+
function candidateVersion(binPath) {
|
|
30
|
+
const result = spawnSync(process.execPath, [binPath, "--version"], {
|
|
31
|
+
encoding: "utf8",
|
|
32
|
+
timeout: 10_000
|
|
33
|
+
});
|
|
34
|
+
if (result.status !== 0)
|
|
35
|
+
return undefined;
|
|
36
|
+
const match = result.stdout.match(/(?:^|\s)(\d+\.\d+\.\d+(?:[-+][^\s]+)?)(?:\s|$)/);
|
|
37
|
+
return match?.[1];
|
|
38
|
+
}
|
|
39
|
+
export async function startRouteKitDaemonHost(options) {
|
|
40
|
+
if (!cluster.isPrimary)
|
|
41
|
+
throw new Error("daemon host must run as the cluster primary");
|
|
42
|
+
const env = options.env ?? process.env;
|
|
43
|
+
const home = options.stateHome ?? routekitHome(env);
|
|
44
|
+
const host = options.host ?? "127.0.0.1";
|
|
45
|
+
const dataPort = options.port === undefined || options.port === 0 ? await reservePort(host) : options.port;
|
|
46
|
+
const controlPort = await reservePort("127.0.0.1");
|
|
47
|
+
const drainGraceMs = options.drainGraceMs ?? 30_000;
|
|
48
|
+
const hostStartedAt = new Date().toISOString();
|
|
49
|
+
const controlToken = generateControlToken();
|
|
50
|
+
const store = createServiceRecordStore({ home, product: ROUTEKIT_PRODUCT });
|
|
51
|
+
const authority = await acquireLifecycleLock(join(store.directory, "daemon-authority.lock"));
|
|
52
|
+
const revisions = readDaemonRevisions(home);
|
|
53
|
+
const previous = store.read(ROUTEKIT_DAEMON_KIND);
|
|
54
|
+
let generation = nextServiceGeneration(Math.max(previous?.generation ?? 0, revisions.daemon));
|
|
55
|
+
revisions.daemon = generation;
|
|
56
|
+
writeDaemonRevisions(home, revisions);
|
|
57
|
+
let dataUrl = loopbackUrl(host, dataPort);
|
|
58
|
+
let rolling = false;
|
|
59
|
+
let active;
|
|
60
|
+
let closed = false;
|
|
61
|
+
let closeRun;
|
|
62
|
+
let rollRun;
|
|
63
|
+
let requestSequence = 0;
|
|
64
|
+
const pending = new Map();
|
|
65
|
+
const readyWaiters = new Map();
|
|
66
|
+
const sidecar = createCliproxySidecar({ env });
|
|
67
|
+
let sidecarTail = Promise.resolve();
|
|
68
|
+
let portless;
|
|
69
|
+
let record;
|
|
70
|
+
let recoveryRun;
|
|
71
|
+
let retirementFallback;
|
|
72
|
+
const sendResponse = (worker, response) => {
|
|
73
|
+
if (worker.isConnected())
|
|
74
|
+
worker.send(response);
|
|
75
|
+
};
|
|
76
|
+
const requestWorker = async (worker, request) => {
|
|
77
|
+
const requestId = `host-${++requestSequence}`;
|
|
78
|
+
const response = new Promise((resolve, reject) => {
|
|
79
|
+
const timer = setTimeout(() => {
|
|
80
|
+
pending.delete(requestId);
|
|
81
|
+
reject(new Error(`daemon worker request ${request.type} timed out`));
|
|
82
|
+
}, WORKER_REQUEST_TIMEOUT_MS);
|
|
83
|
+
timer.unref();
|
|
84
|
+
pending.set(requestId, {
|
|
85
|
+
workerId: worker.id,
|
|
86
|
+
timer,
|
|
87
|
+
resolve: (value) => resolve(value),
|
|
88
|
+
reject
|
|
89
|
+
});
|
|
90
|
+
});
|
|
91
|
+
try {
|
|
92
|
+
worker.send({ ...request, requestId });
|
|
93
|
+
}
|
|
94
|
+
catch (error) {
|
|
95
|
+
const waiter = pending.get(requestId);
|
|
96
|
+
if (waiter !== undefined)
|
|
97
|
+
clearTimeout(waiter.timer);
|
|
98
|
+
pending.delete(requestId);
|
|
99
|
+
throw error;
|
|
100
|
+
}
|
|
101
|
+
return await response;
|
|
102
|
+
};
|
|
103
|
+
const sidecarOperation = async (request) => {
|
|
104
|
+
sidecarTail = sidecarTail.then(async () => {
|
|
105
|
+
switch (request.operation) {
|
|
106
|
+
case "reconcile":
|
|
107
|
+
await sidecar.reconcile(request.wanted === true);
|
|
108
|
+
return { managed: sidecar.managed(), running: sidecar.running() };
|
|
109
|
+
case "refresh":
|
|
110
|
+
await sidecar.refresh();
|
|
111
|
+
return { managed: sidecar.managed(), running: sidecar.running() };
|
|
112
|
+
case "reachable":
|
|
113
|
+
return await sidecar.reachable(request.timeoutMs);
|
|
114
|
+
case "status":
|
|
115
|
+
return { managed: sidecar.managed(), running: sidecar.running() };
|
|
116
|
+
}
|
|
117
|
+
});
|
|
118
|
+
return await sidecarTail;
|
|
119
|
+
};
|
|
120
|
+
const writeRecord = (managed) => {
|
|
121
|
+
const ready = managed.ready;
|
|
122
|
+
const next = store.write({
|
|
123
|
+
kind: ROUTEKIT_DAEMON_KIND,
|
|
124
|
+
pid: process.pid,
|
|
125
|
+
...(processIdentity(process.pid) !== undefined
|
|
126
|
+
? { processIdentity: processIdentity(process.pid) }
|
|
127
|
+
: {}),
|
|
128
|
+
workerPid: ready.workerPid,
|
|
129
|
+
...(ready.workerProcessIdentity !== undefined
|
|
130
|
+
? { workerProcessIdentity: ready.workerProcessIdentity }
|
|
131
|
+
: {}),
|
|
132
|
+
workerStartedAt: ready.workerStartedAt,
|
|
133
|
+
hostProtocolVersion: DAEMON_HOST_PROTOCOL_VERSION,
|
|
134
|
+
url: loopbackUrl("127.0.0.1", controlPort),
|
|
135
|
+
port: controlPort,
|
|
136
|
+
startedAt: hostStartedAt,
|
|
137
|
+
version: ready.packageVersion,
|
|
138
|
+
protocolVersion: CONTROL_PROTOCOL_VERSION,
|
|
139
|
+
controlToken,
|
|
140
|
+
dataUrl,
|
|
141
|
+
dataPort,
|
|
142
|
+
host,
|
|
143
|
+
portless: portless?.enabled ?? false,
|
|
144
|
+
drainGraceMs,
|
|
145
|
+
authTokenFile: options.authTokenFile ?? join(home, "secrets", "data-token"),
|
|
146
|
+
generation: ready.generation,
|
|
147
|
+
supervisor: supervisorFromEnv(env),
|
|
148
|
+
binPath: managed.binPath,
|
|
149
|
+
args: process.argv.slice(2),
|
|
150
|
+
cwd: process.cwd()
|
|
151
|
+
});
|
|
152
|
+
writeDaemonPublicRecord(home, {
|
|
153
|
+
product: ROUTEKIT_PRODUCT,
|
|
154
|
+
kind: ROUTEKIT_DAEMON_KIND,
|
|
155
|
+
url: next.url,
|
|
156
|
+
port: controlPort,
|
|
157
|
+
generation: ready.generation,
|
|
158
|
+
protocolVersion: CONTROL_PROTOCOL_VERSION,
|
|
159
|
+
dataUrl,
|
|
160
|
+
dataPort,
|
|
161
|
+
startedAt: hostStartedAt
|
|
162
|
+
});
|
|
163
|
+
return next;
|
|
164
|
+
};
|
|
165
|
+
const shutdownWorker = async (worker) => {
|
|
166
|
+
if (worker.isDead())
|
|
167
|
+
return;
|
|
168
|
+
try {
|
|
169
|
+
await requestWorker(worker, { type: "worker.shutdown" });
|
|
170
|
+
}
|
|
171
|
+
catch {
|
|
172
|
+
worker.process.kill("SIGTERM");
|
|
173
|
+
}
|
|
174
|
+
};
|
|
175
|
+
const retireWorker = (managed) => {
|
|
176
|
+
setTimeout(() => {
|
|
177
|
+
if (retirementFallback?.worker.id !== managed.worker.id)
|
|
178
|
+
return;
|
|
179
|
+
retirementFallback = undefined;
|
|
180
|
+
void requestWorker(managed.worker, { type: "worker.retire", graceMs: drainGraceMs }).catch(() => {
|
|
181
|
+
managed.worker.process.kill("SIGTERM");
|
|
182
|
+
});
|
|
183
|
+
const forced = setTimeout(() => {
|
|
184
|
+
if (!managed.worker.isDead())
|
|
185
|
+
managed.worker.process.kill("SIGKILL");
|
|
186
|
+
}, drainGraceMs + RETIRE_FORCE_EXTRA_MS);
|
|
187
|
+
forced.unref();
|
|
188
|
+
}, 250).unref();
|
|
189
|
+
};
|
|
190
|
+
const verifySharedGateway = async () => {
|
|
191
|
+
const tokenPath = options.authTokenFile ?? join(home, "secrets", "data-token");
|
|
192
|
+
const { readFileSync } = await import("node:fs");
|
|
193
|
+
const token = readFileSync(tokenPath, "utf8").trim();
|
|
194
|
+
const health = await fetch(`${dataUrl}/health`, { signal: AbortSignal.timeout(5_000) });
|
|
195
|
+
if (!health.ok)
|
|
196
|
+
throw new Error(`candidate gateway health failed (${health.status})`);
|
|
197
|
+
const models = await fetch(`${dataUrl}/v1/models`, {
|
|
198
|
+
headers: { authorization: `Bearer ${token}` },
|
|
199
|
+
signal: AbortSignal.timeout(15_000)
|
|
200
|
+
});
|
|
201
|
+
if (!models.ok)
|
|
202
|
+
throw new Error(`candidate model readiness failed (${models.status})`);
|
|
203
|
+
};
|
|
204
|
+
const spawnWorker = async (input) => {
|
|
205
|
+
cluster.setupPrimary({ exec: input.binPath, args: process.argv.slice(2), silent: false });
|
|
206
|
+
const worker = cluster.fork({
|
|
207
|
+
...env,
|
|
208
|
+
[ROUTEKIT_DAEMON_WORKER_ENV]: "1",
|
|
209
|
+
[ROUTEKIT_DAEMON_GENERATION_ENV]: String(input.generation),
|
|
210
|
+
[ROUTEKIT_DAEMON_CONTROL_TOKEN_ENV]: controlToken,
|
|
211
|
+
[ROUTEKIT_DAEMON_CONTROL_PORT_ENV]: String(controlPort),
|
|
212
|
+
[ROUTEKIT_DAEMON_DATA_PORT_ENV]: String(dataPort),
|
|
213
|
+
[ROUTEKIT_DAEMON_DATA_URL_ENV]: dataUrl,
|
|
214
|
+
[ROUTEKIT_DAEMON_HOST_PID_ENV]: String(process.pid),
|
|
215
|
+
[ROUTEKIT_DAEMON_HOST_STARTED_AT_ENV]: hostStartedAt,
|
|
216
|
+
[ROUTEKIT_DAEMON_INITIAL_PAUSED_ENV]: input.initiallyPaused ? "1" : "0"
|
|
217
|
+
});
|
|
218
|
+
const ready = await new Promise((resolve, reject) => {
|
|
219
|
+
const timer = setTimeout(() => {
|
|
220
|
+
readyWaiters.delete(worker.id);
|
|
221
|
+
reject(new Error(`daemon worker ${worker.process.pid ?? worker.id} did not become ready`));
|
|
222
|
+
}, WORKER_READY_TIMEOUT_MS);
|
|
223
|
+
timer.unref();
|
|
224
|
+
readyWaiters.set(worker.id, { resolve, reject, timer });
|
|
225
|
+
worker.once("exit", (code, signal) => {
|
|
226
|
+
const waiter = readyWaiters.get(worker.id);
|
|
227
|
+
if (waiter === undefined)
|
|
228
|
+
return;
|
|
229
|
+
readyWaiters.delete(worker.id);
|
|
230
|
+
clearTimeout(waiter.timer);
|
|
231
|
+
reject(new Error(`daemon worker exited before readiness (${signal ?? `code ${code ?? "unknown"}`})`));
|
|
232
|
+
});
|
|
233
|
+
});
|
|
234
|
+
if (ready.protocolVersion !== DAEMON_HOST_PROTOCOL_VERSION) {
|
|
235
|
+
await shutdownWorker(worker);
|
|
236
|
+
throw new Error(`daemon worker host protocol ${ready.protocolVersion} is incompatible with ${DAEMON_HOST_PROTOCOL_VERSION}`);
|
|
237
|
+
}
|
|
238
|
+
if (ready.packageVersion !== input.expectedVersion) {
|
|
239
|
+
await shutdownWorker(worker);
|
|
240
|
+
throw new Error(`daemon worker version ${ready.packageVersion} did not match expected ${input.expectedVersion}`);
|
|
241
|
+
}
|
|
242
|
+
return { worker, ready, binPath: input.binPath };
|
|
243
|
+
};
|
|
244
|
+
const roll = async (params) => {
|
|
245
|
+
if (rollRun !== undefined)
|
|
246
|
+
return await rollRun;
|
|
247
|
+
rollRun = (async () => {
|
|
248
|
+
const previousActive = active;
|
|
249
|
+
if (previousActive === undefined)
|
|
250
|
+
throw new Error("RouteKit daemon has no active worker");
|
|
251
|
+
if (params.expectedGeneration !== previousActive.ready.generation) {
|
|
252
|
+
throw new Error(`daemon generation conflict: expected ${params.expectedGeneration}, current ${previousActive.ready.generation}`);
|
|
253
|
+
}
|
|
254
|
+
const binPath = params.candidate?.binPath ?? previousActive.binPath;
|
|
255
|
+
const expectedVersion = params.candidate?.expectedVersion ?? previousActive.ready.packageVersion;
|
|
256
|
+
console.error("routekit daemon roll requested", {
|
|
257
|
+
reason: params.reason,
|
|
258
|
+
generation: previousActive.ready.generation,
|
|
259
|
+
fromVersion: previousActive.ready.packageVersion,
|
|
260
|
+
toVersion: expectedVersion
|
|
261
|
+
});
|
|
262
|
+
if (!isAbsolute(binPath))
|
|
263
|
+
throw new Error("daemon roll candidate path must be absolute");
|
|
264
|
+
const probed = candidateVersion(binPath);
|
|
265
|
+
if (probed !== expectedVersion) {
|
|
266
|
+
throw new Error(`daemon roll candidate reported ${probed ?? "no version"}; expected ${expectedVersion}`);
|
|
267
|
+
}
|
|
268
|
+
rolling = true;
|
|
269
|
+
await requestWorker(previousActive.worker, {
|
|
270
|
+
type: "worker.hostState",
|
|
271
|
+
dataUrl,
|
|
272
|
+
hostPid: process.pid,
|
|
273
|
+
hostStartedAt,
|
|
274
|
+
rolling: true
|
|
275
|
+
});
|
|
276
|
+
const stable = await requestWorker(previousActive.worker, { type: "worker.pause" });
|
|
277
|
+
const nextGeneration = nextServiceGeneration(previousActive.ready.generation);
|
|
278
|
+
let candidate;
|
|
279
|
+
try {
|
|
280
|
+
candidate = await spawnWorker({
|
|
281
|
+
binPath,
|
|
282
|
+
generation: nextGeneration,
|
|
283
|
+
initiallyPaused: true,
|
|
284
|
+
expectedVersion
|
|
285
|
+
});
|
|
286
|
+
console.error("routekit daemon candidate prepared", {
|
|
287
|
+
generation: candidate.ready.generation,
|
|
288
|
+
workerPid: candidate.ready.workerPid,
|
|
289
|
+
version: candidate.ready.packageVersion
|
|
290
|
+
});
|
|
291
|
+
if (candidate.ready.configRevision !== stable.configRevision ||
|
|
292
|
+
candidate.ready.accountRevision !== stable.accountRevision ||
|
|
293
|
+
candidate.ready.configHash !== stable.configHash) {
|
|
294
|
+
throw new Error("candidate state changed while synchronizing; retry the daemon roll");
|
|
295
|
+
}
|
|
296
|
+
console.error("routekit daemon candidate synchronized", {
|
|
297
|
+
generation: candidate.ready.generation,
|
|
298
|
+
configRevision: candidate.ready.configRevision,
|
|
299
|
+
accountRevision: candidate.ready.accountRevision
|
|
300
|
+
});
|
|
301
|
+
await verifySharedGateway();
|
|
302
|
+
console.error("routekit daemon candidate active", {
|
|
303
|
+
generation: candidate.ready.generation,
|
|
304
|
+
workerPid: candidate.ready.workerPid
|
|
305
|
+
});
|
|
306
|
+
await requestWorker(candidate.worker, {
|
|
307
|
+
type: "worker.hostState",
|
|
308
|
+
dataUrl,
|
|
309
|
+
hostPid: process.pid,
|
|
310
|
+
hostStartedAt,
|
|
311
|
+
rolling: false
|
|
312
|
+
});
|
|
313
|
+
await requestWorker(candidate.worker, { type: "worker.resume" });
|
|
314
|
+
generation = nextGeneration;
|
|
315
|
+
revisions.daemon = generation;
|
|
316
|
+
writeDaemonRevisions(home, revisions);
|
|
317
|
+
active = candidate;
|
|
318
|
+
record = writeRecord(candidate);
|
|
319
|
+
console.error("routekit daemon worker generation committed", {
|
|
320
|
+
generation,
|
|
321
|
+
workerPid: candidate.ready.workerPid
|
|
322
|
+
});
|
|
323
|
+
rolling = false;
|
|
324
|
+
retirementFallback = previousActive;
|
|
325
|
+
console.error("routekit daemon previous worker retiring", {
|
|
326
|
+
generation: previousActive.ready.generation,
|
|
327
|
+
workerPid: previousActive.ready.workerPid
|
|
328
|
+
});
|
|
329
|
+
retireWorker(previousActive);
|
|
330
|
+
return {
|
|
331
|
+
rolled: true,
|
|
332
|
+
reason: params.reason,
|
|
333
|
+
previousGeneration: previousActive.ready.generation,
|
|
334
|
+
generation,
|
|
335
|
+
previousWorkerPid: previousActive.ready.workerPid,
|
|
336
|
+
workerPid: candidate.ready.workerPid,
|
|
337
|
+
packageVersion: candidate.ready.packageVersion
|
|
338
|
+
};
|
|
339
|
+
}
|
|
340
|
+
catch (error) {
|
|
341
|
+
console.error("routekit daemon roll rolled back", {
|
|
342
|
+
reason: params.reason,
|
|
343
|
+
stage: candidate === undefined ? "preparation" : "activation",
|
|
344
|
+
error: error instanceof Error ? error.message : String(error)
|
|
345
|
+
});
|
|
346
|
+
if (candidate !== undefined)
|
|
347
|
+
await shutdownWorker(candidate.worker);
|
|
348
|
+
active = previousActive;
|
|
349
|
+
generation = previousActive.ready.generation;
|
|
350
|
+
revisions.daemon = generation;
|
|
351
|
+
writeDaemonRevisions(home, revisions);
|
|
352
|
+
record = writeRecord(previousActive);
|
|
353
|
+
await requestWorker(previousActive.worker, { type: "worker.resume" });
|
|
354
|
+
await requestWorker(previousActive.worker, {
|
|
355
|
+
type: "worker.hostState",
|
|
356
|
+
dataUrl,
|
|
357
|
+
hostPid: process.pid,
|
|
358
|
+
hostStartedAt,
|
|
359
|
+
rolling: false
|
|
360
|
+
});
|
|
361
|
+
rolling = false;
|
|
362
|
+
throw error;
|
|
363
|
+
}
|
|
364
|
+
})();
|
|
365
|
+
try {
|
|
366
|
+
return await rollRun;
|
|
367
|
+
}
|
|
368
|
+
finally {
|
|
369
|
+
rollRun = undefined;
|
|
370
|
+
}
|
|
371
|
+
};
|
|
372
|
+
const close = () => {
|
|
373
|
+
closeRun ??= (async () => {
|
|
374
|
+
if (closed)
|
|
375
|
+
return;
|
|
376
|
+
closed = true;
|
|
377
|
+
const workers = Object.values(cluster.workers ?? {}).filter((worker) => worker !== undefined);
|
|
378
|
+
await Promise.allSettled(workers.map(async (worker) => await shutdownWorker(worker)));
|
|
379
|
+
await sidecar.close();
|
|
380
|
+
if (portless?.enabled)
|
|
381
|
+
portless.unregister("gateway");
|
|
382
|
+
store.remove(ROUTEKIT_DAEMON_KIND, { ifPid: process.pid });
|
|
383
|
+
removeDaemonPublicRecord(home);
|
|
384
|
+
authority.release();
|
|
385
|
+
})();
|
|
386
|
+
return closeRun;
|
|
387
|
+
};
|
|
388
|
+
const handleMessage = (worker, message) => {
|
|
389
|
+
if (message.type === "worker.ready") {
|
|
390
|
+
const waiter = readyWaiters.get(worker.id);
|
|
391
|
+
if (waiter === undefined)
|
|
392
|
+
return;
|
|
393
|
+
readyWaiters.delete(worker.id);
|
|
394
|
+
clearTimeout(waiter.timer);
|
|
395
|
+
waiter.resolve(message);
|
|
396
|
+
return;
|
|
397
|
+
}
|
|
398
|
+
if (message.type === "worker.response") {
|
|
399
|
+
const waiter = pending.get(message.requestId);
|
|
400
|
+
if (waiter === undefined || waiter.workerId !== worker.id)
|
|
401
|
+
return;
|
|
402
|
+
pending.delete(message.requestId);
|
|
403
|
+
clearTimeout(waiter.timer);
|
|
404
|
+
if (message.ok)
|
|
405
|
+
waiter.resolve(message.result);
|
|
406
|
+
else
|
|
407
|
+
waiter.reject(new Error(message.error));
|
|
408
|
+
return;
|
|
409
|
+
}
|
|
410
|
+
if (message.type === "host.sidecar") {
|
|
411
|
+
void sidecarOperation(message).then((result) => sendResponse(worker, { type: "host.response", requestId: message.requestId, ok: true, result }), (error) => sendResponse(worker, {
|
|
412
|
+
type: "host.response",
|
|
413
|
+
requestId: message.requestId,
|
|
414
|
+
ok: false,
|
|
415
|
+
error: error instanceof Error ? error.message : String(error)
|
|
416
|
+
}));
|
|
417
|
+
return;
|
|
418
|
+
}
|
|
419
|
+
if (message.type === "host.roll") {
|
|
420
|
+
if (active?.worker.id !== worker.id) {
|
|
421
|
+
sendResponse(worker, {
|
|
422
|
+
type: "host.response",
|
|
423
|
+
requestId: message.requestId,
|
|
424
|
+
ok: false,
|
|
425
|
+
error: "only the active daemon worker may request a roll"
|
|
426
|
+
});
|
|
427
|
+
return;
|
|
428
|
+
}
|
|
429
|
+
void roll(message.params).then((result) => sendResponse(worker, { type: "host.response", requestId: message.requestId, ok: true, result }), (error) => sendResponse(worker, {
|
|
430
|
+
type: "host.response",
|
|
431
|
+
requestId: message.requestId,
|
|
432
|
+
ok: false,
|
|
433
|
+
error: error instanceof Error ? error.message : String(error)
|
|
434
|
+
}));
|
|
435
|
+
return;
|
|
436
|
+
}
|
|
437
|
+
if (message.type === "host.shutdown") {
|
|
438
|
+
void close().finally(() => process.exit(0));
|
|
439
|
+
}
|
|
440
|
+
};
|
|
441
|
+
cluster.on("message", (worker, message) => handleMessage(worker, message));
|
|
442
|
+
cluster.on("exit", (worker) => {
|
|
443
|
+
for (const [requestId, waiter] of pending) {
|
|
444
|
+
if (waiter.workerId !== worker.id)
|
|
445
|
+
continue;
|
|
446
|
+
pending.delete(requestId);
|
|
447
|
+
clearTimeout(waiter.timer);
|
|
448
|
+
waiter.reject(new Error("daemon worker exited"));
|
|
449
|
+
}
|
|
450
|
+
if (!closed && active?.worker.id === worker.id) {
|
|
451
|
+
const failed = active;
|
|
452
|
+
active = undefined;
|
|
453
|
+
const fallback = retirementFallback;
|
|
454
|
+
if (fallback !== undefined && !fallback.worker.isDead()) {
|
|
455
|
+
retirementFallback = undefined;
|
|
456
|
+
recoveryRun ??= (async () => {
|
|
457
|
+
try {
|
|
458
|
+
active = fallback;
|
|
459
|
+
generation = fallback.ready.generation;
|
|
460
|
+
revisions.daemon = generation;
|
|
461
|
+
writeDaemonRevisions(home, revisions);
|
|
462
|
+
record = writeRecord(fallback);
|
|
463
|
+
await requestWorker(fallback.worker, {
|
|
464
|
+
type: "worker.hostState",
|
|
465
|
+
dataUrl,
|
|
466
|
+
hostPid: process.pid,
|
|
467
|
+
hostStartedAt,
|
|
468
|
+
rolling: false
|
|
469
|
+
});
|
|
470
|
+
await requestWorker(fallback.worker, { type: "worker.resume" });
|
|
471
|
+
rolling = false;
|
|
472
|
+
console.error("routekit daemon roll rolled back after committed worker exit", {
|
|
473
|
+
generation,
|
|
474
|
+
workerPid: fallback.ready.workerPid
|
|
475
|
+
});
|
|
476
|
+
}
|
|
477
|
+
catch (error) {
|
|
478
|
+
console.error("routekit daemon committed rollback failed", error);
|
|
479
|
+
await close();
|
|
480
|
+
process.exit(1);
|
|
481
|
+
}
|
|
482
|
+
finally {
|
|
483
|
+
recoveryRun = undefined;
|
|
484
|
+
}
|
|
485
|
+
})();
|
|
486
|
+
return;
|
|
487
|
+
}
|
|
488
|
+
recoveryRun ??= (async () => {
|
|
489
|
+
console.error("routekit daemon active worker exited; attempting committed respawn", {
|
|
490
|
+
generation: failed.ready.generation,
|
|
491
|
+
workerPid: failed.ready.workerPid
|
|
492
|
+
});
|
|
493
|
+
try {
|
|
494
|
+
const replacement = await spawnWorker({
|
|
495
|
+
binPath: failed.binPath,
|
|
496
|
+
generation: failed.ready.generation,
|
|
497
|
+
initiallyPaused: false,
|
|
498
|
+
expectedVersion: failed.ready.packageVersion
|
|
499
|
+
});
|
|
500
|
+
active = replacement;
|
|
501
|
+
record = writeRecord(replacement);
|
|
502
|
+
await requestWorker(replacement.worker, {
|
|
503
|
+
type: "worker.hostState",
|
|
504
|
+
dataUrl,
|
|
505
|
+
hostPid: process.pid,
|
|
506
|
+
hostStartedAt,
|
|
507
|
+
rolling: false
|
|
508
|
+
});
|
|
509
|
+
if (retirementFallback !== undefined)
|
|
510
|
+
retireWorker(retirementFallback);
|
|
511
|
+
console.error("routekit daemon committed worker respawned", {
|
|
512
|
+
generation: replacement.ready.generation,
|
|
513
|
+
workerPid: replacement.ready.workerPid
|
|
514
|
+
});
|
|
515
|
+
}
|
|
516
|
+
catch (error) {
|
|
517
|
+
console.error("routekit daemon committed worker respawn failed", error);
|
|
518
|
+
await close();
|
|
519
|
+
process.exit(1);
|
|
520
|
+
}
|
|
521
|
+
finally {
|
|
522
|
+
recoveryRun = undefined;
|
|
523
|
+
}
|
|
524
|
+
})();
|
|
525
|
+
}
|
|
526
|
+
});
|
|
527
|
+
extendCleanupGrace(drainGraceMs + RETIRE_FORCE_EXTRA_MS);
|
|
528
|
+
registerCleanup(close);
|
|
529
|
+
try {
|
|
530
|
+
active = await spawnWorker({
|
|
531
|
+
binPath: options.entryPath,
|
|
532
|
+
generation,
|
|
533
|
+
initiallyPaused: false,
|
|
534
|
+
expectedVersion: options.packageVersion
|
|
535
|
+
});
|
|
536
|
+
portless = await createPortlessSession(options.portless ?? env.ROUTEKIT_PORTLESS !== "0", {
|
|
537
|
+
project: ROUTEKIT_PRODUCT,
|
|
538
|
+
ownerLabel: "routekit-daemon",
|
|
539
|
+
bareNames: []
|
|
540
|
+
});
|
|
541
|
+
dataUrl = portless.enabled ? portless.register("gateway", dataPort) : loopbackUrl(host, dataPort);
|
|
542
|
+
await requestWorker(active.worker, {
|
|
543
|
+
type: "worker.hostState",
|
|
544
|
+
dataUrl,
|
|
545
|
+
hostPid: process.pid,
|
|
546
|
+
hostStartedAt,
|
|
547
|
+
rolling: false
|
|
548
|
+
});
|
|
549
|
+
record = writeRecord(active);
|
|
550
|
+
return { record, dataUrl, controlUrl: record.url, close };
|
|
551
|
+
}
|
|
552
|
+
catch (error) {
|
|
553
|
+
await close();
|
|
554
|
+
throw error;
|
|
555
|
+
}
|
|
556
|
+
}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { ServiceRecord } from "@velum-labs/routekit-runtime";
|
|
2
|
+
import type { CliproxySidecar } from "./cliproxy-sidecar.js";
|
|
2
3
|
import { type TelemetryTransportFactory } from "./telemetry.js";
|
|
3
4
|
export declare const ROUTEKIT_DAEMON_KIND = "daemon";
|
|
4
5
|
export declare const ROUTEKIT_PRODUCT = "routekit";
|
|
@@ -9,11 +10,23 @@ export type RouteKitDaemonOptions = {
|
|
|
9
10
|
configPath?: string;
|
|
10
11
|
host?: string;
|
|
11
12
|
port?: number;
|
|
13
|
+
controlPort?: number;
|
|
12
14
|
authToken?: string;
|
|
13
15
|
authTokenFile?: string;
|
|
14
16
|
portless?: boolean;
|
|
15
17
|
drainGraceMs?: number;
|
|
16
18
|
onShutdownRequested?: (reason: "stop" | "restart" | "upgrade") => void;
|
|
19
|
+
onRollRequested?: (params: import("@velum-labs/routekit-control").RouteKitControlParams["daemon.roll"]) => Promise<import("@velum-labs/routekit-control").RouteKitControlResults["daemon.roll"]>;
|
|
20
|
+
hosted?: {
|
|
21
|
+
generation: number;
|
|
22
|
+
controlToken: string;
|
|
23
|
+
dataUrl: () => string;
|
|
24
|
+
hostPid: number;
|
|
25
|
+
hostStartedAt: string;
|
|
26
|
+
rolling: () => boolean;
|
|
27
|
+
sidecar: CliproxySidecar;
|
|
28
|
+
initiallyPaused?: boolean;
|
|
29
|
+
};
|
|
17
30
|
/** Test seam for a network-free telemetry transport. */
|
|
18
31
|
telemetryTransportFactory?: TelemetryTransportFactory;
|
|
19
32
|
telemetryFlushIntervalMs?: number;
|
|
@@ -25,6 +38,18 @@ export type RunningRouteKitDaemon = {
|
|
|
25
38
|
dataUrl: string;
|
|
26
39
|
controlUrl: string;
|
|
27
40
|
close(): Promise<void>;
|
|
41
|
+
retire(graceMs?: number): Promise<void>;
|
|
42
|
+
pauseMutations(): Promise<{
|
|
43
|
+
configRevision: number;
|
|
44
|
+
accountRevision: number;
|
|
45
|
+
configHash: string;
|
|
46
|
+
}>;
|
|
47
|
+
resumeMutations(): void;
|
|
48
|
+
snapshot(): {
|
|
49
|
+
configRevision: number;
|
|
50
|
+
accountRevision: number;
|
|
51
|
+
configHash: string;
|
|
52
|
+
};
|
|
28
53
|
reload(): Promise<void>;
|
|
29
54
|
};
|
|
30
55
|
export type DaemonPublicRecord = {
|
|
@@ -38,4 +63,20 @@ export type DaemonPublicRecord = {
|
|
|
38
63
|
dataPort?: number;
|
|
39
64
|
startedAt: string;
|
|
40
65
|
};
|
|
66
|
+
export declare function daemonPublicRecordPath(home: string): string;
|
|
67
|
+
/** Publish a secret-free discovery file peers can read across OS accounts. */
|
|
68
|
+
export declare function writeDaemonPublicRecord(home: string, record: DaemonPublicRecord): void;
|
|
69
|
+
export declare function removeDaemonPublicRecord(home: string): void;
|
|
70
|
+
type RevisionState = {
|
|
71
|
+
config: number;
|
|
72
|
+
accounts: number;
|
|
73
|
+
daemon: number;
|
|
74
|
+
};
|
|
75
|
+
export declare function readDaemonRevisions(home: string): RevisionState;
|
|
76
|
+
export declare function writeDaemonRevisions(home: string, revisions: RevisionState): void;
|
|
41
77
|
export declare function startRouteKitDaemon(options: RouteKitDaemonOptions): Promise<RunningRouteKitDaemon>;
|
|
78
|
+
export { ROUTEKIT_DAEMON_WORKER_ENV } from "./host-protocol.js";
|
|
79
|
+
export declare function startRouteKitDaemonHost(options: RouteKitDaemonOptions & {
|
|
80
|
+
entryPath: string;
|
|
81
|
+
}): Promise<import("./host.js").RunningRouteKitDaemonHost>;
|
|
82
|
+
export declare function runRouteKitDaemonWorker(options: RouteKitDaemonOptions): Promise<never>;
|