@forgezero/agent 0.1.86 → 0.1.88
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-handover.d.ts +19 -0
- package/dist/agent-heartbeat.d.ts +3 -3
- package/dist/agent-heartbeat.js +247 -60
- package/dist/agent-update-helper.d.ts +12 -2
- package/dist/agent-update-helper.js +210 -45
- package/dist/agent-update.d.ts +4 -0
- package/dist/agent-update.js +19 -1
- package/dist/bootstrap.js +48 -5
- package/dist/fz-agent.js +569 -256
- package/dist/fz.js +87 -5
- package/dist/metal-bootstrap.js +40 -1
- package/dist/operator-bootstrap.js +87 -5
- package/dist/platform-fleet-verification.js +415 -211
- package/dist/provision.d.ts +16 -1
- package/dist/provision.js +332 -124
- package/dist/socket.d.ts +6 -0
- package/dist/version.d.ts +1 -1
- package/package.json +1 -1
|
@@ -16,6 +16,7 @@ import {
|
|
|
16
16
|
} from "node:fs";
|
|
17
17
|
import { dirname, join, resolve } from "node:path";
|
|
18
18
|
var DEFAULT_AGENT_RELEASE_ROOT = "/opt/forgezero/agent";
|
|
19
|
+
var DEFAULT_AGENT_CANDIDATE_LINK = `${DEFAULT_AGENT_RELEASE_ROOT}/candidate`;
|
|
19
20
|
var DEFAULT_AGENT_UPDATE_SOCKET = "/run/forgezero-update/helper.sock";
|
|
20
21
|
var MAX_AGENT_TARBALL_BYTES = 32 * 1024 * 1024;
|
|
21
22
|
var VERSION = /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)$/;
|
|
@@ -209,25 +210,90 @@ function restoreAgentRelease(staged) {
|
|
|
209
210
|
rmSync(next, { force: true });
|
|
210
211
|
}
|
|
211
212
|
}
|
|
213
|
+
function selectAgentCandidate(staged) {
|
|
214
|
+
const link = join(dirname(staged.currentLink), "candidate");
|
|
215
|
+
const next = `${link}.${randomUUID()}.next`;
|
|
216
|
+
try {
|
|
217
|
+
symlinkSync(staged.nextTarget, next);
|
|
218
|
+
renameSync(next, link);
|
|
219
|
+
syncPath(dirname(link));
|
|
220
|
+
} finally {
|
|
221
|
+
rmSync(next, { force: true });
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
function clearAgentCandidate(root = DEFAULT_AGENT_RELEASE_ROOT) {
|
|
225
|
+
rmSync(join(resolve(root), "candidate"), { force: true });
|
|
226
|
+
}
|
|
212
227
|
|
|
213
228
|
// src/agent-update-helper.ts
|
|
214
|
-
import { randomUUID as
|
|
229
|
+
import { randomUUID as randomUUID3 } from "node:crypto";
|
|
215
230
|
import {
|
|
216
|
-
chmodSync as
|
|
231
|
+
chmodSync as chmodSync3,
|
|
217
232
|
closeSync as closeSync2,
|
|
218
|
-
existsSync as
|
|
233
|
+
existsSync as existsSync3,
|
|
219
234
|
fsyncSync as fsyncSync2,
|
|
220
235
|
mkdirSync as mkdirSync2,
|
|
221
236
|
openSync as openSync2,
|
|
222
237
|
readFileSync as readFileSync2,
|
|
223
|
-
renameSync as
|
|
224
|
-
rmSync as
|
|
225
|
-
unlinkSync,
|
|
238
|
+
renameSync as renameSync3,
|
|
239
|
+
rmSync as rmSync3,
|
|
240
|
+
unlinkSync as unlinkSync2,
|
|
226
241
|
writeFileSync as writeFileSync2
|
|
227
242
|
} from "node:fs";
|
|
228
|
-
import { connect, createServer } from "node:net";
|
|
229
|
-
import { dirname as
|
|
243
|
+
import { connect as connect2, createServer as createServer2 } from "node:net";
|
|
244
|
+
import { dirname as dirname3, join as join2, resolve as resolve2 } from "node:path";
|
|
230
245
|
import { DEFAULT_SOCKET } from "@forgezero/vault";
|
|
246
|
+
|
|
247
|
+
// src/agent-handover.ts
|
|
248
|
+
import { randomUUID as randomUUID2 } from "node:crypto";
|
|
249
|
+
import { chmodSync as chmodSync2, existsSync as existsSync2, renameSync as renameSync2, rmSync as rmSync2, symlinkSync as symlinkSync2, unlinkSync } from "node:fs";
|
|
250
|
+
import { basename, dirname as dirname2 } from "node:path";
|
|
251
|
+
import { connect, createServer } from "node:net";
|
|
252
|
+
var DEFAULT_AGENT_CANDIDATE_READY_SOCKET = "/run/forgezero/candidate-ready.sock";
|
|
253
|
+
function switchAgentSocketRoute(route, backend) {
|
|
254
|
+
if (dirname2(route) !== dirname2(backend))
|
|
255
|
+
throw new Error("Agent handover sockets must share one runtime directory");
|
|
256
|
+
const target = basename(backend);
|
|
257
|
+
const next = `${route}.${randomUUID2()}.next`;
|
|
258
|
+
try {
|
|
259
|
+
symlinkSync2(target, next);
|
|
260
|
+
renameSync2(next, route);
|
|
261
|
+
} finally {
|
|
262
|
+
rmSync2(next, { force: true });
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
function probeAgentCandidate(expected, socketPath = DEFAULT_AGENT_CANDIDATE_READY_SOCKET, timeoutMs = 5000) {
|
|
266
|
+
return new Promise((resolve2) => {
|
|
267
|
+
const socket = connect(socketPath);
|
|
268
|
+
let buffer = "";
|
|
269
|
+
let settled = false;
|
|
270
|
+
const finish = (value) => {
|
|
271
|
+
if (settled)
|
|
272
|
+
return;
|
|
273
|
+
settled = true;
|
|
274
|
+
clearTimeout(timer);
|
|
275
|
+
socket.destroy();
|
|
276
|
+
resolve2(value);
|
|
277
|
+
};
|
|
278
|
+
const timer = setTimeout(() => finish(false), timeoutMs);
|
|
279
|
+
socket.on("data", (chunk) => {
|
|
280
|
+
buffer += chunk.toString("utf8");
|
|
281
|
+
const newline = buffer.indexOf(`
|
|
282
|
+
`);
|
|
283
|
+
if (newline < 0)
|
|
284
|
+
return;
|
|
285
|
+
try {
|
|
286
|
+
const value = JSON.parse(buffer.slice(0, newline));
|
|
287
|
+
finish(value.version === expected.version && value.nodeKey === expected.nodeKey && value.bound === expected.bound && expected.vault.includes(value.vault));
|
|
288
|
+
} catch {
|
|
289
|
+
finish(false);
|
|
290
|
+
}
|
|
291
|
+
});
|
|
292
|
+
socket.on("error", () => finish(false));
|
|
293
|
+
});
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
// src/agent-update-helper.ts
|
|
231
297
|
var AGENT_UPDATE_GROUP = "forgezero-update";
|
|
232
298
|
var AGENT_UPDATE_HELPER_UNIT_PATH = "/etc/systemd/system/forgezero-agent-update-helper.service";
|
|
233
299
|
var AGENT_UPDATE_JOURNAL = "/var/lib/forgezero/agent-update.json";
|
|
@@ -322,13 +388,13 @@ function validateJournal(value, root) {
|
|
|
322
388
|
return journal;
|
|
323
389
|
}
|
|
324
390
|
function readJournal(path, root) {
|
|
325
|
-
if (!
|
|
391
|
+
if (!existsSync3(path))
|
|
326
392
|
return;
|
|
327
393
|
return validateJournal(JSON.parse(readFileSync2(path, "utf8")), root);
|
|
328
394
|
}
|
|
329
395
|
function writeAtomic(path, value, mode) {
|
|
330
|
-
mkdirSync2(
|
|
331
|
-
const next = `${path}.${
|
|
396
|
+
mkdirSync2(dirname3(path), { recursive: true, mode: 493 });
|
|
397
|
+
const next = `${path}.${randomUUID3()}.next`;
|
|
332
398
|
let file;
|
|
333
399
|
try {
|
|
334
400
|
file = openSync2(next, "wx", mode);
|
|
@@ -337,8 +403,8 @@ function writeAtomic(path, value, mode) {
|
|
|
337
403
|
fsyncSync2(file);
|
|
338
404
|
closeSync2(file);
|
|
339
405
|
file = undefined;
|
|
340
|
-
|
|
341
|
-
const directory = openSync2(
|
|
406
|
+
renameSync3(next, path);
|
|
407
|
+
const directory = openSync2(dirname3(path), "r");
|
|
342
408
|
try {
|
|
343
409
|
fsyncSync2(directory);
|
|
344
410
|
} finally {
|
|
@@ -347,7 +413,7 @@ function writeAtomic(path, value, mode) {
|
|
|
347
413
|
} finally {
|
|
348
414
|
if (file !== undefined)
|
|
349
415
|
closeSync2(file);
|
|
350
|
-
|
|
416
|
+
rmSync3(next, { force: true });
|
|
351
417
|
}
|
|
352
418
|
}
|
|
353
419
|
var publicReceipt = (journal) => {
|
|
@@ -380,7 +446,7 @@ function writeUpdateState(journalPath, receiptPath, journal) {
|
|
|
380
446
|
}
|
|
381
447
|
function readAgentUpdateReceipt(path = AGENT_UPDATE_RECEIPT) {
|
|
382
448
|
try {
|
|
383
|
-
if (!
|
|
449
|
+
if (!existsSync3(path))
|
|
384
450
|
return;
|
|
385
451
|
return validateReceipt(JSON.parse(readFileSync2(path, "utf8")));
|
|
386
452
|
} catch {
|
|
@@ -420,10 +486,37 @@ var restartAgent = async (target, run) => {
|
|
|
420
486
|
throw new Error(`systemd could not restart ${service}`);
|
|
421
487
|
}
|
|
422
488
|
};
|
|
489
|
+
var candidateUnit = (target) => target === "compute" ? "forgezero-agent-candidate.service" : "forgezero-metal-agent-candidate.service";
|
|
490
|
+
var startCandidate = async (staged, target, expectedNodeKey, run, probe = (expected) => probeAgentCandidate(expected, DEFAULT_AGENT_CANDIDATE_READY_SOCKET)) => {
|
|
491
|
+
selectAgentCandidate(staged);
|
|
492
|
+
if (!await runOk(run, "/usr/bin/systemctl", ["restart", candidateUnit(target)])) {
|
|
493
|
+
clearAgentCandidate(dirname3(staged.currentLink));
|
|
494
|
+
throw new Error(`systemd could not start ${candidateUnit(target)}`);
|
|
495
|
+
}
|
|
496
|
+
if (!await probe({
|
|
497
|
+
version: staged.version,
|
|
498
|
+
nodeKey: expectedNodeKey,
|
|
499
|
+
bound: true,
|
|
500
|
+
vault: target === "compute" ? ["ready"] : ["unbound"]
|
|
501
|
+
})) {
|
|
502
|
+
await run({ command: "/usr/bin/systemctl", args: ["stop", candidateUnit(target)] });
|
|
503
|
+
clearAgentCandidate(dirname3(staged.currentLink));
|
|
504
|
+
throw new Error("the candidate Agent did not prove its identity and durable state");
|
|
505
|
+
}
|
|
506
|
+
};
|
|
507
|
+
var stopCandidate = async (staged, target, run) => {
|
|
508
|
+
await run({ command: "/usr/bin/systemctl", args: ["stop", candidateUnit(target)] });
|
|
509
|
+
clearAgentCandidate(dirname3(staged.currentLink));
|
|
510
|
+
};
|
|
511
|
+
var socketPaths = (publicSocket = process.env.FZ_AGENT_PUBLIC_SOCKET ?? DEFAULT_SOCKET) => ({
|
|
512
|
+
route: `${publicSocket}.backend`,
|
|
513
|
+
active: `${publicSocket}.backend.active`,
|
|
514
|
+
candidate: `${publicSocket}.backend.candidate`
|
|
515
|
+
});
|
|
423
516
|
var targetProbe = (target, run) => target === "compute" ? () => probeAgentSocket() : async () => await runOk(run, "/usr/bin/systemctl", ["is-active", "--quiet", "forgezero-metal-agent.service"]) && await runOk(run, "/usr/bin/systemctl", ["is-active", "--quiet", "forgezero-metal-helper.service"]);
|
|
424
517
|
function probeAgentSocket(socketPath = DEFAULT_SOCKET, timeoutMs = 5000) {
|
|
425
518
|
return new Promise((resolve3) => {
|
|
426
|
-
const socket =
|
|
519
|
+
const socket = connect2(socketPath);
|
|
427
520
|
let settled = false;
|
|
428
521
|
let buffer = "";
|
|
429
522
|
const finish = (value) => {
|
|
@@ -456,12 +549,13 @@ function probeAgentSocket(socketPath = DEFAULT_SOCKET, timeoutMs = 5000) {
|
|
|
456
549
|
async function activateAgentRelease(staged, options = {}) {
|
|
457
550
|
const run = options.run ?? runCommand;
|
|
458
551
|
const target = options.target ?? "compute";
|
|
459
|
-
const
|
|
552
|
+
const paths = socketPaths(options.publicSocketPath);
|
|
553
|
+
const probe = options.probe ?? (target === "compute" ? () => probeAgentSocket(paths.active) : targetProbe(target, run));
|
|
460
554
|
const now = options.now ?? Date.now;
|
|
461
555
|
const journalPath = options.journalPath ?? AGENT_UPDATE_JOURNAL;
|
|
462
556
|
const receiptPath = options.receiptPath ?? AGENT_UPDATE_RECEIPT;
|
|
463
|
-
const previous = readJournal(journalPath,
|
|
464
|
-
const attemptId = options.attemptId ??
|
|
557
|
+
const previous = readJournal(journalPath, dirname3(staged.currentLink));
|
|
558
|
+
const attemptId = options.attemptId ?? randomUUID3();
|
|
465
559
|
if (!ATTEMPT_ID.test(attemptId))
|
|
466
560
|
throw new Error("Agent update attempt ID is invalid");
|
|
467
561
|
const startedAtTs = now();
|
|
@@ -483,11 +577,19 @@ async function activateAgentRelease(staged, options = {}) {
|
|
|
483
577
|
writeUpdateState(journalPath, receiptPath, journal);
|
|
484
578
|
let selectionAttempted = false;
|
|
485
579
|
try {
|
|
580
|
+
if (!options.candidatePrepared) {
|
|
581
|
+
throw new Error("direct activation requires a separately identity-bound candidate preparation");
|
|
582
|
+
}
|
|
583
|
+
if (target === "compute")
|
|
584
|
+
switchAgentSocketRoute(paths.route, paths.candidate);
|
|
486
585
|
selectionAttempted = true;
|
|
487
586
|
selectAgentRelease(staged);
|
|
488
587
|
await restartAgent(target, run);
|
|
489
588
|
if (!await probe())
|
|
490
|
-
throw new Error("the replacement Agent did not answer its
|
|
589
|
+
throw new Error("the replacement Agent did not answer its active Vault backend");
|
|
590
|
+
if (target === "compute")
|
|
591
|
+
switchAgentSocketRoute(paths.route, paths.active);
|
|
592
|
+
await stopCandidate(staged, target, run);
|
|
491
593
|
journal = { ...journal, outcome: "active", updatedAtTs: now(), rollbackHealthy: undefined };
|
|
492
594
|
writeUpdateState(journalPath, receiptPath, journal);
|
|
493
595
|
run({
|
|
@@ -505,10 +607,13 @@ async function activateAgentRelease(staged, options = {}) {
|
|
|
505
607
|
restored = true;
|
|
506
608
|
await restartAgent(target, run);
|
|
507
609
|
rollbackHealthy = await probe();
|
|
610
|
+
if (rollbackHealthy && target === "compute")
|
|
611
|
+
switchAgentSocketRoute(paths.route, paths.active);
|
|
508
612
|
} catch {
|
|
509
613
|
rollbackHealthy = false;
|
|
510
614
|
}
|
|
511
615
|
}
|
|
616
|
+
await stopCandidate(staged, target, run).catch(() => {});
|
|
512
617
|
const failures = failureCount + 1;
|
|
513
618
|
const updatedAtTs = now();
|
|
514
619
|
journal = {
|
|
@@ -528,28 +633,38 @@ async function recoverInterruptedAgentUpdate(options = {}) {
|
|
|
528
633
|
const root = resolve2(options.root ?? DEFAULT_AGENT_RELEASE_ROOT);
|
|
529
634
|
const journalPath = options.journalPath ?? AGENT_UPDATE_JOURNAL;
|
|
530
635
|
const receiptPath = options.receiptPath ?? AGENT_UPDATE_RECEIPT;
|
|
636
|
+
const run = options.run ?? runCommand;
|
|
531
637
|
const journal = readJournal(journalPath, root);
|
|
532
|
-
if (!journal)
|
|
638
|
+
if (!journal) {
|
|
639
|
+
for (const target of ["compute", "metal"]) {
|
|
640
|
+
await run({ command: "/usr/bin/systemctl", args: ["stop", candidateUnit(target)] });
|
|
641
|
+
}
|
|
642
|
+
clearAgentCandidate(root);
|
|
533
643
|
return;
|
|
644
|
+
}
|
|
534
645
|
if (journal.outcome !== "activating") {
|
|
646
|
+
await stopCandidate(stagedFromJournal(journal, root), journal.target, run).catch(() => {});
|
|
535
647
|
writeAtomic(receiptPath, publicReceipt(journal), 416);
|
|
536
648
|
return publicReceipt(journal);
|
|
537
649
|
}
|
|
538
650
|
const staged = stagedFromJournal(journal, root);
|
|
539
|
-
if (!
|
|
651
|
+
if (!existsSync3(join2(root, journal.previousTarget))) {
|
|
540
652
|
throw new Error("Agent update rollback release is missing");
|
|
541
653
|
}
|
|
542
|
-
const
|
|
543
|
-
const probe = options.probe ?? targetProbe(journal.target, run);
|
|
654
|
+
const paths = socketPaths(options.publicSocketPath);
|
|
655
|
+
const probe = options.probe ?? (journal.target === "compute" ? () => probeAgentSocket(paths.active) : targetProbe(journal.target, run));
|
|
544
656
|
restoreAgentRelease(staged);
|
|
545
657
|
let rollbackHealthy = false;
|
|
546
658
|
let failureMessage = "activation was interrupted before its health verdict became durable";
|
|
547
659
|
try {
|
|
548
660
|
await restartAgent(journal.target, run);
|
|
549
661
|
rollbackHealthy = await probe();
|
|
662
|
+
if (rollbackHealthy && journal.target === "compute")
|
|
663
|
+
switchAgentSocketRoute(paths.route, paths.active);
|
|
550
664
|
} catch (cause) {
|
|
551
665
|
failureMessage = `${failureMessage}; ${cause instanceof Error ? cause.message : String(cause)}`;
|
|
552
666
|
}
|
|
667
|
+
await stopCandidate(staged, journal.target, run).catch(() => {});
|
|
553
668
|
const failures = journal.failureCount + 1;
|
|
554
669
|
const updatedAtTs = (options.now ?? Date.now)();
|
|
555
670
|
const recovered = {
|
|
@@ -566,15 +681,23 @@ async function recoverInterruptedAgentUpdate(options = {}) {
|
|
|
566
681
|
}
|
|
567
682
|
function startAgentUpdateHelper(options = {}) {
|
|
568
683
|
const socketPath = options.socketPath ?? DEFAULT_AGENT_UPDATE_SOCKET;
|
|
569
|
-
if (
|
|
570
|
-
|
|
571
|
-
mkdirSync2(
|
|
572
|
-
const setTimer = options.setTimer ?? ((callback, ms) => setTimeout(callback, ms));
|
|
684
|
+
if (existsSync3(socketPath))
|
|
685
|
+
unlinkSync2(socketPath);
|
|
686
|
+
mkdirSync2(dirname3(socketPath), { recursive: true, mode: 488 });
|
|
573
687
|
const receiptPath = options.receiptPath ?? AGENT_UPDATE_RECEIPT;
|
|
574
688
|
const journalPath = options.journalPath ?? AGENT_UPDATE_JOURNAL;
|
|
575
689
|
const releaseRoot = options.root ?? DEFAULT_AGENT_RELEASE_ROOT;
|
|
576
|
-
const
|
|
690
|
+
const setTimer = options.setTimer ?? ((callback, ms) => setTimeout(callback, ms));
|
|
691
|
+
const activate = options.activate ?? ((staged, target, attemptId) => activateAgentRelease(staged, {
|
|
692
|
+
target,
|
|
693
|
+
attemptId,
|
|
694
|
+
journalPath,
|
|
695
|
+
receiptPath,
|
|
696
|
+
now: options.now,
|
|
697
|
+
candidatePrepared: true
|
|
698
|
+
}));
|
|
577
699
|
let busy = true;
|
|
700
|
+
let pending;
|
|
578
701
|
let blocked;
|
|
579
702
|
(options.recover ?? (() => recoverInterruptedAgentUpdate({
|
|
580
703
|
root: releaseRoot,
|
|
@@ -586,7 +709,7 @@ function startAgentUpdateHelper(options = {}) {
|
|
|
586
709
|
}).finally(() => {
|
|
587
710
|
busy = false;
|
|
588
711
|
});
|
|
589
|
-
const server =
|
|
712
|
+
const server = createServer2((socket) => {
|
|
590
713
|
let buffer = "";
|
|
591
714
|
socket.on("data", (chunk) => {
|
|
592
715
|
buffer += chunk.toString("utf8");
|
|
@@ -605,16 +728,52 @@ function startAgentUpdateHelper(options = {}) {
|
|
|
605
728
|
Promise.resolve().then(() => JSON.parse(line)).then(async (request) => {
|
|
606
729
|
if (blocked)
|
|
607
730
|
throw new Error(`update journal needs operator recovery: ${blocked}`);
|
|
608
|
-
if (busy)
|
|
609
|
-
throw new Error("another Agent update or recovery is already active");
|
|
610
|
-
if (request.op !== "apply")
|
|
611
|
-
throw new Error("unknown update operation");
|
|
612
731
|
if (request.target !== "compute" && request.target !== "metal") {
|
|
613
732
|
throw new Error("agent update target is invalid");
|
|
614
733
|
}
|
|
615
|
-
const attemptId = request.attemptId ??
|
|
734
|
+
const attemptId = request.attemptId ?? randomUUID3();
|
|
616
735
|
if (!ATTEMPT_ID.test(attemptId))
|
|
617
736
|
throw new Error("Agent update attempt ID is invalid");
|
|
737
|
+
if (request.op === "commit" || request.op === "abort") {
|
|
738
|
+
if (!pending || pending.attemptId !== attemptId || pending.target !== request.target || pending.staged.fromVersion !== request.currentVersion || pending.staged.version !== request.targetVersion) {
|
|
739
|
+
throw new Error("Agent update handover does not match the prepared candidate");
|
|
740
|
+
}
|
|
741
|
+
const selected = pending;
|
|
742
|
+
pending = undefined;
|
|
743
|
+
if (request.op === "abort") {
|
|
744
|
+
await stopCandidate(selected.staged, selected.target, runCommand);
|
|
745
|
+
busy = false;
|
|
746
|
+
const response3 = {
|
|
747
|
+
ok: true,
|
|
748
|
+
status: "aborted",
|
|
749
|
+
version: selected.staged.version,
|
|
750
|
+
attemptId
|
|
751
|
+
};
|
|
752
|
+
socket.end(`${JSON.stringify(response3)}
|
|
753
|
+
`);
|
|
754
|
+
return;
|
|
755
|
+
}
|
|
756
|
+
const outcome = await activate(selected.staged, selected.target, attemptId);
|
|
757
|
+
busy = false;
|
|
758
|
+
if (outcome?.ok === false)
|
|
759
|
+
throw new Error(outcome.reason ?? "Agent handover failed");
|
|
760
|
+
const response2 = {
|
|
761
|
+
ok: true,
|
|
762
|
+
status: "active",
|
|
763
|
+
version: selected.staged.version,
|
|
764
|
+
attemptId
|
|
765
|
+
};
|
|
766
|
+
socket.end(`${JSON.stringify(response2)}
|
|
767
|
+
`);
|
|
768
|
+
return;
|
|
769
|
+
}
|
|
770
|
+
if (request.op !== "prepare")
|
|
771
|
+
throw new Error("unknown update operation");
|
|
772
|
+
if (typeof request.expectedNodeKey !== "string" || request.expectedNodeKey.length < 16 || Buffer.byteLength(request.expectedNodeKey, "utf8") > 512 || /[\0\r\n]/.test(request.expectedNodeKey)) {
|
|
773
|
+
throw new Error("Agent update expected node identity is invalid");
|
|
774
|
+
}
|
|
775
|
+
if (busy)
|
|
776
|
+
throw new Error("another Agent update or recovery is already active");
|
|
618
777
|
const prior = readJournal(journalPath, releaseRoot);
|
|
619
778
|
const now = (options.now ?? Date.now)();
|
|
620
779
|
if (prior?.targetVersion === request.release.version && (prior.outcome === "rolled-back" || prior.outcome === "failed") && (prior.retryAfterTs ?? 0) > now)
|
|
@@ -625,17 +784,23 @@ function startAgentUpdateHelper(options = {}) {
|
|
|
625
784
|
currentVersion: request.currentVersion,
|
|
626
785
|
root: releaseRoot
|
|
627
786
|
});
|
|
628
|
-
|
|
787
|
+
await startCandidate(staged, request.target, request.expectedNodeKey, runCommand);
|
|
788
|
+
pending = { staged, target: request.target, attemptId, expectedNodeKey: request.expectedNodeKey };
|
|
789
|
+
ownsBusy = false;
|
|
790
|
+
setTimer(() => {
|
|
791
|
+
if (pending?.attemptId !== attemptId)
|
|
792
|
+
return;
|
|
793
|
+
const expired = pending;
|
|
794
|
+
pending = undefined;
|
|
795
|
+
stopCandidate(expired.staged, expired.target, runCommand).finally(() => {
|
|
796
|
+
busy = false;
|
|
797
|
+
});
|
|
798
|
+
}, 180000);
|
|
799
|
+
const response = { ok: true, status: "prepared", version: staged.version, attemptId };
|
|
629
800
|
socket.end(`${JSON.stringify(response)}
|
|
630
801
|
`);
|
|
631
|
-
setTimer(() => void activate(staged, request.target, attemptId).catch((cause) => {
|
|
632
|
-
blocked = cause instanceof Error ? cause.message : String(cause);
|
|
633
|
-
}).finally(() => {
|
|
634
|
-
busy = false;
|
|
635
|
-
}), 100);
|
|
636
|
-
ownsBusy = false;
|
|
637
802
|
}).catch((cause) => {
|
|
638
|
-
if (ownsBusy)
|
|
803
|
+
if (ownsBusy && !pending)
|
|
639
804
|
busy = false;
|
|
640
805
|
const response = {
|
|
641
806
|
ok: false,
|
|
@@ -647,12 +812,12 @@ function startAgentUpdateHelper(options = {}) {
|
|
|
647
812
|
});
|
|
648
813
|
socket.on("error", () => socket.destroy());
|
|
649
814
|
});
|
|
650
|
-
server.listen(socketPath, () =>
|
|
815
|
+
server.listen(socketPath, () => chmodSync3(socketPath, 432));
|
|
651
816
|
return server;
|
|
652
817
|
}
|
|
653
818
|
function requestAgentUpdate(request, socketPath = DEFAULT_AGENT_UPDATE_SOCKET, timeoutMs = 90000) {
|
|
654
819
|
return new Promise((resolve3, reject) => {
|
|
655
|
-
const socket =
|
|
820
|
+
const socket = connect2(socketPath, () => socket.write(`${JSON.stringify(request)}
|
|
656
821
|
`));
|
|
657
822
|
let buffer = "";
|
|
658
823
|
socket.setTimeout(timeoutMs, () => {
|
package/dist/agent-update.d.ts
CHANGED
|
@@ -23,6 +23,7 @@ export interface StagedAgentRelease {
|
|
|
23
23
|
currentLink: string;
|
|
24
24
|
}
|
|
25
25
|
export declare const DEFAULT_AGENT_RELEASE_ROOT = "/opt/forgezero/agent";
|
|
26
|
+
export declare const DEFAULT_AGENT_CANDIDATE_LINK = "/opt/forgezero/agent/candidate";
|
|
26
27
|
export declare const DEFAULT_AGENT_UPDATE_SOCKET = "/run/forgezero-update/helper.sock";
|
|
27
28
|
export declare const MAX_AGENT_TARBALL_BYTES: number;
|
|
28
29
|
/** Root helper input is data, never a command or filesystem path. */
|
|
@@ -45,3 +46,6 @@ export declare function stageAgentRelease(releaseInput: AgentRelease, options: {
|
|
|
45
46
|
export declare function selectAgentRelease(staged: StagedAgentRelease): void;
|
|
46
47
|
/** Roll back only to the exact link target captured before activation. */
|
|
47
48
|
export declare function restoreAgentRelease(staged: StagedAgentRelease): void;
|
|
49
|
+
/** Select a verified candidate without changing the active generation. */
|
|
50
|
+
export declare function selectAgentCandidate(staged: StagedAgentRelease): void;
|
|
51
|
+
export declare function clearAgentCandidate(root?: string): void;
|
package/dist/agent-update.js
CHANGED
|
@@ -16,6 +16,7 @@ import {
|
|
|
16
16
|
} from "node:fs";
|
|
17
17
|
import { dirname, join, resolve } from "node:path";
|
|
18
18
|
var DEFAULT_AGENT_RELEASE_ROOT = "/opt/forgezero/agent";
|
|
19
|
+
var DEFAULT_AGENT_CANDIDATE_LINK = `${DEFAULT_AGENT_RELEASE_ROOT}/candidate`;
|
|
19
20
|
var DEFAULT_AGENT_UPDATE_SOCKET = "/run/forgezero-update/helper.sock";
|
|
20
21
|
var MAX_AGENT_TARBALL_BYTES = 32 * 1024 * 1024;
|
|
21
22
|
var VERSION = /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)$/;
|
|
@@ -209,13 +210,30 @@ function restoreAgentRelease(staged) {
|
|
|
209
210
|
rmSync(next, { force: true });
|
|
210
211
|
}
|
|
211
212
|
}
|
|
213
|
+
function selectAgentCandidate(staged) {
|
|
214
|
+
const link = join(dirname(staged.currentLink), "candidate");
|
|
215
|
+
const next = `${link}.${randomUUID()}.next`;
|
|
216
|
+
try {
|
|
217
|
+
symlinkSync(staged.nextTarget, next);
|
|
218
|
+
renameSync(next, link);
|
|
219
|
+
syncPath(dirname(link));
|
|
220
|
+
} finally {
|
|
221
|
+
rmSync(next, { force: true });
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
function clearAgentCandidate(root = DEFAULT_AGENT_RELEASE_ROOT) {
|
|
225
|
+
rmSync(join(resolve(root), "candidate"), { force: true });
|
|
226
|
+
}
|
|
212
227
|
export {
|
|
213
228
|
validateAgentRelease,
|
|
214
229
|
stageAgentRelease,
|
|
215
230
|
selectAgentRelease,
|
|
231
|
+
selectAgentCandidate,
|
|
216
232
|
restoreAgentRelease,
|
|
217
233
|
compareVersions,
|
|
234
|
+
clearAgentCandidate,
|
|
218
235
|
MAX_AGENT_TARBALL_BYTES,
|
|
219
236
|
DEFAULT_AGENT_UPDATE_SOCKET,
|
|
220
|
-
DEFAULT_AGENT_RELEASE_ROOT
|
|
237
|
+
DEFAULT_AGENT_RELEASE_ROOT,
|
|
238
|
+
DEFAULT_AGENT_CANDIDATE_LINK
|
|
221
239
|
};
|
package/dist/bootstrap.js
CHANGED
|
@@ -1405,9 +1405,13 @@ import { DEFAULT_SOCKET } from "@forgezero/vault";
|
|
|
1405
1405
|
|
|
1406
1406
|
// src/agent-update.ts
|
|
1407
1407
|
var DEFAULT_AGENT_RELEASE_ROOT = "/opt/forgezero/agent";
|
|
1408
|
+
var DEFAULT_AGENT_CANDIDATE_LINK = `${DEFAULT_AGENT_RELEASE_ROOT}/candidate`;
|
|
1408
1409
|
var DEFAULT_AGENT_UPDATE_SOCKET = "/run/forgezero-update/helper.sock";
|
|
1409
1410
|
var MAX_AGENT_TARBALL_BYTES = 32 * 1024 * 1024;
|
|
1410
1411
|
|
|
1412
|
+
// src/agent-handover.ts
|
|
1413
|
+
var DEFAULT_AGENT_CANDIDATE_READY_SOCKET = "/run/forgezero/candidate-ready.sock";
|
|
1414
|
+
|
|
1411
1415
|
// src/agent-update-helper.ts
|
|
1412
1416
|
var AGENT_UPDATE_GROUP = "forgezero-update";
|
|
1413
1417
|
var AGENT_UPDATE_HELPER_UNIT_PATH = "/etc/systemd/system/forgezero-agent-update-helper.service";
|
|
@@ -1416,7 +1420,7 @@ var UPDATE_RETRY_BASE_MS = 5 * 60000;
|
|
|
1416
1420
|
var UPDATE_RETRY_MAX_MS = 24 * 60 * 60000;
|
|
1417
1421
|
|
|
1418
1422
|
// src/version.ts
|
|
1419
|
-
var VERSION = "0.1.
|
|
1423
|
+
var VERSION = "0.1.88";
|
|
1420
1424
|
|
|
1421
1425
|
// src/software.ts
|
|
1422
1426
|
var PINNED_BUN_VERSION = "1.3.14";
|
|
@@ -1674,6 +1678,7 @@ var LIFECYCLE_GROUP = "forgezero-lifecycle";
|
|
|
1674
1678
|
var DEPLOYMENT_RUNNER_UNIT_PATH = "/etc/systemd/system/forgezero-deploy-runner.service";
|
|
1675
1679
|
var AGENT_SOCKET_UNIT_PATH = "/etc/systemd/system/forgezero-agent.socket";
|
|
1676
1680
|
var AGENT_SOCKET_PROXY_UNIT_PATH = "/etc/systemd/system/forgezero-agent-proxy.service";
|
|
1681
|
+
var AGENT_CANDIDATE_UNIT_PATH = "/etc/systemd/system/forgezero-agent-candidate.service";
|
|
1677
1682
|
var DEPLOYMENT_RUNNER_SOCKET = "/run/forgezero-deploy/runner.sock";
|
|
1678
1683
|
var ENROLMENT_UNIT_PATH = "/etc/systemd/system/forgezero-agent-enrol.service";
|
|
1679
1684
|
var LIFECYCLE_HELPER_UNIT_PATH = "/etc/systemd/system/forgezero-lifecycle-helper.service";
|
|
@@ -1785,6 +1790,7 @@ Type=simple
|
|
|
1785
1790
|
User=root
|
|
1786
1791
|
Group=${AGENT_UPDATE_GROUP}
|
|
1787
1792
|
Environment=FZ_AGENT_UPDATE_SOCKET=${DEFAULT_AGENT_UPDATE_SOCKET}
|
|
1793
|
+
Environment=FZ_AGENT_PUBLIC_SOCKET=${systemdPath(options.socketPath, "agent socket")}
|
|
1788
1794
|
ExecStart=${bin} update-helper
|
|
1789
1795
|
Restart=always
|
|
1790
1796
|
RestartSec=2
|
|
@@ -1829,13 +1835,11 @@ WantedBy=sockets.target
|
|
|
1829
1835
|
`;
|
|
1830
1836
|
}
|
|
1831
1837
|
function agentSocketProxyUnit(options) {
|
|
1832
|
-
const backend =
|
|
1838
|
+
const backend = agentRoutingSocketPath(options.socketPath);
|
|
1833
1839
|
const user = options.user ?? "forgezero";
|
|
1834
1840
|
return `[Unit]
|
|
1835
1841
|
Description=ForgeZero application Vault socket proxy
|
|
1836
1842
|
Documentation=https://www.forgezero.net/docs/agent
|
|
1837
|
-
Requires=forgezero-agent.service
|
|
1838
|
-
After=forgezero-agent.service
|
|
1839
1843
|
|
|
1840
1844
|
[Service]
|
|
1841
1845
|
User=${user}
|
|
@@ -1856,12 +1860,26 @@ RestrictAddressFamilies=AF_UNIX
|
|
|
1856
1860
|
`;
|
|
1857
1861
|
}
|
|
1858
1862
|
function agentBackendSocketPath(publicSocketPath) {
|
|
1863
|
+
const socket = systemdPath(publicSocketPath, "agent socket");
|
|
1864
|
+
const backend = `${socket}.backend.active`;
|
|
1865
|
+
if (Buffer.byteLength(backend) > 100)
|
|
1866
|
+
throw new Error("agent socket path is too long for a Unix socket");
|
|
1867
|
+
return backend;
|
|
1868
|
+
}
|
|
1869
|
+
function agentRoutingSocketPath(publicSocketPath) {
|
|
1859
1870
|
const socket = systemdPath(publicSocketPath, "agent socket");
|
|
1860
1871
|
const backend = `${socket}.backend`;
|
|
1861
1872
|
if (Buffer.byteLength(backend) > 100)
|
|
1862
1873
|
throw new Error("agent socket path is too long for a Unix socket");
|
|
1863
1874
|
return backend;
|
|
1864
1875
|
}
|
|
1876
|
+
function agentCandidateSocketPath(publicSocketPath) {
|
|
1877
|
+
const socket = systemdPath(publicSocketPath, "agent socket");
|
|
1878
|
+
const backend = `${socket}.backend.candidate`;
|
|
1879
|
+
if (Buffer.byteLength(backend) > 100)
|
|
1880
|
+
throw new Error("agent socket path is too long for a Unix socket");
|
|
1881
|
+
return backend;
|
|
1882
|
+
}
|
|
1865
1883
|
var systemdPath = (value, label) => {
|
|
1866
1884
|
if (!value || !/^\/[A-Za-z0-9._@/-]+$/.test(value))
|
|
1867
1885
|
throw new Error(`invalid ${label} path`);
|
|
@@ -2145,7 +2163,11 @@ function agentUnit(options) {
|
|
|
2145
2163
|
}
|
|
2146
2164
|
const environment = [
|
|
2147
2165
|
"NODE_ENV=production",
|
|
2148
|
-
`FZ_SOCKET_PATH=${agentBackendSocketPath(options.socketPath)}`,
|
|
2166
|
+
`FZ_SOCKET_PATH=${options.backendSocketPath ?? agentBackendSocketPath(options.socketPath)}`,
|
|
2167
|
+
`FZ_AGENT_PUBLIC_SOCKET=${options.socketPath}`,
|
|
2168
|
+
`FZ_AGENT_ROUTE_SOCKET=${agentRoutingSocketPath(options.socketPath)}`,
|
|
2169
|
+
options.handoverCandidate ? "FZ_AGENT_HANDOVER_CANDIDATE=true" : null,
|
|
2170
|
+
options.handoverCandidate ? `FZ_AGENT_HANDOVER_READY_SOCKET=${DEFAULT_AGENT_CANDIDATE_READY_SOCKET}` : null,
|
|
2149
2171
|
`FZ_CONTROL_SOCKET=${controlSocketPath}`,
|
|
2150
2172
|
`FZ_SEED_CREDENTIAL=agent-seed`,
|
|
2151
2173
|
`FZ_AGENT_MODE=${options.mode}`,
|
|
@@ -2282,6 +2304,26 @@ ${deploymentWrites}
|
|
|
2282
2304
|
WantedBy=multi-user.target
|
|
2283
2305
|
`;
|
|
2284
2306
|
}
|
|
2307
|
+
function agentCandidateUnit(options) {
|
|
2308
|
+
return agentUnit({
|
|
2309
|
+
...options,
|
|
2310
|
+
binPath: `${DEFAULT_AGENT_RELEASE_ROOT}/candidate/dist/fz-agent.js`,
|
|
2311
|
+
backendSocketPath: agentCandidateSocketPath(options.socketPath),
|
|
2312
|
+
handoverCandidate: true,
|
|
2313
|
+
gitCredentialPath: undefined,
|
|
2314
|
+
deploymentCredentials: {},
|
|
2315
|
+
bootstrapSshCredentialPath: undefined,
|
|
2316
|
+
bootstrapSshPublicKeyPath: undefined,
|
|
2317
|
+
pullBootstrap: false,
|
|
2318
|
+
pullDeployments: false,
|
|
2319
|
+
pullMigrations: false,
|
|
2320
|
+
lifecycleProfilePath: undefined,
|
|
2321
|
+
bootstrapTargetTelemetryEndpoint: undefined,
|
|
2322
|
+
repository: undefined,
|
|
2323
|
+
bootstrapBundlePath: undefined,
|
|
2324
|
+
bootstrapBundleManifestPath: undefined
|
|
2325
|
+
}).replace("Description=ForgeZero node agent", "Description=ForgeZero candidate node agent");
|
|
2326
|
+
}
|
|
2285
2327
|
var renderOperation = (operation) => {
|
|
2286
2328
|
if (operation.kind === "commands")
|
|
2287
2329
|
return operation.commands.map(({ argv }) => argv.join(" ")).join(`
|
|
@@ -2417,6 +2459,7 @@ function planProvision(options) {
|
|
|
2417
2459
|
auxiliaryUnits: [
|
|
2418
2460
|
{ path: AGENT_SOCKET_UNIT_PATH, unit: agentSocketUnit(options) },
|
|
2419
2461
|
{ path: AGENT_SOCKET_PROXY_UNIT_PATH, unit: agentSocketProxyUnit(options) },
|
|
2462
|
+
{ path: AGENT_CANDIDATE_UNIT_PATH, unit: agentCandidateUnit(options) },
|
|
2420
2463
|
{ path: AGENT_UPDATE_HELPER_UNIT_PATH, unit: agentUpdateHelperUnit(options) },
|
|
2421
2464
|
...options.enforceEgress ? [
|
|
2422
2465
|
{ path: AGENT_EGRESS_UNIT_PATH, unit: agentEgressUnit(options) }
|