@forgezero/agent 0.1.86 → 0.1.87
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-handover.d.ts +13 -0
- package/dist/agent-heartbeat.d.ts +3 -3
- package/dist/agent-heartbeat.js +231 -60
- package/dist/agent-update-helper.d.ts +11 -2
- package/dist/agent-update-helper.js +201 -45
- package/dist/agent-update.d.ts +4 -0
- package/dist/agent-update.js +19 -1
- package/dist/bootstrap.js +48 -5
- package/dist/fz-agent.js +553 -256
- package/dist/fz.js +87 -5
- package/dist/metal-bootstrap.js +40 -1
- package/dist/operator-bootstrap.js +87 -5
- package/dist/platform-fleet-verification.js +406 -211
- package/dist/provision.d.ts +16 -1
- package/dist/provision.js +323 -124
- package/dist/socket.d.ts +6 -0
- package/dist/version.d.ts +1 -1
- package/package.json +1 -1
|
@@ -16,6 +16,7 @@ import {
|
|
|
16
16
|
} from "node:fs";
|
|
17
17
|
import { dirname, join, resolve } from "node:path";
|
|
18
18
|
var DEFAULT_AGENT_RELEASE_ROOT = "/opt/forgezero/agent";
|
|
19
|
+
var DEFAULT_AGENT_CANDIDATE_LINK = `${DEFAULT_AGENT_RELEASE_ROOT}/candidate`;
|
|
19
20
|
var DEFAULT_AGENT_UPDATE_SOCKET = "/run/forgezero-update/helper.sock";
|
|
20
21
|
var MAX_AGENT_TARBALL_BYTES = 32 * 1024 * 1024;
|
|
21
22
|
var VERSION = /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)$/;
|
|
@@ -209,25 +210,90 @@ function restoreAgentRelease(staged) {
|
|
|
209
210
|
rmSync(next, { force: true });
|
|
210
211
|
}
|
|
211
212
|
}
|
|
213
|
+
function selectAgentCandidate(staged) {
|
|
214
|
+
const link = join(dirname(staged.currentLink), "candidate");
|
|
215
|
+
const next = `${link}.${randomUUID()}.next`;
|
|
216
|
+
try {
|
|
217
|
+
symlinkSync(staged.nextTarget, next);
|
|
218
|
+
renameSync(next, link);
|
|
219
|
+
syncPath(dirname(link));
|
|
220
|
+
} finally {
|
|
221
|
+
rmSync(next, { force: true });
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
function clearAgentCandidate(root = DEFAULT_AGENT_RELEASE_ROOT) {
|
|
225
|
+
rmSync(join(resolve(root), "candidate"), { force: true });
|
|
226
|
+
}
|
|
212
227
|
|
|
213
228
|
// src/agent-update-helper.ts
|
|
214
|
-
import { randomUUID as
|
|
229
|
+
import { randomUUID as randomUUID3 } from "node:crypto";
|
|
215
230
|
import {
|
|
216
|
-
chmodSync as
|
|
231
|
+
chmodSync as chmodSync3,
|
|
217
232
|
closeSync as closeSync2,
|
|
218
|
-
existsSync as
|
|
233
|
+
existsSync as existsSync3,
|
|
219
234
|
fsyncSync as fsyncSync2,
|
|
220
235
|
mkdirSync as mkdirSync2,
|
|
221
236
|
openSync as openSync2,
|
|
222
237
|
readFileSync as readFileSync2,
|
|
223
|
-
renameSync as
|
|
224
|
-
rmSync as
|
|
225
|
-
unlinkSync,
|
|
238
|
+
renameSync as renameSync3,
|
|
239
|
+
rmSync as rmSync3,
|
|
240
|
+
unlinkSync as unlinkSync2,
|
|
226
241
|
writeFileSync as writeFileSync2
|
|
227
242
|
} from "node:fs";
|
|
228
|
-
import { connect, createServer } from "node:net";
|
|
229
|
-
import { dirname as
|
|
243
|
+
import { connect as connect2, createServer as createServer2 } from "node:net";
|
|
244
|
+
import { dirname as dirname3, join as join2, resolve as resolve2 } from "node:path";
|
|
230
245
|
import { DEFAULT_SOCKET } from "@forgezero/vault";
|
|
246
|
+
|
|
247
|
+
// src/agent-handover.ts
|
|
248
|
+
import { randomUUID as randomUUID2 } from "node:crypto";
|
|
249
|
+
import { chmodSync as chmodSync2, existsSync as existsSync2, renameSync as renameSync2, rmSync as rmSync2, symlinkSync as symlinkSync2, unlinkSync } from "node:fs";
|
|
250
|
+
import { basename, dirname as dirname2 } from "node:path";
|
|
251
|
+
import { connect, createServer } from "node:net";
|
|
252
|
+
var DEFAULT_AGENT_CANDIDATE_READY_SOCKET = "/run/forgezero/candidate-ready.sock";
|
|
253
|
+
function switchAgentSocketRoute(route, backend) {
|
|
254
|
+
if (dirname2(route) !== dirname2(backend))
|
|
255
|
+
throw new Error("Agent handover sockets must share one runtime directory");
|
|
256
|
+
const target = basename(backend);
|
|
257
|
+
const next = `${route}.${randomUUID2()}.next`;
|
|
258
|
+
try {
|
|
259
|
+
symlinkSync2(target, next);
|
|
260
|
+
renameSync2(next, route);
|
|
261
|
+
} finally {
|
|
262
|
+
rmSync2(next, { force: true });
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
function probeAgentCandidate(expected, socketPath = DEFAULT_AGENT_CANDIDATE_READY_SOCKET, timeoutMs = 5000) {
|
|
266
|
+
return new Promise((resolve2) => {
|
|
267
|
+
const socket = connect(socketPath);
|
|
268
|
+
let buffer = "";
|
|
269
|
+
let settled = false;
|
|
270
|
+
const finish = (value) => {
|
|
271
|
+
if (settled)
|
|
272
|
+
return;
|
|
273
|
+
settled = true;
|
|
274
|
+
clearTimeout(timer);
|
|
275
|
+
socket.destroy();
|
|
276
|
+
resolve2(value);
|
|
277
|
+
};
|
|
278
|
+
const timer = setTimeout(() => finish(false), timeoutMs);
|
|
279
|
+
socket.on("data", (chunk) => {
|
|
280
|
+
buffer += chunk.toString("utf8");
|
|
281
|
+
const newline = buffer.indexOf(`
|
|
282
|
+
`);
|
|
283
|
+
if (newline < 0)
|
|
284
|
+
return;
|
|
285
|
+
try {
|
|
286
|
+
const value = JSON.parse(buffer.slice(0, newline));
|
|
287
|
+
finish(value.version === expected.version && typeof value.nodeKey === "string" && typeof value.bound === "boolean" && ["ready", "partial", "unavailable", "unbound"].includes(value.vault ?? ""));
|
|
288
|
+
} catch {
|
|
289
|
+
finish(false);
|
|
290
|
+
}
|
|
291
|
+
});
|
|
292
|
+
socket.on("error", () => finish(false));
|
|
293
|
+
});
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
// src/agent-update-helper.ts
|
|
231
297
|
var AGENT_UPDATE_GROUP = "forgezero-update";
|
|
232
298
|
var AGENT_UPDATE_HELPER_UNIT_PATH = "/etc/systemd/system/forgezero-agent-update-helper.service";
|
|
233
299
|
var AGENT_UPDATE_JOURNAL = "/var/lib/forgezero/agent-update.json";
|
|
@@ -322,13 +388,13 @@ function validateJournal(value, root) {
|
|
|
322
388
|
return journal;
|
|
323
389
|
}
|
|
324
390
|
function readJournal(path, root) {
|
|
325
|
-
if (!
|
|
391
|
+
if (!existsSync3(path))
|
|
326
392
|
return;
|
|
327
393
|
return validateJournal(JSON.parse(readFileSync2(path, "utf8")), root);
|
|
328
394
|
}
|
|
329
395
|
function writeAtomic(path, value, mode) {
|
|
330
|
-
mkdirSync2(
|
|
331
|
-
const next = `${path}.${
|
|
396
|
+
mkdirSync2(dirname3(path), { recursive: true, mode: 493 });
|
|
397
|
+
const next = `${path}.${randomUUID3()}.next`;
|
|
332
398
|
let file;
|
|
333
399
|
try {
|
|
334
400
|
file = openSync2(next, "wx", mode);
|
|
@@ -337,8 +403,8 @@ function writeAtomic(path, value, mode) {
|
|
|
337
403
|
fsyncSync2(file);
|
|
338
404
|
closeSync2(file);
|
|
339
405
|
file = undefined;
|
|
340
|
-
|
|
341
|
-
const directory = openSync2(
|
|
406
|
+
renameSync3(next, path);
|
|
407
|
+
const directory = openSync2(dirname3(path), "r");
|
|
342
408
|
try {
|
|
343
409
|
fsyncSync2(directory);
|
|
344
410
|
} finally {
|
|
@@ -347,7 +413,7 @@ function writeAtomic(path, value, mode) {
|
|
|
347
413
|
} finally {
|
|
348
414
|
if (file !== undefined)
|
|
349
415
|
closeSync2(file);
|
|
350
|
-
|
|
416
|
+
rmSync3(next, { force: true });
|
|
351
417
|
}
|
|
352
418
|
}
|
|
353
419
|
var publicReceipt = (journal) => {
|
|
@@ -380,7 +446,7 @@ function writeUpdateState(journalPath, receiptPath, journal) {
|
|
|
380
446
|
}
|
|
381
447
|
function readAgentUpdateReceipt(path = AGENT_UPDATE_RECEIPT) {
|
|
382
448
|
try {
|
|
383
|
-
if (!
|
|
449
|
+
if (!existsSync3(path))
|
|
384
450
|
return;
|
|
385
451
|
return validateReceipt(JSON.parse(readFileSync2(path, "utf8")));
|
|
386
452
|
} catch {
|
|
@@ -420,10 +486,32 @@ var restartAgent = async (target, run) => {
|
|
|
420
486
|
throw new Error(`systemd could not restart ${service}`);
|
|
421
487
|
}
|
|
422
488
|
};
|
|
489
|
+
var candidateUnit = (target) => target === "compute" ? "forgezero-agent-candidate.service" : "forgezero-metal-agent-candidate.service";
|
|
490
|
+
var startCandidate = async (staged, target, run, probe = (expected) => probeAgentCandidate(expected, DEFAULT_AGENT_CANDIDATE_READY_SOCKET)) => {
|
|
491
|
+
selectAgentCandidate(staged);
|
|
492
|
+
if (!await runOk(run, "/usr/bin/systemctl", ["restart", candidateUnit(target)])) {
|
|
493
|
+
clearAgentCandidate(dirname3(staged.currentLink));
|
|
494
|
+
throw new Error(`systemd could not start ${candidateUnit(target)}`);
|
|
495
|
+
}
|
|
496
|
+
if (!await probe({ version: staged.version })) {
|
|
497
|
+
await run({ command: "/usr/bin/systemctl", args: ["stop", candidateUnit(target)] });
|
|
498
|
+
clearAgentCandidate(dirname3(staged.currentLink));
|
|
499
|
+
throw new Error("the candidate Agent did not prove its identity and durable state");
|
|
500
|
+
}
|
|
501
|
+
};
|
|
502
|
+
var stopCandidate = async (staged, target, run) => {
|
|
503
|
+
await run({ command: "/usr/bin/systemctl", args: ["stop", candidateUnit(target)] });
|
|
504
|
+
clearAgentCandidate(dirname3(staged.currentLink));
|
|
505
|
+
};
|
|
506
|
+
var socketPaths = (publicSocket = process.env.FZ_AGENT_PUBLIC_SOCKET ?? DEFAULT_SOCKET) => ({
|
|
507
|
+
route: `${publicSocket}.backend`,
|
|
508
|
+
active: `${publicSocket}.backend.active`,
|
|
509
|
+
candidate: `${publicSocket}.backend.candidate`
|
|
510
|
+
});
|
|
423
511
|
var targetProbe = (target, run) => target === "compute" ? () => probeAgentSocket() : async () => await runOk(run, "/usr/bin/systemctl", ["is-active", "--quiet", "forgezero-metal-agent.service"]) && await runOk(run, "/usr/bin/systemctl", ["is-active", "--quiet", "forgezero-metal-helper.service"]);
|
|
424
512
|
function probeAgentSocket(socketPath = DEFAULT_SOCKET, timeoutMs = 5000) {
|
|
425
513
|
return new Promise((resolve3) => {
|
|
426
|
-
const socket =
|
|
514
|
+
const socket = connect2(socketPath);
|
|
427
515
|
let settled = false;
|
|
428
516
|
let buffer = "";
|
|
429
517
|
const finish = (value) => {
|
|
@@ -456,12 +544,13 @@ function probeAgentSocket(socketPath = DEFAULT_SOCKET, timeoutMs = 5000) {
|
|
|
456
544
|
async function activateAgentRelease(staged, options = {}) {
|
|
457
545
|
const run = options.run ?? runCommand;
|
|
458
546
|
const target = options.target ?? "compute";
|
|
459
|
-
const
|
|
547
|
+
const paths = socketPaths(options.publicSocketPath);
|
|
548
|
+
const probe = options.probe ?? (target === "compute" ? () => probeAgentSocket(paths.active) : targetProbe(target, run));
|
|
460
549
|
const now = options.now ?? Date.now;
|
|
461
550
|
const journalPath = options.journalPath ?? AGENT_UPDATE_JOURNAL;
|
|
462
551
|
const receiptPath = options.receiptPath ?? AGENT_UPDATE_RECEIPT;
|
|
463
|
-
const previous = readJournal(journalPath,
|
|
464
|
-
const attemptId = options.attemptId ??
|
|
552
|
+
const previous = readJournal(journalPath, dirname3(staged.currentLink));
|
|
553
|
+
const attemptId = options.attemptId ?? randomUUID3();
|
|
465
554
|
if (!ATTEMPT_ID.test(attemptId))
|
|
466
555
|
throw new Error("Agent update attempt ID is invalid");
|
|
467
556
|
const startedAtTs = now();
|
|
@@ -483,11 +572,18 @@ async function activateAgentRelease(staged, options = {}) {
|
|
|
483
572
|
writeUpdateState(journalPath, receiptPath, journal);
|
|
484
573
|
let selectionAttempted = false;
|
|
485
574
|
try {
|
|
575
|
+
if (!options.candidatePrepared)
|
|
576
|
+
await startCandidate(staged, target, run);
|
|
577
|
+
if (target === "compute")
|
|
578
|
+
switchAgentSocketRoute(paths.route, paths.candidate);
|
|
486
579
|
selectionAttempted = true;
|
|
487
580
|
selectAgentRelease(staged);
|
|
488
581
|
await restartAgent(target, run);
|
|
489
582
|
if (!await probe())
|
|
490
|
-
throw new Error("the replacement Agent did not answer its
|
|
583
|
+
throw new Error("the replacement Agent did not answer its active Vault backend");
|
|
584
|
+
if (target === "compute")
|
|
585
|
+
switchAgentSocketRoute(paths.route, paths.active);
|
|
586
|
+
await stopCandidate(staged, target, run);
|
|
491
587
|
journal = { ...journal, outcome: "active", updatedAtTs: now(), rollbackHealthy: undefined };
|
|
492
588
|
writeUpdateState(journalPath, receiptPath, journal);
|
|
493
589
|
run({
|
|
@@ -505,10 +601,13 @@ async function activateAgentRelease(staged, options = {}) {
|
|
|
505
601
|
restored = true;
|
|
506
602
|
await restartAgent(target, run);
|
|
507
603
|
rollbackHealthy = await probe();
|
|
604
|
+
if (rollbackHealthy && target === "compute")
|
|
605
|
+
switchAgentSocketRoute(paths.route, paths.active);
|
|
508
606
|
} catch {
|
|
509
607
|
rollbackHealthy = false;
|
|
510
608
|
}
|
|
511
609
|
}
|
|
610
|
+
await stopCandidate(staged, target, run).catch(() => {});
|
|
512
611
|
const failures = failureCount + 1;
|
|
513
612
|
const updatedAtTs = now();
|
|
514
613
|
journal = {
|
|
@@ -528,28 +627,38 @@ async function recoverInterruptedAgentUpdate(options = {}) {
|
|
|
528
627
|
const root = resolve2(options.root ?? DEFAULT_AGENT_RELEASE_ROOT);
|
|
529
628
|
const journalPath = options.journalPath ?? AGENT_UPDATE_JOURNAL;
|
|
530
629
|
const receiptPath = options.receiptPath ?? AGENT_UPDATE_RECEIPT;
|
|
630
|
+
const run = options.run ?? runCommand;
|
|
531
631
|
const journal = readJournal(journalPath, root);
|
|
532
|
-
if (!journal)
|
|
632
|
+
if (!journal) {
|
|
633
|
+
for (const target of ["compute", "metal"]) {
|
|
634
|
+
await run({ command: "/usr/bin/systemctl", args: ["stop", candidateUnit(target)] });
|
|
635
|
+
}
|
|
636
|
+
clearAgentCandidate(root);
|
|
533
637
|
return;
|
|
638
|
+
}
|
|
534
639
|
if (journal.outcome !== "activating") {
|
|
640
|
+
await stopCandidate(stagedFromJournal(journal, root), journal.target, run).catch(() => {});
|
|
535
641
|
writeAtomic(receiptPath, publicReceipt(journal), 416);
|
|
536
642
|
return publicReceipt(journal);
|
|
537
643
|
}
|
|
538
644
|
const staged = stagedFromJournal(journal, root);
|
|
539
|
-
if (!
|
|
645
|
+
if (!existsSync3(join2(root, journal.previousTarget))) {
|
|
540
646
|
throw new Error("Agent update rollback release is missing");
|
|
541
647
|
}
|
|
542
|
-
const
|
|
543
|
-
const probe = options.probe ?? targetProbe(journal.target, run);
|
|
648
|
+
const paths = socketPaths(options.publicSocketPath);
|
|
649
|
+
const probe = options.probe ?? (journal.target === "compute" ? () => probeAgentSocket(paths.active) : targetProbe(journal.target, run));
|
|
544
650
|
restoreAgentRelease(staged);
|
|
545
651
|
let rollbackHealthy = false;
|
|
546
652
|
let failureMessage = "activation was interrupted before its health verdict became durable";
|
|
547
653
|
try {
|
|
548
654
|
await restartAgent(journal.target, run);
|
|
549
655
|
rollbackHealthy = await probe();
|
|
656
|
+
if (rollbackHealthy && journal.target === "compute")
|
|
657
|
+
switchAgentSocketRoute(paths.route, paths.active);
|
|
550
658
|
} catch (cause) {
|
|
551
659
|
failureMessage = `${failureMessage}; ${cause instanceof Error ? cause.message : String(cause)}`;
|
|
552
660
|
}
|
|
661
|
+
await stopCandidate(staged, journal.target, run).catch(() => {});
|
|
553
662
|
const failures = journal.failureCount + 1;
|
|
554
663
|
const updatedAtTs = (options.now ?? Date.now)();
|
|
555
664
|
const recovered = {
|
|
@@ -566,15 +675,23 @@ async function recoverInterruptedAgentUpdate(options = {}) {
|
|
|
566
675
|
}
|
|
567
676
|
function startAgentUpdateHelper(options = {}) {
|
|
568
677
|
const socketPath = options.socketPath ?? DEFAULT_AGENT_UPDATE_SOCKET;
|
|
569
|
-
if (
|
|
570
|
-
|
|
571
|
-
mkdirSync2(
|
|
572
|
-
const setTimer = options.setTimer ?? ((callback, ms) => setTimeout(callback, ms));
|
|
678
|
+
if (existsSync3(socketPath))
|
|
679
|
+
unlinkSync2(socketPath);
|
|
680
|
+
mkdirSync2(dirname3(socketPath), { recursive: true, mode: 488 });
|
|
573
681
|
const receiptPath = options.receiptPath ?? AGENT_UPDATE_RECEIPT;
|
|
574
682
|
const journalPath = options.journalPath ?? AGENT_UPDATE_JOURNAL;
|
|
575
683
|
const releaseRoot = options.root ?? DEFAULT_AGENT_RELEASE_ROOT;
|
|
576
|
-
const
|
|
684
|
+
const setTimer = options.setTimer ?? ((callback, ms) => setTimeout(callback, ms));
|
|
685
|
+
const activate = options.activate ?? ((staged, target, attemptId) => activateAgentRelease(staged, {
|
|
686
|
+
target,
|
|
687
|
+
attemptId,
|
|
688
|
+
journalPath,
|
|
689
|
+
receiptPath,
|
|
690
|
+
now: options.now,
|
|
691
|
+
candidatePrepared: true
|
|
692
|
+
}));
|
|
577
693
|
let busy = true;
|
|
694
|
+
let pending;
|
|
578
695
|
let blocked;
|
|
579
696
|
(options.recover ?? (() => recoverInterruptedAgentUpdate({
|
|
580
697
|
root: releaseRoot,
|
|
@@ -586,7 +703,7 @@ function startAgentUpdateHelper(options = {}) {
|
|
|
586
703
|
}).finally(() => {
|
|
587
704
|
busy = false;
|
|
588
705
|
});
|
|
589
|
-
const server =
|
|
706
|
+
const server = createServer2((socket) => {
|
|
590
707
|
let buffer = "";
|
|
591
708
|
socket.on("data", (chunk) => {
|
|
592
709
|
buffer += chunk.toString("utf8");
|
|
@@ -605,16 +722,49 @@ function startAgentUpdateHelper(options = {}) {
|
|
|
605
722
|
Promise.resolve().then(() => JSON.parse(line)).then(async (request) => {
|
|
606
723
|
if (blocked)
|
|
607
724
|
throw new Error(`update journal needs operator recovery: ${blocked}`);
|
|
608
|
-
if (busy)
|
|
609
|
-
throw new Error("another Agent update or recovery is already active");
|
|
610
|
-
if (request.op !== "apply")
|
|
611
|
-
throw new Error("unknown update operation");
|
|
612
725
|
if (request.target !== "compute" && request.target !== "metal") {
|
|
613
726
|
throw new Error("agent update target is invalid");
|
|
614
727
|
}
|
|
615
|
-
const attemptId = request.attemptId ??
|
|
728
|
+
const attemptId = request.attemptId ?? randomUUID3();
|
|
616
729
|
if (!ATTEMPT_ID.test(attemptId))
|
|
617
730
|
throw new Error("Agent update attempt ID is invalid");
|
|
731
|
+
if (request.op === "commit" || request.op === "abort") {
|
|
732
|
+
if (!pending || pending.attemptId !== attemptId || pending.target !== request.target || pending.staged.fromVersion !== request.currentVersion || pending.staged.version !== request.targetVersion) {
|
|
733
|
+
throw new Error("Agent update handover does not match the prepared candidate");
|
|
734
|
+
}
|
|
735
|
+
const selected = pending;
|
|
736
|
+
pending = undefined;
|
|
737
|
+
if (request.op === "abort") {
|
|
738
|
+
await stopCandidate(selected.staged, selected.target, runCommand);
|
|
739
|
+
busy = false;
|
|
740
|
+
const response3 = {
|
|
741
|
+
ok: true,
|
|
742
|
+
status: "aborted",
|
|
743
|
+
version: selected.staged.version,
|
|
744
|
+
attemptId
|
|
745
|
+
};
|
|
746
|
+
socket.end(`${JSON.stringify(response3)}
|
|
747
|
+
`);
|
|
748
|
+
return;
|
|
749
|
+
}
|
|
750
|
+
const outcome = await activate(selected.staged, selected.target, attemptId);
|
|
751
|
+
busy = false;
|
|
752
|
+
if (outcome?.ok === false)
|
|
753
|
+
throw new Error(outcome.reason ?? "Agent handover failed");
|
|
754
|
+
const response2 = {
|
|
755
|
+
ok: true,
|
|
756
|
+
status: "active",
|
|
757
|
+
version: selected.staged.version,
|
|
758
|
+
attemptId
|
|
759
|
+
};
|
|
760
|
+
socket.end(`${JSON.stringify(response2)}
|
|
761
|
+
`);
|
|
762
|
+
return;
|
|
763
|
+
}
|
|
764
|
+
if (request.op !== "prepare")
|
|
765
|
+
throw new Error("unknown update operation");
|
|
766
|
+
if (busy)
|
|
767
|
+
throw new Error("another Agent update or recovery is already active");
|
|
618
768
|
const prior = readJournal(journalPath, releaseRoot);
|
|
619
769
|
const now = (options.now ?? Date.now)();
|
|
620
770
|
if (prior?.targetVersion === request.release.version && (prior.outcome === "rolled-back" || prior.outcome === "failed") && (prior.retryAfterTs ?? 0) > now)
|
|
@@ -625,17 +775,23 @@ function startAgentUpdateHelper(options = {}) {
|
|
|
625
775
|
currentVersion: request.currentVersion,
|
|
626
776
|
root: releaseRoot
|
|
627
777
|
});
|
|
628
|
-
|
|
778
|
+
await startCandidate(staged, request.target, runCommand);
|
|
779
|
+
pending = { staged, target: request.target, attemptId };
|
|
780
|
+
ownsBusy = false;
|
|
781
|
+
setTimer(() => {
|
|
782
|
+
if (pending?.attemptId !== attemptId)
|
|
783
|
+
return;
|
|
784
|
+
const expired = pending;
|
|
785
|
+
pending = undefined;
|
|
786
|
+
stopCandidate(expired.staged, expired.target, runCommand).finally(() => {
|
|
787
|
+
busy = false;
|
|
788
|
+
});
|
|
789
|
+
}, 180000);
|
|
790
|
+
const response = { ok: true, status: "prepared", version: staged.version, attemptId };
|
|
629
791
|
socket.end(`${JSON.stringify(response)}
|
|
630
792
|
`);
|
|
631
|
-
setTimer(() => void activate(staged, request.target, attemptId).catch((cause) => {
|
|
632
|
-
blocked = cause instanceof Error ? cause.message : String(cause);
|
|
633
|
-
}).finally(() => {
|
|
634
|
-
busy = false;
|
|
635
|
-
}), 100);
|
|
636
|
-
ownsBusy = false;
|
|
637
793
|
}).catch((cause) => {
|
|
638
|
-
if (ownsBusy)
|
|
794
|
+
if (ownsBusy && !pending)
|
|
639
795
|
busy = false;
|
|
640
796
|
const response = {
|
|
641
797
|
ok: false,
|
|
@@ -647,12 +803,12 @@ function startAgentUpdateHelper(options = {}) {
|
|
|
647
803
|
});
|
|
648
804
|
socket.on("error", () => socket.destroy());
|
|
649
805
|
});
|
|
650
|
-
server.listen(socketPath, () =>
|
|
806
|
+
server.listen(socketPath, () => chmodSync3(socketPath, 432));
|
|
651
807
|
return server;
|
|
652
808
|
}
|
|
653
809
|
function requestAgentUpdate(request, socketPath = DEFAULT_AGENT_UPDATE_SOCKET, timeoutMs = 90000) {
|
|
654
810
|
return new Promise((resolve3, reject) => {
|
|
655
|
-
const socket =
|
|
811
|
+
const socket = connect2(socketPath, () => socket.write(`${JSON.stringify(request)}
|
|
656
812
|
`));
|
|
657
813
|
let buffer = "";
|
|
658
814
|
socket.setTimeout(timeoutMs, () => {
|
package/dist/agent-update.d.ts
CHANGED
|
@@ -23,6 +23,7 @@ export interface StagedAgentRelease {
|
|
|
23
23
|
currentLink: string;
|
|
24
24
|
}
|
|
25
25
|
export declare const DEFAULT_AGENT_RELEASE_ROOT = "/opt/forgezero/agent";
|
|
26
|
+
export declare const DEFAULT_AGENT_CANDIDATE_LINK = "/opt/forgezero/agent/candidate";
|
|
26
27
|
export declare const DEFAULT_AGENT_UPDATE_SOCKET = "/run/forgezero-update/helper.sock";
|
|
27
28
|
export declare const MAX_AGENT_TARBALL_BYTES: number;
|
|
28
29
|
/** Root helper input is data, never a command or filesystem path. */
|
|
@@ -45,3 +46,6 @@ export declare function stageAgentRelease(releaseInput: AgentRelease, options: {
|
|
|
45
46
|
export declare function selectAgentRelease(staged: StagedAgentRelease): void;
|
|
46
47
|
/** Roll back only to the exact link target captured before activation. */
|
|
47
48
|
export declare function restoreAgentRelease(staged: StagedAgentRelease): void;
|
|
49
|
+
/** Select a verified candidate without changing the active generation. */
|
|
50
|
+
export declare function selectAgentCandidate(staged: StagedAgentRelease): void;
|
|
51
|
+
export declare function clearAgentCandidate(root?: string): void;
|
package/dist/agent-update.js
CHANGED
|
@@ -16,6 +16,7 @@ import {
|
|
|
16
16
|
} from "node:fs";
|
|
17
17
|
import { dirname, join, resolve } from "node:path";
|
|
18
18
|
var DEFAULT_AGENT_RELEASE_ROOT = "/opt/forgezero/agent";
|
|
19
|
+
var DEFAULT_AGENT_CANDIDATE_LINK = `${DEFAULT_AGENT_RELEASE_ROOT}/candidate`;
|
|
19
20
|
var DEFAULT_AGENT_UPDATE_SOCKET = "/run/forgezero-update/helper.sock";
|
|
20
21
|
var MAX_AGENT_TARBALL_BYTES = 32 * 1024 * 1024;
|
|
21
22
|
var VERSION = /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)$/;
|
|
@@ -209,13 +210,30 @@ function restoreAgentRelease(staged) {
|
|
|
209
210
|
rmSync(next, { force: true });
|
|
210
211
|
}
|
|
211
212
|
}
|
|
213
|
+
function selectAgentCandidate(staged) {
|
|
214
|
+
const link = join(dirname(staged.currentLink), "candidate");
|
|
215
|
+
const next = `${link}.${randomUUID()}.next`;
|
|
216
|
+
try {
|
|
217
|
+
symlinkSync(staged.nextTarget, next);
|
|
218
|
+
renameSync(next, link);
|
|
219
|
+
syncPath(dirname(link));
|
|
220
|
+
} finally {
|
|
221
|
+
rmSync(next, { force: true });
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
function clearAgentCandidate(root = DEFAULT_AGENT_RELEASE_ROOT) {
|
|
225
|
+
rmSync(join(resolve(root), "candidate"), { force: true });
|
|
226
|
+
}
|
|
212
227
|
export {
|
|
213
228
|
validateAgentRelease,
|
|
214
229
|
stageAgentRelease,
|
|
215
230
|
selectAgentRelease,
|
|
231
|
+
selectAgentCandidate,
|
|
216
232
|
restoreAgentRelease,
|
|
217
233
|
compareVersions,
|
|
234
|
+
clearAgentCandidate,
|
|
218
235
|
MAX_AGENT_TARBALL_BYTES,
|
|
219
236
|
DEFAULT_AGENT_UPDATE_SOCKET,
|
|
220
|
-
DEFAULT_AGENT_RELEASE_ROOT
|
|
237
|
+
DEFAULT_AGENT_RELEASE_ROOT,
|
|
238
|
+
DEFAULT_AGENT_CANDIDATE_LINK
|
|
221
239
|
};
|
package/dist/bootstrap.js
CHANGED
|
@@ -1405,9 +1405,13 @@ import { DEFAULT_SOCKET } from "@forgezero/vault";
|
|
|
1405
1405
|
|
|
1406
1406
|
// src/agent-update.ts
|
|
1407
1407
|
var DEFAULT_AGENT_RELEASE_ROOT = "/opt/forgezero/agent";
|
|
1408
|
+
var DEFAULT_AGENT_CANDIDATE_LINK = `${DEFAULT_AGENT_RELEASE_ROOT}/candidate`;
|
|
1408
1409
|
var DEFAULT_AGENT_UPDATE_SOCKET = "/run/forgezero-update/helper.sock";
|
|
1409
1410
|
var MAX_AGENT_TARBALL_BYTES = 32 * 1024 * 1024;
|
|
1410
1411
|
|
|
1412
|
+
// src/agent-handover.ts
|
|
1413
|
+
var DEFAULT_AGENT_CANDIDATE_READY_SOCKET = "/run/forgezero/candidate-ready.sock";
|
|
1414
|
+
|
|
1411
1415
|
// src/agent-update-helper.ts
|
|
1412
1416
|
var AGENT_UPDATE_GROUP = "forgezero-update";
|
|
1413
1417
|
var AGENT_UPDATE_HELPER_UNIT_PATH = "/etc/systemd/system/forgezero-agent-update-helper.service";
|
|
@@ -1416,7 +1420,7 @@ var UPDATE_RETRY_BASE_MS = 5 * 60000;
|
|
|
1416
1420
|
var UPDATE_RETRY_MAX_MS = 24 * 60 * 60000;
|
|
1417
1421
|
|
|
1418
1422
|
// src/version.ts
|
|
1419
|
-
var VERSION = "0.1.
|
|
1423
|
+
var VERSION = "0.1.87";
|
|
1420
1424
|
|
|
1421
1425
|
// src/software.ts
|
|
1422
1426
|
var PINNED_BUN_VERSION = "1.3.14";
|
|
@@ -1674,6 +1678,7 @@ var LIFECYCLE_GROUP = "forgezero-lifecycle";
|
|
|
1674
1678
|
var DEPLOYMENT_RUNNER_UNIT_PATH = "/etc/systemd/system/forgezero-deploy-runner.service";
|
|
1675
1679
|
var AGENT_SOCKET_UNIT_PATH = "/etc/systemd/system/forgezero-agent.socket";
|
|
1676
1680
|
var AGENT_SOCKET_PROXY_UNIT_PATH = "/etc/systemd/system/forgezero-agent-proxy.service";
|
|
1681
|
+
var AGENT_CANDIDATE_UNIT_PATH = "/etc/systemd/system/forgezero-agent-candidate.service";
|
|
1677
1682
|
var DEPLOYMENT_RUNNER_SOCKET = "/run/forgezero-deploy/runner.sock";
|
|
1678
1683
|
var ENROLMENT_UNIT_PATH = "/etc/systemd/system/forgezero-agent-enrol.service";
|
|
1679
1684
|
var LIFECYCLE_HELPER_UNIT_PATH = "/etc/systemd/system/forgezero-lifecycle-helper.service";
|
|
@@ -1785,6 +1790,7 @@ Type=simple
|
|
|
1785
1790
|
User=root
|
|
1786
1791
|
Group=${AGENT_UPDATE_GROUP}
|
|
1787
1792
|
Environment=FZ_AGENT_UPDATE_SOCKET=${DEFAULT_AGENT_UPDATE_SOCKET}
|
|
1793
|
+
Environment=FZ_AGENT_PUBLIC_SOCKET=${systemdPath(options.socketPath, "agent socket")}
|
|
1788
1794
|
ExecStart=${bin} update-helper
|
|
1789
1795
|
Restart=always
|
|
1790
1796
|
RestartSec=2
|
|
@@ -1829,13 +1835,11 @@ WantedBy=sockets.target
|
|
|
1829
1835
|
`;
|
|
1830
1836
|
}
|
|
1831
1837
|
function agentSocketProxyUnit(options) {
|
|
1832
|
-
const backend =
|
|
1838
|
+
const backend = agentRoutingSocketPath(options.socketPath);
|
|
1833
1839
|
const user = options.user ?? "forgezero";
|
|
1834
1840
|
return `[Unit]
|
|
1835
1841
|
Description=ForgeZero application Vault socket proxy
|
|
1836
1842
|
Documentation=https://www.forgezero.net/docs/agent
|
|
1837
|
-
Requires=forgezero-agent.service
|
|
1838
|
-
After=forgezero-agent.service
|
|
1839
1843
|
|
|
1840
1844
|
[Service]
|
|
1841
1845
|
User=${user}
|
|
@@ -1856,12 +1860,26 @@ RestrictAddressFamilies=AF_UNIX
|
|
|
1856
1860
|
`;
|
|
1857
1861
|
}
|
|
1858
1862
|
function agentBackendSocketPath(publicSocketPath) {
|
|
1863
|
+
const socket = systemdPath(publicSocketPath, "agent socket");
|
|
1864
|
+
const backend = `${socket}.backend.active`;
|
|
1865
|
+
if (Buffer.byteLength(backend) > 100)
|
|
1866
|
+
throw new Error("agent socket path is too long for a Unix socket");
|
|
1867
|
+
return backend;
|
|
1868
|
+
}
|
|
1869
|
+
function agentRoutingSocketPath(publicSocketPath) {
|
|
1859
1870
|
const socket = systemdPath(publicSocketPath, "agent socket");
|
|
1860
1871
|
const backend = `${socket}.backend`;
|
|
1861
1872
|
if (Buffer.byteLength(backend) > 100)
|
|
1862
1873
|
throw new Error("agent socket path is too long for a Unix socket");
|
|
1863
1874
|
return backend;
|
|
1864
1875
|
}
|
|
1876
|
+
function agentCandidateSocketPath(publicSocketPath) {
|
|
1877
|
+
const socket = systemdPath(publicSocketPath, "agent socket");
|
|
1878
|
+
const backend = `${socket}.backend.candidate`;
|
|
1879
|
+
if (Buffer.byteLength(backend) > 100)
|
|
1880
|
+
throw new Error("agent socket path is too long for a Unix socket");
|
|
1881
|
+
return backend;
|
|
1882
|
+
}
|
|
1865
1883
|
var systemdPath = (value, label) => {
|
|
1866
1884
|
if (!value || !/^\/[A-Za-z0-9._@/-]+$/.test(value))
|
|
1867
1885
|
throw new Error(`invalid ${label} path`);
|
|
@@ -2145,7 +2163,11 @@ function agentUnit(options) {
|
|
|
2145
2163
|
}
|
|
2146
2164
|
const environment = [
|
|
2147
2165
|
"NODE_ENV=production",
|
|
2148
|
-
`FZ_SOCKET_PATH=${agentBackendSocketPath(options.socketPath)}`,
|
|
2166
|
+
`FZ_SOCKET_PATH=${options.backendSocketPath ?? agentBackendSocketPath(options.socketPath)}`,
|
|
2167
|
+
`FZ_AGENT_PUBLIC_SOCKET=${options.socketPath}`,
|
|
2168
|
+
`FZ_AGENT_ROUTE_SOCKET=${agentRoutingSocketPath(options.socketPath)}`,
|
|
2169
|
+
options.handoverCandidate ? "FZ_AGENT_HANDOVER_CANDIDATE=true" : null,
|
|
2170
|
+
options.handoverCandidate ? `FZ_AGENT_HANDOVER_READY_SOCKET=${DEFAULT_AGENT_CANDIDATE_READY_SOCKET}` : null,
|
|
2149
2171
|
`FZ_CONTROL_SOCKET=${controlSocketPath}`,
|
|
2150
2172
|
`FZ_SEED_CREDENTIAL=agent-seed`,
|
|
2151
2173
|
`FZ_AGENT_MODE=${options.mode}`,
|
|
@@ -2282,6 +2304,26 @@ ${deploymentWrites}
|
|
|
2282
2304
|
WantedBy=multi-user.target
|
|
2283
2305
|
`;
|
|
2284
2306
|
}
|
|
2307
|
+
function agentCandidateUnit(options) {
|
|
2308
|
+
return agentUnit({
|
|
2309
|
+
...options,
|
|
2310
|
+
binPath: `${DEFAULT_AGENT_RELEASE_ROOT}/candidate/dist/fz-agent.js`,
|
|
2311
|
+
backendSocketPath: agentCandidateSocketPath(options.socketPath),
|
|
2312
|
+
handoverCandidate: true,
|
|
2313
|
+
gitCredentialPath: undefined,
|
|
2314
|
+
deploymentCredentials: {},
|
|
2315
|
+
bootstrapSshCredentialPath: undefined,
|
|
2316
|
+
bootstrapSshPublicKeyPath: undefined,
|
|
2317
|
+
pullBootstrap: false,
|
|
2318
|
+
pullDeployments: false,
|
|
2319
|
+
pullMigrations: false,
|
|
2320
|
+
lifecycleProfilePath: undefined,
|
|
2321
|
+
bootstrapTargetTelemetryEndpoint: undefined,
|
|
2322
|
+
repository: undefined,
|
|
2323
|
+
bootstrapBundlePath: undefined,
|
|
2324
|
+
bootstrapBundleManifestPath: undefined
|
|
2325
|
+
}).replace("Description=ForgeZero node agent", "Description=ForgeZero candidate node agent");
|
|
2326
|
+
}
|
|
2285
2327
|
var renderOperation = (operation) => {
|
|
2286
2328
|
if (operation.kind === "commands")
|
|
2287
2329
|
return operation.commands.map(({ argv }) => argv.join(" ")).join(`
|
|
@@ -2417,6 +2459,7 @@ function planProvision(options) {
|
|
|
2417
2459
|
auxiliaryUnits: [
|
|
2418
2460
|
{ path: AGENT_SOCKET_UNIT_PATH, unit: agentSocketUnit(options) },
|
|
2419
2461
|
{ path: AGENT_SOCKET_PROXY_UNIT_PATH, unit: agentSocketProxyUnit(options) },
|
|
2462
|
+
{ path: AGENT_CANDIDATE_UNIT_PATH, unit: agentCandidateUnit(options) },
|
|
2420
2463
|
{ path: AGENT_UPDATE_HELPER_UNIT_PATH, unit: agentUpdateHelperUnit(options) },
|
|
2421
2464
|
...options.enforceEgress ? [
|
|
2422
2465
|
{ path: AGENT_EGRESS_UNIT_PATH, unit: agentEgressUnit(options) }
|