@forgezero/agent 0.1.85 → 0.1.87
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -14
- package/dist/agent-handover.d.ts +13 -0
- package/dist/agent-heartbeat.d.ts +3 -3
- package/dist/agent-heartbeat.js +235 -62
- package/dist/agent-update-helper.d.ts +11 -2
- package/dist/agent-update-helper.js +201 -45
- package/dist/agent-update.d.ts +4 -0
- package/dist/agent-update.js +19 -1
- package/dist/bootstrap.js +56 -8
- package/dist/credential-schema.js +16 -4
- package/dist/fz-agent.js +569 -260
- package/dist/fz.js +415 -43
- package/dist/guest-enrolment.js +4 -2
- package/dist/metal-bootstrap.js +40 -1
- package/dist/migration-pull.js +4 -2
- package/dist/node-vault.js +16 -4
- package/dist/operator-bootstrap.js +95 -8
- package/dist/platform-bootstrap-runtime.d.ts +7 -0
- package/dist/platform-bootstrap-runtime.js +5 -2
- package/dist/platform-fleet-verification.js +411 -213
- package/dist/platform-genesis-config.d.ts +6 -1
- package/dist/platform-launch-profile.d.ts +44 -0
- package/dist/provision.d.ts +16 -1
- package/dist/provision.js +323 -124
- package/dist/provisioning-pull.js +4 -2
- package/dist/signed-node-http.d.ts +2 -1
- package/dist/socket.d.ts +6 -0
- package/dist/version.d.ts +1 -1
- package/package.json +4 -4
package/README.md
CHANGED
|
@@ -11,7 +11,7 @@ The node agent deploys on a compute; tenant mode also holds one complete project
|
|
|
11
11
|
|
|
12
12
|
## Package overview
|
|
13
13
|
|
|
14
|
-
Every managed compute. It owns node identity, attestation and repository pipelines.
|
|
14
|
+
Every managed compute. It owns node identity, attestation and repository pipelines. An SNP compute may hold its assigned whole project across environments in RAM after current attestation. Ordinary compute is API-only unless an administrator explicitly records the versioned non-TEE risk acceptance for that concrete machine. Platform mode never replicates realm-wide API authority beside the process that already holds unlocked realm seeds. Supported runtimes: bun, node. Package root: @forgezero/agent. The sections below show the actual named imports emitted by each declaration entry point; wildcard imports are intentionally not used in the documentation.
|
|
15
15
|
|
|
16
16
|
```text
|
|
17
17
|
bun add -g @forgezero/agent
|
|
@@ -38,7 +38,7 @@ Every row links to the detailed explanation and named-import/example area below.
|
|
|
38
38
|
| fz | The operator command: keys, status, genesis, unlock, run, and agent install. | Bun/Node host | [Reference + usage](#fz) |
|
|
39
39
|
| fz run | The fallback for software that cannot read the local socket: start a process with scoped vault values in its environment. | Bun/Node host | [Reference + usage](#fz-run) |
|
|
40
40
|
| fz agent install | Provision the same hardened agent service locally that the platform provisions remotely. | Bun/Node host | [Reference + usage](#fz-agent-install) |
|
|
41
|
-
| fz-agent | The service entry point. Identity and private Git material arrive only as systemd credentials. A platform bootstrap may use a statically configured local control socket; an enrolled tenant guest builds each deployment manager from its signed server claim and reads all commands from the checked-out .fz definition. The signed enrolment binding constructs the exact project Vault
|
|
41
|
+
| fz-agent | The service entry point. Identity and private Git material arrive only as systemd credentials. A platform bootstrap may use a statically configured local control socket; an enrolled tenant guest builds each deployment manager from its signed server claim and reads all commands from the checked-out .fz definition. The signed enrolment binding constructs the exact project scope. Vault synchronization succeeds only when the API sees current SNP evidence or an audited versioned non-TEE acceptance; ordinary compute otherwise remains API-only and a revocation clears its resident cache. | Bun/Node host | [Reference + usage](#fz-agent) |
|
|
42
42
|
| fz-agent (socket) | identity, sign, attest, get, sync and held — the whole interface an application sees. | Bun/Node host | [Reference + usage](#fz-agent-socket) |
|
|
43
43
|
| @forgezero/agent/migration-pull | PQ-authenticated outbound lifecycle claims with fenced renewal, awaited async execution, idempotent acknowledgement and bounded shutdown drain. The executor reports local facts only. | Bun/Node host | [Reference + usage](#forgezero-agent-migration-pull) |
|
|
44
44
|
| @forgezero/agent/lifecycle-helper | The root-owned fixed-operation boundary for migration work. Signed controller claims can request only private reachability probes, local health checks, or stops of units named in a root-owned profile; they cannot carry commands. | Bun/Node host | [Reference + usage](#forgezero-agent-lifecycle-helper) |
|
|
@@ -79,7 +79,7 @@ Every row links to the detailed explanation and named-import/example area below.
|
|
|
79
79
|
| @forgezero/agent/software | Exact managed-software inventory, requirement and evidence contracts. | Bun/Node host | [Reference + usage](#forgezero-agent-software) |
|
|
80
80
|
| @forgezero/agent/software-helper | Root-owned fixed installation/update boundary for declared software. | Bun/Node host | [Reference + usage](#forgezero-agent-software-helper) |
|
|
81
81
|
| @forgezero/agent/ubuntu | The closed guest-image view: pinned Ubuntu 26.04 LTS x64 supports ordinary or SEV-SNP compute, while pinned Ubuntu 24.04 LTS x64 is limited to ordinary non-confidential compute. Every other OS remains rejected until its own pinned strategy and acceptance evidence are added. | Bun/Node host | [Reference + usage](#forgezero-agent-ubuntu) |
|
|
82
|
-
| fz-agent (replica) | The project-scoped vault replica: resident in RAM, never on disk, invalidated by cursor rather than by timer, and refusing a read it can no longer vouch for. | Bun/Node host | [Reference + usage](#fz-agent-replica) |
|
|
82
|
+
| fz-agent (replica) | The authorized project-scoped vault replica: resident in RAM, never on disk, invalidated by cursor rather than by timer, cleared when local authority is revoked, and refusing a read it can no longer vouch for. | Bun/Node host | [Reference + usage](#fz-agent-replica) |
|
|
83
83
|
|
|
84
84
|
## Commands
|
|
85
85
|
|
|
@@ -89,8 +89,8 @@ fz deploy init --runtime containerd --os ubuntu-24.04 --replicas 1 --cpu-cores 2
|
|
|
89
89
|
fz deploy init --runtime kata-snp --os ubuntu-26.04 --replicas 1 --cpu-cores 2 --memory-mib 4096 --storage-gib 40 --reuse require — Create one Kata QEMU SEV-SNP definition on existing eligible capacity. Use reuse prefer only with the explicit provisioning and spend-ceiling flags shown by fz deploy init --help.
|
|
90
90
|
fz deploy compile — Compile and validate TypeScript deployment intent into .fz/deploy.plan.json.
|
|
91
91
|
fz deploy check — Refuse stale plans, unsafe provider/action coordinates and unresolved deployment blockers.
|
|
92
|
-
fz bootstrap platform
|
|
93
|
-
fz bootstrap platform
|
|
92
|
+
fz bootstrap platform launch --profile development --apply — Run the one attended ForgeZero launch: public topology and bundle are derived; answer only email settings, hidden provider tokens and APPLY.
|
|
93
|
+
fz bootstrap platform fleet apply --bootstrap-config /secure/forgezero/development/genesis/platform-fleet-remote.json — Resume or diagnose an already generated fleet plan; this is not the normal first-launch path.
|
|
94
94
|
fz bootstrap platform --help — Inspect attended platform bootstrap inputs before applying them.
|
|
95
95
|
fz bootstrap metal --help — Inspect the identity-only physical provisioner bootstrap.
|
|
96
96
|
fz host transactions|schema-plan|schema-apply — Plan fixed supervised database maintenance; add --apply only after reviewing the exact argv and credential names.
|
|
@@ -103,8 +103,8 @@ fz deploy init --runtime containerd --os ubuntu-24.04 --replicas 1 --cpu-cores 2
|
|
|
103
103
|
fz deploy init --runtime kata-snp --os ubuntu-26.04 --replicas 1 --cpu-cores 2 --memory-mib 4096 --storage-gib 40 --reuse require
|
|
104
104
|
fz deploy compile
|
|
105
105
|
fz deploy check
|
|
106
|
-
fz bootstrap platform
|
|
107
|
-
fz bootstrap platform
|
|
106
|
+
fz bootstrap platform launch --profile development --apply
|
|
107
|
+
fz bootstrap platform fleet apply --bootstrap-config /secure/forgezero/development/genesis/platform-fleet-remote.json
|
|
108
108
|
fz bootstrap platform --help
|
|
109
109
|
fz bootstrap metal --help
|
|
110
110
|
fz host transactions|schema-plan|schema-apply
|
|
@@ -167,7 +167,7 @@ fz agent install --help
|
|
|
167
167
|
<a id="fz-agent"></a>
|
|
168
168
|
## fz-agent
|
|
169
169
|
|
|
170
|
-
The service entry point. Identity and private Git material arrive only as systemd credentials. A platform bootstrap may use a statically configured local control socket; an enrolled tenant guest builds each deployment manager from its signed server claim and reads all commands from the checked-out .fz definition. The signed enrolment binding constructs the exact project Vault
|
|
170
|
+
The service entry point. Identity and private Git material arrive only as systemd credentials. A platform bootstrap may use a statically configured local control socket; an enrolled tenant guest builds each deployment manager from its signed server claim and reads all commands from the checked-out .fz definition. The signed enrolment binding constructs the exact project scope. Vault synchronization succeeds only when the API sees current SNP evidence or an audited versioned non-TEE acceptance; ordinary compute otherwise remains API-only and a revocation clears its resident cache. This entry requires a Bun/Node host runtime. Run the command with `--help` before applying it.
|
|
171
171
|
|
|
172
172
|
```text
|
|
173
173
|
fz-agent --help
|
|
@@ -755,14 +755,13 @@ import type {
|
|
|
755
755
|
} from '@forgezero/agent/operator-bootstrap';
|
|
756
756
|
```
|
|
757
757
|
|
|
758
|
-
## @forgezero/agent/operator-bootstrap —
|
|
758
|
+
## @forgezero/agent/operator-bootstrap — Launch the platform from an operator laptop
|
|
759
759
|
|
|
760
|
-
The
|
|
760
|
+
The selected ForgeZero profile and pinned Metal evidence derive every public coordinate. The CLI builds the clean API bundle, retains cluster material only as an owner-host encrypted systemd credential for safe resume, deploys all three computes, seals hidden credentials, seeds inventory and prints the invitation. Low-level file commands are reserved for resume and repair.
|
|
761
761
|
|
|
762
762
|
```text
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
fz bootstrap platform remote apply --bootstrap-config /secure/forgezero/development/dev-fz-n1-remote.json --apply
|
|
763
|
+
# The command prints the derived plan before its final APPLY confirmation:
|
|
764
|
+
fz bootstrap platform launch --profile development --apply
|
|
766
765
|
```
|
|
767
766
|
|
|
768
767
|
<a id="forgezero-agent-host-maintenance"></a>
|
|
@@ -1809,7 +1808,7 @@ export const selectedCapability = SUPPORTED_GUEST_IMAGE;
|
|
|
1809
1808
|
<a id="fz-agent-replica"></a>
|
|
1810
1809
|
## fz-agent (replica)
|
|
1811
1810
|
|
|
1812
|
-
The project-scoped vault replica: resident in RAM, never on disk, invalidated by cursor rather than by timer, and refusing a read it can no longer vouch for. This is a managed runtime interface label, not an import or standalone executable.
|
|
1811
|
+
The authorized project-scoped vault replica: resident in RAM, never on disk, invalidated by cursor rather than by timer, cleared when local authority is revoked, and refusing a read it can no longer vouch for. This is a managed runtime interface label, not an import or standalone executable.
|
|
1813
1812
|
|
|
1814
1813
|
```text
|
|
1815
1814
|
fz-agent (replica)
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { type Server } from 'node:net';
|
|
2
|
+
export declare const DEFAULT_AGENT_CANDIDATE_READY_SOCKET = "/run/forgezero/candidate-ready.sock";
|
|
3
|
+
export interface AgentCandidateReady {
|
|
4
|
+
version: string;
|
|
5
|
+
nodeKey: string;
|
|
6
|
+
bound: boolean;
|
|
7
|
+
vault: 'ready' | 'partial' | 'unavailable' | 'unbound';
|
|
8
|
+
}
|
|
9
|
+
/** New proxy connections resolve the new backend; established streams are untouched. */
|
|
10
|
+
export declare function switchAgentSocketRoute(route: string, backend: string): void;
|
|
11
|
+
/** Root-observable readiness appears only after identity, binding and Vault initialization. */
|
|
12
|
+
export declare function startAgentCandidateReady(ready: AgentCandidateReady, socketPath?: string): Server;
|
|
13
|
+
export declare function probeAgentCandidate(expected: Pick<AgentCandidateReady, 'version'>, socketPath?: string, timeoutMs?: number): Promise<boolean>;
|
|
@@ -72,11 +72,11 @@ export interface AgentHeartbeatOptions {
|
|
|
72
72
|
applyDeploymentIntake?: (directive: NonNullable<AgentHeartbeatResponse['deploymentIntake']>) => Promise<void> | void;
|
|
73
73
|
receiptPath?: string;
|
|
74
74
|
now?: () => number;
|
|
75
|
-
/** Stop new lifecycle/deploy claims and await current jobs
|
|
75
|
+
/** Stop new lifecycle/deploy claims and await current jobs after candidate readiness. */
|
|
76
76
|
prepareUpdate?: (release: AgentRelease) => Promise<void>;
|
|
77
|
-
/**
|
|
77
|
+
/** Recover only after a prepared candidate could not be committed. */
|
|
78
78
|
recoverUpdate?: (cause: unknown) => Promise<void> | void;
|
|
79
|
-
applyUpdate?: (release: AgentRelease, currentVersion: string, attemptId: string) => Promise<AgentUpdateResponse>;
|
|
79
|
+
applyUpdate?: (phase: 'prepare' | 'commit' | 'abort', release: AgentRelease, currentVersion: string, attemptId: string) => Promise<AgentUpdateResponse>;
|
|
80
80
|
setTimer?: (callback: () => void, ms: number) => unknown;
|
|
81
81
|
clearTimer?: (handle: unknown) => void;
|
|
82
82
|
onEvent?: (event: string, detail?: unknown) => void;
|
package/dist/agent-heartbeat.js
CHANGED
|
@@ -16,6 +16,7 @@ import {
|
|
|
16
16
|
} from "node:fs";
|
|
17
17
|
import { dirname, join, resolve } from "node:path";
|
|
18
18
|
var DEFAULT_AGENT_RELEASE_ROOT = "/opt/forgezero/agent";
|
|
19
|
+
var DEFAULT_AGENT_CANDIDATE_LINK = `${DEFAULT_AGENT_RELEASE_ROOT}/candidate`;
|
|
19
20
|
var DEFAULT_AGENT_UPDATE_SOCKET = "/run/forgezero-update/helper.sock";
|
|
20
21
|
var MAX_AGENT_TARBALL_BYTES = 32 * 1024 * 1024;
|
|
21
22
|
var VERSION = /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)$/;
|
|
@@ -209,25 +210,90 @@ function restoreAgentRelease(staged) {
|
|
|
209
210
|
rmSync(next, { force: true });
|
|
210
211
|
}
|
|
211
212
|
}
|
|
213
|
+
function selectAgentCandidate(staged) {
|
|
214
|
+
const link = join(dirname(staged.currentLink), "candidate");
|
|
215
|
+
const next = `${link}.${randomUUID()}.next`;
|
|
216
|
+
try {
|
|
217
|
+
symlinkSync(staged.nextTarget, next);
|
|
218
|
+
renameSync(next, link);
|
|
219
|
+
syncPath(dirname(link));
|
|
220
|
+
} finally {
|
|
221
|
+
rmSync(next, { force: true });
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
function clearAgentCandidate(root = DEFAULT_AGENT_RELEASE_ROOT) {
|
|
225
|
+
rmSync(join(resolve(root), "candidate"), { force: true });
|
|
226
|
+
}
|
|
212
227
|
|
|
213
228
|
// src/agent-update-helper.ts
|
|
214
|
-
import { randomUUID as
|
|
229
|
+
import { randomUUID as randomUUID3 } from "node:crypto";
|
|
215
230
|
import {
|
|
216
|
-
chmodSync as
|
|
231
|
+
chmodSync as chmodSync3,
|
|
217
232
|
closeSync as closeSync2,
|
|
218
|
-
existsSync as
|
|
233
|
+
existsSync as existsSync3,
|
|
219
234
|
fsyncSync as fsyncSync2,
|
|
220
235
|
mkdirSync as mkdirSync2,
|
|
221
236
|
openSync as openSync2,
|
|
222
237
|
readFileSync as readFileSync2,
|
|
223
|
-
renameSync as
|
|
224
|
-
rmSync as
|
|
225
|
-
unlinkSync,
|
|
238
|
+
renameSync as renameSync3,
|
|
239
|
+
rmSync as rmSync3,
|
|
240
|
+
unlinkSync as unlinkSync2,
|
|
226
241
|
writeFileSync as writeFileSync2
|
|
227
242
|
} from "node:fs";
|
|
228
|
-
import { connect, createServer } from "node:net";
|
|
229
|
-
import { dirname as
|
|
243
|
+
import { connect as connect2, createServer as createServer2 } from "node:net";
|
|
244
|
+
import { dirname as dirname3, join as join2, resolve as resolve2 } from "node:path";
|
|
230
245
|
import { DEFAULT_SOCKET } from "@forgezero/vault";
|
|
246
|
+
|
|
247
|
+
// src/agent-handover.ts
|
|
248
|
+
import { randomUUID as randomUUID2 } from "node:crypto";
|
|
249
|
+
import { chmodSync as chmodSync2, existsSync as existsSync2, renameSync as renameSync2, rmSync as rmSync2, symlinkSync as symlinkSync2, unlinkSync } from "node:fs";
|
|
250
|
+
import { basename, dirname as dirname2 } from "node:path";
|
|
251
|
+
import { connect, createServer } from "node:net";
|
|
252
|
+
var DEFAULT_AGENT_CANDIDATE_READY_SOCKET = "/run/forgezero/candidate-ready.sock";
|
|
253
|
+
function switchAgentSocketRoute(route, backend) {
|
|
254
|
+
if (dirname2(route) !== dirname2(backend))
|
|
255
|
+
throw new Error("Agent handover sockets must share one runtime directory");
|
|
256
|
+
const target = basename(backend);
|
|
257
|
+
const next = `${route}.${randomUUID2()}.next`;
|
|
258
|
+
try {
|
|
259
|
+
symlinkSync2(target, next);
|
|
260
|
+
renameSync2(next, route);
|
|
261
|
+
} finally {
|
|
262
|
+
rmSync2(next, { force: true });
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
function probeAgentCandidate(expected, socketPath = DEFAULT_AGENT_CANDIDATE_READY_SOCKET, timeoutMs = 5000) {
|
|
266
|
+
return new Promise((resolve2) => {
|
|
267
|
+
const socket = connect(socketPath);
|
|
268
|
+
let buffer = "";
|
|
269
|
+
let settled = false;
|
|
270
|
+
const finish = (value) => {
|
|
271
|
+
if (settled)
|
|
272
|
+
return;
|
|
273
|
+
settled = true;
|
|
274
|
+
clearTimeout(timer);
|
|
275
|
+
socket.destroy();
|
|
276
|
+
resolve2(value);
|
|
277
|
+
};
|
|
278
|
+
const timer = setTimeout(() => finish(false), timeoutMs);
|
|
279
|
+
socket.on("data", (chunk) => {
|
|
280
|
+
buffer += chunk.toString("utf8");
|
|
281
|
+
const newline = buffer.indexOf(`
|
|
282
|
+
`);
|
|
283
|
+
if (newline < 0)
|
|
284
|
+
return;
|
|
285
|
+
try {
|
|
286
|
+
const value = JSON.parse(buffer.slice(0, newline));
|
|
287
|
+
finish(value.version === expected.version && typeof value.nodeKey === "string" && typeof value.bound === "boolean" && ["ready", "partial", "unavailable", "unbound"].includes(value.vault ?? ""));
|
|
288
|
+
} catch {
|
|
289
|
+
finish(false);
|
|
290
|
+
}
|
|
291
|
+
});
|
|
292
|
+
socket.on("error", () => finish(false));
|
|
293
|
+
});
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
// src/agent-update-helper.ts
|
|
231
297
|
var AGENT_UPDATE_GROUP = "forgezero-update";
|
|
232
298
|
var AGENT_UPDATE_HELPER_UNIT_PATH = "/etc/systemd/system/forgezero-agent-update-helper.service";
|
|
233
299
|
var AGENT_UPDATE_JOURNAL = "/var/lib/forgezero/agent-update.json";
|
|
@@ -322,13 +388,13 @@ function validateJournal(value, root) {
|
|
|
322
388
|
return journal;
|
|
323
389
|
}
|
|
324
390
|
function readJournal(path, root) {
|
|
325
|
-
if (!
|
|
391
|
+
if (!existsSync3(path))
|
|
326
392
|
return;
|
|
327
393
|
return validateJournal(JSON.parse(readFileSync2(path, "utf8")), root);
|
|
328
394
|
}
|
|
329
395
|
function writeAtomic(path, value, mode) {
|
|
330
|
-
mkdirSync2(
|
|
331
|
-
const next = `${path}.${
|
|
396
|
+
mkdirSync2(dirname3(path), { recursive: true, mode: 493 });
|
|
397
|
+
const next = `${path}.${randomUUID3()}.next`;
|
|
332
398
|
let file;
|
|
333
399
|
try {
|
|
334
400
|
file = openSync2(next, "wx", mode);
|
|
@@ -337,8 +403,8 @@ function writeAtomic(path, value, mode) {
|
|
|
337
403
|
fsyncSync2(file);
|
|
338
404
|
closeSync2(file);
|
|
339
405
|
file = undefined;
|
|
340
|
-
|
|
341
|
-
const directory = openSync2(
|
|
406
|
+
renameSync3(next, path);
|
|
407
|
+
const directory = openSync2(dirname3(path), "r");
|
|
342
408
|
try {
|
|
343
409
|
fsyncSync2(directory);
|
|
344
410
|
} finally {
|
|
@@ -347,7 +413,7 @@ function writeAtomic(path, value, mode) {
|
|
|
347
413
|
} finally {
|
|
348
414
|
if (file !== undefined)
|
|
349
415
|
closeSync2(file);
|
|
350
|
-
|
|
416
|
+
rmSync3(next, { force: true });
|
|
351
417
|
}
|
|
352
418
|
}
|
|
353
419
|
var publicReceipt = (journal) => {
|
|
@@ -380,7 +446,7 @@ function writeUpdateState(journalPath, receiptPath, journal) {
|
|
|
380
446
|
}
|
|
381
447
|
function readAgentUpdateReceipt(path = AGENT_UPDATE_RECEIPT) {
|
|
382
448
|
try {
|
|
383
|
-
if (!
|
|
449
|
+
if (!existsSync3(path))
|
|
384
450
|
return;
|
|
385
451
|
return validateReceipt(JSON.parse(readFileSync2(path, "utf8")));
|
|
386
452
|
} catch {
|
|
@@ -420,10 +486,32 @@ var restartAgent = async (target, run) => {
|
|
|
420
486
|
throw new Error(`systemd could not restart ${service}`);
|
|
421
487
|
}
|
|
422
488
|
};
|
|
489
|
+
var candidateUnit = (target) => target === "compute" ? "forgezero-agent-candidate.service" : "forgezero-metal-agent-candidate.service";
|
|
490
|
+
var startCandidate = async (staged, target, run, probe = (expected) => probeAgentCandidate(expected, DEFAULT_AGENT_CANDIDATE_READY_SOCKET)) => {
|
|
491
|
+
selectAgentCandidate(staged);
|
|
492
|
+
if (!await runOk(run, "/usr/bin/systemctl", ["restart", candidateUnit(target)])) {
|
|
493
|
+
clearAgentCandidate(dirname3(staged.currentLink));
|
|
494
|
+
throw new Error(`systemd could not start ${candidateUnit(target)}`);
|
|
495
|
+
}
|
|
496
|
+
if (!await probe({ version: staged.version })) {
|
|
497
|
+
await run({ command: "/usr/bin/systemctl", args: ["stop", candidateUnit(target)] });
|
|
498
|
+
clearAgentCandidate(dirname3(staged.currentLink));
|
|
499
|
+
throw new Error("the candidate Agent did not prove its identity and durable state");
|
|
500
|
+
}
|
|
501
|
+
};
|
|
502
|
+
var stopCandidate = async (staged, target, run) => {
|
|
503
|
+
await run({ command: "/usr/bin/systemctl", args: ["stop", candidateUnit(target)] });
|
|
504
|
+
clearAgentCandidate(dirname3(staged.currentLink));
|
|
505
|
+
};
|
|
506
|
+
var socketPaths = (publicSocket = process.env.FZ_AGENT_PUBLIC_SOCKET ?? DEFAULT_SOCKET) => ({
|
|
507
|
+
route: `${publicSocket}.backend`,
|
|
508
|
+
active: `${publicSocket}.backend.active`,
|
|
509
|
+
candidate: `${publicSocket}.backend.candidate`
|
|
510
|
+
});
|
|
423
511
|
var targetProbe = (target, run) => target === "compute" ? () => probeAgentSocket() : async () => await runOk(run, "/usr/bin/systemctl", ["is-active", "--quiet", "forgezero-metal-agent.service"]) && await runOk(run, "/usr/bin/systemctl", ["is-active", "--quiet", "forgezero-metal-helper.service"]);
|
|
424
512
|
function probeAgentSocket(socketPath = DEFAULT_SOCKET, timeoutMs = 5000) {
|
|
425
513
|
return new Promise((resolve3) => {
|
|
426
|
-
const socket =
|
|
514
|
+
const socket = connect2(socketPath);
|
|
427
515
|
let settled = false;
|
|
428
516
|
let buffer = "";
|
|
429
517
|
const finish = (value) => {
|
|
@@ -456,12 +544,13 @@ function probeAgentSocket(socketPath = DEFAULT_SOCKET, timeoutMs = 5000) {
|
|
|
456
544
|
async function activateAgentRelease(staged, options = {}) {
|
|
457
545
|
const run = options.run ?? runCommand;
|
|
458
546
|
const target = options.target ?? "compute";
|
|
459
|
-
const
|
|
547
|
+
const paths = socketPaths(options.publicSocketPath);
|
|
548
|
+
const probe = options.probe ?? (target === "compute" ? () => probeAgentSocket(paths.active) : targetProbe(target, run));
|
|
460
549
|
const now = options.now ?? Date.now;
|
|
461
550
|
const journalPath = options.journalPath ?? AGENT_UPDATE_JOURNAL;
|
|
462
551
|
const receiptPath = options.receiptPath ?? AGENT_UPDATE_RECEIPT;
|
|
463
|
-
const previous = readJournal(journalPath,
|
|
464
|
-
const attemptId = options.attemptId ??
|
|
552
|
+
const previous = readJournal(journalPath, dirname3(staged.currentLink));
|
|
553
|
+
const attemptId = options.attemptId ?? randomUUID3();
|
|
465
554
|
if (!ATTEMPT_ID.test(attemptId))
|
|
466
555
|
throw new Error("Agent update attempt ID is invalid");
|
|
467
556
|
const startedAtTs = now();
|
|
@@ -483,11 +572,18 @@ async function activateAgentRelease(staged, options = {}) {
|
|
|
483
572
|
writeUpdateState(journalPath, receiptPath, journal);
|
|
484
573
|
let selectionAttempted = false;
|
|
485
574
|
try {
|
|
575
|
+
if (!options.candidatePrepared)
|
|
576
|
+
await startCandidate(staged, target, run);
|
|
577
|
+
if (target === "compute")
|
|
578
|
+
switchAgentSocketRoute(paths.route, paths.candidate);
|
|
486
579
|
selectionAttempted = true;
|
|
487
580
|
selectAgentRelease(staged);
|
|
488
581
|
await restartAgent(target, run);
|
|
489
582
|
if (!await probe())
|
|
490
|
-
throw new Error("the replacement Agent did not answer its
|
|
583
|
+
throw new Error("the replacement Agent did not answer its active Vault backend");
|
|
584
|
+
if (target === "compute")
|
|
585
|
+
switchAgentSocketRoute(paths.route, paths.active);
|
|
586
|
+
await stopCandidate(staged, target, run);
|
|
491
587
|
journal = { ...journal, outcome: "active", updatedAtTs: now(), rollbackHealthy: undefined };
|
|
492
588
|
writeUpdateState(journalPath, receiptPath, journal);
|
|
493
589
|
run({
|
|
@@ -505,10 +601,13 @@ async function activateAgentRelease(staged, options = {}) {
|
|
|
505
601
|
restored = true;
|
|
506
602
|
await restartAgent(target, run);
|
|
507
603
|
rollbackHealthy = await probe();
|
|
604
|
+
if (rollbackHealthy && target === "compute")
|
|
605
|
+
switchAgentSocketRoute(paths.route, paths.active);
|
|
508
606
|
} catch {
|
|
509
607
|
rollbackHealthy = false;
|
|
510
608
|
}
|
|
511
609
|
}
|
|
610
|
+
await stopCandidate(staged, target, run).catch(() => {});
|
|
512
611
|
const failures = failureCount + 1;
|
|
513
612
|
const updatedAtTs = now();
|
|
514
613
|
journal = {
|
|
@@ -528,28 +627,38 @@ async function recoverInterruptedAgentUpdate(options = {}) {
|
|
|
528
627
|
const root = resolve2(options.root ?? DEFAULT_AGENT_RELEASE_ROOT);
|
|
529
628
|
const journalPath = options.journalPath ?? AGENT_UPDATE_JOURNAL;
|
|
530
629
|
const receiptPath = options.receiptPath ?? AGENT_UPDATE_RECEIPT;
|
|
630
|
+
const run = options.run ?? runCommand;
|
|
531
631
|
const journal = readJournal(journalPath, root);
|
|
532
|
-
if (!journal)
|
|
632
|
+
if (!journal) {
|
|
633
|
+
for (const target of ["compute", "metal"]) {
|
|
634
|
+
await run({ command: "/usr/bin/systemctl", args: ["stop", candidateUnit(target)] });
|
|
635
|
+
}
|
|
636
|
+
clearAgentCandidate(root);
|
|
533
637
|
return;
|
|
638
|
+
}
|
|
534
639
|
if (journal.outcome !== "activating") {
|
|
640
|
+
await stopCandidate(stagedFromJournal(journal, root), journal.target, run).catch(() => {});
|
|
535
641
|
writeAtomic(receiptPath, publicReceipt(journal), 416);
|
|
536
642
|
return publicReceipt(journal);
|
|
537
643
|
}
|
|
538
644
|
const staged = stagedFromJournal(journal, root);
|
|
539
|
-
if (!
|
|
645
|
+
if (!existsSync3(join2(root, journal.previousTarget))) {
|
|
540
646
|
throw new Error("Agent update rollback release is missing");
|
|
541
647
|
}
|
|
542
|
-
const
|
|
543
|
-
const probe = options.probe ?? targetProbe(journal.target, run);
|
|
648
|
+
const paths = socketPaths(options.publicSocketPath);
|
|
649
|
+
const probe = options.probe ?? (journal.target === "compute" ? () => probeAgentSocket(paths.active) : targetProbe(journal.target, run));
|
|
544
650
|
restoreAgentRelease(staged);
|
|
545
651
|
let rollbackHealthy = false;
|
|
546
652
|
let failureMessage = "activation was interrupted before its health verdict became durable";
|
|
547
653
|
try {
|
|
548
654
|
await restartAgent(journal.target, run);
|
|
549
655
|
rollbackHealthy = await probe();
|
|
656
|
+
if (rollbackHealthy && journal.target === "compute")
|
|
657
|
+
switchAgentSocketRoute(paths.route, paths.active);
|
|
550
658
|
} catch (cause) {
|
|
551
659
|
failureMessage = `${failureMessage}; ${cause instanceof Error ? cause.message : String(cause)}`;
|
|
552
660
|
}
|
|
661
|
+
await stopCandidate(staged, journal.target, run).catch(() => {});
|
|
553
662
|
const failures = journal.failureCount + 1;
|
|
554
663
|
const updatedAtTs = (options.now ?? Date.now)();
|
|
555
664
|
const recovered = {
|
|
@@ -566,15 +675,23 @@ async function recoverInterruptedAgentUpdate(options = {}) {
|
|
|
566
675
|
}
|
|
567
676
|
function startAgentUpdateHelper(options = {}) {
|
|
568
677
|
const socketPath = options.socketPath ?? DEFAULT_AGENT_UPDATE_SOCKET;
|
|
569
|
-
if (
|
|
570
|
-
|
|
571
|
-
mkdirSync2(
|
|
572
|
-
const setTimer = options.setTimer ?? ((callback, ms) => setTimeout(callback, ms));
|
|
678
|
+
if (existsSync3(socketPath))
|
|
679
|
+
unlinkSync2(socketPath);
|
|
680
|
+
mkdirSync2(dirname3(socketPath), { recursive: true, mode: 488 });
|
|
573
681
|
const receiptPath = options.receiptPath ?? AGENT_UPDATE_RECEIPT;
|
|
574
682
|
const journalPath = options.journalPath ?? AGENT_UPDATE_JOURNAL;
|
|
575
683
|
const releaseRoot = options.root ?? DEFAULT_AGENT_RELEASE_ROOT;
|
|
576
|
-
const
|
|
684
|
+
const setTimer = options.setTimer ?? ((callback, ms) => setTimeout(callback, ms));
|
|
685
|
+
const activate = options.activate ?? ((staged, target, attemptId) => activateAgentRelease(staged, {
|
|
686
|
+
target,
|
|
687
|
+
attemptId,
|
|
688
|
+
journalPath,
|
|
689
|
+
receiptPath,
|
|
690
|
+
now: options.now,
|
|
691
|
+
candidatePrepared: true
|
|
692
|
+
}));
|
|
577
693
|
let busy = true;
|
|
694
|
+
let pending;
|
|
578
695
|
let blocked;
|
|
579
696
|
(options.recover ?? (() => recoverInterruptedAgentUpdate({
|
|
580
697
|
root: releaseRoot,
|
|
@@ -586,7 +703,7 @@ function startAgentUpdateHelper(options = {}) {
|
|
|
586
703
|
}).finally(() => {
|
|
587
704
|
busy = false;
|
|
588
705
|
});
|
|
589
|
-
const server =
|
|
706
|
+
const server = createServer2((socket) => {
|
|
590
707
|
let buffer = "";
|
|
591
708
|
socket.on("data", (chunk) => {
|
|
592
709
|
buffer += chunk.toString("utf8");
|
|
@@ -605,16 +722,49 @@ function startAgentUpdateHelper(options = {}) {
|
|
|
605
722
|
Promise.resolve().then(() => JSON.parse(line)).then(async (request) => {
|
|
606
723
|
if (blocked)
|
|
607
724
|
throw new Error(`update journal needs operator recovery: ${blocked}`);
|
|
608
|
-
if (busy)
|
|
609
|
-
throw new Error("another Agent update or recovery is already active");
|
|
610
|
-
if (request.op !== "apply")
|
|
611
|
-
throw new Error("unknown update operation");
|
|
612
725
|
if (request.target !== "compute" && request.target !== "metal") {
|
|
613
726
|
throw new Error("agent update target is invalid");
|
|
614
727
|
}
|
|
615
|
-
const attemptId = request.attemptId ??
|
|
728
|
+
const attemptId = request.attemptId ?? randomUUID3();
|
|
616
729
|
if (!ATTEMPT_ID.test(attemptId))
|
|
617
730
|
throw new Error("Agent update attempt ID is invalid");
|
|
731
|
+
if (request.op === "commit" || request.op === "abort") {
|
|
732
|
+
if (!pending || pending.attemptId !== attemptId || pending.target !== request.target || pending.staged.fromVersion !== request.currentVersion || pending.staged.version !== request.targetVersion) {
|
|
733
|
+
throw new Error("Agent update handover does not match the prepared candidate");
|
|
734
|
+
}
|
|
735
|
+
const selected = pending;
|
|
736
|
+
pending = undefined;
|
|
737
|
+
if (request.op === "abort") {
|
|
738
|
+
await stopCandidate(selected.staged, selected.target, runCommand);
|
|
739
|
+
busy = false;
|
|
740
|
+
const response3 = {
|
|
741
|
+
ok: true,
|
|
742
|
+
status: "aborted",
|
|
743
|
+
version: selected.staged.version,
|
|
744
|
+
attemptId
|
|
745
|
+
};
|
|
746
|
+
socket.end(`${JSON.stringify(response3)}
|
|
747
|
+
`);
|
|
748
|
+
return;
|
|
749
|
+
}
|
|
750
|
+
const outcome = await activate(selected.staged, selected.target, attemptId);
|
|
751
|
+
busy = false;
|
|
752
|
+
if (outcome?.ok === false)
|
|
753
|
+
throw new Error(outcome.reason ?? "Agent handover failed");
|
|
754
|
+
const response2 = {
|
|
755
|
+
ok: true,
|
|
756
|
+
status: "active",
|
|
757
|
+
version: selected.staged.version,
|
|
758
|
+
attemptId
|
|
759
|
+
};
|
|
760
|
+
socket.end(`${JSON.stringify(response2)}
|
|
761
|
+
`);
|
|
762
|
+
return;
|
|
763
|
+
}
|
|
764
|
+
if (request.op !== "prepare")
|
|
765
|
+
throw new Error("unknown update operation");
|
|
766
|
+
if (busy)
|
|
767
|
+
throw new Error("another Agent update or recovery is already active");
|
|
618
768
|
const prior = readJournal(journalPath, releaseRoot);
|
|
619
769
|
const now = (options.now ?? Date.now)();
|
|
620
770
|
if (prior?.targetVersion === request.release.version && (prior.outcome === "rolled-back" || prior.outcome === "failed") && (prior.retryAfterTs ?? 0) > now)
|
|
@@ -625,17 +775,23 @@ function startAgentUpdateHelper(options = {}) {
|
|
|
625
775
|
currentVersion: request.currentVersion,
|
|
626
776
|
root: releaseRoot
|
|
627
777
|
});
|
|
628
|
-
|
|
778
|
+
await startCandidate(staged, request.target, runCommand);
|
|
779
|
+
pending = { staged, target: request.target, attemptId };
|
|
780
|
+
ownsBusy = false;
|
|
781
|
+
setTimer(() => {
|
|
782
|
+
if (pending?.attemptId !== attemptId)
|
|
783
|
+
return;
|
|
784
|
+
const expired = pending;
|
|
785
|
+
pending = undefined;
|
|
786
|
+
stopCandidate(expired.staged, expired.target, runCommand).finally(() => {
|
|
787
|
+
busy = false;
|
|
788
|
+
});
|
|
789
|
+
}, 180000);
|
|
790
|
+
const response = { ok: true, status: "prepared", version: staged.version, attemptId };
|
|
629
791
|
socket.end(`${JSON.stringify(response)}
|
|
630
792
|
`);
|
|
631
|
-
setTimer(() => void activate(staged, request.target, attemptId).catch((cause) => {
|
|
632
|
-
blocked = cause instanceof Error ? cause.message : String(cause);
|
|
633
|
-
}).finally(() => {
|
|
634
|
-
busy = false;
|
|
635
|
-
}), 100);
|
|
636
|
-
ownsBusy = false;
|
|
637
793
|
}).catch((cause) => {
|
|
638
|
-
if (ownsBusy)
|
|
794
|
+
if (ownsBusy && !pending)
|
|
639
795
|
busy = false;
|
|
640
796
|
const response = {
|
|
641
797
|
ok: false,
|
|
@@ -647,12 +803,12 @@ function startAgentUpdateHelper(options = {}) {
|
|
|
647
803
|
});
|
|
648
804
|
socket.on("error", () => socket.destroy());
|
|
649
805
|
});
|
|
650
|
-
server.listen(socketPath, () =>
|
|
806
|
+
server.listen(socketPath, () => chmodSync3(socketPath, 432));
|
|
651
807
|
return server;
|
|
652
808
|
}
|
|
653
809
|
function requestAgentUpdate(request, socketPath = DEFAULT_AGENT_UPDATE_SOCKET, timeoutMs = 90000) {
|
|
654
810
|
return new Promise((resolve3, reject) => {
|
|
655
|
-
const socket =
|
|
811
|
+
const socket = connect2(socketPath, () => socket.write(`${JSON.stringify(request)}
|
|
656
812
|
`));
|
|
657
813
|
let buffer = "";
|
|
658
814
|
socket.setTimeout(timeoutMs, () => {
|
|
@@ -677,7 +833,7 @@ function requestAgentUpdate(request, socketPath = DEFAULT_AGENT_UPDATE_SOCKET, t
|
|
|
677
833
|
}
|
|
678
834
|
|
|
679
835
|
// src/agent-heartbeat.ts
|
|
680
|
-
import { existsSync as
|
|
836
|
+
import { existsSync as existsSync4, readFileSync as readFileSync3, statfsSync } from "node:fs";
|
|
681
837
|
import { cpus, freemem, loadavg, totalmem } from "node:os";
|
|
682
838
|
|
|
683
839
|
// src/signed-node-http.ts
|
|
@@ -691,9 +847,11 @@ import {
|
|
|
691
847
|
|
|
692
848
|
class SignedNodeHttpError extends Error {
|
|
693
849
|
status;
|
|
694
|
-
|
|
850
|
+
code;
|
|
851
|
+
constructor(status, message, code) {
|
|
695
852
|
super(message);
|
|
696
853
|
this.status = status;
|
|
854
|
+
this.code = code;
|
|
697
855
|
this.name = "SignedNodeHttpError";
|
|
698
856
|
}
|
|
699
857
|
}
|
|
@@ -740,7 +898,7 @@ async function postSignedNode(options, path, body) {
|
|
|
740
898
|
if (!response.ok) {
|
|
741
899
|
const failure = payload;
|
|
742
900
|
const reason2 = failure ? failure.error?.message ?? failure.message : undefined;
|
|
743
|
-
throw new SignedNodeHttpError(response.status, reason2 || `signed node request returned HTTP ${response.status}
|
|
901
|
+
throw new SignedNodeHttpError(response.status, reason2 || `signed node request returned HTTP ${response.status}`, failure?.error?.code);
|
|
744
902
|
}
|
|
745
903
|
try {
|
|
746
904
|
return await openResponse(recipient.secretKey, signature, payload);
|
|
@@ -750,7 +908,7 @@ async function postSignedNode(options, path, body) {
|
|
|
750
908
|
}
|
|
751
909
|
|
|
752
910
|
// src/version.ts
|
|
753
|
-
var VERSION3 = "0.1.
|
|
911
|
+
var VERSION3 = "0.1.87";
|
|
754
912
|
|
|
755
913
|
// src/agent-heartbeat.ts
|
|
756
914
|
function readAgentHostMetrics() {
|
|
@@ -796,7 +954,7 @@ function observeAgentHost(version = VERSION3, mode = "enrolled", osRelease = rea
|
|
|
796
954
|
runtimeCapabilities: {
|
|
797
955
|
native: true,
|
|
798
956
|
ociRunc: ubuntuX64,
|
|
799
|
-
kataQemuSnp: ubuntuX64 && osVersion === "26.04" &&
|
|
957
|
+
kataQemuSnp: ubuntuX64 && osVersion === "26.04" && existsSync4("/dev/kvm") && existsSync4("/dev/sev") && enabled("/sys/module/kvm_amd/parameters/sev") && enabled("/sys/module/kvm_amd/parameters/sev_snp")
|
|
800
958
|
},
|
|
801
959
|
capacity: {
|
|
802
960
|
logicalCpu: Math.max(1, metrics.logicalCpu),
|
|
@@ -848,18 +1006,24 @@ async function heartbeatAgentOnce(options) {
|
|
|
848
1006
|
});
|
|
849
1007
|
return response;
|
|
850
1008
|
}
|
|
851
|
-
let
|
|
1009
|
+
let candidatePrepared = false;
|
|
1010
|
+
let drained = false;
|
|
852
1011
|
try {
|
|
853
|
-
|
|
854
|
-
|
|
1012
|
+
const update2 = options.applyUpdate ?? ((phase, next, current, attemptId) => requestAgentUpdate(phase === "prepare" ? { op: "prepare", target: options.updateTarget ?? "compute", release: next, currentVersion: current, attemptId } : {
|
|
1013
|
+
op: phase,
|
|
1014
|
+
target: options.updateTarget ?? "compute",
|
|
1015
|
+
currentVersion: current,
|
|
1016
|
+
targetVersion: next.version,
|
|
1017
|
+
attemptId
|
|
1018
|
+
}));
|
|
855
1019
|
const apply = async () => {
|
|
856
|
-
const
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
1020
|
+
const prepared = await update2("prepare", release, observation.version, desired.attemptId);
|
|
1021
|
+
if (!prepared.ok)
|
|
1022
|
+
throw new AgentUpdateRefusedError(`agent update refused: ${prepared.error.message}`);
|
|
1023
|
+
candidatePrepared = true;
|
|
1024
|
+
await options.prepareUpdate?.(release);
|
|
1025
|
+
drained = true;
|
|
1026
|
+
const applied = await update2("commit", release, observation.version, desired.attemptId);
|
|
863
1027
|
if (!applied.ok)
|
|
864
1028
|
throw new AgentUpdateRefusedError(`agent update refused: ${applied.error.message}`);
|
|
865
1029
|
if (applied.attemptId !== desired.attemptId) {
|
|
@@ -872,9 +1036,18 @@ async function heartbeatAgentOnce(options) {
|
|
|
872
1036
|
} else {
|
|
873
1037
|
await apply();
|
|
874
1038
|
}
|
|
875
|
-
options.onEvent?.("update-
|
|
1039
|
+
options.onEvent?.("update-active", { from: observation.version, to: release.version });
|
|
876
1040
|
} catch (cause) {
|
|
877
|
-
if (
|
|
1041
|
+
if (candidatePrepared && !drained) {
|
|
1042
|
+
await (options.applyUpdate ?? ((phase, next, current, attemptId) => requestAgentUpdate({
|
|
1043
|
+
op: phase,
|
|
1044
|
+
target: options.updateTarget ?? "compute",
|
|
1045
|
+
currentVersion: current,
|
|
1046
|
+
targetVersion: next.version,
|
|
1047
|
+
attemptId
|
|
1048
|
+
})))("abort", release, observation.version, desired.attemptId).catch(() => {});
|
|
1049
|
+
}
|
|
1050
|
+
if (drained)
|
|
878
1051
|
await options.recoverUpdate?.(cause);
|
|
879
1052
|
throw cause;
|
|
880
1053
|
}
|