@forgezero/agent 0.1.87 → 0.1.89
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -9
- package/dist/agent-handover.d.ts +7 -1
- package/dist/agent-heartbeat.js +25 -9
- package/dist/agent-update-helper.d.ts +1 -0
- package/dist/agent-update-helper.js +16 -7
- package/dist/bootstrap.d.ts +8 -1
- package/dist/bootstrap.js +228 -54
- package/dist/capacity-calibration.d.ts +35 -5
- package/dist/capacity-calibration.js +66 -22
- package/dist/cli/agent-install.d.ts +3 -0
- package/dist/community-rehearsal-host.js +7 -2
- package/dist/control.d.ts +3 -1
- package/dist/definition.js +83 -28
- package/dist/deploy-file.js +83 -28
- package/dist/deployment.d.ts +5 -0
- package/dist/egress-policy.d.ts +4 -0
- package/dist/fz-agent.js +440 -158
- package/dist/fz.js +413 -129
- package/dist/host-maintenance.js +1 -1
- package/dist/index.d.ts +7 -0
- package/dist/metal-bootstrap.js +1 -1
- package/dist/metal-helper-socket.js +7 -2
- package/dist/metal-provision.js +7 -2
- package/dist/operator-bootstrap.d.ts +12 -0
- package/dist/operator-bootstrap.js +372 -64
- package/dist/platform-bootstrap-runtime.d.ts +1 -0
- package/dist/platform-bootstrap-runtime.js +47 -8
- package/dist/platform-fleet-verification.js +278 -94
- package/dist/platform-genesis.js +7 -2
- package/dist/provision.d.ts +5 -1
- package/dist/provision.js +201 -78
- package/dist/recovery-host.js +1 -1
- package/dist/software-helper.js +144 -48
- package/dist/software.js +7 -2
- package/dist/ubuntu.js +7 -2
- package/dist/version.d.ts +1 -1
- package/package.json +1 -1
- package/schema/deploy-v3.json +5 -2
|
@@ -284,7 +284,7 @@ function probeAgentCandidate(expected, socketPath = DEFAULT_AGENT_CANDIDATE_READ
|
|
|
284
284
|
return;
|
|
285
285
|
try {
|
|
286
286
|
const value = JSON.parse(buffer.slice(0, newline));
|
|
287
|
-
finish(value.version === expected.version &&
|
|
287
|
+
finish(value.version === expected.version && value.nodeKey === expected.nodeKey && value.bound === expected.bound && expected.vault.includes(value.vault));
|
|
288
288
|
} catch {
|
|
289
289
|
finish(false);
|
|
290
290
|
}
|
|
@@ -487,13 +487,18 @@ var restartAgent = async (target, run) => {
|
|
|
487
487
|
}
|
|
488
488
|
};
|
|
489
489
|
var candidateUnit = (target) => target === "compute" ? "forgezero-agent-candidate.service" : "forgezero-metal-agent-candidate.service";
|
|
490
|
-
var startCandidate = async (staged, target, run, probe = (expected) => probeAgentCandidate(expected, DEFAULT_AGENT_CANDIDATE_READY_SOCKET)) => {
|
|
490
|
+
var startCandidate = async (staged, target, expectedNodeKey, run, probe = (expected) => probeAgentCandidate(expected, DEFAULT_AGENT_CANDIDATE_READY_SOCKET)) => {
|
|
491
491
|
selectAgentCandidate(staged);
|
|
492
492
|
if (!await runOk(run, "/usr/bin/systemctl", ["restart", candidateUnit(target)])) {
|
|
493
493
|
clearAgentCandidate(dirname3(staged.currentLink));
|
|
494
494
|
throw new Error(`systemd could not start ${candidateUnit(target)}`);
|
|
495
495
|
}
|
|
496
|
-
if (!await probe({
|
|
496
|
+
if (!await probe({
|
|
497
|
+
version: staged.version,
|
|
498
|
+
nodeKey: expectedNodeKey,
|
|
499
|
+
bound: true,
|
|
500
|
+
vault: target === "compute" ? ["ready"] : ["unbound"]
|
|
501
|
+
})) {
|
|
497
502
|
await run({ command: "/usr/bin/systemctl", args: ["stop", candidateUnit(target)] });
|
|
498
503
|
clearAgentCandidate(dirname3(staged.currentLink));
|
|
499
504
|
throw new Error("the candidate Agent did not prove its identity and durable state");
|
|
@@ -572,8 +577,9 @@ async function activateAgentRelease(staged, options = {}) {
|
|
|
572
577
|
writeUpdateState(journalPath, receiptPath, journal);
|
|
573
578
|
let selectionAttempted = false;
|
|
574
579
|
try {
|
|
575
|
-
if (!options.candidatePrepared)
|
|
576
|
-
|
|
580
|
+
if (!options.candidatePrepared) {
|
|
581
|
+
throw new Error("direct activation requires a separately identity-bound candidate preparation");
|
|
582
|
+
}
|
|
577
583
|
if (target === "compute")
|
|
578
584
|
switchAgentSocketRoute(paths.route, paths.candidate);
|
|
579
585
|
selectionAttempted = true;
|
|
@@ -763,6 +769,9 @@ function startAgentUpdateHelper(options = {}) {
|
|
|
763
769
|
}
|
|
764
770
|
if (request.op !== "prepare")
|
|
765
771
|
throw new Error("unknown update operation");
|
|
772
|
+
if (typeof request.expectedNodeKey !== "string" || request.expectedNodeKey.length < 16 || Buffer.byteLength(request.expectedNodeKey, "utf8") > 512 || /[\0\r\n]/.test(request.expectedNodeKey)) {
|
|
773
|
+
throw new Error("Agent update expected node identity is invalid");
|
|
774
|
+
}
|
|
766
775
|
if (busy)
|
|
767
776
|
throw new Error("another Agent update or recovery is already active");
|
|
768
777
|
const prior = readJournal(journalPath, releaseRoot);
|
|
@@ -775,8 +784,8 @@ function startAgentUpdateHelper(options = {}) {
|
|
|
775
784
|
currentVersion: request.currentVersion,
|
|
776
785
|
root: releaseRoot
|
|
777
786
|
});
|
|
778
|
-
await startCandidate(staged, request.target, runCommand);
|
|
779
|
-
pending = { staged, target: request.target, attemptId };
|
|
787
|
+
await startCandidate(staged, request.target, request.expectedNodeKey, runCommand);
|
|
788
|
+
pending = { staged, target: request.target, attemptId, expectedNodeKey: request.expectedNodeKey };
|
|
780
789
|
ownsBusy = false;
|
|
781
790
|
setTimer(() => {
|
|
782
791
|
if (pending?.attemptId !== attemptId)
|
|
@@ -1189,7 +1198,12 @@ async function executeSoftwareOperation(operation) {
|
|
|
1189
1198
|
}
|
|
1190
1199
|
if (software === "nginx") {
|
|
1191
1200
|
const binary = await run(["/usr/sbin/nginx", "-v"]);
|
|
1192
|
-
|
|
1201
|
+
if (binary.exitCode !== 0)
|
|
1202
|
+
return binary;
|
|
1203
|
+
if (!existsSync4("/usr/lib/nginx/modules/ngx_http_headers_more_filter_module.so")) {
|
|
1204
|
+
return { exitCode: 1, output: "Nginx response-header suppression module is unavailable" };
|
|
1205
|
+
}
|
|
1206
|
+
return run(["/usr/bin/systemctl", "is-active", "--quiet", "nginx.service"]);
|
|
1193
1207
|
}
|
|
1194
1208
|
if (software === "arangodb") {
|
|
1195
1209
|
const binary = await run(["/usr/sbin/arangod", "--version"]);
|
|
@@ -1221,7 +1235,7 @@ async function executeSoftwareOperation(operation) {
|
|
|
1221
1235
|
}
|
|
1222
1236
|
if (software === "docker" || software === "nginx" || software === "ufw" || software === "openssh-client" || software === "git") {
|
|
1223
1237
|
const packageName = software === "openssh-client" ? "openssh-client" : software === "docker" ? "docker.io" : software;
|
|
1224
|
-
const installed = await aptInstall(packageName);
|
|
1238
|
+
const installed = software === "nginx" ? await aptInstallMany(["nginx", "libnginx-mod-http-headers-more-filter"]) : await aptInstall(packageName);
|
|
1225
1239
|
if (installed.exitCode !== 0 || software !== "nginx" && software !== "docker")
|
|
1226
1240
|
return installed;
|
|
1227
1241
|
if (software === "docker") {
|
|
@@ -1373,11 +1387,22 @@ async function ensureSoftwareRequirements(requirementsInput, options) {
|
|
|
1373
1387
|
}
|
|
1374
1388
|
|
|
1375
1389
|
// src/capacity-calibration.ts
|
|
1376
|
-
|
|
1390
|
+
import { cpus, freemem, totalmem } from "node:os";
|
|
1391
|
+
var percentile = (values, quantile) => {
|
|
1377
1392
|
if (values.length === 0)
|
|
1378
1393
|
return Number.POSITIVE_INFINITY;
|
|
1379
1394
|
const sorted = values.toSorted((left, right) => left - right);
|
|
1380
|
-
return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length *
|
|
1395
|
+
return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * quantile) - 1)];
|
|
1396
|
+
};
|
|
1397
|
+
var hostSample = () => {
|
|
1398
|
+
let cpuIdle = 0;
|
|
1399
|
+
let cpuTotal = 0;
|
|
1400
|
+
for (const cpu of cpus()) {
|
|
1401
|
+
cpuIdle += cpu.times.idle;
|
|
1402
|
+
cpuTotal += Object.values(cpu.times).reduce((sum, value) => sum + value, 0);
|
|
1403
|
+
}
|
|
1404
|
+
const memoryTotal = totalmem();
|
|
1405
|
+
return { cpuIdle, cpuTotal, memoryUsed: memoryTotal - freemem(), memoryTotal };
|
|
1381
1406
|
};
|
|
1382
1407
|
function localCalibrationEndpoint(value) {
|
|
1383
1408
|
const endpoint = new URL(value);
|
|
@@ -1392,9 +1417,12 @@ function validateCapacityCalibrationOptions(options) {
|
|
|
1392
1417
|
const requestsPerWorker = options.requestsPerWorker ?? 8;
|
|
1393
1418
|
const maxP95Ms = options.maxP95Ms ?? 250;
|
|
1394
1419
|
const maxErrorRate = options.maxErrorRate ?? 0.01;
|
|
1395
|
-
const
|
|
1420
|
+
const safetyRatio = options.safetyRatio ?? 0.8;
|
|
1396
1421
|
const requestTimeoutMs = options.requestTimeoutMs ?? 5000;
|
|
1397
|
-
|
|
1422
|
+
const minimumStageDurationMs = options.minimumStageDurationMs ?? 5000;
|
|
1423
|
+
const maxCpuUtilizationPercent = options.maxCpuUtilizationPercent ?? 90;
|
|
1424
|
+
const maxMemoryUtilizationPercent = options.maxMemoryUtilizationPercent ?? 90;
|
|
1425
|
+
if (!Number.isSafeInteger(maxConcurrency) || maxConcurrency < 1 || maxConcurrency > 4096 || !Number.isSafeInteger(requestsPerWorker) || requestsPerWorker < 2 || requestsPerWorker > 100 || !Number.isFinite(maxP95Ms) || maxP95Ms < 1 || !Number.isFinite(maxErrorRate) || maxErrorRate < 0 || maxErrorRate > 0.2 || !Number.isFinite(safetyRatio) || safetyRatio < 0.25 || safetyRatio > 0.95 || !Number.isSafeInteger(requestTimeoutMs) || requestTimeoutMs < 100 || requestTimeoutMs > 30000 || !Number.isSafeInteger(minimumStageDurationMs) || minimumStageDurationMs < 100 || minimumStageDurationMs > 60000 || !Number.isFinite(maxCpuUtilizationPercent) || maxCpuUtilizationPercent < 25 || maxCpuUtilizationPercent > 100 || !Number.isFinite(maxMemoryUtilizationPercent) || maxMemoryUtilizationPercent < 25 || maxMemoryUtilizationPercent > 100) {
|
|
1398
1426
|
throw new Error("Capacity calibration bounds are invalid.");
|
|
1399
1427
|
}
|
|
1400
1428
|
return {
|
|
@@ -1403,18 +1431,24 @@ function validateCapacityCalibrationOptions(options) {
|
|
|
1403
1431
|
requestsPerWorker,
|
|
1404
1432
|
maxP95Ms,
|
|
1405
1433
|
maxErrorRate,
|
|
1406
|
-
|
|
1407
|
-
requestTimeoutMs
|
|
1434
|
+
safetyRatio,
|
|
1435
|
+
requestTimeoutMs,
|
|
1436
|
+
minimumStageDurationMs,
|
|
1437
|
+
maxCpuUtilizationPercent,
|
|
1438
|
+
maxMemoryUtilizationPercent
|
|
1408
1439
|
};
|
|
1409
1440
|
}
|
|
1410
|
-
async function stage(endpoint, concurrency, requestsPerWorker, timeoutMs, fetcher) {
|
|
1441
|
+
async function stage(endpoint, concurrency, requestsPerWorker, timeoutMs, minimumDurationMs, fetcher, sampler) {
|
|
1411
1442
|
const latencies = [];
|
|
1412
1443
|
let succeeded = 0;
|
|
1413
1444
|
let failed = 0;
|
|
1414
1445
|
let overloaded = 0;
|
|
1446
|
+
const systemBefore = sampler();
|
|
1415
1447
|
const started = performance.now();
|
|
1416
1448
|
await Promise.all(Array.from({ length: concurrency }, async () => {
|
|
1417
|
-
|
|
1449
|
+
let request = 0;
|
|
1450
|
+
while (request < requestsPerWorker || performance.now() - started < minimumDurationMs) {
|
|
1451
|
+
request += 1;
|
|
1418
1452
|
const requestStarted = performance.now();
|
|
1419
1453
|
try {
|
|
1420
1454
|
const response = await fetcher(endpoint, {
|
|
@@ -1438,50 +1472,74 @@ async function stage(endpoint, concurrency, requestsPerWorker, timeoutMs, fetche
|
|
|
1438
1472
|
}
|
|
1439
1473
|
}
|
|
1440
1474
|
}));
|
|
1441
|
-
const
|
|
1475
|
+
const durationMs = Math.max(performance.now() - started, 1);
|
|
1476
|
+
const elapsedSeconds = durationMs / 1000;
|
|
1477
|
+
const systemAfter = sampler();
|
|
1478
|
+
const cpuTotal = Math.max(0, systemAfter.cpuTotal - systemBefore.cpuTotal);
|
|
1479
|
+
const cpuIdle = Math.max(0, systemAfter.cpuIdle - systemBefore.cpuIdle);
|
|
1480
|
+
const cpuUtilizationPercent = cpuTotal === 0 ? 0 : (cpuTotal - cpuIdle) / cpuTotal * 100;
|
|
1481
|
+
const memoryUtilizationPercent = Math.max(systemBefore.memoryTotal > 0 ? systemBefore.memoryUsed / systemBefore.memoryTotal * 100 : 0, systemAfter.memoryTotal > 0 ? systemAfter.memoryUsed / systemAfter.memoryTotal * 100 : 0);
|
|
1482
|
+
const requests = succeeded + failed;
|
|
1442
1483
|
return {
|
|
1443
1484
|
concurrency,
|
|
1444
|
-
requests
|
|
1485
|
+
requests,
|
|
1445
1486
|
succeeded,
|
|
1446
1487
|
failed,
|
|
1447
1488
|
overloaded,
|
|
1448
|
-
|
|
1449
|
-
|
|
1489
|
+
successfulRequestsPerSecond: Number((succeeded / elapsedSeconds).toFixed(2)),
|
|
1490
|
+
attemptedRequestsPerSecond: Number((requests / elapsedSeconds).toFixed(2)),
|
|
1491
|
+
durationMs: Number(durationMs.toFixed(2)),
|
|
1492
|
+
errorRate: Number((failed / Math.max(requests, 1)).toFixed(6)),
|
|
1493
|
+
p95Ms: Number(percentile(latencies, 0.95).toFixed(2)),
|
|
1494
|
+
p99Ms: Number(percentile(latencies, 0.99).toFixed(2)),
|
|
1495
|
+
cpuUtilizationPercent: Number(cpuUtilizationPercent.toFixed(2)),
|
|
1496
|
+
memoryUtilizationPercent: Number(memoryUtilizationPercent.toFixed(2))
|
|
1450
1497
|
};
|
|
1451
1498
|
}
|
|
1452
|
-
async function calibrateHttpConcurrency(options, fetcher = fetch) {
|
|
1499
|
+
async function calibrateHttpConcurrency(options, fetcher = fetch, sampler = hostSample) {
|
|
1453
1500
|
const {
|
|
1454
1501
|
endpoint,
|
|
1455
1502
|
maxConcurrency,
|
|
1456
1503
|
requestsPerWorker,
|
|
1457
1504
|
maxP95Ms,
|
|
1458
1505
|
maxErrorRate,
|
|
1459
|
-
|
|
1460
|
-
requestTimeoutMs: timeoutMs
|
|
1506
|
+
safetyRatio,
|
|
1507
|
+
requestTimeoutMs: timeoutMs,
|
|
1508
|
+
minimumStageDurationMs,
|
|
1509
|
+
maxCpuUtilizationPercent,
|
|
1510
|
+
maxMemoryUtilizationPercent
|
|
1461
1511
|
} = validateCapacityCalibrationOptions(options);
|
|
1462
1512
|
const stages = [];
|
|
1463
|
-
let lastSafe
|
|
1513
|
+
let lastSafe;
|
|
1464
1514
|
let stopReason = "maximum-tested";
|
|
1465
1515
|
for (let concurrency = 1;; concurrency = Math.min(maxConcurrency, concurrency * 2)) {
|
|
1466
|
-
const measured = await stage(endpoint, concurrency, requestsPerWorker, timeoutMs, fetcher);
|
|
1516
|
+
const measured = await stage(endpoint, concurrency, requestsPerWorker, timeoutMs, minimumStageDurationMs, fetcher, sampler);
|
|
1467
1517
|
stages.push(measured);
|
|
1468
|
-
const errorRate = measured.failed / measured.requests;
|
|
1469
1518
|
const previous = stages.at(-2);
|
|
1470
|
-
const throughputRegressed = Boolean(previous && concurrency > 1 && measured.
|
|
1471
|
-
if (measured.overloaded > 0 || errorRate > maxErrorRate)
|
|
1519
|
+
const throughputRegressed = Boolean(previous && concurrency > 1 && measured.successfulRequestsPerSecond < previous.successfulRequestsPerSecond * 0.9);
|
|
1520
|
+
if (measured.overloaded > 0 || measured.errorRate > maxErrorRate)
|
|
1472
1521
|
stopReason = "errors";
|
|
1473
1522
|
else if (measured.p95Ms > maxP95Ms)
|
|
1474
1523
|
stopReason = "latency";
|
|
1524
|
+
else if (measured.cpuUtilizationPercent > maxCpuUtilizationPercent)
|
|
1525
|
+
stopReason = "cpu";
|
|
1526
|
+
else if (measured.memoryUtilizationPercent > maxMemoryUtilizationPercent)
|
|
1527
|
+
stopReason = "memory";
|
|
1475
1528
|
else if (throughputRegressed)
|
|
1476
1529
|
stopReason = "throughput-regression";
|
|
1477
1530
|
else
|
|
1478
|
-
lastSafe =
|
|
1531
|
+
lastSafe = measured;
|
|
1479
1532
|
if (stopReason !== "maximum-tested" || concurrency === maxConcurrency)
|
|
1480
1533
|
break;
|
|
1481
1534
|
}
|
|
1535
|
+
if (!lastSafe)
|
|
1536
|
+
throw new Error("Capacity calibration found no safe request stage.");
|
|
1482
1537
|
return {
|
|
1483
1538
|
endpoint: endpoint.toString(),
|
|
1484
|
-
|
|
1539
|
+
measuredSustainableRequestsPerSecond: lastSafe.successfulRequestsPerSecond,
|
|
1540
|
+
allowedRequestsPerSecond: Math.max(1, Math.floor(lastSafe.successfulRequestsPerSecond * safetyRatio)),
|
|
1541
|
+
safetyRatio,
|
|
1542
|
+
recommendedConcurrency: Math.max(1, Math.floor(lastSafe.concurrency * safetyRatio)),
|
|
1485
1543
|
stopReason,
|
|
1486
1544
|
stages
|
|
1487
1545
|
};
|
|
@@ -1558,8 +1616,11 @@ function capacityCalibration(value, where) {
|
|
|
1558
1616
|
"requestsPerWorker",
|
|
1559
1617
|
"maxP95Ms",
|
|
1560
1618
|
"maxErrorRate",
|
|
1561
|
-
"
|
|
1562
|
-
"requestTimeoutMs"
|
|
1619
|
+
"safetyRatio",
|
|
1620
|
+
"requestTimeoutMs",
|
|
1621
|
+
"minimumStageDurationMs",
|
|
1622
|
+
"maxCpuUtilizationPercent",
|
|
1623
|
+
"maxMemoryUtilizationPercent"
|
|
1563
1624
|
], where);
|
|
1564
1625
|
const endpoint = text(calibration.endpoint, `${where}.endpoint`);
|
|
1565
1626
|
try {
|
|
@@ -1583,8 +1644,11 @@ function capacityCalibration(value, where) {
|
|
|
1583
1644
|
"requestsPerWorker",
|
|
1584
1645
|
"maxP95Ms",
|
|
1585
1646
|
"maxErrorRate",
|
|
1586
|
-
"
|
|
1587
|
-
"requestTimeoutMs"
|
|
1647
|
+
"safetyRatio",
|
|
1648
|
+
"requestTimeoutMs",
|
|
1649
|
+
"minimumStageDurationMs",
|
|
1650
|
+
"maxCpuUtilizationPercent",
|
|
1651
|
+
"maxMemoryUtilizationPercent"
|
|
1588
1652
|
].flatMap((name) => {
|
|
1589
1653
|
const found = optionalNumber(name);
|
|
1590
1654
|
return found === undefined ? [] : [[name, found]];
|
|
@@ -2775,35 +2839,76 @@ function requestServiceActivation(request, socketPath = DEFAULT_SOFTWARE_HELPER_
|
|
|
2775
2839
|
function requestSoftware(requirements, socketPath = DEFAULT_SOFTWARE_HELPER_SOCKET, timeoutMs = 15 * 60000) {
|
|
2776
2840
|
validateSoftwareRequirements(requirements);
|
|
2777
2841
|
return new Promise((resolve5, reject) => {
|
|
2778
|
-
const
|
|
2779
|
-
|
|
2780
|
-
|
|
2781
|
-
|
|
2782
|
-
socket.destroy();
|
|
2783
|
-
reject(new Error("software helper did not answer before its deadline"));
|
|
2784
|
-
});
|
|
2785
|
-
socket.on("data", (chunk) => {
|
|
2786
|
-
buffer += chunk.toString("utf8");
|
|
2787
|
-
const newline = buffer.indexOf(`
|
|
2788
|
-
`);
|
|
2789
|
-
if (newline < 0)
|
|
2842
|
+
const deadline = Date.now() + timeoutMs;
|
|
2843
|
+
let settled = false;
|
|
2844
|
+
const finish = (cause, results) => {
|
|
2845
|
+
if (settled)
|
|
2790
2846
|
return;
|
|
2791
|
-
|
|
2847
|
+
settled = true;
|
|
2848
|
+
if (cause)
|
|
2849
|
+
reject(cause);
|
|
2850
|
+
else
|
|
2851
|
+
resolve5(results);
|
|
2852
|
+
};
|
|
2853
|
+
const attempt = () => {
|
|
2854
|
+
if (settled)
|
|
2855
|
+
return;
|
|
2856
|
+
const remaining = deadline - Date.now();
|
|
2857
|
+
if (remaining <= 0)
|
|
2858
|
+
return finish(new Error("software helper did not answer before its deadline"));
|
|
2859
|
+
if (!existsSync7(socketPath)) {
|
|
2860
|
+
setTimeout(attempt, Math.min(100, Math.max(1, remaining)));
|
|
2861
|
+
return;
|
|
2862
|
+
}
|
|
2863
|
+
let buffer = "";
|
|
2864
|
+
let socket;
|
|
2792
2865
|
try {
|
|
2793
|
-
|
|
2794
|
-
|
|
2795
|
-
throw new Error(response.error?.message ?? "software helper refused the request");
|
|
2796
|
-
resolve5(response.results);
|
|
2866
|
+
socket = connect3(socketPath, () => socket.write(`${JSON.stringify({ op: "ensure", requirements })}
|
|
2867
|
+
`));
|
|
2797
2868
|
} catch (cause) {
|
|
2798
|
-
|
|
2869
|
+
const error = cause;
|
|
2870
|
+
if ((error.code === "ENOENT" || error.code === "ECONNREFUSED") && Date.now() < deadline) {
|
|
2871
|
+
setTimeout(attempt, Math.min(100, Math.max(1, deadline - Date.now())));
|
|
2872
|
+
return;
|
|
2873
|
+
}
|
|
2874
|
+
finish(error);
|
|
2875
|
+
return;
|
|
2799
2876
|
}
|
|
2800
|
-
|
|
2801
|
-
|
|
2877
|
+
socket.setTimeout(remaining, () => {
|
|
2878
|
+
socket.destroy();
|
|
2879
|
+
finish(new Error("software helper did not answer before its deadline"));
|
|
2880
|
+
});
|
|
2881
|
+
socket.on("data", (chunk) => {
|
|
2882
|
+
buffer += chunk.toString("utf8");
|
|
2883
|
+
const newline = buffer.indexOf(`
|
|
2884
|
+
`);
|
|
2885
|
+
if (newline < 0)
|
|
2886
|
+
return;
|
|
2887
|
+
socket.end();
|
|
2888
|
+
try {
|
|
2889
|
+
const response = JSON.parse(buffer.slice(0, newline));
|
|
2890
|
+
if (!response.ok || !response.results)
|
|
2891
|
+
throw new Error(response.error?.message ?? "software helper refused the request");
|
|
2892
|
+
finish(undefined, response.results);
|
|
2893
|
+
} catch (cause) {
|
|
2894
|
+
finish(cause);
|
|
2895
|
+
}
|
|
2896
|
+
});
|
|
2897
|
+
socket.on("error", (cause) => {
|
|
2898
|
+
socket.destroy();
|
|
2899
|
+
if ((cause.code === "ENOENT" || cause.code === "ECONNREFUSED") && Date.now() < deadline) {
|
|
2900
|
+
setTimeout(attempt, Math.min(100, Math.max(1, deadline - Date.now())));
|
|
2901
|
+
return;
|
|
2902
|
+
}
|
|
2903
|
+
finish(cause);
|
|
2904
|
+
});
|
|
2905
|
+
};
|
|
2906
|
+
attempt();
|
|
2802
2907
|
});
|
|
2803
2908
|
}
|
|
2804
2909
|
|
|
2805
2910
|
// src/version.ts
|
|
2806
|
-
var VERSION3 = "0.1.
|
|
2911
|
+
var VERSION3 = "0.1.89";
|
|
2807
2912
|
|
|
2808
2913
|
// src/egress-policy.ts
|
|
2809
2914
|
import { realpathSync as realpathSync3 } from "node:fs";
|
|
@@ -2939,6 +3044,7 @@ function agentEgressUnit(options) {
|
|
|
2939
3044
|
}
|
|
2940
3045
|
const deploymentEnabled = Boolean(options.repository || bootstrapBundleEnabled || options.pullDeployments);
|
|
2941
3046
|
const runnerLoopbackPorts = normalizeEgressTcpPorts(options.runnerLoopbackPorts ?? []);
|
|
3047
|
+
const agentLoopbackPorts = normalizeEgressTcpPorts(options.agentLoopbackPorts ?? []);
|
|
2942
3048
|
const runnerPublicTcpPorts = normalizeEgressTcpPorts(options.runnerPublicTcpPorts ?? DEFAULT_RUNNER_PUBLIC_TCP_PORTS);
|
|
2943
3049
|
if (deploymentEnabled && runnerPublicTcpPorts.length < 1) {
|
|
2944
3050
|
throw new Error("deployed project runner needs at least one vetted public TCP port");
|
|
@@ -2946,8 +3052,10 @@ function agentEgressUnit(options) {
|
|
|
2946
3052
|
if (deploymentEnabled)
|
|
2947
3053
|
systemdAgentEgressDirectives(runnerLoopbackPorts);
|
|
2948
3054
|
const users = [user, ...deploymentEnabled ? [DEPLOYMENT_RUNNER_USER] : []];
|
|
2949
|
-
const
|
|
2950
|
-
const
|
|
3055
|
+
const portGrantEnabled = deploymentEnabled || agentLoopbackPorts.length > 0;
|
|
3056
|
+
const restrictedUser = deploymentEnabled ? DEPLOYMENT_RUNNER_USER : user;
|
|
3057
|
+
const portGrant = portGrantEnabled ? ` --loopback-user=${restrictedUser}` + (deploymentEnabled ? runnerLoopbackPorts.map((port) => ` --loopback-tcp-port=${port}`).join("") : "") + agentLoopbackPorts.map((port) => ` --shared-loopback-tcp-port=${port}`).join("") + (deploymentEnabled ? runnerPublicTcpPorts.map((port) => ` --public-tcp-port=${port}`).join("") : "") : "";
|
|
3058
|
+
const policyProof = `ExecStartPost=${bin} egress-policy-check ${users.map((name) => `--user=${name}`).join(" ")}${portGrant}`;
|
|
2951
3059
|
return `[Unit]
|
|
2952
3060
|
Description=ForgeZero Agent host egress policy
|
|
2953
3061
|
Documentation=https://www.forgezero.net/docs/agent
|
|
@@ -2960,7 +3068,7 @@ Type=notify
|
|
|
2960
3068
|
NotifyAccess=all
|
|
2961
3069
|
User=root
|
|
2962
3070
|
Group=root
|
|
2963
|
-
ExecStart=${bin} egress-policy ${users.map((name) => `--user=${name}`).join(" ")}${
|
|
3071
|
+
ExecStart=${bin} egress-policy ${users.map((name) => `--user=${name}`).join(" ")}${portGrant}
|
|
2964
3072
|
${policyProof}
|
|
2965
3073
|
Restart=on-failure
|
|
2966
3074
|
RestartSec=2
|
|
@@ -3244,7 +3352,7 @@ function agentEnrolmentUnit(options) {
|
|
|
3244
3352
|
Requires=forgezero-agent-egress.service
|
|
3245
3353
|
BindsTo=forgezero-agent-egress.service
|
|
3246
3354
|
` : "";
|
|
3247
|
-
const egressDirectives = options.enforceEgress ? systemdAgentEgressDirectives() : "";
|
|
3355
|
+
const egressDirectives = options.enforceEgress ? systemdAgentEgressDirectives(options.agentLoopbackPorts ?? []) : "";
|
|
3248
3356
|
return `[Unit]
|
|
3249
3357
|
Description=Bind this machine to its ForgeZero compute
|
|
3250
3358
|
After=network-online.target
|
|
@@ -3281,7 +3389,7 @@ WantedBy=multi-user.target
|
|
|
3281
3389
|
function deploymentRunnerUnit(options) {
|
|
3282
3390
|
const bin = options.binPath ?? "fz-agent";
|
|
3283
3391
|
const root = options.deployRoot ?? "/opt/forgezero";
|
|
3284
|
-
const agentUser = options.user ?? "forgezero
|
|
3392
|
+
const agentUser = options.user ?? "forgezero";
|
|
3285
3393
|
const egressDependency = options.enforceEgress ? `After=forgezero-agent-egress.service
|
|
3286
3394
|
Requires=forgezero-agent-egress.service
|
|
3287
3395
|
BindsTo=forgezero-agent-egress.service
|
|
@@ -3322,7 +3430,7 @@ RestrictRealtime=true
|
|
|
3322
3430
|
MemoryDenyWriteExecute=true
|
|
3323
3431
|
LockPersonality=true
|
|
3324
3432
|
${egressDirectives}
|
|
3325
|
-
ReadWritePaths=${root}/releases ${root}/runner-home
|
|
3433
|
+
ReadWritePaths=${root}/releases ${root}/runner-home ${root}/slots /etc/nginx/conf.d /var/log/nginx
|
|
3326
3434
|
|
|
3327
3435
|
[Install]
|
|
3328
3436
|
WantedBy=multi-user.target
|
|
@@ -3457,7 +3565,7 @@ function agentUnit(options) {
|
|
|
3457
3565
|
const supplementaryGroups = [
|
|
3458
3566
|
AGENT_UPDATE_GROUP,
|
|
3459
3567
|
deploymentEnabled ? DEPLOYMENT_GROUP : null,
|
|
3460
|
-
|
|
3568
|
+
SOFTWARE_HELPER_GROUP,
|
|
3461
3569
|
lifecycleEnabled ? LIFECYCLE_GROUP : null
|
|
3462
3570
|
].filter((value) => value !== null);
|
|
3463
3571
|
const deploymentGroup = supplementaryGroups.length > 0 ? `SupplementaryGroups=${supplementaryGroups.join(" ")}` : "";
|
|
@@ -3466,7 +3574,7 @@ function agentUnit(options) {
|
|
|
3466
3574
|
"forgezero-agent-update-helper.service",
|
|
3467
3575
|
options.enforceEgress ? "forgezero-agent-egress.service" : null,
|
|
3468
3576
|
deploymentEnabled ? "forgezero-deploy-runner.service" : null,
|
|
3469
|
-
|
|
3577
|
+
"forgezero-software-helper.service",
|
|
3470
3578
|
lifecycleEnabled ? "forgezero-lifecycle-helper.service" : null,
|
|
3471
3579
|
warpEnabled ? "warp-svc.service" : null,
|
|
3472
3580
|
enrolmentEnabled ? "forgezero-agent-enrol.service" : null
|
|
@@ -3475,7 +3583,7 @@ function agentUnit(options) {
|
|
|
3475
3583
|
"forgezero-agent-update-helper.service",
|
|
3476
3584
|
options.enforceEgress ? "forgezero-agent-egress.service" : null,
|
|
3477
3585
|
deploymentEnabled ? "forgezero-deploy-runner.service" : null,
|
|
3478
|
-
|
|
3586
|
+
"forgezero-software-helper.service",
|
|
3479
3587
|
lifecycleEnabled ? "forgezero-lifecycle-helper.service" : null,
|
|
3480
3588
|
warpEnabled ? "warp-svc.service" : null,
|
|
3481
3589
|
enrolmentEnabled ? "forgezero-agent-enrol.service" : null
|
|
@@ -3490,9 +3598,9 @@ function agentUnit(options) {
|
|
|
3490
3598
|
const snpDevice = options.mode === "attested" ? `DevicePolicy=closed
|
|
3491
3599
|
DeviceAllow=/dev/sev-guest rw` : "";
|
|
3492
3600
|
const snpPrepare = options.mode === "attested" ? `ExecStartPre=+/bin/chgrp ${VAULT_GROUP} /dev/sev-guest
|
|
3493
|
-
ExecStartPre=+/bin/chmod
|
|
3601
|
+
ExecStartPre=+/bin/chmod 0660 /dev/sev-guest
|
|
3494
3602
|
` : "";
|
|
3495
|
-
const egressDirectives = options.enforceEgress ? systemdAgentEgressDirectives() : "";
|
|
3603
|
+
const egressDirectives = options.enforceEgress ? systemdAgentEgressDirectives(options.agentLoopbackPorts ?? []) : "";
|
|
3496
3604
|
return `[Unit]
|
|
3497
3605
|
Description=ForgeZero node agent (${options.mode})
|
|
3498
3606
|
Documentation=https://www.forgezero.net/docs/agent
|
|
@@ -3623,6 +3731,7 @@ function planProvision(options) {
|
|
|
3623
3731
|
const deployRoot = options.deployRoot ?? "/opt/forgezero";
|
|
3624
3732
|
const deploymentEnabled = Boolean(options.repository || options.pullDeployments);
|
|
3625
3733
|
const runnerLoopbackPorts = normalizeEgressTcpPorts(options.runnerLoopbackPorts ?? []);
|
|
3734
|
+
const agentLoopbackPorts = normalizeEgressTcpPorts(options.agentLoopbackPorts ?? []);
|
|
3626
3735
|
const runnerPublicTcpPorts = normalizeEgressTcpPorts(options.runnerPublicTcpPorts ?? DEFAULT_RUNNER_PUBLIC_TCP_PORTS);
|
|
3627
3736
|
const lifecycleEnabled = Boolean(options.pullMigrations && options.lifecycleProfilePath);
|
|
3628
3737
|
if (Boolean(options.pullMigrations) !== Boolean(options.lifecycleProfilePath)) {
|
|
@@ -3671,8 +3780,9 @@ function planProvision(options) {
|
|
|
3671
3780
|
const enabledUnits = [
|
|
3672
3781
|
"forgezero-agent.socket",
|
|
3673
3782
|
"forgezero-agent-update-helper.service",
|
|
3783
|
+
"forgezero-software-helper.service",
|
|
3674
3784
|
...options.enforceEgress ? ["forgezero-agent-egress.service"] : [],
|
|
3675
|
-
...deploymentEnabled ? ["forgezero-deploy-runner.service"
|
|
3785
|
+
...deploymentEnabled ? ["forgezero-deploy-runner.service"] : [],
|
|
3676
3786
|
...lifecycleEnabled ? ["forgezero-lifecycle-helper.service"] : [],
|
|
3677
3787
|
...warpEnabled ? ["forgezero-warp-config.service", "warp-svc.service"] : [],
|
|
3678
3788
|
...enrolmentEnabled ? ["forgezero-agent-enrol.service"] : [],
|
|
@@ -3680,8 +3790,9 @@ function planProvision(options) {
|
|
|
3680
3790
|
];
|
|
3681
3791
|
const restartedUnits = [
|
|
3682
3792
|
"forgezero-agent-update-helper.service",
|
|
3793
|
+
"forgezero-software-helper.service",
|
|
3683
3794
|
...options.enforceEgress ? ["forgezero-agent-egress.service"] : [],
|
|
3684
|
-
...deploymentEnabled ? ["forgezero-deploy-runner.service"
|
|
3795
|
+
...deploymentEnabled ? ["forgezero-deploy-runner.service"] : [],
|
|
3685
3796
|
...lifecycleEnabled ? ["forgezero-lifecycle-helper.service"] : [],
|
|
3686
3797
|
...warpEnabled ? ["forgezero-warp-config.service", "warp-svc.service"] : [],
|
|
3687
3798
|
...enrolmentEnabled ? ["forgezero-agent-enrol.service"] : []
|
|
@@ -3705,9 +3816,9 @@ function planProvision(options) {
|
|
|
3705
3816
|
...options.enforceEgress ? [
|
|
3706
3817
|
{ path: AGENT_EGRESS_UNIT_PATH, unit: agentEgressUnit(options) }
|
|
3707
3818
|
] : [],
|
|
3819
|
+
{ path: SOFTWARE_HELPER_UNIT_PATH, unit: softwareHelperUnit(options) },
|
|
3708
3820
|
...deploymentEnabled ? [
|
|
3709
|
-
{ path: DEPLOYMENT_RUNNER_UNIT_PATH, unit: deploymentRunnerUnit(options) }
|
|
3710
|
-
{ path: SOFTWARE_HELPER_UNIT_PATH, unit: softwareHelperUnit(options) }
|
|
3821
|
+
{ path: DEPLOYMENT_RUNNER_UNIT_PATH, unit: deploymentRunnerUnit(options) }
|
|
3711
3822
|
] : [],
|
|
3712
3823
|
...enrolmentEnabled ? [
|
|
3713
3824
|
{ path: ENROLMENT_UNIT_PATH, unit: agentEnrolmentUnit(options) }
|
|
@@ -3740,24 +3851,29 @@ function planProvision(options) {
|
|
|
3740
3851
|
step("Agent update helper access group", { kind: "commands", commands: [{ argv: ["/usr/sbin/groupadd", "--system", AGENT_UPDATE_GROUP], acceptedExitCodes: [0, 9] }] }),
|
|
3741
3852
|
...sourceBinPath && binPath ? [step("root-owned agent runtime", { kind: "install-runtime", source: sourceBinPath, binary: binPath, version: VERSION3 })] : [],
|
|
3742
3853
|
...warpEnabled ? [step("Cloudflare One client for Ubuntu 26.04", { kind: "install-warp" })] : [],
|
|
3743
|
-
|
|
3744
|
-
{ argv: ["/usr/sbin/groupadd", "--system", DEPLOYMENT_GROUP], acceptedExitCodes: [0, 9] },
|
|
3854
|
+
step("software strategy helper group", { kind: "commands", commands: [
|
|
3745
3855
|
{ argv: ["/usr/sbin/groupadd", "--system", SOFTWARE_HELPER_GROUP], acceptedExitCodes: [0, 9] }
|
|
3856
|
+
] }),
|
|
3857
|
+
...deploymentEnabled ? [step("deployment isolation group", { kind: "commands", commands: [
|
|
3858
|
+
{ argv: ["/usr/sbin/groupadd", "--system", DEPLOYMENT_GROUP], acceptedExitCodes: [0, 9] }
|
|
3746
3859
|
] })] : [],
|
|
3747
3860
|
...lifecycleEnabled ? [step("lifecycle helper access group", { kind: "commands", commands: [{ argv: ["/usr/sbin/groupadd", "--system", LIFECYCLE_GROUP], acceptedExitCodes: [0, 9] }] })] : [],
|
|
3748
3861
|
step("service account", { kind: "commands", commands: [{ argv: ["/usr/sbin/useradd", "--system", "--no-create-home", "--shell", "/usr/sbin/nologin", user], acceptedExitCodes: [0, 9] }] }),
|
|
3749
3862
|
step("bind service account to vault group", { kind: "commands", commands: [{ argv: ["/usr/sbin/usermod", "-g", VAULT_GROUP, user] }] }),
|
|
3750
3863
|
step("grant verified Agent update access", { kind: "commands", commands: [{ argv: ["/usr/sbin/usermod", "-a", "-G", AGENT_UPDATE_GROUP, user] }] }),
|
|
3864
|
+
step("grant software helper socket access", { kind: "commands", commands: [
|
|
3865
|
+
{ argv: ["/usr/sbin/usermod", "-a", "-G", SOFTWARE_HELPER_GROUP, user] }
|
|
3866
|
+
] }),
|
|
3751
3867
|
...lifecycleEnabled ? [step("grant lifecycle helper socket access", { kind: "commands", commands: [{ argv: ["/usr/sbin/usermod", "-a", "-G", LIFECYCLE_GROUP, user] }] })] : [],
|
|
3752
3868
|
...deploymentEnabled ? [step("credential-free deployment account", { kind: "commands", commands: [
|
|
3753
3869
|
{ argv: ["/usr/sbin/useradd", "--system", "--no-create-home", "--shell", "/usr/sbin/nologin", "--gid", DEPLOYMENT_GROUP, DEPLOYMENT_RUNNER_USER], acceptedExitCodes: [0, 9] },
|
|
3754
3870
|
{ argv: ["/usr/sbin/useradd", "--system", "--no-create-home", "--shell", "/usr/sbin/nologin", "--gid", VAULT_GROUP, APPLICATION_RUNTIME_USER], acceptedExitCodes: [0, 9] },
|
|
3755
3871
|
{ argv: ["/usr/sbin/usermod", "-a", "-G", DEPLOYMENT_GROUP, APPLICATION_RUNTIME_USER] },
|
|
3756
|
-
{ argv: ["/usr/sbin/usermod", "-a", "-G",
|
|
3872
|
+
{ argv: ["/usr/sbin/usermod", "-a", "-G", DEPLOYMENT_GROUP, user] }
|
|
3757
3873
|
] })] : [],
|
|
3758
3874
|
step("credential and state directories", { kind: "directories", directories: [
|
|
3759
3875
|
{ path: credentialDir, mode: 448, owner: "root", group: "root" },
|
|
3760
|
-
{ path: "/var/lib/forgezero", mode: 488, owner: "root", group:
|
|
3876
|
+
{ path: "/var/lib/forgezero", mode: 488, owner: "root", group: VAULT_GROUP }
|
|
3761
3877
|
] }),
|
|
3762
3878
|
step("encrypted node identity", { kind: "ensure-seed", credential: seedCredentialPath }),
|
|
3763
3879
|
...options.generateGitIdentity && gitCredentialPath && gitPublicKeyPath ? [
|
|
@@ -3808,19 +3924,26 @@ function planProvision(options) {
|
|
|
3808
3924
|
{ argv: ["/usr/bin/systemctl", "enable", ...enabledUnits] },
|
|
3809
3925
|
{ argv: ["/usr/bin/systemctl", "reset-failed", "forgezero-agent.service"], acceptedExitCodes: [0, 1] },
|
|
3810
3926
|
...restartedUnits.length ? [{ argv: ["/usr/bin/systemctl", "restart", ...restartedUnits] }] : [],
|
|
3927
|
+
{ argv: ["/usr/bin/systemctl", "stop", "forgezero-agent-proxy.service"], acceptedExitCodes: [0, 1, 5] },
|
|
3811
3928
|
{ argv: ["/usr/bin/systemctl", "restart", "forgezero-agent.socket"] },
|
|
3812
3929
|
{ argv: ["/usr/bin/systemctl", "reset-failed", "forgezero-agent.service"], acceptedExitCodes: [0, 1] },
|
|
3813
3930
|
{ argv: ["/usr/bin/systemctl", "restart", "forgezero-agent.service"] }
|
|
3814
3931
|
] }),
|
|
3815
3932
|
...enrolmentEnabled ? [step("prove the compute binding is durable", { kind: "verify-file", path: enrolStatePath })] : [],
|
|
3816
|
-
...options.enforceEgress ? [step("prove the Agent egress policy is active", {
|
|
3933
|
+
...options.enforceEgress ? [step("prove the Agent egress policy is active", {
|
|
3934
|
+
kind: "verify-egress",
|
|
3935
|
+
deploymentEnabled,
|
|
3936
|
+
runnerPublicTcpPorts,
|
|
3937
|
+
runnerLoopbackPorts,
|
|
3938
|
+
agentLoopbackPorts
|
|
3939
|
+
})] : [],
|
|
3817
3940
|
step("prove it is running", { kind: "commands", commands: [{ argv: ["/usr/bin/systemctl", "is-active", "forgezero-agent.service"] }] }),
|
|
3818
3941
|
step("prove the public Vault socket exists", { kind: "wait-socket", path: options.socketPath, attempts: 100, intervalMs: 100 }),
|
|
3819
3942
|
step("prove the Agent Vault backend exists", { kind: "wait-socket", path: agentBackendSocketPath(options.socketPath), attempts: 100, intervalMs: 100 }),
|
|
3820
3943
|
step("prove the Agent update helper exists", { kind: "wait-socket", path: DEFAULT_AGENT_UPDATE_SOCKET, attempts: 100, intervalMs: 100 }),
|
|
3944
|
+
step("prove the software strategy helper socket exists", { kind: "wait-socket", path: DEFAULT_SOFTWARE_HELPER_SOCKET, attempts: 100, intervalMs: 100 }),
|
|
3821
3945
|
...deploymentEnabled ? [
|
|
3822
|
-
step("prove the deployment runner socket exists", { kind: "wait-socket", path: DEPLOYMENT_RUNNER_SOCKET, attempts: 100, intervalMs: 100 })
|
|
3823
|
-
step("prove the software strategy helper socket exists", { kind: "wait-socket", path: DEFAULT_SOFTWARE_HELPER_SOCKET, attempts: 100, intervalMs: 100 })
|
|
3946
|
+
step("prove the deployment runner socket exists", { kind: "wait-socket", path: DEPLOYMENT_RUNNER_SOCKET, attempts: 100, intervalMs: 100 })
|
|
3824
3947
|
] : [],
|
|
3825
3948
|
...lifecycleEnabled ? [step("prove the lifecycle helper socket exists", { kind: "wait-socket", path: lifecycleHelperSocketPath, attempts: 100, intervalMs: 100 })] : [],
|
|
3826
3949
|
...warpEnabled ? [step("prove Cloudflare WARP is connected", { kind: "verify-warp" })] : [],
|
|
@@ -4341,12 +4464,26 @@ map $limit_conn_status $forgezero_retry_after { default ''; REJECTED 1; REJECTED
|
|
|
4341
4464
|
server {
|
|
4342
4465
|
listen 127.0.0.1:${input.publicPort};
|
|
4343
4466
|
server_name _;
|
|
4467
|
+
server_tokens off;
|
|
4468
|
+
more_clear_headers Server X-Powered-By;
|
|
4344
4469
|
limit_conn forgezero_admission ${concurrencyLimit};
|
|
4345
4470
|
limit_conn_status 503;
|
|
4346
4471
|
add_header Retry-After $forgezero_retry_after always;
|
|
4347
|
-
|
|
4348
|
-
location
|
|
4349
|
-
location / {
|
|
4472
|
+
error_page 400 403 404 405 408 413 414 429 500 502 503 504 = @transport_failure;
|
|
4473
|
+
location @transport_failure { internal; return 444; }
|
|
4474
|
+
location / {
|
|
4475
|
+
proxy_pass http://forgezero;
|
|
4476
|
+
proxy_http_version 1.1;
|
|
4477
|
+
proxy_intercept_errors off;
|
|
4478
|
+
proxy_hide_header Server;
|
|
4479
|
+
proxy_hide_header X-Powered-By;
|
|
4480
|
+
proxy_set_header Host $host;
|
|
4481
|
+
proxy_set_header X-Forwarded-Proto https;
|
|
4482
|
+
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
|
4483
|
+
proxy_set_header Upgrade $http_upgrade;
|
|
4484
|
+
proxy_set_header Connection $forgezero_connection;
|
|
4485
|
+
proxy_read_timeout 3600s;
|
|
4486
|
+
}
|
|
4350
4487
|
}
|
|
4351
4488
|
`
|
|
4352
4489
|
};
|
|
@@ -4419,11 +4556,29 @@ var replaceLink = (path2, target) => {
|
|
|
4419
4556
|
symlinkSync4(target, pending);
|
|
4420
4557
|
renameSync8(pending, path2);
|
|
4421
4558
|
};
|
|
4559
|
+
var assertReleaseLinksContained = (path2) => {
|
|
4560
|
+
const root = realpathSync4(path2);
|
|
4561
|
+
const visit = (current2) => {
|
|
4562
|
+
const metadata = lstatSync(current2);
|
|
4563
|
+
if (metadata.isSymbolicLink()) {
|
|
4564
|
+
const target = realpathSync4(current2);
|
|
4565
|
+
if (target !== root && !target.startsWith(`${root}/`)) {
|
|
4566
|
+
throw new Error("release contains a symbolic link outside its immutable root");
|
|
4567
|
+
}
|
|
4568
|
+
return;
|
|
4569
|
+
}
|
|
4570
|
+
if (metadata.isDirectory())
|
|
4571
|
+
for (const name of readdirSync3(current2))
|
|
4572
|
+
visit(join5(current2, name));
|
|
4573
|
+
};
|
|
4574
|
+
visit(root);
|
|
4575
|
+
};
|
|
4422
4576
|
var secureRelease = (path2, uid, gid) => {
|
|
4577
|
+
assertReleaseLinksContained(path2);
|
|
4423
4578
|
const visit = (current2) => {
|
|
4424
4579
|
const metadata = lstatSync(current2);
|
|
4425
4580
|
if (metadata.isSymbolicLink())
|
|
4426
|
-
|
|
4581
|
+
return;
|
|
4427
4582
|
chownSync(current2, uid, gid);
|
|
4428
4583
|
chmodSync6(current2, metadata.isDirectory() ? 365 : 292);
|
|
4429
4584
|
if (metadata.isDirectory())
|
|
@@ -4449,7 +4604,7 @@ async function activatePlatformRelease(config, requestedRelease, options = {}) {
|
|
|
4449
4604
|
const sleep = options.sleep ?? Bun.sleep;
|
|
4450
4605
|
const slots = join5(normalized.root, "slots");
|
|
4451
4606
|
mkdirSync8(slots, { recursive: true, mode: 493 });
|
|
4452
|
-
const slotFile = join5(
|
|
4607
|
+
const slotFile = join5(slots, ".active");
|
|
4453
4608
|
const previousSlot = existsSync8(slotFile) ? readFileSync6(slotFile, "utf8").trim() : undefined;
|
|
4454
4609
|
const target = previousSlot === "blue" ? "green" : "blue";
|
|
4455
4610
|
const port = target === "blue" ? normalized.bluePort : normalized.greenPort;
|
|
@@ -4499,6 +4654,7 @@ async function activatePlatformRelease(config, requestedRelease, options = {}) {
|
|
|
4499
4654
|
const test = await exec(["/usr/sbin/nginx", "-t"]);
|
|
4500
4655
|
const reload = test.exitCode === 0 ? await exec(["/usr/sbin/nginx", "-s", "reload"]) : test;
|
|
4501
4656
|
if (reload.exitCode !== 0) {
|
|
4657
|
+
const refusal = (reload.output || test.output).trim().slice(0, 2000);
|
|
4502
4658
|
if (existsSync8(backup))
|
|
4503
4659
|
renameSync8(backup, upstream);
|
|
4504
4660
|
else
|
|
@@ -4506,7 +4662,7 @@ async function activatePlatformRelease(config, requestedRelease, options = {}) {
|
|
|
4506
4662
|
await exec(["/usr/sbin/nginx", "-t"]);
|
|
4507
4663
|
await exec(["/usr/sbin/nginx", "-s", "reload"]);
|
|
4508
4664
|
await stopTarget();
|
|
4509
|
-
throw new Error(
|
|
4665
|
+
throw new Error(`nginx refused the promoted upstream${refusal ? `: ${refusal}` : ""}`);
|
|
4510
4666
|
}
|
|
4511
4667
|
rmSync7(backup, { force: true });
|
|
4512
4668
|
writeFileSync7(slotFile, `${target}
|
|
@@ -4592,6 +4748,25 @@ import {
|
|
|
4592
4748
|
writeFileSync as writeFileSync8
|
|
4593
4749
|
} from "node:fs";
|
|
4594
4750
|
import { dirname as dirname9 } from "node:path";
|
|
4751
|
+
function normalizeEd25519PublicKey(output) {
|
|
4752
|
+
const line = output.trim();
|
|
4753
|
+
if (!/^ssh-ed25519 [A-Za-z0-9+/]+={0,3}(?: [^\r\n]{1,256})?$/.test(line))
|
|
4754
|
+
return;
|
|
4755
|
+
const [algorithm, encoded] = line.split(" ", 3);
|
|
4756
|
+
if (algorithm !== "ssh-ed25519" || !encoded)
|
|
4757
|
+
return;
|
|
4758
|
+
let blob;
|
|
4759
|
+
try {
|
|
4760
|
+
blob = Buffer.from(encoded, "base64");
|
|
4761
|
+
} catch {
|
|
4762
|
+
return;
|
|
4763
|
+
}
|
|
4764
|
+
if (blob.length !== 51 || blob.readUInt32BE(0) !== 11 || blob.subarray(4, 15).toString("ascii") !== "ssh-ed25519" || blob.readUInt32BE(15) !== 32)
|
|
4765
|
+
return;
|
|
4766
|
+
if (blob.toString("base64") !== encoded)
|
|
4767
|
+
return;
|
|
4768
|
+
return `${algorithm} ${encoded}`;
|
|
4769
|
+
}
|
|
4595
4770
|
async function readCapabilities(run2) {
|
|
4596
4771
|
const answers = {};
|
|
4597
4772
|
const checks = Object.entries(CAPABILITY_CHECKS);
|
|
@@ -4731,7 +4906,7 @@ var runProvisionOperation = async (operation) => {
|
|
|
4731
4906
|
return result;
|
|
4732
4907
|
}
|
|
4733
4908
|
result = await fixed(["/usr/bin/ssh-keygen", "-y", "-f", key]);
|
|
4734
|
-
if (result.exitCode !== 0 ||
|
|
4909
|
+
if (result.exitCode !== 0 || !normalizeEd25519PublicKey(result.stdout)) {
|
|
4735
4910
|
return { stdout: "bootstrap SSH private key is not a valid Ed25519 OpenSSH key", exitCode: 1 };
|
|
4736
4911
|
}
|
|
4737
4912
|
result = await fixed(["/usr/bin/systemd-creds", "encrypt", "--name=bootstrap-ssh-key", key, operation.credential]);
|
|
@@ -4747,11 +4922,12 @@ var runProvisionOperation = async (operation) => {
|
|
|
4747
4922
|
chmodSync7(key, 384);
|
|
4748
4923
|
}
|
|
4749
4924
|
const derived = await fixed(["/usr/bin/ssh-keygen", "-y", "-f", key]);
|
|
4750
|
-
|
|
4925
|
+
const publicKey2 = normalizeEd25519PublicKey(derived.stdout);
|
|
4926
|
+
if (derived.exitCode !== 0 || !publicKey2) {
|
|
4751
4927
|
return { stdout: "bootstrap SSH public key derivation failed", exitCode: 1 };
|
|
4752
4928
|
}
|
|
4753
4929
|
mkdirSync9(dirname9(operation.publicKey), { recursive: true, mode: 493 });
|
|
4754
|
-
writeFileSync8(operation.publicKey, `${
|
|
4930
|
+
writeFileSync8(operation.publicKey, `${publicKey2} forgezero-bootstrap-runner
|
|
4755
4931
|
`, { mode: 292 });
|
|
4756
4932
|
chmodSync7(operation.publicKey, 292);
|
|
4757
4933
|
}
|
|
@@ -4794,8 +4970,9 @@ var runProvisionOperation = async (operation) => {
|
|
|
4794
4970
|
const policy = await fixed(["/usr/sbin/nft", "--numeric", "list", "table", "inet", "forgezero_agent_egress"]);
|
|
4795
4971
|
const required = [
|
|
4796
4972
|
"forgezero-agent-egress-v1",
|
|
4797
|
-
`
|
|
4798
|
-
...operation.
|
|
4973
|
+
...operation.agentLoopbackPorts.length ? [`shared-loopback=${operation.agentLoopbackPorts.join(",")}`] : [],
|
|
4974
|
+
...operation.deploymentEnabled && operation.runnerPublicTcpPorts.length ? [`public-tcp=${operation.runnerPublicTcpPorts.join(",")}`] : [],
|
|
4975
|
+
...operation.deploymentEnabled && operation.runnerLoopbackPorts.length ? [`:${operation.runnerLoopbackPorts.join(",")}`] : []
|
|
4799
4976
|
];
|
|
4800
4977
|
return policy.exitCode === 0 && required.every((part) => policy.stdout.includes(part)) ? { stdout: policy.stdout, exitCode: 0 } : { stdout: policy.stdout, exitCode: 1 };
|
|
4801
4978
|
}
|
|
@@ -4861,7 +5038,9 @@ async function applyPlan(plan, run2) {
|
|
|
4861
5038
|
const result = await run2(step2.operation);
|
|
4862
5039
|
transcript.push({ label: step2.label, command: step2.command, exitCode: result.exitCode });
|
|
4863
5040
|
if (result.exitCode !== 0 && !step2.optional) {
|
|
4864
|
-
|
|
5041
|
+
const safeDetail = step2.operation.kind === "ensure-bootstrap-ssh-identity" ? result.stdout.trim().slice(0, 1000) : "";
|
|
5042
|
+
throw new Error(`${step2.label} failed (exit ${result.exitCode}): ${step2.command}${safeDetail ? `
|
|
5043
|
+
${safeDetail}` : ""}`);
|
|
4865
5044
|
}
|
|
4866
5045
|
}
|
|
4867
5046
|
return transcript;
|
|
@@ -4951,7 +5130,6 @@ var SEED_CREDENTIAL = `${CREDS}/seed-sync-root.cred`;
|
|
|
4951
5130
|
var BACKUP_RECOVERY_CREDENTIAL = `${CREDS}/backup-recovery-root.cred`;
|
|
4952
5131
|
var BOOTSTRAP_SSH_CREDENTIAL = `${CREDS}/bootstrap-ssh-key.cred`;
|
|
4953
5132
|
var BOOTSTRAP_SSH_PUBLIC_KEY = "/etc/forgezero/bootstrap/runner.pub";
|
|
4954
|
-
var BOOTSTRAP_RELEASE_EVIDENCE = "/var/lib/forgezero/bootstrap-release.json";
|
|
4955
5133
|
var LIFECYCLE_PROFILE = "/etc/forgezero/lifecycle.json";
|
|
4956
5134
|
var CONTROL_SOCKET = "/run/forgezero/control.sock";
|
|
4957
5135
|
var CLOUDFLARED_METRICS_ADDRESS = "127.0.0.1:20241";
|
|
@@ -5170,6 +5348,8 @@ function planBootstrapAgentInstall(config, phase, context) {
|
|
|
5170
5348
|
bootstrapTargetTelemetryEndpoint: bound && bootstrapRunner ? platformBootstrapRunner(config) ? config.telemetryEndpoint : config.kind === "enrolled-compute" ? config.bootstrapRunner?.targetTelemetryEndpoint : undefined : undefined,
|
|
5171
5349
|
lifecycleProfilePath: bound && config.kind === "platform" ? LIFECYCLE_PROFILE : undefined,
|
|
5172
5350
|
enforceEgress: true,
|
|
5351
|
+
runnerLoopbackPorts: config.kind === "platform" ? [config.runtime.environment.publicApiPort, config.runtime.bluePort, config.runtime.greenPort] : [],
|
|
5352
|
+
agentLoopbackPorts: config.kind === "platform" ? [config.runtime.environment.publicApiPort] : [],
|
|
5173
5353
|
nodeHostname: config.nodeHostname,
|
|
5174
5354
|
telemetryEndpoint: config.telemetryEndpoint,
|
|
5175
5355
|
binPath: "/usr/local/lib/forgezero/agent/fz-agent",
|
|
@@ -5179,7 +5359,7 @@ function planBootstrapAgentInstall(config, phase, context) {
|
|
|
5179
5359
|
enrolStatePath: "/var/lib/forgezero/enrolment.json"
|
|
5180
5360
|
} : bound ? { enrolStatePath: "/var/lib/forgezero/enrolment.json" } : {},
|
|
5181
5361
|
...authenticated ? {
|
|
5182
|
-
apiUrl: config.apiUrl,
|
|
5362
|
+
apiUrl: config.kind === "platform" ? `http://127.0.0.1:${config.runtime.environment.publicApiPort}` : config.apiUrl,
|
|
5183
5363
|
project: config.kind === "enrolled-compute" ? config.realm : "platform",
|
|
5184
5364
|
environment: config.kind === "enrolled-compute" ? undefined : config.environment,
|
|
5185
5365
|
nodeLabel: config.kind === "enrolled-compute" ? config.nodeHostname : config.computeReference
|
|
@@ -5188,6 +5368,7 @@ function planBootstrapAgentInstall(config, phase, context) {
|
|
|
5188
5368
|
return planInstall(options);
|
|
5189
5369
|
}
|
|
5190
5370
|
function localBootstrapHost() {
|
|
5371
|
+
let softwareClientUser;
|
|
5191
5372
|
const execute = async (argv2, options = {}) => {
|
|
5192
5373
|
const child = Bun.spawn([...argv2], { stdin: options.stdin === undefined ? "ignore" : "pipe", stdout: "pipe", stderr: "pipe" });
|
|
5193
5374
|
if (options.stdin !== undefined && child.stdin && typeof child.stdin !== "number") {
|
|
@@ -5220,10 +5401,12 @@ function localBootstrapHost() {
|
|
|
5220
5401
|
exec: execute,
|
|
5221
5402
|
sleep: (milliseconds) => Bun.sleep(milliseconds),
|
|
5222
5403
|
async ensureSoftware(requirements) {
|
|
5404
|
+
if (!softwareClientUser)
|
|
5405
|
+
throw new Error("Agent must be converged before software requirements are applied");
|
|
5223
5406
|
const result = await execute([
|
|
5224
5407
|
"runuser",
|
|
5225
5408
|
"-u",
|
|
5226
|
-
|
|
5409
|
+
softwareClientUser,
|
|
5227
5410
|
"--",
|
|
5228
5411
|
"/usr/local/bin/fz-agent",
|
|
5229
5412
|
"software-ensure",
|
|
@@ -5237,7 +5420,7 @@ function localBootstrapHost() {
|
|
|
5237
5420
|
const capabilities = await readCapabilities(localRunner);
|
|
5238
5421
|
const hasBinding = existsSync11("/var/lib/forgezero/enrolment.json");
|
|
5239
5422
|
const hasEnrolCredential = existsSync11(ENROL_CREDENTIAL);
|
|
5240
|
-
const initialBundle = config.kind === "platform" &&
|
|
5423
|
+
const initialBundle = config.kind === "platform" && phase === "bootstrap" && existsSync11(config.bootstrapBundle.bundleFile) && existsSync11(config.bootstrapBundle.manifestFile) ? {
|
|
5241
5424
|
path: config.bootstrapBundle.bundleFile,
|
|
5242
5425
|
manifestPath: config.bootstrapBundle.manifestFile,
|
|
5243
5426
|
manifest: parseBootstrapBundleManifest(JSON.parse(readFileSync9(config.bootstrapBundle.manifestFile, "utf8")))
|
|
@@ -5268,6 +5451,7 @@ function localBootstrapHost() {
|
|
|
5268
5451
|
writeFileSync10(unit2.path, unit2.unit, { mode: 420 });
|
|
5269
5452
|
}
|
|
5270
5453
|
await applyPlan(plan, localRunner);
|
|
5454
|
+
softwareClientUser = plan.user;
|
|
5271
5455
|
return plan;
|
|
5272
5456
|
}
|
|
5273
5457
|
};
|