@forgezero/agent 0.1.88 → 0.1.90
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -9
- package/dist/agent-heartbeat.d.ts +3 -0
- package/dist/agent-heartbeat.js +2 -1
- package/dist/bootstrap.d.ts +8 -1
- package/dist/bootstrap.js +241 -62
- package/dist/capacity-calibration.d.ts +35 -5
- package/dist/capacity-calibration.js +66 -22
- package/dist/cli/agent-install.d.ts +3 -0
- package/dist/community-rehearsal-host.js +7 -2
- package/dist/control.d.ts +3 -1
- package/dist/definition.js +83 -28
- package/dist/deploy-file.js +83 -28
- package/dist/deployment.d.ts +5 -0
- package/dist/egress-policy.d.ts +4 -0
- package/dist/fz-agent.js +450 -158
- package/dist/fz.js +429 -138
- package/dist/host-maintenance.js +1 -1
- package/dist/index.d.ts +7 -0
- package/dist/metal-bootstrap.js +1 -1
- package/dist/metal-helper-socket.js +7 -2
- package/dist/metal-provision.js +7 -2
- package/dist/migration-pull.d.ts +1 -0
- package/dist/migration-pull.js +4 -0
- package/dist/operator-bootstrap.d.ts +12 -0
- package/dist/operator-bootstrap.js +385 -72
- package/dist/platform-bootstrap-runtime.d.ts +1 -0
- package/dist/platform-bootstrap-runtime.js +47 -8
- package/dist/platform-fleet-verification.js +273 -95
- package/dist/platform-genesis.js +7 -2
- package/dist/platform-launch-profile.d.ts +1 -0
- package/dist/provision.d.ts +5 -1
- package/dist/provision.js +192 -75
- package/dist/recovery-host.js +1 -1
- package/dist/software-helper.js +144 -48
- package/dist/software.js +7 -2
- package/dist/ssh-bootstrap.d.ts +1 -0
- package/dist/ubuntu.js +7 -2
- package/dist/version.d.ts +1 -1
- package/package.json +1 -1
- package/schema/deploy-v3.json +5 -2
|
@@ -1198,7 +1198,12 @@ async function executeSoftwareOperation(operation) {
|
|
|
1198
1198
|
}
|
|
1199
1199
|
if (software === "nginx") {
|
|
1200
1200
|
const binary = await run(["/usr/sbin/nginx", "-v"]);
|
|
1201
|
-
|
|
1201
|
+
if (binary.exitCode !== 0)
|
|
1202
|
+
return binary;
|
|
1203
|
+
if (!existsSync4("/usr/lib/nginx/modules/ngx_http_headers_more_filter_module.so")) {
|
|
1204
|
+
return { exitCode: 1, output: "Nginx response-header suppression module is unavailable" };
|
|
1205
|
+
}
|
|
1206
|
+
return run(["/usr/bin/systemctl", "is-active", "--quiet", "nginx.service"]);
|
|
1202
1207
|
}
|
|
1203
1208
|
if (software === "arangodb") {
|
|
1204
1209
|
const binary = await run(["/usr/sbin/arangod", "--version"]);
|
|
@@ -1230,7 +1235,7 @@ async function executeSoftwareOperation(operation) {
|
|
|
1230
1235
|
}
|
|
1231
1236
|
if (software === "docker" || software === "nginx" || software === "ufw" || software === "openssh-client" || software === "git") {
|
|
1232
1237
|
const packageName = software === "openssh-client" ? "openssh-client" : software === "docker" ? "docker.io" : software;
|
|
1233
|
-
const installed = await aptInstall(packageName);
|
|
1238
|
+
const installed = software === "nginx" ? await aptInstallMany(["nginx", "libnginx-mod-http-headers-more-filter"]) : await aptInstall(packageName);
|
|
1234
1239
|
if (installed.exitCode !== 0 || software !== "nginx" && software !== "docker")
|
|
1235
1240
|
return installed;
|
|
1236
1241
|
if (software === "docker") {
|
|
@@ -1382,11 +1387,22 @@ async function ensureSoftwareRequirements(requirementsInput, options) {
|
|
|
1382
1387
|
}
|
|
1383
1388
|
|
|
1384
1389
|
// src/capacity-calibration.ts
|
|
1385
|
-
|
|
1390
|
+
import { cpus, freemem, totalmem } from "node:os";
|
|
1391
|
+
var percentile = (values, quantile) => {
|
|
1386
1392
|
if (values.length === 0)
|
|
1387
1393
|
return Number.POSITIVE_INFINITY;
|
|
1388
1394
|
const sorted = values.toSorted((left, right) => left - right);
|
|
1389
|
-
return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length *
|
|
1395
|
+
return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * quantile) - 1)];
|
|
1396
|
+
};
|
|
1397
|
+
var hostSample = () => {
|
|
1398
|
+
let cpuIdle = 0;
|
|
1399
|
+
let cpuTotal = 0;
|
|
1400
|
+
for (const cpu of cpus()) {
|
|
1401
|
+
cpuIdle += cpu.times.idle;
|
|
1402
|
+
cpuTotal += Object.values(cpu.times).reduce((sum, value) => sum + value, 0);
|
|
1403
|
+
}
|
|
1404
|
+
const memoryTotal = totalmem();
|
|
1405
|
+
return { cpuIdle, cpuTotal, memoryUsed: memoryTotal - freemem(), memoryTotal };
|
|
1390
1406
|
};
|
|
1391
1407
|
function localCalibrationEndpoint(value) {
|
|
1392
1408
|
const endpoint = new URL(value);
|
|
@@ -1401,9 +1417,12 @@ function validateCapacityCalibrationOptions(options) {
|
|
|
1401
1417
|
const requestsPerWorker = options.requestsPerWorker ?? 8;
|
|
1402
1418
|
const maxP95Ms = options.maxP95Ms ?? 250;
|
|
1403
1419
|
const maxErrorRate = options.maxErrorRate ?? 0.01;
|
|
1404
|
-
const
|
|
1420
|
+
const safetyRatio = options.safetyRatio ?? 0.8;
|
|
1405
1421
|
const requestTimeoutMs = options.requestTimeoutMs ?? 5000;
|
|
1406
|
-
|
|
1422
|
+
const minimumStageDurationMs = options.minimumStageDurationMs ?? 5000;
|
|
1423
|
+
const maxCpuUtilizationPercent = options.maxCpuUtilizationPercent ?? 90;
|
|
1424
|
+
const maxMemoryUtilizationPercent = options.maxMemoryUtilizationPercent ?? 90;
|
|
1425
|
+
if (!Number.isSafeInteger(maxConcurrency) || maxConcurrency < 1 || maxConcurrency > 4096 || !Number.isSafeInteger(requestsPerWorker) || requestsPerWorker < 2 || requestsPerWorker > 100 || !Number.isFinite(maxP95Ms) || maxP95Ms < 1 || !Number.isFinite(maxErrorRate) || maxErrorRate < 0 || maxErrorRate > 0.2 || !Number.isFinite(safetyRatio) || safetyRatio < 0.25 || safetyRatio > 0.95 || !Number.isSafeInteger(requestTimeoutMs) || requestTimeoutMs < 100 || requestTimeoutMs > 30000 || !Number.isSafeInteger(minimumStageDurationMs) || minimumStageDurationMs < 100 || minimumStageDurationMs > 60000 || !Number.isFinite(maxCpuUtilizationPercent) || maxCpuUtilizationPercent < 25 || maxCpuUtilizationPercent > 100 || !Number.isFinite(maxMemoryUtilizationPercent) || maxMemoryUtilizationPercent < 25 || maxMemoryUtilizationPercent > 100) {
|
|
1407
1426
|
throw new Error("Capacity calibration bounds are invalid.");
|
|
1408
1427
|
}
|
|
1409
1428
|
return {
|
|
@@ -1412,18 +1431,24 @@ function validateCapacityCalibrationOptions(options) {
|
|
|
1412
1431
|
requestsPerWorker,
|
|
1413
1432
|
maxP95Ms,
|
|
1414
1433
|
maxErrorRate,
|
|
1415
|
-
|
|
1416
|
-
requestTimeoutMs
|
|
1434
|
+
safetyRatio,
|
|
1435
|
+
requestTimeoutMs,
|
|
1436
|
+
minimumStageDurationMs,
|
|
1437
|
+
maxCpuUtilizationPercent,
|
|
1438
|
+
maxMemoryUtilizationPercent
|
|
1417
1439
|
};
|
|
1418
1440
|
}
|
|
1419
|
-
async function stage(endpoint, concurrency, requestsPerWorker, timeoutMs, fetcher) {
|
|
1441
|
+
async function stage(endpoint, concurrency, requestsPerWorker, timeoutMs, minimumDurationMs, fetcher, sampler) {
|
|
1420
1442
|
const latencies = [];
|
|
1421
1443
|
let succeeded = 0;
|
|
1422
1444
|
let failed = 0;
|
|
1423
1445
|
let overloaded = 0;
|
|
1446
|
+
const systemBefore = sampler();
|
|
1424
1447
|
const started = performance.now();
|
|
1425
1448
|
await Promise.all(Array.from({ length: concurrency }, async () => {
|
|
1426
|
-
|
|
1449
|
+
let request = 0;
|
|
1450
|
+
while (request < requestsPerWorker || performance.now() - started < minimumDurationMs) {
|
|
1451
|
+
request += 1;
|
|
1427
1452
|
const requestStarted = performance.now();
|
|
1428
1453
|
try {
|
|
1429
1454
|
const response = await fetcher(endpoint, {
|
|
@@ -1447,50 +1472,74 @@ async function stage(endpoint, concurrency, requestsPerWorker, timeoutMs, fetche
|
|
|
1447
1472
|
}
|
|
1448
1473
|
}
|
|
1449
1474
|
}));
|
|
1450
|
-
const
|
|
1475
|
+
const durationMs = Math.max(performance.now() - started, 1);
|
|
1476
|
+
const elapsedSeconds = durationMs / 1000;
|
|
1477
|
+
const systemAfter = sampler();
|
|
1478
|
+
const cpuTotal = Math.max(0, systemAfter.cpuTotal - systemBefore.cpuTotal);
|
|
1479
|
+
const cpuIdle = Math.max(0, systemAfter.cpuIdle - systemBefore.cpuIdle);
|
|
1480
|
+
const cpuUtilizationPercent = cpuTotal === 0 ? 0 : (cpuTotal - cpuIdle) / cpuTotal * 100;
|
|
1481
|
+
const memoryUtilizationPercent = Math.max(systemBefore.memoryTotal > 0 ? systemBefore.memoryUsed / systemBefore.memoryTotal * 100 : 0, systemAfter.memoryTotal > 0 ? systemAfter.memoryUsed / systemAfter.memoryTotal * 100 : 0);
|
|
1482
|
+
const requests = succeeded + failed;
|
|
1451
1483
|
return {
|
|
1452
1484
|
concurrency,
|
|
1453
|
-
requests
|
|
1485
|
+
requests,
|
|
1454
1486
|
succeeded,
|
|
1455
1487
|
failed,
|
|
1456
1488
|
overloaded,
|
|
1457
|
-
|
|
1458
|
-
|
|
1489
|
+
successfulRequestsPerSecond: Number((succeeded / elapsedSeconds).toFixed(2)),
|
|
1490
|
+
attemptedRequestsPerSecond: Number((requests / elapsedSeconds).toFixed(2)),
|
|
1491
|
+
durationMs: Number(durationMs.toFixed(2)),
|
|
1492
|
+
errorRate: Number((failed / Math.max(requests, 1)).toFixed(6)),
|
|
1493
|
+
p95Ms: Number(percentile(latencies, 0.95).toFixed(2)),
|
|
1494
|
+
p99Ms: Number(percentile(latencies, 0.99).toFixed(2)),
|
|
1495
|
+
cpuUtilizationPercent: Number(cpuUtilizationPercent.toFixed(2)),
|
|
1496
|
+
memoryUtilizationPercent: Number(memoryUtilizationPercent.toFixed(2))
|
|
1459
1497
|
};
|
|
1460
1498
|
}
|
|
1461
|
-
async function calibrateHttpConcurrency(options, fetcher = fetch) {
|
|
1499
|
+
async function calibrateHttpConcurrency(options, fetcher = fetch, sampler = hostSample) {
|
|
1462
1500
|
const {
|
|
1463
1501
|
endpoint,
|
|
1464
1502
|
maxConcurrency,
|
|
1465
1503
|
requestsPerWorker,
|
|
1466
1504
|
maxP95Ms,
|
|
1467
1505
|
maxErrorRate,
|
|
1468
|
-
|
|
1469
|
-
requestTimeoutMs: timeoutMs
|
|
1506
|
+
safetyRatio,
|
|
1507
|
+
requestTimeoutMs: timeoutMs,
|
|
1508
|
+
minimumStageDurationMs,
|
|
1509
|
+
maxCpuUtilizationPercent,
|
|
1510
|
+
maxMemoryUtilizationPercent
|
|
1470
1511
|
} = validateCapacityCalibrationOptions(options);
|
|
1471
1512
|
const stages = [];
|
|
1472
|
-
let lastSafe
|
|
1513
|
+
let lastSafe;
|
|
1473
1514
|
let stopReason = "maximum-tested";
|
|
1474
1515
|
for (let concurrency = 1;; concurrency = Math.min(maxConcurrency, concurrency * 2)) {
|
|
1475
|
-
const measured = await stage(endpoint, concurrency, requestsPerWorker, timeoutMs, fetcher);
|
|
1516
|
+
const measured = await stage(endpoint, concurrency, requestsPerWorker, timeoutMs, minimumStageDurationMs, fetcher, sampler);
|
|
1476
1517
|
stages.push(measured);
|
|
1477
|
-
const errorRate = measured.failed / measured.requests;
|
|
1478
1518
|
const previous = stages.at(-2);
|
|
1479
|
-
const throughputRegressed = Boolean(previous && concurrency > 1 && measured.
|
|
1480
|
-
if (measured.overloaded > 0 || errorRate > maxErrorRate)
|
|
1519
|
+
const throughputRegressed = Boolean(previous && concurrency > 1 && measured.successfulRequestsPerSecond < previous.successfulRequestsPerSecond * 0.9);
|
|
1520
|
+
if (measured.overloaded > 0 || measured.errorRate > maxErrorRate)
|
|
1481
1521
|
stopReason = "errors";
|
|
1482
1522
|
else if (measured.p95Ms > maxP95Ms)
|
|
1483
1523
|
stopReason = "latency";
|
|
1524
|
+
else if (measured.cpuUtilizationPercent > maxCpuUtilizationPercent)
|
|
1525
|
+
stopReason = "cpu";
|
|
1526
|
+
else if (measured.memoryUtilizationPercent > maxMemoryUtilizationPercent)
|
|
1527
|
+
stopReason = "memory";
|
|
1484
1528
|
else if (throughputRegressed)
|
|
1485
1529
|
stopReason = "throughput-regression";
|
|
1486
1530
|
else
|
|
1487
|
-
lastSafe =
|
|
1531
|
+
lastSafe = measured;
|
|
1488
1532
|
if (stopReason !== "maximum-tested" || concurrency === maxConcurrency)
|
|
1489
1533
|
break;
|
|
1490
1534
|
}
|
|
1535
|
+
if (!lastSafe)
|
|
1536
|
+
throw new Error("Capacity calibration found no safe request stage.");
|
|
1491
1537
|
return {
|
|
1492
1538
|
endpoint: endpoint.toString(),
|
|
1493
|
-
|
|
1539
|
+
measuredSustainableRequestsPerSecond: lastSafe.successfulRequestsPerSecond,
|
|
1540
|
+
allowedRequestsPerSecond: Math.max(1, Math.floor(lastSafe.successfulRequestsPerSecond * safetyRatio)),
|
|
1541
|
+
safetyRatio,
|
|
1542
|
+
recommendedConcurrency: Math.max(1, Math.floor(lastSafe.concurrency * safetyRatio)),
|
|
1494
1543
|
stopReason,
|
|
1495
1544
|
stages
|
|
1496
1545
|
};
|
|
@@ -1567,8 +1616,11 @@ function capacityCalibration(value, where) {
|
|
|
1567
1616
|
"requestsPerWorker",
|
|
1568
1617
|
"maxP95Ms",
|
|
1569
1618
|
"maxErrorRate",
|
|
1570
|
-
"
|
|
1571
|
-
"requestTimeoutMs"
|
|
1619
|
+
"safetyRatio",
|
|
1620
|
+
"requestTimeoutMs",
|
|
1621
|
+
"minimumStageDurationMs",
|
|
1622
|
+
"maxCpuUtilizationPercent",
|
|
1623
|
+
"maxMemoryUtilizationPercent"
|
|
1572
1624
|
], where);
|
|
1573
1625
|
const endpoint = text(calibration.endpoint, `${where}.endpoint`);
|
|
1574
1626
|
try {
|
|
@@ -1592,8 +1644,11 @@ function capacityCalibration(value, where) {
|
|
|
1592
1644
|
"requestsPerWorker",
|
|
1593
1645
|
"maxP95Ms",
|
|
1594
1646
|
"maxErrorRate",
|
|
1595
|
-
"
|
|
1596
|
-
"requestTimeoutMs"
|
|
1647
|
+
"safetyRatio",
|
|
1648
|
+
"requestTimeoutMs",
|
|
1649
|
+
"minimumStageDurationMs",
|
|
1650
|
+
"maxCpuUtilizationPercent",
|
|
1651
|
+
"maxMemoryUtilizationPercent"
|
|
1597
1652
|
].flatMap((name) => {
|
|
1598
1653
|
const found = optionalNumber(name);
|
|
1599
1654
|
return found === undefined ? [] : [[name, found]];
|
|
@@ -2784,35 +2839,76 @@ function requestServiceActivation(request, socketPath = DEFAULT_SOFTWARE_HELPER_
|
|
|
2784
2839
|
function requestSoftware(requirements, socketPath = DEFAULT_SOFTWARE_HELPER_SOCKET, timeoutMs = 15 * 60000) {
|
|
2785
2840
|
validateSoftwareRequirements(requirements);
|
|
2786
2841
|
return new Promise((resolve5, reject) => {
|
|
2787
|
-
const
|
|
2788
|
-
|
|
2789
|
-
|
|
2790
|
-
|
|
2791
|
-
socket.destroy();
|
|
2792
|
-
reject(new Error("software helper did not answer before its deadline"));
|
|
2793
|
-
});
|
|
2794
|
-
socket.on("data", (chunk) => {
|
|
2795
|
-
buffer += chunk.toString("utf8");
|
|
2796
|
-
const newline = buffer.indexOf(`
|
|
2797
|
-
`);
|
|
2798
|
-
if (newline < 0)
|
|
2842
|
+
const deadline = Date.now() + timeoutMs;
|
|
2843
|
+
let settled = false;
|
|
2844
|
+
const finish = (cause, results) => {
|
|
2845
|
+
if (settled)
|
|
2799
2846
|
return;
|
|
2800
|
-
|
|
2847
|
+
settled = true;
|
|
2848
|
+
if (cause)
|
|
2849
|
+
reject(cause);
|
|
2850
|
+
else
|
|
2851
|
+
resolve5(results);
|
|
2852
|
+
};
|
|
2853
|
+
const attempt = () => {
|
|
2854
|
+
if (settled)
|
|
2855
|
+
return;
|
|
2856
|
+
const remaining = deadline - Date.now();
|
|
2857
|
+
if (remaining <= 0)
|
|
2858
|
+
return finish(new Error("software helper did not answer before its deadline"));
|
|
2859
|
+
if (!existsSync7(socketPath)) {
|
|
2860
|
+
setTimeout(attempt, Math.min(100, Math.max(1, remaining)));
|
|
2861
|
+
return;
|
|
2862
|
+
}
|
|
2863
|
+
let buffer = "";
|
|
2864
|
+
let socket;
|
|
2801
2865
|
try {
|
|
2802
|
-
|
|
2803
|
-
|
|
2804
|
-
throw new Error(response.error?.message ?? "software helper refused the request");
|
|
2805
|
-
resolve5(response.results);
|
|
2866
|
+
socket = connect3(socketPath, () => socket.write(`${JSON.stringify({ op: "ensure", requirements })}
|
|
2867
|
+
`));
|
|
2806
2868
|
} catch (cause) {
|
|
2807
|
-
|
|
2869
|
+
const error = cause;
|
|
2870
|
+
if ((error.code === "ENOENT" || error.code === "ECONNREFUSED") && Date.now() < deadline) {
|
|
2871
|
+
setTimeout(attempt, Math.min(100, Math.max(1, deadline - Date.now())));
|
|
2872
|
+
return;
|
|
2873
|
+
}
|
|
2874
|
+
finish(error);
|
|
2875
|
+
return;
|
|
2808
2876
|
}
|
|
2809
|
-
|
|
2810
|
-
|
|
2877
|
+
socket.setTimeout(remaining, () => {
|
|
2878
|
+
socket.destroy();
|
|
2879
|
+
finish(new Error("software helper did not answer before its deadline"));
|
|
2880
|
+
});
|
|
2881
|
+
socket.on("data", (chunk) => {
|
|
2882
|
+
buffer += chunk.toString("utf8");
|
|
2883
|
+
const newline = buffer.indexOf(`
|
|
2884
|
+
`);
|
|
2885
|
+
if (newline < 0)
|
|
2886
|
+
return;
|
|
2887
|
+
socket.end();
|
|
2888
|
+
try {
|
|
2889
|
+
const response = JSON.parse(buffer.slice(0, newline));
|
|
2890
|
+
if (!response.ok || !response.results)
|
|
2891
|
+
throw new Error(response.error?.message ?? "software helper refused the request");
|
|
2892
|
+
finish(undefined, response.results);
|
|
2893
|
+
} catch (cause) {
|
|
2894
|
+
finish(cause);
|
|
2895
|
+
}
|
|
2896
|
+
});
|
|
2897
|
+
socket.on("error", (cause) => {
|
|
2898
|
+
socket.destroy();
|
|
2899
|
+
if ((cause.code === "ENOENT" || cause.code === "ECONNREFUSED") && Date.now() < deadline) {
|
|
2900
|
+
setTimeout(attempt, Math.min(100, Math.max(1, deadline - Date.now())));
|
|
2901
|
+
return;
|
|
2902
|
+
}
|
|
2903
|
+
finish(cause);
|
|
2904
|
+
});
|
|
2905
|
+
};
|
|
2906
|
+
attempt();
|
|
2811
2907
|
});
|
|
2812
2908
|
}
|
|
2813
2909
|
|
|
2814
2910
|
// src/version.ts
|
|
2815
|
-
var VERSION3 = "0.1.
|
|
2911
|
+
var VERSION3 = "0.1.90";
|
|
2816
2912
|
|
|
2817
2913
|
// src/egress-policy.ts
|
|
2818
2914
|
import { realpathSync as realpathSync3 } from "node:fs";
|
|
@@ -2948,6 +3044,7 @@ function agentEgressUnit(options) {
|
|
|
2948
3044
|
}
|
|
2949
3045
|
const deploymentEnabled = Boolean(options.repository || bootstrapBundleEnabled || options.pullDeployments);
|
|
2950
3046
|
const runnerLoopbackPorts = normalizeEgressTcpPorts(options.runnerLoopbackPorts ?? []);
|
|
3047
|
+
const agentLoopbackPorts = normalizeEgressTcpPorts(options.agentLoopbackPorts ?? []);
|
|
2951
3048
|
const runnerPublicTcpPorts = normalizeEgressTcpPorts(options.runnerPublicTcpPorts ?? DEFAULT_RUNNER_PUBLIC_TCP_PORTS);
|
|
2952
3049
|
if (deploymentEnabled && runnerPublicTcpPorts.length < 1) {
|
|
2953
3050
|
throw new Error("deployed project runner needs at least one vetted public TCP port");
|
|
@@ -2955,8 +3052,10 @@ function agentEgressUnit(options) {
|
|
|
2955
3052
|
if (deploymentEnabled)
|
|
2956
3053
|
systemdAgentEgressDirectives(runnerLoopbackPorts);
|
|
2957
3054
|
const users = [user, ...deploymentEnabled ? [DEPLOYMENT_RUNNER_USER] : []];
|
|
2958
|
-
const
|
|
2959
|
-
const
|
|
3055
|
+
const portGrantEnabled = deploymentEnabled || agentLoopbackPorts.length > 0;
|
|
3056
|
+
const restrictedUser = deploymentEnabled ? DEPLOYMENT_RUNNER_USER : user;
|
|
3057
|
+
const portGrant = portGrantEnabled ? ` --loopback-user=${restrictedUser}` + (deploymentEnabled ? runnerLoopbackPorts.map((port) => ` --loopback-tcp-port=${port}`).join("") : "") + agentLoopbackPorts.map((port) => ` --shared-loopback-tcp-port=${port}`).join("") + (deploymentEnabled ? runnerPublicTcpPorts.map((port) => ` --public-tcp-port=${port}`).join("") : "") : "";
|
|
3058
|
+
const policyProof = `ExecStartPost=${bin} egress-policy-check ${users.map((name) => `--user=${name}`).join(" ")}${portGrant}`;
|
|
2960
3059
|
return `[Unit]
|
|
2961
3060
|
Description=ForgeZero Agent host egress policy
|
|
2962
3061
|
Documentation=https://www.forgezero.net/docs/agent
|
|
@@ -2969,7 +3068,7 @@ Type=notify
|
|
|
2969
3068
|
NotifyAccess=all
|
|
2970
3069
|
User=root
|
|
2971
3070
|
Group=root
|
|
2972
|
-
ExecStart=${bin} egress-policy ${users.map((name) => `--user=${name}`).join(" ")}${
|
|
3071
|
+
ExecStart=${bin} egress-policy ${users.map((name) => `--user=${name}`).join(" ")}${portGrant}
|
|
2973
3072
|
${policyProof}
|
|
2974
3073
|
Restart=on-failure
|
|
2975
3074
|
RestartSec=2
|
|
@@ -3253,7 +3352,7 @@ function agentEnrolmentUnit(options) {
|
|
|
3253
3352
|
Requires=forgezero-agent-egress.service
|
|
3254
3353
|
BindsTo=forgezero-agent-egress.service
|
|
3255
3354
|
` : "";
|
|
3256
|
-
const egressDirectives = options.enforceEgress ? systemdAgentEgressDirectives() : "";
|
|
3355
|
+
const egressDirectives = options.enforceEgress ? systemdAgentEgressDirectives(options.agentLoopbackPorts ?? []) : "";
|
|
3257
3356
|
return `[Unit]
|
|
3258
3357
|
Description=Bind this machine to its ForgeZero compute
|
|
3259
3358
|
After=network-online.target
|
|
@@ -3290,7 +3389,7 @@ WantedBy=multi-user.target
|
|
|
3290
3389
|
function deploymentRunnerUnit(options) {
|
|
3291
3390
|
const bin = options.binPath ?? "fz-agent";
|
|
3292
3391
|
const root = options.deployRoot ?? "/opt/forgezero";
|
|
3293
|
-
const agentUser = options.user ?? "forgezero
|
|
3392
|
+
const agentUser = options.user ?? "forgezero";
|
|
3294
3393
|
const egressDependency = options.enforceEgress ? `After=forgezero-agent-egress.service
|
|
3295
3394
|
Requires=forgezero-agent-egress.service
|
|
3296
3395
|
BindsTo=forgezero-agent-egress.service
|
|
@@ -3331,7 +3430,7 @@ RestrictRealtime=true
|
|
|
3331
3430
|
MemoryDenyWriteExecute=true
|
|
3332
3431
|
LockPersonality=true
|
|
3333
3432
|
${egressDirectives}
|
|
3334
|
-
ReadWritePaths=${root}/releases ${root}/runner-home
|
|
3433
|
+
ReadWritePaths=${root}/releases ${root}/runner-home ${root}/slots /etc/nginx/conf.d /var/log/nginx
|
|
3335
3434
|
|
|
3336
3435
|
[Install]
|
|
3337
3436
|
WantedBy=multi-user.target
|
|
@@ -3341,7 +3440,7 @@ function agentUnit(options) {
|
|
|
3341
3440
|
if (!validNodeHostname(options.nodeHostname))
|
|
3342
3441
|
throw new Error("node hostname is invalid");
|
|
3343
3442
|
if (!options.telemetryEndpoint) {
|
|
3344
|
-
throw new Error("compute Agent provisioning requires
|
|
3443
|
+
throw new Error("compute Agent provisioning requires an explicit supervised OTLP collector coordinate");
|
|
3345
3444
|
}
|
|
3346
3445
|
let telemetryEndpoint;
|
|
3347
3446
|
{
|
|
@@ -3349,10 +3448,13 @@ function agentUnit(options) {
|
|
|
3349
3448
|
try {
|
|
3350
3449
|
endpoint = new URL(options.telemetryEndpoint);
|
|
3351
3450
|
} catch {
|
|
3352
|
-
throw new Error("compute telemetry endpoint must be an absolute
|
|
3451
|
+
throw new Error("compute telemetry endpoint must be an absolute collector URL");
|
|
3452
|
+
}
|
|
3453
|
+
const localCollector = endpoint.protocol === "http:" && endpoint.hostname === "127.0.0.1" && endpoint.port === "4318" && endpoint.pathname === "/";
|
|
3454
|
+
const publicCollector = endpoint.protocol === "https:" && !endpoint.username && !endpoint.password && !endpoint.search && !endpoint.hash && isIP(endpoint.hostname) === 0 && endpoint.hostname.includes(".") && endpoint.hostname !== "localhost" && !endpoint.hostname.endsWith(".local");
|
|
3455
|
+
if (!localCollector && !publicCollector || endpoint.username || endpoint.password || endpoint.search || endpoint.hash) {
|
|
3456
|
+
throw new Error("compute telemetry endpoint must be the supervised loopback collector or credential-free public HTTPS");
|
|
3353
3457
|
}
|
|
3354
|
-
if (endpoint.protocol !== "https:" || endpoint.username || endpoint.password || endpoint.search || endpoint.hash || isIP(endpoint.hostname) !== 0 || !endpoint.hostname.includes(".") || endpoint.hostname === "localhost" || endpoint.hostname.endsWith(".local"))
|
|
3355
|
-
throw new Error("compute telemetry endpoint must be an absolute public HTTPS URL without credentials, query or fragment");
|
|
3356
3458
|
telemetryEndpoint = endpoint.toString().replace(/\/$/, "");
|
|
3357
3459
|
}
|
|
3358
3460
|
const bin = options.binPath ?? "fz-agent";
|
|
@@ -3466,7 +3568,7 @@ function agentUnit(options) {
|
|
|
3466
3568
|
const supplementaryGroups = [
|
|
3467
3569
|
AGENT_UPDATE_GROUP,
|
|
3468
3570
|
deploymentEnabled ? DEPLOYMENT_GROUP : null,
|
|
3469
|
-
|
|
3571
|
+
SOFTWARE_HELPER_GROUP,
|
|
3470
3572
|
lifecycleEnabled ? LIFECYCLE_GROUP : null
|
|
3471
3573
|
].filter((value) => value !== null);
|
|
3472
3574
|
const deploymentGroup = supplementaryGroups.length > 0 ? `SupplementaryGroups=${supplementaryGroups.join(" ")}` : "";
|
|
@@ -3475,7 +3577,7 @@ function agentUnit(options) {
|
|
|
3475
3577
|
"forgezero-agent-update-helper.service",
|
|
3476
3578
|
options.enforceEgress ? "forgezero-agent-egress.service" : null,
|
|
3477
3579
|
deploymentEnabled ? "forgezero-deploy-runner.service" : null,
|
|
3478
|
-
|
|
3580
|
+
"forgezero-software-helper.service",
|
|
3479
3581
|
lifecycleEnabled ? "forgezero-lifecycle-helper.service" : null,
|
|
3480
3582
|
warpEnabled ? "warp-svc.service" : null,
|
|
3481
3583
|
enrolmentEnabled ? "forgezero-agent-enrol.service" : null
|
|
@@ -3484,7 +3586,7 @@ function agentUnit(options) {
|
|
|
3484
3586
|
"forgezero-agent-update-helper.service",
|
|
3485
3587
|
options.enforceEgress ? "forgezero-agent-egress.service" : null,
|
|
3486
3588
|
deploymentEnabled ? "forgezero-deploy-runner.service" : null,
|
|
3487
|
-
|
|
3589
|
+
"forgezero-software-helper.service",
|
|
3488
3590
|
lifecycleEnabled ? "forgezero-lifecycle-helper.service" : null,
|
|
3489
3591
|
warpEnabled ? "warp-svc.service" : null,
|
|
3490
3592
|
enrolmentEnabled ? "forgezero-agent-enrol.service" : null
|
|
@@ -3499,9 +3601,9 @@ function agentUnit(options) {
|
|
|
3499
3601
|
const snpDevice = options.mode === "attested" ? `DevicePolicy=closed
|
|
3500
3602
|
DeviceAllow=/dev/sev-guest rw` : "";
|
|
3501
3603
|
const snpPrepare = options.mode === "attested" ? `ExecStartPre=+/bin/chgrp ${VAULT_GROUP} /dev/sev-guest
|
|
3502
|
-
ExecStartPre=+/bin/chmod
|
|
3604
|
+
ExecStartPre=+/bin/chmod 0660 /dev/sev-guest
|
|
3503
3605
|
` : "";
|
|
3504
|
-
const egressDirectives = options.enforceEgress ? systemdAgentEgressDirectives() : "";
|
|
3606
|
+
const egressDirectives = options.enforceEgress ? systemdAgentEgressDirectives(options.agentLoopbackPorts ?? []) : "";
|
|
3505
3607
|
return `[Unit]
|
|
3506
3608
|
Description=ForgeZero node agent (${options.mode})
|
|
3507
3609
|
Documentation=https://www.forgezero.net/docs/agent
|
|
@@ -3632,6 +3734,7 @@ function planProvision(options) {
|
|
|
3632
3734
|
const deployRoot = options.deployRoot ?? "/opt/forgezero";
|
|
3633
3735
|
const deploymentEnabled = Boolean(options.repository || options.pullDeployments);
|
|
3634
3736
|
const runnerLoopbackPorts = normalizeEgressTcpPorts(options.runnerLoopbackPorts ?? []);
|
|
3737
|
+
const agentLoopbackPorts = normalizeEgressTcpPorts(options.agentLoopbackPorts ?? []);
|
|
3635
3738
|
const runnerPublicTcpPorts = normalizeEgressTcpPorts(options.runnerPublicTcpPorts ?? DEFAULT_RUNNER_PUBLIC_TCP_PORTS);
|
|
3636
3739
|
const lifecycleEnabled = Boolean(options.pullMigrations && options.lifecycleProfilePath);
|
|
3637
3740
|
if (Boolean(options.pullMigrations) !== Boolean(options.lifecycleProfilePath)) {
|
|
@@ -3680,8 +3783,9 @@ function planProvision(options) {
|
|
|
3680
3783
|
const enabledUnits = [
|
|
3681
3784
|
"forgezero-agent.socket",
|
|
3682
3785
|
"forgezero-agent-update-helper.service",
|
|
3786
|
+
"forgezero-software-helper.service",
|
|
3683
3787
|
...options.enforceEgress ? ["forgezero-agent-egress.service"] : [],
|
|
3684
|
-
...deploymentEnabled ? ["forgezero-deploy-runner.service"
|
|
3788
|
+
...deploymentEnabled ? ["forgezero-deploy-runner.service"] : [],
|
|
3685
3789
|
...lifecycleEnabled ? ["forgezero-lifecycle-helper.service"] : [],
|
|
3686
3790
|
...warpEnabled ? ["forgezero-warp-config.service", "warp-svc.service"] : [],
|
|
3687
3791
|
...enrolmentEnabled ? ["forgezero-agent-enrol.service"] : [],
|
|
@@ -3689,8 +3793,9 @@ function planProvision(options) {
|
|
|
3689
3793
|
];
|
|
3690
3794
|
const restartedUnits = [
|
|
3691
3795
|
"forgezero-agent-update-helper.service",
|
|
3796
|
+
"forgezero-software-helper.service",
|
|
3692
3797
|
...options.enforceEgress ? ["forgezero-agent-egress.service"] : [],
|
|
3693
|
-
...deploymentEnabled ? ["forgezero-deploy-runner.service"
|
|
3798
|
+
...deploymentEnabled ? ["forgezero-deploy-runner.service"] : [],
|
|
3694
3799
|
...lifecycleEnabled ? ["forgezero-lifecycle-helper.service"] : [],
|
|
3695
3800
|
...warpEnabled ? ["forgezero-warp-config.service", "warp-svc.service"] : [],
|
|
3696
3801
|
...enrolmentEnabled ? ["forgezero-agent-enrol.service"] : []
|
|
@@ -3714,9 +3819,9 @@ function planProvision(options) {
|
|
|
3714
3819
|
...options.enforceEgress ? [
|
|
3715
3820
|
{ path: AGENT_EGRESS_UNIT_PATH, unit: agentEgressUnit(options) }
|
|
3716
3821
|
] : [],
|
|
3822
|
+
{ path: SOFTWARE_HELPER_UNIT_PATH, unit: softwareHelperUnit(options) },
|
|
3717
3823
|
...deploymentEnabled ? [
|
|
3718
|
-
{ path: DEPLOYMENT_RUNNER_UNIT_PATH, unit: deploymentRunnerUnit(options) }
|
|
3719
|
-
{ path: SOFTWARE_HELPER_UNIT_PATH, unit: softwareHelperUnit(options) }
|
|
3824
|
+
{ path: DEPLOYMENT_RUNNER_UNIT_PATH, unit: deploymentRunnerUnit(options) }
|
|
3720
3825
|
] : [],
|
|
3721
3826
|
...enrolmentEnabled ? [
|
|
3722
3827
|
{ path: ENROLMENT_UNIT_PATH, unit: agentEnrolmentUnit(options) }
|
|
@@ -3749,24 +3854,29 @@ function planProvision(options) {
|
|
|
3749
3854
|
step("Agent update helper access group", { kind: "commands", commands: [{ argv: ["/usr/sbin/groupadd", "--system", AGENT_UPDATE_GROUP], acceptedExitCodes: [0, 9] }] }),
|
|
3750
3855
|
...sourceBinPath && binPath ? [step("root-owned agent runtime", { kind: "install-runtime", source: sourceBinPath, binary: binPath, version: VERSION3 })] : [],
|
|
3751
3856
|
...warpEnabled ? [step("Cloudflare One client for Ubuntu 26.04", { kind: "install-warp" })] : [],
|
|
3752
|
-
|
|
3753
|
-
{ argv: ["/usr/sbin/groupadd", "--system", DEPLOYMENT_GROUP], acceptedExitCodes: [0, 9] },
|
|
3857
|
+
step("software strategy helper group", { kind: "commands", commands: [
|
|
3754
3858
|
{ argv: ["/usr/sbin/groupadd", "--system", SOFTWARE_HELPER_GROUP], acceptedExitCodes: [0, 9] }
|
|
3859
|
+
] }),
|
|
3860
|
+
...deploymentEnabled ? [step("deployment isolation group", { kind: "commands", commands: [
|
|
3861
|
+
{ argv: ["/usr/sbin/groupadd", "--system", DEPLOYMENT_GROUP], acceptedExitCodes: [0, 9] }
|
|
3755
3862
|
] })] : [],
|
|
3756
3863
|
...lifecycleEnabled ? [step("lifecycle helper access group", { kind: "commands", commands: [{ argv: ["/usr/sbin/groupadd", "--system", LIFECYCLE_GROUP], acceptedExitCodes: [0, 9] }] })] : [],
|
|
3757
3864
|
step("service account", { kind: "commands", commands: [{ argv: ["/usr/sbin/useradd", "--system", "--no-create-home", "--shell", "/usr/sbin/nologin", user], acceptedExitCodes: [0, 9] }] }),
|
|
3758
3865
|
step("bind service account to vault group", { kind: "commands", commands: [{ argv: ["/usr/sbin/usermod", "-g", VAULT_GROUP, user] }] }),
|
|
3759
3866
|
step("grant verified Agent update access", { kind: "commands", commands: [{ argv: ["/usr/sbin/usermod", "-a", "-G", AGENT_UPDATE_GROUP, user] }] }),
|
|
3867
|
+
step("grant software helper socket access", { kind: "commands", commands: [
|
|
3868
|
+
{ argv: ["/usr/sbin/usermod", "-a", "-G", SOFTWARE_HELPER_GROUP, user] }
|
|
3869
|
+
] }),
|
|
3760
3870
|
...lifecycleEnabled ? [step("grant lifecycle helper socket access", { kind: "commands", commands: [{ argv: ["/usr/sbin/usermod", "-a", "-G", LIFECYCLE_GROUP, user] }] })] : [],
|
|
3761
3871
|
...deploymentEnabled ? [step("credential-free deployment account", { kind: "commands", commands: [
|
|
3762
3872
|
{ argv: ["/usr/sbin/useradd", "--system", "--no-create-home", "--shell", "/usr/sbin/nologin", "--gid", DEPLOYMENT_GROUP, DEPLOYMENT_RUNNER_USER], acceptedExitCodes: [0, 9] },
|
|
3763
3873
|
{ argv: ["/usr/sbin/useradd", "--system", "--no-create-home", "--shell", "/usr/sbin/nologin", "--gid", VAULT_GROUP, APPLICATION_RUNTIME_USER], acceptedExitCodes: [0, 9] },
|
|
3764
3874
|
{ argv: ["/usr/sbin/usermod", "-a", "-G", DEPLOYMENT_GROUP, APPLICATION_RUNTIME_USER] },
|
|
3765
|
-
{ argv: ["/usr/sbin/usermod", "-a", "-G",
|
|
3875
|
+
{ argv: ["/usr/sbin/usermod", "-a", "-G", DEPLOYMENT_GROUP, user] }
|
|
3766
3876
|
] })] : [],
|
|
3767
3877
|
step("credential and state directories", { kind: "directories", directories: [
|
|
3768
3878
|
{ path: credentialDir, mode: 448, owner: "root", group: "root" },
|
|
3769
|
-
{ path: "/var/lib/forgezero", mode: 488, owner: "root", group:
|
|
3879
|
+
{ path: "/var/lib/forgezero", mode: 488, owner: "root", group: VAULT_GROUP }
|
|
3770
3880
|
] }),
|
|
3771
3881
|
step("encrypted node identity", { kind: "ensure-seed", credential: seedCredentialPath }),
|
|
3772
3882
|
...options.generateGitIdentity && gitCredentialPath && gitPublicKeyPath ? [
|
|
@@ -3817,19 +3927,26 @@ function planProvision(options) {
|
|
|
3817
3927
|
{ argv: ["/usr/bin/systemctl", "enable", ...enabledUnits] },
|
|
3818
3928
|
{ argv: ["/usr/bin/systemctl", "reset-failed", "forgezero-agent.service"], acceptedExitCodes: [0, 1] },
|
|
3819
3929
|
...restartedUnits.length ? [{ argv: ["/usr/bin/systemctl", "restart", ...restartedUnits] }] : [],
|
|
3930
|
+
{ argv: ["/usr/bin/systemctl", "stop", "forgezero-agent-proxy.service"], acceptedExitCodes: [0, 1, 5] },
|
|
3820
3931
|
{ argv: ["/usr/bin/systemctl", "restart", "forgezero-agent.socket"] },
|
|
3821
3932
|
{ argv: ["/usr/bin/systemctl", "reset-failed", "forgezero-agent.service"], acceptedExitCodes: [0, 1] },
|
|
3822
3933
|
{ argv: ["/usr/bin/systemctl", "restart", "forgezero-agent.service"] }
|
|
3823
3934
|
] }),
|
|
3824
3935
|
...enrolmentEnabled ? [step("prove the compute binding is durable", { kind: "verify-file", path: enrolStatePath })] : [],
|
|
3825
|
-
...options.enforceEgress ? [step("prove the Agent egress policy is active", {
|
|
3936
|
+
...options.enforceEgress ? [step("prove the Agent egress policy is active", {
|
|
3937
|
+
kind: "verify-egress",
|
|
3938
|
+
deploymentEnabled,
|
|
3939
|
+
runnerPublicTcpPorts,
|
|
3940
|
+
runnerLoopbackPorts,
|
|
3941
|
+
agentLoopbackPorts
|
|
3942
|
+
})] : [],
|
|
3826
3943
|
step("prove it is running", { kind: "commands", commands: [{ argv: ["/usr/bin/systemctl", "is-active", "forgezero-agent.service"] }] }),
|
|
3827
3944
|
step("prove the public Vault socket exists", { kind: "wait-socket", path: options.socketPath, attempts: 100, intervalMs: 100 }),
|
|
3828
3945
|
step("prove the Agent Vault backend exists", { kind: "wait-socket", path: agentBackendSocketPath(options.socketPath), attempts: 100, intervalMs: 100 }),
|
|
3829
3946
|
step("prove the Agent update helper exists", { kind: "wait-socket", path: DEFAULT_AGENT_UPDATE_SOCKET, attempts: 100, intervalMs: 100 }),
|
|
3947
|
+
step("prove the software strategy helper socket exists", { kind: "wait-socket", path: DEFAULT_SOFTWARE_HELPER_SOCKET, attempts: 100, intervalMs: 100 }),
|
|
3830
3948
|
...deploymentEnabled ? [
|
|
3831
|
-
step("prove the deployment runner socket exists", { kind: "wait-socket", path: DEPLOYMENT_RUNNER_SOCKET, attempts: 100, intervalMs: 100 })
|
|
3832
|
-
step("prove the software strategy helper socket exists", { kind: "wait-socket", path: DEFAULT_SOFTWARE_HELPER_SOCKET, attempts: 100, intervalMs: 100 })
|
|
3949
|
+
step("prove the deployment runner socket exists", { kind: "wait-socket", path: DEPLOYMENT_RUNNER_SOCKET, attempts: 100, intervalMs: 100 })
|
|
3833
3950
|
] : [],
|
|
3834
3951
|
...lifecycleEnabled ? [step("prove the lifecycle helper socket exists", { kind: "wait-socket", path: lifecycleHelperSocketPath, attempts: 100, intervalMs: 100 })] : [],
|
|
3835
3952
|
...warpEnabled ? [step("prove Cloudflare WARP is connected", { kind: "verify-warp" })] : [],
|
|
@@ -4350,12 +4467,26 @@ map $limit_conn_status $forgezero_retry_after { default ''; REJECTED 1; REJECTED
|
|
|
4350
4467
|
server {
|
|
4351
4468
|
listen 127.0.0.1:${input.publicPort};
|
|
4352
4469
|
server_name _;
|
|
4470
|
+
server_tokens off;
|
|
4471
|
+
more_clear_headers Server X-Powered-By;
|
|
4353
4472
|
limit_conn forgezero_admission ${concurrencyLimit};
|
|
4354
4473
|
limit_conn_status 503;
|
|
4355
4474
|
add_header Retry-After $forgezero_retry_after always;
|
|
4356
|
-
|
|
4357
|
-
location
|
|
4358
|
-
location / {
|
|
4475
|
+
error_page 400 403 404 405 408 413 414 429 500 502 503 504 = @transport_failure;
|
|
4476
|
+
location @transport_failure { internal; return 444; }
|
|
4477
|
+
location / {
|
|
4478
|
+
proxy_pass http://forgezero;
|
|
4479
|
+
proxy_http_version 1.1;
|
|
4480
|
+
proxy_intercept_errors off;
|
|
4481
|
+
proxy_hide_header Server;
|
|
4482
|
+
proxy_hide_header X-Powered-By;
|
|
4483
|
+
proxy_set_header Host $host;
|
|
4484
|
+
proxy_set_header X-Forwarded-Proto https;
|
|
4485
|
+
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
|
4486
|
+
proxy_set_header Upgrade $http_upgrade;
|
|
4487
|
+
proxy_set_header Connection $forgezero_connection;
|
|
4488
|
+
proxy_read_timeout 3600s;
|
|
4489
|
+
}
|
|
4359
4490
|
}
|
|
4360
4491
|
`
|
|
4361
4492
|
};
|
|
@@ -4428,11 +4559,29 @@ var replaceLink = (path2, target) => {
|
|
|
4428
4559
|
symlinkSync4(target, pending);
|
|
4429
4560
|
renameSync8(pending, path2);
|
|
4430
4561
|
};
|
|
4562
|
+
var assertReleaseLinksContained = (path2) => {
|
|
4563
|
+
const root = realpathSync4(path2);
|
|
4564
|
+
const visit = (current2) => {
|
|
4565
|
+
const metadata = lstatSync(current2);
|
|
4566
|
+
if (metadata.isSymbolicLink()) {
|
|
4567
|
+
const target = realpathSync4(current2);
|
|
4568
|
+
if (target !== root && !target.startsWith(`${root}/`)) {
|
|
4569
|
+
throw new Error("release contains a symbolic link outside its immutable root");
|
|
4570
|
+
}
|
|
4571
|
+
return;
|
|
4572
|
+
}
|
|
4573
|
+
if (metadata.isDirectory())
|
|
4574
|
+
for (const name of readdirSync3(current2))
|
|
4575
|
+
visit(join5(current2, name));
|
|
4576
|
+
};
|
|
4577
|
+
visit(root);
|
|
4578
|
+
};
|
|
4431
4579
|
var secureRelease = (path2, uid, gid) => {
|
|
4580
|
+
assertReleaseLinksContained(path2);
|
|
4432
4581
|
const visit = (current2) => {
|
|
4433
4582
|
const metadata = lstatSync(current2);
|
|
4434
4583
|
if (metadata.isSymbolicLink())
|
|
4435
|
-
|
|
4584
|
+
return;
|
|
4436
4585
|
chownSync(current2, uid, gid);
|
|
4437
4586
|
chmodSync6(current2, metadata.isDirectory() ? 365 : 292);
|
|
4438
4587
|
if (metadata.isDirectory())
|
|
@@ -4458,7 +4607,7 @@ async function activatePlatformRelease(config, requestedRelease, options = {}) {
|
|
|
4458
4607
|
const sleep = options.sleep ?? Bun.sleep;
|
|
4459
4608
|
const slots = join5(normalized.root, "slots");
|
|
4460
4609
|
mkdirSync8(slots, { recursive: true, mode: 493 });
|
|
4461
|
-
const slotFile = join5(
|
|
4610
|
+
const slotFile = join5(slots, ".active");
|
|
4462
4611
|
const previousSlot = existsSync8(slotFile) ? readFileSync6(slotFile, "utf8").trim() : undefined;
|
|
4463
4612
|
const target = previousSlot === "blue" ? "green" : "blue";
|
|
4464
4613
|
const port = target === "blue" ? normalized.bluePort : normalized.greenPort;
|
|
@@ -4508,6 +4657,7 @@ async function activatePlatformRelease(config, requestedRelease, options = {}) {
|
|
|
4508
4657
|
const test = await exec(["/usr/sbin/nginx", "-t"]);
|
|
4509
4658
|
const reload = test.exitCode === 0 ? await exec(["/usr/sbin/nginx", "-s", "reload"]) : test;
|
|
4510
4659
|
if (reload.exitCode !== 0) {
|
|
4660
|
+
const refusal = (reload.output || test.output).trim().slice(0, 2000);
|
|
4511
4661
|
if (existsSync8(backup))
|
|
4512
4662
|
renameSync8(backup, upstream);
|
|
4513
4663
|
else
|
|
@@ -4515,7 +4665,7 @@ async function activatePlatformRelease(config, requestedRelease, options = {}) {
|
|
|
4515
4665
|
await exec(["/usr/sbin/nginx", "-t"]);
|
|
4516
4666
|
await exec(["/usr/sbin/nginx", "-s", "reload"]);
|
|
4517
4667
|
await stopTarget();
|
|
4518
|
-
throw new Error(
|
|
4668
|
+
throw new Error(`nginx refused the promoted upstream${refusal ? `: ${refusal}` : ""}`);
|
|
4519
4669
|
}
|
|
4520
4670
|
rmSync7(backup, { force: true });
|
|
4521
4671
|
writeFileSync7(slotFile, `${target}
|
|
@@ -4601,6 +4751,25 @@ import {
|
|
|
4601
4751
|
writeFileSync as writeFileSync8
|
|
4602
4752
|
} from "node:fs";
|
|
4603
4753
|
import { dirname as dirname9 } from "node:path";
|
|
4754
|
+
function normalizeEd25519PublicKey(output) {
|
|
4755
|
+
const line = output.trim();
|
|
4756
|
+
if (!/^ssh-ed25519 [A-Za-z0-9+/]+={0,3}(?: [^\r\n]{1,256})?$/.test(line))
|
|
4757
|
+
return;
|
|
4758
|
+
const [algorithm, encoded] = line.split(" ", 3);
|
|
4759
|
+
if (algorithm !== "ssh-ed25519" || !encoded)
|
|
4760
|
+
return;
|
|
4761
|
+
let blob;
|
|
4762
|
+
try {
|
|
4763
|
+
blob = Buffer.from(encoded, "base64");
|
|
4764
|
+
} catch {
|
|
4765
|
+
return;
|
|
4766
|
+
}
|
|
4767
|
+
if (blob.length !== 51 || blob.readUInt32BE(0) !== 11 || blob.subarray(4, 15).toString("ascii") !== "ssh-ed25519" || blob.readUInt32BE(15) !== 32)
|
|
4768
|
+
return;
|
|
4769
|
+
if (blob.toString("base64") !== encoded)
|
|
4770
|
+
return;
|
|
4771
|
+
return `${algorithm} ${encoded}`;
|
|
4772
|
+
}
|
|
4604
4773
|
async function readCapabilities(run2) {
|
|
4605
4774
|
const answers = {};
|
|
4606
4775
|
const checks = Object.entries(CAPABILITY_CHECKS);
|
|
@@ -4740,7 +4909,7 @@ var runProvisionOperation = async (operation) => {
|
|
|
4740
4909
|
return result;
|
|
4741
4910
|
}
|
|
4742
4911
|
result = await fixed(["/usr/bin/ssh-keygen", "-y", "-f", key]);
|
|
4743
|
-
if (result.exitCode !== 0 ||
|
|
4912
|
+
if (result.exitCode !== 0 || !normalizeEd25519PublicKey(result.stdout)) {
|
|
4744
4913
|
return { stdout: "bootstrap SSH private key is not a valid Ed25519 OpenSSH key", exitCode: 1 };
|
|
4745
4914
|
}
|
|
4746
4915
|
result = await fixed(["/usr/bin/systemd-creds", "encrypt", "--name=bootstrap-ssh-key", key, operation.credential]);
|
|
@@ -4756,11 +4925,12 @@ var runProvisionOperation = async (operation) => {
|
|
|
4756
4925
|
chmodSync7(key, 384);
|
|
4757
4926
|
}
|
|
4758
4927
|
const derived = await fixed(["/usr/bin/ssh-keygen", "-y", "-f", key]);
|
|
4759
|
-
|
|
4928
|
+
const publicKey2 = normalizeEd25519PublicKey(derived.stdout);
|
|
4929
|
+
if (derived.exitCode !== 0 || !publicKey2) {
|
|
4760
4930
|
return { stdout: "bootstrap SSH public key derivation failed", exitCode: 1 };
|
|
4761
4931
|
}
|
|
4762
4932
|
mkdirSync9(dirname9(operation.publicKey), { recursive: true, mode: 493 });
|
|
4763
|
-
writeFileSync8(operation.publicKey, `${
|
|
4933
|
+
writeFileSync8(operation.publicKey, `${publicKey2} forgezero-bootstrap-runner
|
|
4764
4934
|
`, { mode: 292 });
|
|
4765
4935
|
chmodSync7(operation.publicKey, 292);
|
|
4766
4936
|
}
|
|
@@ -4803,8 +4973,9 @@ var runProvisionOperation = async (operation) => {
|
|
|
4803
4973
|
const policy = await fixed(["/usr/sbin/nft", "--numeric", "list", "table", "inet", "forgezero_agent_egress"]);
|
|
4804
4974
|
const required = [
|
|
4805
4975
|
"forgezero-agent-egress-v1",
|
|
4806
|
-
`
|
|
4807
|
-
...operation.
|
|
4976
|
+
...operation.agentLoopbackPorts.length ? [`shared-loopback=${operation.agentLoopbackPorts.join(",")}`] : [],
|
|
4977
|
+
...operation.deploymentEnabled && operation.runnerPublicTcpPorts.length ? [`public-tcp=${operation.runnerPublicTcpPorts.join(",")}`] : [],
|
|
4978
|
+
...operation.deploymentEnabled && operation.runnerLoopbackPorts.length ? [`:${operation.runnerLoopbackPorts.join(",")}`] : []
|
|
4808
4979
|
];
|
|
4809
4980
|
return policy.exitCode === 0 && required.every((part) => policy.stdout.includes(part)) ? { stdout: policy.stdout, exitCode: 0 } : { stdout: policy.stdout, exitCode: 1 };
|
|
4810
4981
|
}
|
|
@@ -4870,7 +5041,9 @@ async function applyPlan(plan, run2) {
|
|
|
4870
5041
|
const result = await run2(step2.operation);
|
|
4871
5042
|
transcript.push({ label: step2.label, command: step2.command, exitCode: result.exitCode });
|
|
4872
5043
|
if (result.exitCode !== 0 && !step2.optional) {
|
|
4873
|
-
|
|
5044
|
+
const safeDetail = step2.operation.kind === "ensure-bootstrap-ssh-identity" ? result.stdout.trim().slice(0, 1000) : "";
|
|
5045
|
+
throw new Error(`${step2.label} failed (exit ${result.exitCode}): ${step2.command}${safeDetail ? `
|
|
5046
|
+
${safeDetail}` : ""}`);
|
|
4874
5047
|
}
|
|
4875
5048
|
}
|
|
4876
5049
|
return transcript;
|
|
@@ -4960,7 +5133,6 @@ var SEED_CREDENTIAL = `${CREDS}/seed-sync-root.cred`;
|
|
|
4960
5133
|
var BACKUP_RECOVERY_CREDENTIAL = `${CREDS}/backup-recovery-root.cred`;
|
|
4961
5134
|
var BOOTSTRAP_SSH_CREDENTIAL = `${CREDS}/bootstrap-ssh-key.cred`;
|
|
4962
5135
|
var BOOTSTRAP_SSH_PUBLIC_KEY = "/etc/forgezero/bootstrap/runner.pub";
|
|
4963
|
-
var BOOTSTRAP_RELEASE_EVIDENCE = "/var/lib/forgezero/bootstrap-release.json";
|
|
4964
5136
|
var LIFECYCLE_PROFILE = "/etc/forgezero/lifecycle.json";
|
|
4965
5137
|
var CONTROL_SOCKET = "/run/forgezero/control.sock";
|
|
4966
5138
|
var CLOUDFLARED_METRICS_ADDRESS = "127.0.0.1:20241";
|
|
@@ -5051,7 +5223,7 @@ async function bootstrapStatus(host = localBootstrapHost()) {
|
|
|
5051
5223
|
if (result.exitCode !== 0)
|
|
5052
5224
|
problems.push(`${unit2} is not active`);
|
|
5053
5225
|
}
|
|
5054
|
-
for (const socket of [DEFAULT_SOCKET2
|
|
5226
|
+
for (const socket of [DEFAULT_SOCKET2]) {
|
|
5055
5227
|
services[socket] = host.exists(socket);
|
|
5056
5228
|
if (!services[socket])
|
|
5057
5229
|
problems.push(`${socket} is missing`);
|
|
@@ -5140,8 +5312,8 @@ async function bootstrapStatus(host = localBootstrapHost()) {
|
|
|
5140
5312
|
function planBootstrapAgentInstall(config, phase, context) {
|
|
5141
5313
|
const { capabilities, hasBinding, hasEnrolCredential, initialBundle } = context;
|
|
5142
5314
|
if (phase === "bootstrap") {
|
|
5143
|
-
if (config.kind !== "platform" ||
|
|
5144
|
-
throw new Error("
|
|
5315
|
+
if (config.kind !== "platform" || !initialBundle) {
|
|
5316
|
+
throw new Error("platform bundle Agent install requires a verified bootstrap bundle");
|
|
5145
5317
|
}
|
|
5146
5318
|
} else if (phase === "enrol") {
|
|
5147
5319
|
if (hasBinding || !hasEnrolCredential) {
|
|
@@ -5179,8 +5351,10 @@ function planBootstrapAgentInstall(config, phase, context) {
|
|
|
5179
5351
|
bootstrapTargetTelemetryEndpoint: bound && bootstrapRunner ? platformBootstrapRunner(config) ? config.telemetryEndpoint : config.kind === "enrolled-compute" ? config.bootstrapRunner?.targetTelemetryEndpoint : undefined : undefined,
|
|
5180
5352
|
lifecycleProfilePath: bound && config.kind === "platform" ? LIFECYCLE_PROFILE : undefined,
|
|
5181
5353
|
enforceEgress: true,
|
|
5354
|
+
runnerLoopbackPorts: config.kind === "platform" ? [config.runtime.environment.publicApiPort, config.runtime.bluePort, config.runtime.greenPort] : [],
|
|
5355
|
+
agentLoopbackPorts: config.kind === "platform" ? [config.runtime.environment.publicApiPort] : [],
|
|
5182
5356
|
nodeHostname: config.nodeHostname,
|
|
5183
|
-
telemetryEndpoint: config.
|
|
5357
|
+
telemetryEndpoint: config.kind === "platform" ? config.runtime.environment.otlpEndpoint : "http://127.0.0.1:4318",
|
|
5184
5358
|
binPath: "/usr/local/lib/forgezero/agent/fz-agent",
|
|
5185
5359
|
sourceBinPath: PACKAGED_AGENT_BIN,
|
|
5186
5360
|
...enrolling ? {
|
|
@@ -5188,7 +5362,7 @@ function planBootstrapAgentInstall(config, phase, context) {
|
|
|
5188
5362
|
enrolStatePath: "/var/lib/forgezero/enrolment.json"
|
|
5189
5363
|
} : bound ? { enrolStatePath: "/var/lib/forgezero/enrolment.json" } : {},
|
|
5190
5364
|
...authenticated ? {
|
|
5191
|
-
apiUrl: config.apiUrl,
|
|
5365
|
+
apiUrl: config.kind === "platform" ? `http://127.0.0.1:${config.runtime.environment.publicApiPort}` : config.apiUrl,
|
|
5192
5366
|
project: config.kind === "enrolled-compute" ? config.realm : "platform",
|
|
5193
5367
|
environment: config.kind === "enrolled-compute" ? undefined : config.environment,
|
|
5194
5368
|
nodeLabel: config.kind === "enrolled-compute" ? config.nodeHostname : config.computeReference
|
|
@@ -5197,6 +5371,7 @@ function planBootstrapAgentInstall(config, phase, context) {
|
|
|
5197
5371
|
return planInstall(options);
|
|
5198
5372
|
}
|
|
5199
5373
|
function localBootstrapHost() {
|
|
5374
|
+
let softwareClientUser;
|
|
5200
5375
|
const execute = async (argv2, options = {}) => {
|
|
5201
5376
|
const child = Bun.spawn([...argv2], { stdin: options.stdin === undefined ? "ignore" : "pipe", stdout: "pipe", stderr: "pipe" });
|
|
5202
5377
|
if (options.stdin !== undefined && child.stdin && typeof child.stdin !== "number") {
|
|
@@ -5229,10 +5404,12 @@ function localBootstrapHost() {
|
|
|
5229
5404
|
exec: execute,
|
|
5230
5405
|
sleep: (milliseconds) => Bun.sleep(milliseconds),
|
|
5231
5406
|
async ensureSoftware(requirements) {
|
|
5407
|
+
if (!softwareClientUser)
|
|
5408
|
+
throw new Error("Agent must be converged before software requirements are applied");
|
|
5232
5409
|
const result = await execute([
|
|
5233
5410
|
"runuser",
|
|
5234
5411
|
"-u",
|
|
5235
|
-
|
|
5412
|
+
softwareClientUser,
|
|
5236
5413
|
"--",
|
|
5237
5414
|
"/usr/local/bin/fz-agent",
|
|
5238
5415
|
"software-ensure",
|
|
@@ -5246,7 +5423,7 @@ function localBootstrapHost() {
|
|
|
5246
5423
|
const capabilities = await readCapabilities(localRunner);
|
|
5247
5424
|
const hasBinding = existsSync11("/var/lib/forgezero/enrolment.json");
|
|
5248
5425
|
const hasEnrolCredential = existsSync11(ENROL_CREDENTIAL);
|
|
5249
|
-
const initialBundle = config.kind === "platform" &&
|
|
5426
|
+
const initialBundle = config.kind === "platform" && phase === "bootstrap" && existsSync11(config.bootstrapBundle.bundleFile) && existsSync11(config.bootstrapBundle.manifestFile) ? {
|
|
5250
5427
|
path: config.bootstrapBundle.bundleFile,
|
|
5251
5428
|
manifestPath: config.bootstrapBundle.manifestFile,
|
|
5252
5429
|
manifest: parseBootstrapBundleManifest(JSON.parse(readFileSync9(config.bootstrapBundle.manifestFile, "utf8")))
|
|
@@ -5277,6 +5454,7 @@ function localBootstrapHost() {
|
|
|
5277
5454
|
writeFileSync10(unit2.path, unit2.unit, { mode: 420 });
|
|
5278
5455
|
}
|
|
5279
5456
|
await applyPlan(plan, localRunner);
|
|
5457
|
+
softwareClientUser = plan.user;
|
|
5280
5458
|
return plan;
|
|
5281
5459
|
}
|
|
5282
5460
|
};
|