@forgezero/agent 0.1.88 → 0.1.89
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -9
- package/dist/agent-heartbeat.js +1 -1
- package/dist/bootstrap.d.ts +8 -1
- package/dist/bootstrap.js +228 -54
- package/dist/capacity-calibration.d.ts +35 -5
- package/dist/capacity-calibration.js +66 -22
- package/dist/cli/agent-install.d.ts +3 -0
- package/dist/community-rehearsal-host.js +7 -2
- package/dist/control.d.ts +3 -1
- package/dist/definition.js +83 -28
- package/dist/deploy-file.js +83 -28
- package/dist/deployment.d.ts +5 -0
- package/dist/egress-policy.d.ts +4 -0
- package/dist/fz-agent.js +416 -150
- package/dist/fz.js +413 -129
- package/dist/host-maintenance.js +1 -1
- package/dist/index.d.ts +7 -0
- package/dist/metal-bootstrap.js +1 -1
- package/dist/metal-helper-socket.js +7 -2
- package/dist/metal-provision.js +7 -2
- package/dist/operator-bootstrap.d.ts +12 -0
- package/dist/operator-bootstrap.js +372 -64
- package/dist/platform-bootstrap-runtime.d.ts +1 -0
- package/dist/platform-bootstrap-runtime.js +47 -8
- package/dist/platform-fleet-verification.js +262 -87
- package/dist/platform-genesis.js +7 -2
- package/dist/provision.d.ts +5 -1
- package/dist/provision.js +185 -71
- package/dist/recovery-host.js +1 -1
- package/dist/software-helper.js +144 -48
- package/dist/software.js +7 -2
- package/dist/ubuntu.js +7 -2
- package/dist/version.d.ts +1 -1
- package/package.json +1 -1
- package/schema/deploy-v3.json +5 -2
package/dist/platform-genesis.js
CHANGED
|
@@ -355,7 +355,12 @@ async function executeSoftwareOperation(operation) {
|
|
|
355
355
|
}
|
|
356
356
|
if (software === "nginx") {
|
|
357
357
|
const binary = await run(["/usr/sbin/nginx", "-v"]);
|
|
358
|
-
|
|
358
|
+
if (binary.exitCode !== 0)
|
|
359
|
+
return binary;
|
|
360
|
+
if (!existsSync("/usr/lib/nginx/modules/ngx_http_headers_more_filter_module.so")) {
|
|
361
|
+
return { exitCode: 1, output: "Nginx response-header suppression module is unavailable" };
|
|
362
|
+
}
|
|
363
|
+
return run(["/usr/bin/systemctl", "is-active", "--quiet", "nginx.service"]);
|
|
359
364
|
}
|
|
360
365
|
if (software === "arangodb") {
|
|
361
366
|
const binary = await run(["/usr/sbin/arangod", "--version"]);
|
|
@@ -387,7 +392,7 @@ async function executeSoftwareOperation(operation) {
|
|
|
387
392
|
}
|
|
388
393
|
if (software === "docker" || software === "nginx" || software === "ufw" || software === "openssh-client" || software === "git") {
|
|
389
394
|
const packageName = software === "openssh-client" ? "openssh-client" : software === "docker" ? "docker.io" : software;
|
|
390
|
-
const installed = await aptInstall(packageName);
|
|
395
|
+
const installed = software === "nginx" ? await aptInstallMany(["nginx", "libnginx-mod-http-headers-more-filter"]) : await aptInstall(packageName);
|
|
391
396
|
if (installed.exitCode !== 0 || software !== "nginx" && software !== "docker")
|
|
392
397
|
return installed;
|
|
393
398
|
if (software === "docker") {
|
package/dist/provision.d.ts
CHANGED
|
@@ -112,6 +112,8 @@ export interface UnitOptions {
|
|
|
112
112
|
enforceEgress?: boolean;
|
|
113
113
|
/** Provisioning-owned runner health endpoints; repository commands cannot change them. */
|
|
114
114
|
runnerLoopbackPorts?: readonly number[];
|
|
115
|
+
/** Local platform API ports needed by the credential-bearing Agent. */
|
|
116
|
+
agentLoopbackPorts?: readonly number[];
|
|
115
117
|
/** Vetted public TCP surface for project commands. Defaults to HTTPS only. */
|
|
116
118
|
runnerPublicTcpPorts?: readonly number[];
|
|
117
119
|
/** Enable PQ-authenticated lifecycle claims through a constrained root helper. */
|
|
@@ -172,7 +174,7 @@ export declare const WARP_SERVICE_DROP_IN_PATH = "/etc/systemd/system/warp-svc.s
|
|
|
172
174
|
export declare const AGENT_EGRESS_UNIT_PATH = "/etc/systemd/system/forgezero-agent-egress.service";
|
|
173
175
|
export declare const DEFAULT_RUNNER_PUBLIC_TCP_PORTS: readonly [443];
|
|
174
176
|
/** Root-owned policy monitor. The credential-bearing Agent is bound to it. */
|
|
175
|
-
export declare function agentEgressUnit(options: Pick<UnitOptions, 'binPath' | 'user' | 'repository' | 'pullDeployments' | 'bootstrapBundlePath' | 'bootstrapBundleManifestPath' | 'runnerLoopbackPorts' | 'runnerPublicTcpPorts'>): string;
|
|
177
|
+
export declare function agentEgressUnit(options: Pick<UnitOptions, 'binPath' | 'user' | 'repository' | 'pullDeployments' | 'bootstrapBundlePath' | 'bootstrapBundleManifestPath' | 'runnerLoopbackPorts' | 'runnerPublicTcpPorts' | 'agentLoopbackPorts'>): string;
|
|
176
178
|
/** Root may execute only package-owned OS strategies selected by id + version. */
|
|
177
179
|
export declare function softwareHelperUnit(options: Pick<UnitOptions, 'binPath' | 'deployRoot'>): string;
|
|
178
180
|
/** Fixed root boundary for verified, health-gated Agent replacement. */
|
|
@@ -281,8 +283,10 @@ export type ProvisionOperation = {
|
|
|
281
283
|
path: string;
|
|
282
284
|
} | {
|
|
283
285
|
kind: 'verify-egress';
|
|
286
|
+
deploymentEnabled: boolean;
|
|
284
287
|
runnerPublicTcpPorts: readonly number[];
|
|
285
288
|
runnerLoopbackPorts: readonly number[];
|
|
289
|
+
agentLoopbackPorts: readonly number[];
|
|
286
290
|
} | {
|
|
287
291
|
kind: 'verify-resolved-stub';
|
|
288
292
|
} | {
|
package/dist/provision.js
CHANGED
|
@@ -1198,7 +1198,12 @@ async function executeSoftwareOperation(operation) {
|
|
|
1198
1198
|
}
|
|
1199
1199
|
if (software === "nginx") {
|
|
1200
1200
|
const binary = await run(["/usr/sbin/nginx", "-v"]);
|
|
1201
|
-
|
|
1201
|
+
if (binary.exitCode !== 0)
|
|
1202
|
+
return binary;
|
|
1203
|
+
if (!existsSync4("/usr/lib/nginx/modules/ngx_http_headers_more_filter_module.so")) {
|
|
1204
|
+
return { exitCode: 1, output: "Nginx response-header suppression module is unavailable" };
|
|
1205
|
+
}
|
|
1206
|
+
return run(["/usr/bin/systemctl", "is-active", "--quiet", "nginx.service"]);
|
|
1202
1207
|
}
|
|
1203
1208
|
if (software === "arangodb") {
|
|
1204
1209
|
const binary = await run(["/usr/sbin/arangod", "--version"]);
|
|
@@ -1230,7 +1235,7 @@ async function executeSoftwareOperation(operation) {
|
|
|
1230
1235
|
}
|
|
1231
1236
|
if (software === "docker" || software === "nginx" || software === "ufw" || software === "openssh-client" || software === "git") {
|
|
1232
1237
|
const packageName = software === "openssh-client" ? "openssh-client" : software === "docker" ? "docker.io" : software;
|
|
1233
|
-
const installed = await aptInstall(packageName);
|
|
1238
|
+
const installed = software === "nginx" ? await aptInstallMany(["nginx", "libnginx-mod-http-headers-more-filter"]) : await aptInstall(packageName);
|
|
1234
1239
|
if (installed.exitCode !== 0 || software !== "nginx" && software !== "docker")
|
|
1235
1240
|
return installed;
|
|
1236
1241
|
if (software === "docker") {
|
|
@@ -1382,11 +1387,22 @@ async function ensureSoftwareRequirements(requirementsInput, options) {
|
|
|
1382
1387
|
}
|
|
1383
1388
|
|
|
1384
1389
|
// src/capacity-calibration.ts
|
|
1385
|
-
|
|
1390
|
+
import { cpus, freemem, totalmem } from "node:os";
|
|
1391
|
+
var percentile = (values, quantile) => {
|
|
1386
1392
|
if (values.length === 0)
|
|
1387
1393
|
return Number.POSITIVE_INFINITY;
|
|
1388
1394
|
const sorted = values.toSorted((left, right) => left - right);
|
|
1389
|
-
return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length *
|
|
1395
|
+
return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * quantile) - 1)];
|
|
1396
|
+
};
|
|
1397
|
+
var hostSample = () => {
|
|
1398
|
+
let cpuIdle = 0;
|
|
1399
|
+
let cpuTotal = 0;
|
|
1400
|
+
for (const cpu of cpus()) {
|
|
1401
|
+
cpuIdle += cpu.times.idle;
|
|
1402
|
+
cpuTotal += Object.values(cpu.times).reduce((sum, value) => sum + value, 0);
|
|
1403
|
+
}
|
|
1404
|
+
const memoryTotal = totalmem();
|
|
1405
|
+
return { cpuIdle, cpuTotal, memoryUsed: memoryTotal - freemem(), memoryTotal };
|
|
1390
1406
|
};
|
|
1391
1407
|
function localCalibrationEndpoint(value) {
|
|
1392
1408
|
const endpoint = new URL(value);
|
|
@@ -1401,9 +1417,12 @@ function validateCapacityCalibrationOptions(options) {
|
|
|
1401
1417
|
const requestsPerWorker = options.requestsPerWorker ?? 8;
|
|
1402
1418
|
const maxP95Ms = options.maxP95Ms ?? 250;
|
|
1403
1419
|
const maxErrorRate = options.maxErrorRate ?? 0.01;
|
|
1404
|
-
const
|
|
1420
|
+
const safetyRatio = options.safetyRatio ?? 0.8;
|
|
1405
1421
|
const requestTimeoutMs = options.requestTimeoutMs ?? 5000;
|
|
1406
|
-
|
|
1422
|
+
const minimumStageDurationMs = options.minimumStageDurationMs ?? 5000;
|
|
1423
|
+
const maxCpuUtilizationPercent = options.maxCpuUtilizationPercent ?? 90;
|
|
1424
|
+
const maxMemoryUtilizationPercent = options.maxMemoryUtilizationPercent ?? 90;
|
|
1425
|
+
if (!Number.isSafeInteger(maxConcurrency) || maxConcurrency < 1 || maxConcurrency > 4096 || !Number.isSafeInteger(requestsPerWorker) || requestsPerWorker < 2 || requestsPerWorker > 100 || !Number.isFinite(maxP95Ms) || maxP95Ms < 1 || !Number.isFinite(maxErrorRate) || maxErrorRate < 0 || maxErrorRate > 0.2 || !Number.isFinite(safetyRatio) || safetyRatio < 0.25 || safetyRatio > 0.95 || !Number.isSafeInteger(requestTimeoutMs) || requestTimeoutMs < 100 || requestTimeoutMs > 30000 || !Number.isSafeInteger(minimumStageDurationMs) || minimumStageDurationMs < 100 || minimumStageDurationMs > 60000 || !Number.isFinite(maxCpuUtilizationPercent) || maxCpuUtilizationPercent < 25 || maxCpuUtilizationPercent > 100 || !Number.isFinite(maxMemoryUtilizationPercent) || maxMemoryUtilizationPercent < 25 || maxMemoryUtilizationPercent > 100) {
|
|
1407
1426
|
throw new Error("Capacity calibration bounds are invalid.");
|
|
1408
1427
|
}
|
|
1409
1428
|
return {
|
|
@@ -1412,18 +1431,24 @@ function validateCapacityCalibrationOptions(options) {
|
|
|
1412
1431
|
requestsPerWorker,
|
|
1413
1432
|
maxP95Ms,
|
|
1414
1433
|
maxErrorRate,
|
|
1415
|
-
|
|
1416
|
-
requestTimeoutMs
|
|
1434
|
+
safetyRatio,
|
|
1435
|
+
requestTimeoutMs,
|
|
1436
|
+
minimumStageDurationMs,
|
|
1437
|
+
maxCpuUtilizationPercent,
|
|
1438
|
+
maxMemoryUtilizationPercent
|
|
1417
1439
|
};
|
|
1418
1440
|
}
|
|
1419
|
-
async function stage(endpoint, concurrency, requestsPerWorker, timeoutMs, fetcher) {
|
|
1441
|
+
async function stage(endpoint, concurrency, requestsPerWorker, timeoutMs, minimumDurationMs, fetcher, sampler) {
|
|
1420
1442
|
const latencies = [];
|
|
1421
1443
|
let succeeded = 0;
|
|
1422
1444
|
let failed = 0;
|
|
1423
1445
|
let overloaded = 0;
|
|
1446
|
+
const systemBefore = sampler();
|
|
1424
1447
|
const started = performance.now();
|
|
1425
1448
|
await Promise.all(Array.from({ length: concurrency }, async () => {
|
|
1426
|
-
|
|
1449
|
+
let request = 0;
|
|
1450
|
+
while (request < requestsPerWorker || performance.now() - started < minimumDurationMs) {
|
|
1451
|
+
request += 1;
|
|
1427
1452
|
const requestStarted = performance.now();
|
|
1428
1453
|
try {
|
|
1429
1454
|
const response = await fetcher(endpoint, {
|
|
@@ -1447,50 +1472,74 @@ async function stage(endpoint, concurrency, requestsPerWorker, timeoutMs, fetche
|
|
|
1447
1472
|
}
|
|
1448
1473
|
}
|
|
1449
1474
|
}));
|
|
1450
|
-
const
|
|
1475
|
+
const durationMs = Math.max(performance.now() - started, 1);
|
|
1476
|
+
const elapsedSeconds = durationMs / 1000;
|
|
1477
|
+
const systemAfter = sampler();
|
|
1478
|
+
const cpuTotal = Math.max(0, systemAfter.cpuTotal - systemBefore.cpuTotal);
|
|
1479
|
+
const cpuIdle = Math.max(0, systemAfter.cpuIdle - systemBefore.cpuIdle);
|
|
1480
|
+
const cpuUtilizationPercent = cpuTotal === 0 ? 0 : (cpuTotal - cpuIdle) / cpuTotal * 100;
|
|
1481
|
+
const memoryUtilizationPercent = Math.max(systemBefore.memoryTotal > 0 ? systemBefore.memoryUsed / systemBefore.memoryTotal * 100 : 0, systemAfter.memoryTotal > 0 ? systemAfter.memoryUsed / systemAfter.memoryTotal * 100 : 0);
|
|
1482
|
+
const requests = succeeded + failed;
|
|
1451
1483
|
return {
|
|
1452
1484
|
concurrency,
|
|
1453
|
-
requests
|
|
1485
|
+
requests,
|
|
1454
1486
|
succeeded,
|
|
1455
1487
|
failed,
|
|
1456
1488
|
overloaded,
|
|
1457
|
-
|
|
1458
|
-
|
|
1489
|
+
successfulRequestsPerSecond: Number((succeeded / elapsedSeconds).toFixed(2)),
|
|
1490
|
+
attemptedRequestsPerSecond: Number((requests / elapsedSeconds).toFixed(2)),
|
|
1491
|
+
durationMs: Number(durationMs.toFixed(2)),
|
|
1492
|
+
errorRate: Number((failed / Math.max(requests, 1)).toFixed(6)),
|
|
1493
|
+
p95Ms: Number(percentile(latencies, 0.95).toFixed(2)),
|
|
1494
|
+
p99Ms: Number(percentile(latencies, 0.99).toFixed(2)),
|
|
1495
|
+
cpuUtilizationPercent: Number(cpuUtilizationPercent.toFixed(2)),
|
|
1496
|
+
memoryUtilizationPercent: Number(memoryUtilizationPercent.toFixed(2))
|
|
1459
1497
|
};
|
|
1460
1498
|
}
|
|
1461
|
-
async function calibrateHttpConcurrency(options, fetcher = fetch) {
|
|
1499
|
+
async function calibrateHttpConcurrency(options, fetcher = fetch, sampler = hostSample) {
|
|
1462
1500
|
const {
|
|
1463
1501
|
endpoint,
|
|
1464
1502
|
maxConcurrency,
|
|
1465
1503
|
requestsPerWorker,
|
|
1466
1504
|
maxP95Ms,
|
|
1467
1505
|
maxErrorRate,
|
|
1468
|
-
|
|
1469
|
-
requestTimeoutMs: timeoutMs
|
|
1506
|
+
safetyRatio,
|
|
1507
|
+
requestTimeoutMs: timeoutMs,
|
|
1508
|
+
minimumStageDurationMs,
|
|
1509
|
+
maxCpuUtilizationPercent,
|
|
1510
|
+
maxMemoryUtilizationPercent
|
|
1470
1511
|
} = validateCapacityCalibrationOptions(options);
|
|
1471
1512
|
const stages = [];
|
|
1472
|
-
let lastSafe
|
|
1513
|
+
let lastSafe;
|
|
1473
1514
|
let stopReason = "maximum-tested";
|
|
1474
1515
|
for (let concurrency = 1;; concurrency = Math.min(maxConcurrency, concurrency * 2)) {
|
|
1475
|
-
const measured = await stage(endpoint, concurrency, requestsPerWorker, timeoutMs, fetcher);
|
|
1516
|
+
const measured = await stage(endpoint, concurrency, requestsPerWorker, timeoutMs, minimumStageDurationMs, fetcher, sampler);
|
|
1476
1517
|
stages.push(measured);
|
|
1477
|
-
const errorRate = measured.failed / measured.requests;
|
|
1478
1518
|
const previous = stages.at(-2);
|
|
1479
|
-
const throughputRegressed = Boolean(previous && concurrency > 1 && measured.
|
|
1480
|
-
if (measured.overloaded > 0 || errorRate > maxErrorRate)
|
|
1519
|
+
const throughputRegressed = Boolean(previous && concurrency > 1 && measured.successfulRequestsPerSecond < previous.successfulRequestsPerSecond * 0.9);
|
|
1520
|
+
if (measured.overloaded > 0 || measured.errorRate > maxErrorRate)
|
|
1481
1521
|
stopReason = "errors";
|
|
1482
1522
|
else if (measured.p95Ms > maxP95Ms)
|
|
1483
1523
|
stopReason = "latency";
|
|
1524
|
+
else if (measured.cpuUtilizationPercent > maxCpuUtilizationPercent)
|
|
1525
|
+
stopReason = "cpu";
|
|
1526
|
+
else if (measured.memoryUtilizationPercent > maxMemoryUtilizationPercent)
|
|
1527
|
+
stopReason = "memory";
|
|
1484
1528
|
else if (throughputRegressed)
|
|
1485
1529
|
stopReason = "throughput-regression";
|
|
1486
1530
|
else
|
|
1487
|
-
lastSafe =
|
|
1531
|
+
lastSafe = measured;
|
|
1488
1532
|
if (stopReason !== "maximum-tested" || concurrency === maxConcurrency)
|
|
1489
1533
|
break;
|
|
1490
1534
|
}
|
|
1535
|
+
if (!lastSafe)
|
|
1536
|
+
throw new Error("Capacity calibration found no safe request stage.");
|
|
1491
1537
|
return {
|
|
1492
1538
|
endpoint: endpoint.toString(),
|
|
1493
|
-
|
|
1539
|
+
measuredSustainableRequestsPerSecond: lastSafe.successfulRequestsPerSecond,
|
|
1540
|
+
allowedRequestsPerSecond: Math.max(1, Math.floor(lastSafe.successfulRequestsPerSecond * safetyRatio)),
|
|
1541
|
+
safetyRatio,
|
|
1542
|
+
recommendedConcurrency: Math.max(1, Math.floor(lastSafe.concurrency * safetyRatio)),
|
|
1494
1543
|
stopReason,
|
|
1495
1544
|
stages
|
|
1496
1545
|
};
|
|
@@ -1567,8 +1616,11 @@ function capacityCalibration(value, where) {
|
|
|
1567
1616
|
"requestsPerWorker",
|
|
1568
1617
|
"maxP95Ms",
|
|
1569
1618
|
"maxErrorRate",
|
|
1570
|
-
"
|
|
1571
|
-
"requestTimeoutMs"
|
|
1619
|
+
"safetyRatio",
|
|
1620
|
+
"requestTimeoutMs",
|
|
1621
|
+
"minimumStageDurationMs",
|
|
1622
|
+
"maxCpuUtilizationPercent",
|
|
1623
|
+
"maxMemoryUtilizationPercent"
|
|
1572
1624
|
], where);
|
|
1573
1625
|
const endpoint = text(calibration.endpoint, `${where}.endpoint`);
|
|
1574
1626
|
try {
|
|
@@ -1592,8 +1644,11 @@ function capacityCalibration(value, where) {
|
|
|
1592
1644
|
"requestsPerWorker",
|
|
1593
1645
|
"maxP95Ms",
|
|
1594
1646
|
"maxErrorRate",
|
|
1595
|
-
"
|
|
1596
|
-
"requestTimeoutMs"
|
|
1647
|
+
"safetyRatio",
|
|
1648
|
+
"requestTimeoutMs",
|
|
1649
|
+
"minimumStageDurationMs",
|
|
1650
|
+
"maxCpuUtilizationPercent",
|
|
1651
|
+
"maxMemoryUtilizationPercent"
|
|
1597
1652
|
].flatMap((name) => {
|
|
1598
1653
|
const found = optionalNumber(name);
|
|
1599
1654
|
return found === undefined ? [] : [[name, found]];
|
|
@@ -2784,35 +2839,76 @@ function requestServiceActivation(request, socketPath = DEFAULT_SOFTWARE_HELPER_
|
|
|
2784
2839
|
function requestSoftware(requirements, socketPath = DEFAULT_SOFTWARE_HELPER_SOCKET, timeoutMs = 15 * 60000) {
|
|
2785
2840
|
validateSoftwareRequirements(requirements);
|
|
2786
2841
|
return new Promise((resolve5, reject) => {
|
|
2787
|
-
const
|
|
2788
|
-
|
|
2789
|
-
|
|
2790
|
-
|
|
2791
|
-
socket.destroy();
|
|
2792
|
-
reject(new Error("software helper did not answer before its deadline"));
|
|
2793
|
-
});
|
|
2794
|
-
socket.on("data", (chunk) => {
|
|
2795
|
-
buffer += chunk.toString("utf8");
|
|
2796
|
-
const newline = buffer.indexOf(`
|
|
2797
|
-
`);
|
|
2798
|
-
if (newline < 0)
|
|
2842
|
+
const deadline = Date.now() + timeoutMs;
|
|
2843
|
+
let settled = false;
|
|
2844
|
+
const finish = (cause, results) => {
|
|
2845
|
+
if (settled)
|
|
2799
2846
|
return;
|
|
2800
|
-
|
|
2847
|
+
settled = true;
|
|
2848
|
+
if (cause)
|
|
2849
|
+
reject(cause);
|
|
2850
|
+
else
|
|
2851
|
+
resolve5(results);
|
|
2852
|
+
};
|
|
2853
|
+
const attempt = () => {
|
|
2854
|
+
if (settled)
|
|
2855
|
+
return;
|
|
2856
|
+
const remaining = deadline - Date.now();
|
|
2857
|
+
if (remaining <= 0)
|
|
2858
|
+
return finish(new Error("software helper did not answer before its deadline"));
|
|
2859
|
+
if (!existsSync7(socketPath)) {
|
|
2860
|
+
setTimeout(attempt, Math.min(100, Math.max(1, remaining)));
|
|
2861
|
+
return;
|
|
2862
|
+
}
|
|
2863
|
+
let buffer = "";
|
|
2864
|
+
let socket;
|
|
2801
2865
|
try {
|
|
2802
|
-
|
|
2803
|
-
|
|
2804
|
-
throw new Error(response.error?.message ?? "software helper refused the request");
|
|
2805
|
-
resolve5(response.results);
|
|
2866
|
+
socket = connect3(socketPath, () => socket.write(`${JSON.stringify({ op: "ensure", requirements })}
|
|
2867
|
+
`));
|
|
2806
2868
|
} catch (cause) {
|
|
2807
|
-
|
|
2869
|
+
const error = cause;
|
|
2870
|
+
if ((error.code === "ENOENT" || error.code === "ECONNREFUSED") && Date.now() < deadline) {
|
|
2871
|
+
setTimeout(attempt, Math.min(100, Math.max(1, deadline - Date.now())));
|
|
2872
|
+
return;
|
|
2873
|
+
}
|
|
2874
|
+
finish(error);
|
|
2875
|
+
return;
|
|
2808
2876
|
}
|
|
2809
|
-
|
|
2810
|
-
|
|
2877
|
+
socket.setTimeout(remaining, () => {
|
|
2878
|
+
socket.destroy();
|
|
2879
|
+
finish(new Error("software helper did not answer before its deadline"));
|
|
2880
|
+
});
|
|
2881
|
+
socket.on("data", (chunk) => {
|
|
2882
|
+
buffer += chunk.toString("utf8");
|
|
2883
|
+
const newline = buffer.indexOf(`
|
|
2884
|
+
`);
|
|
2885
|
+
if (newline < 0)
|
|
2886
|
+
return;
|
|
2887
|
+
socket.end();
|
|
2888
|
+
try {
|
|
2889
|
+
const response = JSON.parse(buffer.slice(0, newline));
|
|
2890
|
+
if (!response.ok || !response.results)
|
|
2891
|
+
throw new Error(response.error?.message ?? "software helper refused the request");
|
|
2892
|
+
finish(undefined, response.results);
|
|
2893
|
+
} catch (cause) {
|
|
2894
|
+
finish(cause);
|
|
2895
|
+
}
|
|
2896
|
+
});
|
|
2897
|
+
socket.on("error", (cause) => {
|
|
2898
|
+
socket.destroy();
|
|
2899
|
+
if ((cause.code === "ENOENT" || cause.code === "ECONNREFUSED") && Date.now() < deadline) {
|
|
2900
|
+
setTimeout(attempt, Math.min(100, Math.max(1, deadline - Date.now())));
|
|
2901
|
+
return;
|
|
2902
|
+
}
|
|
2903
|
+
finish(cause);
|
|
2904
|
+
});
|
|
2905
|
+
};
|
|
2906
|
+
attempt();
|
|
2811
2907
|
});
|
|
2812
2908
|
}
|
|
2813
2909
|
|
|
2814
2910
|
// src/version.ts
|
|
2815
|
-
var VERSION3 = "0.1.
|
|
2911
|
+
var VERSION3 = "0.1.89";
|
|
2816
2912
|
|
|
2817
2913
|
// src/egress-policy.ts
|
|
2818
2914
|
import { realpathSync as realpathSync3 } from "node:fs";
|
|
@@ -2948,6 +3044,7 @@ function agentEgressUnit(options) {
|
|
|
2948
3044
|
}
|
|
2949
3045
|
const deploymentEnabled = Boolean(options.repository || bootstrapBundleEnabled || options.pullDeployments);
|
|
2950
3046
|
const runnerLoopbackPorts = normalizeEgressTcpPorts(options.runnerLoopbackPorts ?? []);
|
|
3047
|
+
const agentLoopbackPorts = normalizeEgressTcpPorts(options.agentLoopbackPorts ?? []);
|
|
2951
3048
|
const runnerPublicTcpPorts = normalizeEgressTcpPorts(options.runnerPublicTcpPorts ?? DEFAULT_RUNNER_PUBLIC_TCP_PORTS);
|
|
2952
3049
|
if (deploymentEnabled && runnerPublicTcpPorts.length < 1) {
|
|
2953
3050
|
throw new Error("deployed project runner needs at least one vetted public TCP port");
|
|
@@ -2955,8 +3052,10 @@ function agentEgressUnit(options) {
|
|
|
2955
3052
|
if (deploymentEnabled)
|
|
2956
3053
|
systemdAgentEgressDirectives(runnerLoopbackPorts);
|
|
2957
3054
|
const users = [user, ...deploymentEnabled ? [DEPLOYMENT_RUNNER_USER] : []];
|
|
2958
|
-
const
|
|
2959
|
-
const
|
|
3055
|
+
const portGrantEnabled = deploymentEnabled || agentLoopbackPorts.length > 0;
|
|
3056
|
+
const restrictedUser = deploymentEnabled ? DEPLOYMENT_RUNNER_USER : user;
|
|
3057
|
+
const portGrant = portGrantEnabled ? ` --loopback-user=${restrictedUser}` + (deploymentEnabled ? runnerLoopbackPorts.map((port) => ` --loopback-tcp-port=${port}`).join("") : "") + agentLoopbackPorts.map((port) => ` --shared-loopback-tcp-port=${port}`).join("") + (deploymentEnabled ? runnerPublicTcpPorts.map((port) => ` --public-tcp-port=${port}`).join("") : "") : "";
|
|
3058
|
+
const policyProof = `ExecStartPost=${bin} egress-policy-check ${users.map((name) => `--user=${name}`).join(" ")}${portGrant}`;
|
|
2960
3059
|
return `[Unit]
|
|
2961
3060
|
Description=ForgeZero Agent host egress policy
|
|
2962
3061
|
Documentation=https://www.forgezero.net/docs/agent
|
|
@@ -2969,7 +3068,7 @@ Type=notify
|
|
|
2969
3068
|
NotifyAccess=all
|
|
2970
3069
|
User=root
|
|
2971
3070
|
Group=root
|
|
2972
|
-
ExecStart=${bin} egress-policy ${users.map((name) => `--user=${name}`).join(" ")}${
|
|
3071
|
+
ExecStart=${bin} egress-policy ${users.map((name) => `--user=${name}`).join(" ")}${portGrant}
|
|
2973
3072
|
${policyProof}
|
|
2974
3073
|
Restart=on-failure
|
|
2975
3074
|
RestartSec=2
|
|
@@ -3253,7 +3352,7 @@ function agentEnrolmentUnit(options) {
|
|
|
3253
3352
|
Requires=forgezero-agent-egress.service
|
|
3254
3353
|
BindsTo=forgezero-agent-egress.service
|
|
3255
3354
|
` : "";
|
|
3256
|
-
const egressDirectives = options.enforceEgress ? systemdAgentEgressDirectives() : "";
|
|
3355
|
+
const egressDirectives = options.enforceEgress ? systemdAgentEgressDirectives(options.agentLoopbackPorts ?? []) : "";
|
|
3257
3356
|
return `[Unit]
|
|
3258
3357
|
Description=Bind this machine to its ForgeZero compute
|
|
3259
3358
|
After=network-online.target
|
|
@@ -3290,7 +3389,7 @@ WantedBy=multi-user.target
|
|
|
3290
3389
|
function deploymentRunnerUnit(options) {
|
|
3291
3390
|
const bin = options.binPath ?? "fz-agent";
|
|
3292
3391
|
const root = options.deployRoot ?? "/opt/forgezero";
|
|
3293
|
-
const agentUser = options.user ?? "forgezero
|
|
3392
|
+
const agentUser = options.user ?? "forgezero";
|
|
3294
3393
|
const egressDependency = options.enforceEgress ? `After=forgezero-agent-egress.service
|
|
3295
3394
|
Requires=forgezero-agent-egress.service
|
|
3296
3395
|
BindsTo=forgezero-agent-egress.service
|
|
@@ -3331,7 +3430,7 @@ RestrictRealtime=true
|
|
|
3331
3430
|
MemoryDenyWriteExecute=true
|
|
3332
3431
|
LockPersonality=true
|
|
3333
3432
|
${egressDirectives}
|
|
3334
|
-
ReadWritePaths=${root}/releases ${root}/runner-home
|
|
3433
|
+
ReadWritePaths=${root}/releases ${root}/runner-home ${root}/slots /etc/nginx/conf.d /var/log/nginx
|
|
3335
3434
|
|
|
3336
3435
|
[Install]
|
|
3337
3436
|
WantedBy=multi-user.target
|
|
@@ -3466,7 +3565,7 @@ function agentUnit(options) {
|
|
|
3466
3565
|
const supplementaryGroups = [
|
|
3467
3566
|
AGENT_UPDATE_GROUP,
|
|
3468
3567
|
deploymentEnabled ? DEPLOYMENT_GROUP : null,
|
|
3469
|
-
|
|
3568
|
+
SOFTWARE_HELPER_GROUP,
|
|
3470
3569
|
lifecycleEnabled ? LIFECYCLE_GROUP : null
|
|
3471
3570
|
].filter((value) => value !== null);
|
|
3472
3571
|
const deploymentGroup = supplementaryGroups.length > 0 ? `SupplementaryGroups=${supplementaryGroups.join(" ")}` : "";
|
|
@@ -3475,7 +3574,7 @@ function agentUnit(options) {
|
|
|
3475
3574
|
"forgezero-agent-update-helper.service",
|
|
3476
3575
|
options.enforceEgress ? "forgezero-agent-egress.service" : null,
|
|
3477
3576
|
deploymentEnabled ? "forgezero-deploy-runner.service" : null,
|
|
3478
|
-
|
|
3577
|
+
"forgezero-software-helper.service",
|
|
3479
3578
|
lifecycleEnabled ? "forgezero-lifecycle-helper.service" : null,
|
|
3480
3579
|
warpEnabled ? "warp-svc.service" : null,
|
|
3481
3580
|
enrolmentEnabled ? "forgezero-agent-enrol.service" : null
|
|
@@ -3484,7 +3583,7 @@ function agentUnit(options) {
|
|
|
3484
3583
|
"forgezero-agent-update-helper.service",
|
|
3485
3584
|
options.enforceEgress ? "forgezero-agent-egress.service" : null,
|
|
3486
3585
|
deploymentEnabled ? "forgezero-deploy-runner.service" : null,
|
|
3487
|
-
|
|
3586
|
+
"forgezero-software-helper.service",
|
|
3488
3587
|
lifecycleEnabled ? "forgezero-lifecycle-helper.service" : null,
|
|
3489
3588
|
warpEnabled ? "warp-svc.service" : null,
|
|
3490
3589
|
enrolmentEnabled ? "forgezero-agent-enrol.service" : null
|
|
@@ -3499,9 +3598,9 @@ function agentUnit(options) {
|
|
|
3499
3598
|
const snpDevice = options.mode === "attested" ? `DevicePolicy=closed
|
|
3500
3599
|
DeviceAllow=/dev/sev-guest rw` : "";
|
|
3501
3600
|
const snpPrepare = options.mode === "attested" ? `ExecStartPre=+/bin/chgrp ${VAULT_GROUP} /dev/sev-guest
|
|
3502
|
-
ExecStartPre=+/bin/chmod
|
|
3601
|
+
ExecStartPre=+/bin/chmod 0660 /dev/sev-guest
|
|
3503
3602
|
` : "";
|
|
3504
|
-
const egressDirectives = options.enforceEgress ? systemdAgentEgressDirectives() : "";
|
|
3603
|
+
const egressDirectives = options.enforceEgress ? systemdAgentEgressDirectives(options.agentLoopbackPorts ?? []) : "";
|
|
3505
3604
|
return `[Unit]
|
|
3506
3605
|
Description=ForgeZero node agent (${options.mode})
|
|
3507
3606
|
Documentation=https://www.forgezero.net/docs/agent
|
|
@@ -3632,6 +3731,7 @@ function planProvision(options) {
|
|
|
3632
3731
|
const deployRoot = options.deployRoot ?? "/opt/forgezero";
|
|
3633
3732
|
const deploymentEnabled = Boolean(options.repository || options.pullDeployments);
|
|
3634
3733
|
const runnerLoopbackPorts = normalizeEgressTcpPorts(options.runnerLoopbackPorts ?? []);
|
|
3734
|
+
const agentLoopbackPorts = normalizeEgressTcpPorts(options.agentLoopbackPorts ?? []);
|
|
3635
3735
|
const runnerPublicTcpPorts = normalizeEgressTcpPorts(options.runnerPublicTcpPorts ?? DEFAULT_RUNNER_PUBLIC_TCP_PORTS);
|
|
3636
3736
|
const lifecycleEnabled = Boolean(options.pullMigrations && options.lifecycleProfilePath);
|
|
3637
3737
|
if (Boolean(options.pullMigrations) !== Boolean(options.lifecycleProfilePath)) {
|
|
@@ -3680,8 +3780,9 @@ function planProvision(options) {
|
|
|
3680
3780
|
const enabledUnits = [
|
|
3681
3781
|
"forgezero-agent.socket",
|
|
3682
3782
|
"forgezero-agent-update-helper.service",
|
|
3783
|
+
"forgezero-software-helper.service",
|
|
3683
3784
|
...options.enforceEgress ? ["forgezero-agent-egress.service"] : [],
|
|
3684
|
-
...deploymentEnabled ? ["forgezero-deploy-runner.service"
|
|
3785
|
+
...deploymentEnabled ? ["forgezero-deploy-runner.service"] : [],
|
|
3685
3786
|
...lifecycleEnabled ? ["forgezero-lifecycle-helper.service"] : [],
|
|
3686
3787
|
...warpEnabled ? ["forgezero-warp-config.service", "warp-svc.service"] : [],
|
|
3687
3788
|
...enrolmentEnabled ? ["forgezero-agent-enrol.service"] : [],
|
|
@@ -3689,8 +3790,9 @@ function planProvision(options) {
|
|
|
3689
3790
|
];
|
|
3690
3791
|
const restartedUnits = [
|
|
3691
3792
|
"forgezero-agent-update-helper.service",
|
|
3793
|
+
"forgezero-software-helper.service",
|
|
3692
3794
|
...options.enforceEgress ? ["forgezero-agent-egress.service"] : [],
|
|
3693
|
-
...deploymentEnabled ? ["forgezero-deploy-runner.service"
|
|
3795
|
+
...deploymentEnabled ? ["forgezero-deploy-runner.service"] : [],
|
|
3694
3796
|
...lifecycleEnabled ? ["forgezero-lifecycle-helper.service"] : [],
|
|
3695
3797
|
...warpEnabled ? ["forgezero-warp-config.service", "warp-svc.service"] : [],
|
|
3696
3798
|
...enrolmentEnabled ? ["forgezero-agent-enrol.service"] : []
|
|
@@ -3714,9 +3816,9 @@ function planProvision(options) {
|
|
|
3714
3816
|
...options.enforceEgress ? [
|
|
3715
3817
|
{ path: AGENT_EGRESS_UNIT_PATH, unit: agentEgressUnit(options) }
|
|
3716
3818
|
] : [],
|
|
3819
|
+
{ path: SOFTWARE_HELPER_UNIT_PATH, unit: softwareHelperUnit(options) },
|
|
3717
3820
|
...deploymentEnabled ? [
|
|
3718
|
-
{ path: DEPLOYMENT_RUNNER_UNIT_PATH, unit: deploymentRunnerUnit(options) }
|
|
3719
|
-
{ path: SOFTWARE_HELPER_UNIT_PATH, unit: softwareHelperUnit(options) }
|
|
3821
|
+
{ path: DEPLOYMENT_RUNNER_UNIT_PATH, unit: deploymentRunnerUnit(options) }
|
|
3720
3822
|
] : [],
|
|
3721
3823
|
...enrolmentEnabled ? [
|
|
3722
3824
|
{ path: ENROLMENT_UNIT_PATH, unit: agentEnrolmentUnit(options) }
|
|
@@ -3749,24 +3851,29 @@ function planProvision(options) {
|
|
|
3749
3851
|
step("Agent update helper access group", { kind: "commands", commands: [{ argv: ["/usr/sbin/groupadd", "--system", AGENT_UPDATE_GROUP], acceptedExitCodes: [0, 9] }] }),
|
|
3750
3852
|
...sourceBinPath && binPath ? [step("root-owned agent runtime", { kind: "install-runtime", source: sourceBinPath, binary: binPath, version: VERSION3 })] : [],
|
|
3751
3853
|
...warpEnabled ? [step("Cloudflare One client for Ubuntu 26.04", { kind: "install-warp" })] : [],
|
|
3752
|
-
|
|
3753
|
-
{ argv: ["/usr/sbin/groupadd", "--system", DEPLOYMENT_GROUP], acceptedExitCodes: [0, 9] },
|
|
3854
|
+
step("software strategy helper group", { kind: "commands", commands: [
|
|
3754
3855
|
{ argv: ["/usr/sbin/groupadd", "--system", SOFTWARE_HELPER_GROUP], acceptedExitCodes: [0, 9] }
|
|
3856
|
+
] }),
|
|
3857
|
+
...deploymentEnabled ? [step("deployment isolation group", { kind: "commands", commands: [
|
|
3858
|
+
{ argv: ["/usr/sbin/groupadd", "--system", DEPLOYMENT_GROUP], acceptedExitCodes: [0, 9] }
|
|
3755
3859
|
] })] : [],
|
|
3756
3860
|
...lifecycleEnabled ? [step("lifecycle helper access group", { kind: "commands", commands: [{ argv: ["/usr/sbin/groupadd", "--system", LIFECYCLE_GROUP], acceptedExitCodes: [0, 9] }] })] : [],
|
|
3757
3861
|
step("service account", { kind: "commands", commands: [{ argv: ["/usr/sbin/useradd", "--system", "--no-create-home", "--shell", "/usr/sbin/nologin", user], acceptedExitCodes: [0, 9] }] }),
|
|
3758
3862
|
step("bind service account to vault group", { kind: "commands", commands: [{ argv: ["/usr/sbin/usermod", "-g", VAULT_GROUP, user] }] }),
|
|
3759
3863
|
step("grant verified Agent update access", { kind: "commands", commands: [{ argv: ["/usr/sbin/usermod", "-a", "-G", AGENT_UPDATE_GROUP, user] }] }),
|
|
3864
|
+
step("grant software helper socket access", { kind: "commands", commands: [
|
|
3865
|
+
{ argv: ["/usr/sbin/usermod", "-a", "-G", SOFTWARE_HELPER_GROUP, user] }
|
|
3866
|
+
] }),
|
|
3760
3867
|
...lifecycleEnabled ? [step("grant lifecycle helper socket access", { kind: "commands", commands: [{ argv: ["/usr/sbin/usermod", "-a", "-G", LIFECYCLE_GROUP, user] }] })] : [],
|
|
3761
3868
|
...deploymentEnabled ? [step("credential-free deployment account", { kind: "commands", commands: [
|
|
3762
3869
|
{ argv: ["/usr/sbin/useradd", "--system", "--no-create-home", "--shell", "/usr/sbin/nologin", "--gid", DEPLOYMENT_GROUP, DEPLOYMENT_RUNNER_USER], acceptedExitCodes: [0, 9] },
|
|
3763
3870
|
{ argv: ["/usr/sbin/useradd", "--system", "--no-create-home", "--shell", "/usr/sbin/nologin", "--gid", VAULT_GROUP, APPLICATION_RUNTIME_USER], acceptedExitCodes: [0, 9] },
|
|
3764
3871
|
{ argv: ["/usr/sbin/usermod", "-a", "-G", DEPLOYMENT_GROUP, APPLICATION_RUNTIME_USER] },
|
|
3765
|
-
{ argv: ["/usr/sbin/usermod", "-a", "-G",
|
|
3872
|
+
{ argv: ["/usr/sbin/usermod", "-a", "-G", DEPLOYMENT_GROUP, user] }
|
|
3766
3873
|
] })] : [],
|
|
3767
3874
|
step("credential and state directories", { kind: "directories", directories: [
|
|
3768
3875
|
{ path: credentialDir, mode: 448, owner: "root", group: "root" },
|
|
3769
|
-
{ path: "/var/lib/forgezero", mode: 488, owner: "root", group:
|
|
3876
|
+
{ path: "/var/lib/forgezero", mode: 488, owner: "root", group: VAULT_GROUP }
|
|
3770
3877
|
] }),
|
|
3771
3878
|
step("encrypted node identity", { kind: "ensure-seed", credential: seedCredentialPath }),
|
|
3772
3879
|
...options.generateGitIdentity && gitCredentialPath && gitPublicKeyPath ? [
|
|
@@ -3817,19 +3924,26 @@ function planProvision(options) {
|
|
|
3817
3924
|
{ argv: ["/usr/bin/systemctl", "enable", ...enabledUnits] },
|
|
3818
3925
|
{ argv: ["/usr/bin/systemctl", "reset-failed", "forgezero-agent.service"], acceptedExitCodes: [0, 1] },
|
|
3819
3926
|
...restartedUnits.length ? [{ argv: ["/usr/bin/systemctl", "restart", ...restartedUnits] }] : [],
|
|
3927
|
+
{ argv: ["/usr/bin/systemctl", "stop", "forgezero-agent-proxy.service"], acceptedExitCodes: [0, 1, 5] },
|
|
3820
3928
|
{ argv: ["/usr/bin/systemctl", "restart", "forgezero-agent.socket"] },
|
|
3821
3929
|
{ argv: ["/usr/bin/systemctl", "reset-failed", "forgezero-agent.service"], acceptedExitCodes: [0, 1] },
|
|
3822
3930
|
{ argv: ["/usr/bin/systemctl", "restart", "forgezero-agent.service"] }
|
|
3823
3931
|
] }),
|
|
3824
3932
|
...enrolmentEnabled ? [step("prove the compute binding is durable", { kind: "verify-file", path: enrolStatePath })] : [],
|
|
3825
|
-
...options.enforceEgress ? [step("prove the Agent egress policy is active", {
|
|
3933
|
+
...options.enforceEgress ? [step("prove the Agent egress policy is active", {
|
|
3934
|
+
kind: "verify-egress",
|
|
3935
|
+
deploymentEnabled,
|
|
3936
|
+
runnerPublicTcpPorts,
|
|
3937
|
+
runnerLoopbackPorts,
|
|
3938
|
+
agentLoopbackPorts
|
|
3939
|
+
})] : [],
|
|
3826
3940
|
step("prove it is running", { kind: "commands", commands: [{ argv: ["/usr/bin/systemctl", "is-active", "forgezero-agent.service"] }] }),
|
|
3827
3941
|
step("prove the public Vault socket exists", { kind: "wait-socket", path: options.socketPath, attempts: 100, intervalMs: 100 }),
|
|
3828
3942
|
step("prove the Agent Vault backend exists", { kind: "wait-socket", path: agentBackendSocketPath(options.socketPath), attempts: 100, intervalMs: 100 }),
|
|
3829
3943
|
step("prove the Agent update helper exists", { kind: "wait-socket", path: DEFAULT_AGENT_UPDATE_SOCKET, attempts: 100, intervalMs: 100 }),
|
|
3944
|
+
step("prove the software strategy helper socket exists", { kind: "wait-socket", path: DEFAULT_SOFTWARE_HELPER_SOCKET, attempts: 100, intervalMs: 100 }),
|
|
3830
3945
|
...deploymentEnabled ? [
|
|
3831
|
-
step("prove the deployment runner socket exists", { kind: "wait-socket", path: DEPLOYMENT_RUNNER_SOCKET, attempts: 100, intervalMs: 100 })
|
|
3832
|
-
step("prove the software strategy helper socket exists", { kind: "wait-socket", path: DEFAULT_SOFTWARE_HELPER_SOCKET, attempts: 100, intervalMs: 100 })
|
|
3946
|
+
step("prove the deployment runner socket exists", { kind: "wait-socket", path: DEPLOYMENT_RUNNER_SOCKET, attempts: 100, intervalMs: 100 })
|
|
3833
3947
|
] : [],
|
|
3834
3948
|
...lifecycleEnabled ? [step("prove the lifecycle helper socket exists", { kind: "wait-socket", path: lifecycleHelperSocketPath, attempts: 100, intervalMs: 100 })] : [],
|
|
3835
3949
|
...warpEnabled ? [step("prove Cloudflare WARP is connected", { kind: "verify-warp" })] : [],
|
package/dist/recovery-host.js
CHANGED
|
@@ -14,7 +14,7 @@ var checked = async (run, argv, expected = 0) => {
|
|
|
14
14
|
return result;
|
|
15
15
|
};
|
|
16
16
|
var activeSlot = () => {
|
|
17
|
-
const slot = readFileSync("/opt/forgezero/.
|
|
17
|
+
const slot = readFileSync("/opt/forgezero/slots/.active", "utf8").trim();
|
|
18
18
|
if (slot !== "blue" && slot !== "green")
|
|
19
19
|
throw new Error("invalid active slot");
|
|
20
20
|
return slot;
|