@forgezero/agent 0.1.88 → 0.1.90

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. package/README.md +23 -9
  2. package/dist/agent-heartbeat.d.ts +3 -0
  3. package/dist/agent-heartbeat.js +2 -1
  4. package/dist/bootstrap.d.ts +8 -1
  5. package/dist/bootstrap.js +241 -62
  6. package/dist/capacity-calibration.d.ts +35 -5
  7. package/dist/capacity-calibration.js +66 -22
  8. package/dist/cli/agent-install.d.ts +3 -0
  9. package/dist/community-rehearsal-host.js +7 -2
  10. package/dist/control.d.ts +3 -1
  11. package/dist/definition.js +83 -28
  12. package/dist/deploy-file.js +83 -28
  13. package/dist/deployment.d.ts +5 -0
  14. package/dist/egress-policy.d.ts +4 -0
  15. package/dist/fz-agent.js +450 -158
  16. package/dist/fz.js +429 -138
  17. package/dist/host-maintenance.js +1 -1
  18. package/dist/index.d.ts +7 -0
  19. package/dist/metal-bootstrap.js +1 -1
  20. package/dist/metal-helper-socket.js +7 -2
  21. package/dist/metal-provision.js +7 -2
  22. package/dist/migration-pull.d.ts +1 -0
  23. package/dist/migration-pull.js +4 -0
  24. package/dist/operator-bootstrap.d.ts +12 -0
  25. package/dist/operator-bootstrap.js +385 -72
  26. package/dist/platform-bootstrap-runtime.d.ts +1 -0
  27. package/dist/platform-bootstrap-runtime.js +47 -8
  28. package/dist/platform-fleet-verification.js +273 -95
  29. package/dist/platform-genesis.js +7 -2
  30. package/dist/platform-launch-profile.d.ts +1 -0
  31. package/dist/provision.d.ts +5 -1
  32. package/dist/provision.js +192 -75
  33. package/dist/recovery-host.js +1 -1
  34. package/dist/software-helper.js +144 -48
  35. package/dist/software.js +7 -2
  36. package/dist/ssh-bootstrap.d.ts +1 -0
  37. package/dist/ubuntu.js +7 -2
  38. package/dist/version.d.ts +1 -1
  39. package/package.json +1 -1
  40. package/schema/deploy-v3.json +5 -2
@@ -355,7 +355,12 @@ async function executeSoftwareOperation(operation) {
355
355
  }
356
356
  if (software === "nginx") {
357
357
  const binary = await run(["/usr/sbin/nginx", "-v"]);
358
- return binary.exitCode === 0 ? run(["/usr/bin/systemctl", "is-active", "--quiet", "nginx.service"]) : binary;
358
+ if (binary.exitCode !== 0)
359
+ return binary;
360
+ if (!existsSync("/usr/lib/nginx/modules/ngx_http_headers_more_filter_module.so")) {
361
+ return { exitCode: 1, output: "Nginx response-header suppression module is unavailable" };
362
+ }
363
+ return run(["/usr/bin/systemctl", "is-active", "--quiet", "nginx.service"]);
359
364
  }
360
365
  if (software === "arangodb") {
361
366
  const binary = await run(["/usr/sbin/arangod", "--version"]);
@@ -387,7 +392,7 @@ async function executeSoftwareOperation(operation) {
387
392
  }
388
393
  if (software === "docker" || software === "nginx" || software === "ufw" || software === "openssh-client" || software === "git") {
389
394
  const packageName = software === "openssh-client" ? "openssh-client" : software === "docker" ? "docker.io" : software;
390
- const installed = await aptInstall(packageName);
395
+ const installed = software === "nginx" ? await aptInstallMany(["nginx", "libnginx-mod-http-headers-more-filter"]) : await aptInstall(packageName);
391
396
  if (installed.exitCode !== 0 || software !== "nginx" && software !== "docker")
392
397
  return installed;
393
398
  if (software === "docker") {
@@ -17,6 +17,7 @@ export interface ForgeZeroLaunchProfile {
17
17
  city?: string;
18
18
  };
19
19
  agentOtlpEndpoint: string;
20
+ seedSyncEpoch: string;
20
21
  privateCidrs: readonly string[];
21
22
  nodeHostname(nodeIndex: number): string;
22
23
  }
@@ -112,6 +112,8 @@ export interface UnitOptions {
112
112
  enforceEgress?: boolean;
113
113
  /** Provisioning-owned runner health endpoints; repository commands cannot change them. */
114
114
  runnerLoopbackPorts?: readonly number[];
115
+ /** Local platform API ports needed by the credential-bearing Agent. */
116
+ agentLoopbackPorts?: readonly number[];
115
117
  /** Vetted public TCP surface for project commands. Defaults to HTTPS only. */
116
118
  runnerPublicTcpPorts?: readonly number[];
117
119
  /** Enable PQ-authenticated lifecycle claims through a constrained root helper. */
@@ -172,7 +174,7 @@ export declare const WARP_SERVICE_DROP_IN_PATH = "/etc/systemd/system/warp-svc.s
172
174
  export declare const AGENT_EGRESS_UNIT_PATH = "/etc/systemd/system/forgezero-agent-egress.service";
173
175
  export declare const DEFAULT_RUNNER_PUBLIC_TCP_PORTS: readonly [443];
174
176
  /** Root-owned policy monitor. The credential-bearing Agent is bound to it. */
175
- export declare function agentEgressUnit(options: Pick<UnitOptions, 'binPath' | 'user' | 'repository' | 'pullDeployments' | 'bootstrapBundlePath' | 'bootstrapBundleManifestPath' | 'runnerLoopbackPorts' | 'runnerPublicTcpPorts'>): string;
177
+ export declare function agentEgressUnit(options: Pick<UnitOptions, 'binPath' | 'user' | 'repository' | 'pullDeployments' | 'bootstrapBundlePath' | 'bootstrapBundleManifestPath' | 'runnerLoopbackPorts' | 'runnerPublicTcpPorts' | 'agentLoopbackPorts'>): string;
176
178
  /** Root may execute only package-owned OS strategies selected by id + version. */
177
179
  export declare function softwareHelperUnit(options: Pick<UnitOptions, 'binPath' | 'deployRoot'>): string;
178
180
  /** Fixed root boundary for verified, health-gated Agent replacement. */
@@ -281,8 +283,10 @@ export type ProvisionOperation = {
281
283
  path: string;
282
284
  } | {
283
285
  kind: 'verify-egress';
286
+ deploymentEnabled: boolean;
284
287
  runnerPublicTcpPorts: readonly number[];
285
288
  runnerLoopbackPorts: readonly number[];
289
+ agentLoopbackPorts: readonly number[];
286
290
  } | {
287
291
  kind: 'verify-resolved-stub';
288
292
  } | {
package/dist/provision.js CHANGED
@@ -1198,7 +1198,12 @@ async function executeSoftwareOperation(operation) {
1198
1198
  }
1199
1199
  if (software === "nginx") {
1200
1200
  const binary = await run(["/usr/sbin/nginx", "-v"]);
1201
- return binary.exitCode === 0 ? run(["/usr/bin/systemctl", "is-active", "--quiet", "nginx.service"]) : binary;
1201
+ if (binary.exitCode !== 0)
1202
+ return binary;
1203
+ if (!existsSync4("/usr/lib/nginx/modules/ngx_http_headers_more_filter_module.so")) {
1204
+ return { exitCode: 1, output: "Nginx response-header suppression module is unavailable" };
1205
+ }
1206
+ return run(["/usr/bin/systemctl", "is-active", "--quiet", "nginx.service"]);
1202
1207
  }
1203
1208
  if (software === "arangodb") {
1204
1209
  const binary = await run(["/usr/sbin/arangod", "--version"]);
@@ -1230,7 +1235,7 @@ async function executeSoftwareOperation(operation) {
1230
1235
  }
1231
1236
  if (software === "docker" || software === "nginx" || software === "ufw" || software === "openssh-client" || software === "git") {
1232
1237
  const packageName = software === "openssh-client" ? "openssh-client" : software === "docker" ? "docker.io" : software;
1233
- const installed = await aptInstall(packageName);
1238
+ const installed = software === "nginx" ? await aptInstallMany(["nginx", "libnginx-mod-http-headers-more-filter"]) : await aptInstall(packageName);
1234
1239
  if (installed.exitCode !== 0 || software !== "nginx" && software !== "docker")
1235
1240
  return installed;
1236
1241
  if (software === "docker") {
@@ -1382,11 +1387,22 @@ async function ensureSoftwareRequirements(requirementsInput, options) {
1382
1387
  }
1383
1388
 
1384
1389
  // src/capacity-calibration.ts
1385
- var percentile95 = (values) => {
1390
+ import { cpus, freemem, totalmem } from "node:os";
1391
+ var percentile = (values, quantile) => {
1386
1392
  if (values.length === 0)
1387
1393
  return Number.POSITIVE_INFINITY;
1388
1394
  const sorted = values.toSorted((left, right) => left - right);
1389
- return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * 0.95) - 1)];
1395
+ return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * quantile) - 1)];
1396
+ };
1397
+ var hostSample = () => {
1398
+ let cpuIdle = 0;
1399
+ let cpuTotal = 0;
1400
+ for (const cpu of cpus()) {
1401
+ cpuIdle += cpu.times.idle;
1402
+ cpuTotal += Object.values(cpu.times).reduce((sum, value) => sum + value, 0);
1403
+ }
1404
+ const memoryTotal = totalmem();
1405
+ return { cpuIdle, cpuTotal, memoryUsed: memoryTotal - freemem(), memoryTotal };
1390
1406
  };
1391
1407
  function localCalibrationEndpoint(value) {
1392
1408
  const endpoint = new URL(value);
@@ -1401,9 +1417,12 @@ function validateCapacityCalibrationOptions(options) {
1401
1417
  const requestsPerWorker = options.requestsPerWorker ?? 8;
1402
1418
  const maxP95Ms = options.maxP95Ms ?? 250;
1403
1419
  const maxErrorRate = options.maxErrorRate ?? 0.01;
1404
- const headroomRatio = options.headroomRatio ?? 0.8;
1420
+ const safetyRatio = options.safetyRatio ?? 0.8;
1405
1421
  const requestTimeoutMs = options.requestTimeoutMs ?? 5000;
1406
- if (!Number.isSafeInteger(maxConcurrency) || maxConcurrency < 1 || maxConcurrency > 4096 || !Number.isSafeInteger(requestsPerWorker) || requestsPerWorker < 2 || requestsPerWorker > 100 || !Number.isFinite(maxP95Ms) || maxP95Ms < 1 || !Number.isFinite(maxErrorRate) || maxErrorRate < 0 || maxErrorRate > 0.2 || !Number.isFinite(headroomRatio) || headroomRatio < 0.25 || headroomRatio > 0.95 || !Number.isSafeInteger(requestTimeoutMs) || requestTimeoutMs < 100 || requestTimeoutMs > 30000) {
1422
+ const minimumStageDurationMs = options.minimumStageDurationMs ?? 5000;
1423
+ const maxCpuUtilizationPercent = options.maxCpuUtilizationPercent ?? 90;
1424
+ const maxMemoryUtilizationPercent = options.maxMemoryUtilizationPercent ?? 90;
1425
+ if (!Number.isSafeInteger(maxConcurrency) || maxConcurrency < 1 || maxConcurrency > 4096 || !Number.isSafeInteger(requestsPerWorker) || requestsPerWorker < 2 || requestsPerWorker > 100 || !Number.isFinite(maxP95Ms) || maxP95Ms < 1 || !Number.isFinite(maxErrorRate) || maxErrorRate < 0 || maxErrorRate > 0.2 || !Number.isFinite(safetyRatio) || safetyRatio < 0.25 || safetyRatio > 0.95 || !Number.isSafeInteger(requestTimeoutMs) || requestTimeoutMs < 100 || requestTimeoutMs > 30000 || !Number.isSafeInteger(minimumStageDurationMs) || minimumStageDurationMs < 100 || minimumStageDurationMs > 60000 || !Number.isFinite(maxCpuUtilizationPercent) || maxCpuUtilizationPercent < 25 || maxCpuUtilizationPercent > 100 || !Number.isFinite(maxMemoryUtilizationPercent) || maxMemoryUtilizationPercent < 25 || maxMemoryUtilizationPercent > 100) {
1407
1426
  throw new Error("Capacity calibration bounds are invalid.");
1408
1427
  }
1409
1428
  return {
@@ -1412,18 +1431,24 @@ function validateCapacityCalibrationOptions(options) {
1412
1431
  requestsPerWorker,
1413
1432
  maxP95Ms,
1414
1433
  maxErrorRate,
1415
- headroomRatio,
1416
- requestTimeoutMs
1434
+ safetyRatio,
1435
+ requestTimeoutMs,
1436
+ minimumStageDurationMs,
1437
+ maxCpuUtilizationPercent,
1438
+ maxMemoryUtilizationPercent
1417
1439
  };
1418
1440
  }
1419
- async function stage(endpoint, concurrency, requestsPerWorker, timeoutMs, fetcher) {
1441
+ async function stage(endpoint, concurrency, requestsPerWorker, timeoutMs, minimumDurationMs, fetcher, sampler) {
1420
1442
  const latencies = [];
1421
1443
  let succeeded = 0;
1422
1444
  let failed = 0;
1423
1445
  let overloaded = 0;
1446
+ const systemBefore = sampler();
1424
1447
  const started = performance.now();
1425
1448
  await Promise.all(Array.from({ length: concurrency }, async () => {
1426
- for (let request = 0;request < requestsPerWorker; request += 1) {
1449
+ let request = 0;
1450
+ while (request < requestsPerWorker || performance.now() - started < minimumDurationMs) {
1451
+ request += 1;
1427
1452
  const requestStarted = performance.now();
1428
1453
  try {
1429
1454
  const response = await fetcher(endpoint, {
@@ -1447,50 +1472,74 @@ async function stage(endpoint, concurrency, requestsPerWorker, timeoutMs, fetche
1447
1472
  }
1448
1473
  }
1449
1474
  }));
1450
- const elapsedSeconds = Math.max((performance.now() - started) / 1000, 0.001);
1475
+ const durationMs = Math.max(performance.now() - started, 1);
1476
+ const elapsedSeconds = durationMs / 1000;
1477
+ const systemAfter = sampler();
1478
+ const cpuTotal = Math.max(0, systemAfter.cpuTotal - systemBefore.cpuTotal);
1479
+ const cpuIdle = Math.max(0, systemAfter.cpuIdle - systemBefore.cpuIdle);
1480
+ const cpuUtilizationPercent = cpuTotal === 0 ? 0 : (cpuTotal - cpuIdle) / cpuTotal * 100;
1481
+ const memoryUtilizationPercent = Math.max(systemBefore.memoryTotal > 0 ? systemBefore.memoryUsed / systemBefore.memoryTotal * 100 : 0, systemAfter.memoryTotal > 0 ? systemAfter.memoryUsed / systemAfter.memoryTotal * 100 : 0);
1482
+ const requests = succeeded + failed;
1451
1483
  return {
1452
1484
  concurrency,
1453
- requests: concurrency * requestsPerWorker,
1485
+ requests,
1454
1486
  succeeded,
1455
1487
  failed,
1456
1488
  overloaded,
1457
- throughputPerSecond: Number(((succeeded + failed) / elapsedSeconds).toFixed(2)),
1458
- p95Ms: Number(percentile95(latencies).toFixed(2))
1489
+ successfulRequestsPerSecond: Number((succeeded / elapsedSeconds).toFixed(2)),
1490
+ attemptedRequestsPerSecond: Number((requests / elapsedSeconds).toFixed(2)),
1491
+ durationMs: Number(durationMs.toFixed(2)),
1492
+ errorRate: Number((failed / Math.max(requests, 1)).toFixed(6)),
1493
+ p95Ms: Number(percentile(latencies, 0.95).toFixed(2)),
1494
+ p99Ms: Number(percentile(latencies, 0.99).toFixed(2)),
1495
+ cpuUtilizationPercent: Number(cpuUtilizationPercent.toFixed(2)),
1496
+ memoryUtilizationPercent: Number(memoryUtilizationPercent.toFixed(2))
1459
1497
  };
1460
1498
  }
1461
- async function calibrateHttpConcurrency(options, fetcher = fetch) {
1499
+ async function calibrateHttpConcurrency(options, fetcher = fetch, sampler = hostSample) {
1462
1500
  const {
1463
1501
  endpoint,
1464
1502
  maxConcurrency,
1465
1503
  requestsPerWorker,
1466
1504
  maxP95Ms,
1467
1505
  maxErrorRate,
1468
- headroomRatio,
1469
- requestTimeoutMs: timeoutMs
1506
+ safetyRatio,
1507
+ requestTimeoutMs: timeoutMs,
1508
+ minimumStageDurationMs,
1509
+ maxCpuUtilizationPercent,
1510
+ maxMemoryUtilizationPercent
1470
1511
  } = validateCapacityCalibrationOptions(options);
1471
1512
  const stages = [];
1472
- let lastSafe = 1;
1513
+ let lastSafe;
1473
1514
  let stopReason = "maximum-tested";
1474
1515
  for (let concurrency = 1;; concurrency = Math.min(maxConcurrency, concurrency * 2)) {
1475
- const measured = await stage(endpoint, concurrency, requestsPerWorker, timeoutMs, fetcher);
1516
+ const measured = await stage(endpoint, concurrency, requestsPerWorker, timeoutMs, minimumStageDurationMs, fetcher, sampler);
1476
1517
  stages.push(measured);
1477
- const errorRate = measured.failed / measured.requests;
1478
1518
  const previous = stages.at(-2);
1479
- const throughputRegressed = Boolean(previous && concurrency > 1 && measured.throughputPerSecond < previous.throughputPerSecond * 0.9);
1480
- if (measured.overloaded > 0 || errorRate > maxErrorRate)
1519
+ const throughputRegressed = Boolean(previous && concurrency > 1 && measured.successfulRequestsPerSecond < previous.successfulRequestsPerSecond * 0.9);
1520
+ if (measured.overloaded > 0 || measured.errorRate > maxErrorRate)
1481
1521
  stopReason = "errors";
1482
1522
  else if (measured.p95Ms > maxP95Ms)
1483
1523
  stopReason = "latency";
1524
+ else if (measured.cpuUtilizationPercent > maxCpuUtilizationPercent)
1525
+ stopReason = "cpu";
1526
+ else if (measured.memoryUtilizationPercent > maxMemoryUtilizationPercent)
1527
+ stopReason = "memory";
1484
1528
  else if (throughputRegressed)
1485
1529
  stopReason = "throughput-regression";
1486
1530
  else
1487
- lastSafe = concurrency;
1531
+ lastSafe = measured;
1488
1532
  if (stopReason !== "maximum-tested" || concurrency === maxConcurrency)
1489
1533
  break;
1490
1534
  }
1535
+ if (!lastSafe)
1536
+ throw new Error("Capacity calibration found no safe request stage.");
1491
1537
  return {
1492
1538
  endpoint: endpoint.toString(),
1493
- recommendedConcurrency: Math.max(1, Math.floor(lastSafe * headroomRatio)),
1539
+ measuredSustainableRequestsPerSecond: lastSafe.successfulRequestsPerSecond,
1540
+ allowedRequestsPerSecond: Math.max(1, Math.floor(lastSafe.successfulRequestsPerSecond * safetyRatio)),
1541
+ safetyRatio,
1542
+ recommendedConcurrency: Math.max(1, Math.floor(lastSafe.concurrency * safetyRatio)),
1494
1543
  stopReason,
1495
1544
  stages
1496
1545
  };
@@ -1567,8 +1616,11 @@ function capacityCalibration(value, where) {
1567
1616
  "requestsPerWorker",
1568
1617
  "maxP95Ms",
1569
1618
  "maxErrorRate",
1570
- "headroomRatio",
1571
- "requestTimeoutMs"
1619
+ "safetyRatio",
1620
+ "requestTimeoutMs",
1621
+ "minimumStageDurationMs",
1622
+ "maxCpuUtilizationPercent",
1623
+ "maxMemoryUtilizationPercent"
1572
1624
  ], where);
1573
1625
  const endpoint = text(calibration.endpoint, `${where}.endpoint`);
1574
1626
  try {
@@ -1592,8 +1644,11 @@ function capacityCalibration(value, where) {
1592
1644
  "requestsPerWorker",
1593
1645
  "maxP95Ms",
1594
1646
  "maxErrorRate",
1595
- "headroomRatio",
1596
- "requestTimeoutMs"
1647
+ "safetyRatio",
1648
+ "requestTimeoutMs",
1649
+ "minimumStageDurationMs",
1650
+ "maxCpuUtilizationPercent",
1651
+ "maxMemoryUtilizationPercent"
1597
1652
  ].flatMap((name) => {
1598
1653
  const found = optionalNumber(name);
1599
1654
  return found === undefined ? [] : [[name, found]];
@@ -2784,35 +2839,76 @@ function requestServiceActivation(request, socketPath = DEFAULT_SOFTWARE_HELPER_
2784
2839
  function requestSoftware(requirements, socketPath = DEFAULT_SOFTWARE_HELPER_SOCKET, timeoutMs = 15 * 60000) {
2785
2840
  validateSoftwareRequirements(requirements);
2786
2841
  return new Promise((resolve5, reject) => {
2787
- const socket = connect3(socketPath, () => socket.write(`${JSON.stringify({ op: "ensure", requirements })}
2788
- `));
2789
- let buffer = "";
2790
- socket.setTimeout(timeoutMs, () => {
2791
- socket.destroy();
2792
- reject(new Error("software helper did not answer before its deadline"));
2793
- });
2794
- socket.on("data", (chunk) => {
2795
- buffer += chunk.toString("utf8");
2796
- const newline = buffer.indexOf(`
2797
- `);
2798
- if (newline < 0)
2842
+ const deadline = Date.now() + timeoutMs;
2843
+ let settled = false;
2844
+ const finish = (cause, results) => {
2845
+ if (settled)
2799
2846
  return;
2800
- socket.end();
2847
+ settled = true;
2848
+ if (cause)
2849
+ reject(cause);
2850
+ else
2851
+ resolve5(results);
2852
+ };
2853
+ const attempt = () => {
2854
+ if (settled)
2855
+ return;
2856
+ const remaining = deadline - Date.now();
2857
+ if (remaining <= 0)
2858
+ return finish(new Error("software helper did not answer before its deadline"));
2859
+ if (!existsSync7(socketPath)) {
2860
+ setTimeout(attempt, Math.min(100, Math.max(1, remaining)));
2861
+ return;
2862
+ }
2863
+ let buffer = "";
2864
+ let socket;
2801
2865
  try {
2802
- const response = JSON.parse(buffer.slice(0, newline));
2803
- if (!response.ok || !response.results)
2804
- throw new Error(response.error?.message ?? "software helper refused the request");
2805
- resolve5(response.results);
2866
+ socket = connect3(socketPath, () => socket.write(`${JSON.stringify({ op: "ensure", requirements })}
2867
+ `));
2806
2868
  } catch (cause) {
2807
- reject(cause);
2869
+ const error = cause;
2870
+ if ((error.code === "ENOENT" || error.code === "ECONNREFUSED") && Date.now() < deadline) {
2871
+ setTimeout(attempt, Math.min(100, Math.max(1, deadline - Date.now())));
2872
+ return;
2873
+ }
2874
+ finish(error);
2875
+ return;
2808
2876
  }
2809
- });
2810
- socket.on("error", reject);
2877
+ socket.setTimeout(remaining, () => {
2878
+ socket.destroy();
2879
+ finish(new Error("software helper did not answer before its deadline"));
2880
+ });
2881
+ socket.on("data", (chunk) => {
2882
+ buffer += chunk.toString("utf8");
2883
+ const newline = buffer.indexOf(`
2884
+ `);
2885
+ if (newline < 0)
2886
+ return;
2887
+ socket.end();
2888
+ try {
2889
+ const response = JSON.parse(buffer.slice(0, newline));
2890
+ if (!response.ok || !response.results)
2891
+ throw new Error(response.error?.message ?? "software helper refused the request");
2892
+ finish(undefined, response.results);
2893
+ } catch (cause) {
2894
+ finish(cause);
2895
+ }
2896
+ });
2897
+ socket.on("error", (cause) => {
2898
+ socket.destroy();
2899
+ if ((cause.code === "ENOENT" || cause.code === "ECONNREFUSED") && Date.now() < deadline) {
2900
+ setTimeout(attempt, Math.min(100, Math.max(1, deadline - Date.now())));
2901
+ return;
2902
+ }
2903
+ finish(cause);
2904
+ });
2905
+ };
2906
+ attempt();
2811
2907
  });
2812
2908
  }
2813
2909
 
2814
2910
  // src/version.ts
2815
- var VERSION3 = "0.1.88";
2911
+ var VERSION3 = "0.1.90";
2816
2912
 
2817
2913
  // src/egress-policy.ts
2818
2914
  import { realpathSync as realpathSync3 } from "node:fs";
@@ -2948,6 +3044,7 @@ function agentEgressUnit(options) {
2948
3044
  }
2949
3045
  const deploymentEnabled = Boolean(options.repository || bootstrapBundleEnabled || options.pullDeployments);
2950
3046
  const runnerLoopbackPorts = normalizeEgressTcpPorts(options.runnerLoopbackPorts ?? []);
3047
+ const agentLoopbackPorts = normalizeEgressTcpPorts(options.agentLoopbackPorts ?? []);
2951
3048
  const runnerPublicTcpPorts = normalizeEgressTcpPorts(options.runnerPublicTcpPorts ?? DEFAULT_RUNNER_PUBLIC_TCP_PORTS);
2952
3049
  if (deploymentEnabled && runnerPublicTcpPorts.length < 1) {
2953
3050
  throw new Error("deployed project runner needs at least one vetted public TCP port");
@@ -2955,8 +3052,10 @@ function agentEgressUnit(options) {
2955
3052
  if (deploymentEnabled)
2956
3053
  systemdAgentEgressDirectives(runnerLoopbackPorts);
2957
3054
  const users = [user, ...deploymentEnabled ? [DEPLOYMENT_RUNNER_USER] : []];
2958
- const runnerGrant = deploymentEnabled ? ` --loopback-user=${DEPLOYMENT_RUNNER_USER}` + runnerLoopbackPorts.map((port) => ` --loopback-tcp-port=${port}`).join("") + runnerPublicTcpPorts.map((port) => ` --public-tcp-port=${port}`).join("") : "";
2959
- const policyProof = `ExecStartPost=${bin} egress-policy-check ${users.map((name) => `--user=${name}`).join(" ")}${runnerGrant}`;
3055
+ const portGrantEnabled = deploymentEnabled || agentLoopbackPorts.length > 0;
3056
+ const restrictedUser = deploymentEnabled ? DEPLOYMENT_RUNNER_USER : user;
3057
+ const portGrant = portGrantEnabled ? ` --loopback-user=${restrictedUser}` + (deploymentEnabled ? runnerLoopbackPorts.map((port) => ` --loopback-tcp-port=${port}`).join("") : "") + agentLoopbackPorts.map((port) => ` --shared-loopback-tcp-port=${port}`).join("") + (deploymentEnabled ? runnerPublicTcpPorts.map((port) => ` --public-tcp-port=${port}`).join("") : "") : "";
3058
+ const policyProof = `ExecStartPost=${bin} egress-policy-check ${users.map((name) => `--user=${name}`).join(" ")}${portGrant}`;
2960
3059
  return `[Unit]
2961
3060
  Description=ForgeZero Agent host egress policy
2962
3061
  Documentation=https://www.forgezero.net/docs/agent
@@ -2969,7 +3068,7 @@ Type=notify
2969
3068
  NotifyAccess=all
2970
3069
  User=root
2971
3070
  Group=root
2972
- ExecStart=${bin} egress-policy ${users.map((name) => `--user=${name}`).join(" ")}${runnerGrant}
3071
+ ExecStart=${bin} egress-policy ${users.map((name) => `--user=${name}`).join(" ")}${portGrant}
2973
3072
  ${policyProof}
2974
3073
  Restart=on-failure
2975
3074
  RestartSec=2
@@ -3253,7 +3352,7 @@ function agentEnrolmentUnit(options) {
3253
3352
  Requires=forgezero-agent-egress.service
3254
3353
  BindsTo=forgezero-agent-egress.service
3255
3354
  ` : "";
3256
- const egressDirectives = options.enforceEgress ? systemdAgentEgressDirectives() : "";
3355
+ const egressDirectives = options.enforceEgress ? systemdAgentEgressDirectives(options.agentLoopbackPorts ?? []) : "";
3257
3356
  return `[Unit]
3258
3357
  Description=Bind this machine to its ForgeZero compute
3259
3358
  After=network-online.target
@@ -3290,7 +3389,7 @@ WantedBy=multi-user.target
3290
3389
  function deploymentRunnerUnit(options) {
3291
3390
  const bin = options.binPath ?? "fz-agent";
3292
3391
  const root = options.deployRoot ?? "/opt/forgezero";
3293
- const agentUser = options.user ?? "forgezero-agent";
3392
+ const agentUser = options.user ?? "forgezero";
3294
3393
  const egressDependency = options.enforceEgress ? `After=forgezero-agent-egress.service
3295
3394
  Requires=forgezero-agent-egress.service
3296
3395
  BindsTo=forgezero-agent-egress.service
@@ -3331,7 +3430,7 @@ RestrictRealtime=true
3331
3430
  MemoryDenyWriteExecute=true
3332
3431
  LockPersonality=true
3333
3432
  ${egressDirectives}
3334
- ReadWritePaths=${root}/releases ${root}/runner-home
3433
+ ReadWritePaths=${root}/releases ${root}/runner-home ${root}/slots /etc/nginx/conf.d /var/log/nginx
3335
3434
 
3336
3435
  [Install]
3337
3436
  WantedBy=multi-user.target
@@ -3341,7 +3440,7 @@ function agentUnit(options) {
3341
3440
  if (!validNodeHostname(options.nodeHostname))
3342
3441
  throw new Error("node hostname is invalid");
3343
3442
  if (!options.telemetryEndpoint) {
3344
- throw new Error("compute Agent provisioning requires OTEL_EXPORTER_OTLP_ENDPOINT as an explicit public HTTPS collector coordinate");
3443
+ throw new Error("compute Agent provisioning requires an explicit supervised OTLP collector coordinate");
3345
3444
  }
3346
3445
  let telemetryEndpoint;
3347
3446
  {
@@ -3349,10 +3448,13 @@ function agentUnit(options) {
3349
3448
  try {
3350
3449
  endpoint = new URL(options.telemetryEndpoint);
3351
3450
  } catch {
3352
- throw new Error("compute telemetry endpoint must be an absolute public HTTPS URL");
3451
+ throw new Error("compute telemetry endpoint must be an absolute collector URL");
3452
+ }
3453
+ const localCollector = endpoint.protocol === "http:" && endpoint.hostname === "127.0.0.1" && endpoint.port === "4318" && endpoint.pathname === "/";
3454
+ const publicCollector = endpoint.protocol === "https:" && !endpoint.username && !endpoint.password && !endpoint.search && !endpoint.hash && isIP(endpoint.hostname) === 0 && endpoint.hostname.includes(".") && endpoint.hostname !== "localhost" && !endpoint.hostname.endsWith(".local");
3455
+ if (!localCollector && !publicCollector || endpoint.username || endpoint.password || endpoint.search || endpoint.hash) {
3456
+ throw new Error("compute telemetry endpoint must be the supervised loopback collector or credential-free public HTTPS");
3353
3457
  }
3354
- if (endpoint.protocol !== "https:" || endpoint.username || endpoint.password || endpoint.search || endpoint.hash || isIP(endpoint.hostname) !== 0 || !endpoint.hostname.includes(".") || endpoint.hostname === "localhost" || endpoint.hostname.endsWith(".local"))
3355
- throw new Error("compute telemetry endpoint must be an absolute public HTTPS URL without credentials, query or fragment");
3356
3458
  telemetryEndpoint = endpoint.toString().replace(/\/$/, "");
3357
3459
  }
3358
3460
  const bin = options.binPath ?? "fz-agent";
@@ -3466,7 +3568,7 @@ function agentUnit(options) {
3466
3568
  const supplementaryGroups = [
3467
3569
  AGENT_UPDATE_GROUP,
3468
3570
  deploymentEnabled ? DEPLOYMENT_GROUP : null,
3469
- deploymentEnabled ? SOFTWARE_HELPER_GROUP : null,
3571
+ SOFTWARE_HELPER_GROUP,
3470
3572
  lifecycleEnabled ? LIFECYCLE_GROUP : null
3471
3573
  ].filter((value) => value !== null);
3472
3574
  const deploymentGroup = supplementaryGroups.length > 0 ? `SupplementaryGroups=${supplementaryGroups.join(" ")}` : "";
@@ -3475,7 +3577,7 @@ function agentUnit(options) {
3475
3577
  "forgezero-agent-update-helper.service",
3476
3578
  options.enforceEgress ? "forgezero-agent-egress.service" : null,
3477
3579
  deploymentEnabled ? "forgezero-deploy-runner.service" : null,
3478
- deploymentEnabled ? "forgezero-software-helper.service" : null,
3580
+ "forgezero-software-helper.service",
3479
3581
  lifecycleEnabled ? "forgezero-lifecycle-helper.service" : null,
3480
3582
  warpEnabled ? "warp-svc.service" : null,
3481
3583
  enrolmentEnabled ? "forgezero-agent-enrol.service" : null
@@ -3484,7 +3586,7 @@ function agentUnit(options) {
3484
3586
  "forgezero-agent-update-helper.service",
3485
3587
  options.enforceEgress ? "forgezero-agent-egress.service" : null,
3486
3588
  deploymentEnabled ? "forgezero-deploy-runner.service" : null,
3487
- deploymentEnabled ? "forgezero-software-helper.service" : null,
3589
+ "forgezero-software-helper.service",
3488
3590
  lifecycleEnabled ? "forgezero-lifecycle-helper.service" : null,
3489
3591
  warpEnabled ? "warp-svc.service" : null,
3490
3592
  enrolmentEnabled ? "forgezero-agent-enrol.service" : null
@@ -3499,9 +3601,9 @@ function agentUnit(options) {
3499
3601
  const snpDevice = options.mode === "attested" ? `DevicePolicy=closed
3500
3602
  DeviceAllow=/dev/sev-guest rw` : "";
3501
3603
  const snpPrepare = options.mode === "attested" ? `ExecStartPre=+/bin/chgrp ${VAULT_GROUP} /dev/sev-guest
3502
- ExecStartPre=+/bin/chmod 0640 /dev/sev-guest
3604
+ ExecStartPre=+/bin/chmod 0660 /dev/sev-guest
3503
3605
  ` : "";
3504
- const egressDirectives = options.enforceEgress ? systemdAgentEgressDirectives() : "";
3606
+ const egressDirectives = options.enforceEgress ? systemdAgentEgressDirectives(options.agentLoopbackPorts ?? []) : "";
3505
3607
  return `[Unit]
3506
3608
  Description=ForgeZero node agent (${options.mode})
3507
3609
  Documentation=https://www.forgezero.net/docs/agent
@@ -3632,6 +3734,7 @@ function planProvision(options) {
3632
3734
  const deployRoot = options.deployRoot ?? "/opt/forgezero";
3633
3735
  const deploymentEnabled = Boolean(options.repository || options.pullDeployments);
3634
3736
  const runnerLoopbackPorts = normalizeEgressTcpPorts(options.runnerLoopbackPorts ?? []);
3737
+ const agentLoopbackPorts = normalizeEgressTcpPorts(options.agentLoopbackPorts ?? []);
3635
3738
  const runnerPublicTcpPorts = normalizeEgressTcpPorts(options.runnerPublicTcpPorts ?? DEFAULT_RUNNER_PUBLIC_TCP_PORTS);
3636
3739
  const lifecycleEnabled = Boolean(options.pullMigrations && options.lifecycleProfilePath);
3637
3740
  if (Boolean(options.pullMigrations) !== Boolean(options.lifecycleProfilePath)) {
@@ -3680,8 +3783,9 @@ function planProvision(options) {
3680
3783
  const enabledUnits = [
3681
3784
  "forgezero-agent.socket",
3682
3785
  "forgezero-agent-update-helper.service",
3786
+ "forgezero-software-helper.service",
3683
3787
  ...options.enforceEgress ? ["forgezero-agent-egress.service"] : [],
3684
- ...deploymentEnabled ? ["forgezero-deploy-runner.service", "forgezero-software-helper.service"] : [],
3788
+ ...deploymentEnabled ? ["forgezero-deploy-runner.service"] : [],
3685
3789
  ...lifecycleEnabled ? ["forgezero-lifecycle-helper.service"] : [],
3686
3790
  ...warpEnabled ? ["forgezero-warp-config.service", "warp-svc.service"] : [],
3687
3791
  ...enrolmentEnabled ? ["forgezero-agent-enrol.service"] : [],
@@ -3689,8 +3793,9 @@ function planProvision(options) {
3689
3793
  ];
3690
3794
  const restartedUnits = [
3691
3795
  "forgezero-agent-update-helper.service",
3796
+ "forgezero-software-helper.service",
3692
3797
  ...options.enforceEgress ? ["forgezero-agent-egress.service"] : [],
3693
- ...deploymentEnabled ? ["forgezero-deploy-runner.service", "forgezero-software-helper.service"] : [],
3798
+ ...deploymentEnabled ? ["forgezero-deploy-runner.service"] : [],
3694
3799
  ...lifecycleEnabled ? ["forgezero-lifecycle-helper.service"] : [],
3695
3800
  ...warpEnabled ? ["forgezero-warp-config.service", "warp-svc.service"] : [],
3696
3801
  ...enrolmentEnabled ? ["forgezero-agent-enrol.service"] : []
@@ -3714,9 +3819,9 @@ function planProvision(options) {
3714
3819
  ...options.enforceEgress ? [
3715
3820
  { path: AGENT_EGRESS_UNIT_PATH, unit: agentEgressUnit(options) }
3716
3821
  ] : [],
3822
+ { path: SOFTWARE_HELPER_UNIT_PATH, unit: softwareHelperUnit(options) },
3717
3823
  ...deploymentEnabled ? [
3718
- { path: DEPLOYMENT_RUNNER_UNIT_PATH, unit: deploymentRunnerUnit(options) },
3719
- { path: SOFTWARE_HELPER_UNIT_PATH, unit: softwareHelperUnit(options) }
3824
+ { path: DEPLOYMENT_RUNNER_UNIT_PATH, unit: deploymentRunnerUnit(options) }
3720
3825
  ] : [],
3721
3826
  ...enrolmentEnabled ? [
3722
3827
  { path: ENROLMENT_UNIT_PATH, unit: agentEnrolmentUnit(options) }
@@ -3749,24 +3854,29 @@ function planProvision(options) {
3749
3854
  step("Agent update helper access group", { kind: "commands", commands: [{ argv: ["/usr/sbin/groupadd", "--system", AGENT_UPDATE_GROUP], acceptedExitCodes: [0, 9] }] }),
3750
3855
  ...sourceBinPath && binPath ? [step("root-owned agent runtime", { kind: "install-runtime", source: sourceBinPath, binary: binPath, version: VERSION3 })] : [],
3751
3856
  ...warpEnabled ? [step("Cloudflare One client for Ubuntu 26.04", { kind: "install-warp" })] : [],
3752
- ...deploymentEnabled ? [step("deployment isolation group", { kind: "commands", commands: [
3753
- { argv: ["/usr/sbin/groupadd", "--system", DEPLOYMENT_GROUP], acceptedExitCodes: [0, 9] },
3857
+ step("software strategy helper group", { kind: "commands", commands: [
3754
3858
  { argv: ["/usr/sbin/groupadd", "--system", SOFTWARE_HELPER_GROUP], acceptedExitCodes: [0, 9] }
3859
+ ] }),
3860
+ ...deploymentEnabled ? [step("deployment isolation group", { kind: "commands", commands: [
3861
+ { argv: ["/usr/sbin/groupadd", "--system", DEPLOYMENT_GROUP], acceptedExitCodes: [0, 9] }
3755
3862
  ] })] : [],
3756
3863
  ...lifecycleEnabled ? [step("lifecycle helper access group", { kind: "commands", commands: [{ argv: ["/usr/sbin/groupadd", "--system", LIFECYCLE_GROUP], acceptedExitCodes: [0, 9] }] })] : [],
3757
3864
  step("service account", { kind: "commands", commands: [{ argv: ["/usr/sbin/useradd", "--system", "--no-create-home", "--shell", "/usr/sbin/nologin", user], acceptedExitCodes: [0, 9] }] }),
3758
3865
  step("bind service account to vault group", { kind: "commands", commands: [{ argv: ["/usr/sbin/usermod", "-g", VAULT_GROUP, user] }] }),
3759
3866
  step("grant verified Agent update access", { kind: "commands", commands: [{ argv: ["/usr/sbin/usermod", "-a", "-G", AGENT_UPDATE_GROUP, user] }] }),
3867
+ step("grant software helper socket access", { kind: "commands", commands: [
3868
+ { argv: ["/usr/sbin/usermod", "-a", "-G", SOFTWARE_HELPER_GROUP, user] }
3869
+ ] }),
3760
3870
  ...lifecycleEnabled ? [step("grant lifecycle helper socket access", { kind: "commands", commands: [{ argv: ["/usr/sbin/usermod", "-a", "-G", LIFECYCLE_GROUP, user] }] })] : [],
3761
3871
  ...deploymentEnabled ? [step("credential-free deployment account", { kind: "commands", commands: [
3762
3872
  { argv: ["/usr/sbin/useradd", "--system", "--no-create-home", "--shell", "/usr/sbin/nologin", "--gid", DEPLOYMENT_GROUP, DEPLOYMENT_RUNNER_USER], acceptedExitCodes: [0, 9] },
3763
3873
  { argv: ["/usr/sbin/useradd", "--system", "--no-create-home", "--shell", "/usr/sbin/nologin", "--gid", VAULT_GROUP, APPLICATION_RUNTIME_USER], acceptedExitCodes: [0, 9] },
3764
3874
  { argv: ["/usr/sbin/usermod", "-a", "-G", DEPLOYMENT_GROUP, APPLICATION_RUNTIME_USER] },
3765
- { argv: ["/usr/sbin/usermod", "-a", "-G", `${DEPLOYMENT_GROUP},${SOFTWARE_HELPER_GROUP}`, user] }
3875
+ { argv: ["/usr/sbin/usermod", "-a", "-G", DEPLOYMENT_GROUP, user] }
3766
3876
  ] })] : [],
3767
3877
  step("credential and state directories", { kind: "directories", directories: [
3768
3878
  { path: credentialDir, mode: 448, owner: "root", group: "root" },
3769
- { path: "/var/lib/forgezero", mode: 488, owner: "root", group: "root" }
3879
+ { path: "/var/lib/forgezero", mode: 488, owner: "root", group: VAULT_GROUP }
3770
3880
  ] }),
3771
3881
  step("encrypted node identity", { kind: "ensure-seed", credential: seedCredentialPath }),
3772
3882
  ...options.generateGitIdentity && gitCredentialPath && gitPublicKeyPath ? [
@@ -3817,19 +3927,26 @@ function planProvision(options) {
3817
3927
  { argv: ["/usr/bin/systemctl", "enable", ...enabledUnits] },
3818
3928
  { argv: ["/usr/bin/systemctl", "reset-failed", "forgezero-agent.service"], acceptedExitCodes: [0, 1] },
3819
3929
  ...restartedUnits.length ? [{ argv: ["/usr/bin/systemctl", "restart", ...restartedUnits] }] : [],
3930
+ { argv: ["/usr/bin/systemctl", "stop", "forgezero-agent-proxy.service"], acceptedExitCodes: [0, 1, 5] },
3820
3931
  { argv: ["/usr/bin/systemctl", "restart", "forgezero-agent.socket"] },
3821
3932
  { argv: ["/usr/bin/systemctl", "reset-failed", "forgezero-agent.service"], acceptedExitCodes: [0, 1] },
3822
3933
  { argv: ["/usr/bin/systemctl", "restart", "forgezero-agent.service"] }
3823
3934
  ] }),
3824
3935
  ...enrolmentEnabled ? [step("prove the compute binding is durable", { kind: "verify-file", path: enrolStatePath })] : [],
3825
- ...options.enforceEgress ? [step("prove the Agent egress policy is active", { kind: "verify-egress", runnerPublicTcpPorts, runnerLoopbackPorts })] : [],
3936
+ ...options.enforceEgress ? [step("prove the Agent egress policy is active", {
3937
+ kind: "verify-egress",
3938
+ deploymentEnabled,
3939
+ runnerPublicTcpPorts,
3940
+ runnerLoopbackPorts,
3941
+ agentLoopbackPorts
3942
+ })] : [],
3826
3943
  step("prove it is running", { kind: "commands", commands: [{ argv: ["/usr/bin/systemctl", "is-active", "forgezero-agent.service"] }] }),
3827
3944
  step("prove the public Vault socket exists", { kind: "wait-socket", path: options.socketPath, attempts: 100, intervalMs: 100 }),
3828
3945
  step("prove the Agent Vault backend exists", { kind: "wait-socket", path: agentBackendSocketPath(options.socketPath), attempts: 100, intervalMs: 100 }),
3829
3946
  step("prove the Agent update helper exists", { kind: "wait-socket", path: DEFAULT_AGENT_UPDATE_SOCKET, attempts: 100, intervalMs: 100 }),
3947
+ step("prove the software strategy helper socket exists", { kind: "wait-socket", path: DEFAULT_SOFTWARE_HELPER_SOCKET, attempts: 100, intervalMs: 100 }),
3830
3948
  ...deploymentEnabled ? [
3831
- step("prove the deployment runner socket exists", { kind: "wait-socket", path: DEPLOYMENT_RUNNER_SOCKET, attempts: 100, intervalMs: 100 }),
3832
- step("prove the software strategy helper socket exists", { kind: "wait-socket", path: DEFAULT_SOFTWARE_HELPER_SOCKET, attempts: 100, intervalMs: 100 })
3949
+ step("prove the deployment runner socket exists", { kind: "wait-socket", path: DEPLOYMENT_RUNNER_SOCKET, attempts: 100, intervalMs: 100 })
3833
3950
  ] : [],
3834
3951
  ...lifecycleEnabled ? [step("prove the lifecycle helper socket exists", { kind: "wait-socket", path: lifecycleHelperSocketPath, attempts: 100, intervalMs: 100 })] : [],
3835
3952
  ...warpEnabled ? [step("prove Cloudflare WARP is connected", { kind: "verify-warp" })] : [],
@@ -14,7 +14,7 @@ var checked = async (run, argv, expected = 0) => {
14
14
  return result;
15
15
  };
16
16
  var activeSlot = () => {
17
- const slot = readFileSync("/opt/forgezero/.forge-slot", "utf8").trim();
17
+ const slot = readFileSync("/opt/forgezero/slots/.active", "utf8").trim();
18
18
  if (slot !== "blue" && slot !== "green")
19
19
  throw new Error("invalid active slot");
20
20
  return slot;