@forgezero/agent 0.1.88 → 0.1.89

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/fz-agent.js CHANGED
@@ -4,7 +4,7 @@
4
4
  // src/index.ts
5
5
  import { randomBytes as randomBytes7 } from "crypto";
6
6
  import { readFileSync as readFileSync20, writeFileSync as writeFileSync18, existsSync as existsSync23, mkdirSync as mkdirSync18, chmodSync as chmodSync19, lstatSync as lstatSync10, statfsSync as statfsSync3 } from "fs";
7
- import { cpus as cpus2, totalmem as totalmem2 } from "os";
7
+ import { cpus as cpus3, totalmem as totalmem3 } from "os";
8
8
  import { dirname as dirname16, join as join12 } from "path";
9
9
 
10
10
  // ../access/dist/security.js
@@ -5351,7 +5351,12 @@ async function executeSoftwareOperation(operation) {
5351
5351
  }
5352
5352
  if (software === "nginx") {
5353
5353
  const binary = await run(["/usr/sbin/nginx", "-v"]);
5354
- return binary.exitCode === 0 ? run(["/usr/bin/systemctl", "is-active", "--quiet", "nginx.service"]) : binary;
5354
+ if (binary.exitCode !== 0)
5355
+ return binary;
5356
+ if (!existsSync2("/usr/lib/nginx/modules/ngx_http_headers_more_filter_module.so")) {
5357
+ return { exitCode: 1, output: "Nginx response-header suppression module is unavailable" };
5358
+ }
5359
+ return run(["/usr/bin/systemctl", "is-active", "--quiet", "nginx.service"]);
5355
5360
  }
5356
5361
  if (software === "arangodb") {
5357
5362
  const binary = await run(["/usr/sbin/arangod", "--version"]);
@@ -5383,7 +5388,7 @@ async function executeSoftwareOperation(operation) {
5383
5388
  }
5384
5389
  if (software === "docker" || software === "nginx" || software === "ufw" || software === "openssh-client" || software === "git") {
5385
5390
  const packageName = software === "openssh-client" ? "openssh-client" : software === "docker" ? "docker.io" : software;
5386
- const installed = await aptInstall(packageName);
5391
+ const installed = software === "nginx" ? await aptInstallMany(["nginx", "libnginx-mod-http-headers-more-filter"]) : await aptInstall(packageName);
5387
5392
  if (installed.exitCode !== 0 || software !== "nginx" && software !== "docker")
5388
5393
  return installed;
5389
5394
  if (software === "docker") {
@@ -5535,11 +5540,22 @@ async function ensureSoftwareRequirements(requirementsInput, options) {
5535
5540
  }
5536
5541
 
5537
5542
  // src/capacity-calibration.ts
5538
- var percentile95 = (values) => {
5543
+ import { cpus, freemem, totalmem } from "os";
5544
+ var percentile = (values, quantile) => {
5539
5545
  if (values.length === 0)
5540
5546
  return Number.POSITIVE_INFINITY;
5541
5547
  const sorted = values.toSorted((left, right) => left - right);
5542
- return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * 0.95) - 1)];
5548
+ return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * quantile) - 1)];
5549
+ };
5550
+ var hostSample = () => {
5551
+ let cpuIdle = 0;
5552
+ let cpuTotal = 0;
5553
+ for (const cpu of cpus()) {
5554
+ cpuIdle += cpu.times.idle;
5555
+ cpuTotal += Object.values(cpu.times).reduce((sum, value) => sum + value, 0);
5556
+ }
5557
+ const memoryTotal = totalmem();
5558
+ return { cpuIdle, cpuTotal, memoryUsed: memoryTotal - freemem(), memoryTotal };
5543
5559
  };
5544
5560
  function localCalibrationEndpoint(value) {
5545
5561
  const endpoint = new URL(value);
@@ -5554,9 +5570,12 @@ function validateCapacityCalibrationOptions(options) {
5554
5570
  const requestsPerWorker = options.requestsPerWorker ?? 8;
5555
5571
  const maxP95Ms = options.maxP95Ms ?? 250;
5556
5572
  const maxErrorRate = options.maxErrorRate ?? 0.01;
5557
- const headroomRatio = options.headroomRatio ?? 0.8;
5573
+ const safetyRatio = options.safetyRatio ?? 0.8;
5558
5574
  const requestTimeoutMs = options.requestTimeoutMs ?? 5000;
5559
- if (!Number.isSafeInteger(maxConcurrency) || maxConcurrency < 1 || maxConcurrency > 4096 || !Number.isSafeInteger(requestsPerWorker) || requestsPerWorker < 2 || requestsPerWorker > 100 || !Number.isFinite(maxP95Ms) || maxP95Ms < 1 || !Number.isFinite(maxErrorRate) || maxErrorRate < 0 || maxErrorRate > 0.2 || !Number.isFinite(headroomRatio) || headroomRatio < 0.25 || headroomRatio > 0.95 || !Number.isSafeInteger(requestTimeoutMs) || requestTimeoutMs < 100 || requestTimeoutMs > 30000) {
5575
+ const minimumStageDurationMs = options.minimumStageDurationMs ?? 5000;
5576
+ const maxCpuUtilizationPercent = options.maxCpuUtilizationPercent ?? 90;
5577
+ const maxMemoryUtilizationPercent = options.maxMemoryUtilizationPercent ?? 90;
5578
+ if (!Number.isSafeInteger(maxConcurrency) || maxConcurrency < 1 || maxConcurrency > 4096 || !Number.isSafeInteger(requestsPerWorker) || requestsPerWorker < 2 || requestsPerWorker > 100 || !Number.isFinite(maxP95Ms) || maxP95Ms < 1 || !Number.isFinite(maxErrorRate) || maxErrorRate < 0 || maxErrorRate > 0.2 || !Number.isFinite(safetyRatio) || safetyRatio < 0.25 || safetyRatio > 0.95 || !Number.isSafeInteger(requestTimeoutMs) || requestTimeoutMs < 100 || requestTimeoutMs > 30000 || !Number.isSafeInteger(minimumStageDurationMs) || minimumStageDurationMs < 100 || minimumStageDurationMs > 60000 || !Number.isFinite(maxCpuUtilizationPercent) || maxCpuUtilizationPercent < 25 || maxCpuUtilizationPercent > 100 || !Number.isFinite(maxMemoryUtilizationPercent) || maxMemoryUtilizationPercent < 25 || maxMemoryUtilizationPercent > 100) {
5560
5579
  throw new Error("Capacity calibration bounds are invalid.");
5561
5580
  }
5562
5581
  return {
@@ -5565,18 +5584,24 @@ function validateCapacityCalibrationOptions(options) {
5565
5584
  requestsPerWorker,
5566
5585
  maxP95Ms,
5567
5586
  maxErrorRate,
5568
- headroomRatio,
5569
- requestTimeoutMs
5587
+ safetyRatio,
5588
+ requestTimeoutMs,
5589
+ minimumStageDurationMs,
5590
+ maxCpuUtilizationPercent,
5591
+ maxMemoryUtilizationPercent
5570
5592
  };
5571
5593
  }
5572
- async function stage(endpoint, concurrency, requestsPerWorker, timeoutMs, fetcher) {
5594
+ async function stage(endpoint, concurrency, requestsPerWorker, timeoutMs, minimumDurationMs, fetcher, sampler) {
5573
5595
  const latencies = [];
5574
5596
  let succeeded = 0;
5575
5597
  let failed = 0;
5576
5598
  let overloaded = 0;
5599
+ const systemBefore = sampler();
5577
5600
  const started = performance.now();
5578
5601
  await Promise.all(Array.from({ length: concurrency }, async () => {
5579
- for (let request = 0;request < requestsPerWorker; request += 1) {
5602
+ let request = 0;
5603
+ while (request < requestsPerWorker || performance.now() - started < minimumDurationMs) {
5604
+ request += 1;
5580
5605
  const requestStarted = performance.now();
5581
5606
  try {
5582
5607
  const response = await fetcher(endpoint, {
@@ -5600,50 +5625,74 @@ async function stage(endpoint, concurrency, requestsPerWorker, timeoutMs, fetche
5600
5625
  }
5601
5626
  }
5602
5627
  }));
5603
- const elapsedSeconds = Math.max((performance.now() - started) / 1000, 0.001);
5628
+ const durationMs = Math.max(performance.now() - started, 1);
5629
+ const elapsedSeconds = durationMs / 1000;
5630
+ const systemAfter = sampler();
5631
+ const cpuTotal = Math.max(0, systemAfter.cpuTotal - systemBefore.cpuTotal);
5632
+ const cpuIdle = Math.max(0, systemAfter.cpuIdle - systemBefore.cpuIdle);
5633
+ const cpuUtilizationPercent = cpuTotal === 0 ? 0 : (cpuTotal - cpuIdle) / cpuTotal * 100;
5634
+ const memoryUtilizationPercent = Math.max(systemBefore.memoryTotal > 0 ? systemBefore.memoryUsed / systemBefore.memoryTotal * 100 : 0, systemAfter.memoryTotal > 0 ? systemAfter.memoryUsed / systemAfter.memoryTotal * 100 : 0);
5635
+ const requests = succeeded + failed;
5604
5636
  return {
5605
5637
  concurrency,
5606
- requests: concurrency * requestsPerWorker,
5638
+ requests,
5607
5639
  succeeded,
5608
5640
  failed,
5609
5641
  overloaded,
5610
- throughputPerSecond: Number(((succeeded + failed) / elapsedSeconds).toFixed(2)),
5611
- p95Ms: Number(percentile95(latencies).toFixed(2))
5642
+ successfulRequestsPerSecond: Number((succeeded / elapsedSeconds).toFixed(2)),
5643
+ attemptedRequestsPerSecond: Number((requests / elapsedSeconds).toFixed(2)),
5644
+ durationMs: Number(durationMs.toFixed(2)),
5645
+ errorRate: Number((failed / Math.max(requests, 1)).toFixed(6)),
5646
+ p95Ms: Number(percentile(latencies, 0.95).toFixed(2)),
5647
+ p99Ms: Number(percentile(latencies, 0.99).toFixed(2)),
5648
+ cpuUtilizationPercent: Number(cpuUtilizationPercent.toFixed(2)),
5649
+ memoryUtilizationPercent: Number(memoryUtilizationPercent.toFixed(2))
5612
5650
  };
5613
5651
  }
5614
- async function calibrateHttpConcurrency(options, fetcher = fetch) {
5652
+ async function calibrateHttpConcurrency(options, fetcher = fetch, sampler = hostSample) {
5615
5653
  const {
5616
5654
  endpoint,
5617
5655
  maxConcurrency,
5618
5656
  requestsPerWorker,
5619
5657
  maxP95Ms,
5620
5658
  maxErrorRate,
5621
- headroomRatio,
5622
- requestTimeoutMs: timeoutMs
5659
+ safetyRatio,
5660
+ requestTimeoutMs: timeoutMs,
5661
+ minimumStageDurationMs,
5662
+ maxCpuUtilizationPercent,
5663
+ maxMemoryUtilizationPercent
5623
5664
  } = validateCapacityCalibrationOptions(options);
5624
5665
  const stages = [];
5625
- let lastSafe = 1;
5666
+ let lastSafe;
5626
5667
  let stopReason = "maximum-tested";
5627
5668
  for (let concurrency = 1;; concurrency = Math.min(maxConcurrency, concurrency * 2)) {
5628
- const measured = await stage(endpoint, concurrency, requestsPerWorker, timeoutMs, fetcher);
5669
+ const measured = await stage(endpoint, concurrency, requestsPerWorker, timeoutMs, minimumStageDurationMs, fetcher, sampler);
5629
5670
  stages.push(measured);
5630
- const errorRate = measured.failed / measured.requests;
5631
5671
  const previous = stages.at(-2);
5632
- const throughputRegressed = Boolean(previous && concurrency > 1 && measured.throughputPerSecond < previous.throughputPerSecond * 0.9);
5633
- if (measured.overloaded > 0 || errorRate > maxErrorRate)
5672
+ const throughputRegressed = Boolean(previous && concurrency > 1 && measured.successfulRequestsPerSecond < previous.successfulRequestsPerSecond * 0.9);
5673
+ if (measured.overloaded > 0 || measured.errorRate > maxErrorRate)
5634
5674
  stopReason = "errors";
5635
5675
  else if (measured.p95Ms > maxP95Ms)
5636
5676
  stopReason = "latency";
5677
+ else if (measured.cpuUtilizationPercent > maxCpuUtilizationPercent)
5678
+ stopReason = "cpu";
5679
+ else if (measured.memoryUtilizationPercent > maxMemoryUtilizationPercent)
5680
+ stopReason = "memory";
5637
5681
  else if (throughputRegressed)
5638
5682
  stopReason = "throughput-regression";
5639
5683
  else
5640
- lastSafe = concurrency;
5684
+ lastSafe = measured;
5641
5685
  if (stopReason !== "maximum-tested" || concurrency === maxConcurrency)
5642
5686
  break;
5643
5687
  }
5688
+ if (!lastSafe)
5689
+ throw new Error("Capacity calibration found no safe request stage.");
5644
5690
  return {
5645
5691
  endpoint: endpoint.toString(),
5646
- recommendedConcurrency: Math.max(1, Math.floor(lastSafe * headroomRatio)),
5692
+ measuredSustainableRequestsPerSecond: lastSafe.successfulRequestsPerSecond,
5693
+ allowedRequestsPerSecond: Math.max(1, Math.floor(lastSafe.successfulRequestsPerSecond * safetyRatio)),
5694
+ safetyRatio,
5695
+ recommendedConcurrency: Math.max(1, Math.floor(lastSafe.concurrency * safetyRatio)),
5647
5696
  stopReason,
5648
5697
  stages
5649
5698
  };
@@ -5720,8 +5769,11 @@ function capacityCalibration(value, where) {
5720
5769
  "requestsPerWorker",
5721
5770
  "maxP95Ms",
5722
5771
  "maxErrorRate",
5723
- "headroomRatio",
5724
- "requestTimeoutMs"
5772
+ "safetyRatio",
5773
+ "requestTimeoutMs",
5774
+ "minimumStageDurationMs",
5775
+ "maxCpuUtilizationPercent",
5776
+ "maxMemoryUtilizationPercent"
5725
5777
  ], where);
5726
5778
  const endpoint = text(calibration.endpoint, `${where}.endpoint`);
5727
5779
  try {
@@ -5745,8 +5797,11 @@ function capacityCalibration(value, where) {
5745
5797
  "requestsPerWorker",
5746
5798
  "maxP95Ms",
5747
5799
  "maxErrorRate",
5748
- "headroomRatio",
5749
- "requestTimeoutMs"
5800
+ "safetyRatio",
5801
+ "requestTimeoutMs",
5802
+ "minimumStageDurationMs",
5803
+ "maxCpuUtilizationPercent",
5804
+ "maxMemoryUtilizationPercent"
5750
5805
  ].flatMap((name) => {
5751
5806
  const found = optionalNumber(name);
5752
5807
  return found === undefined ? [] : [[name, found]];
@@ -7867,6 +7922,9 @@ function parseBootstrapBundleManifest(value) {
7867
7922
  }
7868
7923
 
7869
7924
  // src/deployment.ts
7925
+ var RELEASE_GROUP = "forgezero-deploy";
7926
+ var deploymentAttestationChallenge = (revision) => createHash5("sha256").update("forgezero/deployment-attestation/v1\x00").update(revision, "utf8").digest("hex");
7927
+
7870
7928
  class DeploymentError extends Error {
7871
7929
  code;
7872
7930
  constructor(code, message) {
@@ -7926,6 +7984,7 @@ function persistCapacityCalibration(args) {
7926
7984
  const path2 = join2(args.directory, `${identity}-${args.measuredAtTs}-${randomUUID().slice(0, 8)}.json`);
7927
7985
  const next = `${path2}.next`;
7928
7986
  const recommendedCoordinate = {
7987
+ FZ_REQUESTS_PER_SECOND_LIMIT: String(args.calibration.allowedRequestsPerSecond),
7929
7988
  FZ_CONCURRENCY_LIMIT: String(args.calibration.recommendedConcurrency)
7930
7989
  };
7931
7990
  writeFileSync3(next, `${JSON.stringify({
@@ -7961,6 +8020,10 @@ function createDeploymentManager(options) {
7961
8020
  const activeRevisions = new Map;
7962
8021
  const exec = options.exec ?? spawnExact;
7963
8022
  const projectExec = options.projectExec ?? exec;
8023
+ const releaseGroup = options.releaseGroup ?? RELEASE_GROUP;
8024
+ if (!/^(?:[a-z_][a-z0-9_-]{0,30}|[0-9]{1,10})$/.test(releaseGroup)) {
8025
+ throw new DeploymentError("SOURCE_FAILED", "release group is invalid");
8026
+ }
7964
8027
  const now = options.now ?? Date.now;
7965
8028
  const readDefinition = options.readDefinition ?? ((path2) => JSON.parse(readFileSync3(path2, "utf8")));
7966
8029
  const credentialsDirectory = process.env.CREDENTIALS_DIRECTORY;
@@ -8100,7 +8163,6 @@ function createDeploymentManager(options) {
8100
8163
  const env = await gitEnvironment();
8101
8164
  const releasesDirectory = join2(options.root, "releases");
8102
8165
  await checked({ argv: ["/usr/bin/test", "-d", releasesDirectory] }, "SOURCE_FAILED");
8103
- await checked({ argv: ["/usr/bin/test", "-w", releasesDirectory] }, "SOURCE_FAILED");
8104
8166
  await checked({ argv: ["/usr/bin/install", "-d", "-m", "0770", release] }, "SOURCE_FAILED");
8105
8167
  if (bootstrapBundle) {
8106
8168
  let metadata;
@@ -8112,7 +8174,14 @@ function createDeploymentManager(options) {
8112
8174
  if (!metadata.isFile() || metadata.isSymbolicLink() || metadata.nlink !== 1 || metadata.size !== bootstrapBundle.manifest.bytes || await sha256File(bootstrapBundle.path) !== bootstrapBundle.manifest.sha256) {
8113
8175
  throw new DeploymentError("SOURCE_FAILED", "The attended bootstrap bundle failed its manifest check.");
8114
8176
  }
8115
- await checked({ argv: ["git", "bundle", "verify", bootstrapBundle.path], env }, "SOURCE_FAILED");
8177
+ const verifier = `${release}.bundle-verify`;
8178
+ try {
8179
+ await checked({ argv: ["/usr/bin/install", "-d", "-m", "0700", verifier] }, "SOURCE_FAILED");
8180
+ await checked({ argv: ["git", "-C", verifier, "init", "--bare", "--quiet"], env }, "SOURCE_FAILED");
8181
+ await checked({ argv: ["git", "-C", verifier, "bundle", "verify", bootstrapBundle.path], env }, "SOURCE_FAILED");
8182
+ } finally {
8183
+ await exec({ argv: ["/usr/bin/rm", "-rf", "--", verifier] });
8184
+ }
8116
8185
  }
8117
8186
  await checked({
8118
8187
  argv: ["git", "clone", "--quiet", "--no-checkout", "--branch", options.branch, "--depth", "100", options.repository, release],
@@ -8150,6 +8219,10 @@ function createDeploymentManager(options) {
8150
8219
  cwd: release,
8151
8220
  env
8152
8221
  }, "SOURCE_FAILED");
8222
+ if (options.projectExec) {
8223
+ await checked({ argv: ["/bin/chgrp", "-hR", releaseGroup, release] }, "SOURCE_FAILED");
8224
+ await checked({ argv: ["/bin/chmod", "-R", "g+rX,o-rwx", release] }, "SOURCE_FAILED");
8225
+ }
8153
8226
  const head = (await checked({ argv: ["git", "rev-parse", "HEAD"], cwd: release, env }, "SOURCE_FAILED")).output.trim();
8154
8227
  if ((request.revision || bootstrapBundle) && head !== desiredRevision) {
8155
8228
  throw new DeploymentError("SOURCE_FAILED", `Git checked out ${head}, not requested ${desiredRevision}.`);
@@ -8172,7 +8245,7 @@ function createDeploymentManager(options) {
8172
8245
  throw new DeploymentError("PIPELINE_FAILED", "deployment plan requires attestation but this Agent has no source");
8173
8246
  if (options.attestation) {
8174
8247
  try {
8175
- await options.attestation.report(head);
8248
+ await options.attestation.report(deploymentAttestationChallenge(head));
8176
8249
  } catch (cause) {
8177
8250
  if (assurance === "required")
8178
8251
  throw new DeploymentError("PIPELINE_FAILED", `deployment attestation failed: ${cause.message}`);
@@ -8543,7 +8616,7 @@ function createDeploymentManager(options) {
8543
8616
  }
8544
8617
  return options.cache.get(name);
8545
8618
  };
8546
- const attest = options.attestation ? async () => ({ report: await options.attestation.report(head), source: options.attestation.name }) : undefined;
8619
+ const attest = options.attestation ? async () => ({ report: await options.attestation.report(deploymentAttestationChallenge(head)), source: options.attestation.name }) : undefined;
8547
8620
  const pipelines = [
8548
8621
  ...["build", "migrate", "release", "health"].map((phase) => {
8549
8622
  return {
@@ -8605,6 +8678,7 @@ function createDeploymentManager(options) {
8605
8678
  ...calibration,
8606
8679
  evidencePath,
8607
8680
  recommendedCoordinate: {
8681
+ FZ_REQUESTS_PER_SECOND_LIMIT: String(calibration.allowedRequestsPerSecond),
8608
8682
  FZ_CONCURRENCY_LIMIT: String(calibration.recommendedConcurrency)
8609
8683
  }
8610
8684
  };
@@ -8753,27 +8827,64 @@ function startControlServer(manager, socketPath = DEFAULT_CONTROL_SOCKET) {
8753
8827
  server.listen(socketPath, () => chmodSync5(socketPath, 384));
8754
8828
  return server;
8755
8829
  }
8756
- function requestControl(request, socketPath = DEFAULT_CONTROL_SOCKET) {
8830
+ function requestControl(request, socketPath = DEFAULT_CONTROL_SOCKET, timeoutMs = 30000) {
8757
8831
  return new Promise((resolve, reject) => {
8758
- const socket = connect(socketPath, () => socket.write(`${JSON.stringify(request)}
8759
- `));
8760
- let buffer = "";
8761
- socket.on("data", (chunk) => {
8762
- buffer += chunk.toString("utf8");
8763
- const newline = buffer.indexOf(`
8764
- `);
8765
- if (newline === -1)
8832
+ const deadline = Date.now() + timeoutMs;
8833
+ let settled = false;
8834
+ const finish = (cause, response) => {
8835
+ if (settled)
8766
8836
  return;
8767
- socket.end();
8768
- try {
8769
- resolve(JSON.parse(buffer.slice(0, newline)));
8770
- } catch (cause) {
8837
+ settled = true;
8838
+ if (cause)
8771
8839
  reject(cause);
8840
+ else
8841
+ resolve(response);
8842
+ };
8843
+ const attempt = () => {
8844
+ if (settled)
8845
+ return;
8846
+ const remaining = deadline - Date.now();
8847
+ if (remaining <= 0)
8848
+ return finish(new Error("agent control socket did not answer before its deadline"));
8849
+ if (!existsSync5(socketPath)) {
8850
+ setTimeout(attempt, Math.min(100, Math.max(1, remaining)));
8851
+ return;
8772
8852
  }
8773
- });
8774
- socket.on("error", reject);
8853
+ let buffer = "";
8854
+ const socket = connect(socketPath, () => socket.write(`${JSON.stringify(request)}
8855
+ `));
8856
+ socket.setTimeout(remaining, () => {
8857
+ socket.destroy();
8858
+ finish(new Error("agent control socket did not answer before its deadline"));
8859
+ });
8860
+ socket.on("data", (chunk) => {
8861
+ buffer += chunk.toString("utf8");
8862
+ const newline = buffer.indexOf(`
8863
+ `);
8864
+ if (newline === -1)
8865
+ return;
8866
+ socket.end();
8867
+ try {
8868
+ finish(undefined, JSON.parse(buffer.slice(0, newline)));
8869
+ } catch (cause) {
8870
+ finish(cause);
8871
+ }
8872
+ });
8873
+ socket.on("error", (cause) => {
8874
+ socket.destroy();
8875
+ if ((cause.code === "ENOENT" || cause.code === "ECONNREFUSED") && Date.now() < deadline) {
8876
+ setTimeout(attempt, Math.min(100, Math.max(1, deadline - Date.now())));
8877
+ return;
8878
+ }
8879
+ finish(cause);
8880
+ });
8881
+ };
8882
+ attempt();
8775
8883
  });
8776
8884
  }
8885
+ function controlRequestTimeout(operation) {
8886
+ return operation === "deploy" ? 600000 : 30000;
8887
+ }
8777
8888
 
8778
8889
  // src/deployment-pull.ts
8779
8890
  var remoteOutcome = (cause) => cause instanceof SignedNodeHttpError && cause.status < 500 ? "refused" : "retryable";
@@ -8917,6 +9028,9 @@ async function pullDeploymentOnce(options) {
8917
9028
  ...target.topology ? { topology: structuredClone(target.topology) } : {}
8918
9029
  })) } : {},
8919
9030
  ...result.capacity ? { capacity: {
9031
+ measuredSustainableRequestsPerSecond: result.capacity.measuredSustainableRequestsPerSecond,
9032
+ allowedRequestsPerSecond: result.capacity.allowedRequestsPerSecond,
9033
+ safetyRatio: result.capacity.safetyRatio,
8920
9034
  recommendedConcurrency: result.capacity.recommendedConcurrency,
8921
9035
  stopReason: result.capacity.stopReason,
8922
9036
  stages: result.capacity.stages.map((stage2) => ({ ...stage2 }))
@@ -9378,7 +9492,7 @@ async function writeAndCloseProcessInput(input, value) {
9378
9492
  }
9379
9493
 
9380
9494
  // src/version.ts
9381
- var VERSION2 = "0.1.88";
9495
+ var VERSION2 = "0.1.89";
9382
9496
 
9383
9497
  // src/ssh-bootstrap.ts
9384
9498
  class SshBootstrapError extends Error {
@@ -10271,15 +10385,15 @@ function linuxList(members) {
10271
10385
  return ranges.join(",");
10272
10386
  }
10273
10387
  function physicalCoreGroups(pool) {
10274
- const cpus = membersOfLinuxList(pool.cpus, "CPU");
10275
- if (cpus.length % pool.physicalCores !== 0) {
10388
+ const cpus2 = membersOfLinuxList(pool.cpus, "CPU");
10389
+ if (cpus2.length % pool.physicalCores !== 0) {
10276
10390
  throw new MetalProvisionError("CPU pool threads must divide evenly across physical cores");
10277
10391
  }
10278
- const threadsPerCore = cpus.length / pool.physicalCores;
10392
+ const threadsPerCore = cpus2.length / pool.physicalCores;
10279
10393
  if (!Number.isInteger(threadsPerCore) || threadsPerCore < 1 || threadsPerCore > 8) {
10280
10394
  throw new MetalProvisionError("invalid CPU pool threads-per-core topology");
10281
10395
  }
10282
- return Array.from({ length: pool.physicalCores }, (_, core) => Array.from({ length: threadsPerCore }, (_2, thread) => cpus[core + thread * pool.physicalCores]));
10396
+ return Array.from({ length: pool.physicalCores }, (_, core) => Array.from({ length: threadsPerCore }, (_2, thread) => cpus2[core + thread * pool.physicalCores]));
10283
10397
  }
10284
10398
  var guestNameFor = (computeKey) => `fzg-${createHash7("sha256").update(computeKey).digest("hex").slice(0, 16)}`;
10285
10399
  var tapNameFor = (computeKey) => `fzt${createHash7("sha256").update(computeKey).digest("hex").slice(0, 12)}`;
@@ -10334,12 +10448,12 @@ function validateMetalProfile(profile) {
10334
10448
  if (!SAFE_NAME.test(pool.key) || keys.has(pool.key))
10335
10449
  throw new MetalProvisionError("invalid or duplicate CPU pool key");
10336
10450
  keys.add(pool.key);
10337
- const cpus = membersOfLinuxList(pool.cpus, "CPU");
10338
- if (!Number.isInteger(pool.physicalCores) || pool.physicalCores < 1 || pool.physicalCores > cpus.length) {
10451
+ const cpus2 = membersOfLinuxList(pool.cpus, "CPU");
10452
+ if (!Number.isInteger(pool.physicalCores) || pool.physicalCores < 1 || pool.physicalCores > cpus2.length) {
10339
10453
  throw new MetalProvisionError("invalid CPU pool physical-core count");
10340
10454
  }
10341
10455
  physicalCoreGroups(pool);
10342
- for (const cpu of cpus) {
10456
+ for (const cpu of cpus2) {
10343
10457
  if (assigned.has(cpu))
10344
10458
  throw new MetalProvisionError("CPU pools overlap");
10345
10459
  assigned.add(cpu);
@@ -11293,13 +11407,13 @@ var memoryDirective = (nodes) => nodes ? `AllowedMemoryNodes=${nodes}
11293
11407
  ` : "";
11294
11408
  function metalGuestSliceUnit(profile) {
11295
11409
  validateMetalProfile(profile);
11296
- const cpus = compact(profile.cpuPools.flatMap((pool) => members(pool.cpus)));
11410
+ const cpus2 = compact(profile.cpuPools.flatMap((pool) => members(pool.cpus)));
11297
11411
  const nodes = compact(profile.cpuPools.flatMap((pool) => pool.memoryNodes ? members(pool.memoryNodes) : [])) || undefined;
11298
11412
  return `[Unit]
11299
11413
  Description=ForgeZero exclusive guest CPU and memory boundary
11300
11414
 
11301
11415
  [Slice]
11302
- AllowedCPUs=${cpus}
11416
+ AllowedCPUs=${cpus2}
11303
11417
  ${memoryDirective(nodes)}`;
11304
11418
  }
11305
11419
  function metalHousekeepingDropIn(profile, kind) {
@@ -12546,13 +12660,13 @@ function requestAgentUpdate(request, socketPath = DEFAULT_AGENT_UPDATE_SOCKET, t
12546
12660
 
12547
12661
  // src/agent-heartbeat.ts
12548
12662
  import { existsSync as existsSync15, readFileSync as readFileSync11, statfsSync as statfsSync2 } from "fs";
12549
- import { cpus, freemem, loadavg, totalmem } from "os";
12663
+ import { cpus as cpus2, freemem as freemem2, loadavg, totalmem as totalmem2 } from "os";
12550
12664
  function readAgentHostMetrics() {
12551
12665
  const filesystem = statfsSync2("/");
12552
12666
  return {
12553
- logicalCpu: cpus().length,
12554
- memoryBytes: totalmem(),
12555
- memoryFreeBytes: freemem(),
12667
+ logicalCpu: cpus2().length,
12668
+ memoryBytes: totalmem2(),
12669
+ memoryFreeBytes: freemem2(),
12556
12670
  storageBytes: Number(filesystem.blocks) * Number(filesystem.bsize),
12557
12671
  storageFreeBytes: Number(filesystem.bavail) * Number(filesystem.bsize),
12558
12672
  load1m: loadavg()[0]
@@ -13695,30 +13809,71 @@ function requestServiceActivation(request, socketPath = DEFAULT_SOFTWARE_HELPER_
13695
13809
  function requestSoftware(requirements, socketPath = DEFAULT_SOFTWARE_HELPER_SOCKET, timeoutMs = 15 * 60000) {
13696
13810
  validateSoftwareRequirements(requirements);
13697
13811
  return new Promise((resolve6, reject) => {
13698
- const socket = connect7(socketPath, () => socket.write(`${JSON.stringify({ op: "ensure", requirements })}
13699
- `));
13700
- let buffer = "";
13701
- socket.setTimeout(timeoutMs, () => {
13702
- socket.destroy();
13703
- reject(new Error("software helper did not answer before its deadline"));
13704
- });
13705
- socket.on("data", (chunk) => {
13706
- buffer += chunk.toString("utf8");
13707
- const newline = buffer.indexOf(`
13708
- `);
13709
- if (newline < 0)
13812
+ const deadline = Date.now() + timeoutMs;
13813
+ let settled = false;
13814
+ const finish = (cause, results) => {
13815
+ if (settled)
13710
13816
  return;
13711
- socket.end();
13817
+ settled = true;
13818
+ if (cause)
13819
+ reject(cause);
13820
+ else
13821
+ resolve6(results);
13822
+ };
13823
+ const attempt = () => {
13824
+ if (settled)
13825
+ return;
13826
+ const remaining = deadline - Date.now();
13827
+ if (remaining <= 0)
13828
+ return finish(new Error("software helper did not answer before its deadline"));
13829
+ if (!existsSync18(socketPath)) {
13830
+ setTimeout(attempt, Math.min(100, Math.max(1, remaining)));
13831
+ return;
13832
+ }
13833
+ let buffer = "";
13834
+ let socket;
13712
13835
  try {
13713
- const response = JSON.parse(buffer.slice(0, newline));
13714
- if (!response.ok || !response.results)
13715
- throw new Error(response.error?.message ?? "software helper refused the request");
13716
- resolve6(response.results);
13836
+ socket = connect7(socketPath, () => socket.write(`${JSON.stringify({ op: "ensure", requirements })}
13837
+ `));
13717
13838
  } catch (cause) {
13718
- reject(cause);
13839
+ const error = cause;
13840
+ if ((error.code === "ENOENT" || error.code === "ECONNREFUSED") && Date.now() < deadline) {
13841
+ setTimeout(attempt, Math.min(100, Math.max(1, deadline - Date.now())));
13842
+ return;
13843
+ }
13844
+ finish(error);
13845
+ return;
13719
13846
  }
13720
- });
13721
- socket.on("error", reject);
13847
+ socket.setTimeout(remaining, () => {
13848
+ socket.destroy();
13849
+ finish(new Error("software helper did not answer before its deadline"));
13850
+ });
13851
+ socket.on("data", (chunk) => {
13852
+ buffer += chunk.toString("utf8");
13853
+ const newline = buffer.indexOf(`
13854
+ `);
13855
+ if (newline < 0)
13856
+ return;
13857
+ socket.end();
13858
+ try {
13859
+ const response = JSON.parse(buffer.slice(0, newline));
13860
+ if (!response.ok || !response.results)
13861
+ throw new Error(response.error?.message ?? "software helper refused the request");
13862
+ finish(undefined, response.results);
13863
+ } catch (cause) {
13864
+ finish(cause);
13865
+ }
13866
+ });
13867
+ socket.on("error", (cause) => {
13868
+ socket.destroy();
13869
+ if ((cause.code === "ENOENT" || cause.code === "ECONNREFUSED") && Date.now() < deadline) {
13870
+ setTimeout(attempt, Math.min(100, Math.max(1, deadline - Date.now())));
13871
+ return;
13872
+ }
13873
+ finish(cause);
13874
+ });
13875
+ };
13876
+ attempt();
13722
13877
  });
13723
13878
  }
13724
13879
 
@@ -13807,26 +13962,33 @@ function renderAgentEgressNft(uids, replace = false, loopback) {
13807
13962
  if (!uniqueUids.includes(loopback.uid))
13808
13963
  throw new Error("Loopback egress UID must be protected.");
13809
13964
  const ports = normalizeEgressTcpPorts(loopback.tcpPorts);
13965
+ const sharedPorts = normalizeEgressTcpPorts(loopback.sharedTcpPorts ?? []);
13810
13966
  const publicPorts = loopback.publicTcpPorts === undefined ? undefined : normalizeEgressTcpPorts(loopback.publicTcpPorts);
13811
13967
  if (publicPorts && publicPorts.length < 1) {
13812
13968
  throw new Error("Restricted public egress needs at least one TCP port.");
13813
13969
  }
13814
- if (ports.length < 1 && !publicPorts) {
13970
+ if (ports.length < 1 && sharedPorts.length < 1 && !publicPorts) {
13815
13971
  throw new Error("Restricted egress needs a loopback or public TCP port.");
13816
13972
  }
13817
- loopbackComment = (ports.length > 0 ? ` loopback=${loopback.uid}:${ports.join(",")}` : "") + (publicPorts ? ` public-tcp=${publicPorts.join(",")}` : "");
13973
+ loopbackComment = (ports.length > 0 ? ` loopback=${loopback.uid}:${ports.join(",")}` : "") + (sharedPorts.length > 0 ? ` shared-loopback=${sharedPorts.join(",")}` : "") + (publicPorts ? ` public-tcp=${publicPorts.join(",")}` : "");
13818
13974
  loopbackSet = `${ports.length > 0 ? ` set loopback_tcp_ports {
13819
13975
  type inet_service
13820
13976
  elements = { ${ports.join(", ")} }
13821
13977
  }
13978
+ ` : ""}${sharedPorts.length > 0 ? ` set shared_loopback_tcp_ports {
13979
+ type inet_service
13980
+ elements = { ${sharedPorts.join(", ")} }
13981
+ }
13822
13982
  ` : ""}${publicPorts ? ` set public_tcp_ports {
13823
13983
  type inet_service
13824
13984
  elements = { ${publicPorts.join(", ")} }
13825
13985
  }
13826
13986
  ` : ""}`;
13827
- loopbackRules = ports.length > 0 ? ` meta skuid ${loopback.uid} tcp dport @loopback_tcp_ports ip daddr 127.0.0.1 accept
13987
+ loopbackRules = `${sharedPorts.length > 0 ? ` meta skuid @service_uids tcp dport @shared_loopback_tcp_ports ip daddr 127.0.0.1 accept
13988
+ meta skuid @service_uids tcp dport @shared_loopback_tcp_ports ip6 daddr ::1 accept
13989
+ ` : ""}${ports.length > 0 ? ` meta skuid ${loopback.uid} tcp dport @loopback_tcp_ports ip daddr 127.0.0.1 accept
13828
13990
  meta skuid ${loopback.uid} tcp dport @loopback_tcp_ports ip6 daddr ::1 accept
13829
- ` : "";
13991
+ ` : ""}`;
13830
13992
  restrictedPublicRules = publicPorts ? ` meta skuid ${loopback.uid} tcp dport @public_tcp_ports accept
13831
13993
  meta skuid ${loopback.uid} reject
13832
13994
  ` : "";
@@ -13921,12 +14083,15 @@ function verifyAgentEgressPolicy(uids, command3 = systemCommand, loopback) {
13921
14083
  if (!uniqueUids.includes(loopback.uid))
13922
14084
  return false;
13923
14085
  const ports = normalizeEgressTcpPorts(loopback.tcpPorts);
14086
+ const sharedPorts = normalizeEgressTcpPorts(loopback.sharedTcpPorts ?? []);
13924
14087
  const publicPorts = loopback.publicTcpPorts === undefined ? undefined : normalizeEgressTcpPorts(loopback.publicTcpPorts);
13925
14088
  if (publicPorts && publicPorts.length < 1)
13926
14089
  return false;
13927
- if (ports.length < 1 && !publicPorts)
14090
+ if (ports.length < 1 && sharedPorts.length < 1 && !publicPorts)
13928
14091
  return false;
13929
- loopbackComment = (ports.length > 0 ? ` loopback=${loopback.uid}:${ports.join(",")}` : "") + (publicPorts ? ` public-tcp=${publicPorts.join(",")}` : "");
14092
+ loopbackComment = (ports.length > 0 ? ` loopback=${loopback.uid}:${ports.join(",")}` : "") + (sharedPorts.length > 0 ? ` shared-loopback=${sharedPorts.join(",")}` : "") + (publicPorts ? ` public-tcp=${publicPorts.join(",")}` : "");
14093
+ if (sharedPorts.length > 0)
14094
+ loopbackRequired.push(`elements = { ${sharedPorts.join(", ")} }`, "meta skuid @service_uids tcp dport @shared_loopback_tcp_ports ip daddr 127.0.0.1 accept", "meta skuid @service_uids tcp dport @shared_loopback_tcp_ports ip6 daddr ::1 accept");
13930
14095
  if (ports.length > 0)
13931
14096
  loopbackRequired.push(`elements = { ${ports.join(", ")} }`, `meta skuid ${loopback.uid} tcp dport @loopback_tcp_ports ip daddr 127.0.0.1 accept`, `meta skuid ${loopback.uid} tcp dport @loopback_tcp_ports ip6 daddr ::1 accept`);
13932
14097
  if (publicPorts)
@@ -13965,6 +14130,7 @@ function applyAgentEgressPolicy(options) {
13965
14130
  const loopback = options.loopback ? {
13966
14131
  uid: resolveServiceUid(options.loopback.user, command3),
13967
14132
  tcpPorts: normalizeEgressTcpPorts(options.loopback.tcpPorts),
14133
+ sharedTcpPorts: normalizeEgressTcpPorts(options.loopback.sharedTcpPorts ?? []),
13968
14134
  publicTcpPorts: options.loopback.publicTcpPorts === undefined ? undefined : normalizeEgressTcpPorts(options.loopback.publicTcpPorts)
13969
14135
  } : undefined;
13970
14136
  if (loopback && !uids.includes(loopback.uid))
@@ -13991,6 +14157,7 @@ async function superviseAgentEgressPolicy(options) {
13991
14157
  const loopback = options.loopback ? {
13992
14158
  uid: resolveServiceUid(options.loopback.user, command3),
13993
14159
  tcpPorts: normalizeEgressTcpPorts(options.loopback.tcpPorts),
14160
+ sharedTcpPorts: normalizeEgressTcpPorts(options.loopback.sharedTcpPorts ?? []),
13994
14161
  publicTcpPorts: options.loopback.publicTcpPorts === undefined ? undefined : normalizeEgressTcpPorts(options.loopback.publicTcpPorts)
13995
14162
  } : undefined;
13996
14163
  (options.notifyReady ?? (() => {
@@ -14730,11 +14897,29 @@ var replaceLink = (path2, target2) => {
14730
14897
  symlinkSync6(target2, pending);
14731
14898
  renameSync13(pending, path2);
14732
14899
  };
14900
+ var assertReleaseLinksContained = (path2) => {
14901
+ const root = realpathSync5(path2);
14902
+ const visit = (current2) => {
14903
+ const metadata = lstatSync4(current2);
14904
+ if (metadata.isSymbolicLink()) {
14905
+ const target2 = realpathSync5(current2);
14906
+ if (target2 !== root && !target2.startsWith(`${root}/`)) {
14907
+ throw new Error("release contains a symbolic link outside its immutable root");
14908
+ }
14909
+ return;
14910
+ }
14911
+ if (metadata.isDirectory())
14912
+ for (const name of readdirSync4(current2))
14913
+ visit(join10(current2, name));
14914
+ };
14915
+ visit(root);
14916
+ };
14733
14917
  var secureRelease = (path2, uid, gid) => {
14918
+ assertReleaseLinksContained(path2);
14734
14919
  const visit = (current2) => {
14735
14920
  const metadata = lstatSync4(current2);
14736
14921
  if (metadata.isSymbolicLink())
14737
- throw new Error("release contains a symbolic link");
14922
+ return;
14738
14923
  chownSync(current2, uid, gid);
14739
14924
  chmodSync16(current2, metadata.isDirectory() ? 365 : 292);
14740
14925
  if (metadata.isDirectory())
@@ -14760,7 +14945,7 @@ async function activatePlatformRelease(config, requestedRelease, options = {}) {
14760
14945
  const sleep2 = options.sleep ?? Bun.sleep;
14761
14946
  const slots = join10(normalized.root, "slots");
14762
14947
  mkdirSync14(slots, { recursive: true, mode: 493 });
14763
- const slotFile = join10(normalized.root, ".forge-slot");
14948
+ const slotFile = join10(slots, ".active");
14764
14949
  const previousSlot = existsSync19(slotFile) ? readFileSync14(slotFile, "utf8").trim() : undefined;
14765
14950
  const target2 = previousSlot === "blue" ? "green" : "blue";
14766
14951
  const port = target2 === "blue" ? normalized.bluePort : normalized.greenPort;
@@ -14810,6 +14995,7 @@ async function activatePlatformRelease(config, requestedRelease, options = {}) {
14810
14995
  const test = await exec(["/usr/sbin/nginx", "-t"]);
14811
14996
  const reload = test.exitCode === 0 ? await exec(["/usr/sbin/nginx", "-s", "reload"]) : test;
14812
14997
  if (reload.exitCode !== 0) {
14998
+ const refusal = (reload.output || test.output).trim().slice(0, 2000);
14813
14999
  if (existsSync19(backup))
14814
15000
  renameSync13(backup, upstream);
14815
15001
  else
@@ -14817,7 +15003,7 @@ async function activatePlatformRelease(config, requestedRelease, options = {}) {
14817
15003
  await exec(["/usr/sbin/nginx", "-t"]);
14818
15004
  await exec(["/usr/sbin/nginx", "-s", "reload"]);
14819
15005
  await stopTarget();
14820
- throw new Error("nginx refused the promoted upstream");
15006
+ throw new Error(`nginx refused the promoted upstream${refusal ? `: ${refusal}` : ""}`);
14821
15007
  }
14822
15008
  rmSync10(backup, { force: true });
14823
15009
  writeFileSync14(slotFile, `${target2}
@@ -14849,7 +15035,7 @@ var checked8 = async (run2, argv2, expected = 0) => {
14849
15035
  return result;
14850
15036
  };
14851
15037
  var activeSlot = () => {
14852
- const slot = readFileSync15("/opt/forgezero/.forge-slot", "utf8").trim();
15038
+ const slot = readFileSync15("/opt/forgezero/slots/.active", "utf8").trim();
14853
15039
  if (slot !== "blue" && slot !== "green")
14854
15040
  throw new Error("invalid active slot");
14855
15041
  return slot;
@@ -15046,6 +15232,7 @@ function agentEgressUnit(options) {
15046
15232
  }
15047
15233
  const deploymentEnabled = Boolean(options.repository || bootstrapBundleEnabled || options.pullDeployments);
15048
15234
  const runnerLoopbackPorts = normalizeEgressTcpPorts(options.runnerLoopbackPorts ?? []);
15235
+ const agentLoopbackPorts = normalizeEgressTcpPorts(options.agentLoopbackPorts ?? []);
15049
15236
  const runnerPublicTcpPorts = normalizeEgressTcpPorts(options.runnerPublicTcpPorts ?? DEFAULT_RUNNER_PUBLIC_TCP_PORTS);
15050
15237
  if (deploymentEnabled && runnerPublicTcpPorts.length < 1) {
15051
15238
  throw new Error("deployed project runner needs at least one vetted public TCP port");
@@ -15053,8 +15240,10 @@ function agentEgressUnit(options) {
15053
15240
  if (deploymentEnabled)
15054
15241
  systemdAgentEgressDirectives(runnerLoopbackPorts);
15055
15242
  const users = [user, ...deploymentEnabled ? [DEPLOYMENT_RUNNER_USER] : []];
15056
- const runnerGrant = deploymentEnabled ? ` --loopback-user=${DEPLOYMENT_RUNNER_USER}` + runnerLoopbackPorts.map((port) => ` --loopback-tcp-port=${port}`).join("") + runnerPublicTcpPorts.map((port) => ` --public-tcp-port=${port}`).join("") : "";
15057
- const policyProof = `ExecStartPost=${bin} egress-policy-check ${users.map((name) => `--user=${name}`).join(" ")}${runnerGrant}`;
15243
+ const portGrantEnabled = deploymentEnabled || agentLoopbackPorts.length > 0;
15244
+ const restrictedUser = deploymentEnabled ? DEPLOYMENT_RUNNER_USER : user;
15245
+ const portGrant = portGrantEnabled ? ` --loopback-user=${restrictedUser}` + (deploymentEnabled ? runnerLoopbackPorts.map((port) => ` --loopback-tcp-port=${port}`).join("") : "") + agentLoopbackPorts.map((port) => ` --shared-loopback-tcp-port=${port}`).join("") + (deploymentEnabled ? runnerPublicTcpPorts.map((port) => ` --public-tcp-port=${port}`).join("") : "") : "";
15246
+ const policyProof = `ExecStartPost=${bin} egress-policy-check ${users.map((name) => `--user=${name}`).join(" ")}${portGrant}`;
15058
15247
  return `[Unit]
15059
15248
  Description=ForgeZero Agent host egress policy
15060
15249
  Documentation=https://www.forgezero.net/docs/agent
@@ -15067,7 +15256,7 @@ Type=notify
15067
15256
  NotifyAccess=all
15068
15257
  User=root
15069
15258
  Group=root
15070
- ExecStart=${bin} egress-policy ${users.map((name) => `--user=${name}`).join(" ")}${runnerGrant}
15259
+ ExecStart=${bin} egress-policy ${users.map((name) => `--user=${name}`).join(" ")}${portGrant}
15071
15260
  ${policyProof}
15072
15261
  Restart=on-failure
15073
15262
  RestartSec=2
@@ -15351,7 +15540,7 @@ function agentEnrolmentUnit(options) {
15351
15540
  Requires=forgezero-agent-egress.service
15352
15541
  BindsTo=forgezero-agent-egress.service
15353
15542
  ` : "";
15354
- const egressDirectives = options.enforceEgress ? systemdAgentEgressDirectives() : "";
15543
+ const egressDirectives = options.enforceEgress ? systemdAgentEgressDirectives(options.agentLoopbackPorts ?? []) : "";
15355
15544
  return `[Unit]
15356
15545
  Description=Bind this machine to its ForgeZero compute
15357
15546
  After=network-online.target
@@ -15388,7 +15577,7 @@ WantedBy=multi-user.target
15388
15577
  function deploymentRunnerUnit(options) {
15389
15578
  const bin = options.binPath ?? "fz-agent";
15390
15579
  const root = options.deployRoot ?? "/opt/forgezero";
15391
- const agentUser = options.user ?? "forgezero-agent";
15580
+ const agentUser = options.user ?? "forgezero";
15392
15581
  const egressDependency = options.enforceEgress ? `After=forgezero-agent-egress.service
15393
15582
  Requires=forgezero-agent-egress.service
15394
15583
  BindsTo=forgezero-agent-egress.service
@@ -15429,7 +15618,7 @@ RestrictRealtime=true
15429
15618
  MemoryDenyWriteExecute=true
15430
15619
  LockPersonality=true
15431
15620
  ${egressDirectives}
15432
- ReadWritePaths=${root}/releases ${root}/runner-home
15621
+ ReadWritePaths=${root}/releases ${root}/runner-home ${root}/slots /etc/nginx/conf.d /var/log/nginx
15433
15622
 
15434
15623
  [Install]
15435
15624
  WantedBy=multi-user.target
@@ -15564,7 +15753,7 @@ function agentUnit(options) {
15564
15753
  const supplementaryGroups = [
15565
15754
  AGENT_UPDATE_GROUP,
15566
15755
  deploymentEnabled ? DEPLOYMENT_GROUP : null,
15567
- deploymentEnabled ? SOFTWARE_HELPER_GROUP : null,
15756
+ SOFTWARE_HELPER_GROUP,
15568
15757
  lifecycleEnabled ? LIFECYCLE_GROUP : null
15569
15758
  ].filter((value) => value !== null);
15570
15759
  const deploymentGroup = supplementaryGroups.length > 0 ? `SupplementaryGroups=${supplementaryGroups.join(" ")}` : "";
@@ -15573,7 +15762,7 @@ function agentUnit(options) {
15573
15762
  "forgezero-agent-update-helper.service",
15574
15763
  options.enforceEgress ? "forgezero-agent-egress.service" : null,
15575
15764
  deploymentEnabled ? "forgezero-deploy-runner.service" : null,
15576
- deploymentEnabled ? "forgezero-software-helper.service" : null,
15765
+ "forgezero-software-helper.service",
15577
15766
  lifecycleEnabled ? "forgezero-lifecycle-helper.service" : null,
15578
15767
  warpEnabled ? "warp-svc.service" : null,
15579
15768
  enrolmentEnabled ? "forgezero-agent-enrol.service" : null
@@ -15582,7 +15771,7 @@ function agentUnit(options) {
15582
15771
  "forgezero-agent-update-helper.service",
15583
15772
  options.enforceEgress ? "forgezero-agent-egress.service" : null,
15584
15773
  deploymentEnabled ? "forgezero-deploy-runner.service" : null,
15585
- deploymentEnabled ? "forgezero-software-helper.service" : null,
15774
+ "forgezero-software-helper.service",
15586
15775
  lifecycleEnabled ? "forgezero-lifecycle-helper.service" : null,
15587
15776
  warpEnabled ? "warp-svc.service" : null,
15588
15777
  enrolmentEnabled ? "forgezero-agent-enrol.service" : null
@@ -15597,9 +15786,9 @@ function agentUnit(options) {
15597
15786
  const snpDevice = options.mode === "attested" ? `DevicePolicy=closed
15598
15787
  DeviceAllow=/dev/sev-guest rw` : "";
15599
15788
  const snpPrepare = options.mode === "attested" ? `ExecStartPre=+/bin/chgrp ${VAULT_GROUP} /dev/sev-guest
15600
- ExecStartPre=+/bin/chmod 0640 /dev/sev-guest
15789
+ ExecStartPre=+/bin/chmod 0660 /dev/sev-guest
15601
15790
  ` : "";
15602
- const egressDirectives = options.enforceEgress ? systemdAgentEgressDirectives() : "";
15791
+ const egressDirectives = options.enforceEgress ? systemdAgentEgressDirectives(options.agentLoopbackPorts ?? []) : "";
15603
15792
  return `[Unit]
15604
15793
  Description=ForgeZero node agent (${options.mode})
15605
15794
  Documentation=https://www.forgezero.net/docs/agent
@@ -15730,6 +15919,7 @@ function planProvision(options) {
15730
15919
  const deployRoot = options.deployRoot ?? "/opt/forgezero";
15731
15920
  const deploymentEnabled = Boolean(options.repository || options.pullDeployments);
15732
15921
  const runnerLoopbackPorts = normalizeEgressTcpPorts(options.runnerLoopbackPorts ?? []);
15922
+ const agentLoopbackPorts = normalizeEgressTcpPorts(options.agentLoopbackPorts ?? []);
15733
15923
  const runnerPublicTcpPorts = normalizeEgressTcpPorts(options.runnerPublicTcpPorts ?? DEFAULT_RUNNER_PUBLIC_TCP_PORTS);
15734
15924
  const lifecycleEnabled = Boolean(options.pullMigrations && options.lifecycleProfilePath);
15735
15925
  if (Boolean(options.pullMigrations) !== Boolean(options.lifecycleProfilePath)) {
@@ -15778,8 +15968,9 @@ function planProvision(options) {
15778
15968
  const enabledUnits = [
15779
15969
  "forgezero-agent.socket",
15780
15970
  "forgezero-agent-update-helper.service",
15971
+ "forgezero-software-helper.service",
15781
15972
  ...options.enforceEgress ? ["forgezero-agent-egress.service"] : [],
15782
- ...deploymentEnabled ? ["forgezero-deploy-runner.service", "forgezero-software-helper.service"] : [],
15973
+ ...deploymentEnabled ? ["forgezero-deploy-runner.service"] : [],
15783
15974
  ...lifecycleEnabled ? ["forgezero-lifecycle-helper.service"] : [],
15784
15975
  ...warpEnabled ? ["forgezero-warp-config.service", "warp-svc.service"] : [],
15785
15976
  ...enrolmentEnabled ? ["forgezero-agent-enrol.service"] : [],
@@ -15787,8 +15978,9 @@ function planProvision(options) {
15787
15978
  ];
15788
15979
  const restartedUnits = [
15789
15980
  "forgezero-agent-update-helper.service",
15981
+ "forgezero-software-helper.service",
15790
15982
  ...options.enforceEgress ? ["forgezero-agent-egress.service"] : [],
15791
- ...deploymentEnabled ? ["forgezero-deploy-runner.service", "forgezero-software-helper.service"] : [],
15983
+ ...deploymentEnabled ? ["forgezero-deploy-runner.service"] : [],
15792
15984
  ...lifecycleEnabled ? ["forgezero-lifecycle-helper.service"] : [],
15793
15985
  ...warpEnabled ? ["forgezero-warp-config.service", "warp-svc.service"] : [],
15794
15986
  ...enrolmentEnabled ? ["forgezero-agent-enrol.service"] : []
@@ -15812,9 +16004,9 @@ function planProvision(options) {
15812
16004
  ...options.enforceEgress ? [
15813
16005
  { path: AGENT_EGRESS_UNIT_PATH, unit: agentEgressUnit(options) }
15814
16006
  ] : [],
16007
+ { path: SOFTWARE_HELPER_UNIT_PATH, unit: softwareHelperUnit(options) },
15815
16008
  ...deploymentEnabled ? [
15816
- { path: DEPLOYMENT_RUNNER_UNIT_PATH, unit: deploymentRunnerUnit(options) },
15817
- { path: SOFTWARE_HELPER_UNIT_PATH, unit: softwareHelperUnit(options) }
16009
+ { path: DEPLOYMENT_RUNNER_UNIT_PATH, unit: deploymentRunnerUnit(options) }
15818
16010
  ] : [],
15819
16011
  ...enrolmentEnabled ? [
15820
16012
  { path: ENROLMENT_UNIT_PATH, unit: agentEnrolmentUnit(options) }
@@ -15847,24 +16039,29 @@ function planProvision(options) {
15847
16039
  step2("Agent update helper access group", { kind: "commands", commands: [{ argv: ["/usr/sbin/groupadd", "--system", AGENT_UPDATE_GROUP], acceptedExitCodes: [0, 9] }] }),
15848
16040
  ...sourceBinPath && binPath ? [step2("root-owned agent runtime", { kind: "install-runtime", source: sourceBinPath, binary: binPath, version: VERSION2 })] : [],
15849
16041
  ...warpEnabled ? [step2("Cloudflare One client for Ubuntu 26.04", { kind: "install-warp" })] : [],
15850
- ...deploymentEnabled ? [step2("deployment isolation group", { kind: "commands", commands: [
15851
- { argv: ["/usr/sbin/groupadd", "--system", DEPLOYMENT_GROUP], acceptedExitCodes: [0, 9] },
16042
+ step2("software strategy helper group", { kind: "commands", commands: [
15852
16043
  { argv: ["/usr/sbin/groupadd", "--system", SOFTWARE_HELPER_GROUP], acceptedExitCodes: [0, 9] }
16044
+ ] }),
16045
+ ...deploymentEnabled ? [step2("deployment isolation group", { kind: "commands", commands: [
16046
+ { argv: ["/usr/sbin/groupadd", "--system", DEPLOYMENT_GROUP], acceptedExitCodes: [0, 9] }
15853
16047
  ] })] : [],
15854
16048
  ...lifecycleEnabled ? [step2("lifecycle helper access group", { kind: "commands", commands: [{ argv: ["/usr/sbin/groupadd", "--system", LIFECYCLE_GROUP], acceptedExitCodes: [0, 9] }] })] : [],
15855
16049
  step2("service account", { kind: "commands", commands: [{ argv: ["/usr/sbin/useradd", "--system", "--no-create-home", "--shell", "/usr/sbin/nologin", user], acceptedExitCodes: [0, 9] }] }),
15856
16050
  step2("bind service account to vault group", { kind: "commands", commands: [{ argv: ["/usr/sbin/usermod", "-g", VAULT_GROUP, user] }] }),
15857
16051
  step2("grant verified Agent update access", { kind: "commands", commands: [{ argv: ["/usr/sbin/usermod", "-a", "-G", AGENT_UPDATE_GROUP, user] }] }),
16052
+ step2("grant software helper socket access", { kind: "commands", commands: [
16053
+ { argv: ["/usr/sbin/usermod", "-a", "-G", SOFTWARE_HELPER_GROUP, user] }
16054
+ ] }),
15858
16055
  ...lifecycleEnabled ? [step2("grant lifecycle helper socket access", { kind: "commands", commands: [{ argv: ["/usr/sbin/usermod", "-a", "-G", LIFECYCLE_GROUP, user] }] })] : [],
15859
16056
  ...deploymentEnabled ? [step2("credential-free deployment account", { kind: "commands", commands: [
15860
16057
  { argv: ["/usr/sbin/useradd", "--system", "--no-create-home", "--shell", "/usr/sbin/nologin", "--gid", DEPLOYMENT_GROUP, DEPLOYMENT_RUNNER_USER], acceptedExitCodes: [0, 9] },
15861
16058
  { argv: ["/usr/sbin/useradd", "--system", "--no-create-home", "--shell", "/usr/sbin/nologin", "--gid", VAULT_GROUP, APPLICATION_RUNTIME_USER], acceptedExitCodes: [0, 9] },
15862
16059
  { argv: ["/usr/sbin/usermod", "-a", "-G", DEPLOYMENT_GROUP, APPLICATION_RUNTIME_USER] },
15863
- { argv: ["/usr/sbin/usermod", "-a", "-G", `${DEPLOYMENT_GROUP},${SOFTWARE_HELPER_GROUP}`, user] }
16060
+ { argv: ["/usr/sbin/usermod", "-a", "-G", DEPLOYMENT_GROUP, user] }
15864
16061
  ] })] : [],
15865
16062
  step2("credential and state directories", { kind: "directories", directories: [
15866
16063
  { path: credentialDir, mode: 448, owner: "root", group: "root" },
15867
- { path: "/var/lib/forgezero", mode: 488, owner: "root", group: "root" }
16064
+ { path: "/var/lib/forgezero", mode: 488, owner: "root", group: VAULT_GROUP }
15868
16065
  ] }),
15869
16066
  step2("encrypted node identity", { kind: "ensure-seed", credential: seedCredentialPath }),
15870
16067
  ...options.generateGitIdentity && gitCredentialPath && gitPublicKeyPath ? [
@@ -15915,19 +16112,26 @@ function planProvision(options) {
15915
16112
  { argv: ["/usr/bin/systemctl", "enable", ...enabledUnits] },
15916
16113
  { argv: ["/usr/bin/systemctl", "reset-failed", "forgezero-agent.service"], acceptedExitCodes: [0, 1] },
15917
16114
  ...restartedUnits.length ? [{ argv: ["/usr/bin/systemctl", "restart", ...restartedUnits] }] : [],
16115
+ { argv: ["/usr/bin/systemctl", "stop", "forgezero-agent-proxy.service"], acceptedExitCodes: [0, 1, 5] },
15918
16116
  { argv: ["/usr/bin/systemctl", "restart", "forgezero-agent.socket"] },
15919
16117
  { argv: ["/usr/bin/systemctl", "reset-failed", "forgezero-agent.service"], acceptedExitCodes: [0, 1] },
15920
16118
  { argv: ["/usr/bin/systemctl", "restart", "forgezero-agent.service"] }
15921
16119
  ] }),
15922
16120
  ...enrolmentEnabled ? [step2("prove the compute binding is durable", { kind: "verify-file", path: enrolStatePath })] : [],
15923
- ...options.enforceEgress ? [step2("prove the Agent egress policy is active", { kind: "verify-egress", runnerPublicTcpPorts, runnerLoopbackPorts })] : [],
16121
+ ...options.enforceEgress ? [step2("prove the Agent egress policy is active", {
16122
+ kind: "verify-egress",
16123
+ deploymentEnabled,
16124
+ runnerPublicTcpPorts,
16125
+ runnerLoopbackPorts,
16126
+ agentLoopbackPorts
16127
+ })] : [],
15924
16128
  step2("prove it is running", { kind: "commands", commands: [{ argv: ["/usr/bin/systemctl", "is-active", "forgezero-agent.service"] }] }),
15925
16129
  step2("prove the public Vault socket exists", { kind: "wait-socket", path: options.socketPath, attempts: 100, intervalMs: 100 }),
15926
16130
  step2("prove the Agent Vault backend exists", { kind: "wait-socket", path: agentBackendSocketPath(options.socketPath), attempts: 100, intervalMs: 100 }),
15927
16131
  step2("prove the Agent update helper exists", { kind: "wait-socket", path: DEFAULT_AGENT_UPDATE_SOCKET, attempts: 100, intervalMs: 100 }),
16132
+ step2("prove the software strategy helper socket exists", { kind: "wait-socket", path: DEFAULT_SOFTWARE_HELPER_SOCKET, attempts: 100, intervalMs: 100 }),
15928
16133
  ...deploymentEnabled ? [
15929
- step2("prove the deployment runner socket exists", { kind: "wait-socket", path: DEPLOYMENT_RUNNER_SOCKET, attempts: 100, intervalMs: 100 }),
15930
- step2("prove the software strategy helper socket exists", { kind: "wait-socket", path: DEFAULT_SOFTWARE_HELPER_SOCKET, attempts: 100, intervalMs: 100 })
16134
+ step2("prove the deployment runner socket exists", { kind: "wait-socket", path: DEPLOYMENT_RUNNER_SOCKET, attempts: 100, intervalMs: 100 })
15931
16135
  ] : [],
15932
16136
  ...lifecycleEnabled ? [step2("prove the lifecycle helper socket exists", { kind: "wait-socket", path: lifecycleHelperSocketPath, attempts: 100, intervalMs: 100 })] : [],
15933
16137
  ...warpEnabled ? [step2("prove Cloudflare WARP is connected", { kind: "verify-warp" })] : [],
@@ -15952,6 +16156,25 @@ import {
15952
16156
  writeFileSync as writeFileSync15
15953
16157
  } from "fs";
15954
16158
  import { dirname as dirname13 } from "path";
16159
+ function normalizeEd25519PublicKey(output) {
16160
+ const line = output.trim();
16161
+ if (!/^ssh-ed25519 [A-Za-z0-9+/]+={0,3}(?: [^\r\n]{1,256})?$/.test(line))
16162
+ return;
16163
+ const [algorithm, encoded] = line.split(" ", 3);
16164
+ if (algorithm !== "ssh-ed25519" || !encoded)
16165
+ return;
16166
+ let blob;
16167
+ try {
16168
+ blob = Buffer.from(encoded, "base64");
16169
+ } catch {
16170
+ return;
16171
+ }
16172
+ if (blob.length !== 51 || blob.readUInt32BE(0) !== 11 || blob.subarray(4, 15).toString("ascii") !== "ssh-ed25519" || blob.readUInt32BE(15) !== 32)
16173
+ return;
16174
+ if (blob.toString("base64") !== encoded)
16175
+ return;
16176
+ return `${algorithm} ${encoded}`;
16177
+ }
15955
16178
  async function readCapabilities(run2) {
15956
16179
  const answers = {};
15957
16180
  const checks = Object.entries(CAPABILITY_CHECKS);
@@ -16091,7 +16314,7 @@ var runProvisionOperation = async (operation) => {
16091
16314
  return result;
16092
16315
  }
16093
16316
  result = await fixed(["/usr/bin/ssh-keygen", "-y", "-f", key]);
16094
- if (result.exitCode !== 0 || !/^ssh-ed25519 [A-Za-z0-9+/]+={0,3}\s*$/.test(result.stdout)) {
16317
+ if (result.exitCode !== 0 || !normalizeEd25519PublicKey(result.stdout)) {
16095
16318
  return { stdout: "bootstrap SSH private key is not a valid Ed25519 OpenSSH key", exitCode: 1 };
16096
16319
  }
16097
16320
  result = await fixed(["/usr/bin/systemd-creds", "encrypt", "--name=bootstrap-ssh-key", key, operation.credential]);
@@ -16107,11 +16330,12 @@ var runProvisionOperation = async (operation) => {
16107
16330
  chmodSync17(key, 384);
16108
16331
  }
16109
16332
  const derived = await fixed(["/usr/bin/ssh-keygen", "-y", "-f", key]);
16110
- if (derived.exitCode !== 0 || !/^ssh-ed25519 [A-Za-z0-9+/]+={0,3}\s*$/.test(derived.stdout)) {
16333
+ const publicKey = normalizeEd25519PublicKey(derived.stdout);
16334
+ if (derived.exitCode !== 0 || !publicKey) {
16111
16335
  return { stdout: "bootstrap SSH public key derivation failed", exitCode: 1 };
16112
16336
  }
16113
16337
  mkdirSync15(dirname13(operation.publicKey), { recursive: true, mode: 493 });
16114
- writeFileSync15(operation.publicKey, `${derived.stdout.trim()} forgezero-bootstrap-runner
16338
+ writeFileSync15(operation.publicKey, `${publicKey} forgezero-bootstrap-runner
16115
16339
  `, { mode: 292 });
16116
16340
  chmodSync17(operation.publicKey, 292);
16117
16341
  }
@@ -16154,8 +16378,9 @@ var runProvisionOperation = async (operation) => {
16154
16378
  const policy = await fixed(["/usr/sbin/nft", "--numeric", "list", "table", "inet", "forgezero_agent_egress"]);
16155
16379
  const required = [
16156
16380
  "forgezero-agent-egress-v1",
16157
- `public-tcp=${operation.runnerPublicTcpPorts.join(",")}`,
16158
- ...operation.runnerLoopbackPorts.length ? [`:${operation.runnerLoopbackPorts.join(",")}`] : []
16381
+ ...operation.agentLoopbackPorts.length ? [`shared-loopback=${operation.agentLoopbackPorts.join(",")}`] : [],
16382
+ ...operation.deploymentEnabled && operation.runnerPublicTcpPorts.length ? [`public-tcp=${operation.runnerPublicTcpPorts.join(",")}`] : [],
16383
+ ...operation.deploymentEnabled && operation.runnerLoopbackPorts.length ? [`:${operation.runnerLoopbackPorts.join(",")}`] : []
16159
16384
  ];
16160
16385
  return policy.exitCode === 0 && required.every((part) => policy.stdout.includes(part)) ? { stdout: policy.stdout, exitCode: 0 } : { stdout: policy.stdout, exitCode: 1 };
16161
16386
  }
@@ -16221,7 +16446,9 @@ async function applyPlan(plan, run2) {
16221
16446
  const result = await run2(step3.operation);
16222
16447
  transcript.push({ label: step3.label, command: step3.command, exitCode: result.exitCode });
16223
16448
  if (result.exitCode !== 0 && !step3.optional) {
16224
- throw new Error(`${step3.label} failed (exit ${result.exitCode}): ${step3.command}`);
16449
+ const safeDetail = step3.operation.kind === "ensure-bootstrap-ssh-identity" ? result.stdout.trim().slice(0, 1000) : "";
16450
+ throw new Error(`${step3.label} failed (exit ${result.exitCode}): ${step3.command}${safeDetail ? `
16451
+ ${safeDetail}` : ""}`);
16225
16452
  }
16226
16453
  }
16227
16454
  return transcript;
@@ -16248,7 +16475,6 @@ var SEED_CREDENTIAL = `${CREDS}/seed-sync-root.cred`;
16248
16475
  var BACKUP_RECOVERY_CREDENTIAL = `${CREDS}/backup-recovery-root.cred`;
16249
16476
  var BOOTSTRAP_SSH_CREDENTIAL = `${CREDS}/bootstrap-ssh-key.cred`;
16250
16477
  var BOOTSTRAP_SSH_PUBLIC_KEY = "/etc/forgezero/bootstrap/runner.pub";
16251
- var BOOTSTRAP_RELEASE_EVIDENCE = "/var/lib/forgezero/bootstrap-release.json";
16252
16478
  var LIFECYCLE_PROFILE = "/etc/forgezero/lifecycle.json";
16253
16479
  var CONTROL_SOCKET = "/run/forgezero/control.sock";
16254
16480
  var CLOUDFLARED_METRICS_ADDRESS = "127.0.0.1:20241";
@@ -16467,6 +16693,8 @@ function planBootstrapAgentInstall(config, phase, context) {
16467
16693
  bootstrapTargetTelemetryEndpoint: bound && bootstrapRunner ? platformBootstrapRunner(config) ? config.telemetryEndpoint : config.kind === "enrolled-compute" ? config.bootstrapRunner?.targetTelemetryEndpoint : undefined : undefined,
16468
16694
  lifecycleProfilePath: bound && config.kind === "platform" ? LIFECYCLE_PROFILE : undefined,
16469
16695
  enforceEgress: true,
16696
+ runnerLoopbackPorts: config.kind === "platform" ? [config.runtime.environment.publicApiPort, config.runtime.bluePort, config.runtime.greenPort] : [],
16697
+ agentLoopbackPorts: config.kind === "platform" ? [config.runtime.environment.publicApiPort] : [],
16470
16698
  nodeHostname: config.nodeHostname,
16471
16699
  telemetryEndpoint: config.telemetryEndpoint,
16472
16700
  binPath: "/usr/local/lib/forgezero/agent/fz-agent",
@@ -16476,7 +16704,7 @@ function planBootstrapAgentInstall(config, phase, context) {
16476
16704
  enrolStatePath: "/var/lib/forgezero/enrolment.json"
16477
16705
  } : bound ? { enrolStatePath: "/var/lib/forgezero/enrolment.json" } : {},
16478
16706
  ...authenticated ? {
16479
- apiUrl: config.apiUrl,
16707
+ apiUrl: config.kind === "platform" ? `http://127.0.0.1:${config.runtime.environment.publicApiPort}` : config.apiUrl,
16480
16708
  project: config.kind === "enrolled-compute" ? config.realm : "platform",
16481
16709
  environment: config.kind === "enrolled-compute" ? undefined : config.environment,
16482
16710
  nodeLabel: config.kind === "enrolled-compute" ? config.nodeHostname : config.computeReference
@@ -16485,6 +16713,7 @@ function planBootstrapAgentInstall(config, phase, context) {
16485
16713
  return planInstall(options);
16486
16714
  }
16487
16715
  function localBootstrapHost() {
16716
+ let softwareClientUser;
16488
16717
  const execute3 = async (argv2, options = {}) => {
16489
16718
  const child = Bun.spawn([...argv2], { stdin: options.stdin === undefined ? "ignore" : "pipe", stdout: "pipe", stderr: "pipe" });
16490
16719
  if (options.stdin !== undefined && child.stdin && typeof child.stdin !== "number") {
@@ -16517,10 +16746,12 @@ function localBootstrapHost() {
16517
16746
  exec: execute3,
16518
16747
  sleep: (milliseconds) => Bun.sleep(milliseconds),
16519
16748
  async ensureSoftware(requirements) {
16749
+ if (!softwareClientUser)
16750
+ throw new Error("Agent must be converged before software requirements are applied");
16520
16751
  const result = await execute3([
16521
16752
  "runuser",
16522
16753
  "-u",
16523
- "forgezero-agent",
16754
+ softwareClientUser,
16524
16755
  "--",
16525
16756
  "/usr/local/bin/fz-agent",
16526
16757
  "software-ensure",
@@ -16534,7 +16765,7 @@ function localBootstrapHost() {
16534
16765
  const capabilities = await readCapabilities(localRunner);
16535
16766
  const hasBinding = existsSync22("/var/lib/forgezero/enrolment.json");
16536
16767
  const hasEnrolCredential = existsSync22(ENROL_CREDENTIAL);
16537
- const initialBundle = config.kind === "platform" && !existsSync22(BOOTSTRAP_RELEASE_EVIDENCE) ? {
16768
+ const initialBundle = config.kind === "platform" && phase === "bootstrap" && existsSync22(config.bootstrapBundle.bundleFile) && existsSync22(config.bootstrapBundle.manifestFile) ? {
16538
16769
  path: config.bootstrapBundle.bundleFile,
16539
16770
  manifestPath: config.bootstrapBundle.manifestFile,
16540
16771
  manifest: parseBootstrapBundleManifest(JSON.parse(readFileSync17(config.bootstrapBundle.manifestFile, "utf8")))
@@ -16565,6 +16796,7 @@ function localBootstrapHost() {
16565
16796
  writeFileSync16(unit3.path, unit3.unit, { mode: 420 });
16566
16797
  }
16567
16798
  await applyPlan(plan, localRunner);
16799
+ softwareClientUser = plan.user;
16568
16800
  return plan;
16569
16801
  }
16570
16802
  };
@@ -17218,6 +17450,14 @@ async function initializeVaultReplica(cache) {
17218
17450
  return { state: "unavailable", loaded: 0, failed: [] };
17219
17451
  }
17220
17452
  }
17453
+ async function attemptInitialAttestation(attempt, onDeferred) {
17454
+ try {
17455
+ return { verified: true, result: await attempt() };
17456
+ } catch (cause) {
17457
+ onDeferred(cause);
17458
+ return { verified: false };
17459
+ }
17460
+ }
17221
17461
  function runAgent(config = {}) {
17222
17462
  const seed = config.seed ?? configuredAgentSeed({
17223
17463
  credential: config.seedCredential,
@@ -17386,16 +17626,18 @@ if (import.meta.main) {
17386
17626
  const loopbackUser = args.find((arg) => arg.startsWith("--loopback-user="))?.slice("--loopback-user=".length);
17387
17627
  const parsePorts = (prefix) => args.filter((arg) => arg.startsWith(prefix)).map((arg) => Number(arg.slice(prefix.length)));
17388
17628
  const loopbackTcpPorts = parsePorts("--loopback-tcp-port=");
17629
+ const sharedLoopbackTcpPorts = parsePorts("--shared-loopback-tcp-port=");
17389
17630
  const publicTcpPorts = parsePorts("--public-tcp-port=");
17390
- if (!loopbackUser && (loopbackTcpPorts.length > 0 || publicTcpPorts.length > 0)) {
17631
+ if (!loopbackUser && (loopbackTcpPorts.length > 0 || sharedLoopbackTcpPorts.length > 0 || publicTcpPorts.length > 0)) {
17391
17632
  throw new Error("egress-policy port grants require --loopback-user=<service-user>");
17392
17633
  }
17393
- if (loopbackUser && loopbackTcpPorts.length < 1 && publicTcpPorts.length < 1) {
17634
+ if (loopbackUser && loopbackTcpPorts.length < 1 && sharedLoopbackTcpPorts.length < 1 && publicTcpPorts.length < 1) {
17394
17635
  throw new Error("egress-policy restricted user requires an explicit TCP port grant");
17395
17636
  }
17396
17637
  const loopback = loopbackUser ? {
17397
17638
  user: loopbackUser,
17398
17639
  tcpPorts: loopbackTcpPorts,
17640
+ sharedTcpPorts: sharedLoopbackTcpPorts,
17399
17641
  publicTcpPorts: publicTcpPorts.length > 0 ? publicTcpPorts : undefined
17400
17642
  } : undefined;
17401
17643
  if (command3 === "egress-policy-check") {
@@ -17403,6 +17645,7 @@ if (import.meta.main) {
17403
17645
  const grant = loopback ? {
17404
17646
  uid: resolveServiceUid(loopback.user),
17405
17647
  tcpPorts: loopback.tcpPorts,
17648
+ sharedTcpPorts: loopback.sharedTcpPorts,
17406
17649
  publicTcpPorts: loopback.publicTcpPorts
17407
17650
  } : undefined;
17408
17651
  if (!verifyAgentEgressPolicy(uids, undefined, grant))
@@ -17657,8 +17900,8 @@ if (import.meta.main) {
17657
17900
  ...request,
17658
17901
  publicKeys: { ed25519: keys2.ed25519.publicKey, mlDsa: keys2.mlDsa.publicKey },
17659
17902
  preflight: {
17660
- vcpu: cpus2().length,
17661
- memoryGib: Math.max(1, Math.floor(totalmem2() / 1024 ** 3)),
17903
+ vcpu: cpus3().length,
17904
+ memoryGib: Math.max(1, Math.floor(totalmem3() / 1024 ** 3)),
17662
17905
  diskGib: Math.max(1, Math.floor(Number(filesystem.blocks) * Number(filesystem.bsize) / 1024 ** 3)),
17663
17906
  kvm: existsSync23("/dev/kvm"),
17664
17907
  snpHost: existsSync23("/dev/sev"),
@@ -17784,7 +18027,7 @@ if (import.meta.main) {
17784
18027
  key: keyArg?.slice("--key=".length) ?? ""
17785
18028
  } : { op: command3 };
17786
18029
  try {
17787
- const response = await requestControl(request, socketPath);
18030
+ const response = await requestControl(request, socketPath, controlRequestTimeout(request.op));
17788
18031
  console.log(JSON.stringify(response, null, 2));
17789
18032
  process.exit(response.ok ? 0 : 1);
17790
18033
  } catch (cause) {
@@ -17836,14 +18079,17 @@ if (import.meta.main) {
17836
18079
  let secretCache;
17837
18080
  let vaultSync;
17838
18081
  let initialVaultState = "unbound";
18082
+ let initialAttestationVerified = !attestationSource;
17839
18083
  if (binding && attestationSource && nodeApiUrl) {
17840
- const result = await telemetry.observe("attestation.refresh", () => attestNodeOnce({
18084
+ const initial = await attemptInitialAttestation(() => telemetry.observe("attestation.refresh", () => attestNodeOnce({
17841
18085
  apiUrl: nodeApiUrl,
17842
18086
  nodeKey,
17843
18087
  keys,
17844
18088
  source: attestationSource
17845
- }));
17846
- console.log(`[agent] initial SEV-SNP attestation verified ${result.measurement.slice(0, 16)}\u2026`);
18089
+ })), (cause) => console.error(`[agent] initial SEV-SNP attestation deferred: ${cause.message}`));
18090
+ initialAttestationVerified = initial.verified;
18091
+ if (initial.verified)
18092
+ console.log(`[agent] initial SEV-SNP attestation verified ${initial.result.measurement.slice(0, 16)}\u2026`);
17847
18093
  }
17848
18094
  if (binding && nodeApiUrl) {
17849
18095
  secretCache = createNodeVaultCache({
@@ -17852,21 +18098,35 @@ if (import.meta.main) {
17852
18098
  keys,
17853
18099
  projectKey: binding.projectKey
17854
18100
  });
17855
- const initialVault = await telemetry.observe("vault.sync", () => initializeVaultReplica(secretCache), (result) => result.state === "ready" ? "success" : "failed");
17856
- initialVaultState = initialVault.state;
17857
- running.setVault(secretCache, binding.projectKey);
17858
- vaultSync = startNodeVaultSync(secretCache, {
17859
- telemetry,
17860
- onEvent: (event, detail) => console.log(`[agent] vault ${event}${detail ? ` ${JSON.stringify(detail)}` : ""}`)
17861
- });
17862
- if (initialVault.state === "ready") {
17863
- console.log(`[agent] in-memory vault loaded for every environment in project ${binding.projectKey}`);
17864
- } else if (initialVault.state === "partial") {
17865
- console.error(`[agent] vault replica partial (${initialVault.loaded} loaded, ${initialVault.failed.length} unavailable); ` + "explicit same-name systemd deployment fallbacks remain available");
17866
- } else {
17867
- console.error("[agent] vault replica unavailable; authenticated refresh remains active and only explicit " + "same-name systemd deployment fallbacks are available");
17868
- }
17869
18101
  }
18102
+ let vaultActivation;
18103
+ const activateVaultReplica = () => {
18104
+ if (!binding || !secretCache || vaultSync)
18105
+ return Promise.resolve();
18106
+ if (vaultActivation)
18107
+ return vaultActivation;
18108
+ vaultActivation = (async () => {
18109
+ const initialVault = await telemetry.observe("vault.sync", () => initializeVaultReplica(secretCache), (result) => result.state === "ready" ? "success" : "failed");
18110
+ initialVaultState = initialVault.state;
18111
+ running.setVault(secretCache, binding.projectKey);
18112
+ vaultSync = startNodeVaultSync(secretCache, {
18113
+ telemetry,
18114
+ onEvent: (event, detail) => console.log(`[agent] vault ${event}${detail ? ` ${JSON.stringify(detail)}` : ""}`)
18115
+ });
18116
+ if (initialVault.state === "ready") {
18117
+ console.log(`[agent] in-memory vault loaded for every environment in project ${binding.projectKey}`);
18118
+ } else if (initialVault.state === "partial") {
18119
+ console.error(`[agent] vault replica partial (${initialVault.loaded} loaded, ${initialVault.failed.length} unavailable); ` + "explicit same-name systemd deployment fallbacks remain available");
18120
+ } else {
18121
+ console.error("[agent] vault replica unavailable; authenticated refresh remains active and only explicit " + "same-name systemd deployment fallbacks are available");
18122
+ }
18123
+ })().finally(() => {
18124
+ vaultActivation = undefined;
18125
+ });
18126
+ return vaultActivation;
18127
+ };
18128
+ if (initialAttestationVerified)
18129
+ await activateVaultReplica();
17870
18130
  if (handoverCandidate) {
17871
18131
  const ready = startAgentCandidateReady({
17872
18132
  version: VERSION2,
@@ -17898,8 +18158,12 @@ if (import.meta.main) {
17898
18158
  keys,
17899
18159
  telemetry,
17900
18160
  source: attestationSource,
17901
- immediate: !binding,
17902
- onEvent: (event, detail) => console.log(`[agent] attestation ${event}${detail ? ` ${detail instanceof Error ? detail.message : JSON.stringify(detail)}` : ""}`)
18161
+ immediate: !initialAttestationVerified,
18162
+ onEvent: (event, detail) => {
18163
+ console.log(`[agent] attestation ${event}${detail ? ` ${detail instanceof Error ? detail.message : JSON.stringify(detail)}` : ""}`);
18164
+ if (event === "verified")
18165
+ activateVaultReplica().catch((cause) => console.error(`[agent] vault activation deferred: ${cause.message}`));
18166
+ }
17903
18167
  }) : undefined;
17904
18168
  if (attestationLoop)
17905
18169
  console.log("[agent] periodic SEV-SNP attestation enabled");
@@ -17990,6 +18254,7 @@ if (import.meta.main) {
17990
18254
  gitCredentialPath: process.env.CREDENTIALS_DIRECTORY ? `${process.env.CREDENTIALS_DIRECTORY}/git-deploy-key` : undefined,
17991
18255
  knownHostsPath: source.knownHosts ? join12(root, "cache", `known-hosts-${key}`) : undefined,
17992
18256
  knownHostsContent: source.knownHosts,
18257
+ attestation: attestationSource,
17993
18258
  cache: deploymentSecrets,
17994
18259
  capacityEvidenceDirectory: process.env.FZ_CAPACITY_EVIDENCE_DIR ?? join12(root, "cache", "capacity"),
17995
18260
  ensureSoftware: (requirements) => requestSoftware(requirements, process.env.FZ_SOFTWARE_HELPER_SOCKET ?? DEFAULT_SOFTWARE_HELPER_SOCKET),
@@ -17999,10 +18264,10 @@ if (import.meta.main) {
17999
18264
  applyConnectivity: (request) => requestDeploymentConnectivity(request, process.env.FZ_SOFTWARE_HELPER_SOCKET ?? DEFAULT_SOFTWARE_HELPER_SOCKET),
18000
18265
  projectExec: process.env.FZ_DEPLOY_RUNNER_SOCKET ? (input) => requestDeploymentCommand(input, process.env.FZ_DEPLOY_RUNNER_SOCKET) : undefined
18001
18266
  });
18002
- const staticManager = repository && branch && profile ? createDeploymentManager(managerOptions({ repository, branch, profile }, process.env.FZ_DEPLOY_KEY ?? `${repository}:${branch}:${profile}`)) : bootstrapBundle && branch && profile ? createDeploymentManager({
18267
+ const staticManager = bootstrapBundle && branch && profile ? createDeploymentManager({
18003
18268
  ...managerOptions({ repository: bootstrapBundle.path, branch, profile }, process.env.FZ_DEPLOY_KEY ?? `bootstrap:${bootstrapBundle.manifest.sha256}:${profile}`),
18004
18269
  bootstrapBundle
18005
- }) : undefined;
18270
+ }) : repository && branch && profile ? createDeploymentManager(managerOptions({ repository, branch, profile }, process.env.FZ_DEPLOY_KEY ?? `${repository}:${branch}:${profile}`)) : undefined;
18006
18271
  if (staticManager)
18007
18272
  managers.set("__static__", staticManager);
18008
18273
  const control = staticManager ? startControlServer(staticManager, process.env.FZ_CONTROL_SOCKET ?? DEFAULT_CONTROL_SOCKET) : undefined;
@@ -18276,6 +18541,7 @@ export {
18276
18541
  cloudInit,
18277
18542
  calibrateHttpConcurrency,
18278
18543
  attestNodeOnce,
18544
+ attemptInitialAttestation,
18279
18545
  assertSupportedGuestImage,
18280
18546
  applyMetalIsolation,
18281
18547
  applyAgentEgressPolicy,