@forgezero/agent 0.1.87 → 0.1.89

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/fz-agent.js CHANGED
@@ -4,7 +4,7 @@
4
4
  // src/index.ts
5
5
  import { randomBytes as randomBytes7 } from "crypto";
6
6
  import { readFileSync as readFileSync20, writeFileSync as writeFileSync18, existsSync as existsSync23, mkdirSync as mkdirSync18, chmodSync as chmodSync19, lstatSync as lstatSync10, statfsSync as statfsSync3 } from "fs";
7
- import { cpus as cpus2, totalmem as totalmem2 } from "os";
7
+ import { cpus as cpus3, totalmem as totalmem3 } from "os";
8
8
  import { dirname as dirname16, join as join12 } from "path";
9
9
 
10
10
  // ../access/dist/security.js
@@ -5351,7 +5351,12 @@ async function executeSoftwareOperation(operation) {
5351
5351
  }
5352
5352
  if (software === "nginx") {
5353
5353
  const binary = await run(["/usr/sbin/nginx", "-v"]);
5354
- return binary.exitCode === 0 ? run(["/usr/bin/systemctl", "is-active", "--quiet", "nginx.service"]) : binary;
5354
+ if (binary.exitCode !== 0)
5355
+ return binary;
5356
+ if (!existsSync2("/usr/lib/nginx/modules/ngx_http_headers_more_filter_module.so")) {
5357
+ return { exitCode: 1, output: "Nginx response-header suppression module is unavailable" };
5358
+ }
5359
+ return run(["/usr/bin/systemctl", "is-active", "--quiet", "nginx.service"]);
5355
5360
  }
5356
5361
  if (software === "arangodb") {
5357
5362
  const binary = await run(["/usr/sbin/arangod", "--version"]);
@@ -5383,7 +5388,7 @@ async function executeSoftwareOperation(operation) {
5383
5388
  }
5384
5389
  if (software === "docker" || software === "nginx" || software === "ufw" || software === "openssh-client" || software === "git") {
5385
5390
  const packageName = software === "openssh-client" ? "openssh-client" : software === "docker" ? "docker.io" : software;
5386
- const installed = await aptInstall(packageName);
5391
+ const installed = software === "nginx" ? await aptInstallMany(["nginx", "libnginx-mod-http-headers-more-filter"]) : await aptInstall(packageName);
5387
5392
  if (installed.exitCode !== 0 || software !== "nginx" && software !== "docker")
5388
5393
  return installed;
5389
5394
  if (software === "docker") {
@@ -5535,11 +5540,22 @@ async function ensureSoftwareRequirements(requirementsInput, options) {
5535
5540
  }
5536
5541
 
5537
5542
  // src/capacity-calibration.ts
5538
- var percentile95 = (values) => {
5543
+ import { cpus, freemem, totalmem } from "os";
5544
+ var percentile = (values, quantile) => {
5539
5545
  if (values.length === 0)
5540
5546
  return Number.POSITIVE_INFINITY;
5541
5547
  const sorted = values.toSorted((left, right) => left - right);
5542
- return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * 0.95) - 1)];
5548
+ return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * quantile) - 1)];
5549
+ };
5550
+ var hostSample = () => {
5551
+ let cpuIdle = 0;
5552
+ let cpuTotal = 0;
5553
+ for (const cpu of cpus()) {
5554
+ cpuIdle += cpu.times.idle;
5555
+ cpuTotal += Object.values(cpu.times).reduce((sum, value) => sum + value, 0);
5556
+ }
5557
+ const memoryTotal = totalmem();
5558
+ return { cpuIdle, cpuTotal, memoryUsed: memoryTotal - freemem(), memoryTotal };
5543
5559
  };
5544
5560
  function localCalibrationEndpoint(value) {
5545
5561
  const endpoint = new URL(value);
@@ -5554,9 +5570,12 @@ function validateCapacityCalibrationOptions(options) {
5554
5570
  const requestsPerWorker = options.requestsPerWorker ?? 8;
5555
5571
  const maxP95Ms = options.maxP95Ms ?? 250;
5556
5572
  const maxErrorRate = options.maxErrorRate ?? 0.01;
5557
- const headroomRatio = options.headroomRatio ?? 0.8;
5573
+ const safetyRatio = options.safetyRatio ?? 0.8;
5558
5574
  const requestTimeoutMs = options.requestTimeoutMs ?? 5000;
5559
- if (!Number.isSafeInteger(maxConcurrency) || maxConcurrency < 1 || maxConcurrency > 4096 || !Number.isSafeInteger(requestsPerWorker) || requestsPerWorker < 2 || requestsPerWorker > 100 || !Number.isFinite(maxP95Ms) || maxP95Ms < 1 || !Number.isFinite(maxErrorRate) || maxErrorRate < 0 || maxErrorRate > 0.2 || !Number.isFinite(headroomRatio) || headroomRatio < 0.25 || headroomRatio > 0.95 || !Number.isSafeInteger(requestTimeoutMs) || requestTimeoutMs < 100 || requestTimeoutMs > 30000) {
5575
+ const minimumStageDurationMs = options.minimumStageDurationMs ?? 5000;
5576
+ const maxCpuUtilizationPercent = options.maxCpuUtilizationPercent ?? 90;
5577
+ const maxMemoryUtilizationPercent = options.maxMemoryUtilizationPercent ?? 90;
5578
+ if (!Number.isSafeInteger(maxConcurrency) || maxConcurrency < 1 || maxConcurrency > 4096 || !Number.isSafeInteger(requestsPerWorker) || requestsPerWorker < 2 || requestsPerWorker > 100 || !Number.isFinite(maxP95Ms) || maxP95Ms < 1 || !Number.isFinite(maxErrorRate) || maxErrorRate < 0 || maxErrorRate > 0.2 || !Number.isFinite(safetyRatio) || safetyRatio < 0.25 || safetyRatio > 0.95 || !Number.isSafeInteger(requestTimeoutMs) || requestTimeoutMs < 100 || requestTimeoutMs > 30000 || !Number.isSafeInteger(minimumStageDurationMs) || minimumStageDurationMs < 100 || minimumStageDurationMs > 60000 || !Number.isFinite(maxCpuUtilizationPercent) || maxCpuUtilizationPercent < 25 || maxCpuUtilizationPercent > 100 || !Number.isFinite(maxMemoryUtilizationPercent) || maxMemoryUtilizationPercent < 25 || maxMemoryUtilizationPercent > 100) {
5560
5579
  throw new Error("Capacity calibration bounds are invalid.");
5561
5580
  }
5562
5581
  return {
@@ -5565,18 +5584,24 @@ function validateCapacityCalibrationOptions(options) {
5565
5584
  requestsPerWorker,
5566
5585
  maxP95Ms,
5567
5586
  maxErrorRate,
5568
- headroomRatio,
5569
- requestTimeoutMs
5587
+ safetyRatio,
5588
+ requestTimeoutMs,
5589
+ minimumStageDurationMs,
5590
+ maxCpuUtilizationPercent,
5591
+ maxMemoryUtilizationPercent
5570
5592
  };
5571
5593
  }
5572
- async function stage(endpoint, concurrency, requestsPerWorker, timeoutMs, fetcher) {
5594
+ async function stage(endpoint, concurrency, requestsPerWorker, timeoutMs, minimumDurationMs, fetcher, sampler) {
5573
5595
  const latencies = [];
5574
5596
  let succeeded = 0;
5575
5597
  let failed = 0;
5576
5598
  let overloaded = 0;
5599
+ const systemBefore = sampler();
5577
5600
  const started = performance.now();
5578
5601
  await Promise.all(Array.from({ length: concurrency }, async () => {
5579
- for (let request = 0;request < requestsPerWorker; request += 1) {
5602
+ let request = 0;
5603
+ while (request < requestsPerWorker || performance.now() - started < minimumDurationMs) {
5604
+ request += 1;
5580
5605
  const requestStarted = performance.now();
5581
5606
  try {
5582
5607
  const response = await fetcher(endpoint, {
@@ -5600,50 +5625,74 @@ async function stage(endpoint, concurrency, requestsPerWorker, timeoutMs, fetche
5600
5625
  }
5601
5626
  }
5602
5627
  }));
5603
- const elapsedSeconds = Math.max((performance.now() - started) / 1000, 0.001);
5628
+ const durationMs = Math.max(performance.now() - started, 1);
5629
+ const elapsedSeconds = durationMs / 1000;
5630
+ const systemAfter = sampler();
5631
+ const cpuTotal = Math.max(0, systemAfter.cpuTotal - systemBefore.cpuTotal);
5632
+ const cpuIdle = Math.max(0, systemAfter.cpuIdle - systemBefore.cpuIdle);
5633
+ const cpuUtilizationPercent = cpuTotal === 0 ? 0 : (cpuTotal - cpuIdle) / cpuTotal * 100;
5634
+ const memoryUtilizationPercent = Math.max(systemBefore.memoryTotal > 0 ? systemBefore.memoryUsed / systemBefore.memoryTotal * 100 : 0, systemAfter.memoryTotal > 0 ? systemAfter.memoryUsed / systemAfter.memoryTotal * 100 : 0);
5635
+ const requests = succeeded + failed;
5604
5636
  return {
5605
5637
  concurrency,
5606
- requests: concurrency * requestsPerWorker,
5638
+ requests,
5607
5639
  succeeded,
5608
5640
  failed,
5609
5641
  overloaded,
5610
- throughputPerSecond: Number(((succeeded + failed) / elapsedSeconds).toFixed(2)),
5611
- p95Ms: Number(percentile95(latencies).toFixed(2))
5642
+ successfulRequestsPerSecond: Number((succeeded / elapsedSeconds).toFixed(2)),
5643
+ attemptedRequestsPerSecond: Number((requests / elapsedSeconds).toFixed(2)),
5644
+ durationMs: Number(durationMs.toFixed(2)),
5645
+ errorRate: Number((failed / Math.max(requests, 1)).toFixed(6)),
5646
+ p95Ms: Number(percentile(latencies, 0.95).toFixed(2)),
5647
+ p99Ms: Number(percentile(latencies, 0.99).toFixed(2)),
5648
+ cpuUtilizationPercent: Number(cpuUtilizationPercent.toFixed(2)),
5649
+ memoryUtilizationPercent: Number(memoryUtilizationPercent.toFixed(2))
5612
5650
  };
5613
5651
  }
5614
- async function calibrateHttpConcurrency(options, fetcher = fetch) {
5652
+ async function calibrateHttpConcurrency(options, fetcher = fetch, sampler = hostSample) {
5615
5653
  const {
5616
5654
  endpoint,
5617
5655
  maxConcurrency,
5618
5656
  requestsPerWorker,
5619
5657
  maxP95Ms,
5620
5658
  maxErrorRate,
5621
- headroomRatio,
5622
- requestTimeoutMs: timeoutMs
5659
+ safetyRatio,
5660
+ requestTimeoutMs: timeoutMs,
5661
+ minimumStageDurationMs,
5662
+ maxCpuUtilizationPercent,
5663
+ maxMemoryUtilizationPercent
5623
5664
  } = validateCapacityCalibrationOptions(options);
5624
5665
  const stages = [];
5625
- let lastSafe = 1;
5666
+ let lastSafe;
5626
5667
  let stopReason = "maximum-tested";
5627
5668
  for (let concurrency = 1;; concurrency = Math.min(maxConcurrency, concurrency * 2)) {
5628
- const measured = await stage(endpoint, concurrency, requestsPerWorker, timeoutMs, fetcher);
5669
+ const measured = await stage(endpoint, concurrency, requestsPerWorker, timeoutMs, minimumStageDurationMs, fetcher, sampler);
5629
5670
  stages.push(measured);
5630
- const errorRate = measured.failed / measured.requests;
5631
5671
  const previous = stages.at(-2);
5632
- const throughputRegressed = Boolean(previous && concurrency > 1 && measured.throughputPerSecond < previous.throughputPerSecond * 0.9);
5633
- if (measured.overloaded > 0 || errorRate > maxErrorRate)
5672
+ const throughputRegressed = Boolean(previous && concurrency > 1 && measured.successfulRequestsPerSecond < previous.successfulRequestsPerSecond * 0.9);
5673
+ if (measured.overloaded > 0 || measured.errorRate > maxErrorRate)
5634
5674
  stopReason = "errors";
5635
5675
  else if (measured.p95Ms > maxP95Ms)
5636
5676
  stopReason = "latency";
5677
+ else if (measured.cpuUtilizationPercent > maxCpuUtilizationPercent)
5678
+ stopReason = "cpu";
5679
+ else if (measured.memoryUtilizationPercent > maxMemoryUtilizationPercent)
5680
+ stopReason = "memory";
5637
5681
  else if (throughputRegressed)
5638
5682
  stopReason = "throughput-regression";
5639
5683
  else
5640
- lastSafe = concurrency;
5684
+ lastSafe = measured;
5641
5685
  if (stopReason !== "maximum-tested" || concurrency === maxConcurrency)
5642
5686
  break;
5643
5687
  }
5688
+ if (!lastSafe)
5689
+ throw new Error("Capacity calibration found no safe request stage.");
5644
5690
  return {
5645
5691
  endpoint: endpoint.toString(),
5646
- recommendedConcurrency: Math.max(1, Math.floor(lastSafe * headroomRatio)),
5692
+ measuredSustainableRequestsPerSecond: lastSafe.successfulRequestsPerSecond,
5693
+ allowedRequestsPerSecond: Math.max(1, Math.floor(lastSafe.successfulRequestsPerSecond * safetyRatio)),
5694
+ safetyRatio,
5695
+ recommendedConcurrency: Math.max(1, Math.floor(lastSafe.concurrency * safetyRatio)),
5647
5696
  stopReason,
5648
5697
  stages
5649
5698
  };
@@ -5720,8 +5769,11 @@ function capacityCalibration(value, where) {
5720
5769
  "requestsPerWorker",
5721
5770
  "maxP95Ms",
5722
5771
  "maxErrorRate",
5723
- "headroomRatio",
5724
- "requestTimeoutMs"
5772
+ "safetyRatio",
5773
+ "requestTimeoutMs",
5774
+ "minimumStageDurationMs",
5775
+ "maxCpuUtilizationPercent",
5776
+ "maxMemoryUtilizationPercent"
5725
5777
  ], where);
5726
5778
  const endpoint = text(calibration.endpoint, `${where}.endpoint`);
5727
5779
  try {
@@ -5745,8 +5797,11 @@ function capacityCalibration(value, where) {
5745
5797
  "requestsPerWorker",
5746
5798
  "maxP95Ms",
5747
5799
  "maxErrorRate",
5748
- "headroomRatio",
5749
- "requestTimeoutMs"
5800
+ "safetyRatio",
5801
+ "requestTimeoutMs",
5802
+ "minimumStageDurationMs",
5803
+ "maxCpuUtilizationPercent",
5804
+ "maxMemoryUtilizationPercent"
5750
5805
  ].flatMap((name) => {
5751
5806
  const found = optionalNumber(name);
5752
5807
  return found === undefined ? [] : [[name, found]];
@@ -7867,6 +7922,9 @@ function parseBootstrapBundleManifest(value) {
7867
7922
  }
7868
7923
 
7869
7924
  // src/deployment.ts
7925
+ var RELEASE_GROUP = "forgezero-deploy";
7926
+ var deploymentAttestationChallenge = (revision) => createHash5("sha256").update("forgezero/deployment-attestation/v1\x00").update(revision, "utf8").digest("hex");
7927
+
7870
7928
  class DeploymentError extends Error {
7871
7929
  code;
7872
7930
  constructor(code, message) {
@@ -7926,6 +7984,7 @@ function persistCapacityCalibration(args) {
7926
7984
  const path2 = join2(args.directory, `${identity}-${args.measuredAtTs}-${randomUUID().slice(0, 8)}.json`);
7927
7985
  const next = `${path2}.next`;
7928
7986
  const recommendedCoordinate = {
7987
+ FZ_REQUESTS_PER_SECOND_LIMIT: String(args.calibration.allowedRequestsPerSecond),
7929
7988
  FZ_CONCURRENCY_LIMIT: String(args.calibration.recommendedConcurrency)
7930
7989
  };
7931
7990
  writeFileSync3(next, `${JSON.stringify({
@@ -7961,6 +8020,10 @@ function createDeploymentManager(options) {
7961
8020
  const activeRevisions = new Map;
7962
8021
  const exec = options.exec ?? spawnExact;
7963
8022
  const projectExec = options.projectExec ?? exec;
8023
+ const releaseGroup = options.releaseGroup ?? RELEASE_GROUP;
8024
+ if (!/^(?:[a-z_][a-z0-9_-]{0,30}|[0-9]{1,10})$/.test(releaseGroup)) {
8025
+ throw new DeploymentError("SOURCE_FAILED", "release group is invalid");
8026
+ }
7964
8027
  const now = options.now ?? Date.now;
7965
8028
  const readDefinition = options.readDefinition ?? ((path2) => JSON.parse(readFileSync3(path2, "utf8")));
7966
8029
  const credentialsDirectory = process.env.CREDENTIALS_DIRECTORY;
@@ -8100,7 +8163,6 @@ function createDeploymentManager(options) {
8100
8163
  const env = await gitEnvironment();
8101
8164
  const releasesDirectory = join2(options.root, "releases");
8102
8165
  await checked({ argv: ["/usr/bin/test", "-d", releasesDirectory] }, "SOURCE_FAILED");
8103
- await checked({ argv: ["/usr/bin/test", "-w", releasesDirectory] }, "SOURCE_FAILED");
8104
8166
  await checked({ argv: ["/usr/bin/install", "-d", "-m", "0770", release] }, "SOURCE_FAILED");
8105
8167
  if (bootstrapBundle) {
8106
8168
  let metadata;
@@ -8112,7 +8174,14 @@ function createDeploymentManager(options) {
8112
8174
  if (!metadata.isFile() || metadata.isSymbolicLink() || metadata.nlink !== 1 || metadata.size !== bootstrapBundle.manifest.bytes || await sha256File(bootstrapBundle.path) !== bootstrapBundle.manifest.sha256) {
8113
8175
  throw new DeploymentError("SOURCE_FAILED", "The attended bootstrap bundle failed its manifest check.");
8114
8176
  }
8115
- await checked({ argv: ["git", "bundle", "verify", bootstrapBundle.path], env }, "SOURCE_FAILED");
8177
+ const verifier = `${release}.bundle-verify`;
8178
+ try {
8179
+ await checked({ argv: ["/usr/bin/install", "-d", "-m", "0700", verifier] }, "SOURCE_FAILED");
8180
+ await checked({ argv: ["git", "-C", verifier, "init", "--bare", "--quiet"], env }, "SOURCE_FAILED");
8181
+ await checked({ argv: ["git", "-C", verifier, "bundle", "verify", bootstrapBundle.path], env }, "SOURCE_FAILED");
8182
+ } finally {
8183
+ await exec({ argv: ["/usr/bin/rm", "-rf", "--", verifier] });
8184
+ }
8116
8185
  }
8117
8186
  await checked({
8118
8187
  argv: ["git", "clone", "--quiet", "--no-checkout", "--branch", options.branch, "--depth", "100", options.repository, release],
@@ -8150,6 +8219,10 @@ function createDeploymentManager(options) {
8150
8219
  cwd: release,
8151
8220
  env
8152
8221
  }, "SOURCE_FAILED");
8222
+ if (options.projectExec) {
8223
+ await checked({ argv: ["/bin/chgrp", "-hR", releaseGroup, release] }, "SOURCE_FAILED");
8224
+ await checked({ argv: ["/bin/chmod", "-R", "g+rX,o-rwx", release] }, "SOURCE_FAILED");
8225
+ }
8153
8226
  const head = (await checked({ argv: ["git", "rev-parse", "HEAD"], cwd: release, env }, "SOURCE_FAILED")).output.trim();
8154
8227
  if ((request.revision || bootstrapBundle) && head !== desiredRevision) {
8155
8228
  throw new DeploymentError("SOURCE_FAILED", `Git checked out ${head}, not requested ${desiredRevision}.`);
@@ -8172,7 +8245,7 @@ function createDeploymentManager(options) {
8172
8245
  throw new DeploymentError("PIPELINE_FAILED", "deployment plan requires attestation but this Agent has no source");
8173
8246
  if (options.attestation) {
8174
8247
  try {
8175
- await options.attestation.report(head);
8248
+ await options.attestation.report(deploymentAttestationChallenge(head));
8176
8249
  } catch (cause) {
8177
8250
  if (assurance === "required")
8178
8251
  throw new DeploymentError("PIPELINE_FAILED", `deployment attestation failed: ${cause.message}`);
@@ -8543,7 +8616,7 @@ function createDeploymentManager(options) {
8543
8616
  }
8544
8617
  return options.cache.get(name);
8545
8618
  };
8546
- const attest = options.attestation ? async () => ({ report: await options.attestation.report(head), source: options.attestation.name }) : undefined;
8619
+ const attest = options.attestation ? async () => ({ report: await options.attestation.report(deploymentAttestationChallenge(head)), source: options.attestation.name }) : undefined;
8547
8620
  const pipelines = [
8548
8621
  ...["build", "migrate", "release", "health"].map((phase) => {
8549
8622
  return {
@@ -8605,6 +8678,7 @@ function createDeploymentManager(options) {
8605
8678
  ...calibration,
8606
8679
  evidencePath,
8607
8680
  recommendedCoordinate: {
8681
+ FZ_REQUESTS_PER_SECOND_LIMIT: String(calibration.allowedRequestsPerSecond),
8608
8682
  FZ_CONCURRENCY_LIMIT: String(calibration.recommendedConcurrency)
8609
8683
  }
8610
8684
  };
@@ -8753,27 +8827,64 @@ function startControlServer(manager, socketPath = DEFAULT_CONTROL_SOCKET) {
8753
8827
  server.listen(socketPath, () => chmodSync5(socketPath, 384));
8754
8828
  return server;
8755
8829
  }
8756
- function requestControl(request, socketPath = DEFAULT_CONTROL_SOCKET) {
8830
+ function requestControl(request, socketPath = DEFAULT_CONTROL_SOCKET, timeoutMs = 30000) {
8757
8831
  return new Promise((resolve, reject) => {
8758
- const socket = connect(socketPath, () => socket.write(`${JSON.stringify(request)}
8759
- `));
8760
- let buffer = "";
8761
- socket.on("data", (chunk) => {
8762
- buffer += chunk.toString("utf8");
8763
- const newline = buffer.indexOf(`
8764
- `);
8765
- if (newline === -1)
8832
+ const deadline = Date.now() + timeoutMs;
8833
+ let settled = false;
8834
+ const finish = (cause, response) => {
8835
+ if (settled)
8766
8836
  return;
8767
- socket.end();
8768
- try {
8769
- resolve(JSON.parse(buffer.slice(0, newline)));
8770
- } catch (cause) {
8837
+ settled = true;
8838
+ if (cause)
8771
8839
  reject(cause);
8840
+ else
8841
+ resolve(response);
8842
+ };
8843
+ const attempt = () => {
8844
+ if (settled)
8845
+ return;
8846
+ const remaining = deadline - Date.now();
8847
+ if (remaining <= 0)
8848
+ return finish(new Error("agent control socket did not answer before its deadline"));
8849
+ if (!existsSync5(socketPath)) {
8850
+ setTimeout(attempt, Math.min(100, Math.max(1, remaining)));
8851
+ return;
8772
8852
  }
8773
- });
8774
- socket.on("error", reject);
8853
+ let buffer = "";
8854
+ const socket = connect(socketPath, () => socket.write(`${JSON.stringify(request)}
8855
+ `));
8856
+ socket.setTimeout(remaining, () => {
8857
+ socket.destroy();
8858
+ finish(new Error("agent control socket did not answer before its deadline"));
8859
+ });
8860
+ socket.on("data", (chunk) => {
8861
+ buffer += chunk.toString("utf8");
8862
+ const newline = buffer.indexOf(`
8863
+ `);
8864
+ if (newline === -1)
8865
+ return;
8866
+ socket.end();
8867
+ try {
8868
+ finish(undefined, JSON.parse(buffer.slice(0, newline)));
8869
+ } catch (cause) {
8870
+ finish(cause);
8871
+ }
8872
+ });
8873
+ socket.on("error", (cause) => {
8874
+ socket.destroy();
8875
+ if ((cause.code === "ENOENT" || cause.code === "ECONNREFUSED") && Date.now() < deadline) {
8876
+ setTimeout(attempt, Math.min(100, Math.max(1, deadline - Date.now())));
8877
+ return;
8878
+ }
8879
+ finish(cause);
8880
+ });
8881
+ };
8882
+ attempt();
8775
8883
  });
8776
8884
  }
8885
+ function controlRequestTimeout(operation) {
8886
+ return operation === "deploy" ? 600000 : 30000;
8887
+ }
8777
8888
 
8778
8889
  // src/deployment-pull.ts
8779
8890
  var remoteOutcome = (cause) => cause instanceof SignedNodeHttpError && cause.status < 500 ? "refused" : "retryable";
@@ -8917,6 +9028,9 @@ async function pullDeploymentOnce(options) {
8917
9028
  ...target.topology ? { topology: structuredClone(target.topology) } : {}
8918
9029
  })) } : {},
8919
9030
  ...result.capacity ? { capacity: {
9031
+ measuredSustainableRequestsPerSecond: result.capacity.measuredSustainableRequestsPerSecond,
9032
+ allowedRequestsPerSecond: result.capacity.allowedRequestsPerSecond,
9033
+ safetyRatio: result.capacity.safetyRatio,
8920
9034
  recommendedConcurrency: result.capacity.recommendedConcurrency,
8921
9035
  stopReason: result.capacity.stopReason,
8922
9036
  stages: result.capacity.stages.map((stage2) => ({ ...stage2 }))
@@ -9378,7 +9492,7 @@ async function writeAndCloseProcessInput(input, value) {
9378
9492
  }
9379
9493
 
9380
9494
  // src/version.ts
9381
- var VERSION2 = "0.1.87";
9495
+ var VERSION2 = "0.1.89";
9382
9496
 
9383
9497
  // src/ssh-bootstrap.ts
9384
9498
  class SshBootstrapError extends Error {
@@ -10271,15 +10385,15 @@ function linuxList(members) {
10271
10385
  return ranges.join(",");
10272
10386
  }
10273
10387
  function physicalCoreGroups(pool) {
10274
- const cpus = membersOfLinuxList(pool.cpus, "CPU");
10275
- if (cpus.length % pool.physicalCores !== 0) {
10388
+ const cpus2 = membersOfLinuxList(pool.cpus, "CPU");
10389
+ if (cpus2.length % pool.physicalCores !== 0) {
10276
10390
  throw new MetalProvisionError("CPU pool threads must divide evenly across physical cores");
10277
10391
  }
10278
- const threadsPerCore = cpus.length / pool.physicalCores;
10392
+ const threadsPerCore = cpus2.length / pool.physicalCores;
10279
10393
  if (!Number.isInteger(threadsPerCore) || threadsPerCore < 1 || threadsPerCore > 8) {
10280
10394
  throw new MetalProvisionError("invalid CPU pool threads-per-core topology");
10281
10395
  }
10282
- return Array.from({ length: pool.physicalCores }, (_, core) => Array.from({ length: threadsPerCore }, (_2, thread) => cpus[core + thread * pool.physicalCores]));
10396
+ return Array.from({ length: pool.physicalCores }, (_, core) => Array.from({ length: threadsPerCore }, (_2, thread) => cpus2[core + thread * pool.physicalCores]));
10283
10397
  }
10284
10398
  var guestNameFor = (computeKey) => `fzg-${createHash7("sha256").update(computeKey).digest("hex").slice(0, 16)}`;
10285
10399
  var tapNameFor = (computeKey) => `fzt${createHash7("sha256").update(computeKey).digest("hex").slice(0, 12)}`;
@@ -10334,12 +10448,12 @@ function validateMetalProfile(profile) {
10334
10448
  if (!SAFE_NAME.test(pool.key) || keys.has(pool.key))
10335
10449
  throw new MetalProvisionError("invalid or duplicate CPU pool key");
10336
10450
  keys.add(pool.key);
10337
- const cpus = membersOfLinuxList(pool.cpus, "CPU");
10338
- if (!Number.isInteger(pool.physicalCores) || pool.physicalCores < 1 || pool.physicalCores > cpus.length) {
10451
+ const cpus2 = membersOfLinuxList(pool.cpus, "CPU");
10452
+ if (!Number.isInteger(pool.physicalCores) || pool.physicalCores < 1 || pool.physicalCores > cpus2.length) {
10339
10453
  throw new MetalProvisionError("invalid CPU pool physical-core count");
10340
10454
  }
10341
10455
  physicalCoreGroups(pool);
10342
- for (const cpu of cpus) {
10456
+ for (const cpu of cpus2) {
10343
10457
  if (assigned.has(cpu))
10344
10458
  throw new MetalProvisionError("CPU pools overlap");
10345
10459
  assigned.add(cpu);
@@ -11293,13 +11407,13 @@ var memoryDirective = (nodes) => nodes ? `AllowedMemoryNodes=${nodes}
11293
11407
  ` : "";
11294
11408
  function metalGuestSliceUnit(profile) {
11295
11409
  validateMetalProfile(profile);
11296
- const cpus = compact(profile.cpuPools.flatMap((pool) => members(pool.cpus)));
11410
+ const cpus2 = compact(profile.cpuPools.flatMap((pool) => members(pool.cpus)));
11297
11411
  const nodes = compact(profile.cpuPools.flatMap((pool) => pool.memoryNodes ? members(pool.memoryNodes) : [])) || undefined;
11298
11412
  return `[Unit]
11299
11413
  Description=ForgeZero exclusive guest CPU and memory boundary
11300
11414
 
11301
11415
  [Slice]
11302
- AllowedCPUs=${cpus}
11416
+ AllowedCPUs=${cpus2}
11303
11417
  ${memoryDirective(nodes)}`;
11304
11418
  }
11305
11419
  function metalHousekeepingDropIn(profile, kind) {
@@ -11987,7 +12101,7 @@ function probeAgentCandidate(expected, socketPath = DEFAULT_AGENT_CANDIDATE_READ
11987
12101
  return;
11988
12102
  try {
11989
12103
  const value = JSON.parse(buffer.slice(0, newline));
11990
- finish(value.version === expected.version && typeof value.nodeKey === "string" && typeof value.bound === "boolean" && ["ready", "partial", "unavailable", "unbound"].includes(value.vault ?? ""));
12104
+ finish(value.version === expected.version && value.nodeKey === expected.nodeKey && value.bound === expected.bound && expected.vault.includes(value.vault));
11991
12105
  } catch {
11992
12106
  finish(false);
11993
12107
  }
@@ -12190,13 +12304,18 @@ var restartAgent = async (target2, run2) => {
12190
12304
  }
12191
12305
  };
12192
12306
  var candidateUnit = (target2) => target2 === "compute" ? "forgezero-agent-candidate.service" : "forgezero-metal-agent-candidate.service";
12193
- var startCandidate = async (staged, target2, run2, probe = (expected) => probeAgentCandidate(expected, DEFAULT_AGENT_CANDIDATE_READY_SOCKET)) => {
12307
+ var startCandidate = async (staged, target2, expectedNodeKey, run2, probe = (expected) => probeAgentCandidate(expected, DEFAULT_AGENT_CANDIDATE_READY_SOCKET)) => {
12194
12308
  selectAgentCandidate(staged);
12195
12309
  if (!await runOk(run2, "/usr/bin/systemctl", ["restart", candidateUnit(target2)])) {
12196
12310
  clearAgentCandidate(dirname8(staged.currentLink));
12197
12311
  throw new Error(`systemd could not start ${candidateUnit(target2)}`);
12198
12312
  }
12199
- if (!await probe({ version: staged.version })) {
12313
+ if (!await probe({
12314
+ version: staged.version,
12315
+ nodeKey: expectedNodeKey,
12316
+ bound: true,
12317
+ vault: target2 === "compute" ? ["ready"] : ["unbound"]
12318
+ })) {
12200
12319
  await run2({ command: "/usr/bin/systemctl", args: ["stop", candidateUnit(target2)] });
12201
12320
  clearAgentCandidate(dirname8(staged.currentLink));
12202
12321
  throw new Error("the candidate Agent did not prove its identity and durable state");
@@ -12275,8 +12394,9 @@ async function activateAgentRelease(staged, options = {}) {
12275
12394
  writeUpdateState(journalPath, receiptPath, journal);
12276
12395
  let selectionAttempted = false;
12277
12396
  try {
12278
- if (!options.candidatePrepared)
12279
- await startCandidate(staged, target2, run2);
12397
+ if (!options.candidatePrepared) {
12398
+ throw new Error("direct activation requires a separately identity-bound candidate preparation");
12399
+ }
12280
12400
  if (target2 === "compute")
12281
12401
  switchAgentSocketRoute(paths.route, paths.candidate);
12282
12402
  selectionAttempted = true;
@@ -12466,6 +12586,9 @@ function startAgentUpdateHelper(options = {}) {
12466
12586
  }
12467
12587
  if (request.op !== "prepare")
12468
12588
  throw new Error("unknown update operation");
12589
+ if (typeof request.expectedNodeKey !== "string" || request.expectedNodeKey.length < 16 || Buffer.byteLength(request.expectedNodeKey, "utf8") > 512 || /[\0\r\n]/.test(request.expectedNodeKey)) {
12590
+ throw new Error("Agent update expected node identity is invalid");
12591
+ }
12469
12592
  if (busy)
12470
12593
  throw new Error("another Agent update or recovery is already active");
12471
12594
  const prior = readJournal(journalPath, releaseRoot);
@@ -12478,8 +12601,8 @@ function startAgentUpdateHelper(options = {}) {
12478
12601
  currentVersion: request.currentVersion,
12479
12602
  root: releaseRoot
12480
12603
  });
12481
- await startCandidate(staged, request.target, runCommand);
12482
- pending = { staged, target: request.target, attemptId };
12604
+ await startCandidate(staged, request.target, request.expectedNodeKey, runCommand);
12605
+ pending = { staged, target: request.target, attemptId, expectedNodeKey: request.expectedNodeKey };
12483
12606
  ownsBusy = false;
12484
12607
  setTimer(() => {
12485
12608
  if (pending?.attemptId !== attemptId)
@@ -12537,13 +12660,13 @@ function requestAgentUpdate(request, socketPath = DEFAULT_AGENT_UPDATE_SOCKET, t
12537
12660
 
12538
12661
  // src/agent-heartbeat.ts
12539
12662
  import { existsSync as existsSync15, readFileSync as readFileSync11, statfsSync as statfsSync2 } from "fs";
12540
- import { cpus, freemem, loadavg, totalmem } from "os";
12663
+ import { cpus as cpus2, freemem as freemem2, loadavg, totalmem as totalmem2 } from "os";
12541
12664
  function readAgentHostMetrics() {
12542
12665
  const filesystem = statfsSync2("/");
12543
12666
  return {
12544
- logicalCpu: cpus().length,
12545
- memoryBytes: totalmem(),
12546
- memoryFreeBytes: freemem(),
12667
+ logicalCpu: cpus2().length,
12668
+ memoryBytes: totalmem2(),
12669
+ memoryFreeBytes: freemem2(),
12547
12670
  storageBytes: Number(filesystem.blocks) * Number(filesystem.bsize),
12548
12671
  storageFreeBytes: Number(filesystem.bavail) * Number(filesystem.bsize),
12549
12672
  load1m: loadavg()[0]
@@ -12636,7 +12759,14 @@ async function heartbeatAgentOnce(options) {
12636
12759
  let candidatePrepared = false;
12637
12760
  let drained = false;
12638
12761
  try {
12639
- const update2 = options.applyUpdate ?? ((phase, next, current2, attemptId) => requestAgentUpdate(phase === "prepare" ? { op: "prepare", target: options.updateTarget ?? "compute", release: next, currentVersion: current2, attemptId } : {
12762
+ const update2 = options.applyUpdate ?? ((phase, next, current2, attemptId) => requestAgentUpdate(phase === "prepare" ? {
12763
+ op: "prepare",
12764
+ target: options.updateTarget ?? "compute",
12765
+ release: next,
12766
+ currentVersion: current2,
12767
+ expectedNodeKey: options.nodeKey,
12768
+ attemptId
12769
+ } : {
12640
12770
  op: phase,
12641
12771
  target: options.updateTarget ?? "compute",
12642
12772
  currentVersion: current2,
@@ -13679,30 +13809,71 @@ function requestServiceActivation(request, socketPath = DEFAULT_SOFTWARE_HELPER_
13679
13809
  function requestSoftware(requirements, socketPath = DEFAULT_SOFTWARE_HELPER_SOCKET, timeoutMs = 15 * 60000) {
13680
13810
  validateSoftwareRequirements(requirements);
13681
13811
  return new Promise((resolve6, reject) => {
13682
- const socket = connect7(socketPath, () => socket.write(`${JSON.stringify({ op: "ensure", requirements })}
13683
- `));
13684
- let buffer = "";
13685
- socket.setTimeout(timeoutMs, () => {
13686
- socket.destroy();
13687
- reject(new Error("software helper did not answer before its deadline"));
13688
- });
13689
- socket.on("data", (chunk) => {
13690
- buffer += chunk.toString("utf8");
13691
- const newline = buffer.indexOf(`
13692
- `);
13693
- if (newline < 0)
13812
+ const deadline = Date.now() + timeoutMs;
13813
+ let settled = false;
13814
+ const finish = (cause, results) => {
13815
+ if (settled)
13694
13816
  return;
13695
- socket.end();
13817
+ settled = true;
13818
+ if (cause)
13819
+ reject(cause);
13820
+ else
13821
+ resolve6(results);
13822
+ };
13823
+ const attempt = () => {
13824
+ if (settled)
13825
+ return;
13826
+ const remaining = deadline - Date.now();
13827
+ if (remaining <= 0)
13828
+ return finish(new Error("software helper did not answer before its deadline"));
13829
+ if (!existsSync18(socketPath)) {
13830
+ setTimeout(attempt, Math.min(100, Math.max(1, remaining)));
13831
+ return;
13832
+ }
13833
+ let buffer = "";
13834
+ let socket;
13696
13835
  try {
13697
- const response = JSON.parse(buffer.slice(0, newline));
13698
- if (!response.ok || !response.results)
13699
- throw new Error(response.error?.message ?? "software helper refused the request");
13700
- resolve6(response.results);
13836
+ socket = connect7(socketPath, () => socket.write(`${JSON.stringify({ op: "ensure", requirements })}
13837
+ `));
13701
13838
  } catch (cause) {
13702
- reject(cause);
13839
+ const error = cause;
13840
+ if ((error.code === "ENOENT" || error.code === "ECONNREFUSED") && Date.now() < deadline) {
13841
+ setTimeout(attempt, Math.min(100, Math.max(1, deadline - Date.now())));
13842
+ return;
13843
+ }
13844
+ finish(error);
13845
+ return;
13703
13846
  }
13704
- });
13705
- socket.on("error", reject);
13847
+ socket.setTimeout(remaining, () => {
13848
+ socket.destroy();
13849
+ finish(new Error("software helper did not answer before its deadline"));
13850
+ });
13851
+ socket.on("data", (chunk) => {
13852
+ buffer += chunk.toString("utf8");
13853
+ const newline = buffer.indexOf(`
13854
+ `);
13855
+ if (newline < 0)
13856
+ return;
13857
+ socket.end();
13858
+ try {
13859
+ const response = JSON.parse(buffer.slice(0, newline));
13860
+ if (!response.ok || !response.results)
13861
+ throw new Error(response.error?.message ?? "software helper refused the request");
13862
+ finish(undefined, response.results);
13863
+ } catch (cause) {
13864
+ finish(cause);
13865
+ }
13866
+ });
13867
+ socket.on("error", (cause) => {
13868
+ socket.destroy();
13869
+ if ((cause.code === "ENOENT" || cause.code === "ECONNREFUSED") && Date.now() < deadline) {
13870
+ setTimeout(attempt, Math.min(100, Math.max(1, deadline - Date.now())));
13871
+ return;
13872
+ }
13873
+ finish(cause);
13874
+ });
13875
+ };
13876
+ attempt();
13706
13877
  });
13707
13878
  }
13708
13879
 
@@ -13791,26 +13962,33 @@ function renderAgentEgressNft(uids, replace = false, loopback) {
13791
13962
  if (!uniqueUids.includes(loopback.uid))
13792
13963
  throw new Error("Loopback egress UID must be protected.");
13793
13964
  const ports = normalizeEgressTcpPorts(loopback.tcpPorts);
13965
+ const sharedPorts = normalizeEgressTcpPorts(loopback.sharedTcpPorts ?? []);
13794
13966
  const publicPorts = loopback.publicTcpPorts === undefined ? undefined : normalizeEgressTcpPorts(loopback.publicTcpPorts);
13795
13967
  if (publicPorts && publicPorts.length < 1) {
13796
13968
  throw new Error("Restricted public egress needs at least one TCP port.");
13797
13969
  }
13798
- if (ports.length < 1 && !publicPorts) {
13970
+ if (ports.length < 1 && sharedPorts.length < 1 && !publicPorts) {
13799
13971
  throw new Error("Restricted egress needs a loopback or public TCP port.");
13800
13972
  }
13801
- loopbackComment = (ports.length > 0 ? ` loopback=${loopback.uid}:${ports.join(",")}` : "") + (publicPorts ? ` public-tcp=${publicPorts.join(",")}` : "");
13973
+ loopbackComment = (ports.length > 0 ? ` loopback=${loopback.uid}:${ports.join(",")}` : "") + (sharedPorts.length > 0 ? ` shared-loopback=${sharedPorts.join(",")}` : "") + (publicPorts ? ` public-tcp=${publicPorts.join(",")}` : "");
13802
13974
  loopbackSet = `${ports.length > 0 ? ` set loopback_tcp_ports {
13803
13975
  type inet_service
13804
13976
  elements = { ${ports.join(", ")} }
13805
13977
  }
13978
+ ` : ""}${sharedPorts.length > 0 ? ` set shared_loopback_tcp_ports {
13979
+ type inet_service
13980
+ elements = { ${sharedPorts.join(", ")} }
13981
+ }
13806
13982
  ` : ""}${publicPorts ? ` set public_tcp_ports {
13807
13983
  type inet_service
13808
13984
  elements = { ${publicPorts.join(", ")} }
13809
13985
  }
13810
13986
  ` : ""}`;
13811
- loopbackRules = ports.length > 0 ? ` meta skuid ${loopback.uid} tcp dport @loopback_tcp_ports ip daddr 127.0.0.1 accept
13987
+ loopbackRules = `${sharedPorts.length > 0 ? ` meta skuid @service_uids tcp dport @shared_loopback_tcp_ports ip daddr 127.0.0.1 accept
13988
+ meta skuid @service_uids tcp dport @shared_loopback_tcp_ports ip6 daddr ::1 accept
13989
+ ` : ""}${ports.length > 0 ? ` meta skuid ${loopback.uid} tcp dport @loopback_tcp_ports ip daddr 127.0.0.1 accept
13812
13990
  meta skuid ${loopback.uid} tcp dport @loopback_tcp_ports ip6 daddr ::1 accept
13813
- ` : "";
13991
+ ` : ""}`;
13814
13992
  restrictedPublicRules = publicPorts ? ` meta skuid ${loopback.uid} tcp dport @public_tcp_ports accept
13815
13993
  meta skuid ${loopback.uid} reject
13816
13994
  ` : "";
@@ -13905,12 +14083,15 @@ function verifyAgentEgressPolicy(uids, command3 = systemCommand, loopback) {
13905
14083
  if (!uniqueUids.includes(loopback.uid))
13906
14084
  return false;
13907
14085
  const ports = normalizeEgressTcpPorts(loopback.tcpPorts);
14086
+ const sharedPorts = normalizeEgressTcpPorts(loopback.sharedTcpPorts ?? []);
13908
14087
  const publicPorts = loopback.publicTcpPorts === undefined ? undefined : normalizeEgressTcpPorts(loopback.publicTcpPorts);
13909
14088
  if (publicPorts && publicPorts.length < 1)
13910
14089
  return false;
13911
- if (ports.length < 1 && !publicPorts)
14090
+ if (ports.length < 1 && sharedPorts.length < 1 && !publicPorts)
13912
14091
  return false;
13913
- loopbackComment = (ports.length > 0 ? ` loopback=${loopback.uid}:${ports.join(",")}` : "") + (publicPorts ? ` public-tcp=${publicPorts.join(",")}` : "");
14092
+ loopbackComment = (ports.length > 0 ? ` loopback=${loopback.uid}:${ports.join(",")}` : "") + (sharedPorts.length > 0 ? ` shared-loopback=${sharedPorts.join(",")}` : "") + (publicPorts ? ` public-tcp=${publicPorts.join(",")}` : "");
14093
+ if (sharedPorts.length > 0)
14094
+ loopbackRequired.push(`elements = { ${sharedPorts.join(", ")} }`, "meta skuid @service_uids tcp dport @shared_loopback_tcp_ports ip daddr 127.0.0.1 accept", "meta skuid @service_uids tcp dport @shared_loopback_tcp_ports ip6 daddr ::1 accept");
13914
14095
  if (ports.length > 0)
13915
14096
  loopbackRequired.push(`elements = { ${ports.join(", ")} }`, `meta skuid ${loopback.uid} tcp dport @loopback_tcp_ports ip daddr 127.0.0.1 accept`, `meta skuid ${loopback.uid} tcp dport @loopback_tcp_ports ip6 daddr ::1 accept`);
13916
14097
  if (publicPorts)
@@ -13949,6 +14130,7 @@ function applyAgentEgressPolicy(options) {
13949
14130
  const loopback = options.loopback ? {
13950
14131
  uid: resolveServiceUid(options.loopback.user, command3),
13951
14132
  tcpPorts: normalizeEgressTcpPorts(options.loopback.tcpPorts),
14133
+ sharedTcpPorts: normalizeEgressTcpPorts(options.loopback.sharedTcpPorts ?? []),
13952
14134
  publicTcpPorts: options.loopback.publicTcpPorts === undefined ? undefined : normalizeEgressTcpPorts(options.loopback.publicTcpPorts)
13953
14135
  } : undefined;
13954
14136
  if (loopback && !uids.includes(loopback.uid))
@@ -13975,6 +14157,7 @@ async function superviseAgentEgressPolicy(options) {
13975
14157
  const loopback = options.loopback ? {
13976
14158
  uid: resolveServiceUid(options.loopback.user, command3),
13977
14159
  tcpPorts: normalizeEgressTcpPorts(options.loopback.tcpPorts),
14160
+ sharedTcpPorts: normalizeEgressTcpPorts(options.loopback.sharedTcpPorts ?? []),
13978
14161
  publicTcpPorts: options.loopback.publicTcpPorts === undefined ? undefined : normalizeEgressTcpPorts(options.loopback.publicTcpPorts)
13979
14162
  } : undefined;
13980
14163
  (options.notifyReady ?? (() => {
@@ -14714,11 +14897,29 @@ var replaceLink = (path2, target2) => {
14714
14897
  symlinkSync6(target2, pending);
14715
14898
  renameSync13(pending, path2);
14716
14899
  };
14900
+ var assertReleaseLinksContained = (path2) => {
14901
+ const root = realpathSync5(path2);
14902
+ const visit = (current2) => {
14903
+ const metadata = lstatSync4(current2);
14904
+ if (metadata.isSymbolicLink()) {
14905
+ const target2 = realpathSync5(current2);
14906
+ if (target2 !== root && !target2.startsWith(`${root}/`)) {
14907
+ throw new Error("release contains a symbolic link outside its immutable root");
14908
+ }
14909
+ return;
14910
+ }
14911
+ if (metadata.isDirectory())
14912
+ for (const name of readdirSync4(current2))
14913
+ visit(join10(current2, name));
14914
+ };
14915
+ visit(root);
14916
+ };
14717
14917
  var secureRelease = (path2, uid, gid) => {
14918
+ assertReleaseLinksContained(path2);
14718
14919
  const visit = (current2) => {
14719
14920
  const metadata = lstatSync4(current2);
14720
14921
  if (metadata.isSymbolicLink())
14721
- throw new Error("release contains a symbolic link");
14922
+ return;
14722
14923
  chownSync(current2, uid, gid);
14723
14924
  chmodSync16(current2, metadata.isDirectory() ? 365 : 292);
14724
14925
  if (metadata.isDirectory())
@@ -14744,7 +14945,7 @@ async function activatePlatformRelease(config, requestedRelease, options = {}) {
14744
14945
  const sleep2 = options.sleep ?? Bun.sleep;
14745
14946
  const slots = join10(normalized.root, "slots");
14746
14947
  mkdirSync14(slots, { recursive: true, mode: 493 });
14747
- const slotFile = join10(normalized.root, ".forge-slot");
14948
+ const slotFile = join10(slots, ".active");
14748
14949
  const previousSlot = existsSync19(slotFile) ? readFileSync14(slotFile, "utf8").trim() : undefined;
14749
14950
  const target2 = previousSlot === "blue" ? "green" : "blue";
14750
14951
  const port = target2 === "blue" ? normalized.bluePort : normalized.greenPort;
@@ -14794,6 +14995,7 @@ async function activatePlatformRelease(config, requestedRelease, options = {}) {
14794
14995
  const test = await exec(["/usr/sbin/nginx", "-t"]);
14795
14996
  const reload = test.exitCode === 0 ? await exec(["/usr/sbin/nginx", "-s", "reload"]) : test;
14796
14997
  if (reload.exitCode !== 0) {
14998
+ const refusal = (reload.output || test.output).trim().slice(0, 2000);
14797
14999
  if (existsSync19(backup))
14798
15000
  renameSync13(backup, upstream);
14799
15001
  else
@@ -14801,7 +15003,7 @@ async function activatePlatformRelease(config, requestedRelease, options = {}) {
14801
15003
  await exec(["/usr/sbin/nginx", "-t"]);
14802
15004
  await exec(["/usr/sbin/nginx", "-s", "reload"]);
14803
15005
  await stopTarget();
14804
- throw new Error("nginx refused the promoted upstream");
15006
+ throw new Error(`nginx refused the promoted upstream${refusal ? `: ${refusal}` : ""}`);
14805
15007
  }
14806
15008
  rmSync10(backup, { force: true });
14807
15009
  writeFileSync14(slotFile, `${target2}
@@ -14833,7 +15035,7 @@ var checked8 = async (run2, argv2, expected = 0) => {
14833
15035
  return result;
14834
15036
  };
14835
15037
  var activeSlot = () => {
14836
- const slot = readFileSync15("/opt/forgezero/.forge-slot", "utf8").trim();
15038
+ const slot = readFileSync15("/opt/forgezero/slots/.active", "utf8").trim();
14837
15039
  if (slot !== "blue" && slot !== "green")
14838
15040
  throw new Error("invalid active slot");
14839
15041
  return slot;
@@ -15030,6 +15232,7 @@ function agentEgressUnit(options) {
15030
15232
  }
15031
15233
  const deploymentEnabled = Boolean(options.repository || bootstrapBundleEnabled || options.pullDeployments);
15032
15234
  const runnerLoopbackPorts = normalizeEgressTcpPorts(options.runnerLoopbackPorts ?? []);
15235
+ const agentLoopbackPorts = normalizeEgressTcpPorts(options.agentLoopbackPorts ?? []);
15033
15236
  const runnerPublicTcpPorts = normalizeEgressTcpPorts(options.runnerPublicTcpPorts ?? DEFAULT_RUNNER_PUBLIC_TCP_PORTS);
15034
15237
  if (deploymentEnabled && runnerPublicTcpPorts.length < 1) {
15035
15238
  throw new Error("deployed project runner needs at least one vetted public TCP port");
@@ -15037,8 +15240,10 @@ function agentEgressUnit(options) {
15037
15240
  if (deploymentEnabled)
15038
15241
  systemdAgentEgressDirectives(runnerLoopbackPorts);
15039
15242
  const users = [user, ...deploymentEnabled ? [DEPLOYMENT_RUNNER_USER] : []];
15040
- const runnerGrant = deploymentEnabled ? ` --loopback-user=${DEPLOYMENT_RUNNER_USER}` + runnerLoopbackPorts.map((port) => ` --loopback-tcp-port=${port}`).join("") + runnerPublicTcpPorts.map((port) => ` --public-tcp-port=${port}`).join("") : "";
15041
- const policyProof = `ExecStartPost=${bin} egress-policy-check ${users.map((name) => `--user=${name}`).join(" ")}${runnerGrant}`;
15243
+ const portGrantEnabled = deploymentEnabled || agentLoopbackPorts.length > 0;
15244
+ const restrictedUser = deploymentEnabled ? DEPLOYMENT_RUNNER_USER : user;
15245
+ const portGrant = portGrantEnabled ? ` --loopback-user=${restrictedUser}` + (deploymentEnabled ? runnerLoopbackPorts.map((port) => ` --loopback-tcp-port=${port}`).join("") : "") + agentLoopbackPorts.map((port) => ` --shared-loopback-tcp-port=${port}`).join("") + (deploymentEnabled ? runnerPublicTcpPorts.map((port) => ` --public-tcp-port=${port}`).join("") : "") : "";
15246
+ const policyProof = `ExecStartPost=${bin} egress-policy-check ${users.map((name) => `--user=${name}`).join(" ")}${portGrant}`;
15042
15247
  return `[Unit]
15043
15248
  Description=ForgeZero Agent host egress policy
15044
15249
  Documentation=https://www.forgezero.net/docs/agent
@@ -15051,7 +15256,7 @@ Type=notify
15051
15256
  NotifyAccess=all
15052
15257
  User=root
15053
15258
  Group=root
15054
- ExecStart=${bin} egress-policy ${users.map((name) => `--user=${name}`).join(" ")}${runnerGrant}
15259
+ ExecStart=${bin} egress-policy ${users.map((name) => `--user=${name}`).join(" ")}${portGrant}
15055
15260
  ${policyProof}
15056
15261
  Restart=on-failure
15057
15262
  RestartSec=2
@@ -15335,7 +15540,7 @@ function agentEnrolmentUnit(options) {
15335
15540
  Requires=forgezero-agent-egress.service
15336
15541
  BindsTo=forgezero-agent-egress.service
15337
15542
  ` : "";
15338
- const egressDirectives = options.enforceEgress ? systemdAgentEgressDirectives() : "";
15543
+ const egressDirectives = options.enforceEgress ? systemdAgentEgressDirectives(options.agentLoopbackPorts ?? []) : "";
15339
15544
  return `[Unit]
15340
15545
  Description=Bind this machine to its ForgeZero compute
15341
15546
  After=network-online.target
@@ -15372,7 +15577,7 @@ WantedBy=multi-user.target
15372
15577
  function deploymentRunnerUnit(options) {
15373
15578
  const bin = options.binPath ?? "fz-agent";
15374
15579
  const root = options.deployRoot ?? "/opt/forgezero";
15375
- const agentUser = options.user ?? "forgezero-agent";
15580
+ const agentUser = options.user ?? "forgezero";
15376
15581
  const egressDependency = options.enforceEgress ? `After=forgezero-agent-egress.service
15377
15582
  Requires=forgezero-agent-egress.service
15378
15583
  BindsTo=forgezero-agent-egress.service
@@ -15413,7 +15618,7 @@ RestrictRealtime=true
15413
15618
  MemoryDenyWriteExecute=true
15414
15619
  LockPersonality=true
15415
15620
  ${egressDirectives}
15416
- ReadWritePaths=${root}/releases ${root}/runner-home
15621
+ ReadWritePaths=${root}/releases ${root}/runner-home ${root}/slots /etc/nginx/conf.d /var/log/nginx
15417
15622
 
15418
15623
  [Install]
15419
15624
  WantedBy=multi-user.target
@@ -15548,7 +15753,7 @@ function agentUnit(options) {
15548
15753
  const supplementaryGroups = [
15549
15754
  AGENT_UPDATE_GROUP,
15550
15755
  deploymentEnabled ? DEPLOYMENT_GROUP : null,
15551
- deploymentEnabled ? SOFTWARE_HELPER_GROUP : null,
15756
+ SOFTWARE_HELPER_GROUP,
15552
15757
  lifecycleEnabled ? LIFECYCLE_GROUP : null
15553
15758
  ].filter((value) => value !== null);
15554
15759
  const deploymentGroup = supplementaryGroups.length > 0 ? `SupplementaryGroups=${supplementaryGroups.join(" ")}` : "";
@@ -15557,7 +15762,7 @@ function agentUnit(options) {
15557
15762
  "forgezero-agent-update-helper.service",
15558
15763
  options.enforceEgress ? "forgezero-agent-egress.service" : null,
15559
15764
  deploymentEnabled ? "forgezero-deploy-runner.service" : null,
15560
- deploymentEnabled ? "forgezero-software-helper.service" : null,
15765
+ "forgezero-software-helper.service",
15561
15766
  lifecycleEnabled ? "forgezero-lifecycle-helper.service" : null,
15562
15767
  warpEnabled ? "warp-svc.service" : null,
15563
15768
  enrolmentEnabled ? "forgezero-agent-enrol.service" : null
@@ -15566,7 +15771,7 @@ function agentUnit(options) {
15566
15771
  "forgezero-agent-update-helper.service",
15567
15772
  options.enforceEgress ? "forgezero-agent-egress.service" : null,
15568
15773
  deploymentEnabled ? "forgezero-deploy-runner.service" : null,
15569
- deploymentEnabled ? "forgezero-software-helper.service" : null,
15774
+ "forgezero-software-helper.service",
15570
15775
  lifecycleEnabled ? "forgezero-lifecycle-helper.service" : null,
15571
15776
  warpEnabled ? "warp-svc.service" : null,
15572
15777
  enrolmentEnabled ? "forgezero-agent-enrol.service" : null
@@ -15581,9 +15786,9 @@ function agentUnit(options) {
15581
15786
  const snpDevice = options.mode === "attested" ? `DevicePolicy=closed
15582
15787
  DeviceAllow=/dev/sev-guest rw` : "";
15583
15788
  const snpPrepare = options.mode === "attested" ? `ExecStartPre=+/bin/chgrp ${VAULT_GROUP} /dev/sev-guest
15584
- ExecStartPre=+/bin/chmod 0640 /dev/sev-guest
15789
+ ExecStartPre=+/bin/chmod 0660 /dev/sev-guest
15585
15790
  ` : "";
15586
- const egressDirectives = options.enforceEgress ? systemdAgentEgressDirectives() : "";
15791
+ const egressDirectives = options.enforceEgress ? systemdAgentEgressDirectives(options.agentLoopbackPorts ?? []) : "";
15587
15792
  return `[Unit]
15588
15793
  Description=ForgeZero node agent (${options.mode})
15589
15794
  Documentation=https://www.forgezero.net/docs/agent
@@ -15714,6 +15919,7 @@ function planProvision(options) {
15714
15919
  const deployRoot = options.deployRoot ?? "/opt/forgezero";
15715
15920
  const deploymentEnabled = Boolean(options.repository || options.pullDeployments);
15716
15921
  const runnerLoopbackPorts = normalizeEgressTcpPorts(options.runnerLoopbackPorts ?? []);
15922
+ const agentLoopbackPorts = normalizeEgressTcpPorts(options.agentLoopbackPorts ?? []);
15717
15923
  const runnerPublicTcpPorts = normalizeEgressTcpPorts(options.runnerPublicTcpPorts ?? DEFAULT_RUNNER_PUBLIC_TCP_PORTS);
15718
15924
  const lifecycleEnabled = Boolean(options.pullMigrations && options.lifecycleProfilePath);
15719
15925
  if (Boolean(options.pullMigrations) !== Boolean(options.lifecycleProfilePath)) {
@@ -15762,8 +15968,9 @@ function planProvision(options) {
15762
15968
  const enabledUnits = [
15763
15969
  "forgezero-agent.socket",
15764
15970
  "forgezero-agent-update-helper.service",
15971
+ "forgezero-software-helper.service",
15765
15972
  ...options.enforceEgress ? ["forgezero-agent-egress.service"] : [],
15766
- ...deploymentEnabled ? ["forgezero-deploy-runner.service", "forgezero-software-helper.service"] : [],
15973
+ ...deploymentEnabled ? ["forgezero-deploy-runner.service"] : [],
15767
15974
  ...lifecycleEnabled ? ["forgezero-lifecycle-helper.service"] : [],
15768
15975
  ...warpEnabled ? ["forgezero-warp-config.service", "warp-svc.service"] : [],
15769
15976
  ...enrolmentEnabled ? ["forgezero-agent-enrol.service"] : [],
@@ -15771,8 +15978,9 @@ function planProvision(options) {
15771
15978
  ];
15772
15979
  const restartedUnits = [
15773
15980
  "forgezero-agent-update-helper.service",
15981
+ "forgezero-software-helper.service",
15774
15982
  ...options.enforceEgress ? ["forgezero-agent-egress.service"] : [],
15775
- ...deploymentEnabled ? ["forgezero-deploy-runner.service", "forgezero-software-helper.service"] : [],
15983
+ ...deploymentEnabled ? ["forgezero-deploy-runner.service"] : [],
15776
15984
  ...lifecycleEnabled ? ["forgezero-lifecycle-helper.service"] : [],
15777
15985
  ...warpEnabled ? ["forgezero-warp-config.service", "warp-svc.service"] : [],
15778
15986
  ...enrolmentEnabled ? ["forgezero-agent-enrol.service"] : []
@@ -15796,9 +16004,9 @@ function planProvision(options) {
15796
16004
  ...options.enforceEgress ? [
15797
16005
  { path: AGENT_EGRESS_UNIT_PATH, unit: agentEgressUnit(options) }
15798
16006
  ] : [],
16007
+ { path: SOFTWARE_HELPER_UNIT_PATH, unit: softwareHelperUnit(options) },
15799
16008
  ...deploymentEnabled ? [
15800
- { path: DEPLOYMENT_RUNNER_UNIT_PATH, unit: deploymentRunnerUnit(options) },
15801
- { path: SOFTWARE_HELPER_UNIT_PATH, unit: softwareHelperUnit(options) }
16009
+ { path: DEPLOYMENT_RUNNER_UNIT_PATH, unit: deploymentRunnerUnit(options) }
15802
16010
  ] : [],
15803
16011
  ...enrolmentEnabled ? [
15804
16012
  { path: ENROLMENT_UNIT_PATH, unit: agentEnrolmentUnit(options) }
@@ -15831,24 +16039,29 @@ function planProvision(options) {
15831
16039
  step2("Agent update helper access group", { kind: "commands", commands: [{ argv: ["/usr/sbin/groupadd", "--system", AGENT_UPDATE_GROUP], acceptedExitCodes: [0, 9] }] }),
15832
16040
  ...sourceBinPath && binPath ? [step2("root-owned agent runtime", { kind: "install-runtime", source: sourceBinPath, binary: binPath, version: VERSION2 })] : [],
15833
16041
  ...warpEnabled ? [step2("Cloudflare One client for Ubuntu 26.04", { kind: "install-warp" })] : [],
15834
- ...deploymentEnabled ? [step2("deployment isolation group", { kind: "commands", commands: [
15835
- { argv: ["/usr/sbin/groupadd", "--system", DEPLOYMENT_GROUP], acceptedExitCodes: [0, 9] },
16042
+ step2("software strategy helper group", { kind: "commands", commands: [
15836
16043
  { argv: ["/usr/sbin/groupadd", "--system", SOFTWARE_HELPER_GROUP], acceptedExitCodes: [0, 9] }
16044
+ ] }),
16045
+ ...deploymentEnabled ? [step2("deployment isolation group", { kind: "commands", commands: [
16046
+ { argv: ["/usr/sbin/groupadd", "--system", DEPLOYMENT_GROUP], acceptedExitCodes: [0, 9] }
15837
16047
  ] })] : [],
15838
16048
  ...lifecycleEnabled ? [step2("lifecycle helper access group", { kind: "commands", commands: [{ argv: ["/usr/sbin/groupadd", "--system", LIFECYCLE_GROUP], acceptedExitCodes: [0, 9] }] })] : [],
15839
16049
  step2("service account", { kind: "commands", commands: [{ argv: ["/usr/sbin/useradd", "--system", "--no-create-home", "--shell", "/usr/sbin/nologin", user], acceptedExitCodes: [0, 9] }] }),
15840
16050
  step2("bind service account to vault group", { kind: "commands", commands: [{ argv: ["/usr/sbin/usermod", "-g", VAULT_GROUP, user] }] }),
15841
16051
  step2("grant verified Agent update access", { kind: "commands", commands: [{ argv: ["/usr/sbin/usermod", "-a", "-G", AGENT_UPDATE_GROUP, user] }] }),
16052
+ step2("grant software helper socket access", { kind: "commands", commands: [
16053
+ { argv: ["/usr/sbin/usermod", "-a", "-G", SOFTWARE_HELPER_GROUP, user] }
16054
+ ] }),
15842
16055
  ...lifecycleEnabled ? [step2("grant lifecycle helper socket access", { kind: "commands", commands: [{ argv: ["/usr/sbin/usermod", "-a", "-G", LIFECYCLE_GROUP, user] }] })] : [],
15843
16056
  ...deploymentEnabled ? [step2("credential-free deployment account", { kind: "commands", commands: [
15844
16057
  { argv: ["/usr/sbin/useradd", "--system", "--no-create-home", "--shell", "/usr/sbin/nologin", "--gid", DEPLOYMENT_GROUP, DEPLOYMENT_RUNNER_USER], acceptedExitCodes: [0, 9] },
15845
16058
  { argv: ["/usr/sbin/useradd", "--system", "--no-create-home", "--shell", "/usr/sbin/nologin", "--gid", VAULT_GROUP, APPLICATION_RUNTIME_USER], acceptedExitCodes: [0, 9] },
15846
16059
  { argv: ["/usr/sbin/usermod", "-a", "-G", DEPLOYMENT_GROUP, APPLICATION_RUNTIME_USER] },
15847
- { argv: ["/usr/sbin/usermod", "-a", "-G", `${DEPLOYMENT_GROUP},${SOFTWARE_HELPER_GROUP}`, user] }
16060
+ { argv: ["/usr/sbin/usermod", "-a", "-G", DEPLOYMENT_GROUP, user] }
15848
16061
  ] })] : [],
15849
16062
  step2("credential and state directories", { kind: "directories", directories: [
15850
16063
  { path: credentialDir, mode: 448, owner: "root", group: "root" },
15851
- { path: "/var/lib/forgezero", mode: 488, owner: "root", group: "root" }
16064
+ { path: "/var/lib/forgezero", mode: 488, owner: "root", group: VAULT_GROUP }
15852
16065
  ] }),
15853
16066
  step2("encrypted node identity", { kind: "ensure-seed", credential: seedCredentialPath }),
15854
16067
  ...options.generateGitIdentity && gitCredentialPath && gitPublicKeyPath ? [
@@ -15899,19 +16112,26 @@ function planProvision(options) {
15899
16112
  { argv: ["/usr/bin/systemctl", "enable", ...enabledUnits] },
15900
16113
  { argv: ["/usr/bin/systemctl", "reset-failed", "forgezero-agent.service"], acceptedExitCodes: [0, 1] },
15901
16114
  ...restartedUnits.length ? [{ argv: ["/usr/bin/systemctl", "restart", ...restartedUnits] }] : [],
16115
+ { argv: ["/usr/bin/systemctl", "stop", "forgezero-agent-proxy.service"], acceptedExitCodes: [0, 1, 5] },
15902
16116
  { argv: ["/usr/bin/systemctl", "restart", "forgezero-agent.socket"] },
15903
16117
  { argv: ["/usr/bin/systemctl", "reset-failed", "forgezero-agent.service"], acceptedExitCodes: [0, 1] },
15904
16118
  { argv: ["/usr/bin/systemctl", "restart", "forgezero-agent.service"] }
15905
16119
  ] }),
15906
16120
  ...enrolmentEnabled ? [step2("prove the compute binding is durable", { kind: "verify-file", path: enrolStatePath })] : [],
15907
- ...options.enforceEgress ? [step2("prove the Agent egress policy is active", { kind: "verify-egress", runnerPublicTcpPorts, runnerLoopbackPorts })] : [],
16121
+ ...options.enforceEgress ? [step2("prove the Agent egress policy is active", {
16122
+ kind: "verify-egress",
16123
+ deploymentEnabled,
16124
+ runnerPublicTcpPorts,
16125
+ runnerLoopbackPorts,
16126
+ agentLoopbackPorts
16127
+ })] : [],
15908
16128
  step2("prove it is running", { kind: "commands", commands: [{ argv: ["/usr/bin/systemctl", "is-active", "forgezero-agent.service"] }] }),
15909
16129
  step2("prove the public Vault socket exists", { kind: "wait-socket", path: options.socketPath, attempts: 100, intervalMs: 100 }),
15910
16130
  step2("prove the Agent Vault backend exists", { kind: "wait-socket", path: agentBackendSocketPath(options.socketPath), attempts: 100, intervalMs: 100 }),
15911
16131
  step2("prove the Agent update helper exists", { kind: "wait-socket", path: DEFAULT_AGENT_UPDATE_SOCKET, attempts: 100, intervalMs: 100 }),
16132
+ step2("prove the software strategy helper socket exists", { kind: "wait-socket", path: DEFAULT_SOFTWARE_HELPER_SOCKET, attempts: 100, intervalMs: 100 }),
15912
16133
  ...deploymentEnabled ? [
15913
- step2("prove the deployment runner socket exists", { kind: "wait-socket", path: DEPLOYMENT_RUNNER_SOCKET, attempts: 100, intervalMs: 100 }),
15914
- step2("prove the software strategy helper socket exists", { kind: "wait-socket", path: DEFAULT_SOFTWARE_HELPER_SOCKET, attempts: 100, intervalMs: 100 })
16134
+ step2("prove the deployment runner socket exists", { kind: "wait-socket", path: DEPLOYMENT_RUNNER_SOCKET, attempts: 100, intervalMs: 100 })
15915
16135
  ] : [],
15916
16136
  ...lifecycleEnabled ? [step2("prove the lifecycle helper socket exists", { kind: "wait-socket", path: lifecycleHelperSocketPath, attempts: 100, intervalMs: 100 })] : [],
15917
16137
  ...warpEnabled ? [step2("prove Cloudflare WARP is connected", { kind: "verify-warp" })] : [],
@@ -15936,6 +16156,25 @@ import {
15936
16156
  writeFileSync as writeFileSync15
15937
16157
  } from "fs";
15938
16158
  import { dirname as dirname13 } from "path";
16159
+ function normalizeEd25519PublicKey(output) {
16160
+ const line = output.trim();
16161
+ if (!/^ssh-ed25519 [A-Za-z0-9+/]+={0,3}(?: [^\r\n]{1,256})?$/.test(line))
16162
+ return;
16163
+ const [algorithm, encoded] = line.split(" ", 3);
16164
+ if (algorithm !== "ssh-ed25519" || !encoded)
16165
+ return;
16166
+ let blob;
16167
+ try {
16168
+ blob = Buffer.from(encoded, "base64");
16169
+ } catch {
16170
+ return;
16171
+ }
16172
+ if (blob.length !== 51 || blob.readUInt32BE(0) !== 11 || blob.subarray(4, 15).toString("ascii") !== "ssh-ed25519" || blob.readUInt32BE(15) !== 32)
16173
+ return;
16174
+ if (blob.toString("base64") !== encoded)
16175
+ return;
16176
+ return `${algorithm} ${encoded}`;
16177
+ }
15939
16178
  async function readCapabilities(run2) {
15940
16179
  const answers = {};
15941
16180
  const checks = Object.entries(CAPABILITY_CHECKS);
@@ -16075,7 +16314,7 @@ var runProvisionOperation = async (operation) => {
16075
16314
  return result;
16076
16315
  }
16077
16316
  result = await fixed(["/usr/bin/ssh-keygen", "-y", "-f", key]);
16078
- if (result.exitCode !== 0 || !/^ssh-ed25519 [A-Za-z0-9+/]+={0,3}\s*$/.test(result.stdout)) {
16317
+ if (result.exitCode !== 0 || !normalizeEd25519PublicKey(result.stdout)) {
16079
16318
  return { stdout: "bootstrap SSH private key is not a valid Ed25519 OpenSSH key", exitCode: 1 };
16080
16319
  }
16081
16320
  result = await fixed(["/usr/bin/systemd-creds", "encrypt", "--name=bootstrap-ssh-key", key, operation.credential]);
@@ -16091,11 +16330,12 @@ var runProvisionOperation = async (operation) => {
16091
16330
  chmodSync17(key, 384);
16092
16331
  }
16093
16332
  const derived = await fixed(["/usr/bin/ssh-keygen", "-y", "-f", key]);
16094
- if (derived.exitCode !== 0 || !/^ssh-ed25519 [A-Za-z0-9+/]+={0,3}\s*$/.test(derived.stdout)) {
16333
+ const publicKey = normalizeEd25519PublicKey(derived.stdout);
16334
+ if (derived.exitCode !== 0 || !publicKey) {
16095
16335
  return { stdout: "bootstrap SSH public key derivation failed", exitCode: 1 };
16096
16336
  }
16097
16337
  mkdirSync15(dirname13(operation.publicKey), { recursive: true, mode: 493 });
16098
- writeFileSync15(operation.publicKey, `${derived.stdout.trim()} forgezero-bootstrap-runner
16338
+ writeFileSync15(operation.publicKey, `${publicKey} forgezero-bootstrap-runner
16099
16339
  `, { mode: 292 });
16100
16340
  chmodSync17(operation.publicKey, 292);
16101
16341
  }
@@ -16138,8 +16378,9 @@ var runProvisionOperation = async (operation) => {
16138
16378
  const policy = await fixed(["/usr/sbin/nft", "--numeric", "list", "table", "inet", "forgezero_agent_egress"]);
16139
16379
  const required = [
16140
16380
  "forgezero-agent-egress-v1",
16141
- `public-tcp=${operation.runnerPublicTcpPorts.join(",")}`,
16142
- ...operation.runnerLoopbackPorts.length ? [`:${operation.runnerLoopbackPorts.join(",")}`] : []
16381
+ ...operation.agentLoopbackPorts.length ? [`shared-loopback=${operation.agentLoopbackPorts.join(",")}`] : [],
16382
+ ...operation.deploymentEnabled && operation.runnerPublicTcpPorts.length ? [`public-tcp=${operation.runnerPublicTcpPorts.join(",")}`] : [],
16383
+ ...operation.deploymentEnabled && operation.runnerLoopbackPorts.length ? [`:${operation.runnerLoopbackPorts.join(",")}`] : []
16143
16384
  ];
16144
16385
  return policy.exitCode === 0 && required.every((part) => policy.stdout.includes(part)) ? { stdout: policy.stdout, exitCode: 0 } : { stdout: policy.stdout, exitCode: 1 };
16145
16386
  }
@@ -16205,7 +16446,9 @@ async function applyPlan(plan, run2) {
16205
16446
  const result = await run2(step3.operation);
16206
16447
  transcript.push({ label: step3.label, command: step3.command, exitCode: result.exitCode });
16207
16448
  if (result.exitCode !== 0 && !step3.optional) {
16208
- throw new Error(`${step3.label} failed (exit ${result.exitCode}): ${step3.command}`);
16449
+ const safeDetail = step3.operation.kind === "ensure-bootstrap-ssh-identity" ? result.stdout.trim().slice(0, 1000) : "";
16450
+ throw new Error(`${step3.label} failed (exit ${result.exitCode}): ${step3.command}${safeDetail ? `
16451
+ ${safeDetail}` : ""}`);
16209
16452
  }
16210
16453
  }
16211
16454
  return transcript;
@@ -16232,7 +16475,6 @@ var SEED_CREDENTIAL = `${CREDS}/seed-sync-root.cred`;
16232
16475
  var BACKUP_RECOVERY_CREDENTIAL = `${CREDS}/backup-recovery-root.cred`;
16233
16476
  var BOOTSTRAP_SSH_CREDENTIAL = `${CREDS}/bootstrap-ssh-key.cred`;
16234
16477
  var BOOTSTRAP_SSH_PUBLIC_KEY = "/etc/forgezero/bootstrap/runner.pub";
16235
- var BOOTSTRAP_RELEASE_EVIDENCE = "/var/lib/forgezero/bootstrap-release.json";
16236
16478
  var LIFECYCLE_PROFILE = "/etc/forgezero/lifecycle.json";
16237
16479
  var CONTROL_SOCKET = "/run/forgezero/control.sock";
16238
16480
  var CLOUDFLARED_METRICS_ADDRESS = "127.0.0.1:20241";
@@ -16451,6 +16693,8 @@ function planBootstrapAgentInstall(config, phase, context) {
16451
16693
  bootstrapTargetTelemetryEndpoint: bound && bootstrapRunner ? platformBootstrapRunner(config) ? config.telemetryEndpoint : config.kind === "enrolled-compute" ? config.bootstrapRunner?.targetTelemetryEndpoint : undefined : undefined,
16452
16694
  lifecycleProfilePath: bound && config.kind === "platform" ? LIFECYCLE_PROFILE : undefined,
16453
16695
  enforceEgress: true,
16696
+ runnerLoopbackPorts: config.kind === "platform" ? [config.runtime.environment.publicApiPort, config.runtime.bluePort, config.runtime.greenPort] : [],
16697
+ agentLoopbackPorts: config.kind === "platform" ? [config.runtime.environment.publicApiPort] : [],
16454
16698
  nodeHostname: config.nodeHostname,
16455
16699
  telemetryEndpoint: config.telemetryEndpoint,
16456
16700
  binPath: "/usr/local/lib/forgezero/agent/fz-agent",
@@ -16460,7 +16704,7 @@ function planBootstrapAgentInstall(config, phase, context) {
16460
16704
  enrolStatePath: "/var/lib/forgezero/enrolment.json"
16461
16705
  } : bound ? { enrolStatePath: "/var/lib/forgezero/enrolment.json" } : {},
16462
16706
  ...authenticated ? {
16463
- apiUrl: config.apiUrl,
16707
+ apiUrl: config.kind === "platform" ? `http://127.0.0.1:${config.runtime.environment.publicApiPort}` : config.apiUrl,
16464
16708
  project: config.kind === "enrolled-compute" ? config.realm : "platform",
16465
16709
  environment: config.kind === "enrolled-compute" ? undefined : config.environment,
16466
16710
  nodeLabel: config.kind === "enrolled-compute" ? config.nodeHostname : config.computeReference
@@ -16469,6 +16713,7 @@ function planBootstrapAgentInstall(config, phase, context) {
16469
16713
  return planInstall(options);
16470
16714
  }
16471
16715
  function localBootstrapHost() {
16716
+ let softwareClientUser;
16472
16717
  const execute3 = async (argv2, options = {}) => {
16473
16718
  const child = Bun.spawn([...argv2], { stdin: options.stdin === undefined ? "ignore" : "pipe", stdout: "pipe", stderr: "pipe" });
16474
16719
  if (options.stdin !== undefined && child.stdin && typeof child.stdin !== "number") {
@@ -16501,10 +16746,12 @@ function localBootstrapHost() {
16501
16746
  exec: execute3,
16502
16747
  sleep: (milliseconds) => Bun.sleep(milliseconds),
16503
16748
  async ensureSoftware(requirements) {
16749
+ if (!softwareClientUser)
16750
+ throw new Error("Agent must be converged before software requirements are applied");
16504
16751
  const result = await execute3([
16505
16752
  "runuser",
16506
16753
  "-u",
16507
- "forgezero-agent",
16754
+ softwareClientUser,
16508
16755
  "--",
16509
16756
  "/usr/local/bin/fz-agent",
16510
16757
  "software-ensure",
@@ -16518,7 +16765,7 @@ function localBootstrapHost() {
16518
16765
  const capabilities = await readCapabilities(localRunner);
16519
16766
  const hasBinding = existsSync22("/var/lib/forgezero/enrolment.json");
16520
16767
  const hasEnrolCredential = existsSync22(ENROL_CREDENTIAL);
16521
- const initialBundle = config.kind === "platform" && !existsSync22(BOOTSTRAP_RELEASE_EVIDENCE) ? {
16768
+ const initialBundle = config.kind === "platform" && phase === "bootstrap" && existsSync22(config.bootstrapBundle.bundleFile) && existsSync22(config.bootstrapBundle.manifestFile) ? {
16522
16769
  path: config.bootstrapBundle.bundleFile,
16523
16770
  manifestPath: config.bootstrapBundle.manifestFile,
16524
16771
  manifest: parseBootstrapBundleManifest(JSON.parse(readFileSync17(config.bootstrapBundle.manifestFile, "utf8")))
@@ -16549,6 +16796,7 @@ function localBootstrapHost() {
16549
16796
  writeFileSync16(unit3.path, unit3.unit, { mode: 420 });
16550
16797
  }
16551
16798
  await applyPlan(plan, localRunner);
16799
+ softwareClientUser = plan.user;
16552
16800
  return plan;
16553
16801
  }
16554
16802
  };
@@ -17202,6 +17450,14 @@ async function initializeVaultReplica(cache) {
17202
17450
  return { state: "unavailable", loaded: 0, failed: [] };
17203
17451
  }
17204
17452
  }
17453
+ async function attemptInitialAttestation(attempt, onDeferred) {
17454
+ try {
17455
+ return { verified: true, result: await attempt() };
17456
+ } catch (cause) {
17457
+ onDeferred(cause);
17458
+ return { verified: false };
17459
+ }
17460
+ }
17205
17461
  function runAgent(config = {}) {
17206
17462
  const seed = config.seed ?? configuredAgentSeed({
17207
17463
  credential: config.seedCredential,
@@ -17370,16 +17626,18 @@ if (import.meta.main) {
17370
17626
  const loopbackUser = args.find((arg) => arg.startsWith("--loopback-user="))?.slice("--loopback-user=".length);
17371
17627
  const parsePorts = (prefix) => args.filter((arg) => arg.startsWith(prefix)).map((arg) => Number(arg.slice(prefix.length)));
17372
17628
  const loopbackTcpPorts = parsePorts("--loopback-tcp-port=");
17629
+ const sharedLoopbackTcpPorts = parsePorts("--shared-loopback-tcp-port=");
17373
17630
  const publicTcpPorts = parsePorts("--public-tcp-port=");
17374
- if (!loopbackUser && (loopbackTcpPorts.length > 0 || publicTcpPorts.length > 0)) {
17631
+ if (!loopbackUser && (loopbackTcpPorts.length > 0 || sharedLoopbackTcpPorts.length > 0 || publicTcpPorts.length > 0)) {
17375
17632
  throw new Error("egress-policy port grants require --loopback-user=<service-user>");
17376
17633
  }
17377
- if (loopbackUser && loopbackTcpPorts.length < 1 && publicTcpPorts.length < 1) {
17634
+ if (loopbackUser && loopbackTcpPorts.length < 1 && sharedLoopbackTcpPorts.length < 1 && publicTcpPorts.length < 1) {
17378
17635
  throw new Error("egress-policy restricted user requires an explicit TCP port grant");
17379
17636
  }
17380
17637
  const loopback = loopbackUser ? {
17381
17638
  user: loopbackUser,
17382
17639
  tcpPorts: loopbackTcpPorts,
17640
+ sharedTcpPorts: sharedLoopbackTcpPorts,
17383
17641
  publicTcpPorts: publicTcpPorts.length > 0 ? publicTcpPorts : undefined
17384
17642
  } : undefined;
17385
17643
  if (command3 === "egress-policy-check") {
@@ -17387,6 +17645,7 @@ if (import.meta.main) {
17387
17645
  const grant = loopback ? {
17388
17646
  uid: resolveServiceUid(loopback.user),
17389
17647
  tcpPorts: loopback.tcpPorts,
17648
+ sharedTcpPorts: loopback.sharedTcpPorts,
17390
17649
  publicTcpPorts: loopback.publicTcpPorts
17391
17650
  } : undefined;
17392
17651
  if (!verifyAgentEgressPolicy(uids, undefined, grant))
@@ -17641,8 +17900,8 @@ if (import.meta.main) {
17641
17900
  ...request,
17642
17901
  publicKeys: { ed25519: keys2.ed25519.publicKey, mlDsa: keys2.mlDsa.publicKey },
17643
17902
  preflight: {
17644
- vcpu: cpus2().length,
17645
- memoryGib: Math.max(1, Math.floor(totalmem2() / 1024 ** 3)),
17903
+ vcpu: cpus3().length,
17904
+ memoryGib: Math.max(1, Math.floor(totalmem3() / 1024 ** 3)),
17646
17905
  diskGib: Math.max(1, Math.floor(Number(filesystem.blocks) * Number(filesystem.bsize) / 1024 ** 3)),
17647
17906
  kvm: existsSync23("/dev/kvm"),
17648
17907
  snpHost: existsSync23("/dev/sev"),
@@ -17768,7 +18027,7 @@ if (import.meta.main) {
17768
18027
  key: keyArg?.slice("--key=".length) ?? ""
17769
18028
  } : { op: command3 };
17770
18029
  try {
17771
- const response = await requestControl(request, socketPath);
18030
+ const response = await requestControl(request, socketPath, controlRequestTimeout(request.op));
17772
18031
  console.log(JSON.stringify(response, null, 2));
17773
18032
  process.exit(response.ok ? 0 : 1);
17774
18033
  } catch (cause) {
@@ -17820,14 +18079,17 @@ if (import.meta.main) {
17820
18079
  let secretCache;
17821
18080
  let vaultSync;
17822
18081
  let initialVaultState = "unbound";
18082
+ let initialAttestationVerified = !attestationSource;
17823
18083
  if (binding && attestationSource && nodeApiUrl) {
17824
- const result = await telemetry.observe("attestation.refresh", () => attestNodeOnce({
18084
+ const initial = await attemptInitialAttestation(() => telemetry.observe("attestation.refresh", () => attestNodeOnce({
17825
18085
  apiUrl: nodeApiUrl,
17826
18086
  nodeKey,
17827
18087
  keys,
17828
18088
  source: attestationSource
17829
- }));
17830
- console.log(`[agent] initial SEV-SNP attestation verified ${result.measurement.slice(0, 16)}\u2026`);
18089
+ })), (cause) => console.error(`[agent] initial SEV-SNP attestation deferred: ${cause.message}`));
18090
+ initialAttestationVerified = initial.verified;
18091
+ if (initial.verified)
18092
+ console.log(`[agent] initial SEV-SNP attestation verified ${initial.result.measurement.slice(0, 16)}\u2026`);
17831
18093
  }
17832
18094
  if (binding && nodeApiUrl) {
17833
18095
  secretCache = createNodeVaultCache({
@@ -17836,21 +18098,35 @@ if (import.meta.main) {
17836
18098
  keys,
17837
18099
  projectKey: binding.projectKey
17838
18100
  });
17839
- const initialVault = await telemetry.observe("vault.sync", () => initializeVaultReplica(secretCache), (result) => result.state === "ready" ? "success" : "failed");
17840
- initialVaultState = initialVault.state;
17841
- running.setVault(secretCache, binding.projectKey);
17842
- vaultSync = startNodeVaultSync(secretCache, {
17843
- telemetry,
17844
- onEvent: (event, detail) => console.log(`[agent] vault ${event}${detail ? ` ${JSON.stringify(detail)}` : ""}`)
17845
- });
17846
- if (initialVault.state === "ready") {
17847
- console.log(`[agent] in-memory vault loaded for every environment in project ${binding.projectKey}`);
17848
- } else if (initialVault.state === "partial") {
17849
- console.error(`[agent] vault replica partial (${initialVault.loaded} loaded, ${initialVault.failed.length} unavailable); ` + "explicit same-name systemd deployment fallbacks remain available");
17850
- } else {
17851
- console.error("[agent] vault replica unavailable; authenticated refresh remains active and only explicit " + "same-name systemd deployment fallbacks are available");
17852
- }
17853
18101
  }
18102
+ let vaultActivation;
18103
+ const activateVaultReplica = () => {
18104
+ if (!binding || !secretCache || vaultSync)
18105
+ return Promise.resolve();
18106
+ if (vaultActivation)
18107
+ return vaultActivation;
18108
+ vaultActivation = (async () => {
18109
+ const initialVault = await telemetry.observe("vault.sync", () => initializeVaultReplica(secretCache), (result) => result.state === "ready" ? "success" : "failed");
18110
+ initialVaultState = initialVault.state;
18111
+ running.setVault(secretCache, binding.projectKey);
18112
+ vaultSync = startNodeVaultSync(secretCache, {
18113
+ telemetry,
18114
+ onEvent: (event, detail) => console.log(`[agent] vault ${event}${detail ? ` ${JSON.stringify(detail)}` : ""}`)
18115
+ });
18116
+ if (initialVault.state === "ready") {
18117
+ console.log(`[agent] in-memory vault loaded for every environment in project ${binding.projectKey}`);
18118
+ } else if (initialVault.state === "partial") {
18119
+ console.error(`[agent] vault replica partial (${initialVault.loaded} loaded, ${initialVault.failed.length} unavailable); ` + "explicit same-name systemd deployment fallbacks remain available");
18120
+ } else {
18121
+ console.error("[agent] vault replica unavailable; authenticated refresh remains active and only explicit " + "same-name systemd deployment fallbacks are available");
18122
+ }
18123
+ })().finally(() => {
18124
+ vaultActivation = undefined;
18125
+ });
18126
+ return vaultActivation;
18127
+ };
18128
+ if (initialAttestationVerified)
18129
+ await activateVaultReplica();
17854
18130
  if (handoverCandidate) {
17855
18131
  const ready = startAgentCandidateReady({
17856
18132
  version: VERSION2,
@@ -17882,8 +18158,12 @@ if (import.meta.main) {
17882
18158
  keys,
17883
18159
  telemetry,
17884
18160
  source: attestationSource,
17885
- immediate: !binding,
17886
- onEvent: (event, detail) => console.log(`[agent] attestation ${event}${detail ? ` ${detail instanceof Error ? detail.message : JSON.stringify(detail)}` : ""}`)
18161
+ immediate: !initialAttestationVerified,
18162
+ onEvent: (event, detail) => {
18163
+ console.log(`[agent] attestation ${event}${detail ? ` ${detail instanceof Error ? detail.message : JSON.stringify(detail)}` : ""}`);
18164
+ if (event === "verified")
18165
+ activateVaultReplica().catch((cause) => console.error(`[agent] vault activation deferred: ${cause.message}`));
18166
+ }
17887
18167
  }) : undefined;
17888
18168
  if (attestationLoop)
17889
18169
  console.log("[agent] periodic SEV-SNP attestation enabled");
@@ -17974,6 +18254,7 @@ if (import.meta.main) {
17974
18254
  gitCredentialPath: process.env.CREDENTIALS_DIRECTORY ? `${process.env.CREDENTIALS_DIRECTORY}/git-deploy-key` : undefined,
17975
18255
  knownHostsPath: source.knownHosts ? join12(root, "cache", `known-hosts-${key}`) : undefined,
17976
18256
  knownHostsContent: source.knownHosts,
18257
+ attestation: attestationSource,
17977
18258
  cache: deploymentSecrets,
17978
18259
  capacityEvidenceDirectory: process.env.FZ_CAPACITY_EVIDENCE_DIR ?? join12(root, "cache", "capacity"),
17979
18260
  ensureSoftware: (requirements) => requestSoftware(requirements, process.env.FZ_SOFTWARE_HELPER_SOCKET ?? DEFAULT_SOFTWARE_HELPER_SOCKET),
@@ -17983,10 +18264,10 @@ if (import.meta.main) {
17983
18264
  applyConnectivity: (request) => requestDeploymentConnectivity(request, process.env.FZ_SOFTWARE_HELPER_SOCKET ?? DEFAULT_SOFTWARE_HELPER_SOCKET),
17984
18265
  projectExec: process.env.FZ_DEPLOY_RUNNER_SOCKET ? (input) => requestDeploymentCommand(input, process.env.FZ_DEPLOY_RUNNER_SOCKET) : undefined
17985
18266
  });
17986
- const staticManager = repository && branch && profile ? createDeploymentManager(managerOptions({ repository, branch, profile }, process.env.FZ_DEPLOY_KEY ?? `${repository}:${branch}:${profile}`)) : bootstrapBundle && branch && profile ? createDeploymentManager({
18267
+ const staticManager = bootstrapBundle && branch && profile ? createDeploymentManager({
17987
18268
  ...managerOptions({ repository: bootstrapBundle.path, branch, profile }, process.env.FZ_DEPLOY_KEY ?? `bootstrap:${bootstrapBundle.manifest.sha256}:${profile}`),
17988
18269
  bootstrapBundle
17989
- }) : undefined;
18270
+ }) : repository && branch && profile ? createDeploymentManager(managerOptions({ repository, branch, profile }, process.env.FZ_DEPLOY_KEY ?? `${repository}:${branch}:${profile}`)) : undefined;
17990
18271
  if (staticManager)
17991
18272
  managers.set("__static__", staticManager);
17992
18273
  const control = staticManager ? startControlServer(staticManager, process.env.FZ_CONTROL_SOCKET ?? DEFAULT_CONTROL_SOCKET) : undefined;
@@ -18260,6 +18541,7 @@ export {
18260
18541
  cloudInit,
18261
18542
  calibrateHttpConcurrency,
18262
18543
  attestNodeOnce,
18544
+ attemptInitialAttestation,
18263
18545
  assertSupportedGuestImage,
18264
18546
  applyMetalIsolation,
18265
18547
  applyAgentEgressPolicy,