@forgezero/agent 0.1.88 → 0.1.89

package/dist/platform-fleet-verification.js

@@ -1198,7 +1198,12 @@ async function executeSoftwareOperation(operation) {
     }
     if (software === "nginx") {
       const binary = await run(["/usr/sbin/nginx", "-v"]);
-      return binary.exitCode === 0 ? run(["/usr/bin/systemctl", "is-active", "--quiet", "nginx.service"]) : binary;
+      if (binary.exitCode !== 0)
+        return binary;
+      if (!existsSync4("/usr/lib/nginx/modules/ngx_http_headers_more_filter_module.so")) {
+        return { exitCode: 1, output: "Nginx response-header suppression module is unavailable" };
+      }
+      return run(["/usr/bin/systemctl", "is-active", "--quiet", "nginx.service"]);
     }
     if (software === "arangodb") {
       const binary = await run(["/usr/sbin/arangod", "--version"]);
@@ -1230,7 +1235,7 @@ async function executeSoftwareOperation(operation) {
   }
   if (software === "docker" || software === "nginx" || software === "ufw" || software === "openssh-client" || software === "git") {
     const packageName = software === "openssh-client" ? "openssh-client" : software === "docker" ? "docker.io" : software;
-    const installed = await aptInstall(packageName);
+    const installed = software === "nginx" ? await aptInstallMany(["nginx", "libnginx-mod-http-headers-more-filter"]) : await aptInstall(packageName);
     if (installed.exitCode !== 0 || software !== "nginx" && software !== "docker")
       return installed;
     if (software === "docker") {
@@ -1382,11 +1387,22 @@ async function ensureSoftwareRequirements(requirementsInput, options) {
 }
 
 // src/capacity-calibration.ts
-var percentile95 = (values) => {
+import { cpus, freemem, totalmem } from "node:os";
+var percentile = (values, quantile) => {
   if (values.length === 0)
     return Number.POSITIVE_INFINITY;
   const sorted = values.toSorted((left, right) => left - right);
-  return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * 0.95) - 1)];
+  return sorted[Math.min(sorted.length - 1, Math.ceil(sorted.length * quantile) - 1)];
+};
+var hostSample = () => {
+  let cpuIdle = 0;
+  let cpuTotal = 0;
+  for (const cpu of cpus()) {
+    cpuIdle += cpu.times.idle;
+    cpuTotal += Object.values(cpu.times).reduce((sum, value) => sum + value, 0);
+  }
+  const memoryTotal = totalmem();
+  return { cpuIdle, cpuTotal, memoryUsed: memoryTotal - freemem(), memoryTotal };
 };
 function localCalibrationEndpoint(value) {
   const endpoint = new URL(value);
@@ -1401,9 +1417,12 @@ function validateCapacityCalibrationOptions(options) {
   const requestsPerWorker = options.requestsPerWorker ?? 8;
   const maxP95Ms = options.maxP95Ms ?? 250;
   const maxErrorRate = options.maxErrorRate ?? 0.01;
-  const headroomRatio = options.headroomRatio ?? 0.8;
+  const safetyRatio = options.safetyRatio ?? 0.8;
   const requestTimeoutMs = options.requestTimeoutMs ?? 5000;
-  if (!Number.isSafeInteger(maxConcurrency) || maxConcurrency < 1 || maxConcurrency > 4096 || !Number.isSafeInteger(requestsPerWorker) || requestsPerWorker < 2 || requestsPerWorker > 100 || !Number.isFinite(maxP95Ms) || maxP95Ms < 1 || !Number.isFinite(maxErrorRate) || maxErrorRate < 0 || maxErrorRate > 0.2 || !Number.isFinite(headroomRatio) || headroomRatio < 0.25 || headroomRatio > 0.95 || !Number.isSafeInteger(requestTimeoutMs) || requestTimeoutMs < 100 || requestTimeoutMs > 30000) {
+  const minimumStageDurationMs = options.minimumStageDurationMs ?? 5000;
+  const maxCpuUtilizationPercent = options.maxCpuUtilizationPercent ?? 90;
+  const maxMemoryUtilizationPercent = options.maxMemoryUtilizationPercent ?? 90;
+  if (!Number.isSafeInteger(maxConcurrency) || maxConcurrency < 1 || maxConcurrency > 4096 || !Number.isSafeInteger(requestsPerWorker) || requestsPerWorker < 2 || requestsPerWorker > 100 || !Number.isFinite(maxP95Ms) || maxP95Ms < 1 || !Number.isFinite(maxErrorRate) || maxErrorRate < 0 || maxErrorRate > 0.2 || !Number.isFinite(safetyRatio) || safetyRatio < 0.25 || safetyRatio > 0.95 || !Number.isSafeInteger(requestTimeoutMs) || requestTimeoutMs < 100 || requestTimeoutMs > 30000 || !Number.isSafeInteger(minimumStageDurationMs) || minimumStageDurationMs < 100 || minimumStageDurationMs > 60000 || !Number.isFinite(maxCpuUtilizationPercent) || maxCpuUtilizationPercent < 25 || maxCpuUtilizationPercent > 100 || !Number.isFinite(maxMemoryUtilizationPercent) || maxMemoryUtilizationPercent < 25 || maxMemoryUtilizationPercent > 100) {
     throw new Error("Capacity calibration bounds are invalid.");
   }
   return {
@@ -1412,18 +1431,24 @@ function validateCapacityCalibrationOptions(options) {
     requestsPerWorker,
     maxP95Ms,
     maxErrorRate,
-    headroomRatio,
-    requestTimeoutMs
+    safetyRatio,
+    requestTimeoutMs,
+    minimumStageDurationMs,
+    maxCpuUtilizationPercent,
+    maxMemoryUtilizationPercent
   };
 }
-async function stage(endpoint, concurrency, requestsPerWorker, timeoutMs, fetcher) {
+async function stage(endpoint, concurrency, requestsPerWorker, timeoutMs, minimumDurationMs, fetcher, sampler) {
   const latencies = [];
   let succeeded = 0;
   let failed = 0;
   let overloaded = 0;
+  const systemBefore = sampler();
   const started = performance.now();
   await Promise.all(Array.from({ length: concurrency }, async () => {
-    for (let request = 0;request < requestsPerWorker; request += 1) {
+    let request = 0;
+    while (request < requestsPerWorker || performance.now() - started < minimumDurationMs) {
+      request += 1;
       const requestStarted = performance.now();
       try {
         const response = await fetcher(endpoint, {
@@ -1447,50 +1472,74 @@ async function stage(endpoint, concurrency, requestsPerWorker, timeoutMs, fetche
       }
     }
   }));
-  const elapsedSeconds = Math.max((performance.now() - started) / 1000, 0.001);
+  const durationMs = Math.max(performance.now() - started, 1);
+  const elapsedSeconds = durationMs / 1000;
+  const systemAfter = sampler();
+  const cpuTotal = Math.max(0, systemAfter.cpuTotal - systemBefore.cpuTotal);
+  const cpuIdle = Math.max(0, systemAfter.cpuIdle - systemBefore.cpuIdle);
+  const cpuUtilizationPercent = cpuTotal === 0 ? 0 : (cpuTotal - cpuIdle) / cpuTotal * 100;
+  const memoryUtilizationPercent = Math.max(systemBefore.memoryTotal > 0 ? systemBefore.memoryUsed / systemBefore.memoryTotal * 100 : 0, systemAfter.memoryTotal > 0 ? systemAfter.memoryUsed / systemAfter.memoryTotal * 100 : 0);
+  const requests = succeeded + failed;
   return {
     concurrency,
-    requests: concurrency * requestsPerWorker,
+    requests,
     succeeded,
     failed,
     overloaded,
-    throughputPerSecond: Number(((succeeded + failed) / elapsedSeconds).toFixed(2)),
-    p95Ms: Number(percentile95(latencies).toFixed(2))
+    successfulRequestsPerSecond: Number((succeeded / elapsedSeconds).toFixed(2)),
+    attemptedRequestsPerSecond: Number((requests / elapsedSeconds).toFixed(2)),
+    durationMs: Number(durationMs.toFixed(2)),
+    errorRate: Number((failed / Math.max(requests, 1)).toFixed(6)),
+    p95Ms: Number(percentile(latencies, 0.95).toFixed(2)),
+    p99Ms: Number(percentile(latencies, 0.99).toFixed(2)),
+    cpuUtilizationPercent: Number(cpuUtilizationPercent.toFixed(2)),
+    memoryUtilizationPercent: Number(memoryUtilizationPercent.toFixed(2))
   };
 }
-async function calibrateHttpConcurrency(options, fetcher = fetch) {
+async function calibrateHttpConcurrency(options, fetcher = fetch, sampler = hostSample) {
   const {
     endpoint,
     maxConcurrency,
     requestsPerWorker,
     maxP95Ms,
     maxErrorRate,
-    headroomRatio,
-    requestTimeoutMs: timeoutMs
+    safetyRatio,
+    requestTimeoutMs: timeoutMs,
+    minimumStageDurationMs,
+    maxCpuUtilizationPercent,
+    maxMemoryUtilizationPercent
   } = validateCapacityCalibrationOptions(options);
   const stages = [];
-  let lastSafe = 1;
+  let lastSafe;
   let stopReason = "maximum-tested";
   for (let concurrency = 1;; concurrency = Math.min(maxConcurrency, concurrency * 2)) {
-    const measured = await stage(endpoint, concurrency, requestsPerWorker, timeoutMs, fetcher);
+    const measured = await stage(endpoint, concurrency, requestsPerWorker, timeoutMs, minimumStageDurationMs, fetcher, sampler);
     stages.push(measured);
-    const errorRate = measured.failed / measured.requests;
     const previous = stages.at(-2);
-    const throughputRegressed = Boolean(previous && concurrency > 1 && measured.throughputPerSecond < previous.throughputPerSecond * 0.9);
-    if (measured.overloaded > 0 || errorRate > maxErrorRate)
+    const throughputRegressed = Boolean(previous && concurrency > 1 && measured.successfulRequestsPerSecond < previous.successfulRequestsPerSecond * 0.9);
+    if (measured.overloaded > 0 || measured.errorRate > maxErrorRate)
       stopReason = "errors";
     else if (measured.p95Ms > maxP95Ms)
       stopReason = "latency";
+    else if (measured.cpuUtilizationPercent > maxCpuUtilizationPercent)
+      stopReason = "cpu";
+    else if (measured.memoryUtilizationPercent > maxMemoryUtilizationPercent)
+      stopReason = "memory";
     else if (throughputRegressed)
       stopReason = "throughput-regression";
     else
-      lastSafe = concurrency;
+      lastSafe = measured;
     if (stopReason !== "maximum-tested" || concurrency === maxConcurrency)
       break;
   }
+  if (!lastSafe)
+    throw new Error("Capacity calibration found no safe request stage.");
   return {
     endpoint: endpoint.toString(),
-    recommendedConcurrency: Math.max(1, Math.floor(lastSafe * headroomRatio)),
+    measuredSustainableRequestsPerSecond: lastSafe.successfulRequestsPerSecond,
+    allowedRequestsPerSecond: Math.max(1, Math.floor(lastSafe.successfulRequestsPerSecond * safetyRatio)),
+    safetyRatio,
+    recommendedConcurrency: Math.max(1, Math.floor(lastSafe.concurrency * safetyRatio)),
     stopReason,
     stages
   };
@@ -1567,8 +1616,11 @@ function capacityCalibration(value, where) {
     "requestsPerWorker",
     "maxP95Ms",
     "maxErrorRate",
-    "headroomRatio",
-    "requestTimeoutMs"
+    "safetyRatio",
+    "requestTimeoutMs",
+    "minimumStageDurationMs",
+    "maxCpuUtilizationPercent",
+    "maxMemoryUtilizationPercent"
   ], where);
   const endpoint = text(calibration.endpoint, `${where}.endpoint`);
   try {
@@ -1592,8 +1644,11 @@ function capacityCalibration(value, where) {
       "requestsPerWorker",
       "maxP95Ms",
       "maxErrorRate",
-      "headroomRatio",
-      "requestTimeoutMs"
+      "safetyRatio",
+      "requestTimeoutMs",
+      "minimumStageDurationMs",
+      "maxCpuUtilizationPercent",
+      "maxMemoryUtilizationPercent"
     ].flatMap((name) => {
       const found = optionalNumber(name);
       return found === undefined ? [] : [[name, found]];
@@ -2784,35 +2839,76 @@ function requestServiceActivation(request, socketPath = DEFAULT_SOFTWARE_HELPER_
 function requestSoftware(requirements, socketPath = DEFAULT_SOFTWARE_HELPER_SOCKET, timeoutMs = 15 * 60000) {
   validateSoftwareRequirements(requirements);
   return new Promise((resolve5, reject) => {
-    const socket = connect3(socketPath, () => socket.write(`${JSON.stringify({ op: "ensure", requirements })}
-`));
-    let buffer = "";
-    socket.setTimeout(timeoutMs, () => {
-      socket.destroy();
-      reject(new Error("software helper did not answer before its deadline"));
-    });
-    socket.on("data", (chunk) => {
-      buffer += chunk.toString("utf8");
-      const newline = buffer.indexOf(`
-`);
-      if (newline < 0)
+    const deadline = Date.now() + timeoutMs;
+    let settled = false;
+    const finish = (cause, results) => {
+      if (settled)
         return;
-      socket.end();
+      settled = true;
+      if (cause)
+        reject(cause);
+      else
+        resolve5(results);
+    };
+    const attempt = () => {
+      if (settled)
+        return;
+      const remaining = deadline - Date.now();
+      if (remaining <= 0)
+        return finish(new Error("software helper did not answer before its deadline"));
+      if (!existsSync7(socketPath)) {
+        setTimeout(attempt, Math.min(100, Math.max(1, remaining)));
+        return;
+      }
+      let buffer = "";
+      let socket;
       try {
-        const response = JSON.parse(buffer.slice(0, newline));
-        if (!response.ok || !response.results)
-          throw new Error(response.error?.message ?? "software helper refused the request");
-        resolve5(response.results);
+        socket = connect3(socketPath, () => socket.write(`${JSON.stringify({ op: "ensure", requirements })}
+`));
       } catch (cause) {
-        reject(cause);
+        const error = cause;
+        if ((error.code === "ENOENT" || error.code === "ECONNREFUSED") && Date.now() < deadline) {
+          setTimeout(attempt, Math.min(100, Math.max(1, deadline - Date.now())));
+          return;
+        }
+        finish(error);
+        return;
       }
-    });
-    socket.on("error", reject);
+      socket.setTimeout(remaining, () => {
+        socket.destroy();
+        finish(new Error("software helper did not answer before its deadline"));
+      });
+      socket.on("data", (chunk) => {
+        buffer += chunk.toString("utf8");
+        const newline = buffer.indexOf(`
+`);
+        if (newline < 0)
+          return;
+        socket.end();
+        try {
+          const response = JSON.parse(buffer.slice(0, newline));
+          if (!response.ok || !response.results)
+            throw new Error(response.error?.message ?? "software helper refused the request");
+          finish(undefined, response.results);
+        } catch (cause) {
+          finish(cause);
+        }
+      });
+      socket.on("error", (cause) => {
+        socket.destroy();
+        if ((cause.code === "ENOENT" || cause.code === "ECONNREFUSED") && Date.now() < deadline) {
+          setTimeout(attempt, Math.min(100, Math.max(1, deadline - Date.now())));
+          return;
+        }
+        finish(cause);
+      });
+    };
+    attempt();
   });
 }
 
 // src/version.ts
-var VERSION3 = "0.1.88";
+var VERSION3 = "0.1.89";
 
 // src/egress-policy.ts
 import { realpathSync as realpathSync3 } from "node:fs";
@@ -2948,6 +3044,7 @@ function agentEgressUnit(options) {
   }
   const deploymentEnabled = Boolean(options.repository || bootstrapBundleEnabled || options.pullDeployments);
   const runnerLoopbackPorts = normalizeEgressTcpPorts(options.runnerLoopbackPorts ?? []);
+  const agentLoopbackPorts = normalizeEgressTcpPorts(options.agentLoopbackPorts ?? []);
   const runnerPublicTcpPorts = normalizeEgressTcpPorts(options.runnerPublicTcpPorts ?? DEFAULT_RUNNER_PUBLIC_TCP_PORTS);
   if (deploymentEnabled && runnerPublicTcpPorts.length < 1) {
     throw new Error("deployed project runner needs at least one vetted public TCP port");
@@ -2955,8 +3052,10 @@ function agentEgressUnit(options) {
   if (deploymentEnabled)
     systemdAgentEgressDirectives(runnerLoopbackPorts);
   const users = [user, ...deploymentEnabled ? [DEPLOYMENT_RUNNER_USER] : []];
-  const runnerGrant = deploymentEnabled ? ` --loopback-user=${DEPLOYMENT_RUNNER_USER}` + runnerLoopbackPorts.map((port) => ` --loopback-tcp-port=${port}`).join("") + runnerPublicTcpPorts.map((port) => ` --public-tcp-port=${port}`).join("") : "";
-  const policyProof = `ExecStartPost=${bin} egress-policy-check ${users.map((name) => `--user=${name}`).join(" ")}${runnerGrant}`;
+  const portGrantEnabled = deploymentEnabled || agentLoopbackPorts.length > 0;
+  const restrictedUser = deploymentEnabled ? DEPLOYMENT_RUNNER_USER : user;
+  const portGrant = portGrantEnabled ? ` --loopback-user=${restrictedUser}` + (deploymentEnabled ? runnerLoopbackPorts.map((port) => ` --loopback-tcp-port=${port}`).join("") : "") + agentLoopbackPorts.map((port) => ` --shared-loopback-tcp-port=${port}`).join("") + (deploymentEnabled ? runnerPublicTcpPorts.map((port) => ` --public-tcp-port=${port}`).join("") : "") : "";
+  const policyProof = `ExecStartPost=${bin} egress-policy-check ${users.map((name) => `--user=${name}`).join(" ")}${portGrant}`;
   return `[Unit]
 Description=ForgeZero Agent host egress policy
 Documentation=https://www.forgezero.net/docs/agent
@@ -2969,7 +3068,7 @@ Type=notify
 NotifyAccess=all
 User=root
 Group=root
-ExecStart=${bin} egress-policy ${users.map((name) => `--user=${name}`).join(" ")}${runnerGrant}
+ExecStart=${bin} egress-policy ${users.map((name) => `--user=${name}`).join(" ")}${portGrant}
 ${policyProof}
 Restart=on-failure
 RestartSec=2
@@ -3253,7 +3352,7 @@ function agentEnrolmentUnit(options) {
 Requires=forgezero-agent-egress.service
 BindsTo=forgezero-agent-egress.service
 ` : "";
-  const egressDirectives = options.enforceEgress ? systemdAgentEgressDirectives() : "";
+  const egressDirectives = options.enforceEgress ? systemdAgentEgressDirectives(options.agentLoopbackPorts ?? []) : "";
   return `[Unit]
 Description=Bind this machine to its ForgeZero compute
 After=network-online.target
@@ -3290,7 +3389,7 @@ WantedBy=multi-user.target
 function deploymentRunnerUnit(options) {
   const bin = options.binPath ?? "fz-agent";
   const root = options.deployRoot ?? "/opt/forgezero";
-  const agentUser = options.user ?? "forgezero-agent";
+  const agentUser = options.user ?? "forgezero";
   const egressDependency = options.enforceEgress ? `After=forgezero-agent-egress.service
 Requires=forgezero-agent-egress.service
 BindsTo=forgezero-agent-egress.service
@@ -3331,7 +3430,7 @@ RestrictRealtime=true
 MemoryDenyWriteExecute=true
 LockPersonality=true
 ${egressDirectives}
-ReadWritePaths=${root}/releases ${root}/runner-home
+ReadWritePaths=${root}/releases ${root}/runner-home ${root}/slots /etc/nginx/conf.d /var/log/nginx
 
 [Install]
 WantedBy=multi-user.target
@@ -3466,7 +3565,7 @@ function agentUnit(options) {
   const supplementaryGroups = [
     AGENT_UPDATE_GROUP,
     deploymentEnabled ? DEPLOYMENT_GROUP : null,
-    deploymentEnabled ? SOFTWARE_HELPER_GROUP : null,
+    SOFTWARE_HELPER_GROUP,
     lifecycleEnabled ? LIFECYCLE_GROUP : null
   ].filter((value) => value !== null);
   const deploymentGroup = supplementaryGroups.length > 0 ? `SupplementaryGroups=${supplementaryGroups.join(" ")}` : "";
@@ -3475,7 +3574,7 @@ function agentUnit(options) {
     "forgezero-agent-update-helper.service",
     options.enforceEgress ? "forgezero-agent-egress.service" : null,
     deploymentEnabled ? "forgezero-deploy-runner.service" : null,
-    deploymentEnabled ? "forgezero-software-helper.service" : null,
+    "forgezero-software-helper.service",
     lifecycleEnabled ? "forgezero-lifecycle-helper.service" : null,
     warpEnabled ? "warp-svc.service" : null,
     enrolmentEnabled ? "forgezero-agent-enrol.service" : null
@@ -3484,7 +3583,7 @@ function agentUnit(options) {
     "forgezero-agent-update-helper.service",
     options.enforceEgress ? "forgezero-agent-egress.service" : null,
     deploymentEnabled ? "forgezero-deploy-runner.service" : null,
-    deploymentEnabled ? "forgezero-software-helper.service" : null,
+    "forgezero-software-helper.service",
     lifecycleEnabled ? "forgezero-lifecycle-helper.service" : null,
     warpEnabled ? "warp-svc.service" : null,
     enrolmentEnabled ? "forgezero-agent-enrol.service" : null
@@ -3499,9 +3598,9 @@ function agentUnit(options) {
   const snpDevice = options.mode === "attested" ? `DevicePolicy=closed
 DeviceAllow=/dev/sev-guest rw` : "";
   const snpPrepare = options.mode === "attested" ? `ExecStartPre=+/bin/chgrp ${VAULT_GROUP} /dev/sev-guest
-ExecStartPre=+/bin/chmod 0640 /dev/sev-guest
+ExecStartPre=+/bin/chmod 0660 /dev/sev-guest
 ` : "";
-  const egressDirectives = options.enforceEgress ? systemdAgentEgressDirectives() : "";
+  const egressDirectives = options.enforceEgress ? systemdAgentEgressDirectives(options.agentLoopbackPorts ?? []) : "";
   return `[Unit]
 Description=ForgeZero node agent (${options.mode})
 Documentation=https://www.forgezero.net/docs/agent
@@ -3632,6 +3731,7 @@ function planProvision(options) {
   const deployRoot = options.deployRoot ?? "/opt/forgezero";
   const deploymentEnabled = Boolean(options.repository || options.pullDeployments);
   const runnerLoopbackPorts = normalizeEgressTcpPorts(options.runnerLoopbackPorts ?? []);
+  const agentLoopbackPorts = normalizeEgressTcpPorts(options.agentLoopbackPorts ?? []);
   const runnerPublicTcpPorts = normalizeEgressTcpPorts(options.runnerPublicTcpPorts ?? DEFAULT_RUNNER_PUBLIC_TCP_PORTS);
   const lifecycleEnabled = Boolean(options.pullMigrations && options.lifecycleProfilePath);
   if (Boolean(options.pullMigrations) !== Boolean(options.lifecycleProfilePath)) {
@@ -3680,8 +3780,9 @@ function planProvision(options) {
   const enabledUnits = [
     "forgezero-agent.socket",
     "forgezero-agent-update-helper.service",
+    "forgezero-software-helper.service",
     ...options.enforceEgress ? ["forgezero-agent-egress.service"] : [],
-    ...deploymentEnabled ? ["forgezero-deploy-runner.service", "forgezero-software-helper.service"] : [],
+    ...deploymentEnabled ? ["forgezero-deploy-runner.service"] : [],
     ...lifecycleEnabled ? ["forgezero-lifecycle-helper.service"] : [],
     ...warpEnabled ? ["forgezero-warp-config.service", "warp-svc.service"] : [],
     ...enrolmentEnabled ? ["forgezero-agent-enrol.service"] : [],
@@ -3689,8 +3790,9 @@ function planProvision(options) {
   ];
   const restartedUnits = [
     "forgezero-agent-update-helper.service",
+    "forgezero-software-helper.service",
     ...options.enforceEgress ? ["forgezero-agent-egress.service"] : [],
-    ...deploymentEnabled ? ["forgezero-deploy-runner.service", "forgezero-software-helper.service"] : [],
+    ...deploymentEnabled ? ["forgezero-deploy-runner.service"] : [],
     ...lifecycleEnabled ? ["forgezero-lifecycle-helper.service"] : [],
     ...warpEnabled ? ["forgezero-warp-config.service", "warp-svc.service"] : [],
     ...enrolmentEnabled ? ["forgezero-agent-enrol.service"] : []
@@ -3714,9 +3816,9 @@ function planProvision(options) {
       ...options.enforceEgress ? [
         { path: AGENT_EGRESS_UNIT_PATH, unit: agentEgressUnit(options) }
       ] : [],
+      { path: SOFTWARE_HELPER_UNIT_PATH, unit: softwareHelperUnit(options) },
       ...deploymentEnabled ? [
-        { path: DEPLOYMENT_RUNNER_UNIT_PATH, unit: deploymentRunnerUnit(options) },
-        { path: SOFTWARE_HELPER_UNIT_PATH, unit: softwareHelperUnit(options) }
+        { path: DEPLOYMENT_RUNNER_UNIT_PATH, unit: deploymentRunnerUnit(options) }
       ] : [],
       ...enrolmentEnabled ? [
         { path: ENROLMENT_UNIT_PATH, unit: agentEnrolmentUnit(options) }
@@ -3749,24 +3851,29 @@ function planProvision(options) {
       step("Agent update helper access group", { kind: "commands", commands: [{ argv: ["/usr/sbin/groupadd", "--system", AGENT_UPDATE_GROUP], acceptedExitCodes: [0, 9] }] }),
       ...sourceBinPath && binPath ? [step("root-owned agent runtime", { kind: "install-runtime", source: sourceBinPath, binary: binPath, version: VERSION3 })] : [],
       ...warpEnabled ? [step("Cloudflare One client for Ubuntu 26.04", { kind: "install-warp" })] : [],
-      ...deploymentEnabled ? [step("deployment isolation group", { kind: "commands", commands: [
-        { argv: ["/usr/sbin/groupadd", "--system", DEPLOYMENT_GROUP], acceptedExitCodes: [0, 9] },
+      step("software strategy helper group", { kind: "commands", commands: [
         { argv: ["/usr/sbin/groupadd", "--system", SOFTWARE_HELPER_GROUP], acceptedExitCodes: [0, 9] }
+      ] }),
+      ...deploymentEnabled ? [step("deployment isolation group", { kind: "commands", commands: [
+        { argv: ["/usr/sbin/groupadd", "--system", DEPLOYMENT_GROUP], acceptedExitCodes: [0, 9] }
       ] })] : [],
       ...lifecycleEnabled ? [step("lifecycle helper access group", { kind: "commands", commands: [{ argv: ["/usr/sbin/groupadd", "--system", LIFECYCLE_GROUP], acceptedExitCodes: [0, 9] }] })] : [],
       step("service account", { kind: "commands", commands: [{ argv: ["/usr/sbin/useradd", "--system", "--no-create-home", "--shell", "/usr/sbin/nologin", user], acceptedExitCodes: [0, 9] }] }),
       step("bind service account to vault group", { kind: "commands", commands: [{ argv: ["/usr/sbin/usermod", "-g", VAULT_GROUP, user] }] }),
       step("grant verified Agent update access", { kind: "commands", commands: [{ argv: ["/usr/sbin/usermod", "-a", "-G", AGENT_UPDATE_GROUP, user] }] }),
+      step("grant software helper socket access", { kind: "commands", commands: [
+        { argv: ["/usr/sbin/usermod", "-a", "-G", SOFTWARE_HELPER_GROUP, user] }
+      ] }),
       ...lifecycleEnabled ? [step("grant lifecycle helper socket access", { kind: "commands", commands: [{ argv: ["/usr/sbin/usermod", "-a", "-G", LIFECYCLE_GROUP, user] }] })] : [],
       ...deploymentEnabled ? [step("credential-free deployment account", { kind: "commands", commands: [
         { argv: ["/usr/sbin/useradd", "--system", "--no-create-home", "--shell", "/usr/sbin/nologin", "--gid", DEPLOYMENT_GROUP, DEPLOYMENT_RUNNER_USER], acceptedExitCodes: [0, 9] },
         { argv: ["/usr/sbin/useradd", "--system", "--no-create-home", "--shell", "/usr/sbin/nologin", "--gid", VAULT_GROUP, APPLICATION_RUNTIME_USER], acceptedExitCodes: [0, 9] },
         { argv: ["/usr/sbin/usermod", "-a", "-G", DEPLOYMENT_GROUP, APPLICATION_RUNTIME_USER] },
-        { argv: ["/usr/sbin/usermod", "-a", "-G", `${DEPLOYMENT_GROUP},${SOFTWARE_HELPER_GROUP}`, user] }
+        { argv: ["/usr/sbin/usermod", "-a", "-G", DEPLOYMENT_GROUP, user] }
       ] })] : [],
       step("credential and state directories", { kind: "directories", directories: [
         { path: credentialDir, mode: 448, owner: "root", group: "root" },
-        { path: "/var/lib/forgezero", mode: 488, owner: "root", group: "root" }
+        { path: "/var/lib/forgezero", mode: 488, owner: "root", group: VAULT_GROUP }
       ] }),
       step("encrypted node identity", { kind: "ensure-seed", credential: seedCredentialPath }),
       ...options.generateGitIdentity && gitCredentialPath && gitPublicKeyPath ? [
@@ -3817,19 +3924,26 @@ function planProvision(options) {
         { argv: ["/usr/bin/systemctl", "enable", ...enabledUnits] },
         { argv: ["/usr/bin/systemctl", "reset-failed", "forgezero-agent.service"], acceptedExitCodes: [0, 1] },
         ...restartedUnits.length ? [{ argv: ["/usr/bin/systemctl", "restart", ...restartedUnits] }] : [],
+        { argv: ["/usr/bin/systemctl", "stop", "forgezero-agent-proxy.service"], acceptedExitCodes: [0, 1, 5] },
         { argv: ["/usr/bin/systemctl", "restart", "forgezero-agent.socket"] },
         { argv: ["/usr/bin/systemctl", "reset-failed", "forgezero-agent.service"], acceptedExitCodes: [0, 1] },
         { argv: ["/usr/bin/systemctl", "restart", "forgezero-agent.service"] }
       ] }),
       ...enrolmentEnabled ? [step("prove the compute binding is durable", { kind: "verify-file", path: enrolStatePath })] : [],
-      ...options.enforceEgress ? [step("prove the Agent egress policy is active", { kind: "verify-egress", runnerPublicTcpPorts, runnerLoopbackPorts })] : [],
+      ...options.enforceEgress ? [step("prove the Agent egress policy is active", {
+        kind: "verify-egress",
+        deploymentEnabled,
+        runnerPublicTcpPorts,
+        runnerLoopbackPorts,
+        agentLoopbackPorts
+      })] : [],
       step("prove it is running", { kind: "commands", commands: [{ argv: ["/usr/bin/systemctl", "is-active", "forgezero-agent.service"] }] }),
       step("prove the public Vault socket exists", { kind: "wait-socket", path: options.socketPath, attempts: 100, intervalMs: 100 }),
       step("prove the Agent Vault backend exists", { kind: "wait-socket", path: agentBackendSocketPath(options.socketPath), attempts: 100, intervalMs: 100 }),
       step("prove the Agent update helper exists", { kind: "wait-socket", path: DEFAULT_AGENT_UPDATE_SOCKET, attempts: 100, intervalMs: 100 }),
+      step("prove the software strategy helper socket exists", { kind: "wait-socket", path: DEFAULT_SOFTWARE_HELPER_SOCKET, attempts: 100, intervalMs: 100 }),
       ...deploymentEnabled ? [
-        step("prove the deployment runner socket exists", { kind: "wait-socket", path: DEPLOYMENT_RUNNER_SOCKET, attempts: 100, intervalMs: 100 }),
-        step("prove the software strategy helper socket exists", { kind: "wait-socket", path: DEFAULT_SOFTWARE_HELPER_SOCKET, attempts: 100, intervalMs: 100 })
+        step("prove the deployment runner socket exists", { kind: "wait-socket", path: DEPLOYMENT_RUNNER_SOCKET, attempts: 100, intervalMs: 100 })
       ] : [],
       ...lifecycleEnabled ? [step("prove the lifecycle helper socket exists", { kind: "wait-socket", path: lifecycleHelperSocketPath, attempts: 100, intervalMs: 100 })] : [],
       ...warpEnabled ? [step("prove Cloudflare WARP is connected", { kind: "verify-warp" })] : [],
@@ -4350,12 +4464,26 @@ map $limit_conn_status $forgezero_retry_after { default ''; REJECTED 1; REJECTED
 server {
   listen 127.0.0.1:${input.publicPort};
   server_name _;
+  server_tokens off;
+  more_clear_headers Server X-Powered-By;
   limit_conn forgezero_admission ${concurrencyLimit};
   limit_conn_status 503;
   add_header Retry-After $forgezero_retry_after always;
-  location ^~ /api/ { proxy_pass http://forgezero; proxy_http_version 1.1; proxy_set_header Host $host; proxy_set_header X-Forwarded-Proto https; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection $forgezero_connection; proxy_read_timeout 3600s; }
-  location ^~ /v1/ { proxy_pass http://forgezero; proxy_http_version 1.1; proxy_set_header Host $host; proxy_set_header X-Forwarded-Proto https; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection $forgezero_connection; proxy_read_timeout 3600s; }
-  location / { return 404; }
+  error_page 400 403 404 405 408 413 414 429 500 502 503 504 = @transport_failure;
+  location @transport_failure { internal; return 444; }
+  location / {
+    proxy_pass http://forgezero;
+    proxy_http_version 1.1;
+    proxy_intercept_errors off;
+    proxy_hide_header Server;
+    proxy_hide_header X-Powered-By;
+    proxy_set_header Host $host;
+    proxy_set_header X-Forwarded-Proto https;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header Upgrade $http_upgrade;
+    proxy_set_header Connection $forgezero_connection;
+    proxy_read_timeout 3600s;
+  }
 }
 `
   };
@@ -4428,11 +4556,29 @@ var replaceLink = (path2, target) => {
   symlinkSync4(target, pending);
   renameSync8(pending, path2);
 };
+var assertReleaseLinksContained = (path2) => {
+  const root = realpathSync4(path2);
+  const visit = (current2) => {
+    const metadata = lstatSync(current2);
+    if (metadata.isSymbolicLink()) {
+      const target = realpathSync4(current2);
+      if (target !== root && !target.startsWith(`${root}/`)) {
+        throw new Error("release contains a symbolic link outside its immutable root");
+      }
+      return;
+    }
+    if (metadata.isDirectory())
+      for (const name of readdirSync3(current2))
+        visit(join5(current2, name));
+  };
+  visit(root);
+};
 var secureRelease = (path2, uid, gid) => {
+  assertReleaseLinksContained(path2);
   const visit = (current2) => {
     const metadata = lstatSync(current2);
     if (metadata.isSymbolicLink())
-      throw new Error("release contains a symbolic link");
+      return;
     chownSync(current2, uid, gid);
     chmodSync6(current2, metadata.isDirectory() ? 365 : 292);
     if (metadata.isDirectory())
@@ -4458,7 +4604,7 @@ async function activatePlatformRelease(config, requestedRelease, options = {}) {
   const sleep = options.sleep ?? Bun.sleep;
   const slots = join5(normalized.root, "slots");
   mkdirSync8(slots, { recursive: true, mode: 493 });
-  const slotFile = join5(normalized.root, ".forge-slot");
+  const slotFile = join5(slots, ".active");
   const previousSlot = existsSync8(slotFile) ? readFileSync6(slotFile, "utf8").trim() : undefined;
   const target = previousSlot === "blue" ? "green" : "blue";
   const port = target === "blue" ? normalized.bluePort : normalized.greenPort;
@@ -4508,6 +4654,7 @@ async function activatePlatformRelease(config, requestedRelease, options = {}) {
   const test = await exec(["/usr/sbin/nginx", "-t"]);
   const reload = test.exitCode === 0 ? await exec(["/usr/sbin/nginx", "-s", "reload"]) : test;
   if (reload.exitCode !== 0) {
+    const refusal = (reload.output || test.output).trim().slice(0, 2000);
     if (existsSync8(backup))
       renameSync8(backup, upstream);
     else
@@ -4515,7 +4662,7 @@ async function activatePlatformRelease(config, requestedRelease, options = {}) {
     await exec(["/usr/sbin/nginx", "-t"]);
     await exec(["/usr/sbin/nginx", "-s", "reload"]);
     await stopTarget();
-    throw new Error("nginx refused the promoted upstream");
+    throw new Error(`nginx refused the promoted upstream${refusal ? `: ${refusal}` : ""}`);
   }
   rmSync7(backup, { force: true });
   writeFileSync7(slotFile, `${target}
@@ -4601,6 +4748,25 @@ import {
   writeFileSync as writeFileSync8
 } from "node:fs";
 import { dirname as dirname9 } from "node:path";
+function normalizeEd25519PublicKey(output) {
+  const line = output.trim();
+  if (!/^ssh-ed25519 [A-Za-z0-9+/]+={0,3}(?: [^\r\n]{1,256})?$/.test(line))
+    return;
+  const [algorithm, encoded] = line.split(" ", 3);
+  if (algorithm !== "ssh-ed25519" || !encoded)
+    return;
+  let blob;
+  try {
+    blob = Buffer.from(encoded, "base64");
+  } catch {
+    return;
+  }
+  if (blob.length !== 51 || blob.readUInt32BE(0) !== 11 || blob.subarray(4, 15).toString("ascii") !== "ssh-ed25519" || blob.readUInt32BE(15) !== 32)
+    return;
+  if (blob.toString("base64") !== encoded)
+    return;
+  return `${algorithm} ${encoded}`;
+}
 async function readCapabilities(run2) {
   const answers = {};
   const checks = Object.entries(CAPABILITY_CHECKS);
@@ -4740,7 +4906,7 @@ var runProvisionOperation = async (operation) => {
             return result;
         }
         result = await fixed(["/usr/bin/ssh-keygen", "-y", "-f", key]);
-        if (result.exitCode !== 0 || !/^ssh-ed25519 [A-Za-z0-9+/]+={0,3}\s*$/.test(result.stdout)) {
+        if (result.exitCode !== 0 || !normalizeEd25519PublicKey(result.stdout)) {
           return { stdout: "bootstrap SSH private key is not a valid Ed25519 OpenSSH key", exitCode: 1 };
         }
         result = await fixed(["/usr/bin/systemd-creds", "encrypt", "--name=bootstrap-ssh-key", key, operation.credential]);
@@ -4756,11 +4922,12 @@ var runProvisionOperation = async (operation) => {
           chmodSync7(key, 384);
         }
         const derived = await fixed(["/usr/bin/ssh-keygen", "-y", "-f", key]);
-        if (derived.exitCode !== 0 || !/^ssh-ed25519 [A-Za-z0-9+/]+={0,3}\s*$/.test(derived.stdout)) {
+        const publicKey2 = normalizeEd25519PublicKey(derived.stdout);
+        if (derived.exitCode !== 0 || !publicKey2) {
           return { stdout: "bootstrap SSH public key derivation failed", exitCode: 1 };
         }
         mkdirSync9(dirname9(operation.publicKey), { recursive: true, mode: 493 });
-        writeFileSync8(operation.publicKey, `${derived.stdout.trim()} forgezero-bootstrap-runner
+        writeFileSync8(operation.publicKey, `${publicKey2} forgezero-bootstrap-runner
 `, { mode: 292 });
         chmodSync7(operation.publicKey, 292);
       }
@@ -4803,8 +4970,9 @@ var runProvisionOperation = async (operation) => {
     const policy = await fixed(["/usr/sbin/nft", "--numeric", "list", "table", "inet", "forgezero_agent_egress"]);
     const required = [
       "forgezero-agent-egress-v1",
-      `public-tcp=${operation.runnerPublicTcpPorts.join(",")}`,
-      ...operation.runnerLoopbackPorts.length ? [`:${operation.runnerLoopbackPorts.join(",")}`] : []
+      ...operation.agentLoopbackPorts.length ? [`shared-loopback=${operation.agentLoopbackPorts.join(",")}`] : [],
+      ...operation.deploymentEnabled && operation.runnerPublicTcpPorts.length ? [`public-tcp=${operation.runnerPublicTcpPorts.join(",")}`] : [],
+      ...operation.deploymentEnabled && operation.runnerLoopbackPorts.length ? [`:${operation.runnerLoopbackPorts.join(",")}`] : []
     ];
     return policy.exitCode === 0 && required.every((part) => policy.stdout.includes(part)) ? { stdout: policy.stdout, exitCode: 0 } : { stdout: policy.stdout, exitCode: 1 };
   }
@@ -4870,7 +5038,9 @@ async function applyPlan(plan, run2) {
     const result = await run2(step2.operation);
     transcript.push({ label: step2.label, command: step2.command, exitCode: result.exitCode });
     if (result.exitCode !== 0 && !step2.optional) {
-      throw new Error(`${step2.label} failed (exit ${result.exitCode}): ${step2.command}`);
+      const safeDetail = step2.operation.kind === "ensure-bootstrap-ssh-identity" ? result.stdout.trim().slice(0, 1000) : "";
+      throw new Error(`${step2.label} failed (exit ${result.exitCode}): ${step2.command}${safeDetail ? `
+${safeDetail}` : ""}`);
     }
   }
   return transcript;
@@ -4960,7 +5130,6 @@ var SEED_CREDENTIAL = `${CREDS}/seed-sync-root.cred`;
 var BACKUP_RECOVERY_CREDENTIAL = `${CREDS}/backup-recovery-root.cred`;
 var BOOTSTRAP_SSH_CREDENTIAL = `${CREDS}/bootstrap-ssh-key.cred`;
 var BOOTSTRAP_SSH_PUBLIC_KEY = "/etc/forgezero/bootstrap/runner.pub";
-var BOOTSTRAP_RELEASE_EVIDENCE = "/var/lib/forgezero/bootstrap-release.json";
 var LIFECYCLE_PROFILE = "/etc/forgezero/lifecycle.json";
 var CONTROL_SOCKET = "/run/forgezero/control.sock";
 var CLOUDFLARED_METRICS_ADDRESS = "127.0.0.1:20241";
@@ -5179,6 +5348,8 @@ function planBootstrapAgentInstall(config, phase, context) {
     bootstrapTargetTelemetryEndpoint: bound && bootstrapRunner ? platformBootstrapRunner(config) ? config.telemetryEndpoint : config.kind === "enrolled-compute" ? config.bootstrapRunner?.targetTelemetryEndpoint : undefined : undefined,
     lifecycleProfilePath: bound && config.kind === "platform" ? LIFECYCLE_PROFILE : undefined,
     enforceEgress: true,
+    runnerLoopbackPorts: config.kind === "platform" ? [config.runtime.environment.publicApiPort, config.runtime.bluePort, config.runtime.greenPort] : [],
+    agentLoopbackPorts: config.kind === "platform" ? [config.runtime.environment.publicApiPort] : [],
     nodeHostname: config.nodeHostname,
     telemetryEndpoint: config.telemetryEndpoint,
     binPath: "/usr/local/lib/forgezero/agent/fz-agent",
@@ -5188,7 +5359,7 @@ function planBootstrapAgentInstall(config, phase, context) {
       enrolStatePath: "/var/lib/forgezero/enrolment.json"
     } : bound ? { enrolStatePath: "/var/lib/forgezero/enrolment.json" } : {},
     ...authenticated ? {
-      apiUrl: config.apiUrl,
+      apiUrl: config.kind === "platform" ? `http://127.0.0.1:${config.runtime.environment.publicApiPort}` : config.apiUrl,
       project: config.kind === "enrolled-compute" ? config.realm : "platform",
       environment: config.kind === "enrolled-compute" ? undefined : config.environment,
       nodeLabel: config.kind === "enrolled-compute" ? config.nodeHostname : config.computeReference
@@ -5197,6 +5368,7 @@ function planBootstrapAgentInstall(config, phase, context) {
   return planInstall(options);
 }
 function localBootstrapHost() {
+  let softwareClientUser;
   const execute = async (argv2, options = {}) => {
     const child = Bun.spawn([...argv2], { stdin: options.stdin === undefined ? "ignore" : "pipe", stdout: "pipe", stderr: "pipe" });
     if (options.stdin !== undefined && child.stdin && typeof child.stdin !== "number") {
@@ -5229,10 +5401,12 @@ function localBootstrapHost() {
     exec: execute,
     sleep: (milliseconds) => Bun.sleep(milliseconds),
     async ensureSoftware(requirements) {
+      if (!softwareClientUser)
+        throw new Error("Agent must be converged before software requirements are applied");
       const result = await execute([
         "runuser",
         "-u",
-        "forgezero-agent",
+        softwareClientUser,
         "--",
         "/usr/local/bin/fz-agent",
         "software-ensure",
@@ -5246,7 +5420,7 @@ function localBootstrapHost() {
       const capabilities = await readCapabilities(localRunner);
       const hasBinding = existsSync11("/var/lib/forgezero/enrolment.json");
       const hasEnrolCredential = existsSync11(ENROL_CREDENTIAL);
-      const initialBundle = config.kind === "platform" && !existsSync11(BOOTSTRAP_RELEASE_EVIDENCE) ? {
+      const initialBundle = config.kind === "platform" && phase === "bootstrap" && existsSync11(config.bootstrapBundle.bundleFile) && existsSync11(config.bootstrapBundle.manifestFile) ? {
         path: config.bootstrapBundle.bundleFile,
         manifestPath: config.bootstrapBundle.manifestFile,
         manifest: parseBootstrapBundleManifest(JSON.parse(readFileSync9(config.bootstrapBundle.manifestFile, "utf8")))
@@ -5277,6 +5451,7 @@ function localBootstrapHost() {
         writeFileSync10(unit2.path, unit2.unit, { mode: 420 });
       }
       await applyPlan(plan, localRunner);
+      softwareClientUser = plan.user;
       return plan;
     }
   };