@forgezero/agent 0.1.88 → 0.1.90

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. package/README.md +23 -9
  2. package/dist/agent-heartbeat.d.ts +3 -0
  3. package/dist/agent-heartbeat.js +2 -1
  4. package/dist/bootstrap.d.ts +8 -1
  5. package/dist/bootstrap.js +241 -62
  6. package/dist/capacity-calibration.d.ts +35 -5
  7. package/dist/capacity-calibration.js +66 -22
  8. package/dist/cli/agent-install.d.ts +3 -0
  9. package/dist/community-rehearsal-host.js +7 -2
  10. package/dist/control.d.ts +3 -1
  11. package/dist/definition.js +83 -28
  12. package/dist/deploy-file.js +83 -28
  13. package/dist/deployment.d.ts +5 -0
  14. package/dist/egress-policy.d.ts +4 -0
  15. package/dist/fz-agent.js +450 -158
  16. package/dist/fz.js +429 -138
  17. package/dist/host-maintenance.js +1 -1
  18. package/dist/index.d.ts +7 -0
  19. package/dist/metal-bootstrap.js +1 -1
  20. package/dist/metal-helper-socket.js +7 -2
  21. package/dist/metal-provision.js +7 -2
  22. package/dist/migration-pull.d.ts +1 -0
  23. package/dist/migration-pull.js +4 -0
  24. package/dist/operator-bootstrap.d.ts +12 -0
  25. package/dist/operator-bootstrap.js +385 -72
  26. package/dist/platform-bootstrap-runtime.d.ts +1 -0
  27. package/dist/platform-bootstrap-runtime.js +47 -8
  28. package/dist/platform-fleet-verification.js +273 -95
  29. package/dist/platform-genesis.js +7 -2
  30. package/dist/platform-launch-profile.d.ts +1 -0
  31. package/dist/provision.d.ts +5 -1
  32. package/dist/provision.js +192 -75
  33. package/dist/recovery-host.js +1 -1
  34. package/dist/software-helper.js +144 -48
  35. package/dist/software.js +7 -2
  36. package/dist/ssh-bootstrap.d.ts +1 -0
  37. package/dist/ubuntu.js +7 -2
  38. package/dist/version.d.ts +1 -1
  39. package/package.json +1 -1
  40. package/schema/deploy-v3.json +5 -2
package/dist/fz.js CHANGED
@@ -4810,9 +4810,8 @@ async function spawnWith(command, env, report = () => {}, options = {}) {
4810
4810
  }
4811
4811
 
4812
4812
  // src/cli/index.ts
4813
- import { chmodSync as chmodSync7, existsSync as existsSync12, lstatSync as lstatSync10, mkdirSync as mkdirSync13, readFileSync as readFileSync16, writeFileSync as writeFileSync13 } from "fs";
4813
+ import { existsSync as existsSync13, lstatSync as lstatSync10, mkdirSync as mkdirSync13, readFileSync as readFileSync16, writeFileSync as writeFileSync13 } from "fs";
4814
4814
  import { basename as basename3, dirname as dirname14, isAbsolute as isAbsolute7, join as join13, resolve as resolve12 } from "path";
4815
- import { randomBytes as randomBytes10 } from "crypto";
4816
4815
 
4817
4816
  // src/process-input.ts
4818
4817
  async function writeAndCloseProcessInput(input, value) {
@@ -4822,8 +4821,7 @@ async function writeAndCloseProcessInput(input, value) {
4822
4821
 
4823
4822
  // src/cli/index.ts
4824
4823
  init_dist();
4825
- import { fileURLToPath as fileURLToPath4 } from "url";
4826
- import { hostname } from "os";
4824
+ import { homedir as homedir2, hostname } from "os";
4827
4825
 
4828
4826
  // src/agent-update.ts
4829
4827
  var DEFAULT_AGENT_RELEASE_ROOT = "/opt/forgezero/agent";
@@ -4842,7 +4840,7 @@ var UPDATE_RETRY_BASE_MS = 5 * 60000;
4842
4840
  var UPDATE_RETRY_MAX_MS = 24 * 60 * 60000;
4843
4841
 
4844
4842
  // src/version.ts
4845
- var VERSION2 = "0.1.88";
4843
+ var VERSION2 = "0.1.90";
4846
4844
 
4847
4845
  // src/software.ts
4848
4846
  var PINNED_BUN_VERSION = "1.3.14";
@@ -4944,9 +4942,12 @@ function validateCapacityCalibrationOptions(options) {
4944
4942
  const requestsPerWorker = options.requestsPerWorker ?? 8;
4945
4943
  const maxP95Ms = options.maxP95Ms ?? 250;
4946
4944
  const maxErrorRate = options.maxErrorRate ?? 0.01;
4947
- const headroomRatio = options.headroomRatio ?? 0.8;
4945
+ const safetyRatio = options.safetyRatio ?? 0.8;
4948
4946
  const requestTimeoutMs = options.requestTimeoutMs ?? 5000;
4949
- if (!Number.isSafeInteger(maxConcurrency) || maxConcurrency < 1 || maxConcurrency > 4096 || !Number.isSafeInteger(requestsPerWorker) || requestsPerWorker < 2 || requestsPerWorker > 100 || !Number.isFinite(maxP95Ms) || maxP95Ms < 1 || !Number.isFinite(maxErrorRate) || maxErrorRate < 0 || maxErrorRate > 0.2 || !Number.isFinite(headroomRatio) || headroomRatio < 0.25 || headroomRatio > 0.95 || !Number.isSafeInteger(requestTimeoutMs) || requestTimeoutMs < 100 || requestTimeoutMs > 30000) {
4947
+ const minimumStageDurationMs = options.minimumStageDurationMs ?? 5000;
4948
+ const maxCpuUtilizationPercent = options.maxCpuUtilizationPercent ?? 90;
4949
+ const maxMemoryUtilizationPercent = options.maxMemoryUtilizationPercent ?? 90;
4950
+ if (!Number.isSafeInteger(maxConcurrency) || maxConcurrency < 1 || maxConcurrency > 4096 || !Number.isSafeInteger(requestsPerWorker) || requestsPerWorker < 2 || requestsPerWorker > 100 || !Number.isFinite(maxP95Ms) || maxP95Ms < 1 || !Number.isFinite(maxErrorRate) || maxErrorRate < 0 || maxErrorRate > 0.2 || !Number.isFinite(safetyRatio) || safetyRatio < 0.25 || safetyRatio > 0.95 || !Number.isSafeInteger(requestTimeoutMs) || requestTimeoutMs < 100 || requestTimeoutMs > 30000 || !Number.isSafeInteger(minimumStageDurationMs) || minimumStageDurationMs < 100 || minimumStageDurationMs > 60000 || !Number.isFinite(maxCpuUtilizationPercent) || maxCpuUtilizationPercent < 25 || maxCpuUtilizationPercent > 100 || !Number.isFinite(maxMemoryUtilizationPercent) || maxMemoryUtilizationPercent < 25 || maxMemoryUtilizationPercent > 100) {
4950
4951
  throw new Error("Capacity calibration bounds are invalid.");
4951
4952
  }
4952
4953
  return {
@@ -4955,8 +4956,11 @@ function validateCapacityCalibrationOptions(options) {
4955
4956
  requestsPerWorker,
4956
4957
  maxP95Ms,
4957
4958
  maxErrorRate,
4958
- headroomRatio,
4959
- requestTimeoutMs
4959
+ safetyRatio,
4960
+ requestTimeoutMs,
4961
+ minimumStageDurationMs,
4962
+ maxCpuUtilizationPercent,
4963
+ maxMemoryUtilizationPercent
4960
4964
  };
4961
4965
  }
4962
4966
 
@@ -5031,8 +5035,11 @@ function capacityCalibration(value, where) {
5031
5035
  "requestsPerWorker",
5032
5036
  "maxP95Ms",
5033
5037
  "maxErrorRate",
5034
- "headroomRatio",
5035
- "requestTimeoutMs"
5038
+ "safetyRatio",
5039
+ "requestTimeoutMs",
5040
+ "minimumStageDurationMs",
5041
+ "maxCpuUtilizationPercent",
5042
+ "maxMemoryUtilizationPercent"
5036
5043
  ], where);
5037
5044
  const endpoint = text(calibration.endpoint, `${where}.endpoint`);
5038
5045
  try {
@@ -5056,8 +5063,11 @@ function capacityCalibration(value, where) {
5056
5063
  "requestsPerWorker",
5057
5064
  "maxP95Ms",
5058
5065
  "maxErrorRate",
5059
- "headroomRatio",
5060
- "requestTimeoutMs"
5066
+ "safetyRatio",
5067
+ "requestTimeoutMs",
5068
+ "minimumStageDurationMs",
5069
+ "maxCpuUtilizationPercent",
5070
+ "maxMemoryUtilizationPercent"
5061
5071
  ].flatMap((name) => {
5062
5072
  const found = optionalNumber(name);
5063
5073
  return found === undefined ? [] : [[name, found]];
@@ -5479,6 +5489,7 @@ function agentEgressUnit(options) {
5479
5489
  }
5480
5490
  const deploymentEnabled = Boolean(options.repository || bootstrapBundleEnabled || options.pullDeployments);
5481
5491
  const runnerLoopbackPorts = normalizeEgressTcpPorts(options.runnerLoopbackPorts ?? []);
5492
+ const agentLoopbackPorts = normalizeEgressTcpPorts(options.agentLoopbackPorts ?? []);
5482
5493
  const runnerPublicTcpPorts = normalizeEgressTcpPorts(options.runnerPublicTcpPorts ?? DEFAULT_RUNNER_PUBLIC_TCP_PORTS);
5483
5494
  if (deploymentEnabled && runnerPublicTcpPorts.length < 1) {
5484
5495
  throw new Error("deployed project runner needs at least one vetted public TCP port");
@@ -5486,8 +5497,10 @@ function agentEgressUnit(options) {
5486
5497
  if (deploymentEnabled)
5487
5498
  systemdAgentEgressDirectives(runnerLoopbackPorts);
5488
5499
  const users = [user, ...deploymentEnabled ? [DEPLOYMENT_RUNNER_USER] : []];
5489
- const runnerGrant = deploymentEnabled ? ` --loopback-user=${DEPLOYMENT_RUNNER_USER}` + runnerLoopbackPorts.map((port) => ` --loopback-tcp-port=${port}`).join("") + runnerPublicTcpPorts.map((port) => ` --public-tcp-port=${port}`).join("") : "";
5490
- const policyProof = `ExecStartPost=${bin} egress-policy-check ${users.map((name) => `--user=${name}`).join(" ")}${runnerGrant}`;
5500
+ const portGrantEnabled = deploymentEnabled || agentLoopbackPorts.length > 0;
5501
+ const restrictedUser = deploymentEnabled ? DEPLOYMENT_RUNNER_USER : user;
5502
+ const portGrant = portGrantEnabled ? ` --loopback-user=${restrictedUser}` + (deploymentEnabled ? runnerLoopbackPorts.map((port) => ` --loopback-tcp-port=${port}`).join("") : "") + agentLoopbackPorts.map((port) => ` --shared-loopback-tcp-port=${port}`).join("") + (deploymentEnabled ? runnerPublicTcpPorts.map((port) => ` --public-tcp-port=${port}`).join("") : "") : "";
5503
+ const policyProof = `ExecStartPost=${bin} egress-policy-check ${users.map((name) => `--user=${name}`).join(" ")}${portGrant}`;
5491
5504
  return `[Unit]
5492
5505
  Description=ForgeZero Agent host egress policy
5493
5506
  Documentation=https://www.forgezero.net/docs/agent
@@ -5500,7 +5513,7 @@ Type=notify
5500
5513
  NotifyAccess=all
5501
5514
  User=root
5502
5515
  Group=root
5503
- ExecStart=${bin} egress-policy ${users.map((name) => `--user=${name}`).join(" ")}${runnerGrant}
5516
+ ExecStart=${bin} egress-policy ${users.map((name) => `--user=${name}`).join(" ")}${portGrant}
5504
5517
  ${policyProof}
5505
5518
  Restart=on-failure
5506
5519
  RestartSec=2
@@ -5784,7 +5797,7 @@ function agentEnrolmentUnit(options) {
5784
5797
  Requires=forgezero-agent-egress.service
5785
5798
  BindsTo=forgezero-agent-egress.service
5786
5799
  ` : "";
5787
- const egressDirectives = options.enforceEgress ? systemdAgentEgressDirectives() : "";
5800
+ const egressDirectives = options.enforceEgress ? systemdAgentEgressDirectives(options.agentLoopbackPorts ?? []) : "";
5788
5801
  return `[Unit]
5789
5802
  Description=Bind this machine to its ForgeZero compute
5790
5803
  After=network-online.target
@@ -5821,7 +5834,7 @@ WantedBy=multi-user.target
5821
5834
  function deploymentRunnerUnit(options) {
5822
5835
  const bin = options.binPath ?? "fz-agent";
5823
5836
  const root = options.deployRoot ?? "/opt/forgezero";
5824
- const agentUser = options.user ?? "forgezero-agent";
5837
+ const agentUser = options.user ?? "forgezero";
5825
5838
  const egressDependency = options.enforceEgress ? `After=forgezero-agent-egress.service
5826
5839
  Requires=forgezero-agent-egress.service
5827
5840
  BindsTo=forgezero-agent-egress.service
@@ -5862,7 +5875,7 @@ RestrictRealtime=true
5862
5875
  MemoryDenyWriteExecute=true
5863
5876
  LockPersonality=true
5864
5877
  ${egressDirectives}
5865
- ReadWritePaths=${root}/releases ${root}/runner-home
5878
+ ReadWritePaths=${root}/releases ${root}/runner-home ${root}/slots /etc/nginx/conf.d /var/log/nginx
5866
5879
 
5867
5880
  [Install]
5868
5881
  WantedBy=multi-user.target
@@ -5872,7 +5885,7 @@ function agentUnit(options) {
5872
5885
  if (!validNodeHostname(options.nodeHostname))
5873
5886
  throw new Error("node hostname is invalid");
5874
5887
  if (!options.telemetryEndpoint) {
5875
- throw new Error("compute Agent provisioning requires OTEL_EXPORTER_OTLP_ENDPOINT as an explicit public HTTPS collector coordinate");
5888
+ throw new Error("compute Agent provisioning requires an explicit supervised OTLP collector coordinate");
5876
5889
  }
5877
5890
  let telemetryEndpoint;
5878
5891
  {
@@ -5880,10 +5893,13 @@ function agentUnit(options) {
5880
5893
  try {
5881
5894
  endpoint = new URL(options.telemetryEndpoint);
5882
5895
  } catch {
5883
- throw new Error("compute telemetry endpoint must be an absolute public HTTPS URL");
5896
+ throw new Error("compute telemetry endpoint must be an absolute collector URL");
5897
+ }
5898
+ const localCollector = endpoint.protocol === "http:" && endpoint.hostname === "127.0.0.1" && endpoint.port === "4318" && endpoint.pathname === "/";
5899
+ const publicCollector = endpoint.protocol === "https:" && !endpoint.username && !endpoint.password && !endpoint.search && !endpoint.hash && isIP(endpoint.hostname) === 0 && endpoint.hostname.includes(".") && endpoint.hostname !== "localhost" && !endpoint.hostname.endsWith(".local");
5900
+ if (!localCollector && !publicCollector || endpoint.username || endpoint.password || endpoint.search || endpoint.hash) {
5901
+ throw new Error("compute telemetry endpoint must be the supervised loopback collector or credential-free public HTTPS");
5884
5902
  }
5885
- if (endpoint.protocol !== "https:" || endpoint.username || endpoint.password || endpoint.search || endpoint.hash || isIP(endpoint.hostname) !== 0 || !endpoint.hostname.includes(".") || endpoint.hostname === "localhost" || endpoint.hostname.endsWith(".local"))
5886
- throw new Error("compute telemetry endpoint must be an absolute public HTTPS URL without credentials, query or fragment");
5887
5903
  telemetryEndpoint = endpoint.toString().replace(/\/$/, "");
5888
5904
  }
5889
5905
  const bin = options.binPath ?? "fz-agent";
@@ -5997,7 +6013,7 @@ function agentUnit(options) {
5997
6013
  const supplementaryGroups = [
5998
6014
  AGENT_UPDATE_GROUP,
5999
6015
  deploymentEnabled ? DEPLOYMENT_GROUP : null,
6000
- deploymentEnabled ? SOFTWARE_HELPER_GROUP : null,
6016
+ SOFTWARE_HELPER_GROUP,
6001
6017
  lifecycleEnabled ? LIFECYCLE_GROUP : null
6002
6018
  ].filter((value) => value !== null);
6003
6019
  const deploymentGroup = supplementaryGroups.length > 0 ? `SupplementaryGroups=${supplementaryGroups.join(" ")}` : "";
@@ -6006,7 +6022,7 @@ function agentUnit(options) {
6006
6022
  "forgezero-agent-update-helper.service",
6007
6023
  options.enforceEgress ? "forgezero-agent-egress.service" : null,
6008
6024
  deploymentEnabled ? "forgezero-deploy-runner.service" : null,
6009
- deploymentEnabled ? "forgezero-software-helper.service" : null,
6025
+ "forgezero-software-helper.service",
6010
6026
  lifecycleEnabled ? "forgezero-lifecycle-helper.service" : null,
6011
6027
  warpEnabled ? "warp-svc.service" : null,
6012
6028
  enrolmentEnabled ? "forgezero-agent-enrol.service" : null
@@ -6015,7 +6031,7 @@ function agentUnit(options) {
6015
6031
  "forgezero-agent-update-helper.service",
6016
6032
  options.enforceEgress ? "forgezero-agent-egress.service" : null,
6017
6033
  deploymentEnabled ? "forgezero-deploy-runner.service" : null,
6018
- deploymentEnabled ? "forgezero-software-helper.service" : null,
6034
+ "forgezero-software-helper.service",
6019
6035
  lifecycleEnabled ? "forgezero-lifecycle-helper.service" : null,
6020
6036
  warpEnabled ? "warp-svc.service" : null,
6021
6037
  enrolmentEnabled ? "forgezero-agent-enrol.service" : null
@@ -6030,9 +6046,9 @@ function agentUnit(options) {
6030
6046
  const snpDevice = options.mode === "attested" ? `DevicePolicy=closed
6031
6047
  DeviceAllow=/dev/sev-guest rw` : "";
6032
6048
  const snpPrepare = options.mode === "attested" ? `ExecStartPre=+/bin/chgrp ${VAULT_GROUP} /dev/sev-guest
6033
- ExecStartPre=+/bin/chmod 0640 /dev/sev-guest
6049
+ ExecStartPre=+/bin/chmod 0660 /dev/sev-guest
6034
6050
  ` : "";
6035
- const egressDirectives = options.enforceEgress ? systemdAgentEgressDirectives() : "";
6051
+ const egressDirectives = options.enforceEgress ? systemdAgentEgressDirectives(options.agentLoopbackPorts ?? []) : "";
6036
6052
  return `[Unit]
6037
6053
  Description=ForgeZero node agent (${options.mode})
6038
6054
  Documentation=https://www.forgezero.net/docs/agent
@@ -6163,6 +6179,7 @@ function planProvision(options) {
6163
6179
  const deployRoot = options.deployRoot ?? "/opt/forgezero";
6164
6180
  const deploymentEnabled = Boolean(options.repository || options.pullDeployments);
6165
6181
  const runnerLoopbackPorts = normalizeEgressTcpPorts(options.runnerLoopbackPorts ?? []);
6182
+ const agentLoopbackPorts = normalizeEgressTcpPorts(options.agentLoopbackPorts ?? []);
6166
6183
  const runnerPublicTcpPorts = normalizeEgressTcpPorts(options.runnerPublicTcpPorts ?? DEFAULT_RUNNER_PUBLIC_TCP_PORTS);
6167
6184
  const lifecycleEnabled = Boolean(options.pullMigrations && options.lifecycleProfilePath);
6168
6185
  if (Boolean(options.pullMigrations) !== Boolean(options.lifecycleProfilePath)) {
@@ -6211,8 +6228,9 @@ function planProvision(options) {
6211
6228
  const enabledUnits = [
6212
6229
  "forgezero-agent.socket",
6213
6230
  "forgezero-agent-update-helper.service",
6231
+ "forgezero-software-helper.service",
6214
6232
  ...options.enforceEgress ? ["forgezero-agent-egress.service"] : [],
6215
- ...deploymentEnabled ? ["forgezero-deploy-runner.service", "forgezero-software-helper.service"] : [],
6233
+ ...deploymentEnabled ? ["forgezero-deploy-runner.service"] : [],
6216
6234
  ...lifecycleEnabled ? ["forgezero-lifecycle-helper.service"] : [],
6217
6235
  ...warpEnabled ? ["forgezero-warp-config.service", "warp-svc.service"] : [],
6218
6236
  ...enrolmentEnabled ? ["forgezero-agent-enrol.service"] : [],
@@ -6220,8 +6238,9 @@ function planProvision(options) {
6220
6238
  ];
6221
6239
  const restartedUnits = [
6222
6240
  "forgezero-agent-update-helper.service",
6241
+ "forgezero-software-helper.service",
6223
6242
  ...options.enforceEgress ? ["forgezero-agent-egress.service"] : [],
6224
- ...deploymentEnabled ? ["forgezero-deploy-runner.service", "forgezero-software-helper.service"] : [],
6243
+ ...deploymentEnabled ? ["forgezero-deploy-runner.service"] : [],
6225
6244
  ...lifecycleEnabled ? ["forgezero-lifecycle-helper.service"] : [],
6226
6245
  ...warpEnabled ? ["forgezero-warp-config.service", "warp-svc.service"] : [],
6227
6246
  ...enrolmentEnabled ? ["forgezero-agent-enrol.service"] : []
@@ -6245,9 +6264,9 @@ function planProvision(options) {
6245
6264
  ...options.enforceEgress ? [
6246
6265
  { path: AGENT_EGRESS_UNIT_PATH, unit: agentEgressUnit(options) }
6247
6266
  ] : [],
6267
+ { path: SOFTWARE_HELPER_UNIT_PATH, unit: softwareHelperUnit(options) },
6248
6268
  ...deploymentEnabled ? [
6249
- { path: DEPLOYMENT_RUNNER_UNIT_PATH, unit: deploymentRunnerUnit(options) },
6250
- { path: SOFTWARE_HELPER_UNIT_PATH, unit: softwareHelperUnit(options) }
6269
+ { path: DEPLOYMENT_RUNNER_UNIT_PATH, unit: deploymentRunnerUnit(options) }
6251
6270
  ] : [],
6252
6271
  ...enrolmentEnabled ? [
6253
6272
  { path: ENROLMENT_UNIT_PATH, unit: agentEnrolmentUnit(options) }
@@ -6280,24 +6299,29 @@ function planProvision(options) {
6280
6299
  step("Agent update helper access group", { kind: "commands", commands: [{ argv: ["/usr/sbin/groupadd", "--system", AGENT_UPDATE_GROUP], acceptedExitCodes: [0, 9] }] }),
6281
6300
  ...sourceBinPath && binPath ? [step("root-owned agent runtime", { kind: "install-runtime", source: sourceBinPath, binary: binPath, version: VERSION2 })] : [],
6282
6301
  ...warpEnabled ? [step("Cloudflare One client for Ubuntu 26.04", { kind: "install-warp" })] : [],
6283
- ...deploymentEnabled ? [step("deployment isolation group", { kind: "commands", commands: [
6284
- { argv: ["/usr/sbin/groupadd", "--system", DEPLOYMENT_GROUP], acceptedExitCodes: [0, 9] },
6302
+ step("software strategy helper group", { kind: "commands", commands: [
6285
6303
  { argv: ["/usr/sbin/groupadd", "--system", SOFTWARE_HELPER_GROUP], acceptedExitCodes: [0, 9] }
6304
+ ] }),
6305
+ ...deploymentEnabled ? [step("deployment isolation group", { kind: "commands", commands: [
6306
+ { argv: ["/usr/sbin/groupadd", "--system", DEPLOYMENT_GROUP], acceptedExitCodes: [0, 9] }
6286
6307
  ] })] : [],
6287
6308
  ...lifecycleEnabled ? [step("lifecycle helper access group", { kind: "commands", commands: [{ argv: ["/usr/sbin/groupadd", "--system", LIFECYCLE_GROUP], acceptedExitCodes: [0, 9] }] })] : [],
6288
6309
  step("service account", { kind: "commands", commands: [{ argv: ["/usr/sbin/useradd", "--system", "--no-create-home", "--shell", "/usr/sbin/nologin", user], acceptedExitCodes: [0, 9] }] }),
6289
6310
  step("bind service account to vault group", { kind: "commands", commands: [{ argv: ["/usr/sbin/usermod", "-g", VAULT_GROUP, user] }] }),
6290
6311
  step("grant verified Agent update access", { kind: "commands", commands: [{ argv: ["/usr/sbin/usermod", "-a", "-G", AGENT_UPDATE_GROUP, user] }] }),
6312
+ step("grant software helper socket access", { kind: "commands", commands: [
6313
+ { argv: ["/usr/sbin/usermod", "-a", "-G", SOFTWARE_HELPER_GROUP, user] }
6314
+ ] }),
6291
6315
  ...lifecycleEnabled ? [step("grant lifecycle helper socket access", { kind: "commands", commands: [{ argv: ["/usr/sbin/usermod", "-a", "-G", LIFECYCLE_GROUP, user] }] })] : [],
6292
6316
  ...deploymentEnabled ? [step("credential-free deployment account", { kind: "commands", commands: [
6293
6317
  { argv: ["/usr/sbin/useradd", "--system", "--no-create-home", "--shell", "/usr/sbin/nologin", "--gid", DEPLOYMENT_GROUP, DEPLOYMENT_RUNNER_USER], acceptedExitCodes: [0, 9] },
6294
6318
  { argv: ["/usr/sbin/useradd", "--system", "--no-create-home", "--shell", "/usr/sbin/nologin", "--gid", VAULT_GROUP, APPLICATION_RUNTIME_USER], acceptedExitCodes: [0, 9] },
6295
6319
  { argv: ["/usr/sbin/usermod", "-a", "-G", DEPLOYMENT_GROUP, APPLICATION_RUNTIME_USER] },
6296
- { argv: ["/usr/sbin/usermod", "-a", "-G", `${DEPLOYMENT_GROUP},${SOFTWARE_HELPER_GROUP}`, user] }
6320
+ { argv: ["/usr/sbin/usermod", "-a", "-G", DEPLOYMENT_GROUP, user] }
6297
6321
  ] })] : [],
6298
6322
  step("credential and state directories", { kind: "directories", directories: [
6299
6323
  { path: credentialDir, mode: 448, owner: "root", group: "root" },
6300
- { path: "/var/lib/forgezero", mode: 488, owner: "root", group: "root" }
6324
+ { path: "/var/lib/forgezero", mode: 488, owner: "root", group: VAULT_GROUP }
6301
6325
  ] }),
6302
6326
  step("encrypted node identity", { kind: "ensure-seed", credential: seedCredentialPath }),
6303
6327
  ...options.generateGitIdentity && gitCredentialPath && gitPublicKeyPath ? [
@@ -6348,19 +6372,26 @@ function planProvision(options) {
6348
6372
  { argv: ["/usr/bin/systemctl", "enable", ...enabledUnits] },
6349
6373
  { argv: ["/usr/bin/systemctl", "reset-failed", "forgezero-agent.service"], acceptedExitCodes: [0, 1] },
6350
6374
  ...restartedUnits.length ? [{ argv: ["/usr/bin/systemctl", "restart", ...restartedUnits] }] : [],
6375
+ { argv: ["/usr/bin/systemctl", "stop", "forgezero-agent-proxy.service"], acceptedExitCodes: [0, 1, 5] },
6351
6376
  { argv: ["/usr/bin/systemctl", "restart", "forgezero-agent.socket"] },
6352
6377
  { argv: ["/usr/bin/systemctl", "reset-failed", "forgezero-agent.service"], acceptedExitCodes: [0, 1] },
6353
6378
  { argv: ["/usr/bin/systemctl", "restart", "forgezero-agent.service"] }
6354
6379
  ] }),
6355
6380
  ...enrolmentEnabled ? [step("prove the compute binding is durable", { kind: "verify-file", path: enrolStatePath })] : [],
6356
- ...options.enforceEgress ? [step("prove the Agent egress policy is active", { kind: "verify-egress", runnerPublicTcpPorts, runnerLoopbackPorts })] : [],
6381
+ ...options.enforceEgress ? [step("prove the Agent egress policy is active", {
6382
+ kind: "verify-egress",
6383
+ deploymentEnabled,
6384
+ runnerPublicTcpPorts,
6385
+ runnerLoopbackPorts,
6386
+ agentLoopbackPorts
6387
+ })] : [],
6357
6388
  step("prove it is running", { kind: "commands", commands: [{ argv: ["/usr/bin/systemctl", "is-active", "forgezero-agent.service"] }] }),
6358
6389
  step("prove the public Vault socket exists", { kind: "wait-socket", path: options.socketPath, attempts: 100, intervalMs: 100 }),
6359
6390
  step("prove the Agent Vault backend exists", { kind: "wait-socket", path: agentBackendSocketPath(options.socketPath), attempts: 100, intervalMs: 100 }),
6360
6391
  step("prove the Agent update helper exists", { kind: "wait-socket", path: DEFAULT_AGENT_UPDATE_SOCKET, attempts: 100, intervalMs: 100 }),
6392
+ step("prove the software strategy helper socket exists", { kind: "wait-socket", path: DEFAULT_SOFTWARE_HELPER_SOCKET, attempts: 100, intervalMs: 100 }),
6361
6393
  ...deploymentEnabled ? [
6362
- step("prove the deployment runner socket exists", { kind: "wait-socket", path: DEPLOYMENT_RUNNER_SOCKET, attempts: 100, intervalMs: 100 }),
6363
- step("prove the software strategy helper socket exists", { kind: "wait-socket", path: DEFAULT_SOFTWARE_HELPER_SOCKET, attempts: 100, intervalMs: 100 })
6394
+ step("prove the deployment runner socket exists", { kind: "wait-socket", path: DEPLOYMENT_RUNNER_SOCKET, attempts: 100, intervalMs: 100 })
6364
6395
  ] : [],
6365
6396
  ...lifecycleEnabled ? [step("prove the lifecycle helper socket exists", { kind: "wait-socket", path: lifecycleHelperSocketPath, attempts: 100, intervalMs: 100 })] : [],
6366
6397
  ...warpEnabled ? [step("prove Cloudflare WARP is connected", { kind: "verify-warp" })] : [],
@@ -6385,6 +6416,25 @@ import {
6385
6416
  writeFileSync as writeFileSync3
6386
6417
  } from "fs";
6387
6418
  import { dirname as dirname3 } from "path";
6419
+ function normalizeEd25519PublicKey(output) {
6420
+ const line = output.trim();
6421
+ if (!/^ssh-ed25519 [A-Za-z0-9+/]+={0,3}(?: [^\r\n]{1,256})?$/.test(line))
6422
+ return;
6423
+ const [algorithm, encoded] = line.split(" ", 3);
6424
+ if (algorithm !== "ssh-ed25519" || !encoded)
6425
+ return;
6426
+ let blob;
6427
+ try {
6428
+ blob = Buffer.from(encoded, "base64");
6429
+ } catch {
6430
+ return;
6431
+ }
6432
+ if (blob.length !== 51 || blob.readUInt32BE(0) !== 11 || blob.subarray(4, 15).toString("ascii") !== "ssh-ed25519" || blob.readUInt32BE(15) !== 32)
6433
+ return;
6434
+ if (blob.toString("base64") !== encoded)
6435
+ return;
6436
+ return `${algorithm} ${encoded}`;
6437
+ }
6388
6438
  function parseAssignments(value) {
6389
6439
  if (!value?.trim())
6390
6440
  return;
@@ -6557,7 +6607,7 @@ var runProvisionOperation = async (operation) => {
6557
6607
  return result;
6558
6608
  }
6559
6609
  result = await fixed(["/usr/bin/ssh-keygen", "-y", "-f", key]);
6560
- if (result.exitCode !== 0 || !/^ssh-ed25519 [A-Za-z0-9+/]+={0,3}\s*$/.test(result.stdout)) {
6610
+ if (result.exitCode !== 0 || !normalizeEd25519PublicKey(result.stdout)) {
6561
6611
  return { stdout: "bootstrap SSH private key is not a valid Ed25519 OpenSSH key", exitCode: 1 };
6562
6612
  }
6563
6613
  result = await fixed(["/usr/bin/systemd-creds", "encrypt", "--name=bootstrap-ssh-key", key, operation.credential]);
@@ -6573,11 +6623,12 @@ var runProvisionOperation = async (operation) => {
6573
6623
  chmodSync(key, 384);
6574
6624
  }
6575
6625
  const derived = await fixed(["/usr/bin/ssh-keygen", "-y", "-f", key]);
6576
- if (derived.exitCode !== 0 || !/^ssh-ed25519 [A-Za-z0-9+/]+={0,3}\s*$/.test(derived.stdout)) {
6626
+ const publicKey = normalizeEd25519PublicKey(derived.stdout);
6627
+ if (derived.exitCode !== 0 || !publicKey) {
6577
6628
  return { stdout: "bootstrap SSH public key derivation failed", exitCode: 1 };
6578
6629
  }
6579
6630
  mkdirSync3(dirname3(operation.publicKey), { recursive: true, mode: 493 });
6580
- writeFileSync3(operation.publicKey, `${derived.stdout.trim()} forgezero-bootstrap-runner
6631
+ writeFileSync3(operation.publicKey, `${publicKey} forgezero-bootstrap-runner
6581
6632
  `, { mode: 292 });
6582
6633
  chmodSync(operation.publicKey, 292);
6583
6634
  }
@@ -6620,8 +6671,9 @@ var runProvisionOperation = async (operation) => {
6620
6671
  const policy = await fixed(["/usr/sbin/nft", "--numeric", "list", "table", "inet", "forgezero_agent_egress"]);
6621
6672
  const required = [
6622
6673
  "forgezero-agent-egress-v1",
6623
- `public-tcp=${operation.runnerPublicTcpPorts.join(",")}`,
6624
- ...operation.runnerLoopbackPorts.length ? [`:${operation.runnerLoopbackPorts.join(",")}`] : []
6674
+ ...operation.agentLoopbackPorts.length ? [`shared-loopback=${operation.agentLoopbackPorts.join(",")}`] : [],
6675
+ ...operation.deploymentEnabled && operation.runnerPublicTcpPorts.length ? [`public-tcp=${operation.runnerPublicTcpPorts.join(",")}`] : [],
6676
+ ...operation.deploymentEnabled && operation.runnerLoopbackPorts.length ? [`:${operation.runnerLoopbackPorts.join(",")}`] : []
6625
6677
  ];
6626
6678
  return policy.exitCode === 0 && required.every((part) => policy.stdout.includes(part)) ? { stdout: policy.stdout, exitCode: 0 } : { stdout: policy.stdout, exitCode: 1 };
6627
6679
  }
@@ -6697,7 +6749,9 @@ async function applyPlan(plan, run) {
6697
6749
  const result = await run(step2.operation);
6698
6750
  transcript.push({ label: step2.label, command: step2.command, exitCode: result.exitCode });
6699
6751
  if (result.exitCode !== 0 && !step2.optional) {
6700
- throw new Error(`${step2.label} failed (exit ${result.exitCode}): ${step2.command}`);
6752
+ const safeDetail = step2.operation.kind === "ensure-bootstrap-ssh-identity" ? result.stdout.trim().slice(0, 1000) : "";
6753
+ throw new Error(`${step2.label} failed (exit ${result.exitCode}): ${step2.command}${safeDetail ? `
6754
+ ${safeDetail}` : ""}`);
6701
6755
  }
6702
6756
  }
6703
6757
  return transcript;
@@ -12509,12 +12563,26 @@ map $limit_conn_status $forgezero_retry_after { default ''; REJECTED 1; REJECTED
12509
12563
  server {
12510
12564
  listen 127.0.0.1:${input.publicPort};
12511
12565
  server_name _;
12566
+ server_tokens off;
12567
+ more_clear_headers Server X-Powered-By;
12512
12568
  limit_conn forgezero_admission ${concurrencyLimit};
12513
12569
  limit_conn_status 503;
12514
12570
  add_header Retry-After $forgezero_retry_after always;
12515
- location ^~ /api/ { proxy_pass http://forgezero; proxy_http_version 1.1; proxy_set_header Host $host; proxy_set_header X-Forwarded-Proto https; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection $forgezero_connection; proxy_read_timeout 3600s; }
12516
- location ^~ /v1/ { proxy_pass http://forgezero; proxy_http_version 1.1; proxy_set_header Host $host; proxy_set_header X-Forwarded-Proto https; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection $forgezero_connection; proxy_read_timeout 3600s; }
12517
- location / { return 404; }
12571
+ error_page 400 403 404 405 408 413 414 429 500 502 503 504 = @transport_failure;
12572
+ location @transport_failure { internal; return 444; }
12573
+ location / {
12574
+ proxy_pass http://forgezero;
12575
+ proxy_http_version 1.1;
12576
+ proxy_intercept_errors off;
12577
+ proxy_hide_header Server;
12578
+ proxy_hide_header X-Powered-By;
12579
+ proxy_set_header Host $host;
12580
+ proxy_set_header X-Forwarded-Proto https;
12581
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
12582
+ proxy_set_header Upgrade $http_upgrade;
12583
+ proxy_set_header Connection $forgezero_connection;
12584
+ proxy_read_timeout 3600s;
12585
+ }
12518
12586
  }
12519
12587
  `
12520
12588
  };
@@ -14362,7 +14430,7 @@ Type=oneshot
14362
14430
  User=arangodb
14363
14431
  Group=arangodb
14364
14432
  LoadCredentialEncrypted=arangodb-jwt:${JWT_CREDENTIAL}
14365
- ExecStart=/usr/bin/arangosh --server.endpoint tcp://${unitEscape(address)}:8529 --server.jwt-secret-keyfile %d/arangodb-jwt --javascript.execute-string 'const c=require("@arangodb").db._connection;let last;for(let i=0;i<90;i++){try{const v=c.GET("/_api/version?details=true");const s=c.GET("/_admin/status");const m=c.GET("/_admin/server/mode");if(!v.error&&v.version==="3.11.14"&&v.details&&v.details.license==="community"&&!s.error&&s.serverInfo&&s.serverInfo.role==="COORDINATOR"&&!m.error&&m.mode==="default")quit(0);last={v,s,m};}catch(e){last=String(e);}require("internal").wait(2);}throw new Error("writable Community Coordinator verification failed: "+JSON.stringify(last));'
14433
+ ExecStart=/usr/bin/arangosh --server.endpoint tcp://${unitEscape(address)}:8529 --server.jwt-secret-keyfile %d/arangodb-jwt --javascript.execute-string 'const c=require("@arangodb").db._connection;let last;let ok=false;for(let i=0;i<90;i++){try{const v=c.GET("/_api/version?details=true");const s=c.GET("/_admin/status");const m=c.GET("/_admin/server/mode");if(!v.error&&v.version==="3.11.14"&&v.details&&v.details.license==="community"&&!s.error&&s.serverInfo&&s.serverInfo.role==="COORDINATOR"&&!m.error&&m.mode==="default"){ok=true;break;}last={v,s,m};}catch(e){last=String(e);}require("internal").wait(2);}if(!ok)throw new Error("writable Community Coordinator verification failed: "+JSON.stringify(last));'
14366
14434
  RemainAfterExit=yes
14367
14435
  TimeoutStartSec=200
14368
14436
  NoNewPrivileges=true
@@ -14488,11 +14556,11 @@ async function seal(host, name, destination2, value) {
14488
14556
  async function verifyBootstrapBundleOnHost(host, config, verifyGit = false) {
14489
14557
  const manifestMetadata = host.inspect?.(config.bootstrapBundle.manifestFile);
14490
14558
  const bundleMetadata = host.inspect?.(config.bootstrapBundle.bundleFile);
14491
- if (manifestMetadata && (!manifestMetadata.regular || manifestMetadata.symbolic || manifestMetadata.uid !== 0 || manifestMetadata.links !== 1 || (manifestMetadata.mode & 63) !== 0 || manifestMetadata.size > 16 * 1024)) {
14492
- throw new Error("bootstrap bundle manifest must be a root-owned owner-only regular file");
14559
+ if (manifestMetadata && (!manifestMetadata.regular || manifestMetadata.symbolic || manifestMetadata.uid !== 0 || manifestMetadata.links !== 1 || (manifestMetadata.mode & 18) !== 0 || manifestMetadata.size > 16 * 1024)) {
14560
+ throw new Error("bootstrap bundle manifest must be a root-owned non-writable regular file");
14493
14561
  }
14494
- if (bundleMetadata && (!bundleMetadata.regular || bundleMetadata.symbolic || bundleMetadata.uid !== 0 || bundleMetadata.links !== 1 || (bundleMetadata.mode & 63) !== 0 || bundleMetadata.size < 1 || bundleMetadata.size > 512 * 1024 * 1024)) {
14495
- throw new Error("bootstrap bundle must be a root-owned owner-only regular file within 512 MiB");
14562
+ if (bundleMetadata && (!bundleMetadata.regular || bundleMetadata.symbolic || bundleMetadata.uid !== 0 || bundleMetadata.links !== 1 || (bundleMetadata.mode & 18) !== 0 || bundleMetadata.size < 1 || bundleMetadata.size > 512 * 1024 * 1024)) {
14563
+ throw new Error("bootstrap bundle must be a root-owned non-writable regular file within 512 MiB");
14496
14564
  }
14497
14565
  if (!host.exists(config.bootstrapBundle.bundleFile) || !host.exists(config.bootstrapBundle.manifestFile)) {
14498
14566
  throw new Error("attended bootstrap bundle and manifest are required for release generation one");
@@ -14511,11 +14579,25 @@ async function verifyBootstrapBundleOnHost(host, config, verifyGit = false) {
14511
14579
  if (digest !== manifest.sha256)
14512
14580
  throw new Error("bootstrap bundle digest does not match its manifest");
14513
14581
  if (verifyGit) {
14514
- await checked3(host, ["/usr/bin/git", "bundle", "verify", config.bootstrapBundle.bundleFile], "bootstrap Git bundle verification");
14582
+ const verificationRepository = "/run/forgezero-bootstrap-bundle-verify";
14583
+ await checked3(host, ["/usr/bin/rm", "-rf", "--", verificationRepository], "stale bootstrap bundle verifier cleanup");
14584
+ await checked3(host, ["/usr/bin/git", "init", "--bare", "--quiet", verificationRepository], "bootstrap bundle verifier repository");
14585
+ try {
14586
+ await checked3(host, [
14587
+ "/usr/bin/git",
14588
+ "-C",
14589
+ verificationRepository,
14590
+ "bundle",
14591
+ "verify",
14592
+ config.bootstrapBundle.bundleFile
14593
+ ], "bootstrap Git bundle verification");
14594
+ } finally {
14595
+ await checked3(host, ["/usr/bin/rm", "-rf", "--", verificationRepository], "bootstrap bundle verifier cleanup");
14596
+ }
14515
14597
  }
14516
14598
  return manifest;
14517
14599
  }
14518
- function bootstrapIdentity(config) {
14600
+ function bootstrapIdentity(config, includeDeploymentEvidence = false) {
14519
14601
  if (config.kind === "enrolled-compute")
14520
14602
  return {
14521
14603
  kind: config.kind,
@@ -14534,7 +14616,13 @@ function bootstrapIdentity(config) {
14534
14616
  bootstrapRunner: config.bootstrapRunner ? { targetTelemetryEndpoint: config.bootstrapRunner.targetTelemetryEndpoint } : null
14535
14617
  };
14536
14618
  const environment = config.runtime.environment;
14537
- const { cloudflare: _cloudflare, realtime: _realtime, ...stableEnvironment } = environment;
14619
+ const { cloudflare: _cloudflare, realtime: _realtime, initialInventory, ...stableEnvironmentRest } = environment;
14620
+ const stableEnvironment = {
14621
+ ...stableEnvironmentRest,
14622
+ ...initialInventory ? {
14623
+ initialInventory: { ...initialInventory, deployment: includeDeploymentEvidence ? initialInventory.deployment : undefined }
14624
+ } : {}
14625
+ };
14538
14626
  return {
14539
14627
  kind: config.kind,
14540
14628
  environment: config.environment,
@@ -14575,6 +14663,26 @@ function bootstrapIdentity(config) {
14575
14663
  function bootstrapIdentityDigest(config) {
14576
14664
  return createHash4("sha256").update(JSON.stringify(bootstrapIdentity(config))).digest("hex");
14577
14665
  }
14666
+ var legacyBootstrapIdentityDigest = (config) => createHash4("sha256").update(JSON.stringify(bootstrapIdentity(config, true))).digest("hex");
14667
+ function interruptedPlatformReleaseIdentityDigests(config, releaseEvidence, bootstrapManifest) {
14668
+ if (!config.runtime.environment.initialInventory)
14669
+ return [];
14670
+ const previous = structuredClone(config);
14671
+ previous.runtime.environment.initialInventory.deployment = {
14672
+ source: "bootstrap-bundle",
14673
+ branch: releaseEvidence.branch,
14674
+ revision: releaseEvidence.revision,
14675
+ bundleSha256: releaseEvidence.sha256
14676
+ };
14677
+ const expectedCurrentEpoch = bootstrapManifest ? `${config.environment}-${bootstrapManifest.revision.slice(0, 16)}` : undefined;
14678
+ if (expectedCurrentEpoch && config.runtime.environment.seedSyncEpoch === expectedCurrentEpoch) {
14679
+ previous.runtime.environment.seedSyncEpoch = `${config.environment}-${releaseEvidence.revision.slice(0, 16)}`;
14680
+ }
14681
+ return [bootstrapIdentityDigest(previous), legacyBootstrapIdentityDigest(previous)];
14682
+ }
14683
+ function storedPlatformCoordinatesMatch(state, config) {
14684
+ return state.kind === "platform" && state.profile === config.profile && state.nodeHostname === config.nodeHostname && state.apiUrl === config.apiUrl && state.environment === config.environment && state.databaseRole === config.database.role && state.databaseAgency === config.database.agency && state.databaseReadPreferred === config.database.readPreferred && state.databaseServerMode === config.database.serverMode && (state.databaseAddress ?? null) === (config.database.address ?? null) && JSON.stringify(state.databaseCoordinators ?? []) === JSON.stringify(config.database.coordinators) && state.collectorUnit === config.runtime.environment.otlpCollectorUnit && state.publicApiPort === config.runtime.environment.publicApiPort && state.healthPath === config.runtime.healthPath;
14685
+ }
14578
14686
  function parseStoredState(raw) {
14579
14687
  let value;
14580
14688
  try {
@@ -14605,11 +14713,21 @@ function parseStoredIntent(raw) {
14605
14713
  }
14606
14714
  return intent;
14607
14715
  }
14608
- function bindBootstrapIntent(host, config) {
14716
+ function bindBootstrapIntent(host, config, previousReleaseDigests = []) {
14609
14717
  const identityDigest = bootstrapIdentityDigest(config);
14610
14718
  if (host.exists(INTENT_PATH)) {
14611
14719
  const intent = parseStoredIntent(host.read(INTENT_PATH));
14612
14720
  if (intent.kind !== config.kind || intent.identityDigest !== identityDigest) {
14721
+ if (previousReleaseDigests.includes(intent.identityDigest) && intent.kind === "platform" && config.kind === "platform") {
14722
+ host.write(INTENT_PATH, `${JSON.stringify({
14723
+ format: 1,
14724
+ kind: config.kind,
14725
+ identityDigest,
14726
+ createdAt: new Date().toISOString()
14727
+ }, null, 2)}
14728
+ `, 384);
14729
+ return identityDigest;
14730
+ }
14613
14731
  throw new Error("bootstrap resume coordinates do not match the interrupted host intent");
14614
14732
  }
14615
14733
  } else if (!host.exists(STATE_PATH)) {
@@ -14684,7 +14802,7 @@ async function bootstrapStatus(host = localBootstrapHost()) {
14684
14802
  if (result.exitCode !== 0)
14685
14803
  problems.push(`${unit} is not active`);
14686
14804
  }
14687
- for (const socket of [DEFAULT_SOCKET, CONTROL_SOCKET]) {
14805
+ for (const socket of [DEFAULT_SOCKET]) {
14688
14806
  services[socket] = host.exists(socket);
14689
14807
  if (!services[socket])
14690
14808
  problems.push(`${socket} is missing`);
@@ -14777,7 +14895,42 @@ async function applyBootstrap(input, host = localBootstrapHost(), secrets, depen
14777
14895
  let installed;
14778
14896
  if (host.exists(STATE_PATH))
14779
14897
  installed = parseStoredState(host.read(STATE_PATH));
14780
- const bootstrapManifest = config.kind === "platform" && !host.exists(BOOTSTRAP_RELEASE_EVIDENCE) ? await verifyBootstrapBundleOnHost(host, config) : undefined;
14898
+ let bootstrapManifest;
14899
+ let releaseEvidence;
14900
+ if (config.kind === "platform") {
14901
+ const staged = host.exists(config.bootstrapBundle.bundleFile) || host.exists(config.bootstrapBundle.manifestFile);
14902
+ if (staged && !(host.exists(config.bootstrapBundle.bundleFile) && host.exists(config.bootstrapBundle.manifestFile))) {
14903
+ throw new Error("bootstrap bundle and manifest must be staged together");
14904
+ }
14905
+ const reviewed = staged ? await verifyBootstrapBundleOnHost(host, config) : undefined;
14906
+ if (!host.exists(BOOTSTRAP_RELEASE_EVIDENCE)) {
14907
+ if (!reviewed)
14908
+ throw new Error("attended bootstrap bundle and manifest are required for release generation one");
14909
+ bootstrapManifest = reviewed;
14910
+ } else if (reviewed) {
14911
+ let evidence;
14912
+ try {
14913
+ evidence = JSON.parse(host.read(BOOTSTRAP_RELEASE_EVIDENCE));
14914
+ } catch {
14915
+ throw new Error("bootstrap release evidence is malformed");
14916
+ }
14917
+ if (typeof evidence.revision !== "string" || !/^[a-f0-9]{40}$/.test(evidence.revision) || typeof evidence.sha256 !== "string" || !/^[a-f0-9]{64}$/.test(evidence.sha256) || typeof evidence.branch !== "string")
14918
+ throw new Error("bootstrap release evidence is malformed");
14919
+ releaseEvidence = { revision: evidence.revision, sha256: evidence.sha256, branch: evidence.branch };
14920
+ if (evidence.revision !== reviewed.revision || evidence.sha256 !== reviewed.sha256 || evidence.branch !== reviewed.branch)
14921
+ bootstrapManifest = reviewed;
14922
+ } else if (host.exists(BOOTSTRAP_RELEASE_EVIDENCE)) {
14923
+ let evidence;
14924
+ try {
14925
+ evidence = JSON.parse(host.read(BOOTSTRAP_RELEASE_EVIDENCE));
14926
+ } catch {
14927
+ throw new Error("bootstrap release evidence is malformed");
14928
+ }
14929
+ if (typeof evidence.revision !== "string" || !/^[a-f0-9]{40}$/.test(evidence.revision) || typeof evidence.sha256 !== "string" || !/^[a-f0-9]{64}$/.test(evidence.sha256) || typeof evidence.branch !== "string")
14930
+ throw new Error("bootstrap release evidence is malformed");
14931
+ releaseEvidence = { revision: evidence.revision, sha256: evidence.sha256, branch: evidence.branch };
14932
+ }
14933
+ }
14781
14934
  let cloudflare;
14782
14935
  if (config.cloudflareHandoff) {
14783
14936
  if (host.exists(config.cloudflareHandoff.handoffFile)) {
@@ -14827,10 +14980,14 @@ async function applyBootstrap(input, host = localBootstrapHost(), secrets, depen
14827
14980
  throw new Error("platform realtime coordinates require a realtime-enabled Cloudflare handoff");
14828
14981
  }
14829
14982
  }
14983
+ let allowInterruptedReleaseRebind = false;
14830
14984
  if (installed) {
14831
- if (installed.kind !== config.kind || installed.identityDigest !== bootstrapIdentityDigest(config)) {
14985
+ const exactIdentity = installed.kind === config.kind && installed.identityDigest === bootstrapIdentityDigest(config);
14986
+ const boundedReleaseResume = config.kind === "platform" && !host.exists("/var/lib/forgezero/enrolment.json") && host.exists(BOOTSTRAP_RELEASE_EVIDENCE) && storedPlatformCoordinatesMatch(installed, config);
14987
+ if (!exactIdentity && !boundedReleaseResume) {
14832
14988
  throw new Error("bootstrap repair coordinates do not match the installed host identity");
14833
14989
  }
14990
+ allowInterruptedReleaseRebind = !exactIdentity && boundedReleaseResume;
14834
14991
  }
14835
14992
  const platformPrivate = config.kind === "platform" ? (() => {
14836
14993
  if (!secrets)
@@ -14850,10 +15007,14 @@ async function applyBootstrap(input, host = localBootstrapHost(), secrets, depen
14850
15007
  if (config.kind === "enrolled-compute" && config.bootstrapRunner?.sshPrivateKeyFile && !host.exists(BOOTSTRAP_SSH_CREDENTIAL)) {
14851
15008
  privateFile(host, config.bootstrapRunner.sshPrivateKeyFile, "bootstrap runner SSH private key");
14852
15009
  }
14853
- bindBootstrapIntent(host, config);
15010
+ let previousReleaseDigests = [];
15011
+ if (!installed && config.kind === "platform" && releaseEvidence && !host.exists("/var/lib/forgezero/enrolment.json")) {
15012
+ previousReleaseDigests = [...interruptedPlatformReleaseIdentityDigests(config, releaseEvidence, bootstrapManifest)];
15013
+ }
15014
+ bindBootstrapIntent(host, config, allowInterruptedReleaseRebind && installed?.identityDigest ? [installed.identityDigest] : previousReleaseDigests);
14854
15015
  const plan = planBootstrap(config, host.exists(STATE_PATH));
14855
15016
  await checked3(host, ["hostnamectl", "set-hostname", "--static", config.nodeHostname], "compute hostname");
14856
- const alreadyEnrolled = host.exists("/var/lib/forgezero/enrolment.json");
15017
+ let alreadyEnrolled = host.exists("/var/lib/forgezero/enrolment.json");
14857
15018
  host.mkdir(CREDS, 448);
14858
15019
  host.mkdir("/var/lib/forgezero", 448);
14859
15020
  if (!alreadyEnrolled && !host.exists(ENROL_CREDENTIAL)) {
@@ -14882,14 +15043,29 @@ async function applyBootstrap(input, host = localBootstrapHost(), secrets, depen
14882
15043
  if (connectorCapabilities?.meshConnectorToken && !host.exists(WARP_CONNECTOR_CREDENTIAL)) {
14883
15044
  await seal(host, "CF_WARP_CONNECTOR_TOKEN", WARP_CONNECTOR_CREDENTIAL, connectorCapabilities.meshConnectorToken);
14884
15045
  }
15046
+ let installedAgentPlan;
14885
15047
  if (alreadyEnrolled) {
14886
- await host.installAgent(config, "bound");
15048
+ installedAgentPlan = await host.installAgent(config, bootstrapManifest ? "bootstrap" : "bound");
14887
15049
  } else if (config.kind === "platform") {
14888
15050
  if (bootstrapManifest)
14889
- await host.installAgent(config, "bootstrap");
15051
+ installedAgentPlan = await host.installAgent(config, "bootstrap");
15052
+ else if (host.exists(BOOTSTRAP_RELEASE_EVIDENCE)) {
15053
+ await checked3(host, [
15054
+ "curl",
15055
+ "--fail",
15056
+ "--silent",
15057
+ "--show-error",
15058
+ "--max-time",
15059
+ "10",
15060
+ `http://127.0.0.1:${config.runtime.environment.publicApiPort}${config.runtime.healthPath}`
15061
+ ], "persisted release-one API health");
15062
+ await host.installAgent(config, "enrol");
15063
+ installedAgentPlan = await host.installAgent(config, "bound");
15064
+ alreadyEnrolled = true;
15065
+ }
14890
15066
  } else {
14891
15067
  await host.installAgent(config, "enrol");
14892
- await host.installAgent(config, "bound");
15068
+ installedAgentPlan = await host.installAgent(config, "bound");
14893
15069
  }
14894
15070
  if (config.kind === "platform" && config.firewall.enabled) {
14895
15071
  await host.ensureSoftware(plan.software.filter(({ id: id2 }) => id2 === "ufw"));
@@ -14969,6 +15145,7 @@ async function applyBootstrap(input, host = localBootstrapHost(), secrets, depen
14969
15145
  await checked3(host, ["useradd", "--system", "--no-create-home", "--shell", "/usr/sbin/nologin", runtime.serviceUser], "API service account").catch(async () => {
14970
15146
  await checked3(host, ["id", runtime.serviceUser], "existing API service account");
14971
15147
  });
15148
+ await checked3(host, ["usermod", "-a", "-G", DEPLOYMENT_GROUP, runtime.serviceUser], "API deployment group");
14972
15149
  host.mkdir(runtime.environment.sharedDirectory, 488);
14973
15150
  host.mkdir(runtime.slotsDirectory, 493);
14974
15151
  host.write(envPath, renderPlatformSharedEnvironment(runtime.environment), 416);
@@ -14987,7 +15164,7 @@ async function applyBootstrap(input, host = localBootstrapHost(), secrets, depen
14987
15164
  host.write("/etc/sudoers.d/forgezero-runner", activation.sudoers, 288);
14988
15165
  await checked3(host, ["visudo", "-cf", "/etc/sudoers.d/forgezero-runner"], "activation sudo policy");
14989
15166
  const telemetry = planLocalOtlpProof(runtime.environment.otlpEndpoint, runtime.environment.otlpCollectorUnit);
14990
- await checked3(host, telemetry.unitCheck.argv, "OTLP collector supervision");
15167
+ await checked3(host, [telemetry.unitCheck.command, ...telemetry.unitCheck.argv], "OTLP collector supervision");
14991
15168
  const otlpStatus = (await checked3(host, [telemetry.receiverCheck.command, ...telemetry.receiverCheck.argv], "OTLP receiver")).trim();
14992
15169
  if (!/^2\d\d$/.test(otlpStatus))
14993
15170
  throw new Error(`OTLP receiver returned HTTP ${otlpStatus || "unknown"}`);
@@ -15020,10 +15197,12 @@ async function applyBootstrap(input, host = localBootstrapHost(), secrets, depen
15020
15197
  `, 384);
15021
15198
  }
15022
15199
  if (bootstrapManifest) {
15200
+ if (!installedAgentPlan?.user)
15201
+ throw new Error("initial Agent deployment requires the converged Agent user");
15023
15202
  await checked3(host, [
15024
15203
  "runuser",
15025
15204
  "-u",
15026
- "forgezero-agent",
15205
+ installedAgentPlan.user,
15027
15206
  "--",
15028
15207
  "/usr/local/bin/fz-agent",
15029
15208
  "deploy",
@@ -15041,6 +15220,8 @@ async function applyBootstrap(input, host = localBootstrapHost(), secrets, depen
15041
15220
  `, 384);
15042
15221
  host.remove(config.bootstrapBundle.bundleFile);
15043
15222
  host.remove(config.bootstrapBundle.manifestFile);
15223
+ if (alreadyEnrolled)
15224
+ installedAgentPlan = await host.installAgent(config, "bound");
15044
15225
  }
15045
15226
  if (!alreadyEnrolled) {
15046
15227
  await checked3(host, [
@@ -15050,7 +15231,7 @@ async function applyBootstrap(input, host = localBootstrapHost(), secrets, depen
15050
15231
  "--show-error",
15051
15232
  "--max-time",
15052
15233
  "10",
15053
- `http://127.0.0.1:3000${runtime.healthPath}`
15234
+ `http://127.0.0.1:${runtime.environment.publicApiPort}${runtime.healthPath}`
15054
15235
  ], "release-one API health");
15055
15236
  await host.installAgent(config, "enrol");
15056
15237
  await host.installAgent(config, "bound");
@@ -15252,8 +15433,8 @@ function readBootstrapConfig(path) {
15252
15433
  function planBootstrapAgentInstall(config, phase, context) {
15253
15434
  const { capabilities, hasBinding, hasEnrolCredential, initialBundle } = context;
15254
15435
  if (phase === "bootstrap") {
15255
- if (config.kind !== "platform" || hasBinding || !initialBundle) {
15256
- throw new Error("release-one Agent install requires an unbound platform host and verified bootstrap bundle");
15436
+ if (config.kind !== "platform" || !initialBundle) {
15437
+ throw new Error("platform bundle Agent install requires a verified bootstrap bundle");
15257
15438
  }
15258
15439
  } else if (phase === "enrol") {
15259
15440
  if (hasBinding || !hasEnrolCredential) {
@@ -15291,8 +15472,10 @@ function planBootstrapAgentInstall(config, phase, context) {
15291
15472
  bootstrapTargetTelemetryEndpoint: bound && bootstrapRunner ? platformBootstrapRunner(config) ? config.telemetryEndpoint : config.kind === "enrolled-compute" ? config.bootstrapRunner?.targetTelemetryEndpoint : undefined : undefined,
15292
15473
  lifecycleProfilePath: bound && config.kind === "platform" ? LIFECYCLE_PROFILE : undefined,
15293
15474
  enforceEgress: true,
15475
+ runnerLoopbackPorts: config.kind === "platform" ? [config.runtime.environment.publicApiPort, config.runtime.bluePort, config.runtime.greenPort] : [],
15476
+ agentLoopbackPorts: config.kind === "platform" ? [config.runtime.environment.publicApiPort] : [],
15294
15477
  nodeHostname: config.nodeHostname,
15295
- telemetryEndpoint: config.telemetryEndpoint,
15478
+ telemetryEndpoint: config.kind === "platform" ? config.runtime.environment.otlpEndpoint : "http://127.0.0.1:4318",
15296
15479
  binPath: "/usr/local/lib/forgezero/agent/fz-agent",
15297
15480
  sourceBinPath: PACKAGED_AGENT_BIN,
15298
15481
  ...enrolling ? {
@@ -15300,7 +15483,7 @@ function planBootstrapAgentInstall(config, phase, context) {
15300
15483
  enrolStatePath: "/var/lib/forgezero/enrolment.json"
15301
15484
  } : bound ? { enrolStatePath: "/var/lib/forgezero/enrolment.json" } : {},
15302
15485
  ...authenticated ? {
15303
- apiUrl: config.apiUrl,
15486
+ apiUrl: config.kind === "platform" ? `http://127.0.0.1:${config.runtime.environment.publicApiPort}` : config.apiUrl,
15304
15487
  project: config.kind === "enrolled-compute" ? config.realm : "platform",
15305
15488
  environment: config.kind === "enrolled-compute" ? undefined : config.environment,
15306
15489
  nodeLabel: config.kind === "enrolled-compute" ? config.nodeHostname : config.computeReference
@@ -15309,6 +15492,7 @@ function planBootstrapAgentInstall(config, phase, context) {
15309
15492
  return planInstall(options);
15310
15493
  }
15311
15494
  function localBootstrapHost() {
15495
+ let softwareClientUser;
15312
15496
  const execute = async (argv2, options = {}) => {
15313
15497
  const child = Bun.spawn([...argv2], { stdin: options.stdin === undefined ? "ignore" : "pipe", stdout: "pipe", stderr: "pipe" });
15314
15498
  if (options.stdin !== undefined && child.stdin && typeof child.stdin !== "number") {
@@ -15341,10 +15525,12 @@ function localBootstrapHost() {
15341
15525
  exec: execute,
15342
15526
  sleep: (milliseconds) => Bun.sleep(milliseconds),
15343
15527
  async ensureSoftware(requirements) {
15528
+ if (!softwareClientUser)
15529
+ throw new Error("Agent must be converged before software requirements are applied");
15344
15530
  const result = await execute([
15345
15531
  "runuser",
15346
15532
  "-u",
15347
- "forgezero-agent",
15533
+ softwareClientUser,
15348
15534
  "--",
15349
15535
  "/usr/local/bin/fz-agent",
15350
15536
  "software-ensure",
@@ -15358,7 +15544,7 @@ function localBootstrapHost() {
15358
15544
  const capabilities = await readCapabilities(localRunner);
15359
15545
  const hasBinding = existsSync9("/var/lib/forgezero/enrolment.json");
15360
15546
  const hasEnrolCredential = existsSync9(ENROL_CREDENTIAL);
15361
- const initialBundle = config.kind === "platform" && !existsSync9(BOOTSTRAP_RELEASE_EVIDENCE) ? {
15547
+ const initialBundle = config.kind === "platform" && phase === "bootstrap" && existsSync9(config.bootstrapBundle.bundleFile) && existsSync9(config.bootstrapBundle.manifestFile) ? {
15362
15548
  path: config.bootstrapBundle.bundleFile,
15363
15549
  manifestPath: config.bootstrapBundle.manifestFile,
15364
15550
  manifest: parseBootstrapBundleManifest(JSON.parse(readFileSync9(config.bootstrapBundle.manifestFile, "utf8")))
@@ -15389,6 +15575,7 @@ function localBootstrapHost() {
15389
15575
  writeFileSync9(unit.path, unit.unit, { mode: 420 });
15390
15576
  }
15391
15577
  await applyPlan(plan, localRunner);
15578
+ softwareClientUser = plan.user;
15392
15579
  return plan;
15393
15580
  }
15394
15581
  };
@@ -15658,7 +15845,7 @@ async function runCloudflareBootstrapFinalizeCommand(configPath, dependencies =
15658
15845
 
15659
15846
  // src/operator-bootstrap.ts
15660
15847
  import { createHash as createHash6, randomBytes as randomBytes9 } from "crypto";
15661
- import { chmodSync as chmodSync6, lstatSync as lstatSync7, mkdirSync as mkdirSync12, mkdtempSync as mkdtempSync2, readFileSync as readFileSync12, rmSync as rmSync7, writeFileSync as writeFileSync12 } from "fs";
15848
+ import { chmodSync as chmodSync6, existsSync as existsSync11, lstatSync as lstatSync7, mkdirSync as mkdirSync12, mkdtempSync as mkdtempSync2, readFileSync as readFileSync12, rmSync as rmSync7, writeFileSync as writeFileSync12 } from "fs";
15662
15849
  import { isIP as isIP6 } from "net";
15663
15850
  import { tmpdir as tmpdir2 } from "os";
15664
15851
  import { basename as basename2, dirname as dirname13, isAbsolute as isAbsolute5, join as join11, resolve as resolve9 } from "path";
@@ -16630,6 +16817,21 @@ async function metalBootstrapStatus(exec = defaultExec2) {
16630
16817
  }
16631
16818
 
16632
16819
  // src/operator-bootstrap.ts
16820
+ function operatorPackagedBinary(name) {
16821
+ const candidates = [
16822
+ fileURLToPath3(new URL(`./${name}`, import.meta.url)),
16823
+ fileURLToPath3(new URL(`../dist/${name}`, import.meta.url))
16824
+ ];
16825
+ const found = candidates.find(existsSync11);
16826
+ if (!found)
16827
+ throw new Error(`packaged ${name} is missing; checked ${candidates.join(" and ")}`);
16828
+ return found;
16829
+ }
16830
+ var operatorPackagedFzCliPath = () => operatorPackagedBinary("fz.js");
16831
+ var operatorPackagedAgentPath = () => operatorPackagedBinary("fz-agent.js");
16832
+ var operatorPackagedGitSshPath = () => operatorPackagedBinary("fz-git-ssh.js");
16833
+ var PLATFORM_CLUSTER_CREDENTIAL_NAME = "forgezero-platform-cluster-bootstrap-code";
16834
+ var PLATFORM_CLUSTER_CREDENTIAL_PATH = `/etc/forgezero/creds/${PLATFORM_CLUSTER_CREDENTIAL_NAME}.cred`;
16633
16835
  var REQUEST_LIMIT = 64 * 1024;
16634
16836
  var SECRET_LIMIT = 256 * 1024;
16635
16837
  var REMOTE_STAGE = "/run/forgezero-operator-bootstrap";
@@ -17072,9 +17274,11 @@ function operatorPlatformFleetCoordinates(fleet) {
17072
17274
  function planOperatorPlatformFleetBootstrap(fleet, mode) {
17073
17275
  const { requests, configs, environment } = validatedPlatformFleet(fleet);
17074
17276
  const secretInputs = mode === "apply" ? attendedSecretNames(configs[0]) : [];
17075
- for (const config of configs.slice(1)) {
17076
- if (JSON.stringify(attendedSecretNames(config)) !== JSON.stringify(secretInputs)) {
17077
- throw new Error("operator fleet configs do not share one attended secret schema");
17277
+ if (mode === "apply") {
17278
+ for (const config of configs.slice(1)) {
17279
+ if (JSON.stringify(attendedSecretNames(config)) !== JSON.stringify(secretInputs)) {
17280
+ throw new Error("operator fleet configs do not share one attended secret schema");
17281
+ }
17078
17282
  }
17079
17283
  }
17080
17284
  return {
@@ -17112,9 +17316,33 @@ var defaultExec3 = async (argv2, options = {}) => {
17112
17316
  ]);
17113
17317
  return {
17114
17318
  exitCode,
17115
- output: options.secret ? options.safeStdout && exitCode === 0 ? stdout.slice(0, 65536) : "" : `${stdout}${stderr}`.slice(0, 65536)
17319
+ output: options.secret ? exitCode === 0 ? options.safeStdout ? stdout.slice(0, 65536) : "" : redactOperatorSecretDiagnostic(stderr, options.stdin) : `${stdout}${stderr}`.slice(0, 65536)
17116
17320
  };
17117
17321
  };
17322
+ function redactOperatorSecretDiagnostic(stderr, stdin) {
17323
+ if (!stdin)
17324
+ return "";
17325
+ let parsed;
17326
+ try {
17327
+ parsed = JSON.parse(stdin);
17328
+ } catch {
17329
+ return "";
17330
+ }
17331
+ const values = [];
17332
+ const visit = (value) => {
17333
+ if (typeof value === "string" && value)
17334
+ values.push(value);
17335
+ else if (Array.isArray(value))
17336
+ value.forEach(visit);
17337
+ else if (value && typeof value === "object")
17338
+ Object.values(value).forEach(visit);
17339
+ };
17340
+ visit(parsed);
17341
+ let safe = stderr.slice(0, 16384);
17342
+ for (const value of values.sort((left, right) => right.length - left.length))
17343
+ safe = safe.replaceAll(value, "[REDACTED]");
17344
+ return safe.slice(0, 8192);
17345
+ }
17118
17346
  var checked5 = async (exec, argv2, label, options) => {
17119
17347
  const result = await exec(argv2, options);
17120
17348
  if (result.exitCode !== 0)
@@ -17157,7 +17385,7 @@ var sshOptions = (request, knownHosts, scp = false) => [
17157
17385
  "-o",
17158
17386
  "ClearAllForwardings=yes",
17159
17387
  "-o",
17160
- "ConnectTimeout=10",
17388
+ "ConnectTimeout=30",
17161
17389
  "-o",
17162
17390
  request.target.jump ? `ProxyJump=${destination2(request.target.jump)}:${request.target.jump.port}` : "ProxyJump=none",
17163
17391
  scp ? "-P" : "-p",
@@ -17192,6 +17420,93 @@ var remoteRegularFileExists = async (exec, request, knownHosts, path) => {
17192
17420
  return false;
17193
17421
  throw new Error(`remote initialized-state probe failed${result.output ? `: ${result.output.trim()}` : ""}`);
17194
17422
  };
17423
+ async function operatorMetalClusterBootstrapCode(requestInput, options = {}) {
17424
+ const request = validateMetalRequestValue(requestInput, { validateMetalConfig: false });
17425
+ publicIdentity(request.target.identityPublicKeyFile);
17426
+ socketPath(request.target.agentSocket);
17427
+ const exec = options.exec ?? defaultExec3;
17428
+ const directory = mkdtempSync2(join11(tmpdir2(), "forgezero-operator-metal-credential-"));
17429
+ try {
17430
+ const knownHosts = writeKnownHosts(request, directory);
17431
+ const probe = await exec([
17432
+ "ssh",
17433
+ ...sshOptions(request, knownHosts),
17434
+ destination2(request.target),
17435
+ "--",
17436
+ "/usr/bin/sudo",
17437
+ "-n",
17438
+ "/usr/bin/test",
17439
+ "-f",
17440
+ PLATFORM_CLUSTER_CREDENTIAL_PATH
17441
+ ]);
17442
+ if (probe.exitCode !== 0 && probe.exitCode !== 1) {
17443
+ throw new Error("Metal cluster credential probe failed");
17444
+ }
17445
+ if (probe.exitCode === 1) {
17446
+ const value2 = randomBytes9(32).toString("hex");
17447
+ await remote(exec, request, knownHosts, [
17448
+ "/usr/bin/sudo",
17449
+ "-n",
17450
+ "/usr/bin/install",
17451
+ "-d",
17452
+ "-o",
17453
+ "root",
17454
+ "-g",
17455
+ "root",
17456
+ "-m",
17457
+ "0700",
17458
+ dirname13(PLATFORM_CLUSTER_CREDENTIAL_PATH)
17459
+ ], "Metal credential directory");
17460
+ await remote(exec, request, knownHosts, [
17461
+ "/usr/bin/sudo",
17462
+ "-n",
17463
+ "/usr/bin/systemd-creds",
17464
+ "encrypt",
17465
+ `--name=${PLATFORM_CLUSTER_CREDENTIAL_NAME}`,
17466
+ "-",
17467
+ PLATFORM_CLUSTER_CREDENTIAL_PATH
17468
+ ], "seal Metal cluster credential", true, `${value2}
17469
+ `);
17470
+ await remote(exec, request, knownHosts, [
17471
+ "/usr/bin/sudo",
17472
+ "-n",
17473
+ "/usr/bin/chown",
17474
+ "root:root",
17475
+ PLATFORM_CLUSTER_CREDENTIAL_PATH
17476
+ ], "own Metal cluster credential");
17477
+ await remote(exec, request, knownHosts, [
17478
+ "/usr/bin/sudo",
17479
+ "-n",
17480
+ "/usr/bin/chmod",
17481
+ "0600",
17482
+ PLATFORM_CLUSTER_CREDENTIAL_PATH
17483
+ ], "protect Metal cluster credential");
17484
+ }
17485
+ const metadata = await remote(exec, request, knownHosts, [
17486
+ "/usr/bin/sudo",
17487
+ "-n",
17488
+ "/usr/bin/stat",
17489
+ "--format=%u:%g:%a:%h",
17490
+ PLATFORM_CLUSTER_CREDENTIAL_PATH
17491
+ ], "verify Metal cluster credential");
17492
+ if (metadata !== "0:0:600:1")
17493
+ throw new Error("Metal cluster credential is not one root-owned 0600 file");
17494
+ const value = await remote(exec, request, knownHosts, [
17495
+ "/usr/bin/sudo",
17496
+ "-n",
17497
+ "/usr/bin/systemd-creds",
17498
+ "decrypt",
17499
+ `--name=${PLATFORM_CLUSTER_CREDENTIAL_NAME}`,
17500
+ PLATFORM_CLUSTER_CREDENTIAL_PATH,
17501
+ "-"
17502
+ ], "open Metal cluster credential", true, undefined, true);
17503
+ if (!/^[a-f0-9]{64}$/i.test(value))
17504
+ throw new Error("Metal cluster credential is invalid");
17505
+ return value;
17506
+ } finally {
17507
+ rmSync7(directory, { recursive: true, force: true });
17508
+ }
17509
+ }
17195
17510
  var copy = async (exec, request, knownHosts, local, remotePath, secret = false) => {
17196
17511
  safeRemoteArg(remotePath);
17197
17512
  await checked5(exec, [
@@ -17308,9 +17623,9 @@ async function verifiedBunArchive(directory, fetcher) {
17308
17623
  }
17309
17624
  async function installPackagedAgent(request, knownHosts, exec, directory, remoteTemp, options) {
17310
17625
  const artifacts = [
17311
- [options.fzCliPath ?? fileURLToPath3(new URL("./fz.js", import.meta.url)), "fz.js"],
17312
- [options.fzAgentPath ?? fileURLToPath3(new URL("./fz-agent.js", import.meta.url)), "fz-agent.js"],
17313
- [options.fzGitSshPath ?? fileURLToPath3(new URL("./fz-git-ssh.js", import.meta.url)), "fz-git-ssh.js"]
17626
+ [options.fzCliPath ?? operatorPackagedFzCliPath(), "fz.js"],
17627
+ [options.fzAgentPath ?? operatorPackagedAgentPath(), "fz-agent.js"],
17628
+ [options.fzGitSshPath ?? operatorPackagedGitSshPath(), "fz-git-ssh.js"]
17314
17629
  ];
17315
17630
  for (const [artifact] of artifacts) {
17316
17631
  if (!readFileSync12(artifact).length)
@@ -17347,6 +17662,7 @@ async function installPackagedAgent(request, knownHosts, exec, directory, remote
17347
17662
  for (const [, name] of artifacts)
17348
17663
  await remote(exec, request, knownHosts, ["/usr/bin/sudo", "-n", "/usr/bin/install", "-m", "0755", `${remoteTemp}/${name}`, `/usr/local/lib/forgezero/agent/${name}`], "remote Agent install");
17349
17664
  await remote(exec, request, knownHosts, ["/usr/bin/sudo", "-n", "/usr/bin/ln", "-sfn", "/usr/local/lib/forgezero/agent/fz.js", "/usr/local/bin/fz"], "remote fz link");
17665
+ await remote(exec, request, knownHosts, ["/usr/bin/sudo", "-n", "/usr/bin/ln", "-sfn", "/usr/local/lib/forgezero/agent/fz-agent", "/usr/local/bin/fz-agent"], "remote fz-agent link");
17350
17666
  }
17351
17667
  async function applyOperatorPlatformBootstrap(request, mode, options = {}) {
17352
17668
  const plan = planOperatorPlatformBootstrap(request, mode);
@@ -17373,8 +17689,10 @@ async function applyOperatorPlatformBootstrap(request, mode, options = {}) {
17373
17689
  await copy(exec, request, knownHosts, join11(directory, name), `${remoteTemp}/${name}`, true);
17374
17690
  for (const bundle of staged.bundleFiles)
17375
17691
  await copy(exec, request, knownHosts, bundle.source, `${remoteTemp}/${bundle.name}`, true);
17376
- for (const name of ["platform-config.json", ...staged.files, ...staged.bundleFiles.map(({ name: name2 }) => name2)])
17692
+ for (const name of ["platform-config.json", ...staged.files])
17377
17693
  await remote(exec, request, knownHosts, ["/usr/bin/sudo", "-n", "/usr/bin/install", "-m", "0600", `${remoteTemp}/${name}`, `${REMOTE_STAGE}/${name}`], "remote bootstrap handoff", true);
17694
+ for (const { name } of staged.bundleFiles)
17695
+ await remote(exec, request, knownHosts, ["/usr/bin/sudo", "-n", "/usr/bin/install", "-m", "0644", `${remoteTemp}/${name}`, `${REMOTE_STAGE}/${name}`], "remote bootstrap bundle", true);
17378
17696
  const command = ["/usr/bin/sudo", "-n", "/usr/local/bin/fz", "bootstrap", "platform", "credentials-stdin"];
17379
17697
  command.push("--bootstrap-config", `${REMOTE_STAGE}/platform-config.json`, "--apply");
17380
17698
  const output = await remote(exec, request, knownHosts, command, "remote typed bootstrap", mode === "apply", secrets ? `${JSON.stringify(secrets)}
@@ -17596,9 +17914,10 @@ var PROFILES = {
17596
17914
  appOrigin: "https://dev.forgezero.net",
17597
17915
  apiOrigin: "https://dev-api.forgezero.net",
17598
17916
  zoneName: "forgezero.net",
17599
- kvNamespaceTitle: "forgezero-dev-nodes",
17917
+ kvNamespaceTitle: "forgezero dev nodes",
17600
17918
  workerScriptName: "forgezero-api-edge-development",
17601
17919
  region: { key: "in-south", label: "India South", country: "IN" },
17920
+ seedSyncEpoch: "development-23c19168773d69ea",
17602
17921
  agentOtlpEndpoint: "https://otel.forgezero.net",
17603
17922
  privateCidrs: ["10.42.0.0/24"]
17604
17923
  },
@@ -17608,9 +17927,10 @@ var PROFILES = {
17608
17927
  appOrigin: "https://www.forgezero.net",
17609
17928
  apiOrigin: "https://api.forgezero.net",
17610
17929
  zoneName: "forgezero.net",
17611
- kvNamespaceTitle: "forgezero-nodes",
17930
+ kvNamespaceTitle: "forgezero nodes",
17612
17931
  workerScriptName: "forgezero-api-edge-production",
17613
17932
  region: { key: "in-south", label: "India South", country: "IN" },
17933
+ seedSyncEpoch: "production-genesis-v1",
17614
17934
  agentOtlpEndpoint: "https://otel.forgezero.net",
17615
17935
  privateCidrs: ["10.42.0.0/24"]
17616
17936
  }
@@ -17678,7 +17998,7 @@ function forgeZeroLaunchTemplate(profile, guests, bundle, owner) {
17678
17998
  sharedDirectory: "/opt/forgezero/shared",
17679
17999
  seedSyncPeers: fleet.slice(1).flatMap(({ address }) => [3001, 3002].map((port) => `ws://${address}:${port}/_internal/forgezero/seed-mesh/v2`)),
17680
18000
  seedSyncMembers: nodeHostnames,
17681
- seedSyncEpoch: `${profile.environment}-${bundle.manifest.revision.slice(0, 16)}`,
18001
+ seedSyncEpoch: profile.seedSyncEpoch,
17682
18002
  concurrencyLimit: 128,
17683
18003
  drainDeadlineMs: 30000,
17684
18004
  otlpEndpoint: "http://127.0.0.1:4318",
@@ -17707,7 +18027,7 @@ function forgeZeroLaunchTemplate(profile, guests, bundle, owner) {
17707
18027
  // src/host-maintenance.ts
17708
18028
  import { lstatSync as lstatSync8, readFileSync as readFileSync13 } from "fs";
17709
18029
  var ROOT = "/opt/forgezero";
17710
- var SLOT_FILE = `${ROOT}/.forge-slot`;
18030
+ var SLOT_FILE = `${ROOT}/slots/.active`;
17711
18031
  var SHARED_ENV = `${ROOT}/shared/.env`;
17712
18032
  var JWT = "/etc/forgezero/creds/arangodb-jwt.cred";
17713
18033
  var FZ = "/usr/local/bin/fz";
@@ -17836,7 +18156,7 @@ async function applyHostMaintenance(request, runtime = localRuntime()) {
17836
18156
  }
17837
18157
 
17838
18158
  // src/cli/maintenance.ts
17839
- import { existsSync as existsSync11, lstatSync as lstatSync9, readFileSync as readFileSync14, realpathSync as realpathSync5 } from "fs";
18159
+ import { existsSync as existsSync12, lstatSync as lstatSync9, readFileSync as readFileSync14, realpathSync as realpathSync5 } from "fs";
17840
18160
  import { isAbsolute as isAbsolute6, join as join12, relative as relative2, resolve as resolve10 } from "path";
17841
18161
  var API_OPERATION_ENTRYPOINTS = {
17842
18162
  "dev-reset": ["src", "server", "maintenance", "dev-reset.ts"],
@@ -17907,7 +18227,7 @@ function unsupportedRepositoryCliOption(argv2) {
17907
18227
  }
17908
18228
  function manifestName(root) {
17909
18229
  const manifestPath = join12(root, "package.json");
17910
- if (!existsSync11(manifestPath)) {
18230
+ if (!existsSync12(manifestPath)) {
17911
18231
  throw new Error(`No package.json exists at repository root ${root}.`);
17912
18232
  }
17913
18233
  let parsed;
@@ -17923,7 +18243,7 @@ function manifestName(root) {
17923
18243
  }
17924
18244
  function checkedEntrypoint(root, parts) {
17925
18245
  const candidate = join12(root, ...parts);
17926
- if (!existsSync11(candidate) || !lstatSync9(candidate).isFile()) {
18246
+ if (!existsSync12(candidate) || !lstatSync9(candidate).isFile()) {
17927
18247
  throw new Error(`The reviewed operation entrypoint is missing: ${candidate}`);
17928
18248
  }
17929
18249
  if (lstatSync9(candidate).isSymbolicLink()) {
@@ -18046,7 +18366,7 @@ function planAppBuild(input) {
18046
18366
 
18047
18367
  // src/cli/index.ts
18048
18368
  var DEFAULT_MODE = (THRESHOLD_MODES.find((mode) => mode.threshold === 1 && mode.total === 1) ?? THRESHOLD_MODES[0]).id;
18049
- var PACKAGED_AGENT_BIN2 = fileURLToPath4(new URL("./fz-agent.js", import.meta.url));
18369
+ var PACKAGED_AGENT_BIN2 = operatorPackagedAgentPath();
18050
18370
  function parseOptions(argv2) {
18051
18371
  const options = {
18052
18372
  api: process.env.FZ_API ?? "http://localhost:8787",
@@ -18792,7 +19112,7 @@ async function cmdAgent(options, args) {
18792
19112
  out.ok(`Wrote ${auxiliary.path}`);
18793
19113
  }
18794
19114
  if (options.enrol) {
18795
- if (!existsSync12(enrolStatePath) && !existsSync12(enrolTokenCredentialPath)) {
19115
+ if (!existsSync13(enrolStatePath) && !existsSync13(enrolTokenCredentialPath)) {
18796
19116
  const token = await bootstrapSecret("ForgeZero one-time enrolment token");
18797
19117
  if (!/^fze_[A-Za-z0-9_-]{40,100}$/.test(token))
18798
19118
  throw new Error("A valid fze_ enrolment token was not provided.");
@@ -18811,7 +19131,7 @@ async function cmdAgent(options, args) {
18811
19131
  for (const step3 of transcript)
18812
19132
  out.ok(step3.label);
18813
19133
  out.ok("Agent service and socket verified");
18814
- if (gitIdentity.gitPublicKeyPath && existsSync12(gitIdentity.gitPublicKeyPath)) {
19134
+ if (gitIdentity.gitPublicKeyPath && existsSync13(gitIdentity.gitPublicKeyPath)) {
18815
19135
  out.line();
18816
19136
  out.line(" Compatibility SSH deploy public key:");
18817
19137
  out.line();
@@ -18855,7 +19175,6 @@ async function bootstrapSecret(question) {
18855
19175
  throw new Error(`${question} was not provided`);
18856
19176
  return value;
18857
19177
  }
18858
- var PLATFORM_CLUSTER_CREDENTIAL = "forgezero-platform-cluster-bootstrap-code";
18859
19178
  function readPlatformLaunchEnvironment(path) {
18860
19179
  const absolute2 = resolve12(path);
18861
19180
  const stat2 = lstatSync10(absolute2);
@@ -18915,37 +19234,6 @@ function readPlatformLaunchEnvironment(path) {
18915
19234
  cloudflareApiToken: requiredValue("CF_API_TOKEN")
18916
19235
  };
18917
19236
  }
18918
- async function platformLaunchClusterBootstrapCode(outputDirectory) {
18919
- const path = join13(dirname14(outputDirectory), `${PLATFORM_CLUSTER_CREDENTIAL}.cred`);
18920
- if (existsSync12(path)) {
18921
- const stat2 = lstatSync10(path);
18922
- if (!stat2.isFile() || stat2.isSymbolicLink() || stat2.nlink !== 1 || typeof process.getuid === "function" && stat2.uid !== process.getuid() || (stat2.mode & 63) !== 0) {
18923
- throw new Error(`platform cluster credential must be one owner-only regular file: ${path}`);
18924
- }
18925
- const child2 = Bun.spawn(["systemd-creds", "decrypt", `--name=${PLATFORM_CLUSTER_CREDENTIAL}`, path, "-"], { stdin: "ignore", stdout: "pipe", stderr: "pipe" });
18926
- const [stdout, stderr2, exitCode2] = await Promise.all([
18927
- new Response(child2.stdout).text(),
18928
- new Response(child2.stderr).text(),
18929
- child2.exited
18930
- ]);
18931
- if (exitCode2 !== 0)
18932
- throw new Error(`cannot open the retained platform cluster credential: ${stderr2.trim()}`);
18933
- const value2 = stdout.trim();
18934
- if (!/^[a-f0-9]{64}$/i.test(value2))
18935
- throw new Error("retained platform cluster credential is invalid");
18936
- return value2;
18937
- }
18938
- mkdirSync13(dirname14(path), { recursive: true, mode: 448 });
18939
- const value = randomBytes10(32).toString("hex");
18940
- const child = Bun.spawn(["systemd-creds", "encrypt", `--name=${PLATFORM_CLUSTER_CREDENTIAL}`, "-", path], { stdin: "pipe", stdout: "pipe", stderr: "pipe" });
18941
- await writeAndCloseProcessInput(child.stdin, `${value}
18942
- `);
18943
- const [stderr, exitCode] = await Promise.all([new Response(child.stderr).text(), child.exited]);
18944
- if (exitCode !== 0)
18945
- throw new Error(`cannot retain the platform cluster credential: ${stderr.trim()}`);
18946
- chmodSync7(path, 384);
18947
- return value;
18948
- }
18949
19237
  async function promptPlatformBootstrapSecrets(config, generatedClusterBootstrapCode) {
18950
19238
  return validatePlatformBootstrapSecrets(config, {
18951
19239
  clusterBootstrapCode: generatedClusterBootstrapCode ?? await bootstrapSecret("Shared 64-hex cluster bootstrap code"),
@@ -19082,7 +19370,7 @@ async function interactiveCloudflareBootstrap() {
19082
19370
  function genesisOutputDirectory(path) {
19083
19371
  if (!isAbsolute7(path) || resolve12(path) !== path)
19084
19372
  throw new Error("--output must be a canonical absolute directory");
19085
- if (!existsSync12(path))
19373
+ if (!existsSync13(path))
19086
19374
  mkdirSync13(path, { recursive: true, mode: 448 });
19087
19375
  const metadata = lstatSync10(path);
19088
19376
  const uid = process.getuid?.();
@@ -19260,14 +19548,14 @@ async function prepareForgeZeroPlatformLaunch(environment, outputDirectory, proj
19260
19548
  [metalRequestFile, "reviewed Metal request"],
19261
19549
  [guestHostKeysFile, "pinned genesis guest host-key evidence"]
19262
19550
  ]) {
19263
- if (!existsSync12(path))
19551
+ if (!existsSync13(path))
19264
19552
  throw new Error(`${label} is missing at ${path}`);
19265
19553
  }
19266
19554
  const root = resolve12(projectRoot);
19267
19555
  const nestedApi = join13(root, "api");
19268
- const repositoryRoot = existsSync12(join13(nestedApi, "package.json")) ? nestedApi : root;
19556
+ const repositoryRoot = existsSync13(join13(nestedApi, "package.json")) ? nestedApi : root;
19269
19557
  const bundlePath = join13(base, "api.bundle");
19270
- const bundle = existsSync12(bundlePath) || existsSync12(`${bundlePath}.json`) ? await readBootstrapBundle(bundlePath) : await buildBootstrapBundle({ repositoryRoot, outputPath: bundlePath, branch: profile.branch });
19558
+ const bundle = existsSync13(bundlePath) || existsSync13(`${bundlePath}.json`) ? await readBootstrapBundle(bundlePath) : await buildBootstrapBundle({ repositoryRoot, outputPath: bundlePath, branch: profile.branch });
19271
19559
  if (bundle.manifest.branch !== profile.branch) {
19272
19560
  throw new Error(`existing bootstrap bundle is for ${bundle.manifest.branch}, expected ${profile.branch}`);
19273
19561
  }
@@ -19476,14 +19764,17 @@ async function cmdBootstrap(options, args) {
19476
19764
  if (environment === "production" && options.bootstrapSecretsEnvFile) {
19477
19765
  throw new Error("production platform launch requires interactive hidden secret prompts");
19478
19766
  }
19479
- const outputDirectory = options.outputPath ?? `/secure/forgezero/${environment}/genesis`;
19767
+ const outputDirectory = options.outputPath ?? join13(homedir2(), ".forgezero-secure", environment, "genesis");
19480
19768
  const environmentInput = options.bootstrapSecretsEnvFile ? readPlatformLaunchEnvironment(options.bootstrapSecretsEnvFile) : undefined;
19481
- const generated = await prepareForgeZeroPlatformLaunch(environment, outputDirectory, options.projectRoot, environmentInput?.owner);
19482
- const fleet = readOperatorPlatformBootstrapFleetRequest(generated.fleetRequest);
19769
+ const existingFleetPath = join13(outputDirectory, "platform-fleet-remote.json");
19770
+ const fleet = existsSync13(existingFleetPath) ? readOperatorPlatformBootstrapFleetRequest(existingFleetPath) : readOperatorPlatformBootstrapFleetRequest((await prepareForgeZeroPlatformLaunch(environment, outputDirectory, options.projectRoot, environmentInput?.owner)).fleetRequest);
19771
+ if (planOperatorPlatformFleetBootstrap(fleet, "apply").environment !== environment) {
19772
+ throw new Error("existing platform fleet does not match the selected launch profile");
19773
+ }
19483
19774
  if (options.apply && !environmentInput && bootstrapAnswer("Type APPLY to run the reviewed Cloudflare and three-compute plan") !== "APPLY") {
19484
19775
  throw new Error("attended platform bootstrap was not confirmed");
19485
19776
  }
19486
- return await runAttendedPlatformFleet(fleet, options.apply, options.apply ? await platformLaunchClusterBootstrapCode(outputDirectory) : undefined, environmentInput);
19777
+ return await runAttendedPlatformFleet(fleet, options.apply, options.apply ? await operatorMetalClusterBootstrapCode(readOperatorMetalBootstrapRequest(join13(dirname14(outputDirectory), "metal", "metal-remote.json"), { validateMetalConfig: false })) : undefined, environmentInput);
19487
19778
  }
19488
19779
  if (operation === "platform" && args[1] === "fleet") {
19489
19780
  const mode = args[2];
@@ -19606,8 +19897,8 @@ async function cmdBootstrap(options, args) {
19606
19897
  if (credentialStdin && args[2] !== undefined)
19607
19898
  throw new Error("platform credential stdin accepts no additional positional arguments");
19608
19899
  const installedBootstrapKind = () => resolveInstalledBootstrapKind({
19609
- metal: existsSync12(METAL_BOOTSTRAP_STATE_PATH),
19610
- compute: existsSync12(BOOTSTRAP_STATE_PATH)
19900
+ metal: existsSync13(METAL_BOOTSTRAP_STATE_PATH),
19901
+ compute: existsSync13(BOOTSTRAP_STATE_PATH)
19611
19902
  });
19612
19903
  if (operation === "status") {
19613
19904
  if (installedBootstrapKind() === "metal") {
@@ -19978,7 +20269,7 @@ function projectFromCheckout(options) {
19978
20269
  try {
19979
20270
  return loadConfig({
19980
20271
  cwd: options.projectRoot,
19981
- readFile: (path) => existsSync12(path) ? readFileSync16(path, "utf8") : undefined
20272
+ readFile: (path) => existsSync13(path) ? readFileSync16(path, "utf8") : undefined
19982
20273
  }).config.project;
19983
20274
  } catch (cause) {
19984
20275
  throw new Error(`No --project was given and the checkout has no usable .fz/config.json: ${cause instanceof Error ? cause.message : String(cause)}`);
@@ -20100,7 +20391,7 @@ async function cmdDeploy(options, args) {
20100
20391
  return problems.length === 0 ? 0 : 1;
20101
20392
  }
20102
20393
  if (operation === "check" || operation === "sync") {
20103
- if (existsSync12(resolve12(options.projectRoot, DEPLOY_SOURCE_FILE))) {
20394
+ if (existsSync13(resolve12(options.projectRoot, DEPLOY_SOURCE_FILE))) {
20104
20395
  const expected = await compileDeploymentProject(options.projectRoot, { write: false });
20105
20396
  const actual = inspectCompiledDeployment(options.projectRoot);
20106
20397
  const current2 = canonicalJson(expected.plan) === canonicalJson(actual.plan);