@forgezero/agent 0.1.159 → 0.1.161

package/dist/operator-bootstrap.js

@@ -1460,7 +1460,7 @@ var UPDATE_RETRY_BASE_MS = 5 * 60000;
 var UPDATE_RETRY_MAX_MS = 24 * 60 * 60000;
 
 // src/version.ts
-var VERSION = "0.1.159";
+var VERSION = "0.1.161";
 
 // src/software.ts
 var PINNED_BUN_VERSION = "1.3.14";
@@ -3551,12 +3551,23 @@ function renderPlatformActivationFiles(input) {
     throw new Error("Activation root must be absolute and single-line.");
   if (!/^[a-z_][a-z0-9_-]{0,31}$/.test(input.serviceUser))
     throw new Error("Invalid activation service user.");
+  boundedInteger("publicPort", input.publicPort, 1024, 65535);
   boundedInteger("bluePort", input.bluePort, 1024, 65535);
   boundedInteger("greenPort", input.greenPort, 1024, 65535);
-  if (input.bluePort === input.greenPort)
-    throw new Error("Activation slot ports must differ.");
-  if (!/^[A-Za-z0-9.-]{1,253}$/.test(input.seedMeshAddress) || input.seedMeshAddress.startsWith(".") || input.seedMeshAddress.endsWith(".")) {
-    throw new Error("Activation seed mesh address is malformed.");
+  if (new Set([input.publicPort, input.bluePort, input.greenPort]).size !== 3) {
+    throw new Error("Activation public and slot ports must differ.");
+  }
+  const runtimeHandoff = input.runtimeHandoff;
+  if (!/^[A-Za-z0-9.-]{1,253}$/.test(runtimeHandoff.address) || runtimeHandoff.address.startsWith(".") || runtimeHandoff.address.endsWith(".")) {
+    throw new Error("Runtime handoff address is malformed.");
+  }
+  for (const [name, path] of Object.entries({
+    residentPath: runtimeHandoff.residentPath,
+    readyPath: runtimeHandoff.readyPath
+  })) {
+    if (!/^\/[A-Za-z0-9/_.-]{1,192}$/.test(path) || path.includes("..")) {
+      throw new Error(`Runtime handoff ${name} is malformed.`);
+    }
   }
   boundedInteger("keepReleases", input.keepReleases, 2, 100);
   const drainDeadlineMs = boundedInteger("drainDeadlineMs", input.drainDeadlineMs ?? 35000, 1000, 300000);
@@ -3566,16 +3577,17 @@ function renderPlatformActivationFiles(input) {
   const environment = [
     `FZ_DIR=${input.root}`,
     `FZ_USER=${input.serviceUser}`,
+    `FZ_PUBLIC_PORT=${input.publicPort}`,
     `FZ_BLUE_PORT=${input.bluePort}`,
     `FZ_GREEN_PORT=${input.greenPort}`,
-    `FZ_SEED_MESH_ADDRESS=${input.seedMeshAddress}`,
+    `FZ_RUNTIME_HANDOFF_ADDRESS=${runtimeHandoff.address}`,
     `FZ_HEALTH_PATH=${input.healthPath}`,
     `FZ_KEEP_RELEASES=${input.keepReleases}`,
     `FZ_DRAIN_DEADLINE_MS=${drainDeadlineMs}`
   ].join(`
 `) + `
 `;
-  const helper = `${JSON.stringify({ ...input, drainDeadlineMs }, null, 2)}
+  const helper = `${JSON.stringify({ ...input, runtimeHandoff, drainDeadlineMs }, null, 2)}
 `;
   return {
     environment,
@@ -4456,7 +4468,7 @@ function interruptedPlatformReleaseIdentityDigests(config, releaseEvidence, boot
   return [bootstrapIdentityDigest(previous), legacyBootstrapIdentityDigest(previous)];
 }
 function storedPlatformCoordinatesMatch(state, config) {
-  return state.kind === "platform" && state.profile === config.profile && state.nodeHostname === config.nodeHostname && state.apiUrl === config.apiUrl && state.environment === config.environment && state.databaseRole === config.database.role && state.databaseAgency === config.database.agency && state.databaseReadPreferred === config.database.readPreferred && state.databaseServerMode === config.database.serverMode && (state.databaseAddress ?? null) === (config.database.address ?? null) && JSON.stringify(state.databaseCoordinators ?? []) === JSON.stringify(config.database.coordinators) && state.collectorUnit === config.runtime.environment.otlpCollectorUnit && state.publicApiPort === config.runtime.environment.publicApiPort && state.healthPath === config.runtime.healthPath;
+  return state.kind === "platform" && state.profile === config.profile && state.nodeHostname === config.nodeHostname && state.apiUrl === config.apiUrl && state.environment === config.environment && state.databaseRole === config.database.role && state.databaseAgency === config.database.agency && state.databaseReadPreferred === config.database.readPreferred && state.databaseServerMode === config.database.serverMode && (state.databaseAddress ?? null) === (config.database.address ?? null) && JSON.stringify(state.databaseCoordinators ?? []) === JSON.stringify(config.database.coordinators) && state.collectorUnit === config.runtime.environment.otlpCollectorUnit && state.publicApiPort === config.runtime.environment.publicApiPort && (state.bluePort === undefined || state.bluePort === config.runtime.bluePort) && (state.greenPort === undefined || state.greenPort === config.runtime.greenPort) && (state.slotsDirectory === undefined || state.slotsDirectory === config.runtime.slotsDirectory) && (state.seedMeshAddress === undefined || state.seedMeshAddress === (config.runtime.environment.databaseAddress ?? config.database.address)) && state.healthPath === config.runtime.healthPath;
 }
 function parseStoredState(raw) {
   let value;
@@ -4535,6 +4547,12 @@ function stateFor(config, cloudflare, previousCloudflareTunnelId) {
       databaseModeEvidence: config.database.role === "none" ? undefined : DB_MODE_EVIDENCE,
       collectorUnit: config.runtime.environment.otlpCollectorUnit,
       publicApiPort: config.runtime.environment.publicApiPort,
+      bluePort: config.runtime.bluePort,
+      greenPort: config.runtime.greenPort,
+      slotsDirectory: config.runtime.slotsDirectory,
+      seedMeshAddress: config.runtime.environment.databaseAddress ?? config.database.address,
+      runtimeHandoffResidentPath: "/_internal/forgezero/seed-mesh/resident",
+      runtimeHandoffReadyPath: "/_internal/forgezero/seed-mesh/ready",
       healthPath: config.runtime.healthPath,
       cloudflare: config.runtime.environment.cloudflare,
       realtime: config.runtime.environment.realtime,
@@ -4616,6 +4634,50 @@ async function bootstrapStatus(host = localBootstrapHost()) {
     services["forgezero@active.service"] = blue.exitCode === 0 || green.exitCode === 0;
     if (!services["forgezero@active.service"])
       problems.push("neither API slot is active");
+    const activeSlots = [
+      ...blue.exitCode === 0 ? ["blue"] : [],
+      ...green.exitCode === 0 ? ["green"] : []
+    ];
+    const activeSlotFile = state.slotsDirectory ? `${state.slotsDirectory}/.active` : "";
+    const recordedSlot = activeSlotFile && host.exists(activeSlotFile) ? host.read(activeSlotFile).trim() : "";
+    const coordinatesValid = Number.isSafeInteger(state.bluePort) && Number.isSafeInteger(state.greenPort) && Boolean(state.slotsDirectory) && Boolean(state.seedMeshAddress) && Boolean(state.healthPath) && Boolean(state.runtimeHandoffResidentPath) && Boolean(state.runtimeHandoffReadyPath);
+    services["platform-active-slot-consistent"] = coordinatesValid && activeSlots.length === 1 && (recordedSlot === "blue" || recordedSlot === "green") && activeSlots[0] === recordedSlot;
+    if (!services["platform-active-slot-consistent"]) {
+      problems.push("active slot marker, service state, or runtime coordinates disagree");
+    }
+    if (services["platform-active-slot-consistent"]) {
+      const slotPort = recordedSlot === "blue" ? state.bluePort : state.greenPort;
+      const directHealth = await host.exec([
+        "curl",
+        "--fail",
+        "--silent",
+        "--show-error",
+        "--max-time",
+        "5",
+        `http://127.0.0.1:${slotPort}${state.healthPath}`
+      ]);
+      services["platform-active-slot-health"] = directHealth.exitCode === 0;
+      if (directHealth.exitCode !== 0)
+        problems.push("active API slot health check failed");
+      const expectedUpstream = `upstream forgezero { server 127.0.0.1:${slotPort}; }
+`;
+      const upstream = "/etc/nginx/conf.d/forgezero-upstream.conf";
+      services["platform-nginx-active-slot"] = host.exists(upstream) && host.read(upstream) === expectedUpstream;
+      if (!services["platform-nginx-active-slot"])
+        problems.push("nginx upstream does not match the active API slot");
+      const resident = await host.exec([
+        "curl",
+        "--fail",
+        "--silent",
+        "--show-error",
+        "--max-time",
+        "5",
+        `http://${state.seedMeshAddress}:${slotPort}${state.runtimeHandoffResidentPath}`
+      ]);
+      services["platform-vault-resident"] = resident.exitCode === 0;
+      if (resident.exitCode !== 0)
+        problems.push("active API slot has no resident Vault seed");
+    }
     if (!Number.isSafeInteger(state.publicApiPort) || !state.healthPath) {
       problems.push("platform health coordinates are missing from bootstrap state");
     } else {
@@ -4947,9 +5009,14 @@ async function applyBootstrap(input, host = localBootstrapHost(), secrets, depen
     const activation = renderPlatformActivationFiles({
       root: config.deployRoot ?? "/opt/forgezero",
       serviceUser: runtime.serviceUser,
+      publicPort: runtime.environment.publicApiPort,
       bluePort: runtime.bluePort,
       greenPort: runtime.greenPort,
-      seedMeshAddress,
+      runtimeHandoff: {
+        address: seedMeshAddress,
+        residentPath: "/_internal/forgezero/seed-mesh/resident",
+        readyPath: "/_internal/forgezero/seed-mesh/ready"
+      },
       healthPath: runtime.healthPath,
       keepReleases: runtime.keepReleases,
       drainDeadlineMs: runtime.environment.drainDeadlineMs

package/dist/platform-bootstrap-runtime.d.ts

@@ -174,10 +174,19 @@ export declare function planPlatformActivation(releasesDirectory: string, releas
 export interface PlatformActivationConfig {
     root: string;
     serviceUser: string;
+    /** Stable loopback listener served by nginx/cloudflared. */
+    publicPort: number;
     bluePort: number;
     greenPort: number;
-    /** Private address used by the seed mesh; never exposed through nginx. */
-    seedMeshAddress: string;
+    /**
+     * Runtime-owned handoff acknowledgement endpoints. The Agent never reads or
+     * transfers runtime state; it only gates traffic switching on these probes.
+     */
+    runtimeHandoff: {
+        address: string;
+        residentPath: string;
+        readyPath: string;
+    };
     healthPath: string;
     keepReleases: number;
     drainDeadlineMs?: number;
@@ -202,6 +211,8 @@ export declare function activatePlatformRelease(config: PlatformActivationConfig
     exec?: PlatformActivationExec;
     fetch?: typeof fetch;
     sleep?: (ms: number) => Promise<void>;
+    /** Test seam; production always uses the root-owned nginx include. */
+    upstreamPath?: string;
 }): Promise<{
     release: string;
     slot: ApiSlot;

package/dist/platform-bootstrap-runtime.js

@@ -1164,12 +1164,23 @@ function renderPlatformActivationFiles(input) {
     throw new Error("Activation root must be absolute and single-line.");
   if (!/^[a-z_][a-z0-9_-]{0,31}$/.test(input.serviceUser))
     throw new Error("Invalid activation service user.");
+  boundedInteger("publicPort", input.publicPort, 1024, 65535);
   boundedInteger("bluePort", input.bluePort, 1024, 65535);
   boundedInteger("greenPort", input.greenPort, 1024, 65535);
-  if (input.bluePort === input.greenPort)
-    throw new Error("Activation slot ports must differ.");
-  if (!/^[A-Za-z0-9.-]{1,253}$/.test(input.seedMeshAddress) || input.seedMeshAddress.startsWith(".") || input.seedMeshAddress.endsWith(".")) {
-    throw new Error("Activation seed mesh address is malformed.");
+  if (new Set([input.publicPort, input.bluePort, input.greenPort]).size !== 3) {
+    throw new Error("Activation public and slot ports must differ.");
+  }
+  const runtimeHandoff = input.runtimeHandoff;
+  if (!/^[A-Za-z0-9.-]{1,253}$/.test(runtimeHandoff.address) || runtimeHandoff.address.startsWith(".") || runtimeHandoff.address.endsWith(".")) {
+    throw new Error("Runtime handoff address is malformed.");
+  }
+  for (const [name, path2] of Object.entries({
+    residentPath: runtimeHandoff.residentPath,
+    readyPath: runtimeHandoff.readyPath
+  })) {
+    if (!/^\/[A-Za-z0-9/_.-]{1,192}$/.test(path2) || path2.includes("..")) {
+      throw new Error(`Runtime handoff ${name} is malformed.`);
+    }
   }
   boundedInteger("keepReleases", input.keepReleases, 2, 100);
   const drainDeadlineMs = boundedInteger("drainDeadlineMs", input.drainDeadlineMs ?? 35000, 1000, 300000);
@@ -1179,16 +1190,17 @@ function renderPlatformActivationFiles(input) {
   const environment = [
     `FZ_DIR=${input.root}`,
     `FZ_USER=${input.serviceUser}`,
+    `FZ_PUBLIC_PORT=${input.publicPort}`,
     `FZ_BLUE_PORT=${input.bluePort}`,
     `FZ_GREEN_PORT=${input.greenPort}`,
-    `FZ_SEED_MESH_ADDRESS=${input.seedMeshAddress}`,
+    `FZ_RUNTIME_HANDOFF_ADDRESS=${runtimeHandoff.address}`,
     `FZ_HEALTH_PATH=${input.healthPath}`,
     `FZ_KEEP_RELEASES=${input.keepReleases}`,
     `FZ_DRAIN_DEADLINE_MS=${drainDeadlineMs}`
   ].join(`
 `) + `
 `;
-  const helper = `${JSON.stringify({ ...input, drainDeadlineMs }, null, 2)}
+  const helper = `${JSON.stringify({ ...input, runtimeHandoff, drainDeadlineMs }, null, 2)}
 `;
   return {
     environment,
@@ -1266,27 +1278,66 @@ async function activatePlatformRelease(config, requestedRelease, options = {}) {
   mkdirSync2(slots, { recursive: true, mode: 493 });
   const slotFile = join2(slots, ".active");
   let previousSlot = existsSync2(slotFile) ? readFileSync2(slotFile, "utf8").trim() : undefined;
-  if (previousSlot !== "blue" && previousSlot !== "green") {
+  if (previousSlot !== "blue" && previousSlot !== "green")
     previousSlot = undefined;
-    for (const candidate of ["blue", "green"]) {
-      const active = await exec(["/usr/bin/systemctl", "is-active", "--quiet", `forgezero@${candidate}.service`]);
-      if (active.exitCode === 0) {
-        previousSlot = candidate;
-        break;
-      }
+  const slotPort = (slot) => slot === "blue" ? normalized.bluePort : normalized.greenPort;
+  const healthy = async (port2) => {
+    try {
+      const response = await request(`http://127.0.0.1:${port2}${normalized.healthPath}`, {
+        signal: AbortSignal.timeout(2000)
+      });
+      return response.ok;
+    } catch {
+      return false;
     }
+  };
+  const runningHealthySlots = [];
+  for (const candidate of ["blue", "green"]) {
+    const active = await exec(["/usr/bin/systemctl", "is-active", "--quiet", `forgezero@${candidate}.service`]);
+    if (active.exitCode === 0 && await healthy(slotPort(candidate)))
+      runningHealthySlots.push(candidate);
+  }
+  if (previousSlot && !runningHealthySlots.includes(previousSlot)) {
+    throw new Error("active slot evidence disagrees with the running healthy API slot");
+  }
+  if (!previousSlot) {
+    if (runningHealthySlots.length !== 1) {
+      throw new Error("exactly one running healthy API slot is required before promotion");
+    }
+    previousSlot = runningHealthySlots[0];
   }
   const target = previousSlot === "blue" ? "green" : "blue";
-  const port = target === "blue" ? normalized.bluePort : normalized.greenPort;
-  const meshProbe = async (slotPort, path2) => {
+  const previousPort = slotPort(previousSlot);
+  const port = slotPort(target);
+  const handoffProbe = async (slotPort2, path2) => {
     try {
-      const response = await request(`http://${normalized.seedMeshAddress}:${slotPort}/_internal/forgezero/seed-mesh/${path2}`, { signal: AbortSignal.timeout(2000) });
+      const response = await request(`http://${normalized.runtimeHandoff.address}:${slotPort2}${path2}`, { signal: AbortSignal.timeout(2000) });
       return response.ok;
     } catch {
       return false;
     }
   };
-  const previousHeldSeed = previousSlot ? await meshProbe(previousSlot === "blue" ? normalized.bluePort : normalized.greenPort, "resident") : false;
+  const upstream = options.upstreamPath ?? "/etc/nginx/conf.d/forgezero-upstream.conf";
+  const renderUpstream = (upstreamPort) => `upstream forgezero { server 127.0.0.1:${upstreamPort}; }
+`;
+  const reloadNginx = async (upstreamPort) => {
+    const pending = `${upstream}.next`;
+    writeFileSync2(pending, renderUpstream(upstreamPort), { mode: 420, flush: true });
+    renameSync2(pending, upstream);
+    const test = await exec(["/usr/sbin/nginx", "-t"]);
+    const reload = test.exitCode === 0 ? await exec(["/usr/sbin/nginx", "-s", "reload"]) : test;
+    if (reload.exitCode !== 0) {
+      const refusal = (reload.output || test.output).trim().slice(0, 2000);
+      throw new Error(`nginx refused the active-slot upstream${refusal ? `: ${refusal}` : ""}`);
+    }
+  };
+  if (!existsSync2(upstream) || readFileSync2(upstream, "utf8") !== renderUpstream(previousPort)) {
+    await reloadNginx(previousPort);
+  }
+  if (!await healthy(normalized.publicPort)) {
+    throw new Error("stable public listener does not reach the active API slot");
+  }
+  const previousHeldSeed = previousSlot ? await handoffProbe(previousSlot === "blue" ? normalized.bluePort : normalized.greenPort, normalized.runtimeHandoff.residentPath) : false;
   if (previousSlot && !previousHeldSeed) {
     throw new Error("active slot has no resident Vault seed; unlock custody before promotion");
   }
@@ -1307,13 +1358,10 @@ async function activatePlatformRelease(config, requestedRelease, options = {}) {
     replaceLink(targetLink, previousTarget);
     throw new Error("target slot failed to start");
   }
-  let healthy = false;
+  let targetHealthy = false;
   for (let attempt = 0;attempt < 30; attempt += 1) {
-    try {
-      const response = await request(`http://127.0.0.1:${port}${normalized.healthPath}`, { signal: AbortSignal.timeout(2000) });
-      healthy = response.ok;
-    } catch {}
-    if (healthy)
+    targetHealthy = await healthy(port);
+    if (targetHealthy)
       break;
     await sleep(1000);
   }
@@ -1323,14 +1371,18 @@ async function activatePlatformRelease(config, requestedRelease, options = {}) {
     if (previousSlot)
       await exec(["/usr/bin/systemctl", "start", `forgezero@${previousSlot}.service`]);
   };
-  if (!healthy) {
+  if (!targetHealthy) {
     await stopTarget();
     throw new Error("target slot failed its health deadline");
   }
   if (previousHeldSeed) {
     let seedReady = false;
     for (let attempt = 0;attempt < 30; attempt += 1) {
-      seedReady = await meshProbe(port, "ready");
+      const [resident, converged] = await Promise.all([
+        handoffProbe(port, normalized.runtimeHandoff.residentPath),
+        handoffProbe(port, normalized.runtimeHandoff.readyPath)
+      ]);
+      seedReady = resident && converged;
       if (seedReady)
         break;
       await sleep(1000);
@@ -1340,18 +1392,24 @@ async function activatePlatformRelease(config, requestedRelease, options = {}) {
       throw new Error("target slot failed RAM-seed handover deadline");
     }
   }
-  const upstream = "/etc/nginx/conf.d/forgezero-upstream.conf";
   const backup = `${upstream}.forgezero-backup`;
   if (existsSync2(upstream))
     copyFileSync2(upstream, backup);
   else
     rmSync2(backup, { force: true });
-  writeFileSync2(upstream, `upstream forgezero { server 127.0.0.1:${port}; }
-`, { mode: 420 });
-  const test = await exec(["/usr/sbin/nginx", "-t"]);
-  const reload = test.exitCode === 0 ? await exec(["/usr/sbin/nginx", "-s", "reload"]) : test;
-  if (reload.exitCode !== 0) {
-    const refusal = (reload.output || test.output).trim().slice(0, 2000);
+  try {
+    await reloadNginx(port);
+  } catch (cause) {
+    if (existsSync2(backup))
+      renameSync2(backup, upstream);
+    else
+      rmSync2(upstream, { force: true });
+    await exec(["/usr/sbin/nginx", "-t"]);
+    await exec(["/usr/sbin/nginx", "-s", "reload"]);
+    await stopTarget();
+    throw cause;
+  }
+  if (!await healthy(normalized.publicPort)) {
     if (existsSync2(backup))
       renameSync2(backup, upstream);
     else
@@ -1359,14 +1417,19 @@ async function activatePlatformRelease(config, requestedRelease, options = {}) {
     await exec(["/usr/sbin/nginx", "-t"]);
     await exec(["/usr/sbin/nginx", "-s", "reload"]);
     await stopTarget();
-    throw new Error(`nginx refused the promoted upstream${refusal ? `: ${refusal}` : ""}`);
+    throw new Error("stable public listener failed after the slot switch");
   }
   rmSync2(backup, { force: true });
   writeFileSync2(slotFile, `${target}
 `, { mode: 420 });
   if (previousSlot && previousSlot !== target) {
-    await sleep(normalized.drainDeadlineMs);
-    await exec(["/usr/bin/systemctl", "stop", `forgezero@${previousSlot}.service`]);
+    const previousService = `forgezero@${previousSlot}.service`;
+    const stopped = await exec(["/usr/bin/systemctl", "stop", previousService]);
+    const inactive = await exec(["/usr/bin/systemctl", "is-active", "--quiet", previousService]);
+    const status = await exec(["/usr/bin/systemctl", "show", "--property=ExecMainStatus", "--value", previousService]);
+    if (stopped.exitCode !== 0 || inactive.exitCode === 0 || status.exitCode !== 0 || status.output.trim() !== "0") {
+      throw new Error("old API slot failed its clean request and job drain; target remains active");
+    }
   }
   const old = readdirSync(releases, { withFileTypes: true }).filter((entry) => entry.isDirectory()).map((entry) => join2(releases, entry.name)).sort((left, right) => statSync2(right).mtimeMs - statSync2(left).mtimeMs).slice(normalized.keepReleases);
   for (const path2 of old)