@codeyam-editor/codeyam-editor 0.1.0-staging.g83619a4 → 0.1.0-staging.ga747ed2

package/npm/cloud.js

@@ -965,11 +965,61 @@ function buildGcloudArgs(verb, cfg = {}, extras = {}) {
 // real use. Unit tests pin their exports so a refactor cannot drop them.
 // =============================================================================
 
+// Per-call wall-clock budget for a single gcloud / SSH invocation. A hung
+// `gcloud describe` (metadata / IAP hiccup) used to block the whole
+// `cloud:reconcile-ingress` sweep forever — the only timeout in the reconcile
+// path was the outer 10-minute exec budget in the dashboard, and that never
+// cascaded to the orphaned descendants. Bounding every per-call spawn well under
+// that outer budget means a single hang trips HERE first: the call returns a
+// timed-out result (formatGcloudFailure renders it), the sweep skips that VM,
+// and no descendant is left hanging to pile up. Shared by runGcloud and the two
+// reconcile spawnSync sites (status probe, per-BES timeout converge) so all three
+// reference one value. 2 min is generous for any single describe/update/SSH probe
+// yet comfortably under the 600000 ms outer reconcile budget.
+const GCLOUD_CALL_TIMEOUT_MS = 120000;
+
+// The project bootstrap is NOT a probe: it runs `git clone` + `npm install` in
+// the repo root and every `apps[].dir`, each package's `postinstall`, then
+// `npm run setup` — minutes of remote work on a heavy repo. It once ran as ONE
+// blocking `runGcloud("ssh-script")` call, so it inherited GCLOUD_CALL_TIMEOUT_MS
+// (a budget whose own comment scopes it to "any single describe/update/SSH
+// probe"). That SIGTERMd the gcloud child mid-clone; `spawnSync` then returned
+// `status: null`, which `!== 0`, so the timeout was indistinguishable from a
+// genuine remote failure and the caller's abort path DELETED A COMPLETELY
+// HEALTHY VM.
+//
+// Observed twice: 2026-07-28 on client repo `lets-go-compost-portal` (a root
+// postinstall of `playwright install chromium`, ~293 MB), and 2026-07-29
+// provisioning six self-host `codeyam-editor` VMs, whose clone alone exceeds the
+// probe budget — all six were built, torn down mid-clone, and reported as
+// "scripts/gcp-project-bootstrap.sh failed on the VM".
+//
+// The install is now DETACHED (projectBootstrapLaunchCommand + a re-attaching
+// follow loop, outcome read from VM-side state), so no single call carries the
+// whole install any more. This constant is therefore the OVERALL deadline across
+// re-attaches — the point at which we stop waiting on a still-healthy VM and say
+// exactly that. Deliberately well above the outer reconcile budget: a long remote
+// install is not a step in a sweep, so the sweep-relative ceiling that governs
+// GCLOUD_CALL_TIMEOUT_MS does not apply to it.
+const PROJECT_BOOTSTRAP_TIMEOUT_MS = 40 * 60 * 1000;
+
+/** Whether a `spawnSync` result is a LOCAL timeout kill rather than a remote
+ *  failure. `spawnSync` reports a timeout as `status: null` plus an ETIMEDOUT
+ *  error, so a bare `status !== 0` check conflates the two — which is what made
+ *  a timed-out bootstrap look like a broken one. Pure. */
+function isSpawnTimeout(result) {
+  if (!result) return false;
+  if (result.status !== null && result.status !== undefined) return false;
+  const code = result.error && result.error.code;
+  return code === "ETIMEDOUT" || result.signal === "SIGTERM";
+}
+
 function runGcloud(args, opts = {}) {
   return spawnSync("gcloud", args, {
     stdio: opts.stdio || "inherit",
     input: opts.input,
     encoding: opts.encoding || "utf8",
+    timeout: opts.timeout != null ? opts.timeout : GCLOUD_CALL_TIMEOUT_MS,
   });
 }
 
@@ -1572,6 +1622,267 @@ function bootstrapLogTail(cfg, lines = 40) {
   return String(r.stdout || "").trim() || "(log unavailable)";
 }
 
+// ─── Detached PROJECT bootstrap ──────────────────────────────────────────────
+//
+// The VM bootstrap above solved a problem the PROJECT bootstrap still had: read
+// success and failure from VM-SIDE STATE, never from the SSH exit code, so a
+// dropped follow-session over a healthy VM is not misreported as "failed on the
+// VM". The project bootstrap ran instead as ONE blocking ssh-script call, so a
+// local budget elapsing (or an SSH pipe breaking) was indistinguishable from a
+// genuine remote failure and the caller's abort path DELETED A HEALTHY VM.
+// Raising the budget alone just moves the cliff; these helpers remove it by
+// mirroring the VM-bootstrap contract — detached launch, re-attaching follow
+// loop, VM-side outcome probe.
+//
+// All VM-side paths are keyed by repo name so two projects on one VM never
+// collide, and they sit beside the `.done` marker projectBootstrapDoneMarker
+// already defines. Pure string builders (like their VM-bootstrap counterparts)
+// so the whole grammar is unit-testable without a live VM.
+function projectBootstrapScriptPath(repoName) {
+  return `$HOME/.codeyam-project-bootstrap-${repoName}.sh`;
+}
+function projectBootstrapRunnerPath(repoName) {
+  return `$HOME/.codeyam-project-bootstrap-${repoName}-run.sh`;
+}
+function projectBootstrapLogPath(repoName) {
+  return `$HOME/.codeyam-project-bootstrap-${repoName}.log`;
+}
+function projectBootstrapPidPath(repoName) {
+  return `$HOME/.codeyam-project-bootstrap-${repoName}.pid`;
+}
+// The runner's recorded EXIT STATUS. Deliberately `.status`, NOT `.done`:
+// `.done` is already taken by scripts/gcp-project-bootstrap.sh's own terminal
+// SUCCESS marker (a timestamp, written only after `set -euo pipefail` carried
+// every step through — see projectBootstrapDoneMarker). Reusing `.done` for an
+// exit code would overwrite the success signal that both improve40's false-skip
+// fix and the follow loop below depend on. Two files, two distinct meanings:
+// `.done` = "the script reached its last line", `.status` = "the runner exited,
+// here is its code".
+function projectBootstrapStatusPath(repoName) {
+  return `$HOME/.codeyam-project-bootstrap-${repoName}.status`;
+}
+const PROJECT_BOOTSTRAP_LAUNCHED_MARKER = "CODEYAM_PROJECT_BOOTSTRAP_LAUNCHED";
+const PROJECT_BOOTSTRAP_STATUS_MARKER = "CYPROJ";
+// One follow session caps at 15 min; the overall wait caps at
+// PROJECT_BOOTSTRAP_TIMEOUT_MS across re-attaches before we bail loudly with
+// the VM-side log location. Unlike the VM-bootstrap follow loop — whose follow
+// calls inherit the 120 s probe budget and so re-attach every two minutes over
+// a ~14-min build — the project follow call below is given an explicit local
+// budget just above this remote deadline, so a long install streams in one
+// session instead of churning re-attaches.
+const PROJECT_BOOTSTRAP_FOLLOW_SECS = 900;
+const PROJECT_BOOTSTRAP_FOLLOW_TIMEOUT_MS = (PROJECT_BOOTSTRAP_FOLLOW_SECS + 60) * 1000;
+
+// The tiny VM-side wrapper the detached launch execs: run the real bootstrap,
+// redirect all output to the per-repo log, and record the exit status in the
+// `.status` sentinel — the single source of truth for "did the runner exit, and
+// with what?" once the SSH follow-session is gone. Contains NO secrets (tokens
+// arrive via the launch env prefix, never written to disk).
+function projectBootstrapRunnerScript(repoName) {
+  return [
+    "#!/usr/bin/env bash",
+    "# Detached project-bootstrap runner (written by npm/cloud.js",
+    "# ensureProjectBootstrapped). Survives SSH disconnect: the laptop only",
+    "# *follows* the log below; this process keeps running under its own session",
+    "# if the follow drops. Records its own PID so the status probe can check",
+    "# liveness via `kill -0` (a `pgrep` on the script path would also match the",
+    "# probe command itself).",
+    `echo "$$" > "${projectBootstrapPidPath(repoName)}"`,
+    `bash "${projectBootstrapScriptPath(repoName)}" "$@" > "${projectBootstrapLogPath(repoName)}" 2>&1`,
+    "status=$?",
+    `echo "$status" > "${projectBootstrapStatusPath(repoName)}"`,
+    `rm -f "${projectBootstrapPidPath(repoName)}"`,
+    "",
+  ].join("\n");
+}
+
+// Build the remote command that (1) materializes the bootstrap script + runner
+// on the VM, (2) clears the stale sentinels, (3) launches the runner DETACHED
+// (`nohup setsid`, new session, stdio to /dev/null) so it outlives the SSH
+// session, and (4) prints a launched marker. The detach is the durable fix: the
+// install no longer dies when the laptop's SSH pipe breaks, and the VM is no
+// longer torn down because it did.
+//
+// BOTH sentinels are cleared, `.status` and `.done`. Clearing `.done` is what
+// makes the outcome probe honest: after a launch, a present `.done` can only
+// mean THIS run's script reached its terminal line, never a leftover from a
+// previous half-wired attempt.
+function projectBootstrapLaunchCommand(scriptBody, scriptArgs = [], envVars = {}, repoName) {
+  const scriptB64 = Buffer.from(scriptBody, "utf8").toString("base64");
+  const runnerB64 = Buffer.from(projectBootstrapRunnerScript(repoName), "utf8").toString("base64");
+  const argsString = scriptArgs.map(shQuote).join(" ");
+  const envPrefix = envAssignmentPrefix(envVars);
+  // Writes run synchronously (fast, before any backgrounding) so a decode error
+  // aborts before launch; then the runner is detached. The env prefix is on the
+  // setsid command so the tokens are inherited through nohup→setsid→bash but
+  // never appear in any argv. `& echo MARKER` (no `;` before echo) keeps the
+  // launch in the background while still confirming it fired.
+  const writes = [
+    `echo '${scriptB64}' | base64 -d > ${projectBootstrapScriptPath(repoName)}`,
+    `echo '${runnerB64}' | base64 -d > ${projectBootstrapRunnerPath(repoName)}`,
+    `rm -f ${projectBootstrapStatusPath(repoName)} ${projectBootstrapDoneMarker(repoName)}`,
+  ].join(" && ");
+  const launch =
+    `${envPrefix}nohup setsid bash ${projectBootstrapRunnerPath(repoName)} ${argsString}`.trimEnd() +
+    ` </dev/null >/dev/null 2>&1 &`;
+  return `${writes}; ${launch} echo ${PROJECT_BOOTSTRAP_LAUNCHED_MARKER}`;
+}
+
+// Build the remote command that FOLLOWS the detached install: tail the log live
+// for the operator, polling for the `.status` sentinel, and exit when the runner
+// finishes OR a per-session deadline elapses. If the SSH session drops
+// mid-follow this command dies but the detached install keeps going — the caller
+// then re-attaches.
+function projectBootstrapFollowCommand(repoName, deadlineSecs = PROJECT_BOOTSTRAP_FOLLOW_SECS) {
+  const log = projectBootstrapLogPath(repoName);
+  const status = projectBootstrapStatusPath(repoName);
+  return [
+    `touch ${log}`,
+    `tail -n +1 -F ${log} 2>/dev/null & TP=$!`,
+    `waited=0`,
+    `while [ ! -f ${status} ]; do ` +
+      `sleep 3; waited=$((waited+3)); ` +
+      `if [ "$waited" -ge ${deadlineSecs} ]; then kill "$TP" 2>/dev/null; exit 0; fi; ` +
+      `done`,
+    `sleep 1`,
+    `kill "$TP" 2>/dev/null`,
+  ].join("; ");
+}
+
+// Build the remote command that reports terminal VM state in one parseable
+// line: whether the script's terminal `.done` marker exists, the runner's
+// recorded exit status (or `none`), and whether the runner is still alive.
+// Cheap — runs over ssh-probe.
+function projectBootstrapStatusCommand(repoName) {
+  const pid = projectBootstrapPidPath(repoName);
+  const status = projectBootstrapStatusPath(repoName);
+  // Liveness via the runner's PID file + `kill -0`, NOT pgrep: the status
+  // command's own argv contains the bootstrap path, so a `pgrep -f` on it would
+  // match this very probe and always report running. A stale PID file from a
+  // killed runner reads as not-running because `kill -0` fails.
+  return [
+    `done=no; [ -f ${projectBootstrapDoneMarker(repoName)} ] && done=yes`,
+    `st=none; [ -f ${status} ] && st=$(cat ${status} 2>/dev/null)`,
+    `run=no; if [ -f ${pid} ]; then p=$(cat ${pid} 2>/dev/null); ` +
+      `{ [ -n "$p" ] && kill -0 "$p" 2>/dev/null; } && run=yes; fi`,
+    `echo "${PROJECT_BOOTSTRAP_STATUS_MARKER} done=$done status=$st running=$run"`,
+  ].join("; ");
+}
+
+// Parse the single status line emitted by projectBootstrapStatusCommand.
+// Returns `{ done, status, running }` or null when no status line is present
+// (e.g. the probe ssh itself failed and produced no stdout).
+function parseProjectBootstrapStatus(stdout) {
+  const m = String(stdout || "").match(
+    new RegExp(`${PROJECT_BOOTSTRAP_STATUS_MARKER} done=(\\S+) status=(\\S+) running=(\\S+)`),
+  );
+  if (!m) return null;
+  return { done: m[1], status: m[2], running: m[3] };
+}
+
+// Decide the project-bootstrap outcome from VM-side state alone — never the SSH
+// exit code. The `.done` marker is the real artifact (the script writes it only
+// after `set -euo pipefail` carried every step to success), so its presence is
+// success regardless of how the follow session ended — the same reasoning that
+// makes the docker image authoritative for the VM bootstrap. A live runner means
+// "re-attach". A recorded exit status without the marker means the install ran
+// and failed. Nothing at all means it was never started (or the probe could not
+// see it).
+function decideProjectBootstrapOutcome(status) {
+  if (!status) return "absent";
+  const { done, status: exitStatus, running } = status;
+  if (done === "yes") return "success";
+  if (running === "yes") return "running";
+  if (exitStatus && exitStatus !== "none") return "failed";
+  return "absent";
+}
+
+// Reduce one follow-loop iteration to an action: "success" | "failed" |
+// "timeout" | "reattach". Extracted as a pure function because its PRECEDENCE is
+// load-bearing — it decides whether an operator is told a VM is broken:
+//
+//   • success and failed outrank the deadline. Otherwise the final poll of a
+//     slow-but-successful bootstrap reports a timeout, and the operator is sent
+//     to reconcile a VM that actually finished.
+//   • a still-running install past the deadline is a TIMEOUT, never a failure.
+//     That is the VM-8 case: healthy VM, still installing, and saying so is what
+//     gets the add classified retryable instead of destroyed.
+//   • an unseen runner ("absent") before the deadline re-attaches rather than
+//     failing — "I could not see it" is not "it broke" — but is still bounded by
+//     the deadline so the loop cannot spin forever.
+function decideProjectBootstrapFollowAction({ outcome, now, deadline } = {}) {
+  if (outcome === "success") return "success";
+  if (outcome === "failed") return "failed";
+  if (now >= deadline) return "timeout";
+  return "reattach";
+}
+
+// Probe the VM for the detached project bootstrap's terminal state and reduce it
+// to a single outcome string ("success" | "failed" | "running" | "absent").
+// Used both for the idempotent re-attach decision and after each follow session.
+function probeProjectBootstrapOutcome(cfg, repoName) {
+  const r = runGcloud(
+    buildGcloudArgs("ssh-probe", cfg, { command: projectBootstrapStatusCommand(repoName) }),
+    { stdio: ["ignore", "pipe", "pipe"] },
+  );
+  return decideProjectBootstrapOutcome(parseProjectBootstrapStatus(r.stdout));
+}
+
+/** Assemble the env forwarded into the project-bootstrap shell on the VM.
+ *  Extracted from ensureProjectBootstrapped so its CONDITIONAL keys are testable
+ *  without a VM — previously you could not assert "FLEET_DOMAIN is omitted for a
+ *  non-fleet project" or "a re-provision without --admin stays non-admin"
+ *  without invoking the whole gcloud-touching bootstrap.
+ *
+ *  Keys, and why each is here:
+ *    GITHUB_TOKEN                  — clone a private client repo.
+ *    CY_GIT_USER_{NAME,EMAIL}      — rendered into the compose file, then written
+ *                                    to `git config --global` by the entrypoint so
+ *                                    plan_claim_refs::create_claim's commit-tree
+ *                                    call succeeds. Empty is legitimate (a CI
+ *                                    runner with no author); the entrypoint warns
+ *                                    and proceeds, so the keys stay present.
+ *    FLEET_DOMAIN                  — ONLY for a numbered fleet VM with a domain
+ *                                    configured, so the compose render publishes
+ *                                    on 0.0.0.0 for the LB NEG. Omitted entirely
+ *                                    otherwise, preserving the loopback-only
+ *                                    publish a plain client project expects.
+ *    <forwarded launch-form keys>  — DATABASE_URL et al, whitelisted by the
+ *                                    dashboard via CY_FORWARD_ENV_KEYS, so the
+ *                                    bootstrap-time `npm run setup` sees them on
+ *                                    the first try. Empty on plain `cloud:up`.
+ *    CODEYAM_ADMIN                 — the Home-tab Setup door. Defaults to the
+ *                                    strict non-admin "".
+ *
+ *  Later spreads win, matching the original inline assembly exactly. `deps` lets
+ *  a test supply the token and git identity instead of reading the real
+ *  environment and shelling out to git. */
+function projectBootstrapEnv(cfg = {}, { env = process.env, githubToken, gitIdentity } = {}) {
+  const git = gitIdentity || resolveLaptopGitIdentity();
+  const out = {
+    GITHUB_TOKEN: githubToken !== undefined ? githubToken : resolveGithubToken(),
+    CY_GIT_USER_NAME: git.name,
+    CY_GIT_USER_EMAIL: git.email,
+  };
+  const ingressPlan = ingressWiringPlan(cfg, env);
+  if (ingressPlan.wire) out.FLEET_DOMAIN = ingressPlan.domain;
+  Object.assign(out, resolveSetupForwardEnv(env));
+  Object.assign(out, buildProvisionEnv(env, cfg));
+  return out;
+}
+
+// Fetch the tail of the VM-side project-bootstrap log for an actionable failure
+// message — the operator sees WHY the install failed, not a bare exit code.
+function projectBootstrapLogTail(cfg, repoName, lines = 40) {
+  const r = runGcloud(
+    buildGcloudArgs("ssh-probe", cfg, {
+      command: `tail -n ${lines} ${projectBootstrapLogPath(repoName)} 2>/dev/null`,
+    }),
+    { stdio: ["ignore", "pipe", "pipe"] },
+  );
+  return String(r.stdout || "").trim() || "(log unavailable)";
+}
+
 /** Assemble the extra provision-time env forwarded into the bootstrap shell,
  *  where it feeds the compose-render substitution. Currently just the admin
  *  toggle: CODEYAM_ADMIN gates the Home-tab Setup door and is read from the
@@ -1609,7 +1920,7 @@ function buildReconcileAdminCommand(repoName, value) {
   );
 }
 
-async function ensureProjectBootstrapped(cfg) {
+async function ensureProjectBootstrapped(cfg, { sleepFn } = {}) {
   const repoName = resolveProject(cfg).name;
   console.error(`Checking project bootstrap state for ${repoName}...`);
   // Gate on the TERMINAL completion marker the script writes as its very last
@@ -1639,48 +1950,102 @@ async function ensureProjectBootstrapped(cfg) {
   const scriptPath = path.join(rootDir, "scripts", "gcp-project-bootstrap.sh");
   const scriptBody = readStreamedScript(scriptPath, cfg);
   const scriptArgs = clientProjectBootstrapArgs(cfg);
-  // Forward the laptop's git author identity so the rendered compose file
-  // carries CY_GIT_USER_{NAME,EMAIL} → the entrypoint writes them to
-  // `git config --global` → plan_claim_refs::create_claim's commit-tree call
-  // succeeds. Empty strings are fine (laptop has no identity, e.g. a CI
-  // runner); the entrypoint warns and proceeds.
-  const laptopGit = resolveLaptopGitIdentity();
-  // Forward the operator's launch-form env keys (DATABASE_URL, etc.) into the
-  // bootstrap shell so the bootstrap-time `npm run setup` sees them on the
-  // first try — without this, Prisma `db:push`/`db:seed` fail for lack of
-  // env and the bootstrap's fail-soft branch leaves the app schema-less until
-  // the post-add `reapplyVmEnv` re-fires setup. See
-  // `improve37-launch-form-env-injected-into-cloud-up-bootstrap-not-just-applied-after`.
-  // Whitelisted by the dashboard via CY_FORWARD_ENV_KEYS — empty on the
-  // standalone `cloud:up` path, so that flow is unchanged.
-  const forwardEnv = resolveSetupForwardEnv();
-  // Forward the fleet domain so the bootstrap's compose render publishes the
-  // editor/app ports on 0.0.0.0 (the LB NEG reaches them on the VM's internal
-  // IP) instead of loopback-only. Only when a fleet domain is configured AND
-  // this is a numbered fleet VM — a non-fleet client project keeps the
-  // loopback-only publish (legacy parity). Absent ⇒ key omitted entirely.
-  const ingressEnv = {};
-  const ingressPlan = ingressWiringPlan(cfg);
-  if (ingressPlan.wire) ingressEnv.FLEET_DOMAIN = ingressPlan.domain;
-  // Forward the admin-mode toggle (Home-tab Setup door) into the compose
-  // render. cfg.admin carries the `--admin` flag; process.env.CODEYAM_ADMIN
-  // carries the `CODEYAM_ADMIN=1 npm run cloud:up` form. Default "" =
-  // non-admin, so this leaves the normal launch and re-provision unchanged.
-  const provisionEnv = buildProvisionEnv(process.env, cfg);
-  const command = sshScriptCommand(scriptBody, scriptArgs, {
-    GITHUB_TOKEN: resolveGithubToken(),
-    CY_GIT_USER_NAME: laptopGit.name,
-    CY_GIT_USER_EMAIL: laptopGit.email,
-    ...ingressEnv,
-    ...forwardEnv,
-    ...provisionEnv,
-  });
-  const result = runGcloud(
-    buildGcloudArgs("ssh-script", cfg, { command }),
-    { stdio: "inherit" },
-  );
-  if (result.status !== 0) {
-    throw new Error("scripts/gcp-project-bootstrap.sh failed on the VM");
+  // Git identity, forwarded launch-form keys, fleet-domain ingress and the admin
+  // toggle — see projectBootstrapEnv for what each key is for and when it is
+  // omitted. Extracted so those conditionals are testable without a VM.
+  const bootstrapEnv = projectBootstrapEnv(cfg);
+
+  // Idempotent re-attach: a prior cloud:up whose follow-session dropped may have
+  // left a detached install still running. Converge on it instead of starting a
+  // second `npm install` on top of the first.
+  //
+  // NOTE the deliberate asymmetry with ensureBootstrapped, which DOES return
+  // early on an outcome of "success". There, the done-probe and the outcome probe
+  // read the same artifact (the docker image), so they always agree. Here the
+  // done-probe above is a strictly STRONGER three-way AND (`.done` marker +
+  // ~/projects/<repo>/.codeyam + docker-compose.cloud.yml) than this probe's
+  // `.done`-alone success, and it has already said no. Honoring a bare `.done`
+  // here would re-open the improve40 VM-7 false-skip from the other side, so only
+  // "running" short-circuits the launch.
+  const initial = probeProjectBootstrapOutcome(cfg, repoName);
+  if (initial === "running") {
+    console.error("  ↺ a detached project bootstrap is already in progress — re-attaching to its log...");
+  } else {
+    // "absent" (never started), "failed" (a prior run died), or "success" (a bare
+    // marker the stricter done-probe rejected) → (re)launch a fresh detached
+    // install that will survive an SSH disconnect. Every step of
+    // gcp-project-bootstrap.sh is skip-on-exists, so re-entering is cheap.
+    console.error("  launching detached project bootstrap on the VM (survives SSH disconnect)...");
+    const launch = projectBootstrapLaunchCommand(scriptBody, scriptArgs, bootstrapEnv, repoName);
+    const launched = await runGcloudSshWithRetry(
+      "ssh-script",
+      cfg,
+      { command: launch },
+      { stdio: ["ignore", "pipe", "pipe"], label: "project bootstrap launch", sleepFn },
+    );
+    if (isSpawnTimeout(launched)) {
+      // The LOCAL budget elapsed and we SIGTERMd the gcloud child. Say so:
+      // the launch is a short write-and-detach, so this is an SSH-reachability
+      // problem, NOT a broken bootstrap script — and the VM may be perfectly
+      // healthy. Conflating the two is what destroyed VM-8.
+      throw new Error(
+        `launching the detached project bootstrap exceeded the local gcloud call budget ` +
+          `and was SIGTERMd locally — the VM may be healthy and simply unreachable over SSH ` +
+          `right now. Re-run to re-attach (each bootstrap step is skip-on-exists).`,
+      );
+    }
+    if (launched.status !== 0) {
+      throw new Error(
+        `failed to launch the detached project bootstrap on the VM (gcloud ssh exit ${launched.status}).\n` +
+          formatGcloudFailure(launched),
+      );
+    }
+  }
+
+  // Follow the live log, re-attaching across any follow-session drops, until the
+  // VM reports a terminal state or the overall deadline elapses. Success and
+  // failure are read from VM state (the `.done` marker + the `.status` sentinel),
+  // never the SSH exit code — so "my laptop's SSH dropped" or "a heavy npm
+  // install outran a local budget" no longer masquerades as a broken bootstrap
+  // and no longer gets a healthy VM deleted.
+  const deadline = Date.now() + PROJECT_BOOTSTRAP_TIMEOUT_MS;
+  for (;;) {
+    runGcloud(
+      buildGcloudArgs("ssh-script", cfg, { command: projectBootstrapFollowCommand(repoName) }),
+      { stdio: "inherit", timeout: PROJECT_BOOTSTRAP_FOLLOW_TIMEOUT_MS },
+    );
+    const outcome = probeProjectBootstrapOutcome(cfg, repoName);
+    const action = decideProjectBootstrapFollowAction({
+      outcome,
+      now: Date.now(),
+      deadline,
+    });
+    if (action === "success") {
+      console.error(`  ✓ ${repoName} bootstrap complete`);
+      return;
+    }
+    if (action === "failed") {
+      const tail = projectBootstrapLogTail(cfg, repoName);
+      throw new Error(
+        "scripts/gcp-project-bootstrap.sh failed on the VM. " +
+          `Last lines of ${projectBootstrapLogPath(repoName)}:\n${tail}`,
+      );
+    }
+    if (action === "timeout") {
+      // The install is STILL RUNNING on a healthy VM and we simply stopped
+      // waiting. This message is the signature npm/fleet-add-failure.js keys on
+      // to classify the add as `project-bootstrap-timeout` (retryable) instead of
+      // the non-retryable generic cloud-up-failed that tore down VM-8.
+      throw new Error(
+        `project bootstrap exceeded the local ${Math.round(PROJECT_BOOTSTRAP_TIMEOUT_MS / 60000)}-minute budget ` +
+          `— the VM and its detached bootstrap may be healthy and still running. ` +
+          `Re-run to re-attach (each bootstrap step is skip-on-exists), or inspect ` +
+          `${projectBootstrapLogPath(repoName)} on the VM.`,
+      );
+    }
+    // action === "reattach": the follow session ended but the install is alive
+    // (or the probe could not see it yet). Re-attach to its log.
+    console.error("  ↺ follow session ended but the project bootstrap is still running — re-attaching...");
   }
 }
 
@@ -4040,9 +4405,15 @@ async function cmdReconcileIngress(args, projectDir = rootDir) {
       zone: cfg.zone,
       slug,
     });
+    // Bound the per-VM status probe: a hung `bash vm-ingress.sh <N> status`
+    // (its gcloud describes stalling on an IAP/metadata hiccup) must not stall the
+    // whole sweep. On timeout res.stdout is empty, so parseIngressStatusVerdict
+    // returns null → an `unknown` verdict → selectIngressReconcileTargets SKIPS
+    // this VM and the sweep continues to the next one.
     const res = spawnSync(inv.command, inv.args, {
       env: { ...process.env, ...inv.env },
       encoding: "utf8",
+      timeout: GCLOUD_CALL_TIMEOUT_MS,
     });
     return { vmNumber: n, slug, verdict: parseIngressStatusVerdict(res.stdout || "") };
   });
@@ -4097,7 +4468,13 @@ async function cmdReconcileIngress(args, projectDir = rootDir) {
     for (const surface of gi.SURFACES) {
       const bes = gi.backendServiceName(n, surface);
       const inv = backendServiceTimeoutInvocation({ bes, project: cfg.project });
-      const res = spawnSync(inv.command, inv.args, { encoding: "utf8" });
+      // Bound the per-BES converge too: a hung `gcloud backend-services update`
+      // returns a non-zero (timed-out) result, so it falls into the skip branch
+      // below and the sweep moves on to the next BES rather than stalling.
+      const res = spawnSync(inv.command, inv.args, {
+        encoding: "utf8",
+        timeout: GCLOUD_CALL_TIMEOUT_MS,
+      });
       if (res.status === 0) {
         console.error(`  ${bes}: timeout → 86400`);
       } else {
@@ -4342,7 +4719,14 @@ function sweepOrphanIngress({ cfg, domain, roster, projectDir = rootDir } = {},
     });
     const res = run(inv.command, inv.args, { encoding: "utf8", env: { ...process.env, ...inv.env } });
     const verdict = parseIngressTeardownVerdict((res && res.stdout) || "");
-    const outcome = classifyOrphanDownResult({ verdict, status: res ? res.status : null });
+    // Pass `signal` too: a teardown the caller's group-kill reaped has a NULL
+    // status, which the classifier would otherwise report as `spawn-failed`
+    // rather than the truthful `killed=SIGKILL`.
+    const outcome = classifyOrphanDownResult({
+      verdict,
+      status: res ? res.status : null,
+      signal: res ? res.signal : null,
+    });
     if (outcome.cleaned) {
       summary.cleaned.push(n);
       console.error(`  VM-${n}: orphaned ingress → auto-cleaned (verified clean)`);
@@ -4663,6 +5047,7 @@ async function cloudMain() {
 
 module.exports = {
   // pure helpers
+  isSpawnTimeout,
   agentEnvVarName,
   agentNpmPackage,
   AGENT_ENV_VARS,
@@ -4732,6 +5117,22 @@ module.exports = {
   decideBootstrapOutcome,
   projectBootstrapDoneMarker,
   projectBootstrapDoneProbeCommand,
+  projectBootstrapScriptPath,
+  projectBootstrapRunnerPath,
+  projectBootstrapLogPath,
+  projectBootstrapPidPath,
+  projectBootstrapStatusPath,
+  projectBootstrapRunnerScript,
+  projectBootstrapLaunchCommand,
+  projectBootstrapFollowCommand,
+  projectBootstrapStatusCommand,
+  parseProjectBootstrapStatus,
+  decideProjectBootstrapOutcome,
+  decideProjectBootstrapFollowAction,
+  projectBootstrapEnv,
+  PROJECT_BOOTSTRAP_LAUNCHED_MARKER,
+  PROJECT_BOOTSTRAP_FOLLOW_SECS,
+  ensureProjectBootstrapped,
   SSH_KEEPALIVE_FLAGS,
   resolveSetupForwardEnv,
   FORWARD_ENV_KEYS_ENV,