@codeyam-editor/codeyam-editor 0.1.0-staging.gc446a6f → 0.1.0-staging.gde2039f

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. package/npm/cloud.js +1185 -141
  2. package/npm/codesign-plan.js +68 -0
  3. package/npm/compose-conflict.js +59 -0
  4. package/npm/fleet-add-failure.js +223 -17
  5. package/npm/fleet-auth-health.js +121 -5
  6. package/npm/fleet-build-data.js +51 -4
  7. package/npm/fleet-cleanup.js +221 -1
  8. package/npm/fleet-destroy-safety.js +230 -13
  9. package/npm/fleet-early-roster.js +87 -0
  10. package/npm/fleet-env.js +86 -0
  11. package/npm/fleet-ghcr-secret.js +129 -0
  12. package/npm/fleet-git-maintenance.js +142 -0
  13. package/npm/fleet-host-op-limiter.js +68 -0
  14. package/npm/fleet-iap-token.js +165 -0
  15. package/npm/fleet-job-badge.js +28 -12
  16. package/npm/fleet-memory.js +281 -8
  17. package/npm/fleet-plans.js +43 -0
  18. package/npm/fleet-prewarm.js +87 -11
  19. package/npm/fleet-probe-transport.js +155 -0
  20. package/npm/fleet-project-drift.js +58 -0
  21. package/npm/fleet-public-url.js +94 -8
  22. package/npm/fleet-reachability.js +80 -0
  23. package/npm/fleet-revision.js +104 -0
  24. package/npm/fleet-running-plan.js +37 -7
  25. package/npm/fleet-session-token-cache.js +65 -0
  26. package/npm/fleet-status-refs.js +96 -0
  27. package/npm/fleet-strand.js +39 -8
  28. package/npm/fleet-vm-access.js +39 -0
  29. package/npm/fleet-vm-probe-target.js +282 -0
  30. package/npm/ingress-invoke.js +226 -8
  31. package/npm/launch-config.js +235 -6
  32. package/npm/scenario-check.js +263 -11
  33. package/npm/scenario-handlers.js +70 -24
  34. package/npm/scenario-interactivity.js +145 -6
  35. package/npm/scenario-playwright.js +272 -32
  36. package/package.json +5 -5
  37. package/ui/dist/assets/{ContainerInspector-OQ1X2doF.js → ContainerInspector-Cu1tSN-o.js} +1 -1
  38. package/ui/dist/assets/EditorShell--V2ExYQ6.js +260 -0
  39. package/ui/dist/assets/EditorShell-DP5UY20G.css +1 -0
  40. package/ui/dist/assets/{Markdown-BoLHHFXH.js → Markdown-Chgdf3PW.js} +1 -1
  41. package/ui/dist/assets/ProjectLauncherScreen-Dmm6Z0mv.js +1 -0
  42. package/ui/dist/assets/ScenarioDataPanel-DjprZq2L.js +9 -0
  43. package/ui/dist/assets/{ScenarioDataPanelFullPage-BrbAtXE5.js → ScenarioDataPanelFullPage-BqMxWVKW.js} +1 -1
  44. package/ui/dist/assets/ScenarioRenderer-DqLVVMYS.js +21 -0
  45. package/ui/dist/assets/{StepsSimulator-BFldWyoX.js → StepsSimulator-CymGgfbu.js} +1 -1
  46. package/ui/dist/assets/index-CjxWhkSy.css +1 -0
  47. package/ui/dist/assets/{index-iE2jFez3.js → index-VuyqBbGJ.js} +4 -4
  48. package/ui/dist/assets/{markdown-v0F0UEt3.js → markdown-C9tbsVHX.js} +1 -1
  49. package/ui/dist/assets/react-nrLBr15I.js +28 -0
  50. package/ui/dist/assets/{useEvents-DHkLJZR4.js → useEvents-S3Qk1Y_d.js} +1 -1
  51. package/ui/dist/assets/useServerIdentity-BdXy8TQv.js +1 -0
  52. package/ui/dist/index.html +4 -4
  53. package/ui/dist/open-source/codeyam-counter.png +0 -0
  54. package/ui/dist/open-source/el-carot.png +0 -0
  55. package/ui/dist/open-source/tabcommand.png +0 -0
  56. package/ui/dist/assets/EditorShell-D5woOSRy.css +0 -1
  57. package/ui/dist/assets/EditorShell-EKM2i9O9.js +0 -232
  58. package/ui/dist/assets/ProjectLauncherScreen-Bq8CJo9M.js +0 -1
  59. package/ui/dist/assets/ScenarioDataPanel-BU2tHhl4.js +0 -5
  60. package/ui/dist/assets/ScenarioRenderer-d3MLbcAe.js +0 -21
  61. package/ui/dist/assets/index-D3csSagX.css +0 -1
  62. package/ui/dist/assets/react-CSS0HapR.js +0 -28
package/npm/cloud.js CHANGED
@@ -11,6 +11,7 @@
11
11
 
12
12
  const { spawn, spawnSync } = require("child_process");
13
13
  const fs = require("fs");
14
+ const https = require("https");
14
15
  const os = require("os");
15
16
  const path = require("path");
16
17
  const readline = require("readline");
@@ -75,9 +76,11 @@ const {
75
76
  regionFromZone,
76
77
  vmIngressInvocation,
77
78
  vmIngressStatusInvocation,
79
+ vmIngressGrantAccessInvocation,
78
80
  parseIngressStatusVerdict,
79
81
  parseIngressTeardownVerdict,
80
82
  selectIngressReconcileTargets,
83
+ selectIngressGrantOnlyTargets,
81
84
  selectOrphanIngressTargets,
82
85
  classifyOrphanDownResult,
83
86
  backendServicesListInvocation,
@@ -90,6 +93,11 @@ const {
90
93
  iapAccessInvocation,
91
94
  sharedIngressInvocation,
92
95
  backendServiceTimeoutInvocation,
96
+ FLEET_FIREWALL_RULE_NAMES,
97
+ fleetFirewallDescribeInvocation,
98
+ fleetFirewallConvergeInvocation,
99
+ firewallNeedsConverge,
100
+ firewallAdmitsPort,
93
101
  } = require("./ingress-invoke");
94
102
  // Shared fleet-roster writer/reader. `cloud:up` rosters the VM it just
95
103
  // provisioned so the dashboard, the cloudflared-monitor tunnel publisher, and
@@ -97,6 +105,10 @@ const {
97
105
  // edit (the invisible-VM gap improve38 closes). `readRoster` enumerates the
98
106
  // fleet for `reconcile-ingress`.
99
107
  const { addToRoster, readRoster } = require("./fleet-roster");
108
+ // Tells a docker container-name conflict (a concurrent run already started this
109
+ // container) apart from a real `docker compose up -d` failure, so the abort path
110
+ // can converge instead of escalating to instance deletion.
111
+ const { classifyComposeUpFailure } = require("./compose-conflict");
100
112
 
101
113
  // =============================================================================
102
114
  // Pure helpers — every export here is unit-tested in cloud.test.js without
@@ -165,6 +177,13 @@ const KNOWN_FLAGS = [
165
177
  // re-provision because it rides the compose-render substitution, unlike a
166
178
  // manual VM compose edit.
167
179
  "admin",
180
+ // GCE machine type for the VM (e.g. n2-standard-4, e2-standard-4). Persisted
181
+ // in cloud.json so a later bare `cloud:up` keeps it. Defaults to
182
+ // DEFAULT_FLEET_MACHINE_TYPE. The whole point of exposing this is fleet
183
+ // scale-out: the region's E2_CPUS quota (72 in us-central1) caps the fleet at
184
+ // ~14 e2-standard-4 VMs, while N2_CPUS/N2D_CPUS (200, unused) leave room for
185
+ // 30 — so the fleet defaults to N2 and this flag lets a caller override.
186
+ "machine-type",
168
187
  ];
169
188
 
170
189
  // Flags that take no value (presence = true). parseFlags must not demand a
@@ -173,6 +192,13 @@ const BOOLEAN_FLAGS = ["no-tunnel", "skip-setup", "list", "allow-no-git-credenti
173
192
 
174
193
  const FIREWALL_RULE_NAME = "codeyam-editor-iap-ssh";
175
194
  const INSTANCE_TAG = "codeyam-editor";
195
+ // Default GCE machine type for fleet VMs. N2 (not E2) because the fleet must
196
+ // reach 30 VMs: us-central1 E2_CPUS quota is 72, so 30 × e2-standard-4 (120
197
+ // vCPU) + the operator host exceed it and cap the fleet at ~14 regardless of
198
+ // code — while N2_CPUS/N2D_CPUS are 200 and unused, leaving headroom for 30 ×
199
+ // n2-standard-4. Override per-VM with `--machine-type`. See the vm--scale-
200
+ // fleet-to-30-vms plan for the measured quota rationale.
201
+ const DEFAULT_FLEET_MACHINE_TYPE = "n2-standard-4";
176
202
  const EDITOR_PORT = 14199;
177
203
  const VITE_PORT = 5173;
178
204
 
@@ -556,6 +582,32 @@ function isWrongModePreviewOrigin(localJson, sameOriginSafe) {
556
582
  return LOOPBACK_ORIGIN_RE.test(origin);
557
583
  }
558
584
 
585
+ /** Decide whether a VM is presenting as HEALTHY while its Live Preview is
586
+ * silently dead — the VM-9 failure mode. The heal
587
+ * (`reconcilePreviewOriginToSlugHost`) fixes the mechanism; this predicate is
588
+ * the smoke detector that catches a FUTURE mechanism gap (a new stack shape, a
589
+ * new provisioning path) by observation instead of by a user noticing a blank
590
+ * iframe days later.
591
+ *
592
+ * Input is the editor's own `/api/config` payload — the effective truth table
593
+ * (`previewSameOriginSafe` + the resolved `previewOrigin`), not a
594
+ * re-derivation from stack.json over ssh. True iff the stack is NOT
595
+ * same-origin-safe AND its effective previewOrigin is one a remote browser can
596
+ * never load (the `/__codeyam_preview` sentinel, any relative path, or a
597
+ * loopback origin) — i.e. exactly `isWrongModePreviewOrigin`'s wrong-mode
598
+ * condition, evaluated against the live config rather than editor.local.json.
599
+ * Composing the two keeps ONE definition of "wrong-mode origin" in this file.
600
+ *
601
+ * Deliberately conservative: a missing/unparseable payload, an absent
602
+ * `previewSameOriginSafe`, or a legitimate same-origin-safe stack on the
603
+ * subpath all return false. Absence of evidence is never a false alarm — the
604
+ * caller renders "unknown" instead. Pure. */
605
+ function isSilentlyBrokenPreview(apiConfig) {
606
+ if (!apiConfig || typeof apiConfig !== "object") return false;
607
+ if (apiConfig.previewSameOriginSafe !== false) return false;
608
+ return isWrongModePreviewOrigin(apiConfig, false);
609
+ }
610
+
559
611
  /** Read the operator-laptop cloudflared-urls.json sidecar. Missing/unparseable
560
612
  * → {} (the monitor may not have run yet; the seed falls back to localhost). */
561
613
  function readCloudflaredUrls() {
@@ -753,6 +805,28 @@ function buildEditorLocalEnvPatch(localEnv, agent) {
753
805
  return { provider: agent, env: { [varName]: value } };
754
806
  }
755
807
 
808
+ /** Build the editor.local.json patch that seeds a GREENFIELD project's
809
+ * `projectTitle` so its live identity resolves to the real project name from
810
+ * first boot. A greenfield project (`--new-project <name>`, no clientRepo) has
811
+ * no origin remote and is bind-mounted at /workspace on the VM, so the editor's
812
+ * identity resolver would otherwise fall back to the bare mount basename
813
+ * "workspace" — which the fleet dashboard then reads as a permanent false
814
+ * project-dir-drift. Seeding `projectTitle` (deep-merged over editor.json on the
815
+ * VM) makes `project_identity::read` prefer the real name.
816
+ *
817
+ * Returns `null` for a clone (a repo Add already labels from its origin
818
+ * basename, and re-seeding a clone's title would be wrong) or when no
819
+ * greenfield name is available, so the caller can no-op without an SSH round
820
+ * trip. Pure — mirrors buildEditorLocalEnvPatch so the decision is unit-tested
821
+ * without a live VM. */
822
+ function buildGreenfieldTitlePatch(cfg = {}) {
823
+ const p = resolveProject(cfg);
824
+ if (!p.greenfield) return null;
825
+ const name = typeof p.name === 'string' ? p.name.trim() : '';
826
+ if (!name) return null;
827
+ return { projectTitle: name };
828
+ }
829
+
756
830
  /** Parse `docker compose ps --format json` (compose v2 — one JSON object per
757
831
  * line) and return the editor service's `State` string, or null when the
758
832
  * service is absent / output is unparseable. Two callers shared an identical
@@ -775,6 +849,79 @@ function parseEditorComposeState(stdout) {
775
849
  return null;
776
850
  }
777
851
 
852
+ /** Decide whether `cloud:up`'s abort handler may delete the instance it is
853
+ * unwinding. Pure `{ createdThisRun, containerUp, rostered, probeInconclusive }
854
+ * → 'delete' | 'keep'`, matching the `reapVerdict` / `reattachVerdict` /
855
+ * `reconcileRosteredVm` pattern in npm/fleet-cleanup.js, so the whole matrix is
856
+ * exhaustively testable without a VM.
857
+ *
858
+ * `createdThisRun` stays a NECESSARY condition — never delete an instance this
859
+ * run did not create — but 2026-08-07 proved it is not SUFFICIENT. It answers
860
+ * "did I create this?", not "is it safe to destroy?", and in a concurrent
861
+ * provision race the losing run legitimately owns the flag while the winner has
862
+ * already brought the VM up and rostered it. Deleting there destroyed VM-4 and
863
+ * VM-5. So authorization also requires CURRENT-STATE evidence that the VM is
864
+ * out of service:
865
+ *
866
+ * - `containerUp` — the editor container is running (per the compose-ps probe).
867
+ * A VM serving an editor is in use by definition.
868
+ * - `rostered` — the VM is in the fleet roster, the operator-visible source of
869
+ * truth. Note this is true whether the winning run or THIS run rostered it;
870
+ * either way the VM is visible in the dashboard and removable in one command,
871
+ * so it is not the silent orphan teardown exists to prevent.
872
+ * - `probeInconclusive` — liveness could not be determined. This must fail
873
+ * SAFE: a failing probe is how we reach the abort handler in the first place,
874
+ * so it can never count as evidence the VM is dead. An orphan costs one
875
+ * `npm run cloud:destroy`; a wrong delete costs a 15-40 minute rebuild plus
876
+ * any unpushed work on the VM. */
877
+ function abortTeardownDecision({ createdThisRun, containerUp, rostered, probeInconclusive } = {}) {
878
+ if (!createdThisRun) return "keep";
879
+ if (probeInconclusive) return "keep";
880
+ if (containerUp) return "keep";
881
+ if (rostered) return "keep";
882
+ return "delete";
883
+ }
884
+
885
+ /** Gather the CURRENT-STATE inputs {@link abortTeardownDecision} needs, so the
886
+ * abort handler reads gather → decide → act instead of inlining probes among
887
+ * the delete calls.
888
+ *
889
+ * Only runs the probes when `createdThisRun` is true — the decision is `keep`
890
+ * regardless otherwise, and an abort path should not spend SSH round trips
891
+ * proving something it already knows. In that case it returns the
892
+ * probe-inconclusive shape, which is the safe default at every layer.
893
+ *
894
+ * Both lookups are wrapped: a throwing probe or an unreadable roster must fail
895
+ * SAFE (toward `keep`), never take a second exception path into the delete.
896
+ * `deps` is injectable so the matrix is unit-tested without a VM. */
897
+ function probeAbortTeardownState(cfg, createdThisRun, deps = {}) {
898
+ if (!createdThisRun) {
899
+ return { containerUp: false, rostered: false, probeInconclusive: true };
900
+ }
901
+ const probe = deps.probeEditorComposeState || probeEditorComposeState;
902
+ const roster = deps.readRoster || readRoster;
903
+ const resolveVm = deps.resolveFleetVmNumber || resolveFleetVmNumber;
904
+ const log = deps.log || ((m) => console.error(m));
905
+
906
+ let liveness = { state: null, inconclusive: true };
907
+ try {
908
+ liveness = probe(cfg);
909
+ } catch (e) {
910
+ log(` (liveness probe failed: ${e && e.message})`);
911
+ }
912
+ let rostered = false;
913
+ try {
914
+ const vm = resolveVm(cfg.instanceName);
915
+ rostered = vm != null && (roster() || []).includes(vm);
916
+ } catch { /* unreadable roster — treated as not-rostered; liveness still guards */ }
917
+
918
+ return {
919
+ containerUp: liveness.state === "running",
920
+ rostered,
921
+ probeInconclusive: liveness.inconclusive,
922
+ };
923
+ }
924
+
778
925
  /** Translate one of the wrapper's verbs into the exact `gcloud` argv. The
779
926
  * switch's job is to keep every per-verb command shape in one reviewable
780
927
  * place — tests assert the arrays without ever spawning a process. */
@@ -795,7 +942,7 @@ function buildGcloudArgs(verb, cfg = {}, extras = {}) {
795
942
  return [
796
943
  "compute", "instances", "create", instanceName,
797
944
  "--project", project, "--zone", zone,
798
- "--machine-type=e2-standard-4",
945
+ `--machine-type=${cfg.machineType || DEFAULT_FLEET_MACHINE_TYPE}`,
799
946
  "--image-family=debian-12",
800
947
  "--image-project=debian-cloud",
801
948
  "--boot-disk-size=100GB",
@@ -895,11 +1042,61 @@ function buildGcloudArgs(verb, cfg = {}, extras = {}) {
895
1042
  // real use. Unit tests pin their exports so a refactor cannot drop them.
896
1043
  // =============================================================================
897
1044
 
1045
+ // Per-call wall-clock budget for a single gcloud / SSH invocation. A hung
1046
+ // `gcloud describe` (metadata / IAP hiccup) used to block the whole
1047
+ // `cloud:reconcile-ingress` sweep forever — the only timeout in the reconcile
1048
+ // path was the outer 10-minute exec budget in the dashboard, and that never
1049
+ // cascaded to the orphaned descendants. Bounding every per-call spawn well under
1050
+ // that outer budget means a single hang trips HERE first: the call returns a
1051
+ // timed-out result (formatGcloudFailure renders it), the sweep skips that VM,
1052
+ // and no descendant is left hanging to pile up. Shared by runGcloud and the two
1053
+ // reconcile spawnSync sites (status probe, per-BES timeout converge) so all three
1054
+ // reference one value. 2 min is generous for any single describe/update/SSH probe
1055
+ // yet comfortably under the 600000 ms outer reconcile budget.
1056
+ const GCLOUD_CALL_TIMEOUT_MS = 120000;
1057
+
1058
+ // The project bootstrap is NOT a probe: it runs `git clone` + `npm install` in
1059
+ // the repo root and every `apps[].dir`, each package's `postinstall`, then
1060
+ // `npm run setup` — minutes of remote work on a heavy repo. It once ran as ONE
1061
+ // blocking `runGcloud("ssh-script")` call, so it inherited GCLOUD_CALL_TIMEOUT_MS
1062
+ // (a budget whose own comment scopes it to "any single describe/update/SSH
1063
+ // probe"). That SIGTERMd the gcloud child mid-clone; `spawnSync` then returned
1064
+ // `status: null`, which `!== 0`, so the timeout was indistinguishable from a
1065
+ // genuine remote failure and the caller's abort path DELETED A COMPLETELY
1066
+ // HEALTHY VM.
1067
+ //
1068
+ // Observed twice: 2026-07-28 on client repo `lets-go-compost-portal` (a root
1069
+ // postinstall of `playwright install chromium`, ~293 MB), and 2026-07-29
1070
+ // provisioning six self-host `codeyam-editor` VMs, whose clone alone exceeds the
1071
+ // probe budget — all six were built, torn down mid-clone, and reported as
1072
+ // "scripts/gcp-project-bootstrap.sh failed on the VM".
1073
+ //
1074
+ // The install is now DETACHED (projectBootstrapLaunchCommand + a re-attaching
1075
+ // follow loop, outcome read from VM-side state), so no single call carries the
1076
+ // whole install any more. This constant is therefore the OVERALL deadline across
1077
+ // re-attaches — the point at which we stop waiting on a still-healthy VM and say
1078
+ // exactly that. Deliberately well above the outer reconcile budget: a long remote
1079
+ // install is not a step in a sweep, so the sweep-relative ceiling that governs
1080
+ // GCLOUD_CALL_TIMEOUT_MS does not apply to it.
1081
+ const PROJECT_BOOTSTRAP_TIMEOUT_MS = 40 * 60 * 1000;
1082
+
1083
+ /** Whether a `spawnSync` result is a LOCAL timeout kill rather than a remote
1084
+ * failure. `spawnSync` reports a timeout as `status: null` plus an ETIMEDOUT
1085
+ * error, so a bare `status !== 0` check conflates the two — which is what made
1086
+ * a timed-out bootstrap look like a broken one. Pure. */
1087
+ function isSpawnTimeout(result) {
1088
+ if (!result) return false;
1089
+ if (result.status !== null && result.status !== undefined) return false;
1090
+ const code = result.error && result.error.code;
1091
+ return code === "ETIMEDOUT" || result.signal === "SIGTERM";
1092
+ }
1093
+
898
1094
  function runGcloud(args, opts = {}) {
899
1095
  return spawnSync("gcloud", args, {
900
1096
  stdio: opts.stdio || "inherit",
901
1097
  input: opts.input,
902
1098
  encoding: opts.encoding || "utf8",
1099
+ timeout: opts.timeout != null ? opts.timeout : GCLOUD_CALL_TIMEOUT_MS,
903
1100
  });
904
1101
  }
905
1102
 
@@ -1502,6 +1699,267 @@ function bootstrapLogTail(cfg, lines = 40) {
1502
1699
  return String(r.stdout || "").trim() || "(log unavailable)";
1503
1700
  }
1504
1701
 
1702
+ // ─── Detached PROJECT bootstrap ──────────────────────────────────────────────
1703
+ //
1704
+ // The VM bootstrap above solved a problem the PROJECT bootstrap still had: read
1705
+ // success and failure from VM-SIDE STATE, never from the SSH exit code, so a
1706
+ // dropped follow-session over a healthy VM is not misreported as "failed on the
1707
+ // VM". The project bootstrap ran instead as ONE blocking ssh-script call, so a
1708
+ // local budget elapsing (or an SSH pipe breaking) was indistinguishable from a
1709
+ // genuine remote failure and the caller's abort path DELETED A HEALTHY VM.
1710
+ // Raising the budget alone just moves the cliff; these helpers remove it by
1711
+ // mirroring the VM-bootstrap contract — detached launch, re-attaching follow
1712
+ // loop, VM-side outcome probe.
1713
+ //
1714
+ // All VM-side paths are keyed by repo name so two projects on one VM never
1715
+ // collide, and they sit beside the `.done` marker projectBootstrapDoneMarker
1716
+ // already defines. Pure string builders (like their VM-bootstrap counterparts)
1717
+ // so the whole grammar is unit-testable without a live VM.
1718
+ function projectBootstrapScriptPath(repoName) {
1719
+ return `$HOME/.codeyam-project-bootstrap-${repoName}.sh`;
1720
+ }
1721
+ function projectBootstrapRunnerPath(repoName) {
1722
+ return `$HOME/.codeyam-project-bootstrap-${repoName}-run.sh`;
1723
+ }
1724
+ function projectBootstrapLogPath(repoName) {
1725
+ return `$HOME/.codeyam-project-bootstrap-${repoName}.log`;
1726
+ }
1727
+ function projectBootstrapPidPath(repoName) {
1728
+ return `$HOME/.codeyam-project-bootstrap-${repoName}.pid`;
1729
+ }
1730
+ // The runner's recorded EXIT STATUS. Deliberately `.status`, NOT `.done`:
1731
+ // `.done` is already taken by scripts/gcp-project-bootstrap.sh's own terminal
1732
+ // SUCCESS marker (a timestamp, written only after `set -euo pipefail` carried
1733
+ // every step through — see projectBootstrapDoneMarker). Reusing `.done` for an
1734
+ // exit code would overwrite the success signal that both improve40's false-skip
1735
+ // fix and the follow loop below depend on. Two files, two distinct meanings:
1736
+ // `.done` = "the script reached its last line", `.status` = "the runner exited,
1737
+ // here is its code".
1738
+ function projectBootstrapStatusPath(repoName) {
1739
+ return `$HOME/.codeyam-project-bootstrap-${repoName}.status`;
1740
+ }
1741
+ const PROJECT_BOOTSTRAP_LAUNCHED_MARKER = "CODEYAM_PROJECT_BOOTSTRAP_LAUNCHED";
1742
+ const PROJECT_BOOTSTRAP_STATUS_MARKER = "CYPROJ";
1743
+ // One follow session caps at 15 min; the overall wait caps at
1744
+ // PROJECT_BOOTSTRAP_TIMEOUT_MS across re-attaches before we bail loudly with
1745
+ // the VM-side log location. Unlike the VM-bootstrap follow loop — whose follow
1746
+ // calls inherit the 120 s probe budget and so re-attach every two minutes over
1747
+ // a ~14-min build — the project follow call below is given an explicit local
1748
+ // budget just above this remote deadline, so a long install streams in one
1749
+ // session instead of churning re-attaches.
1750
+ const PROJECT_BOOTSTRAP_FOLLOW_SECS = 900;
1751
+ const PROJECT_BOOTSTRAP_FOLLOW_TIMEOUT_MS = (PROJECT_BOOTSTRAP_FOLLOW_SECS + 60) * 1000;
1752
+
1753
+ // The tiny VM-side wrapper the detached launch execs: run the real bootstrap,
1754
+ // redirect all output to the per-repo log, and record the exit status in the
1755
+ // `.status` sentinel — the single source of truth for "did the runner exit, and
1756
+ // with what?" once the SSH follow-session is gone. Contains NO secrets (tokens
1757
+ // arrive via the launch env prefix, never written to disk).
1758
+ function projectBootstrapRunnerScript(repoName) {
1759
+ return [
1760
+ "#!/usr/bin/env bash",
1761
+ "# Detached project-bootstrap runner (written by npm/cloud.js",
1762
+ "# ensureProjectBootstrapped). Survives SSH disconnect: the laptop only",
1763
+ "# *follows* the log below; this process keeps running under its own session",
1764
+ "# if the follow drops. Records its own PID so the status probe can check",
1765
+ "# liveness via `kill -0` (a `pgrep` on the script path would also match the",
1766
+ "# probe command itself).",
1767
+ `echo "$$" > "${projectBootstrapPidPath(repoName)}"`,
1768
+ `bash "${projectBootstrapScriptPath(repoName)}" "$@" > "${projectBootstrapLogPath(repoName)}" 2>&1`,
1769
+ "status=$?",
1770
+ `echo "$status" > "${projectBootstrapStatusPath(repoName)}"`,
1771
+ `rm -f "${projectBootstrapPidPath(repoName)}"`,
1772
+ "",
1773
+ ].join("\n");
1774
+ }
1775
+
1776
+ // Build the remote command that (1) materializes the bootstrap script + runner
1777
+ // on the VM, (2) clears the stale sentinels, (3) launches the runner DETACHED
1778
+ // (`nohup setsid`, new session, stdio to /dev/null) so it outlives the SSH
1779
+ // session, and (4) prints a launched marker. The detach is the durable fix: the
1780
+ // install no longer dies when the laptop's SSH pipe breaks, and the VM is no
1781
+ // longer torn down because it did.
1782
+ //
1783
+ // BOTH sentinels are cleared, `.status` and `.done`. Clearing `.done` is what
1784
+ // makes the outcome probe honest: after a launch, a present `.done` can only
1785
+ // mean THIS run's script reached its terminal line, never a leftover from a
1786
+ // previous half-wired attempt.
1787
+ function projectBootstrapLaunchCommand(scriptBody, scriptArgs = [], envVars = {}, repoName) {
1788
+ const scriptB64 = Buffer.from(scriptBody, "utf8").toString("base64");
1789
+ const runnerB64 = Buffer.from(projectBootstrapRunnerScript(repoName), "utf8").toString("base64");
1790
+ const argsString = scriptArgs.map(shQuote).join(" ");
1791
+ const envPrefix = envAssignmentPrefix(envVars);
1792
+ // Writes run synchronously (fast, before any backgrounding) so a decode error
1793
+ // aborts before launch; then the runner is detached. The env prefix is on the
1794
+ // setsid command so the tokens are inherited through nohup→setsid→bash but
1795
+ // never appear in any argv. `& echo MARKER` (no `;` before echo) keeps the
1796
+ // launch in the background while still confirming it fired.
1797
+ const writes = [
1798
+ `echo '${scriptB64}' | base64 -d > ${projectBootstrapScriptPath(repoName)}`,
1799
+ `echo '${runnerB64}' | base64 -d > ${projectBootstrapRunnerPath(repoName)}`,
1800
+ `rm -f ${projectBootstrapStatusPath(repoName)} ${projectBootstrapDoneMarker(repoName)}`,
1801
+ ].join(" && ");
1802
+ const launch =
1803
+ `${envPrefix}nohup setsid bash ${projectBootstrapRunnerPath(repoName)} ${argsString}`.trimEnd() +
1804
+ ` </dev/null >/dev/null 2>&1 &`;
1805
+ return `${writes}; ${launch} echo ${PROJECT_BOOTSTRAP_LAUNCHED_MARKER}`;
1806
+ }
1807
+
1808
+ // Build the remote command that FOLLOWS the detached install: tail the log live
1809
+ // for the operator, polling for the `.status` sentinel, and exit when the runner
1810
+ // finishes OR a per-session deadline elapses. If the SSH session drops
1811
+ // mid-follow this command dies but the detached install keeps going — the caller
1812
+ // then re-attaches.
1813
+ function projectBootstrapFollowCommand(repoName, deadlineSecs = PROJECT_BOOTSTRAP_FOLLOW_SECS) {
1814
+ const log = projectBootstrapLogPath(repoName);
1815
+ const status = projectBootstrapStatusPath(repoName);
1816
+ return [
1817
+ `touch ${log}`,
1818
+ `tail -n +1 -F ${log} 2>/dev/null & TP=$!`,
1819
+ `waited=0`,
1820
+ `while [ ! -f ${status} ]; do ` +
1821
+ `sleep 3; waited=$((waited+3)); ` +
1822
+ `if [ "$waited" -ge ${deadlineSecs} ]; then kill "$TP" 2>/dev/null; exit 0; fi; ` +
1823
+ `done`,
1824
+ `sleep 1`,
1825
+ `kill "$TP" 2>/dev/null`,
1826
+ ].join("; ");
1827
+ }
1828
+
1829
+ // Build the remote command that reports terminal VM state in one parseable
1830
+ // line: whether the script's terminal `.done` marker exists, the runner's
1831
+ // recorded exit status (or `none`), and whether the runner is still alive.
1832
+ // Cheap — runs over ssh-probe.
1833
+ function projectBootstrapStatusCommand(repoName) {
1834
+ const pid = projectBootstrapPidPath(repoName);
1835
+ const status = projectBootstrapStatusPath(repoName);
1836
+ // Liveness via the runner's PID file + `kill -0`, NOT pgrep: the status
1837
+ // command's own argv contains the bootstrap path, so a `pgrep -f` on it would
1838
+ // match this very probe and always report running. A stale PID file from a
1839
+ // killed runner reads as not-running because `kill -0` fails.
1840
+ return [
1841
+ `done=no; [ -f ${projectBootstrapDoneMarker(repoName)} ] && done=yes`,
1842
+ `st=none; [ -f ${status} ] && st=$(cat ${status} 2>/dev/null)`,
1843
+ `run=no; if [ -f ${pid} ]; then p=$(cat ${pid} 2>/dev/null); ` +
1844
+ `{ [ -n "$p" ] && kill -0 "$p" 2>/dev/null; } && run=yes; fi`,
1845
+ `echo "${PROJECT_BOOTSTRAP_STATUS_MARKER} done=$done status=$st running=$run"`,
1846
+ ].join("; ");
1847
+ }
1848
+
1849
+ // Parse the single status line emitted by projectBootstrapStatusCommand.
1850
+ // Returns `{ done, status, running }` or null when no status line is present
1851
+ // (e.g. the probe ssh itself failed and produced no stdout).
1852
+ function parseProjectBootstrapStatus(stdout) {
1853
+ const m = String(stdout || "").match(
1854
+ new RegExp(`${PROJECT_BOOTSTRAP_STATUS_MARKER} done=(\\S+) status=(\\S+) running=(\\S+)`),
1855
+ );
1856
+ if (!m) return null;
1857
+ return { done: m[1], status: m[2], running: m[3] };
1858
+ }
1859
+
1860
+ // Decide the project-bootstrap outcome from VM-side state alone — never the SSH
1861
+ // exit code. The `.done` marker is the real artifact (the script writes it only
1862
+ // after `set -euo pipefail` carried every step to success), so its presence is
1863
+ // success regardless of how the follow session ended — the same reasoning that
1864
+ // makes the docker image authoritative for the VM bootstrap. A live runner means
1865
+ // "re-attach". A recorded exit status without the marker means the install ran
1866
+ // and failed. Nothing at all means it was never started (or the probe could not
1867
+ // see it).
1868
+ function decideProjectBootstrapOutcome(status) {
1869
+ if (!status) return "absent";
1870
+ const { done, status: exitStatus, running } = status;
1871
+ if (done === "yes") return "success";
1872
+ if (running === "yes") return "running";
1873
+ if (exitStatus && exitStatus !== "none") return "failed";
1874
+ return "absent";
1875
+ }
1876
+
1877
+ // Reduce one follow-loop iteration to an action: "success" | "failed" |
1878
+ // "timeout" | "reattach". Extracted as a pure function because its PRECEDENCE is
1879
+ // load-bearing — it decides whether an operator is told a VM is broken:
1880
+ //
1881
+ // • success and failed outrank the deadline. Otherwise the final poll of a
1882
+ // slow-but-successful bootstrap reports a timeout, and the operator is sent
1883
+ // to reconcile a VM that actually finished.
1884
+ // • a still-running install past the deadline is a TIMEOUT, never a failure.
1885
+ // That is the VM-8 case: healthy VM, still installing, and saying so is what
1886
+ // gets the add classified retryable instead of destroyed.
1887
+ // • an unseen runner ("absent") before the deadline re-attaches rather than
1888
+ // failing — "I could not see it" is not "it broke" — but is still bounded by
1889
+ // the deadline so the loop cannot spin forever.
1890
+ function decideProjectBootstrapFollowAction({ outcome, now, deadline } = {}) {
1891
+ if (outcome === "success") return "success";
1892
+ if (outcome === "failed") return "failed";
1893
+ if (now >= deadline) return "timeout";
1894
+ return "reattach";
1895
+ }
1896
+
1897
+ // Probe the VM for the detached project bootstrap's terminal state and reduce it
1898
+ // to a single outcome string ("success" | "failed" | "running" | "absent").
1899
+ // Used both for the idempotent re-attach decision and after each follow session.
1900
+ function probeProjectBootstrapOutcome(cfg, repoName) {
1901
+ const r = runGcloud(
1902
+ buildGcloudArgs("ssh-probe", cfg, { command: projectBootstrapStatusCommand(repoName) }),
1903
+ { stdio: ["ignore", "pipe", "pipe"] },
1904
+ );
1905
+ return decideProjectBootstrapOutcome(parseProjectBootstrapStatus(r.stdout));
1906
+ }
1907
+
1908
+ /** Assemble the env forwarded into the project-bootstrap shell on the VM.
1909
+ * Extracted from ensureProjectBootstrapped so its CONDITIONAL keys are testable
1910
+ * without a VM — previously you could not assert "FLEET_DOMAIN is omitted for a
1911
+ * non-fleet project" or "a re-provision without --admin stays non-admin"
1912
+ * without invoking the whole gcloud-touching bootstrap.
1913
+ *
1914
+ * Keys, and why each is here:
1915
+ * GITHUB_TOKEN — clone a private client repo.
1916
+ * CY_GIT_USER_{NAME,EMAIL} — rendered into the compose file, then written
1917
+ * to `git config --global` by the entrypoint so
1918
+ * plan_claim_refs::create_claim's commit-tree
1919
+ * call succeeds. Empty is legitimate (a CI
1920
+ * runner with no author); the entrypoint warns
1921
+ * and proceeds, so the keys stay present.
1922
+ * FLEET_DOMAIN — ONLY for a numbered fleet VM with a domain
1923
+ * configured, so the compose render publishes
1924
+ * on 0.0.0.0 for the LB NEG. Omitted entirely
1925
+ * otherwise, preserving the loopback-only
1926
+ * publish a plain client project expects.
1927
+ * <forwarded launch-form keys> — DATABASE_URL et al, whitelisted by the
1928
+ * dashboard via CY_FORWARD_ENV_KEYS, so the
1929
+ * bootstrap-time `npm run setup` sees them on
1930
+ * the first try. Empty on plain `cloud:up`.
1931
+ * CODEYAM_ADMIN — the Home-tab Setup door. Defaults to the
1932
+ * strict non-admin "".
1933
+ *
1934
+ * Later spreads win, matching the original inline assembly exactly. `deps` lets
1935
+ * a test supply the token and git identity instead of reading the real
1936
+ * environment and shelling out to git. */
1937
+ function projectBootstrapEnv(cfg = {}, { env = process.env, githubToken, gitIdentity } = {}) {
1938
+ const git = gitIdentity || resolveLaptopGitIdentity();
1939
+ const out = {
1940
+ GITHUB_TOKEN: githubToken !== undefined ? githubToken : resolveGithubToken(),
1941
+ CY_GIT_USER_NAME: git.name,
1942
+ CY_GIT_USER_EMAIL: git.email,
1943
+ };
1944
+ const ingressPlan = ingressWiringPlan(cfg, env);
1945
+ if (ingressPlan.wire) out.FLEET_DOMAIN = ingressPlan.domain;
1946
+ Object.assign(out, resolveSetupForwardEnv(env));
1947
+ Object.assign(out, buildProvisionEnv(env, cfg));
1948
+ return out;
1949
+ }
1950
+
1951
+ // Fetch the tail of the VM-side project-bootstrap log for an actionable failure
1952
+ // message — the operator sees WHY the install failed, not a bare exit code.
1953
+ function projectBootstrapLogTail(cfg, repoName, lines = 40) {
1954
+ const r = runGcloud(
1955
+ buildGcloudArgs("ssh-probe", cfg, {
1956
+ command: `tail -n ${lines} ${projectBootstrapLogPath(repoName)} 2>/dev/null`,
1957
+ }),
1958
+ { stdio: ["ignore", "pipe", "pipe"] },
1959
+ );
1960
+ return String(r.stdout || "").trim() || "(log unavailable)";
1961
+ }
1962
+
1505
1963
  /** Assemble the extra provision-time env forwarded into the bootstrap shell,
1506
1964
  * where it feeds the compose-render substitution. Currently just the admin
1507
1965
  * toggle: CODEYAM_ADMIN gates the Home-tab Setup door and is read from the
@@ -1539,7 +1997,7 @@ function buildReconcileAdminCommand(repoName, value) {
1539
1997
  );
1540
1998
  }
1541
1999
 
1542
- async function ensureProjectBootstrapped(cfg) {
2000
+ async function ensureProjectBootstrapped(cfg, { sleepFn } = {}) {
1543
2001
  const repoName = resolveProject(cfg).name;
1544
2002
  console.error(`Checking project bootstrap state for ${repoName}...`);
1545
2003
  // Gate on the TERMINAL completion marker the script writes as its very last
@@ -1569,49 +2027,154 @@ async function ensureProjectBootstrapped(cfg) {
1569
2027
  const scriptPath = path.join(rootDir, "scripts", "gcp-project-bootstrap.sh");
1570
2028
  const scriptBody = readStreamedScript(scriptPath, cfg);
1571
2029
  const scriptArgs = clientProjectBootstrapArgs(cfg);
1572
- // Forward the laptop's git author identity so the rendered compose file
1573
- // carries CY_GIT_USER_{NAME,EMAIL} → the entrypoint writes them to
1574
- // `git config --global` → plan_claim_refs::create_claim's commit-tree call
1575
- // succeeds. Empty strings are fine (laptop has no identity, e.g. a CI
1576
- // runner); the entrypoint warns and proceeds.
1577
- const laptopGit = resolveLaptopGitIdentity();
1578
- // Forward the operator's launch-form env keys (DATABASE_URL, etc.) into the
1579
- // bootstrap shell so the bootstrap-time `npm run setup` sees them on the
1580
- // first try — without this, Prisma `db:push`/`db:seed` fail for lack of
1581
- // env and the bootstrap's fail-soft branch leaves the app schema-less until
1582
- // the post-add `reapplyVmEnv` re-fires setup. See
1583
- // `improve37-launch-form-env-injected-into-cloud-up-bootstrap-not-just-applied-after`.
1584
- // Whitelisted by the dashboard via CY_FORWARD_ENV_KEYS — empty on the
1585
- // standalone `cloud:up` path, so that flow is unchanged.
1586
- const forwardEnv = resolveSetupForwardEnv();
1587
- // Forward the fleet domain so the bootstrap's compose render publishes the
1588
- // editor/app ports on 0.0.0.0 (the LB NEG reaches them on the VM's internal
1589
- // IP) instead of loopback-only. Only when a fleet domain is configured AND
1590
- // this is a numbered fleet VM — a non-fleet client project keeps the
1591
- // loopback-only publish (legacy parity). Absent ⇒ key omitted entirely.
1592
- const ingressEnv = {};
1593
- const ingressPlan = ingressWiringPlan(cfg);
1594
- if (ingressPlan.wire) ingressEnv.FLEET_DOMAIN = ingressPlan.domain;
1595
- // Forward the admin-mode toggle (Home-tab Setup door) into the compose
1596
- // render. cfg.admin carries the `--admin` flag; process.env.CODEYAM_ADMIN
1597
- // carries the `CODEYAM_ADMIN=1 npm run cloud:up` form. Default "" =
1598
- // non-admin, so this leaves the normal launch and re-provision unchanged.
1599
- const provisionEnv = buildProvisionEnv(process.env, cfg);
1600
- const command = sshScriptCommand(scriptBody, scriptArgs, {
1601
- GITHUB_TOKEN: resolveGithubToken(),
1602
- CY_GIT_USER_NAME: laptopGit.name,
1603
- CY_GIT_USER_EMAIL: laptopGit.email,
1604
- ...ingressEnv,
1605
- ...forwardEnv,
1606
- ...provisionEnv,
1607
- });
1608
- const result = runGcloud(
1609
- buildGcloudArgs("ssh-script", cfg, { command }),
1610
- { stdio: "inherit" },
2030
+ // Git identity, forwarded launch-form keys, fleet-domain ingress and the admin
2031
+ // toggle — see projectBootstrapEnv for what each key is for and when it is
2032
+ // omitted. Extracted so those conditionals are testable without a VM.
2033
+ const bootstrapEnv = projectBootstrapEnv(cfg);
2034
+
2035
+ // Idempotent re-attach: a prior cloud:up whose follow-session dropped may have
2036
+ // left a detached install still running. Converge on it instead of starting a
2037
+ // second `npm install` on top of the first.
2038
+ //
2039
+ // NOTE the deliberate asymmetry with ensureBootstrapped, which DOES return
2040
+ // early on an outcome of "success". There, the done-probe and the outcome probe
2041
+ // read the same artifact (the docker image), so they always agree. Here the
2042
+ // done-probe above is a strictly STRONGER three-way AND (`.done` marker +
2043
+ // ~/projects/<repo>/.codeyam + docker-compose.cloud.yml) than this probe's
2044
+ // `.done`-alone success, and it has already said no. Honoring a bare `.done`
2045
+ // here would re-open the improve40 VM-7 false-skip from the other side, so only
2046
+ // "running" short-circuits the launch.
2047
+ const initial = probeProjectBootstrapOutcome(cfg, repoName);
2048
+ if (initial === "running") {
2049
+ console.error(" ↺ a detached project bootstrap is already in progress — re-attaching to its log...");
2050
+ } else {
2051
+ // "absent" (never started), "failed" (a prior run died), or "success" (a bare
2052
+ // marker the stricter done-probe rejected) → (re)launch a fresh detached
2053
+ // install that will survive an SSH disconnect. Every step of
2054
+ // gcp-project-bootstrap.sh is skip-on-exists, so re-entering is cheap.
2055
+ console.error(" launching detached project bootstrap on the VM (survives SSH disconnect)...");
2056
+ const launch = projectBootstrapLaunchCommand(scriptBody, scriptArgs, bootstrapEnv, repoName);
2057
+ const launched = await runGcloudSshWithRetry(
2058
+ "ssh-script",
2059
+ cfg,
2060
+ { command: launch },
2061
+ { stdio: ["ignore", "pipe", "pipe"], label: "project bootstrap launch", sleepFn },
2062
+ );
2063
+ if (isSpawnTimeout(launched)) {
2064
+ // The LOCAL budget elapsed and we SIGTERMd the gcloud child. Say so:
2065
+ // the launch is a short write-and-detach, so this is an SSH-reachability
2066
+ // problem, NOT a broken bootstrap script — and the VM may be perfectly
2067
+ // healthy. Conflating the two is what destroyed VM-8.
2068
+ throw new Error(
2069
+ `launching the detached project bootstrap exceeded the local gcloud call budget ` +
2070
+ `and was SIGTERMd locally — the VM may be healthy and simply unreachable over SSH ` +
2071
+ `right now. Re-run to re-attach (each bootstrap step is skip-on-exists).`,
2072
+ );
2073
+ }
2074
+ if (launched.status !== 0) {
2075
+ throw new Error(
2076
+ `failed to launch the detached project bootstrap on the VM (gcloud ssh exit ${launched.status}).\n` +
2077
+ formatGcloudFailure(launched),
2078
+ );
2079
+ }
2080
+ }
2081
+
2082
+ // Follow the live log, re-attaching across any follow-session drops, until the
2083
+ // VM reports a terminal state or the overall deadline elapses. Success and
2084
+ // failure are read from VM state (the `.done` marker + the `.status` sentinel),
2085
+ // never the SSH exit code — so "my laptop's SSH dropped" or "a heavy npm
2086
+ // install outran a local budget" no longer masquerades as a broken bootstrap
2087
+ // and no longer gets a healthy VM deleted.
2088
+ const deadline = Date.now() + PROJECT_BOOTSTRAP_TIMEOUT_MS;
2089
+ for (;;) {
2090
+ runGcloud(
2091
+ buildGcloudArgs("ssh-script", cfg, { command: projectBootstrapFollowCommand(repoName) }),
2092
+ { stdio: "inherit", timeout: PROJECT_BOOTSTRAP_FOLLOW_TIMEOUT_MS },
2093
+ );
2094
+ const outcome = probeProjectBootstrapOutcome(cfg, repoName);
2095
+ const action = decideProjectBootstrapFollowAction({
2096
+ outcome,
2097
+ now: Date.now(),
2098
+ deadline,
2099
+ });
2100
+ if (action === "success") {
2101
+ console.error(` ✓ ${repoName} bootstrap complete`);
2102
+ return;
2103
+ }
2104
+ if (action === "failed") {
2105
+ const tail = projectBootstrapLogTail(cfg, repoName);
2106
+ throw new Error(
2107
+ "scripts/gcp-project-bootstrap.sh failed on the VM. " +
2108
+ `Last lines of ${projectBootstrapLogPath(repoName)}:\n${tail}`,
2109
+ );
2110
+ }
2111
+ if (action === "timeout") {
2112
+ // The install is STILL RUNNING on a healthy VM and we simply stopped
2113
+ // waiting. This message is the signature npm/fleet-add-failure.js keys on
2114
+ // to classify the add as `project-bootstrap-timeout` (retryable) instead of
2115
+ // the non-retryable generic cloud-up-failed that tore down VM-8.
2116
+ throw new Error(
2117
+ `project bootstrap exceeded the local ${Math.round(PROJECT_BOOTSTRAP_TIMEOUT_MS / 60000)}-minute budget ` +
2118
+ `— the VM and its detached bootstrap may be healthy and still running. ` +
2119
+ `Re-run to re-attach (each bootstrap step is skip-on-exists), or inspect ` +
2120
+ `${projectBootstrapLogPath(repoName)} on the VM.`,
2121
+ );
2122
+ }
2123
+ // action === "reattach": the follow session ended but the install is alive
2124
+ // (or the probe could not see it yet). Re-attach to its log.
2125
+ console.error(" ↺ follow session ended but the project bootstrap is still running — re-attaching...");
2126
+ }
2127
+ }
2128
+
2129
+ /** Run the compose-ps liveness probe over SSH and report BOTH the editor
2130
+ * service's state and whether the probe itself could be trusted.
2131
+ *
2132
+ * The `|| true` keeps the remote pipeline's exit status 0 whenever SSH itself
2133
+ * worked, so a non-zero `status` means the transport failed (auth/SSH/IAP) —
2134
+ * not that the container is down. Callers that must fail safe (the abort
2135
+ * teardown) need that distinction; `parseEditorComposeState` alone collapses
2136
+ * "no container" and "could not ask" into the same `null`.
2137
+ *
2138
+ * Returns `{ state, inconclusive }` where `state` is the compose `State` string
2139
+ * (`"running"`, `"exited"`, …) or null when the service is absent, and
2140
+ * `inconclusive` is true only for a transport failure. */
2141
+ function probeEditorComposeState(cfg) {
2142
+ const repoName = resolveProject(cfg).name;
2143
+ const composeCmd = `docker compose -f docker-compose.cloud.yml`;
2144
+ const probe = runGcloud(
2145
+ buildGcloudArgs("ssh-probe", cfg, {
2146
+ command: `cd ~/projects/${repoName} 2>/dev/null && ${composeCmd} ps --format json 2>/dev/null || true`,
2147
+ }),
2148
+ { stdio: ["ignore", "pipe", "ignore"] },
1611
2149
  );
1612
- if (result.status !== 0) {
1613
- throw new Error("scripts/gcp-project-bootstrap.sh failed on the VM");
2150
+ if (probe.status !== 0) {
2151
+ return { state: null, inconclusive: true };
1614
2152
  }
2153
+ return { state: parseEditorComposeState(probe.stdout), inconclusive: false };
2154
+ }
2155
+
2156
+ /** Poll {@link probeEditorComposeState} until the editor service reads
2157
+ * `running`, returning true as soon as it does. Used to close the TOCTOU window
2158
+ * after a container-name conflict: the winning run may still be inside
2159
+ * `Creating`/`Created` when the conflict surfaces, so a single immediate probe
2160
+ * can under-read and report a healthy container as absent.
2161
+ *
2162
+ * Deliberately short (default ~30s): this is not the multi-minute first-boot
2163
+ * budget `waitForContainerRunning` grants, it is the tail of a start that
2164
+ * another process has already begun. `sleepFn` and `probeFn` are injectable so
2165
+ * a unit test drives the whole loop with neither wall-clock delay nor a VM. */
2166
+ async function waitForEditorComposeRunning(cfg, { attempts = 6, delayMs = 5000, sleepFn, probeFn } = {}) {
2167
+ const sleep = sleepFn || ((ms) => new Promise((r) => setTimeout(r, ms)));
2168
+ const probe = probeFn || probeEditorComposeState;
2169
+ for (let i = 0; i < attempts; i++) {
2170
+ if (probe(cfg).state === "running") {
2171
+ return true;
2172
+ }
2173
+ if (i < attempts - 1) {
2174
+ await sleep(delayMs);
2175
+ }
2176
+ }
2177
+ return false;
1615
2178
  }
1616
2179
 
1617
2180
  async function ensureEditorContainerUp(cfg) {
@@ -1621,13 +2184,7 @@ async function ensureEditorContainerUp(cfg) {
1621
2184
  // cloud variant when self-hosting (see gcp-project-bootstrap.sh).
1622
2185
  const composeCmd = `docker compose -f docker-compose.cloud.yml`;
1623
2186
  console.error("Checking editor container state...");
1624
- const probe = runGcloud(
1625
- buildGcloudArgs("ssh-probe", cfg, {
1626
- command: `cd ~/projects/${repoName} && ${composeCmd} ps --format json 2>/dev/null || true`,
1627
- }),
1628
- { stdio: ["ignore", "pipe", "ignore"] },
1629
- );
1630
- if (probe.status === 0 && parseEditorComposeState(probe.stdout) === "running") {
2187
+ if (probeEditorComposeState(cfg).state === "running") {
1631
2188
  console.error(" ✓ editor container is already running");
1632
2189
  // containerStarted: the container is confirmed up, so the CLI install runs
1633
2190
  // its POST-start path with no wait — see the precondition on ensureAgentCliInstalled.
@@ -1635,14 +2192,47 @@ async function ensureEditorContainerUp(cfg) {
1635
2192
  return;
1636
2193
  }
1637
2194
  console.error(" starting editor container (docker compose up -d)...");
2195
+ // CAPTURE rather than `stdio: "inherit"`: the daemon writes its container-name
2196
+ // conflict to stderr, and with "inherit" that text went straight to our parent
2197
+ // and was never observable here — so the conflict could not be told apart from
2198
+ // a real failure (the 2026-08-07 VM-4/VM-5 delete). spawnSync cannot both
2199
+ // stream and capture, so the operator's view is preserved by echoing the
2200
+ // captured streams immediately below instead of live. Do NOT revert this to
2201
+ // "inherit"; classification depends on it.
1638
2202
  const up = runGcloud(
1639
2203
  buildGcloudArgs("ssh-probe", cfg, {
1640
2204
  command: `cd ~/projects/${repoName} && ${composeCmd} up -d`,
1641
2205
  }),
1642
- { stdio: "inherit" },
2206
+ { stdio: ["ignore", "pipe", "pipe"] },
1643
2207
  );
2208
+ if (up.stdout) process.stderr.write(up.stdout);
2209
+ if (up.stderr) process.stderr.write(up.stderr);
1644
2210
  if (up.status !== 0) {
1645
- throw new Error("docker compose up -d failed on the VM");
2211
+ const verdict = classifyComposeUpFailure({ status: up.status, stderr: `${up.stderr || ""}\n${up.stdout || ""}` });
2212
+ if (verdict !== "name-conflict") {
2213
+ throw new Error("docker compose up -d failed on the VM");
2214
+ }
2215
+ // A name conflict means a concurrent run already created this container —
2216
+ // the goal we were about to pursue is being reached by someone else. Confirm
2217
+ // that positively before converging: a conflict whose container never
2218
+ // reaches `running` (e.g. a stale STOPPED container squatting the name) is a
2219
+ // genuine problem and must still fail loudly, or convergence becomes a
2220
+ // catch-all that papers over real breakage.
2221
+ console.error(
2222
+ " a concurrent run already created the editor container — confirming it reaches running...",
2223
+ );
2224
+ if (!(await waitForEditorComposeRunning(cfg))) {
2225
+ throw new Error(
2226
+ "docker compose up -d hit a container-name conflict on the VM, but the conflicting " +
2227
+ "container never reached running (a stale stopped container may be holding the name). " +
2228
+ `Inspect with: gcloud compute ssh ${cfg.instanceName} --command 'cd ~/projects/${repoName} && ${composeCmd} ps -a'`,
2229
+ );
2230
+ }
2231
+ console.error(" ✓ editor container is running (started by a concurrent run)");
2232
+ // containerStarted: the container is confirmed up — same post-start contract
2233
+ // as the already-running short-circuit above.
2234
+ await ensureAgentCliInstalled(cfg, { containerStarted: true });
2235
+ return;
1646
2236
  }
1647
2237
  // containerStarted: `up -d` returned 0, so the CLI install is legitimately a
1648
2238
  // post-start step. A brief wait-for-running here (slow boot settling) is
@@ -1658,18 +2248,10 @@ async function ensureEditorContainerUp(cfg) {
1658
2248
  // success. This throws (non-zero exit) when the editor service isn't
1659
2249
  // `running`, converting a silent partial provision into a loud, actionable
1660
2250
  // failure the operator (and the dashboard's fail-fast-on-nonzero path) can act
1661
- // on. The `|| true` keeps the probe's exit status 0 so the real signal is the
1662
- // parsed compose state, not the SSH/cd exit code.
2251
+ // on. An inconclusive probe is not "running" here — a terminal gate must have
2252
+ // positive evidence, and the caller's job is to fail loudly without it.
1663
2253
  async function assertEditorContainerRunning(cfg) {
1664
- const repoName = resolveProject(cfg).name;
1665
- const composeCmd = `docker compose -f docker-compose.cloud.yml`;
1666
- const probe = runGcloud(
1667
- buildGcloudArgs("ssh-probe", cfg, {
1668
- command: `cd ~/projects/${repoName} 2>/dev/null && ${composeCmd} ps --format json 2>/dev/null || true`,
1669
- }),
1670
- { stdio: ["ignore", "pipe", "ignore"] },
1671
- );
1672
- if (!(probe.status === 0 && parseEditorComposeState(probe.stdout) === "running")) {
2254
+ if (probeEditorComposeState(cfg).state !== "running") {
1673
2255
  throw new Error(
1674
2256
  `provisioning incomplete: editor container is not running on ${cfg.instanceName} ` +
1675
2257
  `(no /workspace or container). Re-run: npm run cloud:up -- --instance-name ${cfg.instanceName}`,
@@ -1812,49 +2394,113 @@ function resolveGhcrPushToken() {
1812
2394
  return process.env.CODEYAM_GHCR_PUSH_TOKEN || "";
1813
2395
  }
1814
2396
 
2397
+ // Registry manifest media types we accept when probing image presence over the
2398
+ // GHCR HTTP API — Docker v2 + manifest lists and their OCI equivalents, so a
2399
+ // multi-arch index answers 200 the same as a single-arch manifest.
2400
+ const MANIFEST_ACCEPT = [
2401
+ "application/vnd.docker.distribution.manifest.v2+json",
2402
+ "application/vnd.docker.distribution.manifest.list.v2+json",
2403
+ "application/vnd.oci.image.manifest.v1+json",
2404
+ "application/vnd.oci.image.index.v1+json",
2405
+ ].join(", ");
2406
+ const GHCR_PROBE_TIMEOUT_MS = 15000;
2407
+
2408
+ /** Default HTTPS client for the registry probe: a GET that resolves to
2409
+ * `{ statusCode }` and NEVER rejects — any socket / DNS / timeout error resolves
2410
+ * to `statusCode: 0` (the probe reads that as "undetermined"). Injected as the
2411
+ * `httpImpl` option so probeUsableEditorImage is unit-testable with no network
2412
+ * (and no Docker daemon). `requestImpl` (defaulting to `https.request`) is
2413
+ * injectable so the response / error / timeout mapping is itself unit-testable
2414
+ * without a real socket. */
2415
+ function httpsManifestGet({ hostname, path: reqPath, headers }, requestImpl = https.request) {
2416
+ return new Promise((resolve) => {
2417
+ const req = requestImpl(
2418
+ { hostname, path: reqPath, method: "GET", headers },
2419
+ (res) => {
2420
+ res.resume(); // drain the body so the socket is freed
2421
+ res.on("end", () => resolve({ statusCode: res.statusCode }));
2422
+ },
2423
+ );
2424
+ req.on("error", () => resolve({ statusCode: 0 }));
2425
+ req.setTimeout(GHCR_PROBE_TIMEOUT_MS, () => req.destroy());
2426
+ req.end();
2427
+ });
2428
+ }
2429
+
2430
+ /** Probe ONE fully-qualified image ref (`host/repo:tag`) over the registry HTTP
2431
+ * API — no docker daemon, so a docker-less operator host can still determine
2432
+ * presence. Returns true (200 present), false (404 absent), or null (auth /
2433
+ * network / malformed-ref → can't tell absent from forbidden). */
2434
+ async function probeManifestRef(fullRef, bearer, httpGet) {
2435
+ const colon = fullRef.lastIndexOf(":");
2436
+ if (colon < 0) return null;
2437
+ const ref = fullRef.slice(0, colon); // host/repo
2438
+ const tag = fullRef.slice(colon + 1); // slug-latest
2439
+ const slash = ref.indexOf("/");
2440
+ if (slash < 0) return null;
2441
+ const hostname = ref.slice(0, slash); // ghcr.io
2442
+ const repo = ref.slice(slash + 1); // codeyam-ai/codeyam-editor-cloud-smoke
2443
+ let res;
2444
+ try {
2445
+ res = await httpGet({
2446
+ hostname,
2447
+ path: `/v2/${repo}/manifests/${encodeURIComponent(tag)}`,
2448
+ headers: { Authorization: `Bearer ${bearer}`, Accept: MANIFEST_ACCEPT },
2449
+ });
2450
+ } catch (_e) {
2451
+ return null;
2452
+ }
2453
+ const code = res && res.statusCode;
2454
+ if (code === 200) return true;
2455
+ if (code === 404) return false;
2456
+ return null; // 401 / 403 / 5xx / 0 (network) → undetermined
2457
+ }
2458
+
1815
2459
  /** Read-only probe: does a usable prebuilt editor image already exist on the
1816
2460
  * registry for `branch`? Used by the fleet dashboard BEFORE a concurrent batch
1817
2461
  * add to decide whether to pace one VM ahead of the rest (see
1818
- * npm/fleet-prewarm.js::decideBatchPrewarm). A `docker manifest inspect` is a
1819
- * HEAD-equivalent — it reads the manifest without pulling the (multi-GB) image.
2462
+ * npm/fleet-prewarm.js::decideBatchPrewarm).
2463
+ *
2464
+ * Queries the registry over its HTTP API (`GET /v2/<repo>/manifests/<tag>`, a
2465
+ * HEAD-equivalent that reads the manifest without pulling the multi-GB image)
2466
+ * instead of `docker login` / `docker manifest inspect`. This is the root fix
2467
+ * for the docker-less GCE operator host: the old docker path errored on the
2468
+ * first `docker login` spawn (the control plane has no docker daemon) and so
2469
+ * returned `null` forever, which decideBatchPrewarm read as "fan out" — the
2470
+ * image was therefore never published and every cold add source-built. Over
2471
+ * HTTP the probe returns a DEFINITIVE verdict with no daemon.
1820
2472
  *
1821
2473
  * Returns:
1822
2474
  * true — at least one of the usable tags exists (a cold add can pull; no pacer)
1823
2475
  * false — every tag was checked and none exists (known-absent; pacing pays)
1824
- * null — undetermined: no read token, docker missing, or a probe error that
1825
- * can't be distinguished from "absent". The caller treats null as
1826
- * "fan out" (today's behavior, never worse) rather than risk a
1827
- * 10-min serialization that might buy nothing.
2476
+ * null — undetermined: no read token, or an auth/network error that can't be
2477
+ * distinguished from "absent".
1828
2478
  *
1829
- * The private packages require auth, so we `docker login` with the read token
1830
- * first; without a token we can't tell absent from forbidden, so → null.
1831
- * `spawnImpl` is injectable for unit tests. */
1832
- function probeUsableEditorImage({ branch, registry, token, spawnImpl } = {}) {
2479
+ * GHCR's private packages require auth; without a read token we can't tell
2480
+ * absent from forbidden, so → null. The read token is sent as the registry
2481
+ * bearer (GHCR accepts a base64-encoded PAT directly — no separate token-
2482
+ * exchange round-trip). `httpImpl` is injectable for unit tests; a legacy
2483
+ * `spawnImpl` option is still accepted (and ignored) for backward-compat — the
2484
+ * HTTP happy path needs no spawn. */
2485
+ async function probeUsableEditorImage({ branch, registry, token, httpImpl } = {}) {
1833
2486
  const fp = require("./fleet-prewarm");
1834
- const spawn = spawnImpl || spawnSync;
1835
2487
  const reg = registry || process.env.CODEYAM_REGISTRY || "ghcr.io/codeyam-ai";
1836
2488
  const readToken = token || resolveGhcrToken();
1837
2489
  if (!readToken) return null; // can't auth → can't distinguish absent from forbidden
1838
- const host = String(reg).split("/")[0];
1839
- const login = spawn(
1840
- "docker",
1841
- ["login", host, "-u", process.env.CODEYAM_GHCR_USER || "codeyam", "--password-stdin"],
1842
- { input: readToken, encoding: "utf8" },
1843
- );
1844
- // spawn error (docker missing) or a non-zero login → undetermined.
1845
- if (!login || login.error || login.status !== 0) return null;
2490
+ const httpGet = httpImpl || httpsManifestGet;
2491
+ // GHCR accepts a base64-encoded PAT as the registry bearer — no token-exchange
2492
+ // round-trip needed.
2493
+ const bearer = Buffer.from(readToken).toString("base64");
1846
2494
  const tags = fp.usableImageTags({ registry: reg, branch });
1847
2495
  let allClean = true;
1848
- for (const tag of tags) {
1849
- const res = spawn("docker", ["manifest", "inspect", tag], { encoding: "utf8" });
1850
- if (!res || res.error) {
1851
- allClean = false; // probe itself failed to run — can't claim "absent"
1852
- continue;
1853
- }
1854
- if (res.status === 0) return true; // a usable image exists
1855
- }
1856
- // Every tag inspected cleanly and none existed → definitively absent. If any
1857
- // probe failed to run, we can't be sure → undetermined.
2496
+ for (const fullRef of tags) {
2497
+ const verdict = await probeManifestRef(fullRef, bearer, httpGet);
2498
+ if (verdict === true) return true; // a usable image exists
2499
+ if (verdict === null) allClean = false; // this tag's probe was inconclusive
2500
+ // verdict === false → this tag is absent; keep checking the rest
2501
+ }
2502
+ // Every tag probed cleanly and none existed → definitively absent. If any probe
2503
+ // was inconclusive we can't claim absence → undetermined.
1858
2504
  return allClean ? false : null;
1859
2505
  }
1860
2506
 
@@ -1962,6 +2608,16 @@ function decideCredentialHeal({ stagingEmpty, lsRemoteOk, lsRemoteStderr }) {
1962
2608
  };
1963
2609
  }
1964
2610
 
2611
+ /** The single GitHub credential line format shared by every seed path: the
2612
+ * host staging file, the durable on-disk `store` fallback, and the live
2613
+ * cache-daemon approve all expect `https://x-access-token:<token>@github.com`
2614
+ * (git's credential-URL form, matched by both the entrypoint parser and
2615
+ * git-credential-store). Centralized so the `x-access-token` shape has one
2616
+ * source of truth instead of being re-typed at each call site. */
2617
+ function githubCredentialLine(token) {
2618
+ return `https://x-access-token:${token}@github.com`;
2619
+ }
2620
+
1965
2621
  /** Write the laptop's GitHub token into ~/.codeyam-cloud-git-credentials on
1966
2622
  * the VM so the cloud compose can mount it into /root/.git-credentials —
1967
2623
  * without this, claude / codex / gemini inside the container can clone the
@@ -1987,21 +2643,28 @@ async function seedGithubCredentials(cfg, opts = {}) {
1987
2643
  console.error(
1988
2644
  `Writing GitHub token (from ${source}) into ~/.codeyam-cloud-git-credentials on the VM...`,
1989
2645
  );
1990
- const credLine = `https://x-access-token:${token}@github.com`;
2646
+ const credLine = githubCredentialLine(token);
1991
2647
  // Quote the token through one shell layer (gcloud ssh --command); the
1992
2648
  // value is base64'd to keep it out of the shell history.
1993
2649
  const b64 = Buffer.from(`${credLine}\n`).toString("base64");
1994
- // Verify-before-move: decode into a chmod-600 temp, assert it is non-empty
1995
- // (`[ -s ]`), and only THEN move it over the live staging file. An empty or
2650
+ // Verify-before-write: decode into a chmod-600 temp, assert it is non-empty
2651
+ // (`[ -s ]`), and only THEN copy it over the live staging file. An empty or
1996
2652
  // unreadable decode must never clobber an existing good file — the entrypoint
1997
2653
  // reads this file on every boot, and a silent empty write is exactly how a
1998
2654
  // reprovision lands a VM that "could not read Username for github.com". The
1999
- // trailing `echo OK` runs only on the move, so an empty write can't mask
2655
+ // trailing `echo OK` runs only on success, so an empty write can't mask
2000
2656
  // success with a `&& echo` (the failure shape seen on the Claude creds sinks).
2657
+ //
2658
+ // Use `cat "$t" > "$f"` (in-place truncate), NOT `mv`: the staging file is
2659
+ // bind-mounted read-only into the running container, and `mv` mints a NEW
2660
+ // inode that the live `:ro` mount does not follow — so a `mv`-based re-seed
2661
+ // never reached the running container until a full recreate. Truncating the
2662
+ // existing file in place keeps its inode, so the durable write is visible to
2663
+ // the container immediately. Then remove the temp.
2001
2664
  const remoteCmd =
2002
2665
  `f=$HOME/.codeyam-cloud-git-credentials && t="$f.tmp.$$" && ` +
2003
2666
  `echo ${b64} | base64 -d > "$t" && chmod 600 "$t" && ` +
2004
- `if [ -s "$t" ]; then mv "$t" "$f" && echo OK; ` +
2667
+ `if [ -s "$t" ]; then cat "$t" > "$f" && chmod 600 "$f" && rm -f "$t" && echo OK; ` +
2005
2668
  `else rm -f "$t"; echo EMPTY-WRITE-REFUSED >&2; exit 1; fi`;
2006
2669
  const result = runGcloud(
2007
2670
  buildGcloudArgs("ssh-probe", cfg, { command: remoteCmd }),
@@ -2012,26 +2675,76 @@ async function seedGithubCredentials(cfg, opts = {}) {
2012
2675
  }
2013
2676
  }
2014
2677
 
2678
+ /** Sentinel the remote read prints when it cannot find an `editor.json` under
2679
+ * any `~/projects/*` directory. Distinguishes "this VM has no scaffolded
2680
+ * project" (a real miss worth warning about) from "the project is scaffolded
2681
+ * but declares no port" (legitimately a `VITE_PORT` default). Without the
2682
+ * distinction both collapse to `{}` and the miss is silent. */
2683
+ const VM_APP_PORT_NO_CONFIG = "<<<NOCONFIG>>>";
2684
+
2685
+ /** Build the remote shell command that locates the project's `.codeyam` dir and
2686
+ * cats its two config files in one round-trip.
2687
+ *
2688
+ * Resolution is glob-based rather than `~/projects/<repoName>` alone: the host
2689
+ * directory is not guaranteed to be named after the repo (a renamed repo, a
2690
+ * manually-cloned dir, a fleet slug), and when the guess missed, the old code
2691
+ * silently returned `VITE_PORT` and forwarded the wrong port. The repo-named
2692
+ * path is still tried FIRST so a multi-project VM resolves deterministically;
2693
+ * the glob is the fallback. When nothing matches, print the sentinel so the
2694
+ * caller can warn instead of guessing. Pure string builder; no I/O. */
2695
+ function buildVmAppPortReadCommand(repoName) {
2696
+ return (
2697
+ `d=""; ` +
2698
+ `for c in ~/projects/${repoName}/.codeyam ~/projects/*/.codeyam; do ` +
2699
+ `if [ -f "$c/editor.json" ]; then d="$c"; break; fi; done; ` +
2700
+ `if [ -z "$d" ]; then echo '${VM_APP_PORT_NO_CONFIG}'; else ` +
2701
+ `cat "$d/editor.json" 2>/dev/null || echo '{}'; ` +
2702
+ `echo '<<<LOCAL>>>'; ` +
2703
+ `cat "$d/editor.local.json" 2>/dev/null || echo '{}'; fi`
2704
+ );
2705
+ }
2706
+
2015
2707
  /** Best-effort read of the VM's resolved app dev-server port so cloud:up's
2016
2708
  * laptop `-L` forward, readiness probe, and previewOrigin seed all target the
2017
2709
  * port the app actually serves on instead of a hardcoded 5173. Cats the VM's
2018
2710
  * editor.json and editor.local.json in one SSH round-trip and resolves via
2019
- * `resolveConfiguredAppPort`. Any failure (SSH error, missing/corrupt JSON)
2020
- * falls back to `VITE_PORT` — the same default `resolved_port()` lands on for
2021
- * a portless config, so a 5173/greenfield project sees no behavior change. */
2711
+ * `resolveConfiguredAppPort`.
2712
+ *
2713
+ * Falls back to `VITE_PORT` on any failure — but fails LOUD, not silent. A
2714
+ * wrong port here is not a cosmetic default: it forwards the wrong `-L` port,
2715
+ * and if the preview ends up on a dedicated origin the viewer gets
2716
+ * "localhost refused to connect" with nothing pointing at the cause. So an SSH
2717
+ * failure or a VM with no scaffolded project warns on stderr naming that
2718
+ * symptom. A scaffolded project that simply declares no port is NOT a miss —
2719
+ * `VITE_PORT` is genuinely correct there (it is what `resolved_port()` lands
2720
+ * on) and stays quiet. */
2022
2721
  async function readVmAppPort(cfg) {
2023
2722
  const repoName = resolveProject(cfg).name;
2024
- const base = `~/projects/${repoName}/.codeyam`;
2025
2723
  const read = runGcloud(
2026
2724
  buildGcloudArgs("ssh-probe", cfg, {
2027
- command:
2028
- `cat ${base}/editor.json 2>/dev/null || echo '{}'; ` +
2029
- `echo '<<<LOCAL>>>'; ` +
2030
- `cat ${base}/editor.local.json 2>/dev/null || echo '{}'`,
2725
+ command: buildVmAppPortReadCommand(repoName),
2031
2726
  }),
2032
2727
  { stdio: ["ignore", "pipe", "ignore"] },
2033
2728
  );
2034
- if (read.status !== 0 || typeof read.stdout !== "string") return VITE_PORT;
2729
+ if (read.status !== 0 || typeof read.stdout !== "string") {
2730
+ console.error(
2731
+ `[cloud] WARNING: could not read the VM's app port over SSH — ` +
2732
+ `falling back to ${VITE_PORT}. If the app does not serve on ` +
2733
+ `${VITE_PORT}, the laptop port-forward targets the wrong port and the ` +
2734
+ `Live Preview will show "localhost refused to connect".`,
2735
+ );
2736
+ return VITE_PORT;
2737
+ }
2738
+ if (read.stdout.includes(VM_APP_PORT_NO_CONFIG)) {
2739
+ console.error(
2740
+ `[cloud] WARNING: no .codeyam/editor.json found under ~/projects on the ` +
2741
+ `VM (looked for '${repoName}', then any project dir) — falling back to ` +
2742
+ `${VITE_PORT}. If the project is scaffolded under a different name and ` +
2743
+ `serves on another port, the port-forward targets the wrong port and ` +
2744
+ `the Live Preview will show "localhost refused to connect".`,
2745
+ );
2746
+ return VITE_PORT;
2747
+ }
2035
2748
  const parse = (s) => {
2036
2749
  try {
2037
2750
  const v = JSON.parse(s);
@@ -2355,6 +3068,66 @@ async function seedAgentApiKey(cfg) {
2355
3068
  // vm--no-cli-install-before-container-started removed; do not re-add it.
2356
3069
  }
2357
3070
 
3071
+ /** Pure: deep-merge `patch` over the VM's current editor.local.json content and
3072
+ * return the base64 of the pretty-printed result, ready for
3073
+ * buildEditorLocalSeedWriteCommand. `currentRaw` is the raw `cat` output of the
3074
+ * remote file (may be undefined, empty, or corrupt) — anything that doesn't
3075
+ * parse to a plain object is treated as `{}`, because overwriting a corrupt
3076
+ * override file with a known-good one is safer than refusing to seed. Extracted
3077
+ * so seedGreenfieldProjectTitle stays a thin I/O passthrough and the
3078
+ * parse+merge+encode is unit-tested without a live VM. */
3079
+ function mergeEditorLocalPatchB64(currentRaw, patch) {
3080
+ let current = {};
3081
+ try {
3082
+ const parsed = JSON.parse(currentRaw);
3083
+ if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
3084
+ current = parsed;
3085
+ }
3086
+ } catch {
3087
+ // Unparseable / missing remote content → start from {}.
3088
+ }
3089
+ const merged = deepMergeJson(current, patch);
3090
+ return Buffer.from(JSON.stringify(merged, null, 2), "utf8").toString("base64");
3091
+ }
3092
+
3093
+ /** Seed a GREENFIELD project's `projectTitle` into the VM's editor.local.json
3094
+ * BEFORE compose up, so the editor caches the real project name at startup and
3095
+ * the fleet dashboard renders the correct card label with no false
3096
+ * project-dir-drift. Like seedAgentApiKey, this MUST run before
3097
+ * ensureEditorContainerUp — the editor reads its config once at startup.
3098
+ *
3099
+ * No-op for a clone (buildGreenfieldTitlePatch returns null): a repo Add labels
3100
+ * from its origin basename, so there is nothing to seed and no SSH round-trip
3101
+ * is spent. Runs on every cloud:up, so a reconfigure re-seeds the title,
3102
+ * converging a VM provisioned before this fix onto the correct identity without
3103
+ * a destroy. Thin I/O passthrough: the parse+merge+encode lives in the pure
3104
+ * mergeEditorLocalPatchB64 helper. */
3105
+ async function seedGreenfieldProjectTitle(cfg) {
3106
+ const patch = buildGreenfieldTitlePatch(cfg);
3107
+ if (!patch) return; // clone (or no greenfield name) — nothing to seed
3108
+
3109
+ const repoName = resolveProject(cfg).name;
3110
+ const remotePath = `~/projects/${repoName}/.codeyam/editor.local.json`;
3111
+ console.error(`Seeding projectTitle='${patch.projectTitle}' into ${remotePath}...`);
3112
+
3113
+ const read = runGcloud(
3114
+ buildGcloudArgs("ssh-probe", cfg, {
3115
+ command: `cat ${remotePath} 2>/dev/null || echo '{}'`,
3116
+ }),
3117
+ { stdio: ["ignore", "pipe", "ignore"] },
3118
+ );
3119
+ const b64 = mergeEditorLocalPatchB64(read.stdout, patch);
3120
+ const write = runGcloud(
3121
+ buildGcloudArgs("ssh-probe", cfg, {
3122
+ command: buildEditorLocalSeedWriteCommand({ repoName, remotePath, b64 }),
3123
+ }),
3124
+ { stdio: ["ignore", "inherit", "inherit"] },
3125
+ );
3126
+ if (write.status !== 0) {
3127
+ throw new Error(`Failed to write ${remotePath} on VM`);
3128
+ }
3129
+ }
3130
+
2358
3131
  /** Tri-state probe of the editor container's running state on the VM host.
2359
3132
  * One ssh round-trip; returns `"running"`, `"not-running"`, or `"unknown"`.
2360
3133
  *
@@ -2617,6 +3390,42 @@ function streamedScriptMergeBase(editorBranch) {
2617
3390
  return mbResult.status === 0 ? mbResult.stdout.trim() : null;
2618
3391
  }
2619
3392
 
3393
+ /** Fetch `editorBranch` from origin into a REAL remote-tracking ref
3394
+ * (`refs/remotes/origin/<branch>`) rather than only into `FETCH_HEAD`.
3395
+ *
3396
+ * Operator hosts are commonly shallow, single-branch clones that track only
3397
+ * `+refs/heads/main`, so `origin/<some-feature-branch>` is not a local ref and
3398
+ * every `git show origin/<branch>:<path>` against it fails. Fetching through an
3399
+ * explicit refspec creates the ref, which keeps the drift verdict and the bytes
3400
+ * actually streamed derived from the same object. Falls back to a plain fetch
3401
+ * (FETCH_HEAD only) when the refspec form is rejected — `readRemoteScriptBlob`
3402
+ * can still read through FETCH_HEAD in that case. */
3403
+ function fetchStreamedScriptRef(editorBranch) {
3404
+ const refspec = `${editorBranch}:refs/remotes/origin/${editorBranch}`;
3405
+ const withRef = spawnSync("git", ["fetch", "--force", "origin", refspec], { encoding: "utf8" });
3406
+ if (withRef.status === 0) return { ok: true, stderr: "" };
3407
+
3408
+ const plain = spawnSync("git", ["fetch", "origin", editorBranch], { encoding: "utf8" });
3409
+ if (plain.status === 0) return { ok: true, stderr: "" };
3410
+ return { ok: false, stderr: (plain.stderr || withRef.stderr || "").trim() };
3411
+ }
3412
+
3413
+ /** Resolve one revision of a streamed script from the fetched remote, trying the
3414
+ * remote-tracking ref first and `FETCH_HEAD` second. `gitArg` is the git verb
3415
+ * (`rev-parse` for the blob hash, `show` for the bytes) so the drift facts and the
3416
+ * streamed bytes resolve through the SAME ref-preference order — a verdict of
3417
+ * "stream remote" can therefore never disagree with what actually got streamed.
3418
+ * Returns `{ ok: false }` only when the blob is unreadable through both refs. */
3419
+ function resolveRemoteScriptRev(gitArg, relPath, editorBranch) {
3420
+ for (const ref of [`origin/${editorBranch}`, "FETCH_HEAD"]) {
3421
+ const result = spawnSync("git", [gitArg, `${ref}:${relPath}`], { encoding: "utf8" });
3422
+ if (result.status === 0 && typeof result.stdout === "string") {
3423
+ return { ok: true, stdout: result.stdout, ref };
3424
+ }
3425
+ }
3426
+ return { ok: false, stdout: "", ref: null };
3427
+ }
3428
+
2620
3429
  /** Gather the drift facts for one streamed script relative to `origin/<editorBranch>`:
2621
3430
  * its local (working-tree) blob hash, the remote-tip blob hash, the merge-base blob
2622
3431
  * hash, and whether the working tree is dirty. Any git failure yields a null hash,
@@ -2628,11 +3437,8 @@ function gatherScriptDriftFacts(relPath, editorBranch, mergeBase) {
2628
3437
  const localHashResult = spawnSync("git", ["hash-object", relPath], { encoding: "utf8" });
2629
3438
  const localHash = localHashResult.status === 0 ? localHashResult.stdout.trim() : null;
2630
3439
 
2631
- let remoteHash = null;
2632
- const remoteHashResult = spawnSync("git", ["rev-parse", `origin/${editorBranch}:${relPath}`], { encoding: "utf8" });
2633
- if (remoteHashResult.status === 0) {
2634
- remoteHash = remoteHashResult.stdout.trim();
2635
- }
3440
+ const remoteRev = resolveRemoteScriptRev("rev-parse", relPath, editorBranch);
3441
+ const remoteHash = remoteRev.ok ? remoteRev.stdout.trim() : null;
2636
3442
 
2637
3443
  let baseHash = null;
2638
3444
  if (mergeBase) {
@@ -2655,7 +3461,15 @@ function gatherScriptDriftFacts(relPath, editorBranch, mergeBase) {
2655
3461
  * un-pulled dashboard can't add VMs" failure. Streams the on-disk copy instead
2656
3462
  * when the operator authored local content the remote tip lacks (dirty edit or a
2657
3463
  * committed local-ahead change — developer mode) or when the remote is unreachable
2658
- * (offline fallback). `absScriptPath` is an absolute path under the repo root. */
3464
+ * (offline fallback — `remoteHash === null`, the only sanctioned on-disk path).
3465
+ *
3466
+ * A "remote" verdict NEVER falls back to the on-disk copy. On a clean-but-behind
3467
+ * checkout the on-disk bytes are precisely the stale bytes the drift guard exists
3468
+ * to keep off the VM, so substituting them silently defeats the guard: the verdict
3469
+ * said "stream the reviewed remote tip" and the VM would instead receive the old
3470
+ * script, with only a `[Warning]` to show for it. When the remote blob cannot be
3471
+ * read through either ref this throws, aborting the add.
3472
+ * `absScriptPath` is an absolute path under the repo root. */
2659
3473
  function readStreamedScript(absScriptPath, cfg = {}) {
2660
3474
  const relPath = path.relative(rootDir, absScriptPath) || absScriptPath;
2661
3475
  const editorBranch = streamedScriptEditorBranch(cfg);
@@ -2664,15 +3478,18 @@ function readStreamedScript(absScriptPath, cfg = {}) {
2664
3478
 
2665
3479
  const { streamedScriptSource } = require("./script-drift");
2666
3480
  if (streamedScriptSource(facts) === "remote") {
2667
- const remote = spawnSync("git", ["show", `origin/${editorBranch}:${relPath}`], { encoding: "utf8" });
2668
- if (remote.status === 0 && typeof remote.stdout === "string") {
2669
- console.error(` streaming canonical ${relPath} from origin/${editorBranch} (checkout behind remote is fine)`);
3481
+ const remote = resolveRemoteScriptRev("show", relPath, editorBranch);
3482
+ if (remote.ok) {
3483
+ console.error(` streaming canonical ${relPath} from ${remote.ref} (checkout behind remote is fine)`);
2670
3484
  return remote.stdout;
2671
3485
  }
2672
- // Remote read failed unexpectedly (raced fetch, ref pruned): fall through to disk.
2673
- console.error(
2674
- ` [Warning] Could not read ${relPath} from origin/${editorBranch} ` +
2675
- `(${(remote.stderr || "").trim() || "unavailable"}). Falling back to the on-disk copy.`,
3486
+ // The facts said "the remote tip is the canonical source" but its bytes are
3487
+ // unreadable. Streaming the on-disk copy here would provision unreviewed stale
3488
+ // bytes under the banner of a successful add — fail loud instead.
3489
+ throw new Error(
3490
+ `could not read ${relPath} from origin/${editorBranch} (or FETCH_HEAD); ` +
3491
+ `refusing to stream a possibly-stale on-disk copy. ` +
3492
+ `Fetch the branch first: git fetch origin ${editorBranch}:refs/remotes/origin/${editorBranch}`,
2676
3493
  );
2677
3494
  }
2678
3495
  return fs.readFileSync(absScriptPath, "utf8");
@@ -2694,10 +3511,11 @@ function checkScriptDrift(cfg) {
2694
3511
 
2695
3512
  console.error(`Checking streamed-script drift against origin/${editorBranch}...`);
2696
3513
 
2697
- // 1. git fetch origin <editorBranch>
2698
- const fetchResult = spawnSync("git", ["fetch", "origin", editorBranch], { encoding: "utf8" });
2699
- if (fetchResult.status !== 0) {
2700
- console.error(` [Warning] Failed to fetch origin ${editorBranch} (${(fetchResult.stderr || "").trim() || "unknown error"}). Skipping drift check.`);
3514
+ // 1. Fetch into refs/remotes/origin/<editorBranch> so the ref resolves even on a
3515
+ // single-branch clone that tracks only +refs/heads/main.
3516
+ const fetchResult = fetchStreamedScriptRef(editorBranch);
3517
+ if (!fetchResult.ok) {
3518
+ console.error(` [Warning] Failed to fetch origin ${editorBranch} (${fetchResult.stderr || "unknown error"}). Skipping drift check.`);
2701
3519
  return;
2702
3520
  }
2703
3521
 
@@ -2810,6 +3628,29 @@ function runVmIngress(cfg, action, vmNumber, domain, appPort, slug) {
2810
3628
  });
2811
3629
  }
2812
3630
 
3631
+ /** Grant-only pass: (re-)apply one VM's default IAP access grant by spawning
3632
+ * `scripts/gcp/vm-ingress.sh <N> grant-access` — the churn-free action that runs
3633
+ * `apply_default_iap_access` without touching any NEG/BES/host-rule resource.
3634
+ * Reconcile uses this for already-`wired` VMs its re-wire loop skips, so a healed
3635
+ * VM whose backends already exist still (re-)grants the automation SA the IAP
3636
+ * accessor. Returns the spawnSync result so the caller logs granted vs failed;
3637
+ * best-effort + loud, never fatal. */
3638
+ function runVmIngressGrantAccess(cfg, vmNumber, domain, slug) {
3639
+ const scriptPath = path.join(rootDir, "scripts", "gcp", "vm-ingress.sh");
3640
+ const inv = vmIngressGrantAccessInvocation({
3641
+ scriptPath,
3642
+ vmNumber,
3643
+ domain,
3644
+ project: cfg.project,
3645
+ zone: cfg.zone,
3646
+ slug,
3647
+ });
3648
+ return spawnSync(inv.command, inv.args, {
3649
+ stdio: "inherit",
3650
+ env: { ...process.env, ...inv.env },
3651
+ });
3652
+ }
3653
+
2813
3654
  /** Add-time hook: wire VM-N onto the shared LB when a fleet domain is set.
2814
3655
  * Best-effort + loud — a wiring hiccup must never abort an otherwise-green Add
2815
3656
  * (the VM is reachable operator-side over SSH regardless), so on failure it
@@ -3353,6 +4194,10 @@ async function cmdUp(args, projectDir = rootDir) {
3353
4194
  }
3354
4195
  }
3355
4196
  await seedAgentApiKey(cfg);
4197
+ // Greenfield only (no-op for a clone): seed projectTitle so the editor
4198
+ // resolves its identity to the real project name instead of the bare
4199
+ // /workspace mount basename — the source of the false project-dir-drift.
4200
+ await seedGreenfieldProjectTitle(cfg);
3356
4201
  await ensureEditorContainerUp(cfg);
3357
4202
  // Provision contract: a just-provisioned VM must actually reach GitHub.
3358
4203
  // Verify in-container git auth now that the container (and its credential
@@ -3424,7 +4269,27 @@ async function cmdUp(args, projectDir = rootDir) {
3424
4269
  );
3425
4270
  }
3426
4271
  } catch (err) {
3427
- if (createdThisRun) {
4272
+ // `createdThisRun` alone MUST NOT authorize deletion. Gather what state the
4273
+ // VM is in right now, then let the pure decision rule — see
4274
+ // abortTeardownDecision's doc comment for why each input is load-bearing.
4275
+ const state = probeAbortTeardownState(cfg, createdThisRun);
4276
+ const verdict = abortTeardownDecision({ createdThisRun, ...state });
4277
+ if (createdThisRun && verdict === "keep") {
4278
+ // Deliberately loud and specific. The old blanket "tearing down the VM
4279
+ // cloud:up created this run (no orphan left behind)" was actively
4280
+ // misleading here — it asserted no orphan while deleting a working VM.
4281
+ const why = state.probeInconclusive
4282
+ ? "its liveness could not be confirmed (probe failed), and an unconfirmed VM is never deleted"
4283
+ : state.containerUp
4284
+ ? "its editor container is RUNNING — another run brought it up"
4285
+ : "it is in the fleet roster and visible to the dashboard";
4286
+ console.error(
4287
+ `✗ provisioning aborted — the VM ${cfg.instanceName} was LEFT IN PLACE because ${why}.\n` +
4288
+ ` This run created the instance, but that alone does not authorize deleting a VM that is in service.\n` +
4289
+ ` If it really should go, remove it deliberately: npm run cloud:destroy`,
4290
+ );
4291
+ }
4292
+ if (verdict === "delete") {
3428
4293
  console.error(
3429
4294
  `✗ provisioning aborted — tearing down the VM cloud:up created this run (no orphan left behind)`,
3430
4295
  );
@@ -3606,14 +4471,7 @@ async function cmdStatus(args, projectDir = rootDir) {
3606
4471
  const imgStatus = String(imgProbe.stdout || "").trim() || "?";
3607
4472
  console.error(` image: ${imgStatus}`);
3608
4473
 
3609
- const repoName = resolveProject(cfg).name;
3610
- const containerProbe = runGcloud(
3611
- buildGcloudArgs("ssh-probe", cfg, {
3612
- command: `cd ~/projects/${repoName} 2>/dev/null && docker compose -f docker-compose.cloud.yml ps --format json 2>/dev/null || echo`,
3613
- }),
3614
- { stdio: ["ignore", "pipe", "ignore"] },
3615
- );
3616
- const containerStatus = parseEditorComposeState(containerProbe.stdout) || "missing";
4474
+ const containerStatus = probeEditorComposeState(cfg).state || "missing";
3617
4475
  console.error(` editor container: ${containerStatus}`);
3618
4476
 
3619
4477
  const appPort = await readVmAppPort(cfg);
@@ -3707,17 +4565,40 @@ async function cmdReconcileIngress(args, projectDir = rootDir) {
3707
4565
  zone: cfg.zone,
3708
4566
  slug,
3709
4567
  });
4568
+ // Bound the per-VM status probe: a hung `bash vm-ingress.sh <N> status`
4569
+ // (its gcloud describes stalling on an IAP/metadata hiccup) must not stall the
4570
+ // whole sweep. On timeout res.stdout is empty, so parseIngressStatusVerdict
4571
+ // returns null → an `unknown` verdict → selectIngressReconcileTargets SKIPS
4572
+ // this VM and the sweep continues to the next one.
3710
4573
  const res = spawnSync(inv.command, inv.args, {
3711
4574
  env: { ...process.env, ...inv.env },
3712
4575
  encoding: "utf8",
4576
+ timeout: GCLOUD_CALL_TIMEOUT_MS,
3713
4577
  });
3714
4578
  return { vmNumber: n, slug, verdict: parseIngressStatusVerdict(res.stdout || "") };
3715
4579
  });
3716
4580
  const targets = selectIngressReconcileTargets(statuses);
3717
4581
  const targetNums = new Set(targets.map((t) => t.vmNumber));
4582
+ // The complement of the re-wire loop: `wired` VMs whose backends already exist
4583
+ // so the re-wire loop skips them, yet still need the SA-accessor (re-)granted —
4584
+ // `up` is the only path that grants it, and it never runs for a wired VM. This
4585
+ // closes the VM-8 heal-path gap (a wired VM whose backends predate the SA-grant
4586
+ // fix would otherwise stay 403 forever). Disjoint from `targets` by construction.
4587
+ const grantOnlyNums = new Set(
4588
+ selectIngressGrantOnlyTargets(statuses).map((g) => g.vmNumber),
4589
+ );
3718
4590
  for (const s of statuses) {
3719
- if (!targetNums.has(s.vmNumber)) {
3720
- const label = s.slug || `vm-${s.vmNumber}`;
4591
+ if (targetNums.has(s.vmNumber)) continue;
4592
+ const label = s.slug || `vm-${s.vmNumber}`;
4593
+ if (grantOnlyNums.has(s.vmNumber)) {
4594
+ // Churn-free grant-only pass: (re-)grant the accessor without re-wiring any
4595
+ // NEG/BES/host-rule resource. Idempotent (a no-op when the SA is already
4596
+ // bound) and best-effort-but-loud — a grant hiccup logs, never aborts.
4597
+ const gres = runVmIngressGrantAccess(cfg, s.vmNumber, domain, s.slug);
4598
+ console.error(
4599
+ ` VM-${s.vmNumber} ${label}: wired → ok (accessor ${gres.status === 0 ? "granted" : "grant FAILED"})`,
4600
+ );
4601
+ } else {
3721
4602
  console.error(` VM-${s.vmNumber} ${label}: ${s.verdict || "unknown"} → ok`);
3722
4603
  }
3723
4604
  }
@@ -3747,7 +4628,13 @@ async function cmdReconcileIngress(args, projectDir = rootDir) {
3747
4628
  for (const surface of gi.SURFACES) {
3748
4629
  const bes = gi.backendServiceName(n, surface);
3749
4630
  const inv = backendServiceTimeoutInvocation({ bes, project: cfg.project });
3750
- const res = spawnSync(inv.command, inv.args, { encoding: "utf8" });
4631
+ // Bound the per-BES converge too: a hung `gcloud backend-services update`
4632
+ // returns a non-zero (timed-out) result, so it falls into the skip branch
4633
+ // below and the sweep moves on to the next BES rather than stalling.
4634
+ const res = spawnSync(inv.command, inv.args, {
4635
+ encoding: "utf8",
4636
+ timeout: GCLOUD_CALL_TIMEOUT_MS,
4637
+ });
3751
4638
  if (res.status === 0) {
3752
4639
  console.error(` ${bes}: timeout → 86400`);
3753
4640
  } else {
@@ -3757,6 +4644,12 @@ async function cmdReconcileIngress(args, projectDir = rootDir) {
3757
4644
  }
3758
4645
  }
3759
4646
 
4647
+ // Converge the shared fleet firewall, then surface any app port it still fails to
4648
+ // admit. Both ride this same boot + periodic cadence so the running fleet
4649
+ // self-heals without a hand-run bootstrap.
4650
+ const firewallState = convergeFleetFirewall({ cfg });
4651
+ await reportFirewallPortGaps({ cfg, roster, firewallState });
4652
+
3760
4653
  // Auto-clean orphaned LB ingress (vm--auto-clean-orphaned-lb-ingress): any
3761
4654
  // backend service whose VM number is gone from the roster is a destroyed VM's
3762
4655
  // leftover — cleaned here automatically with the verified `down`, so the
@@ -3786,6 +4679,106 @@ async function cmdReconcileIngress(args, projectDir = rootDir) {
3786
4679
  * summary (null when the backend-service enumeration itself failed) — the same
3787
4680
  * shape it prints — so the flow is directly assertable. `deps` is injectable so
3788
4681
  * the enumerate → select → down flow is testable without gcloud. */
4682
+ /** Converge the shared fleet firewall rule to all-tcp so the LB reaches whatever
4683
+ * port a project's app actually serves.
4684
+ *
4685
+ * A rule pinned to a hardcoded port list (`tcp:14199,tcp:5173,tcp:3000`) drops
4686
+ * LB→VM traffic for any VM serving outside it — the VM-6-on-8080 Live Preview
4687
+ * outage — while the preview backend service still reads HEALTHY, because its
4688
+ * health check runs on the always-admitted editor port. The failure is therefore
4689
+ * invisible until someone opens the preview. Converging here (not only in the
4690
+ * hand-run google-ingress.sh bootstrap) is what makes a drifted rule self-heal on
4691
+ * the running fleet.
4692
+ *
4693
+ * All-tcp is the right exposure, not a per-stack port list: the rule stays scoped
4694
+ * to Google's LB + health-check source ranges and the fleet instance tag either
4695
+ * way, so it admits any app port (Vite 5173, Next 3000, Astro 4321, Go 8080, …)
4696
+ * without enumeration. EVERY candidate name in FLEET_FIREWALL_RULE_NAMES is
4697
+ * converged: describing only one silently no-ops on a fleet wired under the other.
4698
+ *
4699
+ * Scope-safe by construction — `update --rules=tcp` rewrites only the allowed
4700
+ * protocols/ports, never a rule's source ranges or target tags, so this can never
4701
+ * change who the rule applies to. Best-effort throughout: a describe/update
4702
+ * failure logs and continues, never aborting the rest of the reconcile.
4703
+ *
4704
+ * Returns `[{ ruleName, rules, converged }]` for every rule that exists — `rules`
4705
+ * is the post-converge allowed value — so the flow is assertable without gcloud.
4706
+ * `deps.spawnSync` is injectable for the same reason.
4707
+ * See plan vm--self-healing-app-port-agnostic-fleet-lb-firewall. */
4708
+ function convergeFleetFirewall({ cfg } = {}, deps = {}) {
4709
+ const run = deps.spawnSync || spawnSync;
4710
+ const project = cfg && cfg.project;
4711
+ const state = [];
4712
+ for (const ruleName of FLEET_FIREWALL_RULE_NAMES) {
4713
+ const dinv = fleetFirewallDescribeInvocation({ ruleName, project });
4714
+ const dres = run(dinv.command, dinv.args, { encoding: "utf8" });
4715
+ // A non-zero describe means no rule under this name — expected, not an error:
4716
+ // only one of the candidate names is deployed on any given fleet.
4717
+ if (!dres || dres.status !== 0) continue;
4718
+ const current = ((dres.stdout || "") + "").trim();
4719
+ if (!firewallNeedsConverge(current)) {
4720
+ console.error(` firewall ${ruleName}: already all-tcp → ok`);
4721
+ state.push({ ruleName, rules: "tcp", converged: false });
4722
+ continue;
4723
+ }
4724
+ const cinv = fleetFirewallConvergeInvocation({ ruleName, project });
4725
+ const cres = run(cinv.command, cinv.args, { encoding: "utf8" });
4726
+ if (cres && cres.status === 0) {
4727
+ console.error(` firewall ${ruleName}: rules '${current}' → tcp (all-tcp)`);
4728
+ state.push({ ruleName, rules: "tcp", converged: true });
4729
+ } else {
4730
+ const tail = (
4731
+ (cres && (cres.stderr || (cres.error && cres.error.message))) || ""
4732
+ ).toString().trim().split("\n").pop() || "error";
4733
+ console.error(` firewall ${ruleName}: converge skipped (${tail})`);
4734
+ state.push({ ruleName, rules: current, converged: false });
4735
+ }
4736
+ }
4737
+ if (!state.length) {
4738
+ console.error(
4739
+ ` firewall: no rule found under ${FLEET_FIREWALL_RULE_NAMES.join(" / ")} —` +
4740
+ " LB→VM traffic may be blocked; run scripts/gcp/google-ingress.sh",
4741
+ );
4742
+ }
4743
+ return state;
4744
+ }
4745
+
4746
+ /** Surface any rostered VM whose app port the fleet firewall does not admit — the
4747
+ * tripwire for a masked "HEALTHY but unreachable" preview.
4748
+ *
4749
+ * The preview backend service health-checks the fixed editor port, so a rule that
4750
+ * admits that port but not the app's real one reads HEALTHY while every real
4751
+ * request is dropped. That masking is what made the VM-6 outage invisible. Emits a
4752
+ * machine-readable `CY_FIREWALL_PORT_GAP <json>` line per gap, mirroring the
4753
+ * `CY_ORPHAN_SWEEP` marker convention the dashboard already parses.
4754
+ *
4755
+ * Only runs when a rule is still port-pinned: a converged all-tcp rule admits every
4756
+ * port, so the healthy path cannot produce a gap and skipping it avoids an SSH
4757
+ * round-trip per rostered VM for a check that cannot fail. In practice this fires
4758
+ * only if the converge itself failed (e.g. a permissions regression).
4759
+ *
4760
+ * Returns the emitted `[{ vmNumber, appPort, rule, allowed }]` gaps — the same
4761
+ * shape it prints — so the marker contract is directly assertable.
4762
+ * `deps.readVmAppPort` is injectable so tests never reach for SSH. */
4763
+ async function reportFirewallPortGaps({ cfg, roster, firewallState } = {}, deps = {}) {
4764
+ const readPort = deps.readVmAppPort || readVmAppPort;
4765
+ const pinned = (firewallState || []).filter((f) => firewallNeedsConverge(f.rules));
4766
+ const gaps = [];
4767
+ if (!pinned.length) return gaps;
4768
+ for (const n of roster || []) {
4769
+ const perVmCfg = { ...cfg, instanceName: `${fleetInstancePrefix()}${n}` };
4770
+ const appPort = await readPort(perVmCfg);
4771
+ for (const f of pinned) {
4772
+ if (firewallAdmitsPort(f.rules, appPort)) continue;
4773
+ const gap = { vmNumber: n, appPort, rule: f.ruleName, allowed: f.rules };
4774
+ console.error(` VM-${n}: app port ${appPort} NOT admitted by ${f.ruleName}`);
4775
+ console.log(`CY_FIREWALL_PORT_GAP ${JSON.stringify(gap)}`);
4776
+ gaps.push(gap);
4777
+ }
4778
+ }
4779
+ return gaps;
4780
+ }
4781
+
3789
4782
  function sweepOrphanIngress({ cfg, domain, roster, projectDir = rootDir } = {}, deps = {}) {
3790
4783
  const gi = deps.googleIngress || require("./google-ingress.js");
3791
4784
  const fcl = deps.fleetCleanup || require("./fleet-cleanup.js");
@@ -3886,7 +4879,14 @@ function sweepOrphanIngress({ cfg, domain, roster, projectDir = rootDir } = {},
3886
4879
  });
3887
4880
  const res = run(inv.command, inv.args, { encoding: "utf8", env: { ...process.env, ...inv.env } });
3888
4881
  const verdict = parseIngressTeardownVerdict((res && res.stdout) || "");
3889
- const outcome = classifyOrphanDownResult({ verdict, status: res ? res.status : null });
4882
+ // Pass `signal` too: a teardown the caller's group-kill reaped has a NULL
4883
+ // status, which the classifier would otherwise report as `spawn-failed`
4884
+ // rather than the truthful `killed=SIGKILL`.
4885
+ const outcome = classifyOrphanDownResult({
4886
+ verdict,
4887
+ status: res ? res.status : null,
4888
+ signal: res ? res.signal : null,
4889
+ });
3890
4890
  if (outcome.cleaned) {
3891
4891
  summary.cleaned.push(n);
3892
4892
  console.error(` VM-${n}: orphaned ingress → auto-cleaned (verified clean)`);
@@ -3955,20 +4955,33 @@ async function probeGitCredentials(cfg) {
3955
4955
  /** Seed the LIVE in-memory git credential-cache daemon over its socket — the
3956
4956
  * same `git credential approve` shape the entrypoint uses
3957
4957
  * (scripts/cloud-editor-entrypoint.sh) — so push works immediately with no
3958
- * container restart. base64 keeps the token out of the shell history across
3959
- * the gcloud-ssh → docker layers. Throws if the approve fails. Shared by
3960
- * {@link cmdReconcileCredentials} and {@link assertGitPushCredentials}. */
4958
+ * container restart. ALSO writes the container-local on-disk `store` fallback
4959
+ * file (the fill-only second helper baked into the image) directly, so a
4960
+ * reconcile leaves the VM durable against the cache daemon dying mid-life, not
4961
+ * just healthy for the moment. base64 keeps the token out of the shell history
4962
+ * across the gcloud-ssh → docker layers, and the store write is an in-place
4963
+ * `cat > file` truncate (chmod 600), never touched by git's approve path.
4964
+ * Throws if either write fails. Shared by {@link cmdReconcileCredentials} and
4965
+ * {@link assertGitPushCredentials}. */
3961
4966
  async function seedLiveCredentialCache(cfg, token) {
3962
4967
  const repoName = resolveProject(cfg).name;
3963
4968
  const projectCd = `cd ~/projects/${repoName}`;
3964
4969
  const composeExec = `docker compose -f docker-compose.cloud.yml exec -T editor`;
3965
4970
  const socket = "/run/codeyam/git-credential-cache.socket";
4971
+ const store = "/run/codeyam/git-credentials-store";
3966
4972
  const approveStdin = `protocol=https\nhost=github.com\nusername=x-access-token\npassword=${token}\n\n`;
3967
4973
  const b64 = Buffer.from(approveStdin).toString("base64");
4974
+ // The store-file line is the same one-URL-per-line format as the host staging
4975
+ // file; the image's fill-only `store` helper reads it on a cache miss.
4976
+ const storeB64 = Buffer.from(`${githubCredentialLine(token)}\n`).toString("base64");
3968
4977
  const seedCmd =
3969
- `${projectCd} && echo ${b64} | base64 -d | ${composeExec} ` +
4978
+ `${projectCd} && ` +
4979
+ `echo ${b64} | base64 -d | ${composeExec} ` +
3970
4980
  `git -c 'credential.helper=cache --timeout=31536000 --socket=${socket}' ` +
3971
- `credential approve && echo APPROVED`;
4981
+ `credential approve && ` +
4982
+ `echo ${storeB64} | base64 -d | ${composeExec} ` +
4983
+ `bash -c 'cat > ${store} && chmod 600 ${store}' && ` +
4984
+ `echo APPROVED`;
3972
4985
  const seed = runGcloud(
3973
4986
  buildGcloudArgs("ssh-probe", cfg, { command: seedCmd }),
3974
4987
  { stdio: ["ignore", "pipe", "inherit"] },
@@ -4194,6 +5207,7 @@ async function cloudMain() {
4194
5207
 
4195
5208
  module.exports = {
4196
5209
  // pure helpers
5210
+ isSpawnTimeout,
4197
5211
  agentEnvVarName,
4198
5212
  agentNpmPackage,
4199
5213
  AGENT_ENV_VARS,
@@ -4204,11 +5218,15 @@ module.exports = {
4204
5218
  assertAgentCliInstalled,
4205
5219
  probeContainerRunning,
4206
5220
  probeContainerState,
5221
+ waitForEditorComposeRunning,
5222
+ abortTeardownDecision,
5223
+ probeAbortTeardownState,
4207
5224
  probeAgentBinaryInstalled,
4208
5225
  waitForContainerRunning,
4209
5226
  KNOWN_FLAGS,
4210
5227
  FIREWALL_RULE_NAME,
4211
5228
  INSTANCE_TAG,
5229
+ DEFAULT_FLEET_MACHINE_TYPE,
4212
5230
  EDITOR_PORT,
4213
5231
  VITE_PORT,
4214
5232
  parseFlags,
@@ -4227,6 +5245,7 @@ module.exports = {
4227
5245
  previewSameOriginSafeFromStack,
4228
5246
  isRottedManagedPreviewOrigin,
4229
5247
  isWrongModePreviewOrigin,
5248
+ isSilentlyBrokenPreview,
4230
5249
  resolveEditorConfig,
4231
5250
  validateEditorVariant,
4232
5251
  assertVariantProvisionable,
@@ -4235,6 +5254,7 @@ module.exports = {
4235
5254
  resolveGhcrToken,
4236
5255
  resolveGhcrPushToken,
4237
5256
  probeUsableEditorImage,
5257
+ httpsManifestGet,
4238
5258
  resolveProject,
4239
5259
  clientProjectBootstrapArgs,
4240
5260
  cloudConfigPath,
@@ -4245,6 +5265,8 @@ module.exports = {
4245
5265
  buildReconcileAdminCommand,
4246
5266
  cmdReconcileAdmin,
4247
5267
  buildEditorLocalEnvPatch,
5268
+ buildGreenfieldTitlePatch,
5269
+ mergeEditorLocalPatchB64,
4248
5270
  buildEditorLocalSeedWriteCommand,
4249
5271
  seedPreviewOrigin,
4250
5272
  healPreviewOriginAtReady,
@@ -4258,6 +5280,22 @@ module.exports = {
4258
5280
  decideBootstrapOutcome,
4259
5281
  projectBootstrapDoneMarker,
4260
5282
  projectBootstrapDoneProbeCommand,
5283
+ projectBootstrapScriptPath,
5284
+ projectBootstrapRunnerPath,
5285
+ projectBootstrapLogPath,
5286
+ projectBootstrapPidPath,
5287
+ projectBootstrapStatusPath,
5288
+ projectBootstrapRunnerScript,
5289
+ projectBootstrapLaunchCommand,
5290
+ projectBootstrapFollowCommand,
5291
+ projectBootstrapStatusCommand,
5292
+ parseProjectBootstrapStatus,
5293
+ decideProjectBootstrapOutcome,
5294
+ decideProjectBootstrapFollowAction,
5295
+ projectBootstrapEnv,
5296
+ PROJECT_BOOTSTRAP_LAUNCHED_MARKER,
5297
+ PROJECT_BOOTSTRAP_FOLLOW_SECS,
5298
+ ensureProjectBootstrapped,
4261
5299
  SSH_KEEPALIVE_FLAGS,
4262
5300
  resolveSetupForwardEnv,
4263
5301
  FORWARD_ENV_KEYS_ENV,
@@ -4272,6 +5310,8 @@ module.exports = {
4272
5310
  checkScriptDrift,
4273
5311
  streamedScriptEditorBranch,
4274
5312
  streamedScriptMergeBase,
5313
+ fetchStreamedScriptRef,
5314
+ resolveRemoteScriptRev,
4275
5315
  gatherScriptDriftFacts,
4276
5316
  readStreamedScript,
4277
5317
  STREAMED_SCRIPTS,
@@ -4298,6 +5338,8 @@ module.exports = {
4298
5338
  cmdIngressUp,
4299
5339
  cmdReconcileIngress,
4300
5340
  sweepOrphanIngress,
5341
+ convergeFleetFirewall,
5342
+ reportFirewallPortGaps,
4301
5343
  cmdReconcileCredentials,
4302
5344
  cmdAccess,
4303
5345
  cloudMain,
@@ -4305,12 +5347,14 @@ module.exports = {
4305
5347
  seedLiveCredentialCache,
4306
5348
  assertGitPushCredentials,
4307
5349
  seedGithubCredentials,
5350
+ githubCredentialLine,
4308
5351
  resolveGithubToken,
4309
5352
  resolveGithubTokenWithSource,
4310
5353
  MissingGitPushCredentialsError,
4311
5354
  classifyGitRemoteError,
4312
5355
  decideCredentialHeal,
4313
5356
  readVmAppPort,
5357
+ buildVmAppPortReadCommand,
4314
5358
  resolveLaptopGitIdentity,
4315
5359
  };
4316
5360