@codeyam-editor/codeyam-editor 0.1.0-staging.gac4ca32 → 0.1.0-staging.gac53b4e

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. package/npm/cloud.js +1773 -177
  2. package/npm/codesign-plan.js +68 -0
  3. package/npm/compose-conflict.js +59 -0
  4. package/npm/editor-dev.js +24 -2
  5. package/npm/editor.js +15 -5
  6. package/npm/empty-project-dir.js +92 -7
  7. package/npm/fleet-add-failure.js +742 -0
  8. package/npm/fleet-auth-account.js +61 -0
  9. package/npm/fleet-auth-health.js +121 -5
  10. package/npm/fleet-auth.js +69 -1
  11. package/npm/fleet-branch.js +124 -1
  12. package/npm/fleet-build-data.js +62 -4
  13. package/npm/fleet-cleanup.js +302 -1
  14. package/npm/fleet-datastore-health.js +107 -0
  15. package/npm/fleet-destroy-safety.js +230 -13
  16. package/npm/fleet-early-roster.js +87 -0
  17. package/npm/fleet-effect-poll.js +110 -0
  18. package/npm/fleet-env.js +86 -0
  19. package/npm/fleet-ghcr-secret.js +129 -0
  20. package/npm/fleet-git-maintenance.js +142 -0
  21. package/npm/fleet-host-op-limiter.js +68 -0
  22. package/npm/fleet-iap-token.js +165 -0
  23. package/npm/fleet-job-badge.js +28 -12
  24. package/npm/fleet-memory.js +306 -8
  25. package/npm/fleet-plans.js +81 -3
  26. package/npm/fleet-pool.js +75 -16
  27. package/npm/fleet-prewarm.js +87 -11
  28. package/npm/fleet-probe-transport.js +155 -0
  29. package/npm/fleet-project-drift.js +58 -0
  30. package/npm/fleet-public-url.js +208 -19
  31. package/npm/fleet-queue.js +199 -1
  32. package/npm/fleet-reachability.js +80 -0
  33. package/npm/fleet-reconcile.js +46 -11
  34. package/npm/fleet-revision.js +212 -0
  35. package/npm/fleet-run-setup.js +66 -0
  36. package/npm/fleet-running-plan.js +37 -7
  37. package/npm/fleet-session-activity.js +162 -0
  38. package/npm/fleet-session-token-cache.js +65 -0
  39. package/npm/fleet-status-refs.js +96 -0
  40. package/npm/fleet-strand.js +39 -8
  41. package/npm/fleet-vm-access.js +276 -7
  42. package/npm/fleet-vm-probe-target.js +282 -0
  43. package/npm/fleet-vm-readiness.js +78 -20
  44. package/npm/group-bounded-spawn.js +111 -0
  45. package/npm/ingress-invoke.js +272 -10
  46. package/npm/launch-config.js +494 -29
  47. package/npm/orphan-ssh-reaper.js +81 -0
  48. package/npm/scenario-check.js +1013 -34
  49. package/npm/scenario-handlers.js +103 -27
  50. package/npm/scenario-interactivity.js +612 -19
  51. package/npm/scenario-issues.js +85 -1
  52. package/npm/scenario-mocks.js +340 -31
  53. package/npm/scenario-playwright.js +562 -42
  54. package/npm/utils.js +298 -24
  55. package/package.json +5 -5
  56. package/ui/dist/assets/{ContainerInspector-urDZ9zOC.js → ContainerInspector-DiN91foX.js} +1 -1
  57. package/ui/dist/assets/EditorShell-BEIdHnHd.css +1 -0
  58. package/ui/dist/assets/EditorShell-CPV5vhjo.js +330 -0
  59. package/ui/dist/assets/{Markdown-BoLHHFXH.js → Markdown-Chgdf3PW.js} +1 -1
  60. package/ui/dist/assets/ProjectLauncherScreen-E0NYYiwy.js +1 -0
  61. package/ui/dist/assets/ScenarioDataPanel-DHAHyraq.js +18 -0
  62. package/ui/dist/assets/{ScenarioDataPanelFullPage-DlAIUrnk.js → ScenarioDataPanelFullPage-CHIWip3n.js} +1 -1
  63. package/ui/dist/assets/ScenarioRenderer-D9a91iwS.js +21 -0
  64. package/ui/dist/assets/{StepsSimulator-BFldWyoX.js → StepsSimulator-CymGgfbu.js} +1 -1
  65. package/ui/dist/assets/index--cE0hUJw.css +1 -0
  66. package/ui/dist/assets/index-sW8wcg7v.js +27 -0
  67. package/ui/dist/assets/{markdown-v0F0UEt3.js → markdown-C9tbsVHX.js} +1 -1
  68. package/ui/dist/assets/react-nrLBr15I.js +28 -0
  69. package/ui/dist/assets/{useEvents-DHkLJZR4.js → useEvents-S3Qk1Y_d.js} +1 -1
  70. package/ui/dist/assets/useServerIdentity-CVuHTzjg.js +1 -0
  71. package/ui/dist/index.html +4 -4
  72. package/ui/dist/open-source/codeyam-counter.png +0 -0
  73. package/ui/dist/open-source/el-carot.png +0 -0
  74. package/ui/dist/open-source/tabcommand.png +0 -0
  75. package/ui/dist/preview-subdir-demo/index.html +62 -0
  76. package/ui/dist/preview-subdir-demo/sibling.html +48 -0
  77. package/ui/dist/assets/EditorShell-D7wGyetm.css +0 -1
  78. package/ui/dist/assets/EditorShell-eZm_8CsG.js +0 -230
  79. package/ui/dist/assets/ProjectLauncherScreen-Bq8CJo9M.js +0 -1
  80. package/ui/dist/assets/ScenarioDataPanel-DuU3_hdh.js +0 -5
  81. package/ui/dist/assets/ScenarioRenderer-od1vF8-p.js +0 -21
  82. package/ui/dist/assets/index-B7_KRKaQ.js +0 -27
  83. package/ui/dist/assets/index-D3csSagX.css +0 -1
  84. package/ui/dist/assets/react-CSS0HapR.js +0 -28
package/npm/cloud.js CHANGED
@@ -11,6 +11,7 @@
11
11
 
12
12
  const { spawn, spawnSync } = require("child_process");
13
13
  const fs = require("fs");
14
+ const https = require("https");
14
15
  const os = require("os");
15
16
  const path = require("path");
16
17
  const readline = require("readline");
@@ -22,6 +23,11 @@ const {
22
23
  killChildProcess,
23
24
  } = require("./utils");
24
25
  const { open } = require("./open");
26
+ // Bounded gcloud/ssh spawns run through spawnGroupBoundedSync so a timeout takes
27
+ // gcloud's ssh descendant down with it instead of orphaning it to PID 1; the
28
+ // reaper sweeps sessions already leaked before that existed.
29
+ const { spawnGroupBoundedSync } = require("./group-bounded-spawn");
30
+ const { reapOrphanedFleetSsh } = require("./orphan-ssh-reaper");
25
31
  // `FORWARD_ENV_KEYS_ENV` is the env-var name (`CY_FORWARD_ENV_KEYS`) the
26
32
  // fleet dashboard sets on `cloud:up`'s child to name the subset of
27
33
  // `process.env` keys that came from the operator-supplied launch form.
@@ -58,6 +64,7 @@ const {
58
64
  normalizePrincipal,
59
65
  resolveDefaultOperator,
60
66
  withDefaultOperator,
67
+ unauthorizableAddRefusal,
61
68
  accessFor,
62
69
  setAccess,
63
70
  addAccess,
@@ -75,9 +82,11 @@ const {
75
82
  regionFromZone,
76
83
  vmIngressInvocation,
77
84
  vmIngressStatusInvocation,
85
+ vmIngressGrantAccessInvocation,
78
86
  parseIngressStatusVerdict,
79
87
  parseIngressTeardownVerdict,
80
88
  selectIngressReconcileTargets,
89
+ selectIngressGrantOnlyTargets,
81
90
  selectOrphanIngressTargets,
82
91
  classifyOrphanDownResult,
83
92
  backendServicesListInvocation,
@@ -90,6 +99,11 @@ const {
90
99
  iapAccessInvocation,
91
100
  sharedIngressInvocation,
92
101
  backendServiceTimeoutInvocation,
102
+ FLEET_FIREWALL_RULE_NAMES,
103
+ fleetFirewallDescribeInvocation,
104
+ fleetFirewallConvergeInvocation,
105
+ firewallNeedsConverge,
106
+ firewallAdmitsPort,
93
107
  } = require("./ingress-invoke");
94
108
  // Shared fleet-roster writer/reader. `cloud:up` rosters the VM it just
95
109
  // provisioned so the dashboard, the cloudflared-monitor tunnel publisher, and
@@ -97,6 +111,10 @@ const {
97
111
  // edit (the invisible-VM gap improve38 closes). `readRoster` enumerates the
98
112
  // fleet for `reconcile-ingress`.
99
113
  const { addToRoster, readRoster } = require("./fleet-roster");
114
+ // Tells a docker container-name conflict (a concurrent run already started this
115
+ // container) apart from a real `docker compose up -d` failure, so the abort path
116
+ // can converge instead of escalating to instance deletion.
117
+ const { classifyComposeUpFailure } = require("./compose-conflict");
100
118
 
101
119
  // =============================================================================
102
120
  // Pure helpers — every export here is unit-tested in cloud.test.js without
@@ -159,14 +177,34 @@ const KNOWN_FLAGS = [
159
177
  // default is loud — a tokenless VM cannot sync plan claims and must never
160
178
  // report healthy silently. Use only when push genuinely isn't wanted.
161
179
  "allow-no-git-credentials",
180
+ // Provision the VM in admin mode: forward CODEYAM_ADMIN=1 into the container
181
+ // so the Home-tab Setup door renders. Equivalent to `CODEYAM_ADMIN=1 npm run
182
+ // cloud:up …`; absent = strict non-admin (the safe default). Survives
183
+ // re-provision because it rides the compose-render substitution, unlike a
184
+ // manual VM compose edit.
185
+ "admin",
186
+ // GCE machine type for the VM (e.g. n2-standard-4, e2-standard-4). Persisted
187
+ // in cloud.json so a later bare `cloud:up` keeps it. Defaults to
188
+ // DEFAULT_FLEET_MACHINE_TYPE. The whole point of exposing this is fleet
189
+ // scale-out: the region's E2_CPUS quota (72 in us-central1) caps the fleet at
190
+ // ~14 e2-standard-4 VMs, while N2_CPUS/N2D_CPUS (200, unused) leave room for
191
+ // 30 — so the fleet defaults to N2 and this flag lets a caller override.
192
+ "machine-type",
162
193
  ];
163
194
 
164
195
  // Flags that take no value (presence = true). parseFlags must not demand a
165
196
  // value for these or `--no-tunnel` would error "requires a value".
166
- const BOOLEAN_FLAGS = ["no-tunnel", "skip-setup", "list", "allow-no-git-credentials"];
197
+ const BOOLEAN_FLAGS = ["no-tunnel", "skip-setup", "list", "allow-no-git-credentials", "admin"];
167
198
 
168
199
  const FIREWALL_RULE_NAME = "codeyam-editor-iap-ssh";
169
200
  const INSTANCE_TAG = "codeyam-editor";
201
+ // Default GCE machine type for fleet VMs. N2 (not E2) because the fleet must
202
+ // reach 30 VMs: us-central1 E2_CPUS quota is 72, so 30 × e2-standard-4 (120
203
+ // vCPU) + the operator host exceed it and cap the fleet at ~14 regardless of
204
+ // code — while N2_CPUS/N2D_CPUS are 200 and unused, leaving headroom for 30 ×
205
+ // n2-standard-4. Override per-VM with `--machine-type`. See the vm--scale-
206
+ // fleet-to-30-vms plan for the measured quota rationale.
207
+ const DEFAULT_FLEET_MACHINE_TYPE = "n2-standard-4";
170
208
  const EDITOR_PORT = 14199;
171
209
  const VITE_PORT = 5173;
172
210
 
@@ -481,6 +519,28 @@ function previewSameOriginSafeFromStack(stackJson) {
481
519
  return typeof flag === "boolean" ? flag : true;
482
520
  }
483
521
 
522
+ /** Resolve the sameOriginSafe flag a seed should ACT on, given the static
523
+ * `stack.json` reading and the running editor's own `/api/config` verdict.
524
+ *
525
+ * Precedence: a BOOLEAN effective verdict wins; anything else (`null`,
526
+ * `undefined`, a non-boolean) defers to the static flag. The type check is the
527
+ * whole rule — `null` means "no server was up to ask", which is the
528
+ * pre-container seed and must leave the static read in charge, NOT "the server
529
+ * said same-origin is fine". Collapsing the two would put the static flag back
530
+ * in control on exactly the path that needs the override.
531
+ *
532
+ * Extracted from `seedPreviewOrigin` because that function's inputs arrive over
533
+ * gcloud SSH, so this rule was only reachable through heavy mocking — while
534
+ * being precisely where the defect lived. `stack.json` cannot express the state
535
+ * that matters here: a project escalates off the same-origin subpath at RUNTIME
536
+ * after a hydration failure, and the adaptive-default stacks that happens to
537
+ * (Next.js) deliberately omit the key. So a seeder that re-derives the flag
538
+ * from the file writes back the same-origin sentinel that just failed, and the
539
+ * project sits at `previewDedicatedOriginUnavailable: true` forever. Pure. */
540
+ function resolveEffectiveSameOriginSafe(staticFlag, effectiveVerdict) {
541
+ return typeof effectiveVerdict === "boolean" ? effectiveVerdict : staticFlag;
542
+ }
543
+
484
544
  // Matches an absolute cloudflared quick-tunnel origin
485
545
  // (`https://<sub>.trycloudflare.com`, optionally with a port/path/query). These
486
546
  // URLs ROTATE on every tunnel respawn, which is exactly why a frozen one rots.
@@ -550,6 +610,32 @@ function isWrongModePreviewOrigin(localJson, sameOriginSafe) {
550
610
  return LOOPBACK_ORIGIN_RE.test(origin);
551
611
  }
552
612
 
613
+ /** Decide whether a VM is presenting as HEALTHY while its Live Preview is
614
+ * silently dead — the VM-9 failure mode. The heal
615
+ * (`reconcilePreviewOriginToSlugHost`) fixes the mechanism; this predicate is
616
+ * the smoke detector that catches a FUTURE mechanism gap (a new stack shape, a
617
+ * new provisioning path) by observation instead of by a user noticing a blank
618
+ * iframe days later.
619
+ *
620
+ * Input is the editor's own `/api/config` payload — the effective truth table
621
+ * (`previewSameOriginSafe` + the resolved `previewOrigin`), not a
622
+ * re-derivation from stack.json over ssh. True iff the stack is NOT
623
+ * same-origin-safe AND its effective previewOrigin is one a remote browser can
624
+ * never load (the `/__codeyam_preview` sentinel, any relative path, or a
625
+ * loopback origin) — i.e. exactly `isWrongModePreviewOrigin`'s wrong-mode
626
+ * condition, evaluated against the live config rather than editor.local.json.
627
+ * Composing the two keeps ONE definition of "wrong-mode origin" in this file.
628
+ *
629
+ * Deliberately conservative: a missing/unparseable payload, an absent
630
+ * `previewSameOriginSafe`, or a legitimate same-origin-safe stack on the
631
+ * subpath all return false. Absence of evidence is never a false alarm — the
632
+ * caller renders "unknown" instead. Pure. */
633
+ function isSilentlyBrokenPreview(apiConfig) {
634
+ if (!apiConfig || typeof apiConfig !== "object") return false;
635
+ if (apiConfig.previewSameOriginSafe !== false) return false;
636
+ return isWrongModePreviewOrigin(apiConfig, false);
637
+ }
638
+
553
639
  /** Read the operator-laptop cloudflared-urls.json sidecar. Missing/unparseable
554
640
  * → {} (the monitor may not have run yet; the seed falls back to localhost). */
555
641
  function readCloudflaredUrls() {
@@ -747,6 +833,28 @@ function buildEditorLocalEnvPatch(localEnv, agent) {
747
833
  return { provider: agent, env: { [varName]: value } };
748
834
  }
749
835
 
836
+ /** Build the editor.local.json patch that seeds a GREENFIELD project's
837
+ * `projectTitle` so its live identity resolves to the real project name from
838
+ * first boot. A greenfield project (`--new-project <name>`, no clientRepo) has
839
+ * no origin remote and is bind-mounted at /workspace on the VM, so the editor's
840
+ * identity resolver would otherwise fall back to the bare mount basename
841
+ * "workspace" — which the fleet dashboard then reads as a permanent false
842
+ * project-dir-drift. Seeding `projectTitle` (deep-merged over editor.json on the
843
+ * VM) makes `project_identity::read` prefer the real name.
844
+ *
845
+ * Returns `null` for a clone (a repo Add already labels from its origin
846
+ * basename, and re-seeding a clone's title would be wrong) or when no
847
+ * greenfield name is available, so the caller can no-op without an SSH round
848
+ * trip. Pure — mirrors buildEditorLocalEnvPatch so the decision is unit-tested
849
+ * without a live VM. */
850
+ function buildGreenfieldTitlePatch(cfg = {}) {
851
+ const p = resolveProject(cfg);
852
+ if (!p.greenfield) return null;
853
+ const name = typeof p.name === 'string' ? p.name.trim() : '';
854
+ if (!name) return null;
855
+ return { projectTitle: name };
856
+ }
857
+
750
858
  /** Parse `docker compose ps --format json` (compose v2 — one JSON object per
751
859
  * line) and return the editor service's `State` string, or null when the
752
860
  * service is absent / output is unparseable. Two callers shared an identical
@@ -769,6 +877,79 @@ function parseEditorComposeState(stdout) {
769
877
  return null;
770
878
  }
771
879
 
880
+ /** Decide whether `cloud:up`'s abort handler may delete the instance it is
881
+ * unwinding. Pure `{ createdThisRun, containerUp, rostered, probeInconclusive }
882
+ * → 'delete' | 'keep'`, matching the `reapVerdict` / `reattachVerdict` /
883
+ * `reconcileRosteredVm` pattern in npm/fleet-cleanup.js, so the whole matrix is
884
+ * exhaustively testable without a VM.
885
+ *
886
+ * `createdThisRun` stays a NECESSARY condition — never delete an instance this
887
+ * run did not create — but 2026-08-07 proved it is not SUFFICIENT. It answers
888
+ * "did I create this?", not "is it safe to destroy?", and in a concurrent
889
+ * provision race the losing run legitimately owns the flag while the winner has
890
+ * already brought the VM up and rostered it. Deleting there destroyed VM-4 and
891
+ * VM-5. So authorization also requires CURRENT-STATE evidence that the VM is
892
+ * out of service:
893
+ *
894
+ * - `containerUp` — the editor container is running (per the compose-ps probe).
895
+ * A VM serving an editor is in use by definition.
896
+ * - `rostered` — the VM is in the fleet roster, the operator-visible source of
897
+ * truth. Note this is true whether the winning run or THIS run rostered it;
898
+ * either way the VM is visible in the dashboard and removable in one command,
899
+ * so it is not the silent orphan teardown exists to prevent.
900
+ * - `probeInconclusive` — liveness could not be determined. This must fail
901
+ * SAFE: a failing probe is how we reach the abort handler in the first place,
902
+ * so it can never count as evidence the VM is dead. An orphan costs one
903
+ * `npm run cloud:destroy`; a wrong delete costs a 15-40 minute rebuild plus
904
+ * any unpushed work on the VM. */
905
+ function abortTeardownDecision({ createdThisRun, containerUp, rostered, probeInconclusive } = {}) {
906
+ if (!createdThisRun) return "keep";
907
+ if (probeInconclusive) return "keep";
908
+ if (containerUp) return "keep";
909
+ if (rostered) return "keep";
910
+ return "delete";
911
+ }
912
+
913
+ /** Gather the CURRENT-STATE inputs {@link abortTeardownDecision} needs, so the
914
+ * abort handler reads gather → decide → act instead of inlining probes among
915
+ * the delete calls.
916
+ *
917
+ * Only runs the probes when `createdThisRun` is true — the decision is `keep`
918
+ * regardless otherwise, and an abort path should not spend SSH round trips
919
+ * proving something it already knows. In that case it returns the
920
+ * probe-inconclusive shape, which is the safe default at every layer.
921
+ *
922
+ * Both lookups are wrapped: a throwing probe or an unreadable roster must fail
923
+ * SAFE (toward `keep`), never take a second exception path into the delete.
924
+ * `deps` is injectable so the matrix is unit-tested without a VM. */
925
+ function probeAbortTeardownState(cfg, createdThisRun, deps = {}) {
926
+ if (!createdThisRun) {
927
+ return { containerUp: false, rostered: false, probeInconclusive: true };
928
+ }
929
+ const probe = deps.probeEditorComposeState || probeEditorComposeState;
930
+ const roster = deps.readRoster || readRoster;
931
+ const resolveVm = deps.resolveFleetVmNumber || resolveFleetVmNumber;
932
+ const log = deps.log || ((m) => console.error(m));
933
+
934
+ let liveness = { state: null, inconclusive: true };
935
+ try {
936
+ liveness = probe(cfg);
937
+ } catch (e) {
938
+ log(` (liveness probe failed: ${e && e.message})`);
939
+ }
940
+ let rostered = false;
941
+ try {
942
+ const vm = resolveVm(cfg.instanceName);
943
+ rostered = vm != null && (roster() || []).includes(vm);
944
+ } catch { /* unreadable roster — treated as not-rostered; liveness still guards */ }
945
+
946
+ return {
947
+ containerUp: liveness.state === "running",
948
+ rostered,
949
+ probeInconclusive: liveness.inconclusive,
950
+ };
951
+ }
952
+
772
953
  /** Translate one of the wrapper's verbs into the exact `gcloud` argv. The
773
954
  * switch's job is to keep every per-verb command shape in one reviewable
774
955
  * place — tests assert the arrays without ever spawning a process. */
@@ -789,7 +970,7 @@ function buildGcloudArgs(verb, cfg = {}, extras = {}) {
789
970
  return [
790
971
  "compute", "instances", "create", instanceName,
791
972
  "--project", project, "--zone", zone,
792
- "--machine-type=e2-standard-4",
973
+ `--machine-type=${cfg.machineType || DEFAULT_FLEET_MACHINE_TYPE}`,
793
974
  "--image-family=debian-12",
794
975
  "--image-project=debian-cloud",
795
976
  "--boot-disk-size=100GB",
@@ -804,6 +985,19 @@ function buildGcloudArgs(verb, cfg = {}, extras = {}) {
804
985
  return ["compute", "instances", "delete", instanceName, "--project", project, "--zone", zone, "--quiet"];
805
986
  case "instances-describe":
806
987
  return ["compute", "instances", "describe", instanceName, "--project", project, "--zone", zone, "--format=json"];
988
+ // Outstanding (not-yet-DONE) zone operations targeting this instance. Used to
989
+ // tell a normal transient PROVISIONING from a wedged insert: GCE can accept an
990
+ // insert and then never finish it, leaving an operation parked at RUNNING/0%
991
+ // that holds the instance name forever. The status filter is what makes this
992
+ // cheap and unambiguous — a DONE operation is history and says nothing about
993
+ // whether the slot is currently blocked.
994
+ case "operations-list-pending":
995
+ return [
996
+ "compute", "operations", "list",
997
+ "--project", project,
998
+ "--filter", `zone:(${zone}) AND targetLink~${instanceName}$ AND status!=DONE`,
999
+ "--format=json",
1000
+ ];
807
1001
  case "instances-list-tagged":
808
1002
  return [
809
1003
  "compute", "instances", "list",
@@ -889,11 +1083,98 @@ function buildGcloudArgs(verb, cfg = {}, extras = {}) {
889
1083
  // real use. Unit tests pin their exports so a refactor cannot drop them.
890
1084
  // =============================================================================
891
1085
 
1086
+ // Per-call wall-clock budget for a single gcloud / SSH invocation. A hung
1087
+ // `gcloud describe` (metadata / IAP hiccup) used to block the whole
1088
+ // `cloud:reconcile-ingress` sweep forever — the only timeout in the reconcile
1089
+ // path was the outer 10-minute exec budget in the dashboard, and that never
1090
+ // cascaded to the orphaned descendants. Bounding every per-call spawn well under
1091
+ // that outer budget means a single hang trips HERE first: the call returns a
1092
+ // timed-out result (formatGcloudFailure renders it), the sweep skips that VM,
1093
+ // and no descendant is left hanging to pile up. Shared by runGcloud and the two
1094
+ // reconcile spawnSync sites (status probe, per-BES timeout converge) so all three
1095
+ // reference one value. 2 min is generous for any single describe/update/SSH probe
1096
+ // yet comfortably under the 600000 ms outer reconcile budget.
1097
+ //
1098
+ // All three spawn through spawnGroupBoundedSync, because a bare spawnSync timeout
1099
+ // signals only the direct child: gcloud died and its `ssh` survived, re-parented
1100
+ // to PID 1 — the budget bounded the wrapper, never the descendant.
1101
+ //
1102
+ // It is a PROBE budget, and provisioning is exempt — see
1103
+ // GCLOUD_PROVISION_TIMEOUT_MS below. Do not re-collapse the two: raising this
1104
+ // constant to accommodate a slow create would un-bound the reconcile sweep,
1105
+ // which is the exact orphan-cascade this value exists to stop.
1106
+ const GCLOUD_CALL_TIMEOUT_MS = 120000;
1107
+
1108
+ // `instances create` / `instances start` are NOT probes: each blocks until GCE's
1109
+ // insert (or start) operation reaches a terminal state, which routinely outruns
1110
+ // the probe budget under zonal contention. Inheriting GCLOUD_CALL_TIMEOUT_MS
1111
+ // SIGTERMd gcloud mid-insert; `spawnSync` then returned `status: null`, which
1112
+ // `!== 0`, so a local kill was indistinguishable from a remote rejection and the
1113
+ // caller threw a bare `Failed to create instance <name>` with no cause in it.
1114
+ // That mattered downstream: every arm of classifyAddFailure keys on a transcript
1115
+ // signature, so a causeless log fell through to the optimistic `verifying`
1116
+ // fallback and told the operator to WAIT on a VM GCE had already wedged
1117
+ // (observed 2026-09-04 on VM-8: insert 14:29:50Z, add errored 14:32:26Z — the
1118
+ // 120s budget plus teardown).
1119
+ //
1120
+ // Deliberately above the 600000 ms outer reconcile budget, on the same reasoning
1121
+ // as PROJECT_BOOTSTRAP_TIMEOUT_MS: ensureInstance is only ever reached from the
1122
+ // add path, never from the reconcile sweep, so the sweep-relative ceiling that
1123
+ // governs GCLOUD_CALL_TIMEOUT_MS does not apply. A create still running at 10
1124
+ // minutes is wedged, not slow.
1125
+ const GCLOUD_PROVISION_TIMEOUT_MS = 10 * 60 * 1000;
1126
+
1127
+ /** The per-call spawn budget a given wrapper verb should run under. Provisioning
1128
+ * verbs get GCLOUD_PROVISION_TIMEOUT_MS; everything else keeps the probe budget.
1129
+ * Pure — the mapping is asserted directly, without spawning a process. */
1130
+ function gcloudCallTimeoutMs(verb) {
1131
+ return verb === "instances-create" || verb === "instances-start"
1132
+ ? GCLOUD_PROVISION_TIMEOUT_MS
1133
+ : GCLOUD_CALL_TIMEOUT_MS;
1134
+ }
1135
+
1136
+ // The project bootstrap is NOT a probe: it runs `git clone` + `npm install` in
1137
+ // the repo root and every `apps[].dir`, each package's `postinstall`, then
1138
+ // `npm run setup` — minutes of remote work on a heavy repo. It once ran as ONE
1139
+ // blocking `runGcloud("ssh-script")` call, so it inherited GCLOUD_CALL_TIMEOUT_MS
1140
+ // (a budget whose own comment scopes it to "any single describe/update/SSH
1141
+ // probe"). That SIGTERMd the gcloud child mid-clone; `spawnSync` then returned
1142
+ // `status: null`, which `!== 0`, so the timeout was indistinguishable from a
1143
+ // genuine remote failure and the caller's abort path DELETED A COMPLETELY
1144
+ // HEALTHY VM.
1145
+ //
1146
+ // Observed twice: 2026-07-28 on client repo `lets-go-compost-portal` (a root
1147
+ // postinstall of `playwright install chromium`, ~293 MB), and 2026-07-29
1148
+ // provisioning six self-host `codeyam-editor` VMs, whose clone alone exceeds the
1149
+ // probe budget — all six were built, torn down mid-clone, and reported as
1150
+ // "scripts/gcp-project-bootstrap.sh failed on the VM".
1151
+ //
1152
+ // The install is now DETACHED (projectBootstrapLaunchCommand + a re-attaching
1153
+ // follow loop, outcome read from VM-side state), so no single call carries the
1154
+ // whole install any more. This constant is therefore the OVERALL deadline across
1155
+ // re-attaches — the point at which we stop waiting on a still-healthy VM and say
1156
+ // exactly that. Deliberately well above the outer reconcile budget: a long remote
1157
+ // install is not a step in a sweep, so the sweep-relative ceiling that governs
1158
+ // GCLOUD_CALL_TIMEOUT_MS does not apply to it.
1159
+ const PROJECT_BOOTSTRAP_TIMEOUT_MS = 40 * 60 * 1000;
1160
+
1161
+ /** Whether a `spawnSync` result is a LOCAL timeout kill rather than a remote
1162
+ * failure. `spawnSync` reports a timeout as `status: null` plus an ETIMEDOUT
1163
+ * error, so a bare `status !== 0` check conflates the two — which is what made
1164
+ * a timed-out bootstrap look like a broken one. Pure. */
1165
+ function isSpawnTimeout(result) {
1166
+ if (!result) return false;
1167
+ if (result.status !== null && result.status !== undefined) return false;
1168
+ const code = result.error && result.error.code;
1169
+ return code === "ETIMEDOUT" || result.signal === "SIGTERM";
1170
+ }
1171
+
892
1172
  function runGcloud(args, opts = {}) {
893
- return spawnSync("gcloud", args, {
1173
+ return spawnGroupBoundedSync("gcloud", args, {
894
1174
  stdio: opts.stdio || "inherit",
895
1175
  input: opts.input,
896
1176
  encoding: opts.encoding || "utf8",
1177
+ timeout: opts.timeout != null ? opts.timeout : GCLOUD_CALL_TIMEOUT_MS,
897
1178
  });
898
1179
  }
899
1180
 
@@ -924,6 +1205,66 @@ function formatGcloudFailure(result) {
924
1205
  return "(no output captured from gcloud)";
925
1206
  }
926
1207
 
1208
+ /** The thrown message for a failed `instances create` / `instances start`.
1209
+ *
1210
+ * Splits the two outcomes the old bare `Failed to create instance <name>`
1211
+ * collapsed, because they are different operator actions: a LOCAL timeout means
1212
+ * GCE may still be working the operation and it should be checked, while a
1213
+ * remote rejection means GCE said no and its own stderr is the answer. Both
1214
+ * arms put a cause in the transcript — classifyAddFailure keys on transcript
1215
+ * text, so a causeless message is what lands an add on the optimistic
1216
+ * `verifying` fallback. Pure: result in, string out.
1217
+ *
1218
+ * `verb` is the operator-facing word ("create" / "start"), not a wrapper verb. */
1219
+ /** Operator-facing copy for a create GCE accepted and then abandoned, kept
1220
+ * separate from the guard that detects it for the same reason
1221
+ * provisionFailureMessage is: the decision of WHEN to refuse and the wording of
1222
+ * WHAT to say are different concerns, and only the wording needs asserting.
1223
+ * Extracting it is what lets the copy be tested without a live gcloud — and on
1224
+ * this feature the copy IS the fix, since the bug was a message that told the
1225
+ * operator to wait for a VM that would never arrive.
1226
+ *
1227
+ * Every field is rendered defensively because this runs on a degraded path: the
1228
+ * operations read that produced `pending` is best-effort, so a partial record
1229
+ * must still yield a message that names the problem rather than printing
1230
+ * "undefined" at the operator. */
1231
+ function wedgedOperationMessage(instanceName, pending = {}) {
1232
+ const detail = [
1233
+ `${pending.name || "unnamed"}: ${pending.operationType || "unknown type"}`,
1234
+ `status ${pending.status || "unknown"}`,
1235
+ `progress ${pending.progress == null ? "unknown" : `${pending.progress}%`}`,
1236
+ `error ${pending.errorCode || "unknown"}${pending.errorMessage ? `: ${pending.errorMessage}` : ""}`,
1237
+ ].join(", ");
1238
+ return (
1239
+ `Instance ${instanceName} is stuck in PROVISIONING behind a wedged create operation (${detail}). ` +
1240
+ "GCE accepted the create and never finished it, so this slot cannot be re-added and a delete will queue " +
1241
+ "behind the wedged operation rather than clear it. This needs GCE-side resolution — open a Google Cloud " +
1242
+ "Support case citing that error. Use a different VM number if you need a box now."
1243
+ );
1244
+ }
1245
+
1246
+ function provisionFailureMessage(verb, instanceName, result) {
1247
+ if (isSpawnTimeout(result)) {
1248
+ const minutes = Math.round(GCLOUD_PROVISION_TIMEOUT_MS / 60000);
1249
+ // A SIGTERMd child legitimately often captured nothing, and appending
1250
+ // formatGcloudFailure's "(no output captured)" placeholder to a message that
1251
+ // already states the cause is noise. Append it only when it is real output.
1252
+ const detail = formatGcloudFailure(result);
1253
+ const captured = detail === "(no output captured from gcloud)" ? "" : `\n${detail}`;
1254
+ return (
1255
+ `Failed to ${verb} instance ${instanceName}: the gcloud call exceeded the local ` +
1256
+ `${minutes}-minute provisioning budget and was SIGTERMd locally. GCE may still be ` +
1257
+ `working the operation — check it with \`gcloud compute operations list\` before ` +
1258
+ `retrying, and delete the half-built instance if one exists.` +
1259
+ captured
1260
+ );
1261
+ }
1262
+ return (
1263
+ `Failed to ${verb} instance ${instanceName} (gcloud exit ${result && result.status}).\n` +
1264
+ formatGcloudFailure(result)
1265
+ );
1266
+ }
1267
+
927
1268
  // A bounded, pure classifier that keys on status === 255 (gcloud-ssh's exit
928
1269
  // for an ssh failure) and a signature match against known transient strings.
929
1270
  function isTransientSshFailure(result) {
@@ -999,6 +1340,38 @@ function fetchInstanceState(cfg) {
999
1340
  }
1000
1341
  }
1001
1342
 
1343
+ /** Read the instance's outstanding (not-DONE) zone operations and summarize the
1344
+ * first one, or null when there are none / the call failed. Shaped as
1345
+ * { name, operationType, status, progress, errorCode, errorMessage } — the fields
1346
+ * the wedged-insert refusal message and the fleet classifier both need.
1347
+ *
1348
+ * Returning null on a failed or unparseable call is deliberate: this is a
1349
+ * DIAGNOSTIC read used to make a refusal more informative, so it must never turn
1350
+ * a working provision into a failing one just because the operations API hiccuped. */
1351
+ function fetchPendingOperation(cfg) {
1352
+ const result = runGcloud(buildGcloudArgs("operations-list-pending", cfg), {
1353
+ stdio: ["ignore", "pipe", "ignore"],
1354
+ });
1355
+ if (result.status !== 0) return null;
1356
+ let ops;
1357
+ try {
1358
+ ops = JSON.parse(result.stdout);
1359
+ } catch {
1360
+ return null;
1361
+ }
1362
+ if (!Array.isArray(ops) || ops.length === 0) return null;
1363
+ const op = ops[0];
1364
+ const err = op && op.error && Array.isArray(op.error.errors) ? op.error.errors[0] : null;
1365
+ return {
1366
+ name: op.name || null,
1367
+ operationType: op.operationType || null,
1368
+ status: op.status || null,
1369
+ progress: typeof op.progress === "number" ? op.progress : null,
1370
+ errorCode: (err && err.code) || null,
1371
+ errorMessage: (err && err.message) || null,
1372
+ };
1373
+ }
1374
+
1002
1375
  function loadOrInitCloudConfig(projectDir, flags) {
1003
1376
  const existing = loadCloudConfig(projectDir);
1004
1377
  // noTunnel is a per-invocation behavior, never persisted — else a later manual
@@ -1080,7 +1453,11 @@ async function ensureFirewallRule(cfg) {
1080
1453
  }
1081
1454
  }
1082
1455
 
1083
- async function ensureInstance(cfg) {
1456
+ // `sleepFn` is injectable so the 60-iteration wait loop can be driven with a
1457
+ // no-op instead of burning 2 real minutes of clock — the same seam
1458
+ // runGcloudSshWithRetry / waitForEditorComposeRunning already expose.
1459
+ async function ensureInstance(cfg, { sleepFn } = {}) {
1460
+ const sleep = sleepFn || ((ms) => new Promise((r) => setTimeout(r, ms)));
1084
1461
  console.error(`Checking instance ${cfg.instanceName}...`);
1085
1462
  const initialState = fetchInstanceState(cfg);
1086
1463
  const createdThisRun = initialState === null;
@@ -1091,29 +1468,59 @@ async function ensureInstance(cfg) {
1091
1468
  }
1092
1469
  if (initialState === "TERMINATED") {
1093
1470
  console.error(` ${cfg.instanceName} is TERMINATED — starting...`);
1094
- const start = runGcloud(buildGcloudArgs("instances-start", cfg));
1471
+ const start = runGcloud(buildGcloudArgs("instances-start", cfg), {
1472
+ timeout: gcloudCallTimeoutMs("instances-start"),
1473
+ });
1095
1474
  if (start.status !== 0) {
1096
- throw new Error(`Failed to start instance ${cfg.instanceName}`);
1475
+ throw new Error(provisionFailureMessage("start", cfg.instanceName, start));
1097
1476
  }
1098
1477
  } else if (initialState === null) {
1099
1478
  console.error(` ${cfg.instanceName} not found — creating...`);
1100
- const create = runGcloud(buildGcloudArgs("instances-create", cfg));
1479
+ const create = runGcloud(buildGcloudArgs("instances-create", cfg), {
1480
+ timeout: gcloudCallTimeoutMs("instances-create"),
1481
+ });
1101
1482
  if (create.status !== 0) {
1102
- throw new Error(`Failed to create instance ${cfg.instanceName}`);
1483
+ throw new Error(provisionFailureMessage("create", cfg.instanceName, create));
1103
1484
  }
1104
1485
  } else {
1486
+ // A slot found in PROVISIONING is ambiguous, and the two readings need
1487
+ // opposite handling. On a normal add it is a transient the poll loop below
1488
+ // resolves in seconds. But GCE can also accept an insert and then never
1489
+ // finish it (2026-09-04, VM-8): the operation parks at RUNNING/0%, eventually
1490
+ // attaches an INTERNAL_ERROR without ever reaching DONE, and holds the
1491
+ // instance name indefinitely — a delete queues behind it rather than clearing
1492
+ // it. Polling that for 2 minutes only rediscovers, slowly and with a message
1493
+ // naming neither the operation nor the wedge, a state the first describe
1494
+ // already showed. Read the outstanding operation instead and fail immediately
1495
+ // when it carries a terminal error, so the transcript carries the signature
1496
+ // fleet-add-failure.js's wedged-insert arm keys on.
1497
+ if (initialState === "PROVISIONING") {
1498
+ const pending = fetchPendingOperation(cfg);
1499
+ if (pending && pending.errorCode) {
1500
+ throw new Error(wedgedOperationMessage(cfg.instanceName, pending));
1501
+ }
1502
+ }
1105
1503
  console.error(` ${cfg.instanceName} is ${initialState}; waiting...`);
1106
1504
  }
1107
1505
 
1506
+ let lastState = initialState;
1108
1507
  for (let i = 0; i < 60; i++) {
1109
- if (fetchInstanceState(cfg) === "RUNNING") {
1508
+ lastState = fetchInstanceState(cfg);
1509
+ if (lastState === "RUNNING") {
1110
1510
  console.error(` ✓ ${cfg.instanceName} is RUNNING`);
1111
1511
  await waitForIapReady(cfg);
1112
1512
  return { createdThisRun };
1113
1513
  }
1114
- await new Promise((r) => setTimeout(r, 2000));
1514
+ await sleep(2000);
1115
1515
  }
1116
- throw new Error(`Instance ${cfg.instanceName} did not reach RUNNING within 2 minutes`);
1516
+ // Name the state it was actually stuck in. An instance parked in PROVISIONING
1517
+ // is a different operator action from one that stopped answering describe at
1518
+ // all, and the old message ("did not reach RUNNING") distinguished neither.
1519
+ throw new Error(
1520
+ `Instance ${cfg.instanceName} did not reach RUNNING within 2 minutes ` +
1521
+ `(last observed state: ${lastState || "unknown — describe returned no state"}` +
1522
+ `${createdThisRun ? "; a create was issued this run" : ""}).`,
1523
+ );
1117
1524
  }
1118
1525
 
1119
1526
  /** GCE reports the instance as RUNNING the moment the boot disk is attached,
@@ -1496,7 +1903,320 @@ function bootstrapLogTail(cfg, lines = 40) {
1496
1903
  return String(r.stdout || "").trim() || "(log unavailable)";
1497
1904
  }
1498
1905
 
1499
- async function ensureProjectBootstrapped(cfg) {
1906
+ // ─── Detached PROJECT bootstrap ──────────────────────────────────────────────
1907
+ //
1908
+ // The VM bootstrap above solved a problem the PROJECT bootstrap still had: read
1909
+ // success and failure from VM-SIDE STATE, never from the SSH exit code, so a
1910
+ // dropped follow-session over a healthy VM is not misreported as "failed on the
1911
+ // VM". The project bootstrap ran instead as ONE blocking ssh-script call, so a
1912
+ // local budget elapsing (or an SSH pipe breaking) was indistinguishable from a
1913
+ // genuine remote failure and the caller's abort path DELETED A HEALTHY VM.
1914
+ // Raising the budget alone just moves the cliff; these helpers remove it by
1915
+ // mirroring the VM-bootstrap contract — detached launch, re-attaching follow
1916
+ // loop, VM-side outcome probe.
1917
+ //
1918
+ // All VM-side paths are keyed by repo name so two projects on one VM never
1919
+ // collide, and they sit beside the `.done` marker projectBootstrapDoneMarker
1920
+ // already defines. Pure string builders (like their VM-bootstrap counterparts)
1921
+ // so the whole grammar is unit-testable without a live VM.
1922
+ function projectBootstrapScriptPath(repoName) {
1923
+ return `$HOME/.codeyam-project-bootstrap-${repoName}.sh`;
1924
+ }
1925
+ function projectBootstrapRunnerPath(repoName) {
1926
+ return `$HOME/.codeyam-project-bootstrap-${repoName}-run.sh`;
1927
+ }
1928
+ function projectBootstrapLogPath(repoName) {
1929
+ return `$HOME/.codeyam-project-bootstrap-${repoName}.log`;
1930
+ }
1931
+ function projectBootstrapPidPath(repoName) {
1932
+ return `$HOME/.codeyam-project-bootstrap-${repoName}.pid`;
1933
+ }
1934
+ // The runner's recorded EXIT STATUS. Deliberately `.status`, NOT `.done`:
1935
+ // `.done` is already taken by scripts/gcp-project-bootstrap.sh's own terminal
1936
+ // SUCCESS marker (a timestamp, written only after `set -euo pipefail` carried
1937
+ // every step through — see projectBootstrapDoneMarker). Reusing `.done` for an
1938
+ // exit code would overwrite the success signal that both improve40's false-skip
1939
+ // fix and the follow loop below depend on. Two files, two distinct meanings:
1940
+ // `.done` = "the script reached its last line", `.status` = "the runner exited,
1941
+ // here is its code".
1942
+ function projectBootstrapStatusPath(repoName) {
1943
+ return `$HOME/.codeyam-project-bootstrap-${repoName}.status`;
1944
+ }
1945
+ const PROJECT_BOOTSTRAP_LAUNCHED_MARKER = "CODEYAM_PROJECT_BOOTSTRAP_LAUNCHED";
1946
+ const PROJECT_BOOTSTRAP_STATUS_MARKER = "CYPROJ";
1947
+ // One follow session caps at 15 min; the overall wait caps at
1948
+ // PROJECT_BOOTSTRAP_TIMEOUT_MS across re-attaches before we bail loudly with
1949
+ // the VM-side log location. Unlike the VM-bootstrap follow loop — whose follow
1950
+ // calls inherit the 120 s probe budget and so re-attach every two minutes over
1951
+ // a ~14-min build — the project follow call below is given an explicit local
1952
+ // budget just above this remote deadline, so a long install streams in one
1953
+ // session instead of churning re-attaches.
1954
+ const PROJECT_BOOTSTRAP_FOLLOW_SECS = 900;
1955
+ const PROJECT_BOOTSTRAP_FOLLOW_TIMEOUT_MS = (PROJECT_BOOTSTRAP_FOLLOW_SECS + 60) * 1000;
1956
+
1957
+ // The tiny VM-side wrapper the detached launch execs: run the real bootstrap,
1958
+ // redirect all output to the per-repo log, and record the exit status in the
1959
+ // `.status` sentinel — the single source of truth for "did the runner exit, and
1960
+ // with what?" once the SSH follow-session is gone. Contains NO secrets (tokens
1961
+ // arrive via the launch env prefix, never written to disk).
1962
+ function projectBootstrapRunnerScript(repoName) {
1963
+ return [
1964
+ "#!/usr/bin/env bash",
1965
+ "# Detached project-bootstrap runner (written by npm/cloud.js",
1966
+ "# ensureProjectBootstrapped). Survives SSH disconnect: the laptop only",
1967
+ "# *follows* the log below; this process keeps running under its own session",
1968
+ "# if the follow drops. Records its own PID so the status probe can check",
1969
+ "# liveness via `kill -0` (a `pgrep` on the script path would also match the",
1970
+ "# probe command itself).",
1971
+ `echo "$$" > "${projectBootstrapPidPath(repoName)}"`,
1972
+ `bash "${projectBootstrapScriptPath(repoName)}" "$@" > "${projectBootstrapLogPath(repoName)}" 2>&1`,
1973
+ "status=$?",
1974
+ `echo "$status" > "${projectBootstrapStatusPath(repoName)}"`,
1975
+ `rm -f "${projectBootstrapPidPath(repoName)}"`,
1976
+ "",
1977
+ ].join("\n");
1978
+ }
1979
+
1980
+ // Build the remote command that (1) materializes the bootstrap script + runner
1981
+ // on the VM, (2) clears the stale sentinels, (3) launches the runner DETACHED
1982
+ // (`nohup setsid`, new session, stdio to /dev/null) so it outlives the SSH
1983
+ // session, and (4) prints a launched marker. The detach is the durable fix: the
1984
+ // install no longer dies when the laptop's SSH pipe breaks, and the VM is no
1985
+ // longer torn down because it did.
1986
+ //
1987
+ // BOTH sentinels are cleared, `.status` and `.done`. Clearing `.done` is what
1988
+ // makes the outcome probe honest: after a launch, a present `.done` can only
1989
+ // mean THIS run's script reached its terminal line, never a leftover from a
1990
+ // previous half-wired attempt.
1991
+ function projectBootstrapLaunchCommand(scriptBody, scriptArgs = [], envVars = {}, repoName, libFiles = {}) {
1992
+ const scriptB64 = Buffer.from(scriptBody, "utf8").toString("base64");
1993
+ const runnerB64 = Buffer.from(projectBootstrapRunnerScript(repoName), "utf8").toString("base64");
1994
+ const argsString = scriptArgs.map(shQuote).join(" ");
1995
+ const envPrefix = envAssignmentPrefix(envVars);
1996
+ // The bootstrap script is written STANDALONE to $HOME, so any `source`/`.` it
1997
+ // does relative to its own directory resolves against $HOME — not against the
1998
+ // repo layout it was authored in. `libFiles` carries those siblings, keyed by
1999
+ // the $HOME-relative path the script will look for them at (e.g.
2000
+ // "lib/npm-pin.sh"). Their directories do not exist on a fresh VM, hence the
2001
+ // mkdir. Omitting a sourced library is not a soft failure: the script aborts
2002
+ // under `set -euo pipefail` and cloud:up tears the new VM down, which is the
2003
+ // 2026-09-04 fleet-wide add outage.
2004
+ const libWrites = Object.entries(libFiles).flatMap(([relPath, body]) => {
2005
+ const b64 = Buffer.from(body, "utf8").toString("base64");
2006
+ const dir = path.posix.dirname(relPath);
2007
+ const mkdir = dir && dir !== "." ? [`mkdir -p $HOME/${dir}`] : [];
2008
+ return [...mkdir, `echo '${b64}' | base64 -d > $HOME/${relPath}`];
2009
+ });
2010
+ // Writes run synchronously (fast, before any backgrounding) so a decode error
2011
+ // aborts before launch; then the runner is detached. The env prefix is on the
2012
+ // setsid command so the tokens are inherited through nohup→setsid→bash but
2013
+ // never appear in any argv. `& echo MARKER` (no `;` before echo) keeps the
2014
+ // launch in the background while still confirming it fired.
2015
+ const writes = [
2016
+ ...libWrites,
2017
+ `echo '${scriptB64}' | base64 -d > ${projectBootstrapScriptPath(repoName)}`,
2018
+ `echo '${runnerB64}' | base64 -d > ${projectBootstrapRunnerPath(repoName)}`,
2019
+ `rm -f ${projectBootstrapStatusPath(repoName)} ${projectBootstrapDoneMarker(repoName)}`,
2020
+ ].join(" && ");
2021
+ const launch =
2022
+ `${envPrefix}nohup setsid bash ${projectBootstrapRunnerPath(repoName)} ${argsString}`.trimEnd() +
2023
+ ` </dev/null >/dev/null 2>&1 &`;
2024
+ return `${writes}; ${launch} echo ${PROJECT_BOOTSTRAP_LAUNCHED_MARKER}`;
2025
+ }
2026
+
2027
+ // Build the remote command that FOLLOWS the detached install: tail the log live
2028
+ // for the operator, polling for the `.status` sentinel, and exit when the runner
2029
+ // finishes OR a per-session deadline elapses. If the SSH session drops
2030
+ // mid-follow this command dies but the detached install keeps going — the caller
2031
+ // then re-attaches.
2032
+ function projectBootstrapFollowCommand(repoName, deadlineSecs = PROJECT_BOOTSTRAP_FOLLOW_SECS) {
2033
+ const log = projectBootstrapLogPath(repoName);
2034
+ const status = projectBootstrapStatusPath(repoName);
2035
+ return [
2036
+ `touch ${log}`,
2037
+ `tail -n +1 -F ${log} 2>/dev/null & TP=$!`,
2038
+ `waited=0`,
2039
+ `while [ ! -f ${status} ]; do ` +
2040
+ `sleep 3; waited=$((waited+3)); ` +
2041
+ `if [ "$waited" -ge ${deadlineSecs} ]; then kill "$TP" 2>/dev/null; exit 0; fi; ` +
2042
+ `done`,
2043
+ `sleep 1`,
2044
+ `kill "$TP" 2>/dev/null`,
2045
+ ].join("; ");
2046
+ }
2047
+
2048
+ // Build the remote command that reports terminal VM state in one parseable
2049
+ // line: whether the script's terminal `.done` marker exists, the runner's
2050
+ // recorded exit status (or `none`), and whether the runner is still alive.
2051
+ // Cheap — runs over ssh-probe.
2052
+ function projectBootstrapStatusCommand(repoName) {
2053
+ const pid = projectBootstrapPidPath(repoName);
2054
+ const status = projectBootstrapStatusPath(repoName);
2055
+ // Liveness via the runner's PID file + `kill -0`, NOT pgrep: the status
2056
+ // command's own argv contains the bootstrap path, so a `pgrep -f` on it would
2057
+ // match this very probe and always report running. A stale PID file from a
2058
+ // killed runner reads as not-running because `kill -0` fails.
2059
+ return [
2060
+ `done=no; [ -f ${projectBootstrapDoneMarker(repoName)} ] && done=yes`,
2061
+ `st=none; [ -f ${status} ] && st=$(cat ${status} 2>/dev/null)`,
2062
+ `run=no; if [ -f ${pid} ]; then p=$(cat ${pid} 2>/dev/null); ` +
2063
+ `{ [ -n "$p" ] && kill -0 "$p" 2>/dev/null; } && run=yes; fi`,
2064
+ `echo "${PROJECT_BOOTSTRAP_STATUS_MARKER} done=$done status=$st running=$run"`,
2065
+ ].join("; ");
2066
+ }
2067
+
2068
+ // Parse the single status line emitted by projectBootstrapStatusCommand.
2069
+ // Returns `{ done, status, running }` or null when no status line is present
2070
+ // (e.g. the probe ssh itself failed and produced no stdout).
2071
+ function parseProjectBootstrapStatus(stdout) {
2072
+ const m = String(stdout || "").match(
2073
+ new RegExp(`${PROJECT_BOOTSTRAP_STATUS_MARKER} done=(\\S+) status=(\\S+) running=(\\S+)`),
2074
+ );
2075
+ if (!m) return null;
2076
+ return { done: m[1], status: m[2], running: m[3] };
2077
+ }
2078
+
2079
+ // Decide the project-bootstrap outcome from VM-side state alone — never the SSH
2080
+ // exit code. The `.done` marker is the real artifact (the script writes it only
2081
+ // after `set -euo pipefail` carried every step to success), so its presence is
2082
+ // success regardless of how the follow session ended — the same reasoning that
2083
+ // makes the docker image authoritative for the VM bootstrap. A live runner means
2084
+ // "re-attach". A recorded exit status without the marker means the install ran
2085
+ // and failed. Nothing at all means it was never started (or the probe could not
2086
+ // see it).
2087
+ function decideProjectBootstrapOutcome(status) {
2088
+ if (!status) return "absent";
2089
+ const { done, status: exitStatus, running } = status;
2090
+ if (done === "yes") return "success";
2091
+ if (running === "yes") return "running";
2092
+ if (exitStatus && exitStatus !== "none") return "failed";
2093
+ return "absent";
2094
+ }
2095
+
2096
+ // Reduce one follow-loop iteration to an action: "success" | "failed" |
2097
+ // "timeout" | "reattach". Extracted as a pure function because its PRECEDENCE is
2098
+ // load-bearing — it decides whether an operator is told a VM is broken:
2099
+ //
2100
+ // • success and failed outrank the deadline. Otherwise the final poll of a
2101
+ // slow-but-successful bootstrap reports a timeout, and the operator is sent
2102
+ // to reconcile a VM that actually finished.
2103
+ // • a still-running install past the deadline is a TIMEOUT, never a failure.
2104
+ // That is the VM-8 case: healthy VM, still installing, and saying so is what
2105
+ // gets the add classified retryable instead of destroyed.
2106
+ // • an unseen runner ("absent") before the deadline re-attaches rather than
2107
+ // failing — "I could not see it" is not "it broke" — but is still bounded by
2108
+ // the deadline so the loop cannot spin forever.
2109
+ function decideProjectBootstrapFollowAction({ outcome, now, deadline } = {}) {
2110
+ if (outcome === "success") return "success";
2111
+ if (outcome === "failed") return "failed";
2112
+ if (now >= deadline) return "timeout";
2113
+ return "reattach";
2114
+ }
2115
+
2116
+ // Probe the VM for the detached project bootstrap's terminal state and reduce it
2117
+ // to a single outcome string ("success" | "failed" | "running" | "absent").
2118
+ // Used both for the idempotent re-attach decision and after each follow session.
2119
+ function probeProjectBootstrapOutcome(cfg, repoName) {
2120
+ const r = runGcloud(
2121
+ buildGcloudArgs("ssh-probe", cfg, { command: projectBootstrapStatusCommand(repoName) }),
2122
+ { stdio: ["ignore", "pipe", "pipe"] },
2123
+ );
2124
+ return decideProjectBootstrapOutcome(parseProjectBootstrapStatus(r.stdout));
2125
+ }
2126
+
2127
+ /** Assemble the env forwarded into the project-bootstrap shell on the VM.
2128
+ * Extracted from ensureProjectBootstrapped so its CONDITIONAL keys are testable
2129
+ * without a VM — previously you could not assert "FLEET_DOMAIN is omitted for a
2130
+ * non-fleet project" or "a re-provision without --admin stays non-admin"
2131
+ * without invoking the whole gcloud-touching bootstrap.
2132
+ *
2133
+ * Keys, and why each is here:
2134
+ * GITHUB_TOKEN — clone a private client repo.
2135
+ * CY_GIT_USER_{NAME,EMAIL} — rendered into the compose file, then written
2136
+ * to `git config --global` by the entrypoint so
2137
+ * plan_claim_refs::create_claim's commit-tree
2138
+ * call succeeds. Empty is legitimate (a CI
2139
+ * runner with no author); the entrypoint warns
2140
+ * and proceeds, so the keys stay present.
2141
+ * FLEET_DOMAIN — ONLY for a numbered fleet VM with a domain
2142
+ * configured, so the compose render publishes
2143
+ * on 0.0.0.0 for the LB NEG. Omitted entirely
2144
+ * otherwise, preserving the loopback-only
2145
+ * publish a plain client project expects.
2146
+ * <forwarded launch-form keys> — DATABASE_URL et al, whitelisted by the
2147
+ * dashboard via CY_FORWARD_ENV_KEYS, so the
2148
+ * bootstrap-time `npm run setup` sees them on
2149
+ * the first try. Empty on plain `cloud:up`.
2150
+ * CODEYAM_ADMIN — the Home-tab Setup door. Defaults to the
2151
+ * strict non-admin "".
2152
+ *
2153
+ * Later spreads win, matching the original inline assembly exactly. `deps` lets
2154
+ * a test supply the token and git identity instead of reading the real
2155
+ * environment and shelling out to git. */
2156
+ function projectBootstrapEnv(cfg = {}, { env = process.env, githubToken, gitIdentity } = {}) {
2157
+ const git = gitIdentity || resolveLaptopGitIdentity();
2158
+ const out = {
2159
+ GITHUB_TOKEN: githubToken !== undefined ? githubToken : resolveGithubToken(),
2160
+ CY_GIT_USER_NAME: git.name,
2161
+ CY_GIT_USER_EMAIL: git.email,
2162
+ };
2163
+ const ingressPlan = ingressWiringPlan(cfg, env);
2164
+ if (ingressPlan.wire) out.FLEET_DOMAIN = ingressPlan.domain;
2165
+ Object.assign(out, resolveSetupForwardEnv(env));
2166
+ Object.assign(out, buildProvisionEnv(env, cfg));
2167
+ return out;
2168
+ }
2169
+
2170
+ // Fetch the tail of the VM-side project-bootstrap log for an actionable failure
2171
+ // message — the operator sees WHY the install failed, not a bare exit code.
2172
+ function projectBootstrapLogTail(cfg, repoName, lines = 40) {
2173
+ const r = runGcloud(
2174
+ buildGcloudArgs("ssh-probe", cfg, {
2175
+ command: `tail -n ${lines} ${projectBootstrapLogPath(repoName)} 2>/dev/null`,
2176
+ }),
2177
+ { stdio: ["ignore", "pipe", "pipe"] },
2178
+ );
2179
+ return String(r.stdout || "").trim() || "(log unavailable)";
2180
+ }
2181
+
2182
+ /** Assemble the extra provision-time env forwarded into the bootstrap shell,
2183
+ * where it feeds the compose-render substitution. Currently just the admin
2184
+ * toggle: CODEYAM_ADMIN gates the Home-tab Setup door and is read from the
2185
+ * editor-server process env (handlers.rs::admin_enabled, strict literal "1").
2186
+ * The operator opts in with `CODEYAM_ADMIN=1 npm run cloud:up …` or `--admin`
2187
+ * (which lands as cfg.admin); anything else forwards "" so a normal launch —
2188
+ * and a re-provision without the flag — is strict non-admin. Pure over its
2189
+ * inputs so cloud.test.js can pin it without touching gcloud. */
2190
+ function buildProvisionEnv(env = process.env, flags = {}) {
2191
+ const admin = flags.admin === true || env.CODEYAM_ADMIN === "1";
2192
+ return { CODEYAM_ADMIN: admin ? "1" : "" };
2193
+ }
2194
+
2195
+ /** Build the one-shot SSH shell command that reconciles a running VM's admin
2196
+ * mode without a full re-provision: rewrite the rendered compose's
2197
+ * CODEYAM_ADMIN line to `value` — or insert it under `environment:` on a legacy
2198
+ * VM whose compose predates the placeholder — then recreate the editor service
2199
+ * so admin_enabled() re-reads it. `value` is always "1" or "" (the caller
2200
+ * normalizes), so inlining it into sed is safe. Pure/string-only so
2201
+ * cloud.test.js can assert its shape without SSHing. */
2202
+ function buildReconcileAdminCommand(repoName, value) {
2203
+ const f = "docker-compose.cloud.yml";
2204
+ const entry = ` - CODEYAM_ADMIN=${value}`;
2205
+ // Capture only the leading indentation (\1) and re-emit the literal key, so a
2206
+ // value that starts with a digit (e.g. "1") never lands right after the
2207
+ // backreference — `\11` would be read by sed as group 11, not `\1` + "1".
2208
+ return (
2209
+ `cd ~/projects/${repoName} && ` +
2210
+ `if grep -qE '^[[:space:]]*- CODEYAM_ADMIN=' ${f}; then ` +
2211
+ `sed -i -E 's#^([[:space:]]*)- CODEYAM_ADMIN=.*#\\1- CODEYAM_ADMIN=${value}#' ${f}; ` +
2212
+ `else ` +
2213
+ `sed -i -E 's#^([[:space:]]*environment:[[:space:]]*)$#\\1\\n${entry}#' ${f}; ` +
2214
+ `fi && ` +
2215
+ `docker compose -f ${f} up -d editor`
2216
+ );
2217
+ }
2218
+
2219
+ async function ensureProjectBootstrapped(cfg, { sleepFn } = {}) {
1500
2220
  const repoName = resolveProject(cfg).name;
1501
2221
  console.error(`Checking project bootstrap state for ${repoName}...`);
1502
2222
  // Gate on the TERMINAL completion marker the script writes as its very last
@@ -1525,44 +2245,162 @@ async function ensureProjectBootstrapped(cfg) {
1525
2245
  console.error(` streaming scripts/gcp-project-bootstrap.sh to VM for ${cfg.repo}...`);
1526
2246
  const scriptPath = path.join(rootDir, "scripts", "gcp-project-bootstrap.sh");
1527
2247
  const scriptBody = readStreamedScript(scriptPath, cfg);
2248
+ // gcp-project-bootstrap.sh sources lib/npm-pin.sh relative to its own
2249
+ // directory. Read it through readStreamedScript too, so the library is subject
2250
+ // to the same drift guard as the script that sources it — streaming a reviewed
2251
+ // remote script beside a stale on-disk library would defeat the guard.
2252
+ const bootstrapLibs = {
2253
+ "lib/npm-pin.sh": readStreamedScript(path.join(rootDir, "scripts", "lib", "npm-pin.sh"), cfg),
2254
+ };
1528
2255
  const scriptArgs = clientProjectBootstrapArgs(cfg);
1529
- // Forward the laptop's git author identity so the rendered compose file
1530
- // carries CY_GIT_USER_{NAME,EMAIL} → the entrypoint writes them to
1531
- // `git config --global` → plan_claim_refs::create_claim's commit-tree call
1532
- // succeeds. Empty strings are fine (laptop has no identity, e.g. a CI
1533
- // runner); the entrypoint warns and proceeds.
1534
- const laptopGit = resolveLaptopGitIdentity();
1535
- // Forward the operator's launch-form env keys (DATABASE_URL, etc.) into the
1536
- // bootstrap shell so the bootstrap-time `npm run setup` sees them on the
1537
- // first try — without this, Prisma `db:push`/`db:seed` fail for lack of
1538
- // env and the bootstrap's fail-soft branch leaves the app schema-less until
1539
- // the post-add `reapplyVmEnv` re-fires setup. See
1540
- // `improve37-launch-form-env-injected-into-cloud-up-bootstrap-not-just-applied-after`.
1541
- // Whitelisted by the dashboard via CY_FORWARD_ENV_KEYS — empty on the
1542
- // standalone `cloud:up` path, so that flow is unchanged.
1543
- const forwardEnv = resolveSetupForwardEnv();
1544
- // Forward the fleet domain so the bootstrap's compose render publishes the
1545
- // editor/app ports on 0.0.0.0 (the LB NEG reaches them on the VM's internal
1546
- // IP) instead of loopback-only. Only when a fleet domain is configured AND
1547
- // this is a numbered fleet VM — a non-fleet client project keeps the
1548
- // loopback-only publish (legacy parity). Absent ⇒ key omitted entirely.
1549
- const ingressEnv = {};
1550
- const ingressPlan = ingressWiringPlan(cfg);
1551
- if (ingressPlan.wire) ingressEnv.FLEET_DOMAIN = ingressPlan.domain;
1552
- const command = sshScriptCommand(scriptBody, scriptArgs, {
1553
- GITHUB_TOKEN: resolveGithubToken(),
1554
- CY_GIT_USER_NAME: laptopGit.name,
1555
- CY_GIT_USER_EMAIL: laptopGit.email,
1556
- ...ingressEnv,
1557
- ...forwardEnv,
1558
- });
1559
- const result = runGcloud(
1560
- buildGcloudArgs("ssh-script", cfg, { command }),
1561
- { stdio: "inherit" },
2256
+ // Git identity, forwarded launch-form keys, fleet-domain ingress and the admin
2257
+ // toggle — see projectBootstrapEnv for what each key is for and when it is
2258
+ // omitted. Extracted so those conditionals are testable without a VM.
2259
+ const bootstrapEnv = projectBootstrapEnv(cfg);
2260
+
2261
+ // Idempotent re-attach: a prior cloud:up whose follow-session dropped may have
2262
+ // left a detached install still running. Converge on it instead of starting a
2263
+ // second `npm install` on top of the first.
2264
+ //
2265
+ // NOTE the deliberate asymmetry with ensureBootstrapped, which DOES return
2266
+ // early on an outcome of "success". There, the done-probe and the outcome probe
2267
+ // read the same artifact (the docker image), so they always agree. Here the
2268
+ // done-probe above is a strictly STRONGER three-way AND (`.done` marker +
2269
+ // ~/projects/<repo>/.codeyam + docker-compose.cloud.yml) than this probe's
2270
+ // `.done`-alone success, and it has already said no. Honoring a bare `.done`
2271
+ // here would re-open the improve40 VM-7 false-skip from the other side, so only
2272
+ // "running" short-circuits the launch.
2273
+ const initial = probeProjectBootstrapOutcome(cfg, repoName);
2274
+ if (initial === "running") {
2275
+ console.error(" ↺ a detached project bootstrap is already in progress — re-attaching to its log...");
2276
+ } else {
2277
+ // "absent" (never started), "failed" (a prior run died), or "success" (a bare
2278
+ // marker the stricter done-probe rejected) → (re)launch a fresh detached
2279
+ // install that will survive an SSH disconnect. Every step of
2280
+ // gcp-project-bootstrap.sh is skip-on-exists, so re-entering is cheap.
2281
+ console.error(" launching detached project bootstrap on the VM (survives SSH disconnect)...");
2282
+ const launch = projectBootstrapLaunchCommand(scriptBody, scriptArgs, bootstrapEnv, repoName, bootstrapLibs);
2283
+ const launched = await runGcloudSshWithRetry(
2284
+ "ssh-script",
2285
+ cfg,
2286
+ { command: launch },
2287
+ { stdio: ["ignore", "pipe", "pipe"], label: "project bootstrap launch", sleepFn },
2288
+ );
2289
+ if (isSpawnTimeout(launched)) {
2290
+ // The LOCAL budget elapsed and we SIGTERMd the gcloud child. Say so:
2291
+ // the launch is a short write-and-detach, so this is an SSH-reachability
2292
+ // problem, NOT a broken bootstrap script — and the VM may be perfectly
2293
+ // healthy. Conflating the two is what destroyed VM-8.
2294
+ throw new Error(
2295
+ `launching the detached project bootstrap exceeded the local gcloud call budget ` +
2296
+ `and was SIGTERMd locally — the VM may be healthy and simply unreachable over SSH ` +
2297
+ `right now. Re-run to re-attach (each bootstrap step is skip-on-exists).`,
2298
+ );
2299
+ }
2300
+ if (launched.status !== 0) {
2301
+ throw new Error(
2302
+ `failed to launch the detached project bootstrap on the VM (gcloud ssh exit ${launched.status}).\n` +
2303
+ formatGcloudFailure(launched),
2304
+ );
2305
+ }
2306
+ }
2307
+
2308
+ // Follow the live log, re-attaching across any follow-session drops, until the
2309
+ // VM reports a terminal state or the overall deadline elapses. Success and
2310
+ // failure are read from VM state (the `.done` marker + the `.status` sentinel),
2311
+ // never the SSH exit code — so "my laptop's SSH dropped" or "a heavy npm
2312
+ // install outran a local budget" no longer masquerades as a broken bootstrap
2313
+ // and no longer gets a healthy VM deleted.
2314
+ const deadline = Date.now() + PROJECT_BOOTSTRAP_TIMEOUT_MS;
2315
+ for (;;) {
2316
+ runGcloud(
2317
+ buildGcloudArgs("ssh-script", cfg, { command: projectBootstrapFollowCommand(repoName) }),
2318
+ { stdio: "inherit", timeout: PROJECT_BOOTSTRAP_FOLLOW_TIMEOUT_MS },
2319
+ );
2320
+ const outcome = probeProjectBootstrapOutcome(cfg, repoName);
2321
+ const action = decideProjectBootstrapFollowAction({
2322
+ outcome,
2323
+ now: Date.now(),
2324
+ deadline,
2325
+ });
2326
+ if (action === "success") {
2327
+ console.error(` ✓ ${repoName} bootstrap complete`);
2328
+ return;
2329
+ }
2330
+ if (action === "failed") {
2331
+ const tail = projectBootstrapLogTail(cfg, repoName);
2332
+ throw new Error(
2333
+ "scripts/gcp-project-bootstrap.sh failed on the VM. " +
2334
+ `Last lines of ${projectBootstrapLogPath(repoName)}:\n${tail}`,
2335
+ );
2336
+ }
2337
+ if (action === "timeout") {
2338
+ // The install is STILL RUNNING on a healthy VM and we simply stopped
2339
+ // waiting. This message is the signature npm/fleet-add-failure.js keys on
2340
+ // to classify the add as `project-bootstrap-timeout` (retryable) instead of
2341
+ // the non-retryable generic cloud-up-failed that tore down VM-8.
2342
+ throw new Error(
2343
+ `project bootstrap exceeded the local ${Math.round(PROJECT_BOOTSTRAP_TIMEOUT_MS / 60000)}-minute budget ` +
2344
+ `— the VM and its detached bootstrap may be healthy and still running. ` +
2345
+ `Re-run to re-attach (each bootstrap step is skip-on-exists), or inspect ` +
2346
+ `${projectBootstrapLogPath(repoName)} on the VM.`,
2347
+ );
2348
+ }
2349
+ // action === "reattach": the follow session ended but the install is alive
2350
+ // (or the probe could not see it yet). Re-attach to its log.
2351
+ console.error(" ↺ follow session ended but the project bootstrap is still running — re-attaching...");
2352
+ }
2353
+ }
2354
+
2355
+ /** Run the compose-ps liveness probe over SSH and report BOTH the editor
2356
+ * service's state and whether the probe itself could be trusted.
2357
+ *
2358
+ * The `|| true` keeps the remote pipeline's exit status 0 whenever SSH itself
2359
+ * worked, so a non-zero `status` means the transport failed (auth/SSH/IAP) —
2360
+ * not that the container is down. Callers that must fail safe (the abort
2361
+ * teardown) need that distinction; `parseEditorComposeState` alone collapses
2362
+ * "no container" and "could not ask" into the same `null`.
2363
+ *
2364
+ * Returns `{ state, inconclusive }` where `state` is the compose `State` string
2365
+ * (`"running"`, `"exited"`, …) or null when the service is absent, and
2366
+ * `inconclusive` is true only for a transport failure. */
2367
+ function probeEditorComposeState(cfg) {
2368
+ const repoName = resolveProject(cfg).name;
2369
+ const composeCmd = `docker compose -f docker-compose.cloud.yml`;
2370
+ const probe = runGcloud(
2371
+ buildGcloudArgs("ssh-probe", cfg, {
2372
+ command: `cd ~/projects/${repoName} 2>/dev/null && ${composeCmd} ps --format json 2>/dev/null || true`,
2373
+ }),
2374
+ { stdio: ["ignore", "pipe", "ignore"] },
1562
2375
  );
1563
- if (result.status !== 0) {
1564
- throw new Error("scripts/gcp-project-bootstrap.sh failed on the VM");
2376
+ if (probe.status !== 0) {
2377
+ return { state: null, inconclusive: true };
1565
2378
  }
2379
+ return { state: parseEditorComposeState(probe.stdout), inconclusive: false };
2380
+ }
2381
+
2382
+ /** Poll {@link probeEditorComposeState} until the editor service reads
2383
+ * `running`, returning true as soon as it does. Used to close the TOCTOU window
2384
+ * after a container-name conflict: the winning run may still be inside
2385
+ * `Creating`/`Created` when the conflict surfaces, so a single immediate probe
2386
+ * can under-read and report a healthy container as absent.
2387
+ *
2388
+ * Deliberately short (default ~30s): this is not the multi-minute first-boot
2389
+ * budget `waitForContainerRunning` grants, it is the tail of a start that
2390
+ * another process has already begun. `sleepFn` and `probeFn` are injectable so
2391
+ * a unit test drives the whole loop with neither wall-clock delay nor a VM. */
2392
+ async function waitForEditorComposeRunning(cfg, { attempts = 6, delayMs = 5000, sleepFn, probeFn } = {}) {
2393
+ const sleep = sleepFn || ((ms) => new Promise((r) => setTimeout(r, ms)));
2394
+ const probe = probeFn || probeEditorComposeState;
2395
+ for (let i = 0; i < attempts; i++) {
2396
+ if (probe(cfg).state === "running") {
2397
+ return true;
2398
+ }
2399
+ if (i < attempts - 1) {
2400
+ await sleep(delayMs);
2401
+ }
2402
+ }
2403
+ return false;
1566
2404
  }
1567
2405
 
1568
2406
  async function ensureEditorContainerUp(cfg) {
@@ -1572,13 +2410,7 @@ async function ensureEditorContainerUp(cfg) {
1572
2410
  // cloud variant when self-hosting (see gcp-project-bootstrap.sh).
1573
2411
  const composeCmd = `docker compose -f docker-compose.cloud.yml`;
1574
2412
  console.error("Checking editor container state...");
1575
- const probe = runGcloud(
1576
- buildGcloudArgs("ssh-probe", cfg, {
1577
- command: `cd ~/projects/${repoName} && ${composeCmd} ps --format json 2>/dev/null || true`,
1578
- }),
1579
- { stdio: ["ignore", "pipe", "ignore"] },
1580
- );
1581
- if (probe.status === 0 && parseEditorComposeState(probe.stdout) === "running") {
2413
+ if (probeEditorComposeState(cfg).state === "running") {
1582
2414
  console.error(" ✓ editor container is already running");
1583
2415
  // containerStarted: the container is confirmed up, so the CLI install runs
1584
2416
  // its POST-start path with no wait — see the precondition on ensureAgentCliInstalled.
@@ -1586,14 +2418,47 @@ async function ensureEditorContainerUp(cfg) {
1586
2418
  return;
1587
2419
  }
1588
2420
  console.error(" starting editor container (docker compose up -d)...");
2421
+ // CAPTURE rather than `stdio: "inherit"`: the daemon writes its container-name
2422
+ // conflict to stderr, and with "inherit" that text went straight to our parent
2423
+ // and was never observable here — so the conflict could not be told apart from
2424
+ // a real failure (the 2026-08-07 VM-4/VM-5 delete). spawnSync cannot both
2425
+ // stream and capture, so the operator's view is preserved by echoing the
2426
+ // captured streams immediately below instead of live. Do NOT revert this to
2427
+ // "inherit"; classification depends on it.
1589
2428
  const up = runGcloud(
1590
2429
  buildGcloudArgs("ssh-probe", cfg, {
1591
2430
  command: `cd ~/projects/${repoName} && ${composeCmd} up -d`,
1592
2431
  }),
1593
- { stdio: "inherit" },
2432
+ { stdio: ["ignore", "pipe", "pipe"] },
1594
2433
  );
2434
+ if (up.stdout) process.stderr.write(up.stdout);
2435
+ if (up.stderr) process.stderr.write(up.stderr);
1595
2436
  if (up.status !== 0) {
1596
- throw new Error("docker compose up -d failed on the VM");
2437
+ const verdict = classifyComposeUpFailure({ status: up.status, stderr: `${up.stderr || ""}\n${up.stdout || ""}` });
2438
+ if (verdict !== "name-conflict") {
2439
+ throw new Error("docker compose up -d failed on the VM");
2440
+ }
2441
+ // A name conflict means a concurrent run already created this container —
2442
+ // the goal we were about to pursue is being reached by someone else. Confirm
2443
+ // that positively before converging: a conflict whose container never
2444
+ // reaches `running` (e.g. a stale STOPPED container squatting the name) is a
2445
+ // genuine problem and must still fail loudly, or convergence becomes a
2446
+ // catch-all that papers over real breakage.
2447
+ console.error(
2448
+ " a concurrent run already created the editor container — confirming it reaches running...",
2449
+ );
2450
+ if (!(await waitForEditorComposeRunning(cfg))) {
2451
+ throw new Error(
2452
+ "docker compose up -d hit a container-name conflict on the VM, but the conflicting " +
2453
+ "container never reached running (a stale stopped container may be holding the name). " +
2454
+ `Inspect with: gcloud compute ssh ${cfg.instanceName} --command 'cd ~/projects/${repoName} && ${composeCmd} ps -a'`,
2455
+ );
2456
+ }
2457
+ console.error(" ✓ editor container is running (started by a concurrent run)");
2458
+ // containerStarted: the container is confirmed up — same post-start contract
2459
+ // as the already-running short-circuit above.
2460
+ await ensureAgentCliInstalled(cfg, { containerStarted: true });
2461
+ return;
1597
2462
  }
1598
2463
  // containerStarted: `up -d` returned 0, so the CLI install is legitimately a
1599
2464
  // post-start step. A brief wait-for-running here (slow boot settling) is
@@ -1609,18 +2474,10 @@ async function ensureEditorContainerUp(cfg) {
1609
2474
  // success. This throws (non-zero exit) when the editor service isn't
1610
2475
  // `running`, converting a silent partial provision into a loud, actionable
1611
2476
  // failure the operator (and the dashboard's fail-fast-on-nonzero path) can act
1612
- // on. The `|| true` keeps the probe's exit status 0 so the real signal is the
1613
- // parsed compose state, not the SSH/cd exit code.
2477
+ // on. An inconclusive probe is not "running" here — a terminal gate must have
2478
+ // positive evidence, and the caller's job is to fail loudly without it.
1614
2479
  async function assertEditorContainerRunning(cfg) {
1615
- const repoName = resolveProject(cfg).name;
1616
- const composeCmd = `docker compose -f docker-compose.cloud.yml`;
1617
- const probe = runGcloud(
1618
- buildGcloudArgs("ssh-probe", cfg, {
1619
- command: `cd ~/projects/${repoName} 2>/dev/null && ${composeCmd} ps --format json 2>/dev/null || true`,
1620
- }),
1621
- { stdio: ["ignore", "pipe", "ignore"] },
1622
- );
1623
- if (!(probe.status === 0 && parseEditorComposeState(probe.stdout) === "running")) {
2480
+ if (probeEditorComposeState(cfg).state !== "running") {
1624
2481
  throw new Error(
1625
2482
  `provisioning incomplete: editor container is not running on ${cfg.instanceName} ` +
1626
2483
  `(no /workspace or container). Re-run: npm run cloud:up -- --instance-name ${cfg.instanceName}`,
@@ -1763,49 +2620,113 @@ function resolveGhcrPushToken() {
1763
2620
  return process.env.CODEYAM_GHCR_PUSH_TOKEN || "";
1764
2621
  }
1765
2622
 
2623
+ // Registry manifest media types we accept when probing image presence over the
2624
+ // GHCR HTTP API — Docker v2 + manifest lists and their OCI equivalents, so a
2625
+ // multi-arch index answers 200 the same as a single-arch manifest.
2626
+ const MANIFEST_ACCEPT = [
2627
+ "application/vnd.docker.distribution.manifest.v2+json",
2628
+ "application/vnd.docker.distribution.manifest.list.v2+json",
2629
+ "application/vnd.oci.image.manifest.v1+json",
2630
+ "application/vnd.oci.image.index.v1+json",
2631
+ ].join(", ");
2632
+ const GHCR_PROBE_TIMEOUT_MS = 15000;
2633
+
2634
+ /** Default HTTPS client for the registry probe: a GET that resolves to
2635
+ * `{ statusCode }` and NEVER rejects — any socket / DNS / timeout error resolves
2636
+ * to `statusCode: 0` (the probe reads that as "undetermined"). Injected as the
2637
+ * `httpImpl` option so probeUsableEditorImage is unit-testable with no network
2638
+ * (and no Docker daemon). `requestImpl` (defaulting to `https.request`) is
2639
+ * injectable so the response / error / timeout mapping is itself unit-testable
2640
+ * without a real socket. */
2641
+ function httpsManifestGet({ hostname, path: reqPath, headers }, requestImpl = https.request) {
2642
+ return new Promise((resolve) => {
2643
+ const req = requestImpl(
2644
+ { hostname, path: reqPath, method: "GET", headers },
2645
+ (res) => {
2646
+ res.resume(); // drain the body so the socket is freed
2647
+ res.on("end", () => resolve({ statusCode: res.statusCode }));
2648
+ },
2649
+ );
2650
+ req.on("error", () => resolve({ statusCode: 0 }));
2651
+ req.setTimeout(GHCR_PROBE_TIMEOUT_MS, () => req.destroy());
2652
+ req.end();
2653
+ });
2654
+ }
2655
+
2656
+ /** Probe ONE fully-qualified image ref (`host/repo:tag`) over the registry HTTP
2657
+ * API — no docker daemon, so a docker-less operator host can still determine
2658
+ * presence. Returns true (200 present), false (404 absent), or null (auth /
2659
+ * network / malformed-ref → can't tell absent from forbidden). */
2660
+ async function probeManifestRef(fullRef, bearer, httpGet) {
2661
+ const colon = fullRef.lastIndexOf(":");
2662
+ if (colon < 0) return null;
2663
+ const ref = fullRef.slice(0, colon); // host/repo
2664
+ const tag = fullRef.slice(colon + 1); // slug-latest
2665
+ const slash = ref.indexOf("/");
2666
+ if (slash < 0) return null;
2667
+ const hostname = ref.slice(0, slash); // ghcr.io
2668
+ const repo = ref.slice(slash + 1); // codeyam-ai/codeyam-editor-cloud-smoke
2669
+ let res;
2670
+ try {
2671
+ res = await httpGet({
2672
+ hostname,
2673
+ path: `/v2/${repo}/manifests/${encodeURIComponent(tag)}`,
2674
+ headers: { Authorization: `Bearer ${bearer}`, Accept: MANIFEST_ACCEPT },
2675
+ });
2676
+ } catch (_e) {
2677
+ return null;
2678
+ }
2679
+ const code = res && res.statusCode;
2680
+ if (code === 200) return true;
2681
+ if (code === 404) return false;
2682
+ return null; // 401 / 403 / 5xx / 0 (network) → undetermined
2683
+ }
2684
+
1766
2685
  /** Read-only probe: does a usable prebuilt editor image already exist on the
1767
2686
  * registry for `branch`? Used by the fleet dashboard BEFORE a concurrent batch
1768
2687
  * add to decide whether to pace one VM ahead of the rest (see
1769
- * npm/fleet-prewarm.js::decideBatchPrewarm). A `docker manifest inspect` is a
1770
- * HEAD-equivalent — it reads the manifest without pulling the (multi-GB) image.
2688
+ * npm/fleet-prewarm.js::decideBatchPrewarm).
2689
+ *
2690
+ * Queries the registry over its HTTP API (`GET /v2/<repo>/manifests/<tag>`, a
2691
+ * HEAD-equivalent that reads the manifest without pulling the multi-GB image)
2692
+ * instead of `docker login` / `docker manifest inspect`. This is the root fix
2693
+ * for the docker-less GCE operator host: the old docker path errored on the
2694
+ * first `docker login` spawn (the control plane has no docker daemon) and so
2695
+ * returned `null` forever, which decideBatchPrewarm read as "fan out" — the
2696
+ * image was therefore never published and every cold add source-built. Over
2697
+ * HTTP the probe returns a DEFINITIVE verdict with no daemon.
1771
2698
  *
1772
2699
  * Returns:
1773
2700
  * true — at least one of the usable tags exists (a cold add can pull; no pacer)
1774
2701
  * false — every tag was checked and none exists (known-absent; pacing pays)
1775
- * null — undetermined: no read token, docker missing, or a probe error that
1776
- * can't be distinguished from "absent". The caller treats null as
1777
- * "fan out" (today's behavior, never worse) rather than risk a
1778
- * 10-min serialization that might buy nothing.
2702
+ * null — undetermined: no read token, or an auth/network error that can't be
2703
+ * distinguished from "absent".
1779
2704
  *
1780
- * The private packages require auth, so we `docker login` with the read token
1781
- * first; without a token we can't tell absent from forbidden, so → null.
1782
- * `spawnImpl` is injectable for unit tests. */
1783
- function probeUsableEditorImage({ branch, registry, token, spawnImpl } = {}) {
2705
+ * GHCR's private packages require auth; without a read token we can't tell
2706
+ * absent from forbidden, so → null. The read token is sent as the registry
2707
+ * bearer (GHCR accepts a base64-encoded PAT directly — no separate token-
2708
+ * exchange round-trip). `httpImpl` is injectable for unit tests; a legacy
2709
+ * `spawnImpl` option is still accepted (and ignored) for backward-compat — the
2710
+ * HTTP happy path needs no spawn. */
2711
+ async function probeUsableEditorImage({ branch, registry, token, httpImpl } = {}) {
1784
2712
  const fp = require("./fleet-prewarm");
1785
- const spawn = spawnImpl || spawnSync;
1786
2713
  const reg = registry || process.env.CODEYAM_REGISTRY || "ghcr.io/codeyam-ai";
1787
2714
  const readToken = token || resolveGhcrToken();
1788
2715
  if (!readToken) return null; // can't auth → can't distinguish absent from forbidden
1789
- const host = String(reg).split("/")[0];
1790
- const login = spawn(
1791
- "docker",
1792
- ["login", host, "-u", process.env.CODEYAM_GHCR_USER || "codeyam", "--password-stdin"],
1793
- { input: readToken, encoding: "utf8" },
1794
- );
1795
- // spawn error (docker missing) or a non-zero login → undetermined.
1796
- if (!login || login.error || login.status !== 0) return null;
2716
+ const httpGet = httpImpl || httpsManifestGet;
2717
+ // GHCR accepts a base64-encoded PAT as the registry bearer — no token-exchange
2718
+ // round-trip needed.
2719
+ const bearer = Buffer.from(readToken).toString("base64");
1797
2720
  const tags = fp.usableImageTags({ registry: reg, branch });
1798
2721
  let allClean = true;
1799
- for (const tag of tags) {
1800
- const res = spawn("docker", ["manifest", "inspect", tag], { encoding: "utf8" });
1801
- if (!res || res.error) {
1802
- allClean = false; // probe itself failed to run — can't claim "absent"
1803
- continue;
1804
- }
1805
- if (res.status === 0) return true; // a usable image exists
1806
- }
1807
- // Every tag inspected cleanly and none existed → definitively absent. If any
1808
- // probe failed to run, we can't be sure → undetermined.
2722
+ for (const fullRef of tags) {
2723
+ const verdict = await probeManifestRef(fullRef, bearer, httpGet);
2724
+ if (verdict === true) return true; // a usable image exists
2725
+ if (verdict === null) allClean = false; // this tag's probe was inconclusive
2726
+ // verdict === false → this tag is absent; keep checking the rest
2727
+ }
2728
+ // Every tag probed cleanly and none existed → definitively absent. If any probe
2729
+ // was inconclusive we can't claim absence → undetermined.
1809
2730
  return allClean ? false : null;
1810
2731
  }
1811
2732
 
@@ -1842,6 +2763,15 @@ function classifyGitRemoteError(stderr) {
1842
2763
  ) {
1843
2764
  return "network";
1844
2765
  }
2766
+ // A greenfield project (`--new-project` scaffold) has no origin remote and an
2767
+ // unborn `main`, so `git ls-remote origin` fails with "does not appear to be a
2768
+ // git repository". No token would fix this — there is simply nothing to push
2769
+ // yet — so it is neither `credential` nor `network` nor `git-spawn`. Keep the
2770
+ // match narrow: `repository not found` is a genuine access failure against a
2771
+ // real remote and must NOT be swallowed here.
2772
+ if (lower.includes("does not appear to be a git repository")) {
2773
+ return "no-remote";
2774
+ }
1845
2775
  return "other";
1846
2776
  }
1847
2777
 
@@ -1879,6 +2809,14 @@ function decideCredentialHeal({ stagingEmpty, lsRemoteOk, lsRemoteStderr }) {
1879
2809
  reason: "git binary failed to launch — not a credential gap, not re-seeding",
1880
2810
  };
1881
2811
  }
2812
+ if (category === "no-remote") {
2813
+ return {
2814
+ heal: false,
2815
+ category,
2816
+ reason:
2817
+ "no git remote configured — greenfield project, nothing to push yet (not a credential gap)",
2818
+ };
2819
+ }
1882
2820
  return {
1883
2821
  heal: true,
1884
2822
  category,
@@ -1896,6 +2834,16 @@ function decideCredentialHeal({ stagingEmpty, lsRemoteOk, lsRemoteStderr }) {
1896
2834
  };
1897
2835
  }
1898
2836
 
2837
+ /** The single GitHub credential line format shared by every seed path: the
2838
+ * host staging file, the durable on-disk `store` fallback, and the live
2839
+ * cache-daemon approve all expect `https://x-access-token:<token>@github.com`
2840
+ * (git's credential-URL form, matched by both the entrypoint parser and
2841
+ * git-credential-store). Centralized so the `x-access-token` shape has one
2842
+ * source of truth instead of being re-typed at each call site. */
2843
+ function githubCredentialLine(token) {
2844
+ return `https://x-access-token:${token}@github.com`;
2845
+ }
2846
+
1899
2847
  /** Write the laptop's GitHub token into ~/.codeyam-cloud-git-credentials on
1900
2848
  * the VM so the cloud compose can mount it into /root/.git-credentials —
1901
2849
  * without this, claude / codex / gemini inside the container can clone the
@@ -1921,21 +2869,28 @@ async function seedGithubCredentials(cfg, opts = {}) {
1921
2869
  console.error(
1922
2870
  `Writing GitHub token (from ${source}) into ~/.codeyam-cloud-git-credentials on the VM...`,
1923
2871
  );
1924
- const credLine = `https://x-access-token:${token}@github.com`;
2872
+ const credLine = githubCredentialLine(token);
1925
2873
  // Quote the token through one shell layer (gcloud ssh --command); the
1926
2874
  // value is base64'd to keep it out of the shell history.
1927
2875
  const b64 = Buffer.from(`${credLine}\n`).toString("base64");
1928
- // Verify-before-move: decode into a chmod-600 temp, assert it is non-empty
1929
- // (`[ -s ]`), and only THEN move it over the live staging file. An empty or
2876
+ // Verify-before-write: decode into a chmod-600 temp, assert it is non-empty
2877
+ // (`[ -s ]`), and only THEN copy it over the live staging file. An empty or
1930
2878
  // unreadable decode must never clobber an existing good file — the entrypoint
1931
2879
  // reads this file on every boot, and a silent empty write is exactly how a
1932
2880
  // reprovision lands a VM that "could not read Username for github.com". The
1933
- // trailing `echo OK` runs only on the move, so an empty write can't mask
2881
+ // trailing `echo OK` runs only on success, so an empty write can't mask
1934
2882
  // success with a `&& echo` (the failure shape seen on the Claude creds sinks).
2883
+ //
2884
+ // Use `cat "$t" > "$f"` (in-place truncate), NOT `mv`: the staging file is
2885
+ // bind-mounted read-only into the running container, and `mv` mints a NEW
2886
+ // inode that the live `:ro` mount does not follow — so a `mv`-based re-seed
2887
+ // never reached the running container until a full recreate. Truncating the
2888
+ // existing file in place keeps its inode, so the durable write is visible to
2889
+ // the container immediately. Then remove the temp.
1935
2890
  const remoteCmd =
1936
2891
  `f=$HOME/.codeyam-cloud-git-credentials && t="$f.tmp.$$" && ` +
1937
2892
  `echo ${b64} | base64 -d > "$t" && chmod 600 "$t" && ` +
1938
- `if [ -s "$t" ]; then mv "$t" "$f" && echo OK; ` +
2893
+ `if [ -s "$t" ]; then cat "$t" > "$f" && chmod 600 "$f" && rm -f "$t" && echo OK; ` +
1939
2894
  `else rm -f "$t"; echo EMPTY-WRITE-REFUSED >&2; exit 1; fi`;
1940
2895
  const result = runGcloud(
1941
2896
  buildGcloudArgs("ssh-probe", cfg, { command: remoteCmd }),
@@ -1946,26 +2901,76 @@ async function seedGithubCredentials(cfg, opts = {}) {
1946
2901
  }
1947
2902
  }
1948
2903
 
2904
+ /** Sentinel the remote read prints when it cannot find an `editor.json` under
2905
+ * any `~/projects/*` directory. Distinguishes "this VM has no scaffolded
2906
+ * project" (a real miss worth warning about) from "the project is scaffolded
2907
+ * but declares no port" (legitimately a `VITE_PORT` default). Without the
2908
+ * distinction both collapse to `{}` and the miss is silent. */
2909
+ const VM_APP_PORT_NO_CONFIG = "<<<NOCONFIG>>>";
2910
+
2911
+ /** Build the remote shell command that locates the project's `.codeyam` dir and
2912
+ * cats its two config files in one round-trip.
2913
+ *
2914
+ * Resolution is glob-based rather than `~/projects/<repoName>` alone: the host
2915
+ * directory is not guaranteed to be named after the repo (a renamed repo, a
2916
+ * manually-cloned dir, a fleet slug), and when the guess missed, the old code
2917
+ * silently returned `VITE_PORT` and forwarded the wrong port. The repo-named
2918
+ * path is still tried FIRST so a multi-project VM resolves deterministically;
2919
+ * the glob is the fallback. When nothing matches, print the sentinel so the
2920
+ * caller can warn instead of guessing. Pure string builder; no I/O. */
2921
+ function buildVmAppPortReadCommand(repoName) {
2922
+ return (
2923
+ `d=""; ` +
2924
+ `for c in ~/projects/${repoName}/.codeyam ~/projects/*/.codeyam; do ` +
2925
+ `if [ -f "$c/editor.json" ]; then d="$c"; break; fi; done; ` +
2926
+ `if [ -z "$d" ]; then echo '${VM_APP_PORT_NO_CONFIG}'; else ` +
2927
+ `cat "$d/editor.json" 2>/dev/null || echo '{}'; ` +
2928
+ `echo '<<<LOCAL>>>'; ` +
2929
+ `cat "$d/editor.local.json" 2>/dev/null || echo '{}'; fi`
2930
+ );
2931
+ }
2932
+
1949
2933
  /** Best-effort read of the VM's resolved app dev-server port so cloud:up's
1950
2934
  * laptop `-L` forward, readiness probe, and previewOrigin seed all target the
1951
2935
  * port the app actually serves on instead of a hardcoded 5173. Cats the VM's
1952
2936
  * editor.json and editor.local.json in one SSH round-trip and resolves via
1953
- * `resolveConfiguredAppPort`. Any failure (SSH error, missing/corrupt JSON)
1954
- * falls back to `VITE_PORT` — the same default `resolved_port()` lands on for
1955
- * a portless config, so a 5173/greenfield project sees no behavior change. */
2937
+ * `resolveConfiguredAppPort`.
2938
+ *
2939
+ * Falls back to `VITE_PORT` on any failure — but fails LOUD, not silent. A
2940
+ * wrong port here is not a cosmetic default: it forwards the wrong `-L` port,
2941
+ * and if the preview ends up on a dedicated origin the viewer gets
2942
+ * "localhost refused to connect" with nothing pointing at the cause. So an SSH
2943
+ * failure or a VM with no scaffolded project warns on stderr naming that
2944
+ * symptom. A scaffolded project that simply declares no port is NOT a miss —
2945
+ * `VITE_PORT` is genuinely correct there (it is what `resolved_port()` lands
2946
+ * on) and stays quiet. */
1956
2947
  async function readVmAppPort(cfg) {
1957
2948
  const repoName = resolveProject(cfg).name;
1958
- const base = `~/projects/${repoName}/.codeyam`;
1959
2949
  const read = runGcloud(
1960
2950
  buildGcloudArgs("ssh-probe", cfg, {
1961
- command:
1962
- `cat ${base}/editor.json 2>/dev/null || echo '{}'; ` +
1963
- `echo '<<<LOCAL>>>'; ` +
1964
- `cat ${base}/editor.local.json 2>/dev/null || echo '{}'`,
2951
+ command: buildVmAppPortReadCommand(repoName),
1965
2952
  }),
1966
2953
  { stdio: ["ignore", "pipe", "ignore"] },
1967
2954
  );
1968
- if (read.status !== 0 || typeof read.stdout !== "string") return VITE_PORT;
2955
+ if (read.status !== 0 || typeof read.stdout !== "string") {
2956
+ console.error(
2957
+ `[cloud] WARNING: could not read the VM's app port over SSH — ` +
2958
+ `falling back to ${VITE_PORT}. If the app does not serve on ` +
2959
+ `${VITE_PORT}, the laptop port-forward targets the wrong port and the ` +
2960
+ `Live Preview will show "localhost refused to connect".`,
2961
+ );
2962
+ return VITE_PORT;
2963
+ }
2964
+ if (read.stdout.includes(VM_APP_PORT_NO_CONFIG)) {
2965
+ console.error(
2966
+ `[cloud] WARNING: no .codeyam/editor.json found under ~/projects on the ` +
2967
+ `VM (looked for '${repoName}', then any project dir) — falling back to ` +
2968
+ `${VITE_PORT}. If the project is scaffolded under a different name and ` +
2969
+ `serves on another port, the port-forward targets the wrong port and ` +
2970
+ `the Live Preview will show "localhost refused to connect".`,
2971
+ );
2972
+ return VITE_PORT;
2973
+ }
1969
2974
  const parse = (s) => {
1970
2975
  try {
1971
2976
  const v = JSON.parse(s);
@@ -2025,8 +3030,30 @@ function buildEditorLocalSeedWriteCommand({ repoName, remotePath, b64 }) {
2025
3030
  * `_managedPreviewOrigin` sentinel). We stamp the sentinel here too so the
2026
3031
  * monitor doesn't mistake the cloud:up seed for a user pin. The
2027
3032
  * skip-on-already-set guard stays: a non-empty previewOrigin (whether a user
2028
- * pin or a monitor write) is left untouched on every `cloud:up`. */
2029
- async function seedPreviewOrigin(cfg, appPort = VITE_PORT) {
3033
+ * pin or a monitor write) is left untouched on every `cloud:up`.
3034
+ *
3035
+ * `effectiveSameOriginSafe` is the RUNNING editor's own verdict
3036
+ * (`/api/config`'s `previewSameOriginSafe`), supplied by a caller that has
3037
+ * already fetched it. When it is a boolean it WINS over the static
3038
+ * `stack.json` read; `null`/absent leaves the static read in charge.
3039
+ *
3040
+ * That parameter exists because this function had two sources of truth for one
3041
+ * flag, and they disagreed exactly where the adaptive default lives. A
3042
+ * subpath hydration failure escalates a project off `/__codeyam_preview` at
3043
+ * RUNTIME even when `stack.json` never declared `sameOriginSafe: false` — which
3044
+ * is the documented Next.js scaffold shape (a static `false` broke every
3045
+ * SSH-forward VM, so the key is deliberately absent). `healPreviewOriginAtReady`
3046
+ * decided WHETHER to heal from that runtime verdict and was right, then called
3047
+ * this function, which recomputed WHAT to write from the static flag and wrote
3048
+ * the very same-origin sentinel that had just failed. The project then sat
3049
+ * permanently at `previewDedicatedOriginUnavailable: true` — the editor knowing
3050
+ * it must leave the subpath and having nowhere to go — beside a preview host
3051
+ * that was already provisioned, LB-routed, HEALTHY and unused. */
3052
+ async function seedPreviewOrigin(
3053
+ cfg,
3054
+ appPort = VITE_PORT,
3055
+ effectiveSameOriginSafe = null,
3056
+ ) {
2030
3057
  const repoName = resolveProject(cfg).name;
2031
3058
  const remotePath = `~/projects/${repoName}/.codeyam/editor.local.json`;
2032
3059
  const stackPath = `~/projects/${repoName}/.codeyam/stack.json`;
@@ -2062,6 +3089,12 @@ async function seedPreviewOrigin(cfg, appPort = VITE_PORT) {
2062
3089
  current = parseObj(localJson);
2063
3090
  sameOriginSafe = previewSameOriginSafeFromStack(parseObj(stackJson));
2064
3091
  }
3092
+ // The running editor's verdict overrides the static file — see
3093
+ // `resolveEffectiveSameOriginSafe` for why only a boolean counts.
3094
+ sameOriginSafe = resolveEffectiveSameOriginSafe(
3095
+ sameOriginSafe,
3096
+ effectiveSameOriginSafe,
3097
+ );
2065
3098
 
2066
3099
  // Resolve this VM's custom slug (if any) so a dedicated-origin seed uses the
2067
3100
  // slug host. The slug rides on cfg at Add time (the --vm-slug flag, validated
@@ -2095,6 +3128,12 @@ async function seedPreviewOrigin(cfg, appPort = VITE_PORT) {
2095
3128
  // `sameOriginSafe: true`, and seeded `/__codeyam_preview` — which a
2096
3129
  // remote browser then loads as `127.0.0.1 refused to connect`). Once
2097
3130
  // stack.json exists, heal it forward to the dedicated origin.
3131
+ // `sameOriginSafe` here is the EFFECTIVE verdict, so this exception also
3132
+ // covers the case stack.json can never report: a project whose static
3133
+ // flag is legitimately absent (the Next.js scaffold default) that
3134
+ // escalated off the subpath at runtime after a hydration failure. The
3135
+ // caller passes the editor's own answer; without it this predicate read
3136
+ // `true` and declined to heal the one shape it was needed for.
2098
3137
  // Because every dashboard action (Reset / Update / Add / bootstrap-recovery)
2099
3138
  // re-runs this seed over cloud:up, either heal reaches every touch-point the
2100
3139
  // moment stack.json is present — closing the race without re-ordering
@@ -2190,11 +3229,14 @@ async function healPreviewOriginAtReady(cfg, appPort = VITE_PORT) {
2190
3229
  `READY auto-heal: ${cfg.instanceName} reports a not-same-origin-safe stack ` +
2191
3230
  `but a wrong-mode previewOrigin (${config.previewOrigin}); re-seeding...`,
2192
3231
  );
2193
- // Re-run the seed: stack.json is now readable, so the wrong-mode heal writes
2194
- // the dedicated origin into editor.local.json (a published `<N>-preview`
2195
- // tunnel URL, else the localhost bootstrap for the window before the monitor's
2196
- // first tick).
2197
- await seedPreviewOrigin(cfg, appPort);
3232
+ // Re-run the seed with the verdict we just read, so WHETHER to heal and WHAT
3233
+ // to write come from the same source. Passing it is what makes this heal reach
3234
+ // the adaptive-default stacks it exists for: on a Next.js project whose
3235
+ // `stack.json` omits `sameOriginSafe` (the scaffolded shape), the static read
3236
+ // inside the seed answers `true` and re-writes the same-origin sentinel that
3237
+ // just failed to hydrate. We got here precisely BECAUSE the editor said
3238
+ // `false`, so `false` is the answer the seed must use.
3239
+ await seedPreviewOrigin(cfg, appPort, false);
2198
3240
  // Tell the running server to re-read editor.local.json — no restart (the
2199
3241
  // footgun that killed VM7's live Claude session).
2200
3242
  const reload = runGcloud(
@@ -2289,6 +3331,66 @@ async function seedAgentApiKey(cfg) {
2289
3331
  // vm--no-cli-install-before-container-started removed; do not re-add it.
2290
3332
  }
2291
3333
 
3334
+ /** Pure: deep-merge `patch` over the VM's current editor.local.json content and
3335
+ * return the base64 of the pretty-printed result, ready for
3336
+ * buildEditorLocalSeedWriteCommand. `currentRaw` is the raw `cat` output of the
3337
+ * remote file (may be undefined, empty, or corrupt) — anything that doesn't
3338
+ * parse to a plain object is treated as `{}`, because overwriting a corrupt
3339
+ * override file with a known-good one is safer than refusing to seed. Extracted
3340
+ * so seedGreenfieldProjectTitle stays a thin I/O passthrough and the
3341
+ * parse+merge+encode is unit-tested without a live VM. */
3342
+ function mergeEditorLocalPatchB64(currentRaw, patch) {
3343
+ let current = {};
3344
+ try {
3345
+ const parsed = JSON.parse(currentRaw);
3346
+ if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
3347
+ current = parsed;
3348
+ }
3349
+ } catch {
3350
+ // Unparseable / missing remote content → start from {}.
3351
+ }
3352
+ const merged = deepMergeJson(current, patch);
3353
+ return Buffer.from(JSON.stringify(merged, null, 2), "utf8").toString("base64");
3354
+ }
3355
+
3356
+ /** Seed a GREENFIELD project's `projectTitle` into the VM's editor.local.json
3357
+ * BEFORE compose up, so the editor caches the real project name at startup and
3358
+ * the fleet dashboard renders the correct card label with no false
3359
+ * project-dir-drift. Like seedAgentApiKey, this MUST run before
3360
+ * ensureEditorContainerUp — the editor reads its config once at startup.
3361
+ *
3362
+ * No-op for a clone (buildGreenfieldTitlePatch returns null): a repo Add labels
3363
+ * from its origin basename, so there is nothing to seed and no SSH round-trip
3364
+ * is spent. Runs on every cloud:up, so a reconfigure re-seeds the title,
3365
+ * converging a VM provisioned before this fix onto the correct identity without
3366
+ * a destroy. Thin I/O passthrough: the parse+merge+encode lives in the pure
3367
+ * mergeEditorLocalPatchB64 helper. */
3368
+ async function seedGreenfieldProjectTitle(cfg) {
3369
+ const patch = buildGreenfieldTitlePatch(cfg);
3370
+ if (!patch) return; // clone (or no greenfield name) — nothing to seed
3371
+
3372
+ const repoName = resolveProject(cfg).name;
3373
+ const remotePath = `~/projects/${repoName}/.codeyam/editor.local.json`;
3374
+ console.error(`Seeding projectTitle='${patch.projectTitle}' into ${remotePath}...`);
3375
+
3376
+ const read = runGcloud(
3377
+ buildGcloudArgs("ssh-probe", cfg, {
3378
+ command: `cat ${remotePath} 2>/dev/null || echo '{}'`,
3379
+ }),
3380
+ { stdio: ["ignore", "pipe", "ignore"] },
3381
+ );
3382
+ const b64 = mergeEditorLocalPatchB64(read.stdout, patch);
3383
+ const write = runGcloud(
3384
+ buildGcloudArgs("ssh-probe", cfg, {
3385
+ command: buildEditorLocalSeedWriteCommand({ repoName, remotePath, b64 }),
3386
+ }),
3387
+ { stdio: ["ignore", "inherit", "inherit"] },
3388
+ );
3389
+ if (write.status !== 0) {
3390
+ throw new Error(`Failed to write ${remotePath} on VM`);
3391
+ }
3392
+ }
3393
+
2292
3394
  /** Tri-state probe of the editor container's running state on the VM host.
2293
3395
  * One ssh round-trip; returns `"running"`, `"not-running"`, or `"unknown"`.
2294
3396
  *
@@ -2532,7 +3634,16 @@ function promptForInstanceName(expected) {
2532
3634
  });
2533
3635
  }
2534
3636
 
2535
- const STREAMED_SCRIPTS = ["scripts/gcp-bootstrap.sh", "scripts/gcp-project-bootstrap.sh"];
3637
+ // Every file streamed to a VM, and therefore every file the drift guard must
3638
+ // compare against the reviewed remote tip. `scripts/lib/npm-pin.sh` is here
3639
+ // because gcp-project-bootstrap.sh SOURCES it: a library that reaches the VM is
3640
+ // as much streamed content as the script that sources it, and leaving it out
3641
+ // meant the drift guard silently vouched for only half of what was shipped.
3642
+ const STREAMED_SCRIPTS = [
3643
+ "scripts/gcp-bootstrap.sh",
3644
+ "scripts/gcp-project-bootstrap.sh",
3645
+ "scripts/lib/npm-pin.sh",
3646
+ ];
2536
3647
 
2537
3648
  /** The configured editor branch to compare/stream streamed scripts against.
2538
3649
  * Resolved from config (`resolveEditorConfig` → `editorBranch`, itself derived
@@ -2551,6 +3662,42 @@ function streamedScriptMergeBase(editorBranch) {
2551
3662
  return mbResult.status === 0 ? mbResult.stdout.trim() : null;
2552
3663
  }
2553
3664
 
3665
+ /** Fetch `editorBranch` from origin into a REAL remote-tracking ref
3666
+ * (`refs/remotes/origin/<branch>`) rather than only into `FETCH_HEAD`.
3667
+ *
3668
+ * Operator hosts are commonly shallow, single-branch clones that track only
3669
+ * `+refs/heads/main`, so `origin/<some-feature-branch>` is not a local ref and
3670
+ * every `git show origin/<branch>:<path>` against it fails. Fetching through an
3671
+ * explicit refspec creates the ref, which keeps the drift verdict and the bytes
3672
+ * actually streamed derived from the same object. Falls back to a plain fetch
3673
+ * (FETCH_HEAD only) when the refspec form is rejected — `readRemoteScriptBlob`
3674
+ * can still read through FETCH_HEAD in that case. */
3675
+ function fetchStreamedScriptRef(editorBranch) {
3676
+ const refspec = `${editorBranch}:refs/remotes/origin/${editorBranch}`;
3677
+ const withRef = spawnSync("git", ["fetch", "--force", "origin", refspec], { encoding: "utf8" });
3678
+ if (withRef.status === 0) return { ok: true, stderr: "" };
3679
+
3680
+ const plain = spawnSync("git", ["fetch", "origin", editorBranch], { encoding: "utf8" });
3681
+ if (plain.status === 0) return { ok: true, stderr: "" };
3682
+ return { ok: false, stderr: (plain.stderr || withRef.stderr || "").trim() };
3683
+ }
3684
+
3685
+ /** Resolve one revision of a streamed script from the fetched remote, trying the
3686
+ * remote-tracking ref first and `FETCH_HEAD` second. `gitArg` is the git verb
3687
+ * (`rev-parse` for the blob hash, `show` for the bytes) so the drift facts and the
3688
+ * streamed bytes resolve through the SAME ref-preference order — a verdict of
3689
+ * "stream remote" can therefore never disagree with what actually got streamed.
3690
+ * Returns `{ ok: false }` only when the blob is unreadable through both refs. */
3691
+ function resolveRemoteScriptRev(gitArg, relPath, editorBranch) {
3692
+ for (const ref of [`origin/${editorBranch}`, "FETCH_HEAD"]) {
3693
+ const result = spawnSync("git", [gitArg, `${ref}:${relPath}`], { encoding: "utf8" });
3694
+ if (result.status === 0 && typeof result.stdout === "string") {
3695
+ return { ok: true, stdout: result.stdout, ref };
3696
+ }
3697
+ }
3698
+ return { ok: false, stdout: "", ref: null };
3699
+ }
3700
+
2554
3701
  /** Gather the drift facts for one streamed script relative to `origin/<editorBranch>`:
2555
3702
  * its local (working-tree) blob hash, the remote-tip blob hash, the merge-base blob
2556
3703
  * hash, and whether the working tree is dirty. Any git failure yields a null hash,
@@ -2562,11 +3709,8 @@ function gatherScriptDriftFacts(relPath, editorBranch, mergeBase) {
2562
3709
  const localHashResult = spawnSync("git", ["hash-object", relPath], { encoding: "utf8" });
2563
3710
  const localHash = localHashResult.status === 0 ? localHashResult.stdout.trim() : null;
2564
3711
 
2565
- let remoteHash = null;
2566
- const remoteHashResult = spawnSync("git", ["rev-parse", `origin/${editorBranch}:${relPath}`], { encoding: "utf8" });
2567
- if (remoteHashResult.status === 0) {
2568
- remoteHash = remoteHashResult.stdout.trim();
2569
- }
3712
+ const remoteRev = resolveRemoteScriptRev("rev-parse", relPath, editorBranch);
3713
+ const remoteHash = remoteRev.ok ? remoteRev.stdout.trim() : null;
2570
3714
 
2571
3715
  let baseHash = null;
2572
3716
  if (mergeBase) {
@@ -2589,7 +3733,15 @@ function gatherScriptDriftFacts(relPath, editorBranch, mergeBase) {
2589
3733
  * un-pulled dashboard can't add VMs" failure. Streams the on-disk copy instead
2590
3734
  * when the operator authored local content the remote tip lacks (dirty edit or a
2591
3735
  * committed local-ahead change — developer mode) or when the remote is unreachable
2592
- * (offline fallback). `absScriptPath` is an absolute path under the repo root. */
3736
+ * (offline fallback — `remoteHash === null`, the only sanctioned on-disk path).
3737
+ *
3738
+ * A "remote" verdict NEVER falls back to the on-disk copy. On a clean-but-behind
3739
+ * checkout the on-disk bytes are precisely the stale bytes the drift guard exists
3740
+ * to keep off the VM, so substituting them silently defeats the guard: the verdict
3741
+ * said "stream the reviewed remote tip" and the VM would instead receive the old
3742
+ * script, with only a `[Warning]` to show for it. When the remote blob cannot be
3743
+ * read through either ref this throws, aborting the add.
3744
+ * `absScriptPath` is an absolute path under the repo root. */
2593
3745
  function readStreamedScript(absScriptPath, cfg = {}) {
2594
3746
  const relPath = path.relative(rootDir, absScriptPath) || absScriptPath;
2595
3747
  const editorBranch = streamedScriptEditorBranch(cfg);
@@ -2598,15 +3750,18 @@ function readStreamedScript(absScriptPath, cfg = {}) {
2598
3750
 
2599
3751
  const { streamedScriptSource } = require("./script-drift");
2600
3752
  if (streamedScriptSource(facts) === "remote") {
2601
- const remote = spawnSync("git", ["show", `origin/${editorBranch}:${relPath}`], { encoding: "utf8" });
2602
- if (remote.status === 0 && typeof remote.stdout === "string") {
2603
- console.error(` streaming canonical ${relPath} from origin/${editorBranch} (checkout behind remote is fine)`);
3753
+ const remote = resolveRemoteScriptRev("show", relPath, editorBranch);
3754
+ if (remote.ok) {
3755
+ console.error(` streaming canonical ${relPath} from ${remote.ref} (checkout behind remote is fine)`);
2604
3756
  return remote.stdout;
2605
3757
  }
2606
- // Remote read failed unexpectedly (raced fetch, ref pruned): fall through to disk.
2607
- console.error(
2608
- ` [Warning] Could not read ${relPath} from origin/${editorBranch} ` +
2609
- `(${(remote.stderr || "").trim() || "unavailable"}). Falling back to the on-disk copy.`,
3758
+ // The facts said "the remote tip is the canonical source" but its bytes are
3759
+ // unreadable. Streaming the on-disk copy here would provision unreviewed stale
3760
+ // bytes under the banner of a successful add — fail loud instead.
3761
+ throw new Error(
3762
+ `could not read ${relPath} from origin/${editorBranch} (or FETCH_HEAD); ` +
3763
+ `refusing to stream a possibly-stale on-disk copy. ` +
3764
+ `Fetch the branch first: git fetch origin ${editorBranch}:refs/remotes/origin/${editorBranch}`,
2610
3765
  );
2611
3766
  }
2612
3767
  return fs.readFileSync(absScriptPath, "utf8");
@@ -2628,10 +3783,11 @@ function checkScriptDrift(cfg) {
2628
3783
 
2629
3784
  console.error(`Checking streamed-script drift against origin/${editorBranch}...`);
2630
3785
 
2631
- // 1. git fetch origin <editorBranch>
2632
- const fetchResult = spawnSync("git", ["fetch", "origin", editorBranch], { encoding: "utf8" });
2633
- if (fetchResult.status !== 0) {
2634
- console.error(` [Warning] Failed to fetch origin ${editorBranch} (${(fetchResult.stderr || "").trim() || "unknown error"}). Skipping drift check.`);
3786
+ // 1. Fetch into refs/remotes/origin/<editorBranch> so the ref resolves even on a
3787
+ // single-branch clone that tracks only +refs/heads/main.
3788
+ const fetchResult = fetchStreamedScriptRef(editorBranch);
3789
+ if (!fetchResult.ok) {
3790
+ console.error(` [Warning] Failed to fetch origin ${editorBranch} (${fetchResult.stderr || "unknown error"}). Skipping drift check.`);
2635
3791
  return;
2636
3792
  }
2637
3793
 
@@ -2724,8 +3880,9 @@ function fleetInstancePrefix() {
2724
3880
  * scripts/gcp/vm-ingress.sh with the resolved coordinates. Returns the spawnSync
2725
3881
  * result so callers decide fatality — Add treats a non-zero as a loud warning
2726
3882
  * (the VM is provisioned; ingress can be re-wired), Destroy likewise warns so a
2727
- * teardown still removes the instance. */
2728
- function runVmIngress(cfg, action, vmNumber, domain, appPort, slug) {
3883
+ * teardown still removes the instance. `vmAccess` (up only) is the Add's intended
3884
+ * principal set, which the script's grant step uses in place of its sidecar read. */
3885
+ function runVmIngress(cfg, action, vmNumber, domain, appPort, slug, vmAccess) {
2729
3886
  const scriptPath = path.join(rootDir, "scripts", "gcp", "vm-ingress.sh");
2730
3887
  const inv = vmIngressInvocation({
2731
3888
  scriptPath,
@@ -2737,6 +3894,7 @@ function runVmIngress(cfg, action, vmNumber, domain, appPort, slug) {
2737
3894
  appPort,
2738
3895
  instancePrefix: fleetInstancePrefix(),
2739
3896
  slug,
3897
+ vmAccess,
2740
3898
  });
2741
3899
  return spawnSync(inv.command, inv.args, {
2742
3900
  stdio: "inherit",
@@ -2744,17 +3902,100 @@ function runVmIngress(cfg, action, vmNumber, domain, appPort, slug) {
2744
3902
  });
2745
3903
  }
2746
3904
 
3905
+ /** Grant-only pass: (re-)apply one VM's default IAP access grant by spawning
3906
+ * `scripts/gcp/vm-ingress.sh <N> grant-access` — the churn-free action that runs
3907
+ * `apply_default_iap_access` without touching any NEG/BES/host-rule resource.
3908
+ * Reconcile uses this for already-`wired` VMs its re-wire loop skips, so a healed
3909
+ * VM whose backends already exist still (re-)grants the automation SA the IAP
3910
+ * accessor. Returns the spawnSync result so the caller logs granted vs failed;
3911
+ * best-effort + loud, never fatal. */
3912
+ function runVmIngressGrantAccess(cfg, vmNumber, domain, slug) {
3913
+ const scriptPath = path.join(rootDir, "scripts", "gcp", "vm-ingress.sh");
3914
+ const inv = vmIngressGrantAccessInvocation({
3915
+ scriptPath,
3916
+ vmNumber,
3917
+ domain,
3918
+ project: cfg.project,
3919
+ zone: cfg.zone,
3920
+ slug,
3921
+ });
3922
+ return spawnSync(inv.command, inv.args, {
3923
+ stdio: "inherit",
3924
+ env: { ...process.env, ...inv.env },
3925
+ });
3926
+ }
3927
+
3928
+ /** THE principal set that must have IAP access to this VM after the Add: the
3929
+ * explicit `cfg.vmAccess` collaborators unioned with the resolved default
3930
+ * operator. Both add-time access paths consume this ONE function —
3931
+ * `wireVmIngressOnAdd` (which hands it to vm-ingress.sh's grant) and
3932
+ * `applyVmAccessOnAdd` (which grants it and subtracts it from the stale-revoke
3933
+ * set). They used to decide independently, and that is precisely how one path
3934
+ * granted `user:jared@codeyam.com` at 00:09:31 and the other revoked the same
3935
+ * principal 18 seconds later as a "stale prior tenant" (VMs 4/5/6, 2026-08-09):
3936
+ * the ingress path read the SIDECAR (still holding the prior tenant) while
3937
+ * `applyVmAccessOnAdd` resolved an empty operator. One source of truth removes
3938
+ * the whole class of disagreement. `env` is injectable for tests; the only I/O
3939
+ * is `resolveDefaultOperator`'s config/gcloud read. */
3940
+ function intendedVmPrincipals(cfg = {}, env = process.env) {
3941
+ return withDefaultOperator(
3942
+ normalizeAccessList(cfg.vmAccess || []),
3943
+ resolveDefaultOperator({ env }),
3944
+ );
3945
+ }
3946
+
3947
+ /** Add preflight: throw — BEFORE any GCE instance exists — when this Add would
3948
+ * come up on an IAP-gated public URL that authorizes NOBODY. The whole condition
3949
+ * is env + config (`intendedVmPrincipals` needs no GCP state), which is why it
3950
+ * belongs beside cmdUp's other cheap boundary validations rather than after a
3951
+ * machine has been provisioned: checking later means a half-provisioned VM to
3952
+ * clean up. On 2026-08-09 the un-refused version of this Add reported `done` for
3953
+ * VMs 4/5/6, left three unreachable URLs, and revoked each slot's prior tenant on
3954
+ * the way.
3955
+ *
3956
+ * Only fires in LB mode on a numbered fleet VM with nothing to grant; an explicit
3957
+ * `--vm-access` satisfies it with no default operator configured, so the
3958
+ * collaborator flow is unaffected, as is every legacy quick-tunnel Add. The
3959
+ * refusal text (and its `Next valid action:` remedy naming all three config
3960
+ * tiers) is the pure `unauthorizableAddRefusal`; this function is the composition
3961
+ * and the throw. */
3962
+ function assertAddCanAuthorizeSomeone(cfg, env = process.env) {
3963
+ let plan;
3964
+ try {
3965
+ plan = ingressWiringPlan(cfg, env);
3966
+ } catch {
3967
+ // A malformed domain is `resolveIngressDomain`'s error to raise on the wiring
3968
+ // path; it must not masquerade here as an authorization refusal.
3969
+ plan = { wire: false };
3970
+ }
3971
+ const refusal = unauthorizableAddRefusal({
3972
+ lbMode: !!plan.wire,
3973
+ vmNumber: plan.wire ? plan.vmNumber : null,
3974
+ intendedCount: plan.wire ? intendedVmPrincipals(cfg, env).length : null,
3975
+ });
3976
+ if (refusal) throw new Error(refusal);
3977
+ }
3978
+
2747
3979
  /** Add-time hook: wire VM-N onto the shared LB when a fleet domain is set.
2748
3980
  * Best-effort + loud — a wiring hiccup must never abort an otherwise-green Add
2749
3981
  * (the VM is reachable operator-side over SSH regardless), so on failure it
2750
3982
  * prints the exact re-run command rather than throwing. No-op (and silent) in
2751
- * legacy quick-tunnel mode or for a non-fleet client project. */
3983
+ * legacy quick-tunnel mode or for a non-fleet client project.
3984
+ *
3985
+ * RETURNS the principals this call granted (`intendedVmPrincipals`), or [] when
3986
+ * it did not wire — the explicit dataflow `applyVmAccessOnAdd` needs to know what
3987
+ * the Add has already granted before it computes a stale-revoke. Returned on the
3988
+ * FAILURE path too: a non-zero `vm-ingress.sh up` may still have completed its
3989
+ * grant step, and a principal that was possibly granted must never be revoked. */
2752
3990
  function wireVmIngressOnAdd(cfg, appPort, env = process.env) {
2753
3991
  const plan = ingressWiringPlan(cfg, env);
2754
- if (!plan.wire) return;
3992
+ if (!plan.wire) return [];
2755
3993
  const label = plan.slug || `vm-${plan.vmNumber}`;
3994
+ // Hand the intended set to the script so its `apply_default_iap_access` grants
3995
+ // THIS Add's principals instead of reading the sidecar's prior tenant.
3996
+ const vmAccess = intendedVmPrincipals(cfg, env);
2756
3997
  console.error(` wiring LB ingress for VM-${plan.vmNumber} on ${plan.domain}...`);
2757
- const res = runVmIngress(cfg, "up", plan.vmNumber, plan.domain, appPort, plan.slug);
3998
+ const res = runVmIngress(cfg, "up", plan.vmNumber, plan.domain, appPort, plan.slug, vmAccess);
2758
3999
  if (res.status === 0) {
2759
4000
  console.error(` ✓ LB ingress wired: https://${label}.editor.${plan.domain}`);
2760
4001
  } else {
@@ -2776,6 +4017,7 @@ function wireVmIngressOnAdd(cfg, appPort, env = process.env) {
2776
4017
  `(vm-ingress.sh up exited ${res.status == null ? "abnormally" : res.status}). Re-wire: ${rewire}`,
2777
4018
  );
2778
4019
  }
4020
+ return vmAccess;
2779
4021
  }
2780
4022
 
2781
4023
  /** Destroy-time hook: unwire VM-N's LB backends before the instance is deleted,
@@ -2844,8 +4086,15 @@ function runIapAccess(cfg, vmNumber, action, principals) {
2844
4086
  * a revoke/grant hiccup (e.g. the SA missing IAP-admin) prints the exact re-run
2845
4087
  * command rather than aborting an otherwise-green Add. `cfg.vmAccess` is the
2846
4088
  * array (or comma-separated string) cmdUp validated; absent ⇒ an empty desired
2847
- * list (the clean-slate case). */
2848
- function applyVmAccessOnAdd(cfg, env = process.env) {
4089
+ * list (the clean-slate case).
4090
+ *
4091
+ * `grantedThisRun` is what an EARLIER step of this same Add already granted —
4092
+ * `wireVmIngressOnAdd`'s return value. It is subtracted from the stale-revoke
4093
+ * set, so a principal this Add just authorized can never be revoked moments later
4094
+ * as a prior tenant. That is a second, cheaper guard than the shared
4095
+ * `intendedVmPrincipals` resolver above: it holds even if the two paths drift
4096
+ * apart again, which is exactly how VMs 4/5/6 broke on 2026-08-09. */
4097
+ function applyVmAccessOnAdd(cfg, env = process.env, { grantedThisRun = [] } = {}) {
2849
4098
  const desired = normalizeAccessList(cfg.vmAccess || []);
2850
4099
  let plan;
2851
4100
  try {
@@ -2862,11 +4111,15 @@ function applyVmAccessOnAdd(cfg, env = process.env) {
2862
4111
  // never inaccessible-by-default (the Error-51 footgun). The default operator is
2863
4112
  // NOT persisted to the sidecar (it's a standing grant, not a per-VM
2864
4113
  // collaborator), so `desired` alone drives the sidecar + stale-revoke below.
2865
- const toGrant = withDefaultOperator(desired, resolveDefaultOperator({ env }));
4114
+ const toGrant = intendedVmPrincipals(cfg, env);
2866
4115
  const stale = accessFor(vmNumber);
2867
4116
  // Never revoke a principal we're about to (re-)grant — a prior tenant who was
2868
- // also the default operator must not be revoked-then-regranted for nothing.
2869
- const staleOnly = stale.filter((p) => !toGrant.includes(p));
4117
+ // also the default operator must not be revoked-then-regranted for nothing —
4118
+ // NOR one an earlier step of this same Add already granted. `toGrant` alone
4119
+ // protected nothing when it was empty: the ingress step had granted the operator
4120
+ // seconds earlier and this revoke took it straight back off (VMs 4/5/6).
4121
+ const grantedAlready = new Set([...toGrant, ...normalizeAccessList(grantedThisRun || [])]);
4122
+ const staleOnly = stale.filter((p) => !grantedAlready.has(p));
2870
4123
 
2871
4124
  if (plan.wire) {
2872
4125
  if (staleOnly.length > 0) {
@@ -2983,12 +4236,23 @@ async function convergeVmIngressOnAdd(cfg, appPort, env = process.env, deps = {}
2983
4236
  }
2984
4237
  if (!plan.wire) {
2985
4238
  // Nothing to converge — the no-op wire keeps the clean-slate invariant.
2986
- wireVmIngressOnAdd(cfg, appPort, env);
2987
- return { wired: true, reason: "no-wire" };
2988
- }
2989
-
4239
+ const granted = wireVmIngressOnAdd(cfg, appPort, env);
4240
+ return { wired: true, reason: "no-wire", grantedThisRun: granted };
4241
+ }
4242
+
4243
+ // The intended set, resolved ONCE and handed to every `up` attempt so each
4244
+ // converge iteration grants the same principals — and reported back to the
4245
+ // caller as `grantedThisRun` so the access reconcile that follows cannot revoke
4246
+ // them. Every `up` runs the script's grant step, so these are granted whether
4247
+ // the loop converges or hits its deadline. Injectable via `deps` like the rest
4248
+ // of this loop's slow I/O: the resolve spawns `gcloud config get-value account`
4249
+ // on its last tier, and a unit test must never depend on the gcloud identity
4250
+ // that happens to be active on the machine running it.
4251
+ const grantedThisRun =
4252
+ deps.grantedThisRun != null ? deps.grantedThisRun : intendedVmPrincipals(cfg, env);
2990
4253
  const runUp =
2991
- deps.runUp || ((p) => runVmIngress(cfg, "up", p.vmNumber, p.domain, appPort, p.slug));
4254
+ deps.runUp
4255
+ || ((p) => runVmIngress(cfg, "up", p.vmNumber, p.domain, appPort, p.slug, grantedThisRun));
2992
4256
  const probeVerdict = deps.probeVerdict || ((p) => probeVmIngressVerdict(cfg, p));
2993
4257
  const sleep = deps.sleep || ((ms) => new Promise((r) => setTimeout(r, ms)));
2994
4258
  const now = deps.now || (() => Date.now());
@@ -3031,7 +4295,7 @@ async function convergeVmIngressOnAdd(cfg, appPort, env = process.env, deps = {}
3031
4295
  if (decision.done) {
3032
4296
  if (decision.wired) {
3033
4297
  console.error(` ✓ LB ingress wired: https://${label}.editor.${plan.domain}`);
3034
- return { wired: true, reason: "wired", attempts: attempt + 1 };
4298
+ return { wired: true, reason: "wired", attempts: attempt + 1, grantedThisRun };
3035
4299
  }
3036
4300
  const rewire =
3037
4301
  `FLEET_DOMAIN=${plan.domain}${plan.slug ? ` VM_SLUG=${plan.slug}` : ""} ` +
@@ -3048,7 +4312,7 @@ async function convergeVmIngressOnAdd(cfg, appPort, env = process.env, deps = {}
3048
4312
  `CY_DEGRADED ingress-wiring-failed: VM-${plan.vmNumber} public URL did not wire ` +
3049
4313
  `within the add window (last verdict=${verdict || "unknown"}). Re-wire: ${rewire}`,
3050
4314
  );
3051
- return { wired: false, reason: "deadline", attempts: attempt + 1 };
4315
+ return { wired: false, reason: "deadline", attempts: attempt + 1, grantedThisRun };
3052
4316
  }
3053
4317
  attempt += 1;
3054
4318
  await sleep(decision.delayMs);
@@ -3077,8 +4341,8 @@ function startDetachedAddWiring(cfg, appPort, env = process.env) {
3077
4341
  if (!plan.wire) {
3078
4342
  // Nothing slow to detach: wiring is a no-op and the access reconcile is a cheap
3079
4343
  // sidecar write. Run them inline so the clean-slate invariant still holds.
3080
- wireVmIngressOnAdd(cfg, appPort, env);
3081
- applyVmAccessOnAdd(cfg, env);
4344
+ const grantedThisRun = wireVmIngressOnAdd(cfg, appPort, env);
4345
+ applyVmAccessOnAdd(cfg, env, { grantedThisRun });
3082
4346
  return;
3083
4347
  }
3084
4348
  const marker = ingressInflightMarkerPath(INGRESS_INFLIGHT_DIR, plan.vmNumber);
@@ -3110,8 +4374,8 @@ function startDetachedAddWiring(cfg, appPort, env = process.env) {
3110
4374
  `running it inline instead.`,
3111
4375
  );
3112
4376
  try { fs.unlinkSync(marker); } catch { /* marker may not exist */ }
3113
- wireVmIngressOnAdd(cfg, appPort, env);
3114
- applyVmAccessOnAdd(cfg, env);
4377
+ const grantedThisRun = wireVmIngressOnAdd(cfg, appPort, env);
4378
+ applyVmAccessOnAdd(cfg, env, { grantedThisRun });
3115
4379
  }
3116
4380
  }
3117
4381
 
@@ -3143,8 +4407,11 @@ async function cmdWireAddTail() {
3143
4407
  // Converge the URL to `verdict=wired` (retry-until-wired), not a single-shot
3144
4408
  // `up` left to the slow reconcile sweep — then the IAP access grant, in the
3145
4409
  // SAME order cmdUp ran them inline.
3146
- await convergeVmIngressOnAdd(cfg, appPort);
3147
- applyVmAccessOnAdd(cfg);
4410
+ // Carry what the converge granted into the access reconcile, so the revoke
4411
+ // step cannot take back a principal this same Add just authorized — the exact
4412
+ // grant-then-revoke-18s-later sequence `/tmp/cy-wire-add-vm-4.log` recorded.
4413
+ const { grantedThisRun } = await convergeVmIngressOnAdd(cfg, appPort);
4414
+ applyVmAccessOnAdd(cfg, process.env, { grantedThisRun });
3148
4415
  } finally {
3149
4416
  if (marker) {
3150
4417
  try { fs.unlinkSync(marker); } catch { /* already gone — fine */ }
@@ -3206,6 +4473,75 @@ function releaseVmSlugOnDestroy(cfg) {
3206
4473
  return vmNumber;
3207
4474
  }
3208
4475
 
4476
+ /** Does `branch` still exist on `repo`? Answered with a single
4477
+ * `git ls-remote --heads` — about a second, and no cloud resource touched.
4478
+ *
4479
+ * Tri-state on purpose, and the third state is the load-bearing one:
4480
+ * true — the ref resolved.
4481
+ * false — the remote answered and the ref is NOT there. Provably gone.
4482
+ * null — we could not get an answer (no network, no credentials, git
4483
+ * missing, timeout). NOT evidence of a deleted branch.
4484
+ * Only `false` refuses an add. Collapsing `null` into `false` would turn every
4485
+ * transient GitHub blip into a fleet-wide add outage, which is a far worse
4486
+ * failure than the one this preflight exists to prevent — and the classifier
4487
+ * arm downstream still catches a genuinely dead ref, just later and dearer.
4488
+ *
4489
+ * A private repo needs credentials, so the token is inlined into the URL the
4490
+ * same way clone_editor_repo does it on the VM side (x-access-token), and for
4491
+ * the same reason: the plain URL 401s. The token is confined to this argv and
4492
+ * never logged — the caller prints `repo` (the un-inlined form) instead. */
4493
+ function gitBranchRefExists({ repo, branch }) {
4494
+ if (!repo || !branch) return null;
4495
+ let url = repo;
4496
+ const token = resolveGithubToken();
4497
+ if (token && /^https:\/\/github\.com\//.test(repo)) {
4498
+ url = repo.replace("https://", `https://x-access-token:${token}@`);
4499
+ }
4500
+ const res = spawnSync("git", ["ls-remote", "--heads", url, branch], {
4501
+ encoding: "utf8",
4502
+ timeout: 30000,
4503
+ });
4504
+ // A non-zero exit is an unreachable/unauthorized remote, not an absent branch:
4505
+ // git exits 0 and prints nothing for "connected fine, no such head".
4506
+ if (res.error || res.status !== 0) return null;
4507
+ return String(res.stdout || "").trim().length > 0;
4508
+ }
4509
+
4510
+ /** Refuse an add whose stored branch has been deleted from origin, BEFORE the
4511
+ * first gcloud call.
4512
+ *
4513
+ * Without this the failure lands as late and as expensively as it possibly can:
4514
+ * on 2026-08-25 four fleet VMs each created a real GCE instance, booted it,
4515
+ * installed npm and cloned 13,226 files before `git fetch` inside the VM said
4516
+ * `fatal: couldn't find remote ref editor-improvements-75` — a branch pruned
4517
+ * from origin hours earlier when it merged. One `git ls-remote` establishes the
4518
+ * same fact in about a second, for free. This repo prunes branches on merge, so
4519
+ * ANY stored feature-branch name expires silently; that makes this a recurring
4520
+ * condition, not a one-off.
4521
+ *
4522
+ * Placed beside checkScriptDrift and assertVariantProvisionable — the other
4523
+ * "fail loud before creating a GCE instance" guards — because it is only worth
4524
+ * anything ahead of ensureInstance. It cannot live in scripts/gcp-bootstrap.sh,
4525
+ * which already runs ON the provisioned box and never sees the client branch.
4526
+ *
4527
+ * Throws with the field, the branch and the repo named. Never throws for a ref
4528
+ * it could not check (see gitBranchRefExists). */
4529
+ function assertBranchRefsExist(cfg, { refExists = gitBranchRefExists } = {}) {
4530
+ // Lazy require: launch-config.js requires THIS module at load time, so a
4531
+ // top-level require here would close the cycle. Same pattern as the
4532
+ // isPlaceholderProject call sites above.
4533
+ const lc = require("./launch-config");
4534
+ const { editorRepo, editorBranch } = resolveEditorConfig(cfg);
4535
+ const targets = lc.branchRefTargets(
4536
+ { clientRepo: cfg.repo, clientBranch: cfg.branch, editorBranch },
4537
+ { editorRepo },
4538
+ );
4539
+ if (!targets.length) return;
4540
+ console.error("Checking that the requested branches still exist on origin...");
4541
+ const missing = lc.missingBranchRefs(targets, refExists);
4542
+ if (missing.length) throw new Error(lc.missingBranchRefMessage(missing));
4543
+ }
4544
+
3209
4545
  async function cmdUp(args, projectDir = rootDir) {
3210
4546
  ensureGcloud();
3211
4547
  const flags = parseFlags(args);
@@ -3256,8 +4592,14 @@ async function cmdUp(args, projectDir = rootDir) {
3256
4592
  }
3257
4593
  }
3258
4594
 
4595
+ assertAddCanAuthorizeSomeone(cfg);
4596
+
3259
4597
  checkScriptDrift(cfg);
3260
4598
 
4599
+ // Last of the pre-instance guards: a deleted branch ref is checkable in ~1s
4600
+ // and otherwise costs a full instance + boot + 13k-file clone to discover.
4601
+ assertBranchRefsExist(cfg);
4602
+
3261
4603
  await ensureFirewallRule(cfg);
3262
4604
  const { createdThisRun } = await ensureInstance(cfg);
3263
4605
 
@@ -3287,6 +4629,10 @@ async function cmdUp(args, projectDir = rootDir) {
3287
4629
  }
3288
4630
  }
3289
4631
  await seedAgentApiKey(cfg);
4632
+ // Greenfield only (no-op for a clone): seed projectTitle so the editor
4633
+ // resolves its identity to the real project name instead of the bare
4634
+ // /workspace mount basename — the source of the false project-dir-drift.
4635
+ await seedGreenfieldProjectTitle(cfg);
3290
4636
  await ensureEditorContainerUp(cfg);
3291
4637
  // Provision contract: a just-provisioned VM must actually reach GitHub.
3292
4638
  // Verify in-container git auth now that the container (and its credential
@@ -3358,7 +4704,27 @@ async function cmdUp(args, projectDir = rootDir) {
3358
4704
  );
3359
4705
  }
3360
4706
  } catch (err) {
3361
- if (createdThisRun) {
4707
+ // `createdThisRun` alone MUST NOT authorize deletion. Gather what state the
4708
+ // VM is in right now, then let the pure decision rule — see
4709
+ // abortTeardownDecision's doc comment for why each input is load-bearing.
4710
+ const state = probeAbortTeardownState(cfg, createdThisRun);
4711
+ const verdict = abortTeardownDecision({ createdThisRun, ...state });
4712
+ if (createdThisRun && verdict === "keep") {
4713
+ // Deliberately loud and specific. The old blanket "tearing down the VM
4714
+ // cloud:up created this run (no orphan left behind)" was actively
4715
+ // misleading here — it asserted no orphan while deleting a working VM.
4716
+ const why = state.probeInconclusive
4717
+ ? "its liveness could not be confirmed (probe failed), and an unconfirmed VM is never deleted"
4718
+ : state.containerUp
4719
+ ? "its editor container is RUNNING — another run brought it up"
4720
+ : "it is in the fleet roster and visible to the dashboard";
4721
+ console.error(
4722
+ `✗ provisioning aborted — the VM ${cfg.instanceName} was LEFT IN PLACE because ${why}.\n` +
4723
+ ` This run created the instance, but that alone does not authorize deleting a VM that is in service.\n` +
4724
+ ` If it really should go, remove it deliberately: npm run cloud:destroy`,
4725
+ );
4726
+ }
4727
+ if (verdict === "delete") {
3362
4728
  console.error(
3363
4729
  `✗ provisioning aborted — tearing down the VM cloud:up created this run (no orphan left behind)`,
3364
4730
  );
@@ -3540,14 +4906,7 @@ async function cmdStatus(args, projectDir = rootDir) {
3540
4906
  const imgStatus = String(imgProbe.stdout || "").trim() || "?";
3541
4907
  console.error(` image: ${imgStatus}`);
3542
4908
 
3543
- const repoName = resolveProject(cfg).name;
3544
- const containerProbe = runGcloud(
3545
- buildGcloudArgs("ssh-probe", cfg, {
3546
- command: `cd ~/projects/${repoName} 2>/dev/null && docker compose -f docker-compose.cloud.yml ps --format json 2>/dev/null || echo`,
3547
- }),
3548
- { stdio: ["ignore", "pipe", "ignore"] },
3549
- );
3550
- const containerStatus = parseEditorComposeState(containerProbe.stdout) || "missing";
4909
+ const containerStatus = probeEditorComposeState(cfg).state || "missing";
3551
4910
  console.error(` editor container: ${containerStatus}`);
3552
4911
 
3553
4912
  const appPort = await readVmAppPort(cfg);
@@ -3616,6 +4975,10 @@ async function cmdIngressUp(args, projectDir = rootDir) {
3616
4975
  async function cmdReconcileIngress(args, projectDir = rootDir) {
3617
4976
  ensureGcloud();
3618
4977
  parseFlags(args);
4978
+ // Clear ssh sessions earlier timed-out probes leaked before this sweep can add
4979
+ // load of its own. Only PID-1 orphans with the fleet signature — never a live
4980
+ // sweep's in-flight children.
4981
+ reapOrphanedFleetSsh();
3619
4982
  const cfg = loadCloudConfig(projectDir) || {};
3620
4983
  const domain = resolveIngressDomain(cfg, process.env);
3621
4984
  if (!domain) {
@@ -3641,17 +5004,40 @@ async function cmdReconcileIngress(args, projectDir = rootDir) {
3641
5004
  zone: cfg.zone,
3642
5005
  slug,
3643
5006
  });
3644
- const res = spawnSync(inv.command, inv.args, {
5007
+ // Bound the per-VM status probe: a hung `bash vm-ingress.sh <N> status`
5008
+ // (its gcloud describes stalling on an IAP/metadata hiccup) must not stall the
5009
+ // whole sweep. On timeout res.stdout is empty, so parseIngressStatusVerdict
5010
+ // returns null → an `unknown` verdict → selectIngressReconcileTargets SKIPS
5011
+ // this VM and the sweep continues to the next one.
5012
+ const res = spawnGroupBoundedSync(inv.command, inv.args, {
3645
5013
  env: { ...process.env, ...inv.env },
3646
5014
  encoding: "utf8",
5015
+ timeout: GCLOUD_CALL_TIMEOUT_MS,
3647
5016
  });
3648
5017
  return { vmNumber: n, slug, verdict: parseIngressStatusVerdict(res.stdout || "") };
3649
5018
  });
3650
5019
  const targets = selectIngressReconcileTargets(statuses);
3651
5020
  const targetNums = new Set(targets.map((t) => t.vmNumber));
5021
+ // The complement of the re-wire loop: `wired` VMs whose backends already exist
5022
+ // so the re-wire loop skips them, yet still need the SA-accessor (re-)granted —
5023
+ // `up` is the only path that grants it, and it never runs for a wired VM. This
5024
+ // closes the VM-8 heal-path gap (a wired VM whose backends predate the SA-grant
5025
+ // fix would otherwise stay 403 forever). Disjoint from `targets` by construction.
5026
+ const grantOnlyNums = new Set(
5027
+ selectIngressGrantOnlyTargets(statuses).map((g) => g.vmNumber),
5028
+ );
3652
5029
  for (const s of statuses) {
3653
- if (!targetNums.has(s.vmNumber)) {
3654
- const label = s.slug || `vm-${s.vmNumber}`;
5030
+ if (targetNums.has(s.vmNumber)) continue;
5031
+ const label = s.slug || `vm-${s.vmNumber}`;
5032
+ if (grantOnlyNums.has(s.vmNumber)) {
5033
+ // Churn-free grant-only pass: (re-)grant the accessor without re-wiring any
5034
+ // NEG/BES/host-rule resource. Idempotent (a no-op when the SA is already
5035
+ // bound) and best-effort-but-loud — a grant hiccup logs, never aborts.
5036
+ const gres = runVmIngressGrantAccess(cfg, s.vmNumber, domain, s.slug);
5037
+ console.error(
5038
+ ` VM-${s.vmNumber} ${label}: wired → ok (accessor ${gres.status === 0 ? "granted" : "grant FAILED"})`,
5039
+ );
5040
+ } else {
3655
5041
  console.error(` VM-${s.vmNumber} ${label}: ${s.verdict || "unknown"} → ok`);
3656
5042
  }
3657
5043
  }
@@ -3681,7 +5067,13 @@ async function cmdReconcileIngress(args, projectDir = rootDir) {
3681
5067
  for (const surface of gi.SURFACES) {
3682
5068
  const bes = gi.backendServiceName(n, surface);
3683
5069
  const inv = backendServiceTimeoutInvocation({ bes, project: cfg.project });
3684
- const res = spawnSync(inv.command, inv.args, { encoding: "utf8" });
5070
+ // Bound the per-BES converge too: a hung `gcloud backend-services update`
5071
+ // returns a non-zero (timed-out) result, so it falls into the skip branch
5072
+ // below and the sweep moves on to the next BES rather than stalling.
5073
+ const res = spawnGroupBoundedSync(inv.command, inv.args, {
5074
+ encoding: "utf8",
5075
+ timeout: GCLOUD_CALL_TIMEOUT_MS,
5076
+ });
3685
5077
  if (res.status === 0) {
3686
5078
  console.error(` ${bes}: timeout → 86400`);
3687
5079
  } else {
@@ -3691,6 +5083,12 @@ async function cmdReconcileIngress(args, projectDir = rootDir) {
3691
5083
  }
3692
5084
  }
3693
5085
 
5086
+ // Converge the shared fleet firewall, then surface any app port it still fails to
5087
+ // admit. Both ride this same boot + periodic cadence so the running fleet
5088
+ // self-heals without a hand-run bootstrap.
5089
+ const firewallState = convergeFleetFirewall({ cfg });
5090
+ await reportFirewallPortGaps({ cfg, roster, firewallState });
5091
+
3694
5092
  // Auto-clean orphaned LB ingress (vm--auto-clean-orphaned-lb-ingress): any
3695
5093
  // backend service whose VM number is gone from the roster is a destroyed VM's
3696
5094
  // leftover — cleaned here automatically with the verified `down`, so the
@@ -3720,6 +5118,106 @@ async function cmdReconcileIngress(args, projectDir = rootDir) {
3720
5118
  * summary (null when the backend-service enumeration itself failed) — the same
3721
5119
  * shape it prints — so the flow is directly assertable. `deps` is injectable so
3722
5120
  * the enumerate → select → down flow is testable without gcloud. */
5121
+ /** Converge the shared fleet firewall rule to all-tcp so the LB reaches whatever
5122
+ * port a project's app actually serves.
5123
+ *
5124
+ * A rule pinned to a hardcoded port list (`tcp:14199,tcp:5173,tcp:3000`) drops
5125
+ * LB→VM traffic for any VM serving outside it — the VM-6-on-8080 Live Preview
5126
+ * outage — while the preview backend service still reads HEALTHY, because its
5127
+ * health check runs on the always-admitted editor port. The failure is therefore
5128
+ * invisible until someone opens the preview. Converging here (not only in the
5129
+ * hand-run google-ingress.sh bootstrap) is what makes a drifted rule self-heal on
5130
+ * the running fleet.
5131
+ *
5132
+ * All-tcp is the right exposure, not a per-stack port list: the rule stays scoped
5133
+ * to Google's LB + health-check source ranges and the fleet instance tag either
5134
+ * way, so it admits any app port (Vite 5173, Next 3000, Astro 4321, Go 8080, …)
5135
+ * without enumeration. EVERY candidate name in FLEET_FIREWALL_RULE_NAMES is
5136
+ * converged: describing only one silently no-ops on a fleet wired under the other.
5137
+ *
5138
+ * Scope-safe by construction — `update --rules=tcp` rewrites only the allowed
5139
+ * protocols/ports, never a rule's source ranges or target tags, so this can never
5140
+ * change who the rule applies to. Best-effort throughout: a describe/update
5141
+ * failure logs and continues, never aborting the rest of the reconcile.
5142
+ *
5143
+ * Returns `[{ ruleName, rules, converged }]` for every rule that exists — `rules`
5144
+ * is the post-converge allowed value — so the flow is assertable without gcloud.
5145
+ * `deps.spawnSync` is injectable for the same reason.
5146
+ * See plan vm--self-healing-app-port-agnostic-fleet-lb-firewall. */
5147
+ function convergeFleetFirewall({ cfg } = {}, deps = {}) {
5148
+ const run = deps.spawnSync || spawnSync;
5149
+ const project = cfg && cfg.project;
5150
+ const state = [];
5151
+ for (const ruleName of FLEET_FIREWALL_RULE_NAMES) {
5152
+ const dinv = fleetFirewallDescribeInvocation({ ruleName, project });
5153
+ const dres = run(dinv.command, dinv.args, { encoding: "utf8" });
5154
+ // A non-zero describe means no rule under this name — expected, not an error:
5155
+ // only one of the candidate names is deployed on any given fleet.
5156
+ if (!dres || dres.status !== 0) continue;
5157
+ const current = ((dres.stdout || "") + "").trim();
5158
+ if (!firewallNeedsConverge(current)) {
5159
+ console.error(` firewall ${ruleName}: already all-tcp → ok`);
5160
+ state.push({ ruleName, rules: "tcp", converged: false });
5161
+ continue;
5162
+ }
5163
+ const cinv = fleetFirewallConvergeInvocation({ ruleName, project });
5164
+ const cres = run(cinv.command, cinv.args, { encoding: "utf8" });
5165
+ if (cres && cres.status === 0) {
5166
+ console.error(` firewall ${ruleName}: rules '${current}' → tcp (all-tcp)`);
5167
+ state.push({ ruleName, rules: "tcp", converged: true });
5168
+ } else {
5169
+ const tail = (
5170
+ (cres && (cres.stderr || (cres.error && cres.error.message))) || ""
5171
+ ).toString().trim().split("\n").pop() || "error";
5172
+ console.error(` firewall ${ruleName}: converge skipped (${tail})`);
5173
+ state.push({ ruleName, rules: current, converged: false });
5174
+ }
5175
+ }
5176
+ if (!state.length) {
5177
+ console.error(
5178
+ ` firewall: no rule found under ${FLEET_FIREWALL_RULE_NAMES.join(" / ")} —` +
5179
+ " LB→VM traffic may be blocked; run scripts/gcp/google-ingress.sh",
5180
+ );
5181
+ }
5182
+ return state;
5183
+ }
5184
+
5185
+ /** Surface any rostered VM whose app port the fleet firewall does not admit — the
5186
+ * tripwire for a masked "HEALTHY but unreachable" preview.
5187
+ *
5188
+ * The preview backend service health-checks the fixed editor port, so a rule that
5189
+ * admits that port but not the app's real one reads HEALTHY while every real
5190
+ * request is dropped. That masking is what made the VM-6 outage invisible. Emits a
5191
+ * machine-readable `CY_FIREWALL_PORT_GAP <json>` line per gap, mirroring the
5192
+ * `CY_ORPHAN_SWEEP` marker convention the dashboard already parses.
5193
+ *
5194
+ * Only runs when a rule is still port-pinned: a converged all-tcp rule admits every
5195
+ * port, so the healthy path cannot produce a gap and skipping it avoids an SSH
5196
+ * round-trip per rostered VM for a check that cannot fail. In practice this fires
5197
+ * only if the converge itself failed (e.g. a permissions regression).
5198
+ *
5199
+ * Returns the emitted `[{ vmNumber, appPort, rule, allowed }]` gaps — the same
5200
+ * shape it prints — so the marker contract is directly assertable.
5201
+ * `deps.readVmAppPort` is injectable so tests never reach for SSH. */
5202
+ async function reportFirewallPortGaps({ cfg, roster, firewallState } = {}, deps = {}) {
5203
+ const readPort = deps.readVmAppPort || readVmAppPort;
5204
+ const pinned = (firewallState || []).filter((f) => firewallNeedsConverge(f.rules));
5205
+ const gaps = [];
5206
+ if (!pinned.length) return gaps;
5207
+ for (const n of roster || []) {
5208
+ const perVmCfg = { ...cfg, instanceName: `${fleetInstancePrefix()}${n}` };
5209
+ const appPort = await readPort(perVmCfg);
5210
+ for (const f of pinned) {
5211
+ if (firewallAdmitsPort(f.rules, appPort)) continue;
5212
+ const gap = { vmNumber: n, appPort, rule: f.ruleName, allowed: f.rules };
5213
+ console.error(` VM-${n}: app port ${appPort} NOT admitted by ${f.ruleName}`);
5214
+ console.log(`CY_FIREWALL_PORT_GAP ${JSON.stringify(gap)}`);
5215
+ gaps.push(gap);
5216
+ }
5217
+ }
5218
+ return gaps;
5219
+ }
5220
+
3723
5221
  function sweepOrphanIngress({ cfg, domain, roster, projectDir = rootDir } = {}, deps = {}) {
3724
5222
  const gi = deps.googleIngress || require("./google-ingress.js");
3725
5223
  const fcl = deps.fleetCleanup || require("./fleet-cleanup.js");
@@ -3820,7 +5318,14 @@ function sweepOrphanIngress({ cfg, domain, roster, projectDir = rootDir } = {},
3820
5318
  });
3821
5319
  const res = run(inv.command, inv.args, { encoding: "utf8", env: { ...process.env, ...inv.env } });
3822
5320
  const verdict = parseIngressTeardownVerdict((res && res.stdout) || "");
3823
- const outcome = classifyOrphanDownResult({ verdict, status: res ? res.status : null });
5321
+ // Pass `signal` too: a teardown the caller's group-kill reaped has a NULL
5322
+ // status, which the classifier would otherwise report as `spawn-failed`
5323
+ // rather than the truthful `killed=SIGKILL`.
5324
+ const outcome = classifyOrphanDownResult({
5325
+ verdict,
5326
+ status: res ? res.status : null,
5327
+ signal: res ? res.signal : null,
5328
+ });
3824
5329
  if (outcome.cleaned) {
3825
5330
  summary.cleaned.push(n);
3826
5331
  console.error(` VM-${n}: orphaned ingress → auto-cleaned (verified clean)`);
@@ -3889,20 +5394,33 @@ async function probeGitCredentials(cfg) {
3889
5394
  /** Seed the LIVE in-memory git credential-cache daemon over its socket — the
3890
5395
  * same `git credential approve` shape the entrypoint uses
3891
5396
  * (scripts/cloud-editor-entrypoint.sh) — so push works immediately with no
3892
- * container restart. base64 keeps the token out of the shell history across
3893
- * the gcloud-ssh → docker layers. Throws if the approve fails. Shared by
3894
- * {@link cmdReconcileCredentials} and {@link assertGitPushCredentials}. */
5397
+ * container restart. ALSO writes the container-local on-disk `store` fallback
5398
+ * file (the fill-only second helper baked into the image) directly, so a
5399
+ * reconcile leaves the VM durable against the cache daemon dying mid-life, not
5400
+ * just healthy for the moment. base64 keeps the token out of the shell history
5401
+ * across the gcloud-ssh → docker layers, and the store write is an in-place
5402
+ * `cat > file` truncate (chmod 600), never touched by git's approve path.
5403
+ * Throws if either write fails. Shared by {@link cmdReconcileCredentials} and
5404
+ * {@link assertGitPushCredentials}. */
3895
5405
  async function seedLiveCredentialCache(cfg, token) {
3896
5406
  const repoName = resolveProject(cfg).name;
3897
5407
  const projectCd = `cd ~/projects/${repoName}`;
3898
5408
  const composeExec = `docker compose -f docker-compose.cloud.yml exec -T editor`;
3899
5409
  const socket = "/run/codeyam/git-credential-cache.socket";
5410
+ const store = "/run/codeyam/git-credentials-store";
3900
5411
  const approveStdin = `protocol=https\nhost=github.com\nusername=x-access-token\npassword=${token}\n\n`;
3901
5412
  const b64 = Buffer.from(approveStdin).toString("base64");
5413
+ // The store-file line is the same one-URL-per-line format as the host staging
5414
+ // file; the image's fill-only `store` helper reads it on a cache miss.
5415
+ const storeB64 = Buffer.from(`${githubCredentialLine(token)}\n`).toString("base64");
3902
5416
  const seedCmd =
3903
- `${projectCd} && echo ${b64} | base64 -d | ${composeExec} ` +
5417
+ `${projectCd} && ` +
5418
+ `echo ${b64} | base64 -d | ${composeExec} ` +
3904
5419
  `git -c 'credential.helper=cache --timeout=31536000 --socket=${socket}' ` +
3905
- `credential approve && echo APPROVED`;
5420
+ `credential approve && ` +
5421
+ `echo ${storeB64} | base64 -d | ${composeExec} ` +
5422
+ `bash -c 'cat > ${store} && chmod 600 ${store}' && ` +
5423
+ `echo APPROVED`;
3906
5424
  const seed = runGcloud(
3907
5425
  buildGcloudArgs("ssh-probe", cfg, { command: seedCmd }),
3908
5426
  { stdio: ["ignore", "pipe", "inherit"] },
@@ -3941,6 +5459,39 @@ async function cmdReconcileCredentials(args, projectDir = rootDir) {
3941
5459
  );
3942
5460
  }
3943
5461
 
5462
+ /** `cloud:reconcile-admin` — flip admin mode on/off for an already-running VM
5463
+ * WITHOUT a full re-provision. Resolves the desired value from this invocation
5464
+ * (`--admin` / `CODEYAM_ADMIN`, else off), rewrites the rendered compose's
5465
+ * CODEYAM_ADMIN line, and recreates the editor service — matching what a manual
5466
+ * compose edit + `docker compose up -d editor` does today, so the durable admin
5467
+ * setting no longer requires a full re-provision to change. */
5468
+ async function cmdReconcileAdmin(args, projectDir = rootDir) {
5469
+ ensureGcloud();
5470
+ const flags = parseFlags(args);
5471
+ const cfg = loadCloudConfig(projectDir);
5472
+ if (!cfg) {
5473
+ throw new Error("No .codeyam/cloud.json found. Run `npm run cloud:up` first.");
5474
+ }
5475
+ const desired =
5476
+ buildProvisionEnv(process.env, flags).CODEYAM_ADMIN === "1" ? "1" : "";
5477
+ const repoName = resolveProject(cfg).name;
5478
+ console.error(
5479
+ `Reconciling admin mode to ${desired === "1" ? "ON" : "off"} on ${cfg.instanceName}...`,
5480
+ );
5481
+ const result = runGcloud(
5482
+ buildGcloudArgs("ssh-probe", cfg, {
5483
+ command: buildReconcileAdminCommand(repoName, desired),
5484
+ }),
5485
+ { stdio: "inherit" },
5486
+ );
5487
+ if (result.status !== 0) {
5488
+ throw new Error(`Failed to reconcile admin mode on ${cfg.instanceName}.`);
5489
+ }
5490
+ console.error(
5491
+ ` ✓ editor service recreated; /api/config now reports admin=${desired === "1" ? "true" : "false"}.`,
5492
+ );
5493
+ }
5494
+
3944
5495
  /** Provision-time guarantee that a just-provisioned VM can actually reach
3945
5496
  * GitHub. Probes the staging file + in-container `git ls-remote origin` and
3946
5497
  * routes the outcome through the SAME {@link decideCredentialHeal} classifier
@@ -4072,6 +5623,7 @@ const HANDLERS = {
4072
5623
  "reconcile-ingress": cmdReconcileIngress,
4073
5624
  "wire-add-tail": cmdWireAddTail,
4074
5625
  "reconcile-credentials": cmdReconcileCredentials,
5626
+ "reconcile-admin": cmdReconcileAdmin,
4075
5627
  access: cmdAccess,
4076
5628
  };
4077
5629
 
@@ -4094,6 +5646,10 @@ async function cloudMain() {
4094
5646
 
4095
5647
  module.exports = {
4096
5648
  // pure helpers
5649
+ isSpawnTimeout,
5650
+ gcloudCallTimeoutMs,
5651
+ provisionFailureMessage,
5652
+ GCLOUD_PROVISION_TIMEOUT_MS,
4097
5653
  agentEnvVarName,
4098
5654
  agentNpmPackage,
4099
5655
  AGENT_ENV_VARS,
@@ -4104,11 +5660,15 @@ module.exports = {
4104
5660
  assertAgentCliInstalled,
4105
5661
  probeContainerRunning,
4106
5662
  probeContainerState,
5663
+ waitForEditorComposeRunning,
5664
+ abortTeardownDecision,
5665
+ probeAbortTeardownState,
4107
5666
  probeAgentBinaryInstalled,
4108
5667
  waitForContainerRunning,
4109
5668
  KNOWN_FLAGS,
4110
5669
  FIREWALL_RULE_NAME,
4111
5670
  INSTANCE_TAG,
5671
+ DEFAULT_FLEET_MACHINE_TYPE,
4112
5672
  EDITOR_PORT,
4113
5673
  VITE_PORT,
4114
5674
  parseFlags,
@@ -4125,8 +5685,10 @@ module.exports = {
4125
5685
  resolveConfiguredAppPort,
4126
5686
  resolveSeedPreviewOrigin,
4127
5687
  previewSameOriginSafeFromStack,
5688
+ resolveEffectiveSameOriginSafe,
4128
5689
  isRottedManagedPreviewOrigin,
4129
5690
  isWrongModePreviewOrigin,
5691
+ isSilentlyBrokenPreview,
4130
5692
  resolveEditorConfig,
4131
5693
  validateEditorVariant,
4132
5694
  assertVariantProvisionable,
@@ -4135,13 +5697,19 @@ module.exports = {
4135
5697
  resolveGhcrToken,
4136
5698
  resolveGhcrPushToken,
4137
5699
  probeUsableEditorImage,
5700
+ httpsManifestGet,
4138
5701
  resolveProject,
4139
5702
  clientProjectBootstrapArgs,
4140
5703
  cloudConfigPath,
4141
5704
  loadCloudConfig,
4142
5705
  saveCloudConfig,
4143
5706
  loadOrInitCloudConfig,
5707
+ buildProvisionEnv,
5708
+ buildReconcileAdminCommand,
5709
+ cmdReconcileAdmin,
4144
5710
  buildEditorLocalEnvPatch,
5711
+ buildGreenfieldTitlePatch,
5712
+ mergeEditorLocalPatchB64,
4145
5713
  buildEditorLocalSeedWriteCommand,
4146
5714
  seedPreviewOrigin,
4147
5715
  healPreviewOriginAtReady,
@@ -4155,11 +5723,29 @@ module.exports = {
4155
5723
  decideBootstrapOutcome,
4156
5724
  projectBootstrapDoneMarker,
4157
5725
  projectBootstrapDoneProbeCommand,
5726
+ projectBootstrapScriptPath,
5727
+ projectBootstrapRunnerPath,
5728
+ projectBootstrapLogPath,
5729
+ projectBootstrapPidPath,
5730
+ projectBootstrapStatusPath,
5731
+ projectBootstrapRunnerScript,
5732
+ projectBootstrapLaunchCommand,
5733
+ projectBootstrapFollowCommand,
5734
+ projectBootstrapStatusCommand,
5735
+ parseProjectBootstrapStatus,
5736
+ decideProjectBootstrapOutcome,
5737
+ decideProjectBootstrapFollowAction,
5738
+ projectBootstrapEnv,
5739
+ PROJECT_BOOTSTRAP_LAUNCHED_MARKER,
5740
+ PROJECT_BOOTSTRAP_FOLLOW_SECS,
5741
+ ensureProjectBootstrapped,
4158
5742
  SSH_KEEPALIVE_FLAGS,
4159
5743
  resolveSetupForwardEnv,
4160
5744
  FORWARD_ENV_KEYS_ENV,
4161
5745
  parseEditorComposeState,
4162
5746
  fetchInstanceState,
5747
+ fetchPendingOperation,
5748
+ wedgedOperationMessage,
4163
5749
  formatGcloudFailure,
4164
5750
  isTransientSshFailure,
4165
5751
  runGcloudSshWithRetry,
@@ -4169,6 +5755,8 @@ module.exports = {
4169
5755
  checkScriptDrift,
4170
5756
  streamedScriptEditorBranch,
4171
5757
  streamedScriptMergeBase,
5758
+ fetchStreamedScriptRef,
5759
+ resolveRemoteScriptRev,
4172
5760
  gatherScriptDriftFacts,
4173
5761
  readStreamedScript,
4174
5762
  STREAMED_SCRIPTS,
@@ -4177,6 +5765,8 @@ module.exports = {
4177
5765
  ingressWiringPlan,
4178
5766
  fleetInstancePrefix,
4179
5767
  runVmIngress,
5768
+ intendedVmPrincipals,
5769
+ assertAddCanAuthorizeSomeone,
4180
5770
  wireVmIngressOnAdd,
4181
5771
  probeVmIngressVerdict,
4182
5772
  convergeVmIngressOnAdd,
@@ -4195,6 +5785,8 @@ module.exports = {
4195
5785
  cmdIngressUp,
4196
5786
  cmdReconcileIngress,
4197
5787
  sweepOrphanIngress,
5788
+ convergeFleetFirewall,
5789
+ reportFirewallPortGaps,
4198
5790
  cmdReconcileCredentials,
4199
5791
  cmdAccess,
4200
5792
  cloudMain,
@@ -4202,13 +5794,17 @@ module.exports = {
4202
5794
  seedLiveCredentialCache,
4203
5795
  assertGitPushCredentials,
4204
5796
  seedGithubCredentials,
5797
+ githubCredentialLine,
4205
5798
  resolveGithubToken,
4206
5799
  resolveGithubTokenWithSource,
4207
5800
  MissingGitPushCredentialsError,
4208
5801
  classifyGitRemoteError,
4209
5802
  decideCredentialHeal,
4210
5803
  readVmAppPort,
5804
+ buildVmAppPortReadCommand,
4211
5805
  resolveLaptopGitIdentity,
5806
+ gitBranchRefExists,
5807
+ assertBranchRefsExist,
4212
5808
  };
4213
5809
 
4214
5810
  if (require.main === module) {