@codeyam-editor/codeyam-editor 0.1.5 → 0.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/npm/cli.js +5 -1
- package/npm/cloud.js +2079 -242
- package/npm/codesign-plan.js +68 -0
- package/npm/compose-conflict.js +59 -0
- package/npm/empty-project-dir.js +92 -7
- package/npm/fleet-add-failure.js +361 -0
- package/npm/fleet-asset-version.js +37 -0
- package/npm/fleet-auth-health.js +143 -5
- package/npm/fleet-build-data.js +60 -5
- package/npm/fleet-cleanup.js +243 -9
- package/npm/fleet-destroy-safety.js +230 -13
- package/npm/fleet-early-roster.js +87 -0
- package/npm/fleet-effect-poll.js +110 -0
- package/npm/fleet-env.js +86 -0
- package/npm/fleet-ghcr-secret.js +129 -0
- package/npm/fleet-git-maintenance.js +142 -0
- package/npm/fleet-host-op-limiter.js +68 -0
- package/npm/fleet-iap-token.js +165 -0
- package/npm/fleet-job-badge.js +28 -12
- package/npm/fleet-live-status.js +40 -8
- package/npm/fleet-memory.js +289 -9
- package/npm/fleet-plans.js +98 -4
- package/npm/fleet-pool.js +75 -16
- package/npm/fleet-prewarm.js +87 -11
- package/npm/fleet-probe-transport.js +155 -0
- package/npm/fleet-project-drift.js +58 -0
- package/npm/fleet-public-url.js +234 -0
- package/npm/fleet-queue.js +58 -1
- package/npm/fleet-reachability-debounce.js +69 -0
- package/npm/fleet-reachability.js +80 -0
- package/npm/fleet-reset.js +32 -0
- package/npm/fleet-revision.js +104 -0
- package/npm/fleet-running-plan.js +115 -0
- package/npm/fleet-session-token-cache.js +65 -0
- package/npm/fleet-setup-steps.js +19 -12
- package/npm/fleet-status-refs.js +96 -0
- package/npm/fleet-strand.js +39 -8
- package/npm/fleet-stuck-status.js +13 -7
- package/npm/fleet-vm-access.js +170 -0
- package/npm/fleet-vm-probe-target.js +282 -0
- package/npm/fleet-vm-readiness.js +136 -3
- package/npm/google-ingress.js +15 -0
- package/npm/ingress-invoke.js +478 -0
- package/npm/launch-config.js +302 -10
- package/npm/scenario-check.js +318 -12
- package/npm/scenario-handlers.js +85 -27
- package/npm/scenario-interactivity.js +145 -6
- package/npm/scenario-issues.js +13 -1
- package/npm/scenario-mocks.js +129 -26
- package/npm/scenario-playwright.js +330 -22
- package/npm/script-drift.js +72 -1
- package/npm/utils.js +37 -3
- package/package.json +5 -5
- package/ui/dist/assets/{ContainerInspector-D7VZkkaj.js → ContainerInspector-Cu1tSN-o.js} +1 -1
- package/ui/dist/assets/EditorShell--V2ExYQ6.js +260 -0
- package/ui/dist/assets/EditorShell-DP5UY20G.css +1 -0
- package/ui/dist/assets/{Markdown-BoLHHFXH.js → Markdown-Chgdf3PW.js} +1 -1
- package/ui/dist/assets/ProjectLauncherScreen-Dmm6Z0mv.js +1 -0
- package/ui/dist/assets/{ScenarioDataPanel-DYP7pi_n.css → ScenarioDataPanel-C1LukYI6.css} +1 -1
- package/ui/dist/assets/ScenarioDataPanel-DjprZq2L.js +9 -0
- package/ui/dist/assets/{ScenarioDataPanelFullPage-CBLTGC4-.js → ScenarioDataPanelFullPage-BqMxWVKW.js} +1 -1
- package/ui/dist/assets/ScenarioRenderer-DqLVVMYS.js +21 -0
- package/ui/dist/assets/{StepsSimulator-BFldWyoX.js → StepsSimulator-CymGgfbu.js} +1 -1
- package/ui/dist/assets/index-CjxWhkSy.css +1 -0
- package/ui/dist/assets/{index-QsOiOG9x.js → index-VuyqBbGJ.js} +4 -4
- package/ui/dist/assets/{markdown-v0F0UEt3.js → markdown-C9tbsVHX.js} +1 -1
- package/ui/dist/assets/react-nrLBr15I.js +28 -0
- package/ui/dist/assets/{useEvents-DHkLJZR4.js → useEvents-S3Qk1Y_d.js} +1 -1
- package/ui/dist/assets/useServerIdentity-BdXy8TQv.js +1 -0
- package/ui/dist/index.html +4 -4
- package/ui/dist/open-source/codeyam-counter.png +0 -0
- package/ui/dist/open-source/el-carot.png +0 -0
- package/ui/dist/open-source/tabcommand.png +0 -0
- package/ui/dist/assets/EditorShell-DHhIaotd.css +0 -1
- package/ui/dist/assets/EditorShell-r-796N_W.js +0 -191
- package/ui/dist/assets/ProjectLauncherScreen-Ck-XWkQN.js +0 -1
- package/ui/dist/assets/ScenarioDataPanel-B9iDSUpA.js +0 -5
- package/ui/dist/assets/ScenarioRenderer-BhLfT-nX.js +0 -21
- package/ui/dist/assets/index-CIXZkwfB.css +0 -1
- package/ui/dist/assets/react-CSS0HapR.js +0 -28
package/npm/cloud.js
CHANGED
|
@@ -11,6 +11,7 @@
|
|
|
11
11
|
|
|
12
12
|
const { spawn, spawnSync } = require("child_process");
|
|
13
13
|
const fs = require("fs");
|
|
14
|
+
const https = require("https");
|
|
14
15
|
const os = require("os");
|
|
15
16
|
const path = require("path");
|
|
16
17
|
const readline = require("readline");
|
|
@@ -56,6 +57,8 @@ const {
|
|
|
56
57
|
const {
|
|
57
58
|
normalizeAccessList,
|
|
58
59
|
normalizePrincipal,
|
|
60
|
+
resolveDefaultOperator,
|
|
61
|
+
withDefaultOperator,
|
|
59
62
|
accessFor,
|
|
60
63
|
setAccess,
|
|
61
64
|
addAccess,
|
|
@@ -73,12 +76,28 @@ const {
|
|
|
73
76
|
regionFromZone,
|
|
74
77
|
vmIngressInvocation,
|
|
75
78
|
vmIngressStatusInvocation,
|
|
79
|
+
vmIngressGrantAccessInvocation,
|
|
76
80
|
parseIngressStatusVerdict,
|
|
81
|
+
parseIngressTeardownVerdict,
|
|
77
82
|
selectIngressReconcileTargets,
|
|
83
|
+
selectIngressGrantOnlyTargets,
|
|
84
|
+
selectOrphanIngressTargets,
|
|
85
|
+
classifyOrphanDownResult,
|
|
86
|
+
backendServicesListInvocation,
|
|
87
|
+
negListInvocation,
|
|
88
|
+
instancesListInvocation,
|
|
89
|
+
ingressConvergeDecision,
|
|
78
90
|
detachedAddTailInvocation,
|
|
79
91
|
ingressInflightMarkerPath,
|
|
92
|
+
ingressFailedMarkerPath,
|
|
80
93
|
iapAccessInvocation,
|
|
81
94
|
sharedIngressInvocation,
|
|
95
|
+
backendServiceTimeoutInvocation,
|
|
96
|
+
FLEET_FIREWALL_RULE_NAMES,
|
|
97
|
+
fleetFirewallDescribeInvocation,
|
|
98
|
+
fleetFirewallConvergeInvocation,
|
|
99
|
+
firewallNeedsConverge,
|
|
100
|
+
firewallAdmitsPort,
|
|
82
101
|
} = require("./ingress-invoke");
|
|
83
102
|
// Shared fleet-roster writer/reader. `cloud:up` rosters the VM it just
|
|
84
103
|
// provisioned so the dashboard, the cloudflared-monitor tunnel publisher, and
|
|
@@ -86,6 +105,10 @@ const {
|
|
|
86
105
|
// edit (the invisible-VM gap improve38 closes). `readRoster` enumerates the
|
|
87
106
|
// fleet for `reconcile-ingress`.
|
|
88
107
|
const { addToRoster, readRoster } = require("./fleet-roster");
|
|
108
|
+
// Tells a docker container-name conflict (a concurrent run already started this
|
|
109
|
+
// container) apart from a real `docker compose up -d` failure, so the abort path
|
|
110
|
+
// can converge instead of escalating to instance deletion.
|
|
111
|
+
const { classifyComposeUpFailure } = require("./compose-conflict");
|
|
89
112
|
|
|
90
113
|
// =============================================================================
|
|
91
114
|
// Pure helpers — every export here is unit-tested in cloud.test.js without
|
|
@@ -148,14 +171,34 @@ const KNOWN_FLAGS = [
|
|
|
148
171
|
// default is loud — a tokenless VM cannot sync plan claims and must never
|
|
149
172
|
// report healthy silently. Use only when push genuinely isn't wanted.
|
|
150
173
|
"allow-no-git-credentials",
|
|
174
|
+
// Provision the VM in admin mode: forward CODEYAM_ADMIN=1 into the container
|
|
175
|
+
// so the Home-tab Setup door renders. Equivalent to `CODEYAM_ADMIN=1 npm run
|
|
176
|
+
// cloud:up …`; absent = strict non-admin (the safe default). Survives
|
|
177
|
+
// re-provision because it rides the compose-render substitution, unlike a
|
|
178
|
+
// manual VM compose edit.
|
|
179
|
+
"admin",
|
|
180
|
+
// GCE machine type for the VM (e.g. n2-standard-4, e2-standard-4). Persisted
|
|
181
|
+
// in cloud.json so a later bare `cloud:up` keeps it. Defaults to
|
|
182
|
+
// DEFAULT_FLEET_MACHINE_TYPE. The whole point of exposing this is fleet
|
|
183
|
+
// scale-out: the region's E2_CPUS quota (72 in us-central1) caps the fleet at
|
|
184
|
+
// ~14 e2-standard-4 VMs, while N2_CPUS/N2D_CPUS (200, unused) leave room for
|
|
185
|
+
// 30 — so the fleet defaults to N2 and this flag lets a caller override.
|
|
186
|
+
"machine-type",
|
|
151
187
|
];
|
|
152
188
|
|
|
153
189
|
// Flags that take no value (presence = true). parseFlags must not demand a
|
|
154
190
|
// value for these or `--no-tunnel` would error "requires a value".
|
|
155
|
-
const BOOLEAN_FLAGS = ["no-tunnel", "skip-setup", "list", "allow-no-git-credentials"];
|
|
191
|
+
const BOOLEAN_FLAGS = ["no-tunnel", "skip-setup", "list", "allow-no-git-credentials", "admin"];
|
|
156
192
|
|
|
157
193
|
const FIREWALL_RULE_NAME = "codeyam-editor-iap-ssh";
|
|
158
194
|
const INSTANCE_TAG = "codeyam-editor";
|
|
195
|
+
// Default GCE machine type for fleet VMs. N2 (not E2) because the fleet must
|
|
196
|
+
// reach 30 VMs: us-central1 E2_CPUS quota is 72, so 30 × e2-standard-4 (120
|
|
197
|
+
// vCPU) + the operator host exceed it and cap the fleet at ~14 regardless of
|
|
198
|
+
// code — while N2_CPUS/N2D_CPUS are 200 and unused, leaving headroom for 30 ×
|
|
199
|
+
// n2-standard-4. Override per-VM with `--machine-type`. See the vm--scale-
|
|
200
|
+
// fleet-to-30-vms plan for the measured quota rationale.
|
|
201
|
+
const DEFAULT_FLEET_MACHINE_TYPE = "n2-standard-4";
|
|
159
202
|
const EDITOR_PORT = 14199;
|
|
160
203
|
const VITE_PORT = 5173;
|
|
161
204
|
|
|
@@ -539,6 +582,32 @@ function isWrongModePreviewOrigin(localJson, sameOriginSafe) {
|
|
|
539
582
|
return LOOPBACK_ORIGIN_RE.test(origin);
|
|
540
583
|
}
|
|
541
584
|
|
|
585
|
+
/** Decide whether a VM is presenting as HEALTHY while its Live Preview is
|
|
586
|
+
* silently dead — the VM-9 failure mode. The heal
|
|
587
|
+
* (`reconcilePreviewOriginToSlugHost`) fixes the mechanism; this predicate is
|
|
588
|
+
* the smoke detector that catches a FUTURE mechanism gap (a new stack shape, a
|
|
589
|
+
* new provisioning path) by observation instead of by a user noticing a blank
|
|
590
|
+
* iframe days later.
|
|
591
|
+
*
|
|
592
|
+
* Input is the editor's own `/api/config` payload — the effective truth table
|
|
593
|
+
* (`previewSameOriginSafe` + the resolved `previewOrigin`), not a
|
|
594
|
+
* re-derivation from stack.json over ssh. True iff the stack is NOT
|
|
595
|
+
* same-origin-safe AND its effective previewOrigin is one a remote browser can
|
|
596
|
+
* never load (the `/__codeyam_preview` sentinel, any relative path, or a
|
|
597
|
+
* loopback origin) — i.e. exactly `isWrongModePreviewOrigin`'s wrong-mode
|
|
598
|
+
* condition, evaluated against the live config rather than editor.local.json.
|
|
599
|
+
* Composing the two keeps ONE definition of "wrong-mode origin" in this file.
|
|
600
|
+
*
|
|
601
|
+
* Deliberately conservative: a missing/unparseable payload, an absent
|
|
602
|
+
* `previewSameOriginSafe`, or a legitimate same-origin-safe stack on the
|
|
603
|
+
* subpath all return false. Absence of evidence is never a false alarm — the
|
|
604
|
+
* caller renders "unknown" instead. Pure. */
|
|
605
|
+
function isSilentlyBrokenPreview(apiConfig) {
|
|
606
|
+
if (!apiConfig || typeof apiConfig !== "object") return false;
|
|
607
|
+
if (apiConfig.previewSameOriginSafe !== false) return false;
|
|
608
|
+
return isWrongModePreviewOrigin(apiConfig, false);
|
|
609
|
+
}
|
|
610
|
+
|
|
542
611
|
/** Read the operator-laptop cloudflared-urls.json sidecar. Missing/unparseable
|
|
543
612
|
* → {} (the monitor may not have run yet; the seed falls back to localhost). */
|
|
544
613
|
function readCloudflaredUrls() {
|
|
@@ -736,6 +805,28 @@ function buildEditorLocalEnvPatch(localEnv, agent) {
|
|
|
736
805
|
return { provider: agent, env: { [varName]: value } };
|
|
737
806
|
}
|
|
738
807
|
|
|
808
|
+
/** Build the editor.local.json patch that seeds a GREENFIELD project's
|
|
809
|
+
* `projectTitle` so its live identity resolves to the real project name from
|
|
810
|
+
* first boot. A greenfield project (`--new-project <name>`, no clientRepo) has
|
|
811
|
+
* no origin remote and is bind-mounted at /workspace on the VM, so the editor's
|
|
812
|
+
* identity resolver would otherwise fall back to the bare mount basename
|
|
813
|
+
* "workspace" — which the fleet dashboard then reads as a permanent false
|
|
814
|
+
* project-dir-drift. Seeding `projectTitle` (deep-merged over editor.json on the
|
|
815
|
+
* VM) makes `project_identity::read` prefer the real name.
|
|
816
|
+
*
|
|
817
|
+
* Returns `null` for a clone (a repo Add already labels from its origin
|
|
818
|
+
* basename, and re-seeding a clone's title would be wrong) or when no
|
|
819
|
+
* greenfield name is available, so the caller can no-op without an SSH round
|
|
820
|
+
* trip. Pure — mirrors buildEditorLocalEnvPatch so the decision is unit-tested
|
|
821
|
+
* without a live VM. */
|
|
822
|
+
function buildGreenfieldTitlePatch(cfg = {}) {
|
|
823
|
+
const p = resolveProject(cfg);
|
|
824
|
+
if (!p.greenfield) return null;
|
|
825
|
+
const name = typeof p.name === 'string' ? p.name.trim() : '';
|
|
826
|
+
if (!name) return null;
|
|
827
|
+
return { projectTitle: name };
|
|
828
|
+
}
|
|
829
|
+
|
|
739
830
|
/** Parse `docker compose ps --format json` (compose v2 — one JSON object per
|
|
740
831
|
* line) and return the editor service's `State` string, or null when the
|
|
741
832
|
* service is absent / output is unparseable. Two callers shared an identical
|
|
@@ -758,6 +849,79 @@ function parseEditorComposeState(stdout) {
|
|
|
758
849
|
return null;
|
|
759
850
|
}
|
|
760
851
|
|
|
852
|
+
/** Decide whether `cloud:up`'s abort handler may delete the instance it is
|
|
853
|
+
* unwinding. Pure `{ createdThisRun, containerUp, rostered, probeInconclusive }
|
|
854
|
+
* → 'delete' | 'keep'`, matching the `reapVerdict` / `reattachVerdict` /
|
|
855
|
+
* `reconcileRosteredVm` pattern in npm/fleet-cleanup.js, so the whole matrix is
|
|
856
|
+
* exhaustively testable without a VM.
|
|
857
|
+
*
|
|
858
|
+
* `createdThisRun` stays a NECESSARY condition — never delete an instance this
|
|
859
|
+
* run did not create — but 2026-08-07 proved it is not SUFFICIENT. It answers
|
|
860
|
+
* "did I create this?", not "is it safe to destroy?", and in a concurrent
|
|
861
|
+
* provision race the losing run legitimately owns the flag while the winner has
|
|
862
|
+
* already brought the VM up and rostered it. Deleting there destroyed VM-4 and
|
|
863
|
+
* VM-5. So authorization also requires CURRENT-STATE evidence that the VM is
|
|
864
|
+
* out of service:
|
|
865
|
+
*
|
|
866
|
+
* - `containerUp` — the editor container is running (per the compose-ps probe).
|
|
867
|
+
* A VM serving an editor is in use by definition.
|
|
868
|
+
* - `rostered` — the VM is in the fleet roster, the operator-visible source of
|
|
869
|
+
* truth. Note this is true whether the winning run or THIS run rostered it;
|
|
870
|
+
* either way the VM is visible in the dashboard and removable in one command,
|
|
871
|
+
* so it is not the silent orphan teardown exists to prevent.
|
|
872
|
+
* - `probeInconclusive` — liveness could not be determined. This must fail
|
|
873
|
+
* SAFE: a failing probe is how we reach the abort handler in the first place,
|
|
874
|
+
* so it can never count as evidence the VM is dead. An orphan costs one
|
|
875
|
+
* `npm run cloud:destroy`; a wrong delete costs a 15-40 minute rebuild plus
|
|
876
|
+
* any unpushed work on the VM. */
|
|
877
|
+
function abortTeardownDecision({ createdThisRun, containerUp, rostered, probeInconclusive } = {}) {
|
|
878
|
+
if (!createdThisRun) return "keep";
|
|
879
|
+
if (probeInconclusive) return "keep";
|
|
880
|
+
if (containerUp) return "keep";
|
|
881
|
+
if (rostered) return "keep";
|
|
882
|
+
return "delete";
|
|
883
|
+
}
|
|
884
|
+
|
|
885
|
+
/** Gather the CURRENT-STATE inputs {@link abortTeardownDecision} needs, so the
|
|
886
|
+
* abort handler reads gather → decide → act instead of inlining probes among
|
|
887
|
+
* the delete calls.
|
|
888
|
+
*
|
|
889
|
+
* Only runs the probes when `createdThisRun` is true — the decision is `keep`
|
|
890
|
+
* regardless otherwise, and an abort path should not spend SSH round trips
|
|
891
|
+
* proving something it already knows. In that case it returns the
|
|
892
|
+
* probe-inconclusive shape, which is the safe default at every layer.
|
|
893
|
+
*
|
|
894
|
+
* Both lookups are wrapped: a throwing probe or an unreadable roster must fail
|
|
895
|
+
* SAFE (toward `keep`), never take a second exception path into the delete.
|
|
896
|
+
* `deps` is injectable so the matrix is unit-tested without a VM. */
|
|
897
|
+
function probeAbortTeardownState(cfg, createdThisRun, deps = {}) {
|
|
898
|
+
if (!createdThisRun) {
|
|
899
|
+
return { containerUp: false, rostered: false, probeInconclusive: true };
|
|
900
|
+
}
|
|
901
|
+
const probe = deps.probeEditorComposeState || probeEditorComposeState;
|
|
902
|
+
const roster = deps.readRoster || readRoster;
|
|
903
|
+
const resolveVm = deps.resolveFleetVmNumber || resolveFleetVmNumber;
|
|
904
|
+
const log = deps.log || ((m) => console.error(m));
|
|
905
|
+
|
|
906
|
+
let liveness = { state: null, inconclusive: true };
|
|
907
|
+
try {
|
|
908
|
+
liveness = probe(cfg);
|
|
909
|
+
} catch (e) {
|
|
910
|
+
log(` (liveness probe failed: ${e && e.message})`);
|
|
911
|
+
}
|
|
912
|
+
let rostered = false;
|
|
913
|
+
try {
|
|
914
|
+
const vm = resolveVm(cfg.instanceName);
|
|
915
|
+
rostered = vm != null && (roster() || []).includes(vm);
|
|
916
|
+
} catch { /* unreadable roster — treated as not-rostered; liveness still guards */ }
|
|
917
|
+
|
|
918
|
+
return {
|
|
919
|
+
containerUp: liveness.state === "running",
|
|
920
|
+
rostered,
|
|
921
|
+
probeInconclusive: liveness.inconclusive,
|
|
922
|
+
};
|
|
923
|
+
}
|
|
924
|
+
|
|
761
925
|
/** Translate one of the wrapper's verbs into the exact `gcloud` argv. The
|
|
762
926
|
* switch's job is to keep every per-verb command shape in one reviewable
|
|
763
927
|
* place — tests assert the arrays without ever spawning a process. */
|
|
@@ -778,7 +942,7 @@ function buildGcloudArgs(verb, cfg = {}, extras = {}) {
|
|
|
778
942
|
return [
|
|
779
943
|
"compute", "instances", "create", instanceName,
|
|
780
944
|
"--project", project, "--zone", zone,
|
|
781
|
-
|
|
945
|
+
`--machine-type=${cfg.machineType || DEFAULT_FLEET_MACHINE_TYPE}`,
|
|
782
946
|
"--image-family=debian-12",
|
|
783
947
|
"--image-project=debian-cloud",
|
|
784
948
|
"--boot-disk-size=100GB",
|
|
@@ -878,11 +1042,61 @@ function buildGcloudArgs(verb, cfg = {}, extras = {}) {
|
|
|
878
1042
|
// real use. Unit tests pin their exports so a refactor cannot drop them.
|
|
879
1043
|
// =============================================================================
|
|
880
1044
|
|
|
1045
|
+
// Per-call wall-clock budget for a single gcloud / SSH invocation. A hung
|
|
1046
|
+
// `gcloud describe` (metadata / IAP hiccup) used to block the whole
|
|
1047
|
+
// `cloud:reconcile-ingress` sweep forever — the only timeout in the reconcile
|
|
1048
|
+
// path was the outer 10-minute exec budget in the dashboard, and that never
|
|
1049
|
+
// cascaded to the orphaned descendants. Bounding every per-call spawn well under
|
|
1050
|
+
// that outer budget means a single hang trips HERE first: the call returns a
|
|
1051
|
+
// timed-out result (formatGcloudFailure renders it), the sweep skips that VM,
|
|
1052
|
+
// and no descendant is left hanging to pile up. Shared by runGcloud and the two
|
|
1053
|
+
// reconcile spawnSync sites (status probe, per-BES timeout converge) so all three
|
|
1054
|
+
// reference one value. 2 min is generous for any single describe/update/SSH probe
|
|
1055
|
+
// yet comfortably under the 600000 ms outer reconcile budget.
|
|
1056
|
+
const GCLOUD_CALL_TIMEOUT_MS = 120000;
|
|
1057
|
+
|
|
1058
|
+
// The project bootstrap is NOT a probe: it runs `git clone` + `npm install` in
|
|
1059
|
+
// the repo root and every `apps[].dir`, each package's `postinstall`, then
|
|
1060
|
+
// `npm run setup` — minutes of remote work on a heavy repo. It once ran as ONE
|
|
1061
|
+
// blocking `runGcloud("ssh-script")` call, so it inherited GCLOUD_CALL_TIMEOUT_MS
|
|
1062
|
+
// (a budget whose own comment scopes it to "any single describe/update/SSH
|
|
1063
|
+
// probe"). That SIGTERMd the gcloud child mid-clone; `spawnSync` then returned
|
|
1064
|
+
// `status: null`, which `!== 0`, so the timeout was indistinguishable from a
|
|
1065
|
+
// genuine remote failure and the caller's abort path DELETED A COMPLETELY
|
|
1066
|
+
// HEALTHY VM.
|
|
1067
|
+
//
|
|
1068
|
+
// Observed twice: 2026-07-28 on client repo `lets-go-compost-portal` (a root
|
|
1069
|
+
// postinstall of `playwright install chromium`, ~293 MB), and 2026-07-29
|
|
1070
|
+
// provisioning six self-host `codeyam-editor` VMs, whose clone alone exceeds the
|
|
1071
|
+
// probe budget — all six were built, torn down mid-clone, and reported as
|
|
1072
|
+
// "scripts/gcp-project-bootstrap.sh failed on the VM".
|
|
1073
|
+
//
|
|
1074
|
+
// The install is now DETACHED (projectBootstrapLaunchCommand + a re-attaching
|
|
1075
|
+
// follow loop, outcome read from VM-side state), so no single call carries the
|
|
1076
|
+
// whole install any more. This constant is therefore the OVERALL deadline across
|
|
1077
|
+
// re-attaches — the point at which we stop waiting on a still-healthy VM and say
|
|
1078
|
+
// exactly that. Deliberately well above the outer reconcile budget: a long remote
|
|
1079
|
+
// install is not a step in a sweep, so the sweep-relative ceiling that governs
|
|
1080
|
+
// GCLOUD_CALL_TIMEOUT_MS does not apply to it.
|
|
1081
|
+
const PROJECT_BOOTSTRAP_TIMEOUT_MS = 40 * 60 * 1000;
|
|
1082
|
+
|
|
1083
|
+
/** Whether a `spawnSync` result is a LOCAL timeout kill rather than a remote
|
|
1084
|
+
* failure. `spawnSync` reports a timeout as `status: null` plus an ETIMEDOUT
|
|
1085
|
+
* error, so a bare `status !== 0` check conflates the two — which is what made
|
|
1086
|
+
* a timed-out bootstrap look like a broken one. Pure. */
|
|
1087
|
+
function isSpawnTimeout(result) {
|
|
1088
|
+
if (!result) return false;
|
|
1089
|
+
if (result.status !== null && result.status !== undefined) return false;
|
|
1090
|
+
const code = result.error && result.error.code;
|
|
1091
|
+
return code === "ETIMEDOUT" || result.signal === "SIGTERM";
|
|
1092
|
+
}
|
|
1093
|
+
|
|
881
1094
|
function runGcloud(args, opts = {}) {
|
|
882
1095
|
return spawnSync("gcloud", args, {
|
|
883
1096
|
stdio: opts.stdio || "inherit",
|
|
884
1097
|
input: opts.input,
|
|
885
1098
|
encoding: opts.encoding || "utf8",
|
|
1099
|
+
timeout: opts.timeout != null ? opts.timeout : GCLOUD_CALL_TIMEOUT_MS,
|
|
886
1100
|
});
|
|
887
1101
|
}
|
|
888
1102
|
|
|
@@ -1368,7 +1582,7 @@ async function ensureBootstrapped(cfg, { sleepFn } = {}) {
|
|
|
1368
1582
|
}
|
|
1369
1583
|
console.error(" streaming scripts/gcp-bootstrap.sh to VM (this may take ~10 minutes)...");
|
|
1370
1584
|
const scriptPath = path.join(rootDir, "scripts", "gcp-bootstrap.sh");
|
|
1371
|
-
const scriptBody =
|
|
1585
|
+
const scriptBody = readStreamedScript(scriptPath, cfg);
|
|
1372
1586
|
// GITHUB_TOKEN is passed through to the bootstrap script so it can clone
|
|
1373
1587
|
// a private editor repo (the bootstrap clones $REPO_URL to build the
|
|
1374
1588
|
// codeyam-editor:local image). Without this the git clone hits the
|
|
@@ -1485,7 +1699,305 @@ function bootstrapLogTail(cfg, lines = 40) {
|
|
|
1485
1699
|
return String(r.stdout || "").trim() || "(log unavailable)";
|
|
1486
1700
|
}
|
|
1487
1701
|
|
|
1488
|
-
|
|
1702
|
+
// ─── Detached PROJECT bootstrap ──────────────────────────────────────────────
|
|
1703
|
+
//
|
|
1704
|
+
// The VM bootstrap above solved a problem the PROJECT bootstrap still had: read
|
|
1705
|
+
// success and failure from VM-SIDE STATE, never from the SSH exit code, so a
|
|
1706
|
+
// dropped follow-session over a healthy VM is not misreported as "failed on the
|
|
1707
|
+
// VM". The project bootstrap ran instead as ONE blocking ssh-script call, so a
|
|
1708
|
+
// local budget elapsing (or an SSH pipe breaking) was indistinguishable from a
|
|
1709
|
+
// genuine remote failure and the caller's abort path DELETED A HEALTHY VM.
|
|
1710
|
+
// Raising the budget alone just moves the cliff; these helpers remove it by
|
|
1711
|
+
// mirroring the VM-bootstrap contract — detached launch, re-attaching follow
|
|
1712
|
+
// loop, VM-side outcome probe.
|
|
1713
|
+
//
|
|
1714
|
+
// All VM-side paths are keyed by repo name so two projects on one VM never
|
|
1715
|
+
// collide, and they sit beside the `.done` marker projectBootstrapDoneMarker
|
|
1716
|
+
// already defines. Pure string builders (like their VM-bootstrap counterparts)
|
|
1717
|
+
// so the whole grammar is unit-testable without a live VM.
|
|
1718
|
+
function projectBootstrapScriptPath(repoName) {
|
|
1719
|
+
return `$HOME/.codeyam-project-bootstrap-${repoName}.sh`;
|
|
1720
|
+
}
|
|
1721
|
+
function projectBootstrapRunnerPath(repoName) {
|
|
1722
|
+
return `$HOME/.codeyam-project-bootstrap-${repoName}-run.sh`;
|
|
1723
|
+
}
|
|
1724
|
+
function projectBootstrapLogPath(repoName) {
|
|
1725
|
+
return `$HOME/.codeyam-project-bootstrap-${repoName}.log`;
|
|
1726
|
+
}
|
|
1727
|
+
function projectBootstrapPidPath(repoName) {
|
|
1728
|
+
return `$HOME/.codeyam-project-bootstrap-${repoName}.pid`;
|
|
1729
|
+
}
|
|
1730
|
+
// The runner's recorded EXIT STATUS. Deliberately `.status`, NOT `.done`:
|
|
1731
|
+
// `.done` is already taken by scripts/gcp-project-bootstrap.sh's own terminal
|
|
1732
|
+
// SUCCESS marker (a timestamp, written only after `set -euo pipefail` carried
|
|
1733
|
+
// every step through — see projectBootstrapDoneMarker). Reusing `.done` for an
|
|
1734
|
+
// exit code would overwrite the success signal that both improve40's false-skip
|
|
1735
|
+
// fix and the follow loop below depend on. Two files, two distinct meanings:
|
|
1736
|
+
// `.done` = "the script reached its last line", `.status` = "the runner exited,
|
|
1737
|
+
// here is its code".
|
|
1738
|
+
function projectBootstrapStatusPath(repoName) {
|
|
1739
|
+
return `$HOME/.codeyam-project-bootstrap-${repoName}.status`;
|
|
1740
|
+
}
|
|
1741
|
+
const PROJECT_BOOTSTRAP_LAUNCHED_MARKER = "CODEYAM_PROJECT_BOOTSTRAP_LAUNCHED";
|
|
1742
|
+
const PROJECT_BOOTSTRAP_STATUS_MARKER = "CYPROJ";
|
|
1743
|
+
// One follow session caps at 15 min; the overall wait caps at
|
|
1744
|
+
// PROJECT_BOOTSTRAP_TIMEOUT_MS across re-attaches before we bail loudly with
|
|
1745
|
+
// the VM-side log location. Unlike the VM-bootstrap follow loop — whose follow
|
|
1746
|
+
// calls inherit the 120 s probe budget and so re-attach every two minutes over
|
|
1747
|
+
// a ~14-min build — the project follow call below is given an explicit local
|
|
1748
|
+
// budget just above this remote deadline, so a long install streams in one
|
|
1749
|
+
// session instead of churning re-attaches.
|
|
1750
|
+
const PROJECT_BOOTSTRAP_FOLLOW_SECS = 900;
|
|
1751
|
+
const PROJECT_BOOTSTRAP_FOLLOW_TIMEOUT_MS = (PROJECT_BOOTSTRAP_FOLLOW_SECS + 60) * 1000;
|
|
1752
|
+
|
|
1753
|
+
// The tiny VM-side wrapper the detached launch execs: run the real bootstrap,
|
|
1754
|
+
// redirect all output to the per-repo log, and record the exit status in the
|
|
1755
|
+
// `.status` sentinel — the single source of truth for "did the runner exit, and
|
|
1756
|
+
// with what?" once the SSH follow-session is gone. Contains NO secrets (tokens
|
|
1757
|
+
// arrive via the launch env prefix, never written to disk).
|
|
1758
|
+
function projectBootstrapRunnerScript(repoName) {
|
|
1759
|
+
return [
|
|
1760
|
+
"#!/usr/bin/env bash",
|
|
1761
|
+
"# Detached project-bootstrap runner (written by npm/cloud.js",
|
|
1762
|
+
"# ensureProjectBootstrapped). Survives SSH disconnect: the laptop only",
|
|
1763
|
+
"# *follows* the log below; this process keeps running under its own session",
|
|
1764
|
+
"# if the follow drops. Records its own PID so the status probe can check",
|
|
1765
|
+
"# liveness via `kill -0` (a `pgrep` on the script path would also match the",
|
|
1766
|
+
"# probe command itself).",
|
|
1767
|
+
`echo "$$" > "${projectBootstrapPidPath(repoName)}"`,
|
|
1768
|
+
`bash "${projectBootstrapScriptPath(repoName)}" "$@" > "${projectBootstrapLogPath(repoName)}" 2>&1`,
|
|
1769
|
+
"status=$?",
|
|
1770
|
+
`echo "$status" > "${projectBootstrapStatusPath(repoName)}"`,
|
|
1771
|
+
`rm -f "${projectBootstrapPidPath(repoName)}"`,
|
|
1772
|
+
"",
|
|
1773
|
+
].join("\n");
|
|
1774
|
+
}
|
|
1775
|
+
|
|
1776
|
+
// Build the remote command that (1) materializes the bootstrap script + runner
|
|
1777
|
+
// on the VM, (2) clears the stale sentinels, (3) launches the runner DETACHED
|
|
1778
|
+
// (`nohup setsid`, new session, stdio to /dev/null) so it outlives the SSH
|
|
1779
|
+
// session, and (4) prints a launched marker. The detach is the durable fix: the
|
|
1780
|
+
// install no longer dies when the laptop's SSH pipe breaks, and the VM is no
|
|
1781
|
+
// longer torn down because it did.
|
|
1782
|
+
//
|
|
1783
|
+
// BOTH sentinels are cleared, `.status` and `.done`. Clearing `.done` is what
|
|
1784
|
+
// makes the outcome probe honest: after a launch, a present `.done` can only
|
|
1785
|
+
// mean THIS run's script reached its terminal line, never a leftover from a
|
|
1786
|
+
// previous half-wired attempt.
|
|
1787
|
+
function projectBootstrapLaunchCommand(scriptBody, scriptArgs = [], envVars = {}, repoName) {
|
|
1788
|
+
const scriptB64 = Buffer.from(scriptBody, "utf8").toString("base64");
|
|
1789
|
+
const runnerB64 = Buffer.from(projectBootstrapRunnerScript(repoName), "utf8").toString("base64");
|
|
1790
|
+
const argsString = scriptArgs.map(shQuote).join(" ");
|
|
1791
|
+
const envPrefix = envAssignmentPrefix(envVars);
|
|
1792
|
+
// Writes run synchronously (fast, before any backgrounding) so a decode error
|
|
1793
|
+
// aborts before launch; then the runner is detached. The env prefix is on the
|
|
1794
|
+
// setsid command so the tokens are inherited through nohup→setsid→bash but
|
|
1795
|
+
// never appear in any argv. `& echo MARKER` (no `;` before echo) keeps the
|
|
1796
|
+
// launch in the background while still confirming it fired.
|
|
1797
|
+
const writes = [
|
|
1798
|
+
`echo '${scriptB64}' | base64 -d > ${projectBootstrapScriptPath(repoName)}`,
|
|
1799
|
+
`echo '${runnerB64}' | base64 -d > ${projectBootstrapRunnerPath(repoName)}`,
|
|
1800
|
+
`rm -f ${projectBootstrapStatusPath(repoName)} ${projectBootstrapDoneMarker(repoName)}`,
|
|
1801
|
+
].join(" && ");
|
|
1802
|
+
const launch =
|
|
1803
|
+
`${envPrefix}nohup setsid bash ${projectBootstrapRunnerPath(repoName)} ${argsString}`.trimEnd() +
|
|
1804
|
+
` </dev/null >/dev/null 2>&1 &`;
|
|
1805
|
+
return `${writes}; ${launch} echo ${PROJECT_BOOTSTRAP_LAUNCHED_MARKER}`;
|
|
1806
|
+
}
|
|
1807
|
+
|
|
1808
|
+
// Build the remote command that FOLLOWS the detached install: tail the log live
|
|
1809
|
+
// for the operator, polling for the `.status` sentinel, and exit when the runner
|
|
1810
|
+
// finishes OR a per-session deadline elapses. If the SSH session drops
|
|
1811
|
+
// mid-follow this command dies but the detached install keeps going — the caller
|
|
1812
|
+
// then re-attaches.
|
|
1813
|
+
function projectBootstrapFollowCommand(repoName, deadlineSecs = PROJECT_BOOTSTRAP_FOLLOW_SECS) {
|
|
1814
|
+
const log = projectBootstrapLogPath(repoName);
|
|
1815
|
+
const status = projectBootstrapStatusPath(repoName);
|
|
1816
|
+
return [
|
|
1817
|
+
`touch ${log}`,
|
|
1818
|
+
`tail -n +1 -F ${log} 2>/dev/null & TP=$!`,
|
|
1819
|
+
`waited=0`,
|
|
1820
|
+
`while [ ! -f ${status} ]; do ` +
|
|
1821
|
+
`sleep 3; waited=$((waited+3)); ` +
|
|
1822
|
+
`if [ "$waited" -ge ${deadlineSecs} ]; then kill "$TP" 2>/dev/null; exit 0; fi; ` +
|
|
1823
|
+
`done`,
|
|
1824
|
+
`sleep 1`,
|
|
1825
|
+
`kill "$TP" 2>/dev/null`,
|
|
1826
|
+
].join("; ");
|
|
1827
|
+
}
|
|
1828
|
+
|
|
1829
|
+
// Build the remote command that reports terminal VM state in one parseable
|
|
1830
|
+
// line: whether the script's terminal `.done` marker exists, the runner's
|
|
1831
|
+
// recorded exit status (or `none`), and whether the runner is still alive.
|
|
1832
|
+
// Cheap — runs over ssh-probe.
|
|
1833
|
+
function projectBootstrapStatusCommand(repoName) {
|
|
1834
|
+
const pid = projectBootstrapPidPath(repoName);
|
|
1835
|
+
const status = projectBootstrapStatusPath(repoName);
|
|
1836
|
+
// Liveness via the runner's PID file + `kill -0`, NOT pgrep: the status
|
|
1837
|
+
// command's own argv contains the bootstrap path, so a `pgrep -f` on it would
|
|
1838
|
+
// match this very probe and always report running. A stale PID file from a
|
|
1839
|
+
// killed runner reads as not-running because `kill -0` fails.
|
|
1840
|
+
return [
|
|
1841
|
+
`done=no; [ -f ${projectBootstrapDoneMarker(repoName)} ] && done=yes`,
|
|
1842
|
+
`st=none; [ -f ${status} ] && st=$(cat ${status} 2>/dev/null)`,
|
|
1843
|
+
`run=no; if [ -f ${pid} ]; then p=$(cat ${pid} 2>/dev/null); ` +
|
|
1844
|
+
`{ [ -n "$p" ] && kill -0 "$p" 2>/dev/null; } && run=yes; fi`,
|
|
1845
|
+
`echo "${PROJECT_BOOTSTRAP_STATUS_MARKER} done=$done status=$st running=$run"`,
|
|
1846
|
+
].join("; ");
|
|
1847
|
+
}
|
|
1848
|
+
|
|
1849
|
+
// Parse the single status line emitted by projectBootstrapStatusCommand.
|
|
1850
|
+
// Returns `{ done, status, running }` or null when no status line is present
|
|
1851
|
+
// (e.g. the probe ssh itself failed and produced no stdout).
|
|
1852
|
+
function parseProjectBootstrapStatus(stdout) {
|
|
1853
|
+
const m = String(stdout || "").match(
|
|
1854
|
+
new RegExp(`${PROJECT_BOOTSTRAP_STATUS_MARKER} done=(\\S+) status=(\\S+) running=(\\S+)`),
|
|
1855
|
+
);
|
|
1856
|
+
if (!m) return null;
|
|
1857
|
+
return { done: m[1], status: m[2], running: m[3] };
|
|
1858
|
+
}
|
|
1859
|
+
|
|
1860
|
+
// Decide the project-bootstrap outcome from VM-side state alone — never the SSH
|
|
1861
|
+
// exit code. The `.done` marker is the real artifact (the script writes it only
|
|
1862
|
+
// after `set -euo pipefail` carried every step to success), so its presence is
|
|
1863
|
+
// success regardless of how the follow session ended — the same reasoning that
|
|
1864
|
+
// makes the docker image authoritative for the VM bootstrap. A live runner means
|
|
1865
|
+
// "re-attach". A recorded exit status without the marker means the install ran
|
|
1866
|
+
// and failed. Nothing at all means it was never started (or the probe could not
|
|
1867
|
+
// see it).
|
|
1868
|
+
function decideProjectBootstrapOutcome(status) {
|
|
1869
|
+
if (!status) return "absent";
|
|
1870
|
+
const { done, status: exitStatus, running } = status;
|
|
1871
|
+
if (done === "yes") return "success";
|
|
1872
|
+
if (running === "yes") return "running";
|
|
1873
|
+
if (exitStatus && exitStatus !== "none") return "failed";
|
|
1874
|
+
return "absent";
|
|
1875
|
+
}
|
|
1876
|
+
|
|
1877
|
+
// Reduce one follow-loop iteration to an action: "success" | "failed" |
|
|
1878
|
+
// "timeout" | "reattach". Extracted as a pure function because its PRECEDENCE is
|
|
1879
|
+
// load-bearing — it decides whether an operator is told a VM is broken:
|
|
1880
|
+
//
|
|
1881
|
+
// • success and failed outrank the deadline. Otherwise the final poll of a
|
|
1882
|
+
// slow-but-successful bootstrap reports a timeout, and the operator is sent
|
|
1883
|
+
// to reconcile a VM that actually finished.
|
|
1884
|
+
// • a still-running install past the deadline is a TIMEOUT, never a failure.
|
|
1885
|
+
// That is the VM-8 case: healthy VM, still installing, and saying so is what
|
|
1886
|
+
// gets the add classified retryable instead of destroyed.
|
|
1887
|
+
// • an unseen runner ("absent") before the deadline re-attaches rather than
|
|
1888
|
+
// failing — "I could not see it" is not "it broke" — but is still bounded by
|
|
1889
|
+
// the deadline so the loop cannot spin forever.
|
|
1890
|
+
function decideProjectBootstrapFollowAction({ outcome, now, deadline } = {}) {
|
|
1891
|
+
if (outcome === "success") return "success";
|
|
1892
|
+
if (outcome === "failed") return "failed";
|
|
1893
|
+
if (now >= deadline) return "timeout";
|
|
1894
|
+
return "reattach";
|
|
1895
|
+
}
|
|
1896
|
+
|
|
1897
|
+
// Probe the VM for the detached project bootstrap's terminal state and reduce it
|
|
1898
|
+
// to a single outcome string ("success" | "failed" | "running" | "absent").
|
|
1899
|
+
// Used both for the idempotent re-attach decision and after each follow session.
|
|
1900
|
+
function probeProjectBootstrapOutcome(cfg, repoName) {
|
|
1901
|
+
const r = runGcloud(
|
|
1902
|
+
buildGcloudArgs("ssh-probe", cfg, { command: projectBootstrapStatusCommand(repoName) }),
|
|
1903
|
+
{ stdio: ["ignore", "pipe", "pipe"] },
|
|
1904
|
+
);
|
|
1905
|
+
return decideProjectBootstrapOutcome(parseProjectBootstrapStatus(r.stdout));
|
|
1906
|
+
}
|
|
1907
|
+
|
|
1908
|
+
/** Assemble the env forwarded into the project-bootstrap shell on the VM.
|
|
1909
|
+
* Extracted from ensureProjectBootstrapped so its CONDITIONAL keys are testable
|
|
1910
|
+
* without a VM — previously you could not assert "FLEET_DOMAIN is omitted for a
|
|
1911
|
+
* non-fleet project" or "a re-provision without --admin stays non-admin"
|
|
1912
|
+
* without invoking the whole gcloud-touching bootstrap.
|
|
1913
|
+
*
|
|
1914
|
+
* Keys, and why each is here:
|
|
1915
|
+
* GITHUB_TOKEN — clone a private client repo.
|
|
1916
|
+
* CY_GIT_USER_{NAME,EMAIL} — rendered into the compose file, then written
|
|
1917
|
+
* to `git config --global` by the entrypoint so
|
|
1918
|
+
* plan_claim_refs::create_claim's commit-tree
|
|
1919
|
+
* call succeeds. Empty is legitimate (a CI
|
|
1920
|
+
* runner with no author); the entrypoint warns
|
|
1921
|
+
* and proceeds, so the keys stay present.
|
|
1922
|
+
* FLEET_DOMAIN — ONLY for a numbered fleet VM with a domain
|
|
1923
|
+
* configured, so the compose render publishes
|
|
1924
|
+
* on 0.0.0.0 for the LB NEG. Omitted entirely
|
|
1925
|
+
* otherwise, preserving the loopback-only
|
|
1926
|
+
* publish a plain client project expects.
|
|
1927
|
+
* <forwarded launch-form keys> — DATABASE_URL et al, whitelisted by the
|
|
1928
|
+
* dashboard via CY_FORWARD_ENV_KEYS, so the
|
|
1929
|
+
* bootstrap-time `npm run setup` sees them on
|
|
1930
|
+
* the first try. Empty on plain `cloud:up`.
|
|
1931
|
+
* CODEYAM_ADMIN — the Home-tab Setup door. Defaults to the
|
|
1932
|
+
* strict non-admin "".
|
|
1933
|
+
*
|
|
1934
|
+
* Later spreads win, matching the original inline assembly exactly. `deps` lets
|
|
1935
|
+
* a test supply the token and git identity instead of reading the real
|
|
1936
|
+
* environment and shelling out to git. */
|
|
1937
|
+
function projectBootstrapEnv(cfg = {}, { env = process.env, githubToken, gitIdentity } = {}) {
|
|
1938
|
+
const git = gitIdentity || resolveLaptopGitIdentity();
|
|
1939
|
+
const out = {
|
|
1940
|
+
GITHUB_TOKEN: githubToken !== undefined ? githubToken : resolveGithubToken(),
|
|
1941
|
+
CY_GIT_USER_NAME: git.name,
|
|
1942
|
+
CY_GIT_USER_EMAIL: git.email,
|
|
1943
|
+
};
|
|
1944
|
+
const ingressPlan = ingressWiringPlan(cfg, env);
|
|
1945
|
+
if (ingressPlan.wire) out.FLEET_DOMAIN = ingressPlan.domain;
|
|
1946
|
+
Object.assign(out, resolveSetupForwardEnv(env));
|
|
1947
|
+
Object.assign(out, buildProvisionEnv(env, cfg));
|
|
1948
|
+
return out;
|
|
1949
|
+
}
|
|
1950
|
+
|
|
1951
|
+
// Fetch the tail of the VM-side project-bootstrap log for an actionable failure
|
|
1952
|
+
// message — the operator sees WHY the install failed, not a bare exit code.
|
|
1953
|
+
function projectBootstrapLogTail(cfg, repoName, lines = 40) {
|
|
1954
|
+
const r = runGcloud(
|
|
1955
|
+
buildGcloudArgs("ssh-probe", cfg, {
|
|
1956
|
+
command: `tail -n ${lines} ${projectBootstrapLogPath(repoName)} 2>/dev/null`,
|
|
1957
|
+
}),
|
|
1958
|
+
{ stdio: ["ignore", "pipe", "pipe"] },
|
|
1959
|
+
);
|
|
1960
|
+
return String(r.stdout || "").trim() || "(log unavailable)";
|
|
1961
|
+
}
|
|
1962
|
+
|
|
1963
|
+
/** Assemble the extra provision-time env forwarded into the bootstrap shell,
|
|
1964
|
+
* where it feeds the compose-render substitution. Currently just the admin
|
|
1965
|
+
* toggle: CODEYAM_ADMIN gates the Home-tab Setup door and is read from the
|
|
1966
|
+
* editor-server process env (handlers.rs::admin_enabled, strict literal "1").
|
|
1967
|
+
* The operator opts in with `CODEYAM_ADMIN=1 npm run cloud:up …` or `--admin`
|
|
1968
|
+
* (which lands as cfg.admin); anything else forwards "" so a normal launch —
|
|
1969
|
+
* and a re-provision without the flag — is strict non-admin. Pure over its
|
|
1970
|
+
* inputs so cloud.test.js can pin it without touching gcloud. */
|
|
1971
|
+
function buildProvisionEnv(env = process.env, flags = {}) {
|
|
1972
|
+
const admin = flags.admin === true || env.CODEYAM_ADMIN === "1";
|
|
1973
|
+
return { CODEYAM_ADMIN: admin ? "1" : "" };
|
|
1974
|
+
}
|
|
1975
|
+
|
|
1976
|
+
/** Build the one-shot SSH shell command that reconciles a running VM's admin
|
|
1977
|
+
* mode without a full re-provision: rewrite the rendered compose's
|
|
1978
|
+
* CODEYAM_ADMIN line to `value` — or insert it under `environment:` on a legacy
|
|
1979
|
+
* VM whose compose predates the placeholder — then recreate the editor service
|
|
1980
|
+
* so admin_enabled() re-reads it. `value` is always "1" or "" (the caller
|
|
1981
|
+
* normalizes), so inlining it into sed is safe. Pure/string-only so
|
|
1982
|
+
* cloud.test.js can assert its shape without SSHing. */
|
|
1983
|
+
function buildReconcileAdminCommand(repoName, value) {
|
|
1984
|
+
const f = "docker-compose.cloud.yml";
|
|
1985
|
+
const entry = ` - CODEYAM_ADMIN=${value}`;
|
|
1986
|
+
// Capture only the leading indentation (\1) and re-emit the literal key, so a
|
|
1987
|
+
// value that starts with a digit (e.g. "1") never lands right after the
|
|
1988
|
+
// backreference — `\11` would be read by sed as group 11, not `\1` + "1".
|
|
1989
|
+
return (
|
|
1990
|
+
`cd ~/projects/${repoName} && ` +
|
|
1991
|
+
`if grep -qE '^[[:space:]]*- CODEYAM_ADMIN=' ${f}; then ` +
|
|
1992
|
+
`sed -i -E 's#^([[:space:]]*)- CODEYAM_ADMIN=.*#\\1- CODEYAM_ADMIN=${value}#' ${f}; ` +
|
|
1993
|
+
`else ` +
|
|
1994
|
+
`sed -i -E 's#^([[:space:]]*environment:[[:space:]]*)$#\\1\\n${entry}#' ${f}; ` +
|
|
1995
|
+
`fi && ` +
|
|
1996
|
+
`docker compose -f ${f} up -d editor`
|
|
1997
|
+
);
|
|
1998
|
+
}
|
|
1999
|
+
|
|
2000
|
+
async function ensureProjectBootstrapped(cfg, { sleepFn } = {}) {
|
|
1489
2001
|
const repoName = resolveProject(cfg).name;
|
|
1490
2002
|
console.error(`Checking project bootstrap state for ${repoName}...`);
|
|
1491
2003
|
// Gate on the TERMINAL completion marker the script writes as its very last
|
|
@@ -1513,45 +2025,156 @@ async function ensureProjectBootstrapped(cfg) {
|
|
|
1513
2025
|
}
|
|
1514
2026
|
console.error(` streaming scripts/gcp-project-bootstrap.sh to VM for ${cfg.repo}...`);
|
|
1515
2027
|
const scriptPath = path.join(rootDir, "scripts", "gcp-project-bootstrap.sh");
|
|
1516
|
-
const scriptBody =
|
|
2028
|
+
const scriptBody = readStreamedScript(scriptPath, cfg);
|
|
1517
2029
|
const scriptArgs = clientProjectBootstrapArgs(cfg);
|
|
1518
|
-
//
|
|
1519
|
-
//
|
|
1520
|
-
//
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
//
|
|
1525
|
-
//
|
|
1526
|
-
//
|
|
1527
|
-
//
|
|
1528
|
-
// the
|
|
1529
|
-
//
|
|
1530
|
-
//
|
|
1531
|
-
//
|
|
1532
|
-
|
|
1533
|
-
//
|
|
1534
|
-
//
|
|
1535
|
-
|
|
1536
|
-
|
|
1537
|
-
|
|
1538
|
-
|
|
1539
|
-
|
|
1540
|
-
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
|
|
1548
|
-
|
|
1549
|
-
|
|
1550
|
-
|
|
2030
|
+
// Git identity, forwarded launch-form keys, fleet-domain ingress and the admin
|
|
2031
|
+
// toggle — see projectBootstrapEnv for what each key is for and when it is
|
|
2032
|
+
// omitted. Extracted so those conditionals are testable without a VM.
|
|
2033
|
+
const bootstrapEnv = projectBootstrapEnv(cfg);
|
|
2034
|
+
|
|
2035
|
+
// Idempotent re-attach: a prior cloud:up whose follow-session dropped may have
|
|
2036
|
+
// left a detached install still running. Converge on it instead of starting a
|
|
2037
|
+
// second `npm install` on top of the first.
|
|
2038
|
+
//
|
|
2039
|
+
// NOTE the deliberate asymmetry with ensureBootstrapped, which DOES return
|
|
2040
|
+
// early on an outcome of "success". There, the done-probe and the outcome probe
|
|
2041
|
+
// read the same artifact (the docker image), so they always agree. Here the
|
|
2042
|
+
// done-probe above is a strictly STRONGER three-way AND (`.done` marker +
|
|
2043
|
+
// ~/projects/<repo>/.codeyam + docker-compose.cloud.yml) than this probe's
|
|
2044
|
+
// `.done`-alone success, and it has already said no. Honoring a bare `.done`
|
|
2045
|
+
// here would re-open the improve40 VM-7 false-skip from the other side, so only
|
|
2046
|
+
// "running" short-circuits the launch.
|
|
2047
|
+
const initial = probeProjectBootstrapOutcome(cfg, repoName);
|
|
2048
|
+
if (initial === "running") {
|
|
2049
|
+
console.error(" ↺ a detached project bootstrap is already in progress — re-attaching to its log...");
|
|
2050
|
+
} else {
|
|
2051
|
+
// "absent" (never started), "failed" (a prior run died), or "success" (a bare
|
|
2052
|
+
// marker the stricter done-probe rejected) → (re)launch a fresh detached
|
|
2053
|
+
// install that will survive an SSH disconnect. Every step of
|
|
2054
|
+
// gcp-project-bootstrap.sh is skip-on-exists, so re-entering is cheap.
|
|
2055
|
+
console.error(" launching detached project bootstrap on the VM (survives SSH disconnect)...");
|
|
2056
|
+
const launch = projectBootstrapLaunchCommand(scriptBody, scriptArgs, bootstrapEnv, repoName);
|
|
2057
|
+
const launched = await runGcloudSshWithRetry(
|
|
2058
|
+
"ssh-script",
|
|
2059
|
+
cfg,
|
|
2060
|
+
{ command: launch },
|
|
2061
|
+
{ stdio: ["ignore", "pipe", "pipe"], label: "project bootstrap launch", sleepFn },
|
|
2062
|
+
);
|
|
2063
|
+
if (isSpawnTimeout(launched)) {
|
|
2064
|
+
// The LOCAL budget elapsed and we SIGTERMd the gcloud child. Say so:
|
|
2065
|
+
// the launch is a short write-and-detach, so this is an SSH-reachability
|
|
2066
|
+
// problem, NOT a broken bootstrap script — and the VM may be perfectly
|
|
2067
|
+
// healthy. Conflating the two is what destroyed VM-8.
|
|
2068
|
+
throw new Error(
|
|
2069
|
+
`launching the detached project bootstrap exceeded the local gcloud call budget ` +
|
|
2070
|
+
`and was SIGTERMd locally — the VM may be healthy and simply unreachable over SSH ` +
|
|
2071
|
+
`right now. Re-run to re-attach (each bootstrap step is skip-on-exists).`,
|
|
2072
|
+
);
|
|
2073
|
+
}
|
|
2074
|
+
if (launched.status !== 0) {
|
|
2075
|
+
throw new Error(
|
|
2076
|
+
`failed to launch the detached project bootstrap on the VM (gcloud ssh exit ${launched.status}).\n` +
|
|
2077
|
+
formatGcloudFailure(launched),
|
|
2078
|
+
);
|
|
2079
|
+
}
|
|
2080
|
+
}
|
|
2081
|
+
|
|
2082
|
+
// Follow the live log, re-attaching across any follow-session drops, until the
|
|
2083
|
+
// VM reports a terminal state or the overall deadline elapses. Success and
|
|
2084
|
+
// failure are read from VM state (the `.done` marker + the `.status` sentinel),
|
|
2085
|
+
// never the SSH exit code — so "my laptop's SSH dropped" or "a heavy npm
|
|
2086
|
+
// install outran a local budget" no longer masquerades as a broken bootstrap
|
|
2087
|
+
// and no longer gets a healthy VM deleted.
|
|
2088
|
+
const deadline = Date.now() + PROJECT_BOOTSTRAP_TIMEOUT_MS;
|
|
2089
|
+
for (;;) {
|
|
2090
|
+
runGcloud(
|
|
2091
|
+
buildGcloudArgs("ssh-script", cfg, { command: projectBootstrapFollowCommand(repoName) }),
|
|
2092
|
+
{ stdio: "inherit", timeout: PROJECT_BOOTSTRAP_FOLLOW_TIMEOUT_MS },
|
|
2093
|
+
);
|
|
2094
|
+
const outcome = probeProjectBootstrapOutcome(cfg, repoName);
|
|
2095
|
+
const action = decideProjectBootstrapFollowAction({
|
|
2096
|
+
outcome,
|
|
2097
|
+
now: Date.now(),
|
|
2098
|
+
deadline,
|
|
2099
|
+
});
|
|
2100
|
+
if (action === "success") {
|
|
2101
|
+
console.error(` ✓ ${repoName} bootstrap complete`);
|
|
2102
|
+
return;
|
|
2103
|
+
}
|
|
2104
|
+
if (action === "failed") {
|
|
2105
|
+
const tail = projectBootstrapLogTail(cfg, repoName);
|
|
2106
|
+
throw new Error(
|
|
2107
|
+
"scripts/gcp-project-bootstrap.sh failed on the VM. " +
|
|
2108
|
+
`Last lines of ${projectBootstrapLogPath(repoName)}:\n${tail}`,
|
|
2109
|
+
);
|
|
2110
|
+
}
|
|
2111
|
+
if (action === "timeout") {
|
|
2112
|
+
// The install is STILL RUNNING on a healthy VM and we simply stopped
|
|
2113
|
+
// waiting. This message is the signature npm/fleet-add-failure.js keys on
|
|
2114
|
+
// to classify the add as `project-bootstrap-timeout` (retryable) instead of
|
|
2115
|
+
// the non-retryable generic cloud-up-failed that tore down VM-8.
|
|
2116
|
+
throw new Error(
|
|
2117
|
+
`project bootstrap exceeded the local ${Math.round(PROJECT_BOOTSTRAP_TIMEOUT_MS / 60000)}-minute budget ` +
|
|
2118
|
+
`— the VM and its detached bootstrap may be healthy and still running. ` +
|
|
2119
|
+
`Re-run to re-attach (each bootstrap step is skip-on-exists), or inspect ` +
|
|
2120
|
+
`${projectBootstrapLogPath(repoName)} on the VM.`,
|
|
2121
|
+
);
|
|
2122
|
+
}
|
|
2123
|
+
// action === "reattach": the follow session ended but the install is alive
|
|
2124
|
+
// (or the probe could not see it yet). Re-attach to its log.
|
|
2125
|
+
console.error(" ↺ follow session ended but the project bootstrap is still running — re-attaching...");
|
|
2126
|
+
}
|
|
2127
|
+
}
|
|
2128
|
+
|
|
2129
|
+
/** Run the compose-ps liveness probe over SSH and report BOTH the editor
|
|
2130
|
+
* service's state and whether the probe itself could be trusted.
|
|
2131
|
+
*
|
|
2132
|
+
* The `|| true` keeps the remote pipeline's exit status 0 whenever SSH itself
|
|
2133
|
+
* worked, so a non-zero `status` means the transport failed (auth/SSH/IAP) —
|
|
2134
|
+
* not that the container is down. Callers that must fail safe (the abort
|
|
2135
|
+
* teardown) need that distinction; `parseEditorComposeState` alone collapses
|
|
2136
|
+
* "no container" and "could not ask" into the same `null`.
|
|
2137
|
+
*
|
|
2138
|
+
* Returns `{ state, inconclusive }` where `state` is the compose `State` string
|
|
2139
|
+
* (`"running"`, `"exited"`, …) or null when the service is absent, and
|
|
2140
|
+
* `inconclusive` is true only for a transport failure. */
|
|
2141
|
+
function probeEditorComposeState(cfg) {
|
|
2142
|
+
const repoName = resolveProject(cfg).name;
|
|
2143
|
+
const composeCmd = `docker compose -f docker-compose.cloud.yml`;
|
|
2144
|
+
const probe = runGcloud(
|
|
2145
|
+
buildGcloudArgs("ssh-probe", cfg, {
|
|
2146
|
+
command: `cd ~/projects/${repoName} 2>/dev/null && ${composeCmd} ps --format json 2>/dev/null || true`,
|
|
2147
|
+
}),
|
|
2148
|
+
{ stdio: ["ignore", "pipe", "ignore"] },
|
|
1551
2149
|
);
|
|
1552
|
-
if (
|
|
1553
|
-
|
|
2150
|
+
if (probe.status !== 0) {
|
|
2151
|
+
return { state: null, inconclusive: true };
|
|
2152
|
+
}
|
|
2153
|
+
return { state: parseEditorComposeState(probe.stdout), inconclusive: false };
|
|
2154
|
+
}
|
|
2155
|
+
|
|
2156
|
+
/** Poll {@link probeEditorComposeState} until the editor service reads
|
|
2157
|
+
* `running`, returning true as soon as it does. Used to close the TOCTOU window
|
|
2158
|
+
* after a container-name conflict: the winning run may still be inside
|
|
2159
|
+
* `Creating`/`Created` when the conflict surfaces, so a single immediate probe
|
|
2160
|
+
* can under-read and report a healthy container as absent.
|
|
2161
|
+
*
|
|
2162
|
+
* Deliberately short (default ~30s): this is not the multi-minute first-boot
|
|
2163
|
+
* budget `waitForContainerRunning` grants, it is the tail of a start that
|
|
2164
|
+
* another process has already begun. `sleepFn` and `probeFn` are injectable so
|
|
2165
|
+
* a unit test drives the whole loop with neither wall-clock delay nor a VM. */
|
|
2166
|
+
async function waitForEditorComposeRunning(cfg, { attempts = 6, delayMs = 5000, sleepFn, probeFn } = {}) {
|
|
2167
|
+
const sleep = sleepFn || ((ms) => new Promise((r) => setTimeout(r, ms)));
|
|
2168
|
+
const probe = probeFn || probeEditorComposeState;
|
|
2169
|
+
for (let i = 0; i < attempts; i++) {
|
|
2170
|
+
if (probe(cfg).state === "running") {
|
|
2171
|
+
return true;
|
|
2172
|
+
}
|
|
2173
|
+
if (i < attempts - 1) {
|
|
2174
|
+
await sleep(delayMs);
|
|
2175
|
+
}
|
|
1554
2176
|
}
|
|
2177
|
+
return false;
|
|
1555
2178
|
}
|
|
1556
2179
|
|
|
1557
2180
|
async function ensureEditorContainerUp(cfg) {
|
|
@@ -1561,28 +2184,60 @@ async function ensureEditorContainerUp(cfg) {
|
|
|
1561
2184
|
// cloud variant when self-hosting (see gcp-project-bootstrap.sh).
|
|
1562
2185
|
const composeCmd = `docker compose -f docker-compose.cloud.yml`;
|
|
1563
2186
|
console.error("Checking editor container state...");
|
|
1564
|
-
|
|
1565
|
-
buildGcloudArgs("ssh-probe", cfg, {
|
|
1566
|
-
command: `cd ~/projects/${repoName} && ${composeCmd} ps --format json 2>/dev/null || true`,
|
|
1567
|
-
}),
|
|
1568
|
-
{ stdio: ["ignore", "pipe", "ignore"] },
|
|
1569
|
-
);
|
|
1570
|
-
if (probe.status === 0 && parseEditorComposeState(probe.stdout) === "running") {
|
|
2187
|
+
if (probeEditorComposeState(cfg).state === "running") {
|
|
1571
2188
|
console.error(" ✓ editor container is already running");
|
|
1572
|
-
|
|
2189
|
+
// containerStarted: the container is confirmed up, so the CLI install runs
|
|
2190
|
+
// its POST-start path with no wait — see the precondition on ensureAgentCliInstalled.
|
|
2191
|
+
await ensureAgentCliInstalled(cfg, { containerStarted: true });
|
|
1573
2192
|
return;
|
|
1574
2193
|
}
|
|
1575
2194
|
console.error(" starting editor container (docker compose up -d)...");
|
|
2195
|
+
// CAPTURE rather than `stdio: "inherit"`: the daemon writes its container-name
|
|
2196
|
+
// conflict to stderr, and with "inherit" that text went straight to our parent
|
|
2197
|
+
// and was never observable here — so the conflict could not be told apart from
|
|
2198
|
+
// a real failure (the 2026-08-07 VM-4/VM-5 delete). spawnSync cannot both
|
|
2199
|
+
// stream and capture, so the operator's view is preserved by echoing the
|
|
2200
|
+
// captured streams immediately below instead of live. Do NOT revert this to
|
|
2201
|
+
// "inherit"; classification depends on it.
|
|
1576
2202
|
const up = runGcloud(
|
|
1577
2203
|
buildGcloudArgs("ssh-probe", cfg, {
|
|
1578
2204
|
command: `cd ~/projects/${repoName} && ${composeCmd} up -d`,
|
|
1579
2205
|
}),
|
|
1580
|
-
{ stdio: "
|
|
2206
|
+
{ stdio: ["ignore", "pipe", "pipe"] },
|
|
1581
2207
|
);
|
|
2208
|
+
if (up.stdout) process.stderr.write(up.stdout);
|
|
2209
|
+
if (up.stderr) process.stderr.write(up.stderr);
|
|
1582
2210
|
if (up.status !== 0) {
|
|
1583
|
-
|
|
2211
|
+
const verdict = classifyComposeUpFailure({ status: up.status, stderr: `${up.stderr || ""}\n${up.stdout || ""}` });
|
|
2212
|
+
if (verdict !== "name-conflict") {
|
|
2213
|
+
throw new Error("docker compose up -d failed on the VM");
|
|
2214
|
+
}
|
|
2215
|
+
// A name conflict means a concurrent run already created this container —
|
|
2216
|
+
// the goal we were about to pursue is being reached by someone else. Confirm
|
|
2217
|
+
// that positively before converging: a conflict whose container never
|
|
2218
|
+
// reaches `running` (e.g. a stale STOPPED container squatting the name) is a
|
|
2219
|
+
// genuine problem and must still fail loudly, or convergence becomes a
|
|
2220
|
+
// catch-all that papers over real breakage.
|
|
2221
|
+
console.error(
|
|
2222
|
+
" a concurrent run already created the editor container — confirming it reaches running...",
|
|
2223
|
+
);
|
|
2224
|
+
if (!(await waitForEditorComposeRunning(cfg))) {
|
|
2225
|
+
throw new Error(
|
|
2226
|
+
"docker compose up -d hit a container-name conflict on the VM, but the conflicting " +
|
|
2227
|
+
"container never reached running (a stale stopped container may be holding the name). " +
|
|
2228
|
+
`Inspect with: gcloud compute ssh ${cfg.instanceName} --command 'cd ~/projects/${repoName} && ${composeCmd} ps -a'`,
|
|
2229
|
+
);
|
|
2230
|
+
}
|
|
2231
|
+
console.error(" ✓ editor container is running (started by a concurrent run)");
|
|
2232
|
+
// containerStarted: the container is confirmed up — same post-start contract
|
|
2233
|
+
// as the already-running short-circuit above.
|
|
2234
|
+
await ensureAgentCliInstalled(cfg, { containerStarted: true });
|
|
2235
|
+
return;
|
|
1584
2236
|
}
|
|
1585
|
-
|
|
2237
|
+
// containerStarted: `up -d` returned 0, so the CLI install is legitimately a
|
|
2238
|
+
// post-start step. A brief wait-for-running here (slow boot settling) is
|
|
2239
|
+
// expected and must NOT warn about mis-ordering.
|
|
2240
|
+
await ensureAgentCliInstalled(cfg, { containerStarted: true });
|
|
1586
2241
|
}
|
|
1587
2242
|
|
|
1588
2243
|
// Hard gate: provisioning success REQUIRES a running editor container. Reuses
|
|
@@ -1593,18 +2248,10 @@ async function ensureEditorContainerUp(cfg) {
|
|
|
1593
2248
|
// success. This throws (non-zero exit) when the editor service isn't
|
|
1594
2249
|
// `running`, converting a silent partial provision into a loud, actionable
|
|
1595
2250
|
// failure the operator (and the dashboard's fail-fast-on-nonzero path) can act
|
|
1596
|
-
// on.
|
|
1597
|
-
//
|
|
2251
|
+
// on. An inconclusive probe is not "running" here — a terminal gate must have
|
|
2252
|
+
// positive evidence, and the caller's job is to fail loudly without it.
|
|
1598
2253
|
async function assertEditorContainerRunning(cfg) {
|
|
1599
|
-
|
|
1600
|
-
const composeCmd = `docker compose -f docker-compose.cloud.yml`;
|
|
1601
|
-
const probe = runGcloud(
|
|
1602
|
-
buildGcloudArgs("ssh-probe", cfg, {
|
|
1603
|
-
command: `cd ~/projects/${repoName} 2>/dev/null && ${composeCmd} ps --format json 2>/dev/null || true`,
|
|
1604
|
-
}),
|
|
1605
|
-
{ stdio: ["ignore", "pipe", "ignore"] },
|
|
1606
|
-
);
|
|
1607
|
-
if (!(probe.status === 0 && parseEditorComposeState(probe.stdout) === "running")) {
|
|
2254
|
+
if (probeEditorComposeState(cfg).state !== "running") {
|
|
1608
2255
|
throw new Error(
|
|
1609
2256
|
`provisioning incomplete: editor container is not running on ${cfg.instanceName} ` +
|
|
1610
2257
|
`(no /workspace or container). Re-run: npm run cloud:up -- --instance-name ${cfg.instanceName}`,
|
|
@@ -1612,6 +2259,36 @@ async function assertEditorContainerRunning(cfg) {
|
|
|
1612
2259
|
}
|
|
1613
2260
|
}
|
|
1614
2261
|
|
|
2262
|
+
// Hard gate: provisioning success ALSO REQUIRES the agent CLI present in the
|
|
2263
|
+
// container. The image deliberately ships without it (Dockerfile
|
|
2264
|
+
// INSTALL_BUILD_AGENT_CLI=none), and the in-container install is best-effort —
|
|
2265
|
+
// if it was skipped (container not up when it first ran), nothing else would
|
|
2266
|
+
// catch the absent binary and the editor's auth probe times out forever, leaving
|
|
2267
|
+
// the Build tab stuck on "Checking your account…" (the 2026-06-30 reprovision
|
|
2268
|
+
// defect). This re-probes for `command -v <binary>`, makes one more install
|
|
2269
|
+
// attempt if it's missing, then throws (loud non-zero exit the dashboard's
|
|
2270
|
+
// fail-fast path acts on) if it STILL can't be installed — never rostering a
|
|
2271
|
+
// CLI-less VM. Runs AFTER assertEditorContainerRunning, so the container is
|
|
2272
|
+
// known up by the time we probe the binary.
|
|
2273
|
+
async function assertAgentCliInstalled(cfg) {
|
|
2274
|
+
const agent = cfg.agent;
|
|
2275
|
+
const container = process.env.CONTAINER || "codeyam-editor-editor-1";
|
|
2276
|
+
if (probeAgentBinaryInstalled(cfg, container, agent)) {
|
|
2277
|
+
return;
|
|
2278
|
+
}
|
|
2279
|
+
console.error(` build-agent CLI '${agent}' not found in ${container}; attempting install before failing...`);
|
|
2280
|
+
// containerStarted: assertAgentCliInstalled runs AFTER assertEditorContainerRunning,
|
|
2281
|
+
// so the container is known up; this install is a legitimate post-start retry.
|
|
2282
|
+
await ensureAgentCliInstalled(cfg, { containerStarted: true });
|
|
2283
|
+
if (!probeAgentBinaryInstalled(cfg, container, agent)) {
|
|
2284
|
+
throw new Error(
|
|
2285
|
+
`provisioning incomplete: build-agent CLI '${agent}' is not installed in ${container} on ${cfg.instanceName} ` +
|
|
2286
|
+
`(image ships without it and the in-container install failed). The editor auth probe will hang until it's present. ` +
|
|
2287
|
+
`Re-run: npm run cloud:up -- --instance-name ${cfg.instanceName}`,
|
|
2288
|
+
);
|
|
2289
|
+
}
|
|
2290
|
+
}
|
|
2291
|
+
|
|
1615
2292
|
/** Read the laptop's git author identity (`user.name`, `user.email`) so
|
|
1616
2293
|
* cloud:up can forward it to the VM's container entrypoint, which writes
|
|
1617
2294
|
* it to `git config --global`. Without identity in the container,
|
|
@@ -1717,49 +2394,113 @@ function resolveGhcrPushToken() {
|
|
|
1717
2394
|
return process.env.CODEYAM_GHCR_PUSH_TOKEN || "";
|
|
1718
2395
|
}
|
|
1719
2396
|
|
|
2397
|
+
// Registry manifest media types we accept when probing image presence over the
|
|
2398
|
+
// GHCR HTTP API — Docker v2 + manifest lists and their OCI equivalents, so a
|
|
2399
|
+
// multi-arch index answers 200 the same as a single-arch manifest.
|
|
2400
|
+
const MANIFEST_ACCEPT = [
|
|
2401
|
+
"application/vnd.docker.distribution.manifest.v2+json",
|
|
2402
|
+
"application/vnd.docker.distribution.manifest.list.v2+json",
|
|
2403
|
+
"application/vnd.oci.image.manifest.v1+json",
|
|
2404
|
+
"application/vnd.oci.image.index.v1+json",
|
|
2405
|
+
].join(", ");
|
|
2406
|
+
const GHCR_PROBE_TIMEOUT_MS = 15000;
|
|
2407
|
+
|
|
2408
|
+
/** Default HTTPS client for the registry probe: a GET that resolves to
|
|
2409
|
+
* `{ statusCode }` and NEVER rejects — any socket / DNS / timeout error resolves
|
|
2410
|
+
* to `statusCode: 0` (the probe reads that as "undetermined"). Injected as the
|
|
2411
|
+
* `httpImpl` option so probeUsableEditorImage is unit-testable with no network
|
|
2412
|
+
* (and no Docker daemon). `requestImpl` (defaulting to `https.request`) is
|
|
2413
|
+
* injectable so the response / error / timeout mapping is itself unit-testable
|
|
2414
|
+
* without a real socket. */
|
|
2415
|
+
function httpsManifestGet({ hostname, path: reqPath, headers }, requestImpl = https.request) {
|
|
2416
|
+
return new Promise((resolve) => {
|
|
2417
|
+
const req = requestImpl(
|
|
2418
|
+
{ hostname, path: reqPath, method: "GET", headers },
|
|
2419
|
+
(res) => {
|
|
2420
|
+
res.resume(); // drain the body so the socket is freed
|
|
2421
|
+
res.on("end", () => resolve({ statusCode: res.statusCode }));
|
|
2422
|
+
},
|
|
2423
|
+
);
|
|
2424
|
+
req.on("error", () => resolve({ statusCode: 0 }));
|
|
2425
|
+
req.setTimeout(GHCR_PROBE_TIMEOUT_MS, () => req.destroy());
|
|
2426
|
+
req.end();
|
|
2427
|
+
});
|
|
2428
|
+
}
|
|
2429
|
+
|
|
2430
|
+
/** Probe ONE fully-qualified image ref (`host/repo:tag`) over the registry HTTP
|
|
2431
|
+
* API — no docker daemon, so a docker-less operator host can still determine
|
|
2432
|
+
* presence. Returns true (200 present), false (404 absent), or null (auth /
|
|
2433
|
+
* network / malformed-ref → can't tell absent from forbidden). */
|
|
2434
|
+
async function probeManifestRef(fullRef, bearer, httpGet) {
|
|
2435
|
+
const colon = fullRef.lastIndexOf(":");
|
|
2436
|
+
if (colon < 0) return null;
|
|
2437
|
+
const ref = fullRef.slice(0, colon); // host/repo
|
|
2438
|
+
const tag = fullRef.slice(colon + 1); // slug-latest
|
|
2439
|
+
const slash = ref.indexOf("/");
|
|
2440
|
+
if (slash < 0) return null;
|
|
2441
|
+
const hostname = ref.slice(0, slash); // ghcr.io
|
|
2442
|
+
const repo = ref.slice(slash + 1); // codeyam-ai/codeyam-editor-cloud-smoke
|
|
2443
|
+
let res;
|
|
2444
|
+
try {
|
|
2445
|
+
res = await httpGet({
|
|
2446
|
+
hostname,
|
|
2447
|
+
path: `/v2/${repo}/manifests/${encodeURIComponent(tag)}`,
|
|
2448
|
+
headers: { Authorization: `Bearer ${bearer}`, Accept: MANIFEST_ACCEPT },
|
|
2449
|
+
});
|
|
2450
|
+
} catch (_e) {
|
|
2451
|
+
return null;
|
|
2452
|
+
}
|
|
2453
|
+
const code = res && res.statusCode;
|
|
2454
|
+
if (code === 200) return true;
|
|
2455
|
+
if (code === 404) return false;
|
|
2456
|
+
return null; // 401 / 403 / 5xx / 0 (network) → undetermined
|
|
2457
|
+
}
|
|
2458
|
+
|
|
1720
2459
|
/** Read-only probe: does a usable prebuilt editor image already exist on the
|
|
1721
2460
|
* registry for `branch`? Used by the fleet dashboard BEFORE a concurrent batch
|
|
1722
2461
|
* add to decide whether to pace one VM ahead of the rest (see
|
|
1723
|
-
* npm/fleet-prewarm.js::decideBatchPrewarm).
|
|
1724
|
-
*
|
|
2462
|
+
* npm/fleet-prewarm.js::decideBatchPrewarm).
|
|
2463
|
+
*
|
|
2464
|
+
* Queries the registry over its HTTP API (`GET /v2/<repo>/manifests/<tag>`, a
|
|
2465
|
+
* HEAD-equivalent that reads the manifest without pulling the multi-GB image)
|
|
2466
|
+
* instead of `docker login` / `docker manifest inspect`. This is the root fix
|
|
2467
|
+
* for the docker-less GCE operator host: the old docker path errored on the
|
|
2468
|
+
* first `docker login` spawn (the control plane has no docker daemon) and so
|
|
2469
|
+
* returned `null` forever, which decideBatchPrewarm read as "fan out" — the
|
|
2470
|
+
* image was therefore never published and every cold add source-built. Over
|
|
2471
|
+
* HTTP the probe returns a DEFINITIVE verdict with no daemon.
|
|
1725
2472
|
*
|
|
1726
2473
|
* Returns:
|
|
1727
2474
|
* true — at least one of the usable tags exists (a cold add can pull; no pacer)
|
|
1728
2475
|
* false — every tag was checked and none exists (known-absent; pacing pays)
|
|
1729
|
-
* null — undetermined: no read token,
|
|
1730
|
-
*
|
|
1731
|
-
* "fan out" (today's behavior, never worse) rather than risk a
|
|
1732
|
-
* 10-min serialization that might buy nothing.
|
|
2476
|
+
* null — undetermined: no read token, or an auth/network error that can't be
|
|
2477
|
+
* distinguished from "absent".
|
|
1733
2478
|
*
|
|
1734
|
-
*
|
|
1735
|
-
*
|
|
1736
|
-
*
|
|
1737
|
-
|
|
2479
|
+
* GHCR's private packages require auth; without a read token we can't tell
|
|
2480
|
+
* absent from forbidden, so → null. The read token is sent as the registry
|
|
2481
|
+
* bearer (GHCR accepts a base64-encoded PAT directly — no separate token-
|
|
2482
|
+
* exchange round-trip). `httpImpl` is injectable for unit tests; a legacy
|
|
2483
|
+
* `spawnImpl` option is still accepted (and ignored) for backward-compat — the
|
|
2484
|
+
* HTTP happy path needs no spawn. */
|
|
2485
|
+
async function probeUsableEditorImage({ branch, registry, token, httpImpl } = {}) {
|
|
1738
2486
|
const fp = require("./fleet-prewarm");
|
|
1739
|
-
const spawn = spawnImpl || spawnSync;
|
|
1740
2487
|
const reg = registry || process.env.CODEYAM_REGISTRY || "ghcr.io/codeyam-ai";
|
|
1741
2488
|
const readToken = token || resolveGhcrToken();
|
|
1742
2489
|
if (!readToken) return null; // can't auth → can't distinguish absent from forbidden
|
|
1743
|
-
const
|
|
1744
|
-
|
|
1745
|
-
|
|
1746
|
-
|
|
1747
|
-
{ input: readToken, encoding: "utf8" },
|
|
1748
|
-
);
|
|
1749
|
-
// spawn error (docker missing) or a non-zero login → undetermined.
|
|
1750
|
-
if (!login || login.error || login.status !== 0) return null;
|
|
2490
|
+
const httpGet = httpImpl || httpsManifestGet;
|
|
2491
|
+
// GHCR accepts a base64-encoded PAT as the registry bearer — no token-exchange
|
|
2492
|
+
// round-trip needed.
|
|
2493
|
+
const bearer = Buffer.from(readToken).toString("base64");
|
|
1751
2494
|
const tags = fp.usableImageTags({ registry: reg, branch });
|
|
1752
2495
|
let allClean = true;
|
|
1753
|
-
for (const
|
|
1754
|
-
const
|
|
1755
|
-
if (
|
|
1756
|
-
|
|
1757
|
-
|
|
1758
|
-
}
|
|
1759
|
-
if (res.status === 0) return true; // a usable image exists
|
|
2496
|
+
for (const fullRef of tags) {
|
|
2497
|
+
const verdict = await probeManifestRef(fullRef, bearer, httpGet);
|
|
2498
|
+
if (verdict === true) return true; // a usable image exists
|
|
2499
|
+
if (verdict === null) allClean = false; // this tag's probe was inconclusive
|
|
2500
|
+
// verdict === false → this tag is absent; keep checking the rest
|
|
1760
2501
|
}
|
|
1761
|
-
// Every tag
|
|
1762
|
-
//
|
|
2502
|
+
// Every tag probed cleanly and none existed → definitively absent. If any probe
|
|
2503
|
+
// was inconclusive we can't claim absence → undetermined.
|
|
1763
2504
|
return allClean ? false : null;
|
|
1764
2505
|
}
|
|
1765
2506
|
|
|
@@ -1796,6 +2537,15 @@ function classifyGitRemoteError(stderr) {
|
|
|
1796
2537
|
) {
|
|
1797
2538
|
return "network";
|
|
1798
2539
|
}
|
|
2540
|
+
// A greenfield project (`--new-project` scaffold) has no origin remote and an
|
|
2541
|
+
// unborn `main`, so `git ls-remote origin` fails with "does not appear to be a
|
|
2542
|
+
// git repository". No token would fix this — there is simply nothing to push
|
|
2543
|
+
// yet — so it is neither `credential` nor `network` nor `git-spawn`. Keep the
|
|
2544
|
+
// match narrow: `repository not found` is a genuine access failure against a
|
|
2545
|
+
// real remote and must NOT be swallowed here.
|
|
2546
|
+
if (lower.includes("does not appear to be a git repository")) {
|
|
2547
|
+
return "no-remote";
|
|
2548
|
+
}
|
|
1799
2549
|
return "other";
|
|
1800
2550
|
}
|
|
1801
2551
|
|
|
@@ -1833,6 +2583,14 @@ function decideCredentialHeal({ stagingEmpty, lsRemoteOk, lsRemoteStderr }) {
|
|
|
1833
2583
|
reason: "git binary failed to launch — not a credential gap, not re-seeding",
|
|
1834
2584
|
};
|
|
1835
2585
|
}
|
|
2586
|
+
if (category === "no-remote") {
|
|
2587
|
+
return {
|
|
2588
|
+
heal: false,
|
|
2589
|
+
category,
|
|
2590
|
+
reason:
|
|
2591
|
+
"no git remote configured — greenfield project, nothing to push yet (not a credential gap)",
|
|
2592
|
+
};
|
|
2593
|
+
}
|
|
1836
2594
|
return {
|
|
1837
2595
|
heal: true,
|
|
1838
2596
|
category,
|
|
@@ -1850,6 +2608,16 @@ function decideCredentialHeal({ stagingEmpty, lsRemoteOk, lsRemoteStderr }) {
|
|
|
1850
2608
|
};
|
|
1851
2609
|
}
|
|
1852
2610
|
|
|
2611
|
+
/** The single GitHub credential line format shared by every seed path: the
|
|
2612
|
+
* host staging file, the durable on-disk `store` fallback, and the live
|
|
2613
|
+
* cache-daemon approve all expect `https://x-access-token:<token>@github.com`
|
|
2614
|
+
* (git's credential-URL form, matched by both the entrypoint parser and
|
|
2615
|
+
* git-credential-store). Centralized so the `x-access-token` shape has one
|
|
2616
|
+
* source of truth instead of being re-typed at each call site. */
|
|
2617
|
+
function githubCredentialLine(token) {
|
|
2618
|
+
return `https://x-access-token:${token}@github.com`;
|
|
2619
|
+
}
|
|
2620
|
+
|
|
1853
2621
|
/** Write the laptop's GitHub token into ~/.codeyam-cloud-git-credentials on
|
|
1854
2622
|
* the VM so the cloud compose can mount it into /root/.git-credentials —
|
|
1855
2623
|
* without this, claude / codex / gemini inside the container can clone the
|
|
@@ -1875,13 +2643,29 @@ async function seedGithubCredentials(cfg, opts = {}) {
|
|
|
1875
2643
|
console.error(
|
|
1876
2644
|
`Writing GitHub token (from ${source}) into ~/.codeyam-cloud-git-credentials on the VM...`,
|
|
1877
2645
|
);
|
|
1878
|
-
const credLine =
|
|
2646
|
+
const credLine = githubCredentialLine(token);
|
|
1879
2647
|
// Quote the token through one shell layer (gcloud ssh --command); the
|
|
1880
2648
|
// value is base64'd to keep it out of the shell history.
|
|
1881
2649
|
const b64 = Buffer.from(`${credLine}\n`).toString("base64");
|
|
2650
|
+
// Verify-before-write: decode into a chmod-600 temp, assert it is non-empty
|
|
2651
|
+
// (`[ -s ]`), and only THEN copy it over the live staging file. An empty or
|
|
2652
|
+
// unreadable decode must never clobber an existing good file — the entrypoint
|
|
2653
|
+
// reads this file on every boot, and a silent empty write is exactly how a
|
|
2654
|
+
// reprovision lands a VM that "could not read Username for github.com". The
|
|
2655
|
+
// trailing `echo OK` runs only on success, so an empty write can't mask
|
|
2656
|
+
// success with a `&& echo` (the failure shape seen on the Claude creds sinks).
|
|
2657
|
+
//
|
|
2658
|
+
// Use `cat "$t" > "$f"` (in-place truncate), NOT `mv`: the staging file is
|
|
2659
|
+
// bind-mounted read-only into the running container, and `mv` mints a NEW
|
|
2660
|
+
// inode that the live `:ro` mount does not follow — so a `mv`-based re-seed
|
|
2661
|
+
// never reached the running container until a full recreate. Truncating the
|
|
2662
|
+
// existing file in place keeps its inode, so the durable write is visible to
|
|
2663
|
+
// the container immediately. Then remove the temp.
|
|
1882
2664
|
const remoteCmd =
|
|
1883
|
-
`f=$HOME/.codeyam-cloud-git-credentials && ` +
|
|
1884
|
-
`echo ${b64} | base64 -d > $
|
|
2665
|
+
`f=$HOME/.codeyam-cloud-git-credentials && t="$f.tmp.$$" && ` +
|
|
2666
|
+
`echo ${b64} | base64 -d > "$t" && chmod 600 "$t" && ` +
|
|
2667
|
+
`if [ -s "$t" ]; then cat "$t" > "$f" && chmod 600 "$f" && rm -f "$t" && echo OK; ` +
|
|
2668
|
+
`else rm -f "$t"; echo EMPTY-WRITE-REFUSED >&2; exit 1; fi`;
|
|
1885
2669
|
const result = runGcloud(
|
|
1886
2670
|
buildGcloudArgs("ssh-probe", cfg, { command: remoteCmd }),
|
|
1887
2671
|
{ stdio: ["ignore", "pipe", "inherit"] },
|
|
@@ -1891,26 +2675,76 @@ async function seedGithubCredentials(cfg, opts = {}) {
|
|
|
1891
2675
|
}
|
|
1892
2676
|
}
|
|
1893
2677
|
|
|
2678
|
+
/** Sentinel the remote read prints when it cannot find an `editor.json` under
|
|
2679
|
+
* any `~/projects/*` directory. Distinguishes "this VM has no scaffolded
|
|
2680
|
+
* project" (a real miss worth warning about) from "the project is scaffolded
|
|
2681
|
+
* but declares no port" (legitimately a `VITE_PORT` default). Without the
|
|
2682
|
+
* distinction both collapse to `{}` and the miss is silent. */
|
|
2683
|
+
const VM_APP_PORT_NO_CONFIG = "<<<NOCONFIG>>>";
|
|
2684
|
+
|
|
2685
|
+
/** Build the remote shell command that locates the project's `.codeyam` dir and
|
|
2686
|
+
* cats its two config files in one round-trip.
|
|
2687
|
+
*
|
|
2688
|
+
* Resolution is glob-based rather than `~/projects/<repoName>` alone: the host
|
|
2689
|
+
* directory is not guaranteed to be named after the repo (a renamed repo, a
|
|
2690
|
+
* manually-cloned dir, a fleet slug), and when the guess missed, the old code
|
|
2691
|
+
* silently returned `VITE_PORT` and forwarded the wrong port. The repo-named
|
|
2692
|
+
* path is still tried FIRST so a multi-project VM resolves deterministically;
|
|
2693
|
+
* the glob is the fallback. When nothing matches, print the sentinel so the
|
|
2694
|
+
* caller can warn instead of guessing. Pure string builder; no I/O. */
|
|
2695
|
+
function buildVmAppPortReadCommand(repoName) {
|
|
2696
|
+
return (
|
|
2697
|
+
`d=""; ` +
|
|
2698
|
+
`for c in ~/projects/${repoName}/.codeyam ~/projects/*/.codeyam; do ` +
|
|
2699
|
+
`if [ -f "$c/editor.json" ]; then d="$c"; break; fi; done; ` +
|
|
2700
|
+
`if [ -z "$d" ]; then echo '${VM_APP_PORT_NO_CONFIG}'; else ` +
|
|
2701
|
+
`cat "$d/editor.json" 2>/dev/null || echo '{}'; ` +
|
|
2702
|
+
`echo '<<<LOCAL>>>'; ` +
|
|
2703
|
+
`cat "$d/editor.local.json" 2>/dev/null || echo '{}'; fi`
|
|
2704
|
+
);
|
|
2705
|
+
}
|
|
2706
|
+
|
|
1894
2707
|
/** Best-effort read of the VM's resolved app dev-server port so cloud:up's
|
|
1895
2708
|
* laptop `-L` forward, readiness probe, and previewOrigin seed all target the
|
|
1896
2709
|
* port the app actually serves on instead of a hardcoded 5173. Cats the VM's
|
|
1897
2710
|
* editor.json and editor.local.json in one SSH round-trip and resolves via
|
|
1898
|
-
* `resolveConfiguredAppPort`.
|
|
1899
|
-
*
|
|
1900
|
-
*
|
|
2711
|
+
* `resolveConfiguredAppPort`.
|
|
2712
|
+
*
|
|
2713
|
+
* Falls back to `VITE_PORT` on any failure — but fails LOUD, not silent. A
|
|
2714
|
+
* wrong port here is not a cosmetic default: it forwards the wrong `-L` port,
|
|
2715
|
+
* and if the preview ends up on a dedicated origin the viewer gets
|
|
2716
|
+
* "localhost refused to connect" with nothing pointing at the cause. So an SSH
|
|
2717
|
+
* failure or a VM with no scaffolded project warns on stderr naming that
|
|
2718
|
+
* symptom. A scaffolded project that simply declares no port is NOT a miss —
|
|
2719
|
+
* `VITE_PORT` is genuinely correct there (it is what `resolved_port()` lands
|
|
2720
|
+
* on) and stays quiet. */
|
|
1901
2721
|
async function readVmAppPort(cfg) {
|
|
1902
2722
|
const repoName = resolveProject(cfg).name;
|
|
1903
|
-
const base = `~/projects/${repoName}/.codeyam`;
|
|
1904
2723
|
const read = runGcloud(
|
|
1905
2724
|
buildGcloudArgs("ssh-probe", cfg, {
|
|
1906
|
-
command:
|
|
1907
|
-
`cat ${base}/editor.json 2>/dev/null || echo '{}'; ` +
|
|
1908
|
-
`echo '<<<LOCAL>>>'; ` +
|
|
1909
|
-
`cat ${base}/editor.local.json 2>/dev/null || echo '{}'`,
|
|
2725
|
+
command: buildVmAppPortReadCommand(repoName),
|
|
1910
2726
|
}),
|
|
1911
2727
|
{ stdio: ["ignore", "pipe", "ignore"] },
|
|
1912
2728
|
);
|
|
1913
|
-
if (read.status !== 0 || typeof read.stdout !== "string")
|
|
2729
|
+
if (read.status !== 0 || typeof read.stdout !== "string") {
|
|
2730
|
+
console.error(
|
|
2731
|
+
`[cloud] WARNING: could not read the VM's app port over SSH — ` +
|
|
2732
|
+
`falling back to ${VITE_PORT}. If the app does not serve on ` +
|
|
2733
|
+
`${VITE_PORT}, the laptop port-forward targets the wrong port and the ` +
|
|
2734
|
+
`Live Preview will show "localhost refused to connect".`,
|
|
2735
|
+
);
|
|
2736
|
+
return VITE_PORT;
|
|
2737
|
+
}
|
|
2738
|
+
if (read.stdout.includes(VM_APP_PORT_NO_CONFIG)) {
|
|
2739
|
+
console.error(
|
|
2740
|
+
`[cloud] WARNING: no .codeyam/editor.json found under ~/projects on the ` +
|
|
2741
|
+
`VM (looked for '${repoName}', then any project dir) — falling back to ` +
|
|
2742
|
+
`${VITE_PORT}. If the project is scaffolded under a different name and ` +
|
|
2743
|
+
`serves on another port, the port-forward targets the wrong port and ` +
|
|
2744
|
+
`the Live Preview will show "localhost refused to connect".`,
|
|
2745
|
+
);
|
|
2746
|
+
return VITE_PORT;
|
|
2747
|
+
}
|
|
1914
2748
|
const parse = (s) => {
|
|
1915
2749
|
try {
|
|
1916
2750
|
const v = JSON.parse(s);
|
|
@@ -2224,31 +3058,236 @@ async function seedAgentApiKey(cfg) {
|
|
|
2224
3058
|
if (write.status !== 0) {
|
|
2225
3059
|
throw new Error(`Failed to write ${remotePath} on VM`);
|
|
2226
3060
|
}
|
|
3061
|
+
// NOTE: seedAgentApiKey deliberately does NOT install the build-agent CLI.
|
|
3062
|
+
// The CLI install is a POST-container-start step owned solely by
|
|
3063
|
+
// ensureEditorContainerUp (which runs `docker compose up -d` first). Seeding
|
|
3064
|
+
// the API key runs BEFORE the container is up, so a CLI install here would
|
|
3065
|
+
// enter ensureAgentCliInstalled's wait-for-container branch before any
|
|
3066
|
+
// `up -d` had happened — burning the whole wait window and firing a spurious
|
|
3067
|
+
// `CY_DEGRADED cli-deferred` on every add. That mis-ordering is the bug
|
|
3068
|
+
// vm--no-cli-install-before-container-started removed; do not re-add it.
|
|
3069
|
+
}
|
|
2227
3070
|
|
|
2228
|
-
|
|
3071
|
+
/** Pure: deep-merge `patch` over the VM's current editor.local.json content and
|
|
3072
|
+
* return the base64 of the pretty-printed result, ready for
|
|
3073
|
+
* buildEditorLocalSeedWriteCommand. `currentRaw` is the raw `cat` output of the
|
|
3074
|
+
* remote file (may be undefined, empty, or corrupt) — anything that doesn't
|
|
3075
|
+
* parse to a plain object is treated as `{}`, because overwriting a corrupt
|
|
3076
|
+
* override file with a known-good one is safer than refusing to seed. Extracted
|
|
3077
|
+
* so seedGreenfieldProjectTitle stays a thin I/O passthrough and the
|
|
3078
|
+
* parse+merge+encode is unit-tested without a live VM. */
|
|
3079
|
+
function mergeEditorLocalPatchB64(currentRaw, patch) {
|
|
3080
|
+
let current = {};
|
|
3081
|
+
try {
|
|
3082
|
+
const parsed = JSON.parse(currentRaw);
|
|
3083
|
+
if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
|
|
3084
|
+
current = parsed;
|
|
3085
|
+
}
|
|
3086
|
+
} catch {
|
|
3087
|
+
// Unparseable / missing remote content → start from {}.
|
|
3088
|
+
}
|
|
3089
|
+
const merged = deepMergeJson(current, patch);
|
|
3090
|
+
return Buffer.from(JSON.stringify(merged, null, 2), "utf8").toString("base64");
|
|
2229
3091
|
}
|
|
2230
3092
|
|
|
2231
|
-
/**
|
|
2232
|
-
|
|
2233
|
-
|
|
2234
|
-
|
|
2235
|
-
|
|
3093
|
+
/** Seed a GREENFIELD project's `projectTitle` into the VM's editor.local.json
|
|
3094
|
+
* BEFORE compose up, so the editor caches the real project name at startup and
|
|
3095
|
+
* the fleet dashboard renders the correct card label with no false
|
|
3096
|
+
* project-dir-drift. Like seedAgentApiKey, this MUST run before
|
|
3097
|
+
* ensureEditorContainerUp — the editor reads its config once at startup.
|
|
3098
|
+
*
|
|
3099
|
+
* No-op for a clone (buildGreenfieldTitlePatch returns null): a repo Add labels
|
|
3100
|
+
* from its origin basename, so there is nothing to seed and no SSH round-trip
|
|
3101
|
+
* is spent. Runs on every cloud:up, so a reconfigure re-seeds the title,
|
|
3102
|
+
* converging a VM provisioned before this fix onto the correct identity without
|
|
3103
|
+
* a destroy. Thin I/O passthrough: the parse+merge+encode lives in the pure
|
|
3104
|
+
* mergeEditorLocalPatchB64 helper. */
|
|
3105
|
+
async function seedGreenfieldProjectTitle(cfg) {
|
|
3106
|
+
const patch = buildGreenfieldTitlePatch(cfg);
|
|
3107
|
+
if (!patch) return; // clone (or no greenfield name) — nothing to seed
|
|
2236
3108
|
|
|
2237
|
-
|
|
2238
|
-
|
|
2239
|
-
|
|
3109
|
+
const repoName = resolveProject(cfg).name;
|
|
3110
|
+
const remotePath = `~/projects/${repoName}/.codeyam/editor.local.json`;
|
|
3111
|
+
console.error(`Seeding projectTitle='${patch.projectTitle}' into ${remotePath}...`);
|
|
3112
|
+
|
|
3113
|
+
const read = runGcloud(
|
|
2240
3114
|
buildGcloudArgs("ssh-probe", cfg, {
|
|
2241
|
-
command: `
|
|
3115
|
+
command: `cat ${remotePath} 2>/dev/null || echo '{}'`,
|
|
2242
3116
|
}),
|
|
2243
3117
|
{ stdio: ["ignore", "pipe", "ignore"] },
|
|
2244
3118
|
);
|
|
2245
|
-
|
|
2246
|
-
const
|
|
2247
|
-
|
|
2248
|
-
|
|
2249
|
-
|
|
2250
|
-
|
|
2251
|
-
|
|
3119
|
+
const b64 = mergeEditorLocalPatchB64(read.stdout, patch);
|
|
3120
|
+
const write = runGcloud(
|
|
3121
|
+
buildGcloudArgs("ssh-probe", cfg, {
|
|
3122
|
+
command: buildEditorLocalSeedWriteCommand({ repoName, remotePath, b64 }),
|
|
3123
|
+
}),
|
|
3124
|
+
{ stdio: ["ignore", "inherit", "inherit"] },
|
|
3125
|
+
);
|
|
3126
|
+
if (write.status !== 0) {
|
|
3127
|
+
throw new Error(`Failed to write ${remotePath} on VM`);
|
|
3128
|
+
}
|
|
3129
|
+
}
|
|
3130
|
+
|
|
3131
|
+
/** Tri-state probe of the editor container's running state on the VM host.
|
|
3132
|
+
* One ssh round-trip; returns `"running"`, `"not-running"`, or `"unknown"`.
|
|
3133
|
+
*
|
|
3134
|
+
* The distinction matters: `probeContainerState` runs `docker ps` over
|
|
3135
|
+
* gcloud/IAP-SSH, and a NON-zero exit means the probe itself couldn't run — a
|
|
3136
|
+
* transient operator-auth flap (`[operator-auth] DEGRADED: could not resolve
|
|
3137
|
+
* an account`, observed in the 2026-06-30 incident) or an SSH/transport blip —
|
|
3138
|
+
* NOT proof the container is down. That is `"unknown"` and must never count
|
|
3139
|
+
* toward a give-up/teardown decision. Only a clean `docker ps` (exit 0) is
|
|
3140
|
+
* authoritative: a non-empty line is `"running"`, an empty result is
|
|
3141
|
+
* `"not-running"`.
|
|
3142
|
+
*
|
|
3143
|
+
* The format field is `{{.Names}}` (plural). `docker ps` has NO `.Name` field
|
|
3144
|
+
* (that belongs to `docker inspect`); `'{{.Name}}'` makes `docker ps` exit
|
|
3145
|
+
* non-zero with `can't evaluate field Name in type *formatter.ContainerContext`,
|
|
3146
|
+
* which — pre-fix — made every probe report a non-running container regardless
|
|
3147
|
+
* of its real state and took the whole fleet to zero on 2026-06-30. */
|
|
3148
|
+
function probeContainerState(cfg, container) {
|
|
3149
|
+
const checkProbe = runGcloud(
|
|
3150
|
+
buildGcloudArgs("ssh-probe", cfg, {
|
|
3151
|
+
command: `docker ps --filter name=^/${container}$ --filter status=running --format '{{.Names}}'`,
|
|
3152
|
+
}),
|
|
3153
|
+
{ stdio: ["ignore", "pipe", "ignore"] },
|
|
3154
|
+
);
|
|
3155
|
+
if (checkProbe.status !== 0) {
|
|
3156
|
+
return "unknown";
|
|
3157
|
+
}
|
|
3158
|
+
return checkProbe.stdout.trim() !== "" ? "running" : "not-running";
|
|
3159
|
+
}
|
|
3160
|
+
|
|
3161
|
+
/** Boolean convenience wrapper over {@link probeContainerState}: true ONLY when
|
|
3162
|
+
* the container is definitively running. An `"unknown"` probe (auth/SSH error)
|
|
3163
|
+
* is not "running", so this returns false — callers that must distinguish a
|
|
3164
|
+
* probe failure from a down container use `probeContainerState` directly. */
|
|
3165
|
+
function probeContainerRunning(cfg, container) {
|
|
3166
|
+
return probeContainerState(cfg, container) === "running";
|
|
3167
|
+
}
|
|
3168
|
+
|
|
3169
|
+
/** Probe whether `binary` resolves on PATH inside the running container. */
|
|
3170
|
+
function probeAgentBinaryInstalled(cfg, container, binary) {
|
|
3171
|
+
const probe = runGcloud(
|
|
3172
|
+
buildGcloudArgs("ssh-probe", cfg, {
|
|
3173
|
+
command: `docker exec ${container} bash -lc 'command -v ${binary}'`,
|
|
3174
|
+
}),
|
|
3175
|
+
{ stdio: ["ignore", "pipe", "ignore"] },
|
|
3176
|
+
);
|
|
3177
|
+
return probe.status === 0 && probe.stdout.trim() !== "";
|
|
3178
|
+
}
|
|
3179
|
+
|
|
3180
|
+
/** Poll the container's running state, returning true as soon as it's running.
|
|
3181
|
+
* The accept-on-first-success shape means the common already-running case
|
|
3182
|
+
* returns instantly with zero added latency. Parameterized so a unit test can
|
|
3183
|
+
* drive it with `sleepFn` = a no-op.
|
|
3184
|
+
*
|
|
3185
|
+
* `attempts` (default 48) bounds only DEFINITIVE `"not-running"` probes — at
|
|
3186
|
+
* the capped-backoff cadence below that is a multi-minute budget (~4 min),
|
|
3187
|
+
* generous enough for a slow first-boot editor container under fleet load
|
|
3188
|
+
* (30s was far too short and guillotined healthy slow boots). An `"unknown"`
|
|
3189
|
+
* probe (auth/SSH transport failure) may clear, so it does NOT consume the
|
|
3190
|
+
* budget; a hard iteration cap (2× attempts) still prevents an infinite loop
|
|
3191
|
+
* when the probe is permanently unreachable. Backoff (delayMs → maxDelayMs)
|
|
3192
|
+
* keeps the poll cheap early and patient late. */
|
|
3193
|
+
async function waitForContainerRunning(
|
|
3194
|
+
cfg,
|
|
3195
|
+
container,
|
|
3196
|
+
{ attempts = 48, delayMs = 5000, maxDelayMs = 15000, sleepFn } = {},
|
|
3197
|
+
) {
|
|
3198
|
+
const sleep = sleepFn || ((ms) => new Promise((r) => setTimeout(r, ms)));
|
|
3199
|
+
let notRunning = 0;
|
|
3200
|
+
const maxIterations = attempts * 2;
|
|
3201
|
+
for (let i = 0; i < maxIterations && notRunning < attempts; i++) {
|
|
3202
|
+
const state = probeContainerState(cfg, container);
|
|
3203
|
+
if (state === "running") {
|
|
3204
|
+
return true;
|
|
3205
|
+
}
|
|
3206
|
+
if (state === "not-running") {
|
|
3207
|
+
notRunning += 1;
|
|
3208
|
+
console.error(` waiting for container ${container} to reach running (${notRunning}/${attempts})...`);
|
|
3209
|
+
} else {
|
|
3210
|
+
// unknown: a probe-transport failure (auth/SSH), not proof the container
|
|
3211
|
+
// is down — retry without counting toward the give-up budget.
|
|
3212
|
+
console.error(` container ${container} probe inconclusive (auth/SSH?); retrying without counting toward give-up...`);
|
|
3213
|
+
}
|
|
3214
|
+
if (notRunning >= attempts) {
|
|
3215
|
+
break;
|
|
3216
|
+
}
|
|
3217
|
+
await sleep(Math.min(maxDelayMs, delayMs * 2 ** Math.min(i, 3)));
|
|
3218
|
+
}
|
|
3219
|
+
return probeContainerRunning(cfg, container);
|
|
3220
|
+
}
|
|
3221
|
+
|
|
3222
|
+
/** Install the selected agent's CLI inside the editor container on the live VM.
|
|
3223
|
+
*
|
|
3224
|
+
* PRECONDITION: the caller must have already STARTED the container (via
|
|
3225
|
+
* ensureEditorContainerUp's `docker compose up -d`, or after confirming it is
|
|
3226
|
+
* already running). This is a post-start install helper, NOT a container
|
|
3227
|
+
* starter — its wait-for-container branch only exists to absorb a slow boot
|
|
3228
|
+
* *after* `up -d`. Legitimate callers signal this with `waitOpts.containerStarted`;
|
|
3229
|
+
* entering the wait branch WITHOUT that signal is almost certainly a call-ordering
|
|
3230
|
+
* regression (the mis-ordering that made every add burn the full wait window and
|
|
3231
|
+
* emit a spurious `CY_DEGRADED cli-deferred` — see
|
|
3232
|
+
* vm--no-cli-install-before-container-started) and is warned about loudly.
|
|
3233
|
+
*
|
|
3234
|
+
* `waitOpts` is forwarded to {@link waitForContainerRunning} (a unit test
|
|
3235
|
+
* injects `{ attempts, delayMs, sleepFn }` so the defer path resolves without
|
|
3236
|
+
* the multi-minute real-timer wait); `waitOpts.containerStarted` is the
|
|
3237
|
+
* precondition acknowledgement described above; production start-path calls
|
|
3238
|
+
* pass `{ containerStarted: true }`. */
|
|
3239
|
+
async function ensureAgentCliInstalled(cfg, waitOpts = {}) {
|
|
3240
|
+
const agent = cfg.agent;
|
|
3241
|
+
const pkg = agentNpmPackage(agent);
|
|
3242
|
+
const container = process.env.CONTAINER || "codeyam-editor-editor-1";
|
|
3243
|
+
|
|
3244
|
+
// The container may not be up the instant this runs — a slow `docker compose
|
|
3245
|
+
// up -d` or a recreate still settling. This used to log "Skipping … for now"
|
|
3246
|
+
// and RETURN, but nothing ever re-asserted it: because the image ships WITHOUT
|
|
3247
|
+
// the agent CLI (Dockerfile INSTALL_BUILD_AGENT_CLI=none), a skipped install
|
|
3248
|
+
// left the VM with no `claude` anywhere for its whole lifetime and stranded the
|
|
3249
|
+
// Build-tab auth probe (the 2026-06-30 reprovision defect). Poll for the
|
|
3250
|
+
// container, then DEGRADE LOUD (never throw) if it still isn't up.
|
|
3251
|
+
if (!probeContainerRunning(cfg, container)) {
|
|
3252
|
+
if (!waitOpts.containerStarted) {
|
|
3253
|
+
// Precondition breach: we are about to wait for a container that no
|
|
3254
|
+
// caller has started. On a healthy add this branch is unreachable
|
|
3255
|
+
// (ensureEditorContainerUp starts the container, then calls us with
|
|
3256
|
+
// `containerStarted: true`). Reaching it with no start recorded means an
|
|
3257
|
+
// install was ordered before `docker compose up -d` — the exact
|
|
3258
|
+
// mis-ordering that cost ~4 minutes and a spurious CY_DEGRADED on every
|
|
3259
|
+
// add. Warn loudly so a future regression is caught at the log instead of
|
|
3260
|
+
// silently burning the wait window.
|
|
3261
|
+
console.error(
|
|
3262
|
+
` ⚠ build-agent CLI install ordered before any editor container start ` +
|
|
3263
|
+
`(${container} is not running and no container-start was signalled). ` +
|
|
3264
|
+
`The CLI install is a POST-start step and must run AFTER \`docker compose up -d\` ` +
|
|
3265
|
+
`in ensureEditorContainerUp — this is very likely a call-ordering regression.`,
|
|
3266
|
+
);
|
|
3267
|
+
}
|
|
3268
|
+
console.error(` Container ${container} is not running yet; waiting before installing build-agent CLI...`);
|
|
3269
|
+
const up = await waitForContainerRunning(cfg, container, waitOpts);
|
|
3270
|
+
if (!up) {
|
|
3271
|
+
// DEFENSE-IN-DEPTH: a CLI install that can't confirm the container — a
|
|
3272
|
+
// genuinely slow boot, a transient operator-auth/SSH blip, or a future
|
|
3273
|
+
// probe bug like the {{.Name}}→{{.Names}} regression that took the fleet
|
|
3274
|
+
// to zero on 2026-06-30 — must DEFER, never throw. cloud:up's abort path
|
|
3275
|
+
// catches a throw here and DELETES the instance ("no orphan"); a healthy
|
|
3276
|
+
// VM with a deferred CLI install is fully recoverable (readiness flags it
|
|
3277
|
+
// provision-incomplete / cli-missing; the dashboard's one-click finish-wire
|
|
3278
|
+
// re-runs cloud:up, which re-asserts this install), but a deleted VM is not.
|
|
3279
|
+
// Loud signal (no silent skip — the VM5 CLI-less defect stays fixed), no
|
|
3280
|
+
// teardown. The `CY_DEGRADED` prefix is what the dashboard's add-job scanner
|
|
3281
|
+
// keys off to land an amber card.
|
|
3282
|
+
console.error(
|
|
3283
|
+
` CY_DEGRADED cli-deferred: container ${container} never reached status=running on ${cfg.instanceName} ` +
|
|
3284
|
+
`within the wait window — DEFERRING build-agent CLI install (VM kept). ` +
|
|
3285
|
+
`Readiness will flag cli-missing; re-run: npm run cloud:up -- --instance-name ${cfg.instanceName}`,
|
|
3286
|
+
);
|
|
3287
|
+
return;
|
|
3288
|
+
}
|
|
3289
|
+
}
|
|
3290
|
+
|
|
2252
3291
|
console.error(`Ensuring build-agent CLI for '${agent}' is installed in container '${container}'...`);
|
|
2253
3292
|
|
|
2254
3293
|
const containerScript = [
|
|
@@ -2334,86 +3373,187 @@ function promptForInstanceName(expected) {
|
|
|
2334
3373
|
|
|
2335
3374
|
const STREAMED_SCRIPTS = ["scripts/gcp-bootstrap.sh", "scripts/gcp-project-bootstrap.sh"];
|
|
2336
3375
|
|
|
3376
|
+
/** The configured editor branch to compare/stream streamed scripts against.
|
|
3377
|
+
* Resolved from config (`resolveEditorConfig` → `editorBranch`, itself derived
|
|
3378
|
+
* from `cfg.editorBranch`, or the client branch when self-hosting; the dashboard
|
|
3379
|
+
* feeds this from `FLEET_BRANCH` / the legacy `BRANCH` alias). Never a hardcoded
|
|
3380
|
+
* branch literal — the dashboard tracks different branches over time. */
|
|
3381
|
+
function streamedScriptEditorBranch(cfg) {
|
|
3382
|
+
const { editorBranch = "main" } = resolveEditorConfig(cfg);
|
|
3383
|
+
return editorBranch;
|
|
3384
|
+
}
|
|
3385
|
+
|
|
3386
|
+
/** Merge base between HEAD and the remote tip of `editorBranch`, or null when it
|
|
3387
|
+
* can't be resolved (no remote ref, detached, etc.). */
|
|
3388
|
+
function streamedScriptMergeBase(editorBranch) {
|
|
3389
|
+
const mbResult = spawnSync("git", ["merge-base", "HEAD", `origin/${editorBranch}`], { encoding: "utf8" });
|
|
3390
|
+
return mbResult.status === 0 ? mbResult.stdout.trim() : null;
|
|
3391
|
+
}
|
|
3392
|
+
|
|
3393
|
+
/** Fetch `editorBranch` from origin into a REAL remote-tracking ref
|
|
3394
|
+
* (`refs/remotes/origin/<branch>`) rather than only into `FETCH_HEAD`.
|
|
3395
|
+
*
|
|
3396
|
+
* Operator hosts are commonly shallow, single-branch clones that track only
|
|
3397
|
+
* `+refs/heads/main`, so `origin/<some-feature-branch>` is not a local ref and
|
|
3398
|
+
* every `git show origin/<branch>:<path>` against it fails. Fetching through an
|
|
3399
|
+
* explicit refspec creates the ref, which keeps the drift verdict and the bytes
|
|
3400
|
+
* actually streamed derived from the same object. Falls back to a plain fetch
|
|
3401
|
+
* (FETCH_HEAD only) when the refspec form is rejected — `readRemoteScriptBlob`
|
|
3402
|
+
* can still read through FETCH_HEAD in that case. */
|
|
3403
|
+
function fetchStreamedScriptRef(editorBranch) {
|
|
3404
|
+
const refspec = `${editorBranch}:refs/remotes/origin/${editorBranch}`;
|
|
3405
|
+
const withRef = spawnSync("git", ["fetch", "--force", "origin", refspec], { encoding: "utf8" });
|
|
3406
|
+
if (withRef.status === 0) return { ok: true, stderr: "" };
|
|
3407
|
+
|
|
3408
|
+
const plain = spawnSync("git", ["fetch", "origin", editorBranch], { encoding: "utf8" });
|
|
3409
|
+
if (plain.status === 0) return { ok: true, stderr: "" };
|
|
3410
|
+
return { ok: false, stderr: (plain.stderr || withRef.stderr || "").trim() };
|
|
3411
|
+
}
|
|
3412
|
+
|
|
3413
|
+
/** Resolve one revision of a streamed script from the fetched remote, trying the
|
|
3414
|
+
* remote-tracking ref first and `FETCH_HEAD` second. `gitArg` is the git verb
|
|
3415
|
+
* (`rev-parse` for the blob hash, `show` for the bytes) so the drift facts and the
|
|
3416
|
+
* streamed bytes resolve through the SAME ref-preference order — a verdict of
|
|
3417
|
+
* "stream remote" can therefore never disagree with what actually got streamed.
|
|
3418
|
+
* Returns `{ ok: false }` only when the blob is unreadable through both refs. */
|
|
3419
|
+
function resolveRemoteScriptRev(gitArg, relPath, editorBranch) {
|
|
3420
|
+
for (const ref of [`origin/${editorBranch}`, "FETCH_HEAD"]) {
|
|
3421
|
+
const result = spawnSync("git", [gitArg, `${ref}:${relPath}`], { encoding: "utf8" });
|
|
3422
|
+
if (result.status === 0 && typeof result.stdout === "string") {
|
|
3423
|
+
return { ok: true, stdout: result.stdout, ref };
|
|
3424
|
+
}
|
|
3425
|
+
}
|
|
3426
|
+
return { ok: false, stdout: "", ref: null };
|
|
3427
|
+
}
|
|
3428
|
+
|
|
3429
|
+
/** Gather the drift facts for one streamed script relative to `origin/<editorBranch>`:
|
|
3430
|
+
* its local (working-tree) blob hash, the remote-tip blob hash, the merge-base blob
|
|
3431
|
+
* hash, and whether the working tree is dirty. Any git failure yields a null hash,
|
|
3432
|
+
* treated downstream as "unknown" → fail-soft (never a false STALE). Shared by the
|
|
3433
|
+
* drift guard (`checkScriptDrift`) and the streaming-source decision
|
|
3434
|
+
* (`readStreamedScript`) so both judge identical facts. `relPath` is repo-relative
|
|
3435
|
+
* (the STREAMED_SCRIPTS form) — git is invoked from the repo root. */
|
|
3436
|
+
function gatherScriptDriftFacts(relPath, editorBranch, mergeBase) {
|
|
3437
|
+
const localHashResult = spawnSync("git", ["hash-object", relPath], { encoding: "utf8" });
|
|
3438
|
+
const localHash = localHashResult.status === 0 ? localHashResult.stdout.trim() : null;
|
|
3439
|
+
|
|
3440
|
+
const remoteRev = resolveRemoteScriptRev("rev-parse", relPath, editorBranch);
|
|
3441
|
+
const remoteHash = remoteRev.ok ? remoteRev.stdout.trim() : null;
|
|
3442
|
+
|
|
3443
|
+
let baseHash = null;
|
|
3444
|
+
if (mergeBase) {
|
|
3445
|
+
const baseHashResult = spawnSync("git", ["rev-parse", `${mergeBase}:${relPath}`], { encoding: "utf8" });
|
|
3446
|
+
if (baseHashResult.status === 0) {
|
|
3447
|
+
baseHash = baseHashResult.stdout.trim();
|
|
3448
|
+
}
|
|
3449
|
+
}
|
|
3450
|
+
|
|
3451
|
+
const statusResult = spawnSync("git", ["status", "--porcelain", relPath], { encoding: "utf8" });
|
|
3452
|
+
const dirty = statusResult.status === 0 && statusResult.stdout.trim() !== "";
|
|
3453
|
+
|
|
3454
|
+
return { path: relPath, localHash, remoteHash, baseHash, dirty };
|
|
3455
|
+
}
|
|
3456
|
+
|
|
3457
|
+
/** Read the canonical bytes of a streamed provisioning script for sending to a VM.
|
|
3458
|
+
* Prefers the remote tip (`git show origin/<editorBranch>:<path>`) so a checkout
|
|
3459
|
+
* merely *behind* origin streams exactly the reviewed bytes the drift guard
|
|
3460
|
+
* expects — the primary fix for the "fleet just landed a script commit → every
|
|
3461
|
+
* un-pulled dashboard can't add VMs" failure. Streams the on-disk copy instead
|
|
3462
|
+
* when the operator authored local content the remote tip lacks (dirty edit or a
|
|
3463
|
+
* committed local-ahead change — developer mode) or when the remote is unreachable
|
|
3464
|
+
* (offline fallback — `remoteHash === null`, the only sanctioned on-disk path).
|
|
3465
|
+
*
|
|
3466
|
+
* A "remote" verdict NEVER falls back to the on-disk copy. On a clean-but-behind
|
|
3467
|
+
* checkout the on-disk bytes are precisely the stale bytes the drift guard exists
|
|
3468
|
+
* to keep off the VM, so substituting them silently defeats the guard: the verdict
|
|
3469
|
+
* said "stream the reviewed remote tip" and the VM would instead receive the old
|
|
3470
|
+
* script, with only a `[Warning]` to show for it. When the remote blob cannot be
|
|
3471
|
+
* read through either ref this throws, aborting the add.
|
|
3472
|
+
* `absScriptPath` is an absolute path under the repo root. */
|
|
3473
|
+
function readStreamedScript(absScriptPath, cfg = {}) {
|
|
3474
|
+
const relPath = path.relative(rootDir, absScriptPath) || absScriptPath;
|
|
3475
|
+
const editorBranch = streamedScriptEditorBranch(cfg);
|
|
3476
|
+
const mergeBase = streamedScriptMergeBase(editorBranch);
|
|
3477
|
+
const facts = gatherScriptDriftFacts(relPath, editorBranch, mergeBase);
|
|
3478
|
+
|
|
3479
|
+
const { streamedScriptSource } = require("./script-drift");
|
|
3480
|
+
if (streamedScriptSource(facts) === "remote") {
|
|
3481
|
+
const remote = resolveRemoteScriptRev("show", relPath, editorBranch);
|
|
3482
|
+
if (remote.ok) {
|
|
3483
|
+
console.error(` streaming canonical ${relPath} from ${remote.ref} (checkout behind remote is fine)`);
|
|
3484
|
+
return remote.stdout;
|
|
3485
|
+
}
|
|
3486
|
+
// The facts said "the remote tip is the canonical source" but its bytes are
|
|
3487
|
+
// unreadable. Streaming the on-disk copy here would provision unreviewed stale
|
|
3488
|
+
// bytes under the banner of a successful add — fail loud instead.
|
|
3489
|
+
throw new Error(
|
|
3490
|
+
`could not read ${relPath} from origin/${editorBranch} (or FETCH_HEAD); ` +
|
|
3491
|
+
`refusing to stream a possibly-stale on-disk copy. ` +
|
|
3492
|
+
`Fetch the branch first: git fetch origin ${editorBranch}:refs/remotes/origin/${editorBranch}`,
|
|
3493
|
+
);
|
|
3494
|
+
}
|
|
3495
|
+
return fs.readFileSync(absScriptPath, "utf8");
|
|
3496
|
+
}
|
|
3497
|
+
|
|
2337
3498
|
/** Check if the streamed bootstrap scripts have drifted from the remote repository
|
|
2338
|
-
* and abort provisioning
|
|
3499
|
+
* and abort provisioning when the local copy carries unreviewed content, to prevent
|
|
3500
|
+
* silent container/VM failures. A checkout that is merely *behind* origin is benign:
|
|
3501
|
+
* the add streams the canonical remote-tip bytes (see `readStreamedScript`), so
|
|
3502
|
+
* "behind" no longer blocks. The guard still fires for content the remote tip does
|
|
3503
|
+
* not have — an uncommitted (dirty) local edit or a genuine divergence. */
|
|
2339
3504
|
function checkScriptDrift(cfg) {
|
|
2340
3505
|
if (process.env.CODEYAM_SKIP_SCRIPT_DRIFT_CHECK === "1") {
|
|
2341
3506
|
console.error(" [Warning] CODEYAM_SKIP_SCRIPT_DRIFT_CHECK=1 active. Skipping streamed-script drift preflight check.");
|
|
2342
3507
|
return;
|
|
2343
3508
|
}
|
|
2344
3509
|
|
|
2345
|
-
const
|
|
3510
|
+
const editorBranch = streamedScriptEditorBranch(cfg);
|
|
2346
3511
|
|
|
2347
3512
|
console.error(`Checking streamed-script drift against origin/${editorBranch}...`);
|
|
2348
3513
|
|
|
2349
|
-
// 1.
|
|
2350
|
-
|
|
2351
|
-
|
|
2352
|
-
|
|
3514
|
+
// 1. Fetch into refs/remotes/origin/<editorBranch> so the ref resolves even on a
|
|
3515
|
+
// single-branch clone that tracks only +refs/heads/main.
|
|
3516
|
+
const fetchResult = fetchStreamedScriptRef(editorBranch);
|
|
3517
|
+
if (!fetchResult.ok) {
|
|
3518
|
+
console.error(` [Warning] Failed to fetch origin ${editorBranch} (${fetchResult.stderr || "unknown error"}). Skipping drift check.`);
|
|
2353
3519
|
return;
|
|
2354
3520
|
}
|
|
2355
3521
|
|
|
2356
3522
|
// 2. Find the merge base between HEAD and origin/<editorBranch>
|
|
2357
|
-
const
|
|
2358
|
-
let mergeBase = null;
|
|
2359
|
-
if (mbResult.status === 0) {
|
|
2360
|
-
mergeBase = mbResult.stdout.trim();
|
|
2361
|
-
}
|
|
3523
|
+
const mergeBase = streamedScriptMergeBase(editorBranch);
|
|
2362
3524
|
|
|
2363
3525
|
const filesData = [];
|
|
2364
|
-
|
|
2365
3526
|
for (const scriptPath of STREAMED_SCRIPTS) {
|
|
2366
|
-
|
|
2367
|
-
|
|
2368
|
-
if (localHashResult.status !== 0) {
|
|
3527
|
+
const facts = gatherScriptDriftFacts(scriptPath, editorBranch, mergeBase);
|
|
3528
|
+
if (facts.localHash === null) {
|
|
2369
3529
|
console.error(` [Warning] Failed to compute local hash for ${scriptPath}. Skipping drift check.`);
|
|
2370
3530
|
return;
|
|
2371
3531
|
}
|
|
2372
|
-
|
|
2373
|
-
|
|
2374
|
-
// b. remoteHash (from origin/<editorBranch>)
|
|
2375
|
-
let remoteHash = null;
|
|
2376
|
-
const remoteHashResult = spawnSync("git", ["rev-parse", `origin/${editorBranch}:${scriptPath}`], { encoding: "utf8" });
|
|
2377
|
-
if (remoteHashResult.status === 0) {
|
|
2378
|
-
remoteHash = remoteHashResult.stdout.trim();
|
|
2379
|
-
}
|
|
2380
|
-
|
|
2381
|
-
// c. baseHash (from mergeBase)
|
|
2382
|
-
let baseHash = null;
|
|
2383
|
-
if (mergeBase) {
|
|
2384
|
-
const baseHashResult = spawnSync("git", ["rev-parse", `${mergeBase}:${scriptPath}`], { encoding: "utf8" });
|
|
2385
|
-
if (baseHashResult.status === 0) {
|
|
2386
|
-
baseHash = baseHashResult.stdout.trim();
|
|
2387
|
-
}
|
|
2388
|
-
}
|
|
2389
|
-
|
|
2390
|
-
// d. dirty status
|
|
2391
|
-
const statusResult = spawnSync("git", ["status", "--porcelain", scriptPath], { encoding: "utf8" });
|
|
2392
|
-
const dirty = statusResult.status === 0 && statusResult.stdout.trim() !== "";
|
|
2393
|
-
|
|
2394
|
-
filesData.push({
|
|
2395
|
-
path: scriptPath,
|
|
2396
|
-
localHash,
|
|
2397
|
-
remoteHash,
|
|
2398
|
-
baseHash,
|
|
2399
|
-
dirty
|
|
2400
|
-
});
|
|
3532
|
+
filesData.push(facts);
|
|
2401
3533
|
}
|
|
2402
3534
|
|
|
2403
|
-
const { classifyScriptDrift } = require("./script-drift");
|
|
3535
|
+
const { classifyScriptDrift, scriptDriftGuardAction } = require("./script-drift");
|
|
2404
3536
|
const verdictResult = classifyScriptDrift({ files: filesData });
|
|
3537
|
+
const dirtyFiles = filesData.filter((f) => f.dirty).map((f) => f.path);
|
|
3538
|
+
const action = scriptDriftGuardAction({
|
|
3539
|
+
verdict: verdictResult.verdict,
|
|
3540
|
+
staleFiles: verdictResult.staleFiles,
|
|
3541
|
+
dirtyFiles,
|
|
3542
|
+
});
|
|
2405
3543
|
|
|
2406
|
-
if (
|
|
3544
|
+
if (action.block) {
|
|
3545
|
+
const label = action.reason === "diverged" ? "DIVERGED" : "UNCOMMITTED LOCAL EDITS";
|
|
2407
3546
|
console.error("");
|
|
2408
3547
|
console.error("================================================================================");
|
|
2409
|
-
console.error(`✗ ERROR: Streamed scripts
|
|
2410
|
-
console.error(` Verdict: ${
|
|
2411
|
-
console.error(" The following files
|
|
2412
|
-
for (const f of
|
|
3548
|
+
console.error(`✗ ERROR: Streamed scripts carry unreviewed local content (origin/${editorBranch})!`);
|
|
3549
|
+
console.error(` Verdict: ${label}`);
|
|
3550
|
+
console.error(" The following files differ locally from the reviewed remote tip:");
|
|
3551
|
+
for (const f of action.blockedFiles) {
|
|
2413
3552
|
console.error(` - ${f}`);
|
|
2414
3553
|
}
|
|
2415
3554
|
console.error("");
|
|
2416
|
-
console.error("
|
|
3555
|
+
console.error(" Commit and push these changes, or discard them, before provisioning:");
|
|
3556
|
+
console.error(` git checkout -- <file> # discard local edits`);
|
|
2417
3557
|
console.error(` git pull --no-rebase origin ${editorBranch}`);
|
|
2418
3558
|
console.error("");
|
|
2419
3559
|
console.error(" If you must bypass this check (emergency provisioning), set:");
|
|
@@ -2421,8 +3561,10 @@ function checkScriptDrift(cfg) {
|
|
|
2421
3561
|
console.error("================================================================================");
|
|
2422
3562
|
console.error("");
|
|
2423
3563
|
process.exit(1);
|
|
2424
|
-
} else if (
|
|
2425
|
-
console.error("
|
|
3564
|
+
} else if (action.reason === "behind") {
|
|
3565
|
+
console.error(" ✓ Checkout is behind origin; streaming canonical remote-tip scripts (no pull needed).");
|
|
3566
|
+
} else if (action.reason === "local-ahead") {
|
|
3567
|
+
console.error(" [Warning] Streaming locally-committed scripts to VM (developer mode).");
|
|
2426
3568
|
} else {
|
|
2427
3569
|
console.error(" ✓ Streamed scripts are in sync with remote.");
|
|
2428
3570
|
}
|
|
@@ -2486,6 +3628,29 @@ function runVmIngress(cfg, action, vmNumber, domain, appPort, slug) {
|
|
|
2486
3628
|
});
|
|
2487
3629
|
}
|
|
2488
3630
|
|
|
3631
|
+
/** Grant-only pass: (re-)apply one VM's default IAP access grant by spawning
|
|
3632
|
+
* `scripts/gcp/vm-ingress.sh <N> grant-access` — the churn-free action that runs
|
|
3633
|
+
* `apply_default_iap_access` without touching any NEG/BES/host-rule resource.
|
|
3634
|
+
* Reconcile uses this for already-`wired` VMs its re-wire loop skips, so a healed
|
|
3635
|
+
* VM whose backends already exist still (re-)grants the automation SA the IAP
|
|
3636
|
+
* accessor. Returns the spawnSync result so the caller logs granted vs failed;
|
|
3637
|
+
* best-effort + loud, never fatal. */
|
|
3638
|
+
function runVmIngressGrantAccess(cfg, vmNumber, domain, slug) {
|
|
3639
|
+
const scriptPath = path.join(rootDir, "scripts", "gcp", "vm-ingress.sh");
|
|
3640
|
+
const inv = vmIngressGrantAccessInvocation({
|
|
3641
|
+
scriptPath,
|
|
3642
|
+
vmNumber,
|
|
3643
|
+
domain,
|
|
3644
|
+
project: cfg.project,
|
|
3645
|
+
zone: cfg.zone,
|
|
3646
|
+
slug,
|
|
3647
|
+
});
|
|
3648
|
+
return spawnSync(inv.command, inv.args, {
|
|
3649
|
+
stdio: "inherit",
|
|
3650
|
+
env: { ...process.env, ...inv.env },
|
|
3651
|
+
});
|
|
3652
|
+
}
|
|
3653
|
+
|
|
2489
3654
|
/** Add-time hook: wire VM-N onto the shared LB when a fleet domain is set.
|
|
2490
3655
|
* Best-effort + loud — a wiring hiccup must never abort an otherwise-green Add
|
|
2491
3656
|
* (the VM is reachable operator-side over SSH regardless), so on failure it
|
|
@@ -2599,8 +3764,16 @@ function applyVmAccessOnAdd(cfg, env = process.env) {
|
|
|
2599
3764
|
}
|
|
2600
3765
|
const vmNumber = plan.wire ? plan.vmNumber : resolveFleetVmNumber(cfg.instanceName);
|
|
2601
3766
|
if (vmNumber == null) return; // not a fleet VM — nothing to reconcile
|
|
3767
|
+
// The list we GRANT always unions in the configured default operator, so an
|
|
3768
|
+
// empty `vmAccess` still yields access for the operator — an IAP-enabled VM is
|
|
3769
|
+
// never inaccessible-by-default (the Error-51 footgun). The default operator is
|
|
3770
|
+
// NOT persisted to the sidecar (it's a standing grant, not a per-VM
|
|
3771
|
+
// collaborator), so `desired` alone drives the sidecar + stale-revoke below.
|
|
3772
|
+
const toGrant = withDefaultOperator(desired, resolveDefaultOperator({ env }));
|
|
2602
3773
|
const stale = accessFor(vmNumber);
|
|
2603
|
-
|
|
3774
|
+
// Never revoke a principal we're about to (re-)grant — a prior tenant who was
|
|
3775
|
+
// also the default operator must not be revoked-then-regranted for nothing.
|
|
3776
|
+
const staleOnly = stale.filter((p) => !toGrant.includes(p));
|
|
2604
3777
|
|
|
2605
3778
|
if (plan.wire) {
|
|
2606
3779
|
if (staleOnly.length > 0) {
|
|
@@ -2614,31 +3787,33 @@ function applyVmAccessOnAdd(cfg, env = process.env) {
|
|
|
2614
3787
|
);
|
|
2615
3788
|
}
|
|
2616
3789
|
}
|
|
2617
|
-
if (
|
|
2618
|
-
console.error(` granting IAP access to VM-${vmNumber} for ${
|
|
2619
|
-
const res = runIapAccess(cfg, vmNumber, "grant",
|
|
3790
|
+
if (toGrant.length > 0) {
|
|
3791
|
+
console.error(` granting IAP access to VM-${vmNumber} for ${toGrant.length} principal(s)...`);
|
|
3792
|
+
const res = runIapAccess(cfg, vmNumber, "grant", toGrant);
|
|
2620
3793
|
if (res.status === 0) {
|
|
2621
|
-
console.error(` ✓ IAP access granted: ${
|
|
3794
|
+
console.error(` ✓ IAP access granted: ${toGrant.join(", ")}`);
|
|
2622
3795
|
} else {
|
|
2623
3796
|
console.error(
|
|
2624
3797
|
` ⚠ IAP access grant for VM-${vmNumber} did not complete cleanly. ` +
|
|
2625
3798
|
`The VM is provisioned; re-grant with:\n` +
|
|
2626
|
-
` VM_ACCESS=${
|
|
3799
|
+
` VM_ACCESS=${toGrant.join(",")} bash scripts/gcp/iap-access.sh ${vmNumber} grant`,
|
|
2627
3800
|
);
|
|
2628
3801
|
}
|
|
2629
3802
|
}
|
|
2630
|
-
} else if (
|
|
3803
|
+
} else if (toGrant.length > 0) {
|
|
2631
3804
|
console.error(
|
|
2632
|
-
` (no fleet domain / not a fleet VM — IAP access for ${
|
|
3805
|
+
` (no fleet domain / not a fleet VM — IAP access for ${toGrant.length} principal(s) not applied; ` +
|
|
2633
3806
|
`set a fleet domain and re-run, or use cloud:access)`,
|
|
2634
3807
|
);
|
|
2635
3808
|
}
|
|
2636
3809
|
|
|
2637
|
-
// Sidecar reconcile (always, for a fleet VM): record
|
|
2638
|
-
//
|
|
2639
|
-
// backend services).
|
|
2640
|
-
//
|
|
2641
|
-
//
|
|
3810
|
+
// Sidecar reconcile (always, for a fleet VM): record the explicit collaborator
|
|
3811
|
+
// list (`desired`) when wired, or empty when not (nothing was applied without
|
|
3812
|
+
// backend services). The default operator is deliberately excluded — it is a
|
|
3813
|
+
// standing grant re-applied on every wire, not a per-VM collaborator, so the
|
|
3814
|
+
// sidecar display + a reused number's stale-revoke stay about collaborators
|
|
3815
|
+
// only. setAccess deletes the entry on empty, so a reused number never carries a
|
|
3816
|
+
// prior tenant's stale entry forward into the dashboard display or a re-add.
|
|
2642
3817
|
const granted = plan.wire ? desired : [];
|
|
2643
3818
|
try {
|
|
2644
3819
|
setAccess(vmNumber, granted);
|
|
@@ -2652,6 +3827,141 @@ function applyVmAccessOnAdd(cfg, env = process.env) {
|
|
|
2652
3827
|
// dashboard reads the same path via the shared `ingressInflightMarkerPath` builder.
|
|
2653
3828
|
const INGRESS_INFLIGHT_DIR = os.tmpdir();
|
|
2654
3829
|
|
|
3830
|
+
// Add-time ingress CONVERGE loop bounds (vm--add-time-url-wiring-converges-
|
|
3831
|
+
// reliably). The detached add-tail re-runs `vm-ingress.sh up`+`status` on its own
|
|
3832
|
+
// TIGHT cadence (seconds, not the ~15-min INGRESS_RECONCILE_MS sweep) until the
|
|
3833
|
+
// verdict is `wired` or the deadline elapses — so a fresh VM's public URL is live
|
|
3834
|
+
// in ~a minute rather than whenever the next roster-wide sweep lands. Overridable
|
|
3835
|
+
// via env for tests / operator tuning; the sweep stays the long-run safety net.
|
|
3836
|
+
const INGRESS_CONVERGE_DEADLINE_MS = 8 * 60 * 1000;
|
|
3837
|
+
const INGRESS_CONVERGE_BASE_MS = 5000;
|
|
3838
|
+
const INGRESS_CONVERGE_MAX_MS = 30000;
|
|
3839
|
+
|
|
3840
|
+
/** Read a positive-integer env override, falling back to `fallback` on absent /
|
|
3841
|
+
* non-numeric / non-positive input. Keeps the converge bounds tunable without
|
|
3842
|
+
* scattering `Number(...)||default` guards through the loop. */
|
|
3843
|
+
function envPosInt(env, key, fallback) {
|
|
3844
|
+
const v = Number(env && env[key]);
|
|
3845
|
+
return Number.isInteger(v) && v > 0 ? v : fallback;
|
|
3846
|
+
}
|
|
3847
|
+
|
|
3848
|
+
/** Read one VM's LB-ingress `verdict` (`wired`|`partial`|`unwired`|null) by
|
|
3849
|
+
* spawning `vm-ingress.sh <N> status` and parsing its output — the read-only
|
|
3850
|
+
* probe the converge loop polls between `up` attempts. Best-effort: a script
|
|
3851
|
+
* error / missing verdict line returns null (parsed as "keep converging"),
|
|
3852
|
+
* never throws. */
|
|
3853
|
+
function probeVmIngressVerdict(cfg, plan) {
|
|
3854
|
+
const scriptPath = path.join(rootDir, "scripts", "gcp", "vm-ingress.sh");
|
|
3855
|
+
const inv = vmIngressStatusInvocation({
|
|
3856
|
+
scriptPath,
|
|
3857
|
+
vmNumber: plan.vmNumber,
|
|
3858
|
+
domain: plan.domain,
|
|
3859
|
+
project: cfg.project,
|
|
3860
|
+
zone: cfg.zone,
|
|
3861
|
+
slug: plan.slug,
|
|
3862
|
+
});
|
|
3863
|
+
const res = spawnSync(inv.command, inv.args, {
|
|
3864
|
+
env: { ...process.env, ...inv.env },
|
|
3865
|
+
encoding: "utf8",
|
|
3866
|
+
});
|
|
3867
|
+
return parseIngressStatusVerdict(res.stdout || "");
|
|
3868
|
+
}
|
|
3869
|
+
|
|
3870
|
+
/** Drive a just-added VM's LB-ingress wiring to `verdict=wired` — the reliable
|
|
3871
|
+
* replacement for the old fire-once-and-trust-the-sweep add path
|
|
3872
|
+
* (vm--add-time-url-wiring-converges-reliably). Re-runs `vm-ingress.sh up`
|
|
3873
|
+
* (idempotent: creates only what's absent, healing a `partial`/orphaned state)
|
|
3874
|
+
* then re-probes `status`, looping with exponential backoff (`ingressConverge-
|
|
3875
|
+
* Decision`) until `wired` or the bounded deadline. On success it unlinks any
|
|
3876
|
+
* stale `wiring-failed` marker and returns `{ wired: true }`; on deadline it
|
|
3877
|
+
* drops the failed marker + a `CY_DEGRADED ingress-wiring-failed` line (a REAL
|
|
3878
|
+
* attention reason with the re-run command) rather than leaving a silent
|
|
3879
|
+
* `partial`. A no-op wire (legacy quick-tunnel / non-fleet) delegates to the
|
|
3880
|
+
* inline `wireVmIngressOnAdd`. All slow I/O (`runUp`, `probeVerdict`, `sleep`,
|
|
3881
|
+
* `now`, marker dir, and the loop bounds) is injectable via `deps` so the loop
|
|
3882
|
+
* is unit-testable without spawning the script or actually sleeping. */
|
|
3883
|
+
async function convergeVmIngressOnAdd(cfg, appPort, env = process.env, deps = {}) {
|
|
3884
|
+
let plan;
|
|
3885
|
+
try {
|
|
3886
|
+
plan = ingressWiringPlan(cfg, env);
|
|
3887
|
+
} catch (e) {
|
|
3888
|
+
console.error(` ⚠ skipped LB ingress converge (${e && e.message}).`);
|
|
3889
|
+
return { wired: false, reason: "plan-error" };
|
|
3890
|
+
}
|
|
3891
|
+
if (!plan.wire) {
|
|
3892
|
+
// Nothing to converge — the no-op wire keeps the clean-slate invariant.
|
|
3893
|
+
wireVmIngressOnAdd(cfg, appPort, env);
|
|
3894
|
+
return { wired: true, reason: "no-wire" };
|
|
3895
|
+
}
|
|
3896
|
+
|
|
3897
|
+
const runUp =
|
|
3898
|
+
deps.runUp || ((p) => runVmIngress(cfg, "up", p.vmNumber, p.domain, appPort, p.slug));
|
|
3899
|
+
const probeVerdict = deps.probeVerdict || ((p) => probeVmIngressVerdict(cfg, p));
|
|
3900
|
+
const sleep = deps.sleep || ((ms) => new Promise((r) => setTimeout(r, ms)));
|
|
3901
|
+
const now = deps.now || (() => Date.now());
|
|
3902
|
+
const dir = deps.markerDir || INGRESS_INFLIGHT_DIR;
|
|
3903
|
+
const deadlineMs =
|
|
3904
|
+
deps.deadlineMs != null
|
|
3905
|
+
? deps.deadlineMs
|
|
3906
|
+
: envPosInt(env, "INGRESS_CONVERGE_DEADLINE_MS", INGRESS_CONVERGE_DEADLINE_MS);
|
|
3907
|
+
const baseDelayMs =
|
|
3908
|
+
deps.baseDelayMs != null
|
|
3909
|
+
? deps.baseDelayMs
|
|
3910
|
+
: envPosInt(env, "INGRESS_CONVERGE_BASE_MS", INGRESS_CONVERGE_BASE_MS);
|
|
3911
|
+
const maxDelayMs =
|
|
3912
|
+
deps.maxDelayMs != null
|
|
3913
|
+
? deps.maxDelayMs
|
|
3914
|
+
: envPosInt(env, "INGRESS_CONVERGE_MAX_MS", INGRESS_CONVERGE_MAX_MS);
|
|
3915
|
+
|
|
3916
|
+
const label = plan.slug || `vm-${plan.vmNumber}`;
|
|
3917
|
+
const failedMarker = ingressFailedMarkerPath(dir, plan.vmNumber);
|
|
3918
|
+
// Clear any failed marker a prior add/self-heal left behind before we re-attempt,
|
|
3919
|
+
// so a converge that now succeeds doesn't read as still-failed.
|
|
3920
|
+
try { fs.unlinkSync(failedMarker); } catch { /* not present — fine */ }
|
|
3921
|
+
|
|
3922
|
+
console.error(
|
|
3923
|
+
` wiring LB ingress for VM-${plan.vmNumber} on ${plan.domain} (converge until wired)...`,
|
|
3924
|
+
);
|
|
3925
|
+
const start = now();
|
|
3926
|
+
let attempt = 0;
|
|
3927
|
+
for (;;) {
|
|
3928
|
+
runUp(plan);
|
|
3929
|
+
const verdict = probeVerdict(plan);
|
|
3930
|
+
const decision = ingressConvergeDecision({
|
|
3931
|
+
verdict,
|
|
3932
|
+
elapsedMs: now() - start,
|
|
3933
|
+
attempt,
|
|
3934
|
+
deadlineMs,
|
|
3935
|
+
baseDelayMs,
|
|
3936
|
+
maxDelayMs,
|
|
3937
|
+
});
|
|
3938
|
+
if (decision.done) {
|
|
3939
|
+
if (decision.wired) {
|
|
3940
|
+
console.error(` ✓ LB ingress wired: https://${label}.editor.${plan.domain}`);
|
|
3941
|
+
return { wired: true, reason: "wired", attempts: attempt + 1 };
|
|
3942
|
+
}
|
|
3943
|
+
const rewire =
|
|
3944
|
+
`FLEET_DOMAIN=${plan.domain}${plan.slug ? ` VM_SLUG=${plan.slug}` : ""} ` +
|
|
3945
|
+
`bash scripts/gcp/vm-ingress.sh ${plan.vmNumber} up`;
|
|
3946
|
+
try { fs.writeFileSync(failedMarker, String(now())); } catch { /* best-effort */ }
|
|
3947
|
+
console.error(
|
|
3948
|
+
` ⚠ LB ingress for VM-${plan.vmNumber} did not converge to wired within the ` +
|
|
3949
|
+
`add window (last verdict=${verdict || "unknown"}). Re-wire:\n ${rewire}`,
|
|
3950
|
+
);
|
|
3951
|
+
// Machine-readable marker so the card lands a genuine `wiring-failed`
|
|
3952
|
+
// attention state with its remediation, not a silent partial the sweep may
|
|
3953
|
+
// never reach. Keyed the same way as the other CY_DEGRADED add markers.
|
|
3954
|
+
console.log(
|
|
3955
|
+
`CY_DEGRADED ingress-wiring-failed: VM-${plan.vmNumber} public URL did not wire ` +
|
|
3956
|
+
`within the add window (last verdict=${verdict || "unknown"}). Re-wire: ${rewire}`,
|
|
3957
|
+
);
|
|
3958
|
+
return { wired: false, reason: "deadline", attempts: attempt + 1 };
|
|
3959
|
+
}
|
|
3960
|
+
attempt += 1;
|
|
3961
|
+
await sleep(decision.delayMs);
|
|
3962
|
+
}
|
|
3963
|
+
}
|
|
3964
|
+
|
|
2655
3965
|
/** Start the slow LB-ingress wiring (+ IAP access grant) for a just-added VM as a
|
|
2656
3966
|
* DETACHED background child and return immediately, so the Add completes at
|
|
2657
3967
|
* container-healthy instead of blocking ~10 min on the wiring tail. The detached
|
|
@@ -2714,12 +4024,13 @@ function startDetachedAddWiring(cfg, appPort, env = process.env) {
|
|
|
2714
4024
|
|
|
2715
4025
|
/** `wire-add-tail` (internal): the DETACHED background continuation
|
|
2716
4026
|
* `startDetachedAddWiring` spawns. Reads the Add's `cfg`/`appPort` back out of the
|
|
2717
|
-
* env (so it wires EXACTLY what the foreground Add resolved),
|
|
2718
|
-
*
|
|
2719
|
-
*
|
|
2720
|
-
*
|
|
2721
|
-
*
|
|
2722
|
-
|
|
4027
|
+
* env (so it wires EXACTLY what the foreground Add resolved), CONVERGES the wiring
|
|
4028
|
+
* to `verdict=wired` (retry-until-wired via convergeVmIngressOnAdd, not a lossy
|
|
4029
|
+
* single-shot) then runs the IAP access grant in the SAME order cmdUp ran them
|
|
4030
|
+
* inline, and ALWAYS unlinks the in-flight marker on the way out — even on a
|
|
4031
|
+
* wiring failure — so the dashboard's auto-rewire backstop takes over instead of
|
|
4032
|
+
* being pinned off. Not a user-facing subcommand; invoked only via re-exec. */
|
|
4033
|
+
async function cmdWireAddTail() {
|
|
2723
4034
|
let cfg = {};
|
|
2724
4035
|
try {
|
|
2725
4036
|
cfg = JSON.parse(process.env.CY_WIRE_ADD_CFG || "{}");
|
|
@@ -2736,7 +4047,10 @@ function cmdWireAddTail() {
|
|
|
2736
4047
|
}
|
|
2737
4048
|
} catch { /* fall through — marker cleanup is best-effort */ }
|
|
2738
4049
|
try {
|
|
2739
|
-
|
|
4050
|
+
// Converge the URL to `verdict=wired` (retry-until-wired), not a single-shot
|
|
4051
|
+
// `up` left to the slow reconcile sweep — then the IAP access grant, in the
|
|
4052
|
+
// SAME order cmdUp ran them inline.
|
|
4053
|
+
await convergeVmIngressOnAdd(cfg, appPort);
|
|
2740
4054
|
applyVmAccessOnAdd(cfg);
|
|
2741
4055
|
} finally {
|
|
2742
4056
|
if (marker) {
|
|
@@ -2880,7 +4194,20 @@ async function cmdUp(args, projectDir = rootDir) {
|
|
|
2880
4194
|
}
|
|
2881
4195
|
}
|
|
2882
4196
|
await seedAgentApiKey(cfg);
|
|
4197
|
+
// Greenfield only (no-op for a clone): seed projectTitle so the editor
|
|
4198
|
+
// resolves its identity to the real project name instead of the bare
|
|
4199
|
+
// /workspace mount basename — the source of the false project-dir-drift.
|
|
4200
|
+
await seedGreenfieldProjectTitle(cfg);
|
|
2883
4201
|
await ensureEditorContainerUp(cfg);
|
|
4202
|
+
// Provision contract: a just-provisioned VM must actually reach GitHub.
|
|
4203
|
+
// Verify in-container git auth now that the container (and its credential
|
|
4204
|
+
// cache) is up; re-seed-then-re-assert on a stale cache, and land a loud
|
|
4205
|
+
// CY_DEGRADED on a persistent gap rather than a silently-green VM that
|
|
4206
|
+
// cannot push (VM5's 2026-06-30 could-not-read-Username symptom). Skipped
|
|
4207
|
+
// for an intentionally read-only VM (--allow-no-git-credentials).
|
|
4208
|
+
if (!cfg.allowNoGitCredentials) {
|
|
4209
|
+
await assertGitPushCredentials(cfg);
|
|
4210
|
+
}
|
|
2884
4211
|
// Roster the VM into the fleet dashboard NOW that it's provisioned and the
|
|
2885
4212
|
// editor container is up. Runs BEFORE the --no-tunnel early-return so the
|
|
2886
4213
|
// dashboard's own `cloud:up --no-tunnel` Add path also rosters (idempotent
|
|
@@ -2911,6 +4238,10 @@ async function cmdUp(args, projectDir = rootDir) {
|
|
|
2911
4238
|
// us — a run reported success with no container). A failure here is a loud
|
|
2912
4239
|
// non-zero exit, which the dashboard fail-fast-on-nonzero path acts on.
|
|
2913
4240
|
await assertEditorContainerRunning(cfg);
|
|
4241
|
+
// Terminal gate #2: the editor container is up, but the agent CLI is what the
|
|
4242
|
+
// Build-tab auth probe actually needs. Assert it's present (install + re-check
|
|
4243
|
+
// if not), failing loud rather than rostering a CLI-less VM.
|
|
4244
|
+
await assertAgentCliInstalled(cfg);
|
|
2914
4245
|
// Wire this VM onto the shared Google HTTPS LB when a fleet domain is
|
|
2915
4246
|
// configured — the stable-URL replacement for the quick-tunnel mint. Runs
|
|
2916
4247
|
// AFTER the terminal container gate so the editor/preview ports the NEG
|
|
@@ -2938,7 +4269,27 @@ async function cmdUp(args, projectDir = rootDir) {
|
|
|
2938
4269
|
);
|
|
2939
4270
|
}
|
|
2940
4271
|
} catch (err) {
|
|
2941
|
-
|
|
4272
|
+
// `createdThisRun` alone MUST NOT authorize deletion. Gather what state the
|
|
4273
|
+
// VM is in right now, then let the pure decision rule — see
|
|
4274
|
+
// abortTeardownDecision's doc comment for why each input is load-bearing.
|
|
4275
|
+
const state = probeAbortTeardownState(cfg, createdThisRun);
|
|
4276
|
+
const verdict = abortTeardownDecision({ createdThisRun, ...state });
|
|
4277
|
+
if (createdThisRun && verdict === "keep") {
|
|
4278
|
+
// Deliberately loud and specific. The old blanket "tearing down the VM
|
|
4279
|
+
// cloud:up created this run (no orphan left behind)" was actively
|
|
4280
|
+
// misleading here — it asserted no orphan while deleting a working VM.
|
|
4281
|
+
const why = state.probeInconclusive
|
|
4282
|
+
? "its liveness could not be confirmed (probe failed), and an unconfirmed VM is never deleted"
|
|
4283
|
+
: state.containerUp
|
|
4284
|
+
? "its editor container is RUNNING — another run brought it up"
|
|
4285
|
+
: "it is in the fleet roster and visible to the dashboard";
|
|
4286
|
+
console.error(
|
|
4287
|
+
`✗ provisioning aborted — the VM ${cfg.instanceName} was LEFT IN PLACE because ${why}.\n` +
|
|
4288
|
+
` This run created the instance, but that alone does not authorize deleting a VM that is in service.\n` +
|
|
4289
|
+
` If it really should go, remove it deliberately: npm run cloud:destroy`,
|
|
4290
|
+
);
|
|
4291
|
+
}
|
|
4292
|
+
if (verdict === "delete") {
|
|
2942
4293
|
console.error(
|
|
2943
4294
|
`✗ provisioning aborted — tearing down the VM cloud:up created this run (no orphan left behind)`,
|
|
2944
4295
|
);
|
|
@@ -3120,14 +4471,7 @@ async function cmdStatus(args, projectDir = rootDir) {
|
|
|
3120
4471
|
const imgStatus = String(imgProbe.stdout || "").trim() || "?";
|
|
3121
4472
|
console.error(` image: ${imgStatus}`);
|
|
3122
4473
|
|
|
3123
|
-
const
|
|
3124
|
-
const containerProbe = runGcloud(
|
|
3125
|
-
buildGcloudArgs("ssh-probe", cfg, {
|
|
3126
|
-
command: `cd ~/projects/${repoName} 2>/dev/null && docker compose -f docker-compose.cloud.yml ps --format json 2>/dev/null || echo`,
|
|
3127
|
-
}),
|
|
3128
|
-
{ stdio: ["ignore", "pipe", "ignore"] },
|
|
3129
|
-
);
|
|
3130
|
-
const containerStatus = parseEditorComposeState(containerProbe.stdout) || "missing";
|
|
4474
|
+
const containerStatus = probeEditorComposeState(cfg).state || "missing";
|
|
3131
4475
|
console.error(` editor container: ${containerStatus}`);
|
|
3132
4476
|
|
|
3133
4477
|
const appPort = await readVmAppPort(cfg);
|
|
@@ -3221,17 +4565,40 @@ async function cmdReconcileIngress(args, projectDir = rootDir) {
|
|
|
3221
4565
|
zone: cfg.zone,
|
|
3222
4566
|
slug,
|
|
3223
4567
|
});
|
|
4568
|
+
// Bound the per-VM status probe: a hung `bash vm-ingress.sh <N> status`
|
|
4569
|
+
// (its gcloud describes stalling on an IAP/metadata hiccup) must not stall the
|
|
4570
|
+
// whole sweep. On timeout res.stdout is empty, so parseIngressStatusVerdict
|
|
4571
|
+
// returns null → an `unknown` verdict → selectIngressReconcileTargets SKIPS
|
|
4572
|
+
// this VM and the sweep continues to the next one.
|
|
3224
4573
|
const res = spawnSync(inv.command, inv.args, {
|
|
3225
4574
|
env: { ...process.env, ...inv.env },
|
|
3226
4575
|
encoding: "utf8",
|
|
4576
|
+
timeout: GCLOUD_CALL_TIMEOUT_MS,
|
|
3227
4577
|
});
|
|
3228
4578
|
return { vmNumber: n, slug, verdict: parseIngressStatusVerdict(res.stdout || "") };
|
|
3229
4579
|
});
|
|
3230
4580
|
const targets = selectIngressReconcileTargets(statuses);
|
|
3231
4581
|
const targetNums = new Set(targets.map((t) => t.vmNumber));
|
|
4582
|
+
// The complement of the re-wire loop: `wired` VMs whose backends already exist
|
|
4583
|
+
// so the re-wire loop skips them, yet still need the SA-accessor (re-)granted —
|
|
4584
|
+
// `up` is the only path that grants it, and it never runs for a wired VM. This
|
|
4585
|
+
// closes the VM-8 heal-path gap (a wired VM whose backends predate the SA-grant
|
|
4586
|
+
// fix would otherwise stay 403 forever). Disjoint from `targets` by construction.
|
|
4587
|
+
const grantOnlyNums = new Set(
|
|
4588
|
+
selectIngressGrantOnlyTargets(statuses).map((g) => g.vmNumber),
|
|
4589
|
+
);
|
|
3232
4590
|
for (const s of statuses) {
|
|
3233
|
-
if (
|
|
3234
|
-
|
|
4591
|
+
if (targetNums.has(s.vmNumber)) continue;
|
|
4592
|
+
const label = s.slug || `vm-${s.vmNumber}`;
|
|
4593
|
+
if (grantOnlyNums.has(s.vmNumber)) {
|
|
4594
|
+
// Churn-free grant-only pass: (re-)grant the accessor without re-wiring any
|
|
4595
|
+
// NEG/BES/host-rule resource. Idempotent (a no-op when the SA is already
|
|
4596
|
+
// bound) and best-effort-but-loud — a grant hiccup logs, never aborts.
|
|
4597
|
+
const gres = runVmIngressGrantAccess(cfg, s.vmNumber, domain, s.slug);
|
|
4598
|
+
console.error(
|
|
4599
|
+
` VM-${s.vmNumber} ${label}: wired → ok (accessor ${gres.status === 0 ? "granted" : "grant FAILED"})`,
|
|
4600
|
+
);
|
|
4601
|
+
} else {
|
|
3235
4602
|
console.error(` VM-${s.vmNumber} ${label}: ${s.verdict || "unknown"} → ok`);
|
|
3236
4603
|
}
|
|
3237
4604
|
}
|
|
@@ -3247,6 +4614,296 @@ async function cmdReconcileIngress(args, projectDir = rootDir) {
|
|
|
3247
4614
|
` VM-${t.vmNumber} ${label}: ${t.verdict} → ${res.status === 0 ? "re-wired" : "re-wire FAILED"}`,
|
|
3248
4615
|
);
|
|
3249
4616
|
}
|
|
4617
|
+
|
|
4618
|
+
// Converge each rostered backend-service's timeout to 86400 (24h). A fully-wired
|
|
4619
|
+
// BES still at GCP's 30s default reports `wired` above and is skipped by the
|
|
4620
|
+
// re-wire loop, yet on an External HTTPS LB that 30s is the MAX WebSocket lifetime
|
|
4621
|
+
// — the editor's /ws/terminal socket is guillotined every 30s (the Build-tab
|
|
4622
|
+
// "Reconnecting…" loop). Assert the timeout per rostered BES so the running fleet
|
|
4623
|
+
// self-heals without a manual sweep — idempotent (a no-op when already 86400). The
|
|
4624
|
+
// durable per-`up` converge lives in vm-ingress.sh; this catches already-wired VMs
|
|
4625
|
+
// the periodic reconcile never re-`up`s. See plan vm--lb-backend-service-websocket-timeout.
|
|
4626
|
+
const gi = require("./google-ingress.js");
|
|
4627
|
+
for (const n of roster) {
|
|
4628
|
+
for (const surface of gi.SURFACES) {
|
|
4629
|
+
const bes = gi.backendServiceName(n, surface);
|
|
4630
|
+
const inv = backendServiceTimeoutInvocation({ bes, project: cfg.project });
|
|
4631
|
+
// Bound the per-BES converge too: a hung `gcloud backend-services update`
|
|
4632
|
+
// returns a non-zero (timed-out) result, so it falls into the skip branch
|
|
4633
|
+
// below and the sweep moves on to the next BES rather than stalling.
|
|
4634
|
+
const res = spawnSync(inv.command, inv.args, {
|
|
4635
|
+
encoding: "utf8",
|
|
4636
|
+
timeout: GCLOUD_CALL_TIMEOUT_MS,
|
|
4637
|
+
});
|
|
4638
|
+
if (res.status === 0) {
|
|
4639
|
+
console.error(` ${bes}: timeout → 86400`);
|
|
4640
|
+
} else {
|
|
4641
|
+
const tail = (res.stderr || "").trim().split("\n").pop() || "error";
|
|
4642
|
+
console.error(` ${bes}: timeout converge skipped (${tail})`);
|
|
4643
|
+
}
|
|
4644
|
+
}
|
|
4645
|
+
}
|
|
4646
|
+
|
|
4647
|
+
// Converge the shared fleet firewall, then surface any app port it still fails to
|
|
4648
|
+
// admit. Both ride this same boot + periodic cadence so the running fleet
|
|
4649
|
+
// self-heals without a hand-run bootstrap.
|
|
4650
|
+
const firewallState = convergeFleetFirewall({ cfg });
|
|
4651
|
+
await reportFirewallPortGaps({ cfg, roster, firewallState });
|
|
4652
|
+
|
|
4653
|
+
// Auto-clean orphaned LB ingress (vm--auto-clean-orphaned-lb-ingress): any
|
|
4654
|
+
// backend service whose VM number is gone from the roster is a destroyed VM's
|
|
4655
|
+
// leftover — cleaned here automatically with the verified `down`, so the
|
|
4656
|
+
// operator never sees a manual `vm-ingress.sh <N> down` command. Rides the same
|
|
4657
|
+
// boot + periodic cadence as the wiring reconcile above.
|
|
4658
|
+
sweepOrphanIngress({ cfg, domain, roster, projectDir });
|
|
4659
|
+
}
|
|
4660
|
+
|
|
4661
|
+
/** Auto-clean orphaned LB ingress (vm--auto-clean-orphaned-lb-ingress): LB
|
|
4662
|
+
* resources whose owning VM number is gone from the roster (a destroyed VM) and
|
|
4663
|
+
* has no in-flight add. Enumerates the live backend services AND zonal NEGs,
|
|
4664
|
+
* UNIONs their owning VM numbers (so a partial teardown whose NEG outlived its
|
|
4665
|
+
* backend service — the VM-5/6 case — still surfaces its VM as a candidate),
|
|
4666
|
+
* maps each name back to its VM number, subtracts the roster + in-flight adds +
|
|
4667
|
+
* live instances (the false-orphan guards), and runs the VERIFIED
|
|
4668
|
+
* `vm-ingress.sh <N> down` (asserts neg=no bes=no host=no, with retries) for
|
|
4669
|
+
* each orphan — so an orphan is genuinely cleared or surfaced as a real failure,
|
|
4670
|
+
* never a silent leak and never a manual command. Emits a machine-readable
|
|
4671
|
+
* `CY_ORPHAN_SWEEP <json>` summary line the dashboard parses to reconcile its
|
|
4672
|
+
* orphan-attention banner; the summary carries `stillPresent` (VM numbers that
|
|
4673
|
+
* still own ≥1 LB resource after the sweep) so the dashboard can clear a stale
|
|
4674
|
+
* banner for a VM that is now back in the roster or has no residual resources.
|
|
4675
|
+
* Best-effort per VM: one `down` failing never aborts the rest of the sweep.
|
|
4676
|
+
* Fail-safe: if the live-instance list can't be fetched, teardown is skipped for
|
|
4677
|
+
* this pass (a live-but-unrostered VM must never be torn down) while the banner
|
|
4678
|
+
* reconcile still runs. Returns the `{ cleaned, failed, skipped, stillPresent }`
|
|
4679
|
+
* summary (null when the backend-service enumeration itself failed) — the same
|
|
4680
|
+
* shape it prints — so the flow is directly assertable. `deps` is injectable so
|
|
4681
|
+
* the enumerate → select → down flow is testable without gcloud. */
|
|
4682
|
+
/** Converge the shared fleet firewall rule to all-tcp so the LB reaches whatever
|
|
4683
|
+
* port a project's app actually serves.
|
|
4684
|
+
*
|
|
4685
|
+
* A rule pinned to a hardcoded port list (`tcp:14199,tcp:5173,tcp:3000`) drops
|
|
4686
|
+
* LB→VM traffic for any VM serving outside it — the VM-6-on-8080 Live Preview
|
|
4687
|
+
* outage — while the preview backend service still reads HEALTHY, because its
|
|
4688
|
+
* health check runs on the always-admitted editor port. The failure is therefore
|
|
4689
|
+
* invisible until someone opens the preview. Converging here (not only in the
|
|
4690
|
+
* hand-run google-ingress.sh bootstrap) is what makes a drifted rule self-heal on
|
|
4691
|
+
* the running fleet.
|
|
4692
|
+
*
|
|
4693
|
+
* All-tcp is the right exposure, not a per-stack port list: the rule stays scoped
|
|
4694
|
+
* to Google's LB + health-check source ranges and the fleet instance tag either
|
|
4695
|
+
* way, so it admits any app port (Vite 5173, Next 3000, Astro 4321, Go 8080, …)
|
|
4696
|
+
* without enumeration. EVERY candidate name in FLEET_FIREWALL_RULE_NAMES is
|
|
4697
|
+
* converged: describing only one silently no-ops on a fleet wired under the other.
|
|
4698
|
+
*
|
|
4699
|
+
* Scope-safe by construction — `update --rules=tcp` rewrites only the allowed
|
|
4700
|
+
* protocols/ports, never a rule's source ranges or target tags, so this can never
|
|
4701
|
+
* change who the rule applies to. Best-effort throughout: a describe/update
|
|
4702
|
+
* failure logs and continues, never aborting the rest of the reconcile.
|
|
4703
|
+
*
|
|
4704
|
+
* Returns `[{ ruleName, rules, converged }]` for every rule that exists — `rules`
|
|
4705
|
+
* is the post-converge allowed value — so the flow is assertable without gcloud.
|
|
4706
|
+
* `deps.spawnSync` is injectable for the same reason.
|
|
4707
|
+
* See plan vm--self-healing-app-port-agnostic-fleet-lb-firewall. */
|
|
4708
|
+
function convergeFleetFirewall({ cfg } = {}, deps = {}) {
|
|
4709
|
+
const run = deps.spawnSync || spawnSync;
|
|
4710
|
+
const project = cfg && cfg.project;
|
|
4711
|
+
const state = [];
|
|
4712
|
+
for (const ruleName of FLEET_FIREWALL_RULE_NAMES) {
|
|
4713
|
+
const dinv = fleetFirewallDescribeInvocation({ ruleName, project });
|
|
4714
|
+
const dres = run(dinv.command, dinv.args, { encoding: "utf8" });
|
|
4715
|
+
// A non-zero describe means no rule under this name — expected, not an error:
|
|
4716
|
+
// only one of the candidate names is deployed on any given fleet.
|
|
4717
|
+
if (!dres || dres.status !== 0) continue;
|
|
4718
|
+
const current = ((dres.stdout || "") + "").trim();
|
|
4719
|
+
if (!firewallNeedsConverge(current)) {
|
|
4720
|
+
console.error(` firewall ${ruleName}: already all-tcp → ok`);
|
|
4721
|
+
state.push({ ruleName, rules: "tcp", converged: false });
|
|
4722
|
+
continue;
|
|
4723
|
+
}
|
|
4724
|
+
const cinv = fleetFirewallConvergeInvocation({ ruleName, project });
|
|
4725
|
+
const cres = run(cinv.command, cinv.args, { encoding: "utf8" });
|
|
4726
|
+
if (cres && cres.status === 0) {
|
|
4727
|
+
console.error(` firewall ${ruleName}: rules '${current}' → tcp (all-tcp)`);
|
|
4728
|
+
state.push({ ruleName, rules: "tcp", converged: true });
|
|
4729
|
+
} else {
|
|
4730
|
+
const tail = (
|
|
4731
|
+
(cres && (cres.stderr || (cres.error && cres.error.message))) || ""
|
|
4732
|
+
).toString().trim().split("\n").pop() || "error";
|
|
4733
|
+
console.error(` firewall ${ruleName}: converge skipped (${tail})`);
|
|
4734
|
+
state.push({ ruleName, rules: current, converged: false });
|
|
4735
|
+
}
|
|
4736
|
+
}
|
|
4737
|
+
if (!state.length) {
|
|
4738
|
+
console.error(
|
|
4739
|
+
` firewall: no rule found under ${FLEET_FIREWALL_RULE_NAMES.join(" / ")} —` +
|
|
4740
|
+
" LB→VM traffic may be blocked; run scripts/gcp/google-ingress.sh",
|
|
4741
|
+
);
|
|
4742
|
+
}
|
|
4743
|
+
return state;
|
|
4744
|
+
}
|
|
4745
|
+
|
|
4746
|
+
/** Surface any rostered VM whose app port the fleet firewall does not admit — the
|
|
4747
|
+
* tripwire for a masked "HEALTHY but unreachable" preview.
|
|
4748
|
+
*
|
|
4749
|
+
* The preview backend service health-checks the fixed editor port, so a rule that
|
|
4750
|
+
* admits that port but not the app's real one reads HEALTHY while every real
|
|
4751
|
+
* request is dropped. That masking is what made the VM-6 outage invisible. Emits a
|
|
4752
|
+
* machine-readable `CY_FIREWALL_PORT_GAP <json>` line per gap, mirroring the
|
|
4753
|
+
* `CY_ORPHAN_SWEEP` marker convention the dashboard already parses.
|
|
4754
|
+
*
|
|
4755
|
+
* Only runs when a rule is still port-pinned: a converged all-tcp rule admits every
|
|
4756
|
+
* port, so the healthy path cannot produce a gap and skipping it avoids an SSH
|
|
4757
|
+
* round-trip per rostered VM for a check that cannot fail. In practice this fires
|
|
4758
|
+
* only if the converge itself failed (e.g. a permissions regression).
|
|
4759
|
+
*
|
|
4760
|
+
* Returns the emitted `[{ vmNumber, appPort, rule, allowed }]` gaps — the same
|
|
4761
|
+
* shape it prints — so the marker contract is directly assertable.
|
|
4762
|
+
* `deps.readVmAppPort` is injectable so tests never reach for SSH. */
|
|
4763
|
+
async function reportFirewallPortGaps({ cfg, roster, firewallState } = {}, deps = {}) {
|
|
4764
|
+
const readPort = deps.readVmAppPort || readVmAppPort;
|
|
4765
|
+
const pinned = (firewallState || []).filter((f) => firewallNeedsConverge(f.rules));
|
|
4766
|
+
const gaps = [];
|
|
4767
|
+
if (!pinned.length) return gaps;
|
|
4768
|
+
for (const n of roster || []) {
|
|
4769
|
+
const perVmCfg = { ...cfg, instanceName: `${fleetInstancePrefix()}${n}` };
|
|
4770
|
+
const appPort = await readPort(perVmCfg);
|
|
4771
|
+
for (const f of pinned) {
|
|
4772
|
+
if (firewallAdmitsPort(f.rules, appPort)) continue;
|
|
4773
|
+
const gap = { vmNumber: n, appPort, rule: f.ruleName, allowed: f.rules };
|
|
4774
|
+
console.error(` VM-${n}: app port ${appPort} NOT admitted by ${f.ruleName}`);
|
|
4775
|
+
console.log(`CY_FIREWALL_PORT_GAP ${JSON.stringify(gap)}`);
|
|
4776
|
+
gaps.push(gap);
|
|
4777
|
+
}
|
|
4778
|
+
}
|
|
4779
|
+
return gaps;
|
|
4780
|
+
}
|
|
4781
|
+
|
|
4782
|
+
function sweepOrphanIngress({ cfg, domain, roster, projectDir = rootDir } = {}, deps = {}) {
|
|
4783
|
+
const gi = deps.googleIngress || require("./google-ingress.js");
|
|
4784
|
+
const fcl = deps.fleetCleanup || require("./fleet-cleanup.js");
|
|
4785
|
+
const frecon = deps.fleetReconcile || require("./fleet-reconcile.js");
|
|
4786
|
+
const run = deps.spawnSync || spawnSync;
|
|
4787
|
+
|
|
4788
|
+
const listInv = backendServicesListInvocation({ project: cfg && cfg.project });
|
|
4789
|
+
const listRes = run(listInv.command, listInv.args, { encoding: "utf8" });
|
|
4790
|
+
if (!listRes || listRes.status !== 0) {
|
|
4791
|
+
const tail = (
|
|
4792
|
+
(listRes && (listRes.stderr || (listRes.error && listRes.error.message))) || ""
|
|
4793
|
+
).toString().trim().split("\n").pop() || "error";
|
|
4794
|
+
console.error(` orphan sweep skipped — could not list backend services (${tail}).`);
|
|
4795
|
+
return null;
|
|
4796
|
+
}
|
|
4797
|
+
const besVmNumbers = String(listRes.stdout || "")
|
|
4798
|
+
.split("\n")
|
|
4799
|
+
.map((line) => gi.vmNumberFromResourceName(line.trim()))
|
|
4800
|
+
.filter((n) => n != null);
|
|
4801
|
+
|
|
4802
|
+
// Also enumerate zonal NEGs and union their owners in: a partial teardown can
|
|
4803
|
+
// leave a `vm-<N>-<surface>-neg` behind after its backend service is already
|
|
4804
|
+
// gone, so a backend-service-only candidate set misses that residue. Best-effort
|
|
4805
|
+
// — a NEG-list failure never aborts the backend-service-driven sweep.
|
|
4806
|
+
let negVmNumbers = [];
|
|
4807
|
+
const negInv = negListInvocation({ project: cfg && cfg.project });
|
|
4808
|
+
const negRes = run(negInv.command, negInv.args, { encoding: "utf8" });
|
|
4809
|
+
if (negRes && negRes.status === 0) {
|
|
4810
|
+
negVmNumbers = String(negRes.stdout || "")
|
|
4811
|
+
.split("\n")
|
|
4812
|
+
.map((line) => gi.vmNumberFromResourceName(line.trim()))
|
|
4813
|
+
.filter((n) => n != null);
|
|
4814
|
+
} else {
|
|
4815
|
+
const tail = (
|
|
4816
|
+
(negRes && (negRes.stderr || (negRes.error && negRes.error.message))) || ""
|
|
4817
|
+
).toString().trim().split("\n").pop() || "error";
|
|
4818
|
+
console.error(` orphan sweep: could not list NEGs (${tail}) — sweeping backend services only.`);
|
|
4819
|
+
}
|
|
4820
|
+
const ingressVmNumbers = [...new Set([...besVmNumbers, ...negVmNumbers])].sort((a, b) => a - b);
|
|
4821
|
+
|
|
4822
|
+
// A VM in the roster, or with an in-flight `add-<n>.json` sidecar (provisioning,
|
|
4823
|
+
// not yet rostered), is live — never an orphan.
|
|
4824
|
+
const pendingDir = process.env.PENDING_JOBS_DIR || path.join(projectDir, ".codeyam/logs/pending-jobs");
|
|
4825
|
+
const inFlightAdds = fcl
|
|
4826
|
+
.listPendingJobs(pendingDir)
|
|
4827
|
+
.filter((j) => j.action === "add")
|
|
4828
|
+
.map((j) => j.n);
|
|
4829
|
+
|
|
4830
|
+
// Defensive live-instance guard: never tear down a VM whose GCP instance is
|
|
4831
|
+
// actually running (a roster drift could leave a live VM unrostered; a teardown
|
|
4832
|
+
// that SUCCEEDED against it would knock it offline). `liveVmNumbers === null`
|
|
4833
|
+
// means the list couldn't be fetched — fail safe by skipping teardown this pass
|
|
4834
|
+
// (below) rather than risk a false teardown. The banner reconcile still runs.
|
|
4835
|
+
let liveVmNumbers = null;
|
|
4836
|
+
const instInv = instancesListInvocation({ project: cfg && cfg.project });
|
|
4837
|
+
const instRes = run(instInv.command, instInv.args, { encoding: "utf8" });
|
|
4838
|
+
if (instRes && instRes.status === 0) {
|
|
4839
|
+
liveVmNumbers = frecon.parseRunningFleetVms(instRes.stdout, fleetInstancePrefix());
|
|
4840
|
+
} else {
|
|
4841
|
+
const tail = (
|
|
4842
|
+
(instRes && (instRes.stderr || (instRes.error && instRes.error.message))) || ""
|
|
4843
|
+
).toString().trim().split("\n").pop() || "error";
|
|
4844
|
+
console.error(` orphan sweep: could not list instances (${tail}) — skipping teardown this pass (banners still reconciled).`);
|
|
4845
|
+
}
|
|
4846
|
+
|
|
4847
|
+
const orphans = liveVmNumbers === null
|
|
4848
|
+
? []
|
|
4849
|
+
: selectOrphanIngressTargets({ ingressVmNumbers, roster, inFlightAdds, liveVmNumbers });
|
|
4850
|
+
|
|
4851
|
+
// stillPresent = every VM that owned an LB resource going in, minus the ones we
|
|
4852
|
+
// verifiably clean below. The dashboard clears a banner for any orphan VM not in
|
|
4853
|
+
// this set (or now back in the roster). Computed up front so the no-orphans and
|
|
4854
|
+
// fail-safe-skip paths report it too — that is what unfreezes a VM-3-style banner
|
|
4855
|
+
// (re-rostered, so never a sweep target) with no teardown at all.
|
|
4856
|
+
const summary = { cleaned: [], failed: [], skipped: inFlightAdds.slice(), stillPresent: ingressVmNumbers.slice() };
|
|
4857
|
+
if (!orphans.length) {
|
|
4858
|
+
console.error(" orphan sweep: no orphaned LB ingress found.");
|
|
4859
|
+
console.log(`CY_ORPHAN_SWEEP ${JSON.stringify(summary)}`);
|
|
4860
|
+
return summary;
|
|
4861
|
+
}
|
|
4862
|
+
|
|
4863
|
+
console.error(` orphan sweep: auto-cleaning ${orphans.length} orphaned VM ingress (${orphans.join(", ")})...`);
|
|
4864
|
+
const scriptPath = path.join(projectDir, "scripts", "gcp", "vm-ingress.sh");
|
|
4865
|
+
for (const n of orphans) {
|
|
4866
|
+
// A destroyed VM's slug is normally already released, so slugFor(n) is null and
|
|
4867
|
+
// the numeric vm-<N> host rules are torn down; if a destroy left the slug behind,
|
|
4868
|
+
// slugFor(n) resolves it so the <slug>.<surface> host rules go too.
|
|
4869
|
+
const slug = slugFor(n) || undefined;
|
|
4870
|
+
const inv = vmIngressInvocation({
|
|
4871
|
+
scriptPath,
|
|
4872
|
+
vmNumber: n,
|
|
4873
|
+
action: "down",
|
|
4874
|
+
domain,
|
|
4875
|
+
project: cfg && cfg.project,
|
|
4876
|
+
zone: cfg && cfg.zone,
|
|
4877
|
+
instancePrefix: fleetInstancePrefix(),
|
|
4878
|
+
slug,
|
|
4879
|
+
});
|
|
4880
|
+
const res = run(inv.command, inv.args, { encoding: "utf8", env: { ...process.env, ...inv.env } });
|
|
4881
|
+
const verdict = parseIngressTeardownVerdict((res && res.stdout) || "");
|
|
4882
|
+
// Pass `signal` too: a teardown the caller's group-kill reaped has a NULL
|
|
4883
|
+
// status, which the classifier would otherwise report as `spawn-failed`
|
|
4884
|
+
// rather than the truthful `killed=SIGKILL`.
|
|
4885
|
+
const outcome = classifyOrphanDownResult({
|
|
4886
|
+
verdict,
|
|
4887
|
+
status: res ? res.status : null,
|
|
4888
|
+
signal: res ? res.signal : null,
|
|
4889
|
+
});
|
|
4890
|
+
if (outcome.cleaned) {
|
|
4891
|
+
summary.cleaned.push(n);
|
|
4892
|
+
console.error(` VM-${n}: orphaned ingress → auto-cleaned (verified clean)`);
|
|
4893
|
+
} else {
|
|
4894
|
+
const tail = `${(res && res.stdout) || ""}${(res && res.stderr) || ""}`
|
|
4895
|
+
.trim().split("\n").pop() || "no verdict";
|
|
4896
|
+
summary.failed.push({ n, cause: outcome.cause });
|
|
4897
|
+
console.error(` VM-${n}: orphaned ingress → auto-clean FAILED (${tail.slice(-200)})`);
|
|
4898
|
+
}
|
|
4899
|
+
}
|
|
4900
|
+
// A verifiably-cleaned VM no longer owns any LB resource — drop it from
|
|
4901
|
+
// stillPresent so the dashboard clears its banner (failed VMs stay, keeping
|
|
4902
|
+
// their real attention item).
|
|
4903
|
+
const cleanedSet = new Set(summary.cleaned);
|
|
4904
|
+
summary.stillPresent = ingressVmNumbers.filter((n) => !cleanedSet.has(n));
|
|
4905
|
+
console.log(`CY_ORPHAN_SWEEP ${JSON.stringify(summary)}`);
|
|
4906
|
+
return summary;
|
|
3250
4907
|
}
|
|
3251
4908
|
|
|
3252
4909
|
/** `cloud:reconcile-credentials` — self-heal a running VM that came up (or
|
|
@@ -3258,20 +4915,18 @@ async function cmdReconcileIngress(args, projectDir = rootDir) {
|
|
|
3258
4915
|
* entrypoint) and the live in-memory credential-cache daemon (immediate, no
|
|
3259
4916
|
* restart). This codifies the manual recovery run on all 9 fleet VMs on
|
|
3260
4917
|
* 2026-06-29. */
|
|
3261
|
-
|
|
3262
|
-
|
|
3263
|
-
|
|
3264
|
-
|
|
3265
|
-
|
|
3266
|
-
|
|
3267
|
-
|
|
4918
|
+
/** One SSH round-trip probe of a running VM's git push credentials: reports
|
|
4919
|
+
* whether the host staging file (`~/.codeyam-cloud-git-credentials`) is empty
|
|
4920
|
+
* and the in-container `git ls-remote origin` outcome, parsed into the exact
|
|
4921
|
+
* shape {@link decideCredentialHeal} consumes (`{ stagingEmpty, lsRemoteOk,
|
|
4922
|
+
* lsRemoteStderr }`). Shared by {@link cmdReconcileCredentials} (the manual
|
|
4923
|
+
* self-heal verb) and {@link assertGitPushCredentials} (the provision-time
|
|
4924
|
+
* guarantee) so the two never drift on probe wire-format. Throws on an SSH
|
|
4925
|
+
* error — the caller decides whether that is fatal or best-effort. */
|
|
4926
|
+
async function probeGitCredentials(cfg) {
|
|
3268
4927
|
const repoName = resolveProject(cfg).name;
|
|
3269
4928
|
const projectCd = `cd ~/projects/${repoName}`;
|
|
3270
4929
|
const composeExec = `docker compose -f docker-compose.cloud.yml exec -T editor`;
|
|
3271
|
-
|
|
3272
|
-
// One SSH round-trip: report staging-file emptiness and the in-container
|
|
3273
|
-
// ls-remote outcome with stable markers we parse below.
|
|
3274
|
-
console.error(`Probing git push credentials on ${cfg.instanceName}...`);
|
|
3275
4930
|
const probeCmd =
|
|
3276
4931
|
`f=$HOME/.codeyam-cloud-git-credentials; ` +
|
|
3277
4932
|
`if [ -s "$f" ]; then echo STAGING=nonempty; else echo STAGING=empty; fi; ` +
|
|
@@ -3286,54 +4941,183 @@ async function cmdReconcileCredentials(args, projectDir = rootDir) {
|
|
|
3286
4941
|
throw new Error(`Failed to probe credentials on ${cfg.instanceName} (SSH error).`);
|
|
3287
4942
|
}
|
|
3288
4943
|
const out = probe.stdout;
|
|
3289
|
-
|
|
3290
|
-
|
|
3291
|
-
|
|
3292
|
-
|
|
3293
|
-
|
|
3294
|
-
|
|
3295
|
-
|
|
3296
|
-
|
|
3297
|
-
|
|
3298
|
-
|
|
3299
|
-
if (!decision.heal) {
|
|
3300
|
-
console.error(" ✓ no re-seed needed.");
|
|
3301
|
-
return;
|
|
3302
|
-
}
|
|
3303
|
-
|
|
3304
|
-
// Re-seed needs a token. Reuse seedGithubCredentials for the durable host-file
|
|
3305
|
-
// write (it throws MissingGitPushCredentialsError when none resolves — the
|
|
3306
|
-
// honest "can't heal without a token" outcome).
|
|
3307
|
-
await seedGithubCredentials(cfg);
|
|
4944
|
+
return {
|
|
4945
|
+
stagingEmpty: /STAGING=empty/.test(out),
|
|
4946
|
+
lsRemoteOk: /LSREMOTE=ok/.test(out),
|
|
4947
|
+
lsRemoteStderr: out
|
|
4948
|
+
.split("\n")
|
|
4949
|
+
.filter((l) => l.startsWith("LSRERR:"))
|
|
4950
|
+
.map((l) => l.slice("LSRERR:".length))
|
|
4951
|
+
.join("\n"),
|
|
4952
|
+
};
|
|
4953
|
+
}
|
|
3308
4954
|
|
|
3309
|
-
|
|
3310
|
-
|
|
3311
|
-
|
|
3312
|
-
|
|
3313
|
-
|
|
3314
|
-
|
|
4955
|
+
/** Seed the LIVE in-memory git credential-cache daemon over its socket — the
|
|
4956
|
+
* same `git credential approve` shape the entrypoint uses
|
|
4957
|
+
* (scripts/cloud-editor-entrypoint.sh) — so push works immediately with no
|
|
4958
|
+
* container restart. ALSO writes the container-local on-disk `store` fallback
|
|
4959
|
+
* file (the fill-only second helper baked into the image) directly, so a
|
|
4960
|
+
* reconcile leaves the VM durable against the cache daemon dying mid-life, not
|
|
4961
|
+
* just healthy for the moment. base64 keeps the token out of the shell history
|
|
4962
|
+
* across the gcloud-ssh → docker layers, and the store write is an in-place
|
|
4963
|
+
* `cat > file` truncate (chmod 600), never touched by git's approve path.
|
|
4964
|
+
* Throws if either write fails. Shared by {@link cmdReconcileCredentials} and
|
|
4965
|
+
* {@link assertGitPushCredentials}. */
|
|
4966
|
+
async function seedLiveCredentialCache(cfg, token) {
|
|
4967
|
+
const repoName = resolveProject(cfg).name;
|
|
4968
|
+
const projectCd = `cd ~/projects/${repoName}`;
|
|
4969
|
+
const composeExec = `docker compose -f docker-compose.cloud.yml exec -T editor`;
|
|
3315
4970
|
const socket = "/run/codeyam/git-credential-cache.socket";
|
|
4971
|
+
const store = "/run/codeyam/git-credentials-store";
|
|
3316
4972
|
const approveStdin = `protocol=https\nhost=github.com\nusername=x-access-token\npassword=${token}\n\n`;
|
|
3317
4973
|
const b64 = Buffer.from(approveStdin).toString("base64");
|
|
4974
|
+
// The store-file line is the same one-URL-per-line format as the host staging
|
|
4975
|
+
// file; the image's fill-only `store` helper reads it on a cache miss.
|
|
4976
|
+
const storeB64 = Buffer.from(`${githubCredentialLine(token)}\n`).toString("base64");
|
|
3318
4977
|
const seedCmd =
|
|
3319
|
-
`${projectCd} &&
|
|
4978
|
+
`${projectCd} && ` +
|
|
4979
|
+
`echo ${b64} | base64 -d | ${composeExec} ` +
|
|
3320
4980
|
`git -c 'credential.helper=cache --timeout=31536000 --socket=${socket}' ` +
|
|
3321
|
-
`credential approve &&
|
|
4981
|
+
`credential approve && ` +
|
|
4982
|
+
`echo ${storeB64} | base64 -d | ${composeExec} ` +
|
|
4983
|
+
`bash -c 'cat > ${store} && chmod 600 ${store}' && ` +
|
|
4984
|
+
`echo APPROVED`;
|
|
3322
4985
|
const seed = runGcloud(
|
|
3323
4986
|
buildGcloudArgs("ssh-probe", cfg, { command: seedCmd }),
|
|
3324
4987
|
{ stdio: ["ignore", "pipe", "inherit"] },
|
|
3325
4988
|
);
|
|
3326
4989
|
if (seed.status !== 0 || !/APPROVED/.test(String(seed.stdout || ""))) {
|
|
3327
4990
|
throw new Error(
|
|
3328
|
-
`
|
|
4991
|
+
`Failed to seed the live credential cache on ${cfg.instanceName}.`,
|
|
3329
4992
|
);
|
|
3330
4993
|
}
|
|
4994
|
+
}
|
|
4995
|
+
|
|
4996
|
+
async function cmdReconcileCredentials(args, projectDir = rootDir) {
|
|
4997
|
+
ensureGcloud();
|
|
4998
|
+
parseFlags(args);
|
|
4999
|
+
const cfg = loadCloudConfig(projectDir);
|
|
5000
|
+
if (!cfg) {
|
|
5001
|
+
throw new Error("No .codeyam/cloud.json found. Run `npm run cloud:up` first.");
|
|
5002
|
+
}
|
|
5003
|
+
|
|
5004
|
+
console.error(`Probing git push credentials on ${cfg.instanceName}...`);
|
|
5005
|
+
const decision = decideCredentialHeal(await probeGitCredentials(cfg));
|
|
5006
|
+
console.error(` category=${decision.category}: ${decision.reason}`);
|
|
5007
|
+
if (!decision.heal) {
|
|
5008
|
+
console.error(" ✓ no re-seed needed.");
|
|
5009
|
+
return;
|
|
5010
|
+
}
|
|
5011
|
+
|
|
5012
|
+
// Re-seed needs a token. Reuse seedGithubCredentials for the durable host-file
|
|
5013
|
+
// write (it throws MissingGitPushCredentialsError when none resolves — the
|
|
5014
|
+
// honest "can't heal without a token" outcome), then seed the live cache.
|
|
5015
|
+
await seedGithubCredentials(cfg);
|
|
5016
|
+
await seedLiveCredentialCache(cfg, resolveGithubTokenWithSource().token);
|
|
3331
5017
|
console.error(
|
|
3332
5018
|
` ✓ re-seeded host file + live credential cache on ${cfg.instanceName}; ` +
|
|
3333
5019
|
`plan-claim sync will recover on the next tick.`,
|
|
3334
5020
|
);
|
|
3335
5021
|
}
|
|
3336
5022
|
|
|
5023
|
+
/** `cloud:reconcile-admin` — flip admin mode on/off for an already-running VM
|
|
5024
|
+
* WITHOUT a full re-provision. Resolves the desired value from this invocation
|
|
5025
|
+
* (`--admin` / `CODEYAM_ADMIN`, else off), rewrites the rendered compose's
|
|
5026
|
+
* CODEYAM_ADMIN line, and recreates the editor service — matching what a manual
|
|
5027
|
+
* compose edit + `docker compose up -d editor` does today, so the durable admin
|
|
5028
|
+
* setting no longer requires a full re-provision to change. */
|
|
5029
|
+
async function cmdReconcileAdmin(args, projectDir = rootDir) {
|
|
5030
|
+
ensureGcloud();
|
|
5031
|
+
const flags = parseFlags(args);
|
|
5032
|
+
const cfg = loadCloudConfig(projectDir);
|
|
5033
|
+
if (!cfg) {
|
|
5034
|
+
throw new Error("No .codeyam/cloud.json found. Run `npm run cloud:up` first.");
|
|
5035
|
+
}
|
|
5036
|
+
const desired =
|
|
5037
|
+
buildProvisionEnv(process.env, flags).CODEYAM_ADMIN === "1" ? "1" : "";
|
|
5038
|
+
const repoName = resolveProject(cfg).name;
|
|
5039
|
+
console.error(
|
|
5040
|
+
`Reconciling admin mode to ${desired === "1" ? "ON" : "off"} on ${cfg.instanceName}...`,
|
|
5041
|
+
);
|
|
5042
|
+
const result = runGcloud(
|
|
5043
|
+
buildGcloudArgs("ssh-probe", cfg, {
|
|
5044
|
+
command: buildReconcileAdminCommand(repoName, desired),
|
|
5045
|
+
}),
|
|
5046
|
+
{ stdio: "inherit" },
|
|
5047
|
+
);
|
|
5048
|
+
if (result.status !== 0) {
|
|
5049
|
+
throw new Error(`Failed to reconcile admin mode on ${cfg.instanceName}.`);
|
|
5050
|
+
}
|
|
5051
|
+
console.error(
|
|
5052
|
+
` ✓ editor service recreated; /api/config now reports admin=${desired === "1" ? "true" : "false"}.`,
|
|
5053
|
+
);
|
|
5054
|
+
}
|
|
5055
|
+
|
|
5056
|
+
/** Provision-time guarantee that a just-provisioned VM can actually reach
|
|
5057
|
+
* GitHub. Probes the staging file + in-container `git ls-remote origin` and
|
|
5058
|
+
* routes the outcome through the SAME {@link decideCredentialHeal} classifier
|
|
5059
|
+
* the manual `cloud:reconcile-credentials` verb uses (no parallel logic). On a
|
|
5060
|
+
* credential gap it re-seeds the durable host file + live cache daemon and
|
|
5061
|
+
* re-asserts ONCE. A failure that persists after the re-seed — or no token to
|
|
5062
|
+
* seed with — ends in a loud `CY_DEGRADED git push credentials missing` marker
|
|
5063
|
+
* (the dashboard's amber-card channel), never a silently-green VM that cannot
|
|
5064
|
+
* push (VM5's 2026-06-30 "could not read Username for github.com" symptom).
|
|
5065
|
+
* Best-effort on infra errors: an SSH-probe failure is logged and skipped
|
|
5066
|
+
* rather than aborting an otherwise-green Add, matching seedGithubCredentials'
|
|
5067
|
+
* degraded posture. */
|
|
5068
|
+
async function assertGitPushCredentials(cfg) {
|
|
5069
|
+
let probe;
|
|
5070
|
+
try {
|
|
5071
|
+
probe = await probeGitCredentials(cfg);
|
|
5072
|
+
} catch (err) {
|
|
5073
|
+
// Probe itself failed (SSH error) — can't classify; log loud, don't abort.
|
|
5074
|
+
console.error(` (could not verify git push credentials: ${err.message})`);
|
|
5075
|
+
return;
|
|
5076
|
+
}
|
|
5077
|
+
const decision = decideCredentialHeal(probe);
|
|
5078
|
+
console.error(` git push credentials: category=${decision.category}: ${decision.reason}`);
|
|
5079
|
+
if (!decision.heal) {
|
|
5080
|
+
// Healthy, or a transient network blip that re-seeding wouldn't fix anyway.
|
|
5081
|
+
return;
|
|
5082
|
+
}
|
|
5083
|
+
|
|
5084
|
+
// A re-seed needs a token. Without one we cannot heal — land the loud marker
|
|
5085
|
+
// instead of throwing inside seedGithubCredentials (which would abort the Add).
|
|
5086
|
+
const { token } = resolveGithubTokenWithSource();
|
|
5087
|
+
if (!token) {
|
|
5088
|
+
console.log(
|
|
5089
|
+
"CY_DEGRADED git push credentials missing: no token resolved to seed the VM. " +
|
|
5090
|
+
"Recover: npm run cloud:reconcile-credentials",
|
|
5091
|
+
);
|
|
5092
|
+
return;
|
|
5093
|
+
}
|
|
5094
|
+
|
|
5095
|
+
// Best-effort heal: the container is already up and the VM is otherwise
|
|
5096
|
+
// reachable, so a re-seed/approve/probe error must DEGRADE (loud CY_DEGRADED),
|
|
5097
|
+
// never throw out of cmdUp and abort an otherwise-green Add. Mirrors the
|
|
5098
|
+
// seedGithubCredentials catch in cmdUp.
|
|
5099
|
+
try {
|
|
5100
|
+
await seedGithubCredentials(cfg);
|
|
5101
|
+
await seedLiveCredentialCache(cfg, token);
|
|
5102
|
+
|
|
5103
|
+
// Re-assert once: a stale/incomplete cache should now ls-remote cleanly.
|
|
5104
|
+
const reprobe = await probeGitCredentials(cfg);
|
|
5105
|
+
if (!reprobe.lsRemoteOk) {
|
|
5106
|
+
console.log(
|
|
5107
|
+
"CY_DEGRADED git push credentials missing: git ls-remote still fails after re-seed. " +
|
|
5108
|
+
"Recover: npm run cloud:reconcile-credentials",
|
|
5109
|
+
);
|
|
5110
|
+
return;
|
|
5111
|
+
}
|
|
5112
|
+
console.error(` ✓ git push credentials verified after re-seed on ${cfg.instanceName}.`);
|
|
5113
|
+
} catch (err) {
|
|
5114
|
+
console.log(
|
|
5115
|
+
`CY_DEGRADED git push credentials missing: re-seed failed (${err.message}). ` +
|
|
5116
|
+
"Recover: npm run cloud:reconcile-credentials",
|
|
5117
|
+
);
|
|
5118
|
+
}
|
|
5119
|
+
}
|
|
5120
|
+
|
|
3337
5121
|
/** `cloud:access` — edit an existing VM's IAP access whitelist without
|
|
3338
5122
|
* re-provisioning. Resolves the VM number from --instance-name (or cloud.json),
|
|
3339
5123
|
* mutates the access sidecar, and applies just the delta against the live
|
|
@@ -3400,6 +5184,7 @@ const HANDLERS = {
|
|
|
3400
5184
|
"reconcile-ingress": cmdReconcileIngress,
|
|
3401
5185
|
"wire-add-tail": cmdWireAddTail,
|
|
3402
5186
|
"reconcile-credentials": cmdReconcileCredentials,
|
|
5187
|
+
"reconcile-admin": cmdReconcileAdmin,
|
|
3403
5188
|
access: cmdAccess,
|
|
3404
5189
|
};
|
|
3405
5190
|
|
|
@@ -3422,14 +5207,26 @@ async function cloudMain() {
|
|
|
3422
5207
|
|
|
3423
5208
|
module.exports = {
|
|
3424
5209
|
// pure helpers
|
|
5210
|
+
isSpawnTimeout,
|
|
3425
5211
|
agentEnvVarName,
|
|
3426
5212
|
agentNpmPackage,
|
|
3427
5213
|
AGENT_ENV_VARS,
|
|
3428
5214
|
AGENT_NPM_PACKAGES,
|
|
3429
5215
|
ensureAgentCliInstalled,
|
|
5216
|
+
ensureEditorContainerUp,
|
|
5217
|
+
seedAgentApiKey,
|
|
5218
|
+
assertAgentCliInstalled,
|
|
5219
|
+
probeContainerRunning,
|
|
5220
|
+
probeContainerState,
|
|
5221
|
+
waitForEditorComposeRunning,
|
|
5222
|
+
abortTeardownDecision,
|
|
5223
|
+
probeAbortTeardownState,
|
|
5224
|
+
probeAgentBinaryInstalled,
|
|
5225
|
+
waitForContainerRunning,
|
|
3430
5226
|
KNOWN_FLAGS,
|
|
3431
5227
|
FIREWALL_RULE_NAME,
|
|
3432
5228
|
INSTANCE_TAG,
|
|
5229
|
+
DEFAULT_FLEET_MACHINE_TYPE,
|
|
3433
5230
|
EDITOR_PORT,
|
|
3434
5231
|
VITE_PORT,
|
|
3435
5232
|
parseFlags,
|
|
@@ -3448,6 +5245,7 @@ module.exports = {
|
|
|
3448
5245
|
previewSameOriginSafeFromStack,
|
|
3449
5246
|
isRottedManagedPreviewOrigin,
|
|
3450
5247
|
isWrongModePreviewOrigin,
|
|
5248
|
+
isSilentlyBrokenPreview,
|
|
3451
5249
|
resolveEditorConfig,
|
|
3452
5250
|
validateEditorVariant,
|
|
3453
5251
|
assertVariantProvisionable,
|
|
@@ -3456,13 +5254,19 @@ module.exports = {
|
|
|
3456
5254
|
resolveGhcrToken,
|
|
3457
5255
|
resolveGhcrPushToken,
|
|
3458
5256
|
probeUsableEditorImage,
|
|
5257
|
+
httpsManifestGet,
|
|
3459
5258
|
resolveProject,
|
|
3460
5259
|
clientProjectBootstrapArgs,
|
|
3461
5260
|
cloudConfigPath,
|
|
3462
5261
|
loadCloudConfig,
|
|
3463
5262
|
saveCloudConfig,
|
|
3464
5263
|
loadOrInitCloudConfig,
|
|
5264
|
+
buildProvisionEnv,
|
|
5265
|
+
buildReconcileAdminCommand,
|
|
5266
|
+
cmdReconcileAdmin,
|
|
3465
5267
|
buildEditorLocalEnvPatch,
|
|
5268
|
+
buildGreenfieldTitlePatch,
|
|
5269
|
+
mergeEditorLocalPatchB64,
|
|
3466
5270
|
buildEditorLocalSeedWriteCommand,
|
|
3467
5271
|
seedPreviewOrigin,
|
|
3468
5272
|
healPreviewOriginAtReady,
|
|
@@ -3476,6 +5280,22 @@ module.exports = {
|
|
|
3476
5280
|
decideBootstrapOutcome,
|
|
3477
5281
|
projectBootstrapDoneMarker,
|
|
3478
5282
|
projectBootstrapDoneProbeCommand,
|
|
5283
|
+
projectBootstrapScriptPath,
|
|
5284
|
+
projectBootstrapRunnerPath,
|
|
5285
|
+
projectBootstrapLogPath,
|
|
5286
|
+
projectBootstrapPidPath,
|
|
5287
|
+
projectBootstrapStatusPath,
|
|
5288
|
+
projectBootstrapRunnerScript,
|
|
5289
|
+
projectBootstrapLaunchCommand,
|
|
5290
|
+
projectBootstrapFollowCommand,
|
|
5291
|
+
projectBootstrapStatusCommand,
|
|
5292
|
+
parseProjectBootstrapStatus,
|
|
5293
|
+
decideProjectBootstrapOutcome,
|
|
5294
|
+
decideProjectBootstrapFollowAction,
|
|
5295
|
+
projectBootstrapEnv,
|
|
5296
|
+
PROJECT_BOOTSTRAP_LAUNCHED_MARKER,
|
|
5297
|
+
PROJECT_BOOTSTRAP_FOLLOW_SECS,
|
|
5298
|
+
ensureProjectBootstrapped,
|
|
3479
5299
|
SSH_KEEPALIVE_FLAGS,
|
|
3480
5300
|
resolveSetupForwardEnv,
|
|
3481
5301
|
FORWARD_ENV_KEYS_ENV,
|
|
@@ -3488,12 +5308,21 @@ module.exports = {
|
|
|
3488
5308
|
ensureBootstrapped,
|
|
3489
5309
|
assertEditorContainerRunning,
|
|
3490
5310
|
checkScriptDrift,
|
|
5311
|
+
streamedScriptEditorBranch,
|
|
5312
|
+
streamedScriptMergeBase,
|
|
5313
|
+
fetchStreamedScriptRef,
|
|
5314
|
+
resolveRemoteScriptRev,
|
|
5315
|
+
gatherScriptDriftFacts,
|
|
5316
|
+
readStreamedScript,
|
|
3491
5317
|
STREAMED_SCRIPTS,
|
|
3492
5318
|
// ingress wiring (pure decision + spawn wrappers)
|
|
5319
|
+
envPosInt,
|
|
3493
5320
|
ingressWiringPlan,
|
|
3494
5321
|
fleetInstancePrefix,
|
|
3495
5322
|
runVmIngress,
|
|
3496
5323
|
wireVmIngressOnAdd,
|
|
5324
|
+
probeVmIngressVerdict,
|
|
5325
|
+
convergeVmIngressOnAdd,
|
|
3497
5326
|
unwireVmIngressOnDestroy,
|
|
3498
5327
|
// per-VM IAP access (spawn wrapper + Add/Destroy hooks)
|
|
3499
5328
|
runIapAccess,
|
|
@@ -3508,16 +5337,24 @@ module.exports = {
|
|
|
3508
5337
|
cmdStatus,
|
|
3509
5338
|
cmdIngressUp,
|
|
3510
5339
|
cmdReconcileIngress,
|
|
5340
|
+
sweepOrphanIngress,
|
|
5341
|
+
convergeFleetFirewall,
|
|
5342
|
+
reportFirewallPortGaps,
|
|
3511
5343
|
cmdReconcileCredentials,
|
|
3512
5344
|
cmdAccess,
|
|
3513
5345
|
cloudMain,
|
|
5346
|
+
probeGitCredentials,
|
|
5347
|
+
seedLiveCredentialCache,
|
|
5348
|
+
assertGitPushCredentials,
|
|
3514
5349
|
seedGithubCredentials,
|
|
5350
|
+
githubCredentialLine,
|
|
3515
5351
|
resolveGithubToken,
|
|
3516
5352
|
resolveGithubTokenWithSource,
|
|
3517
5353
|
MissingGitPushCredentialsError,
|
|
3518
5354
|
classifyGitRemoteError,
|
|
3519
5355
|
decideCredentialHeal,
|
|
3520
5356
|
readVmAppPort,
|
|
5357
|
+
buildVmAppPortReadCommand,
|
|
3521
5358
|
resolveLaptopGitIdentity,
|
|
3522
5359
|
};
|
|
3523
5360
|
|