@codeyam-editor/codeyam-editor 0.1.0-staging.gbaebe71 → 0.1.0-staging.gc0a93f0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/npm/cloud.js +1085 -111
- package/npm/compose-conflict.js +59 -0
- package/npm/editor-dev.js +24 -2
- package/npm/editor.js +15 -5
- package/npm/fleet-add-failure.js +574 -39
- package/npm/fleet-auth-health.js +26 -5
- package/npm/fleet-branch.js +124 -1
- package/npm/fleet-build-data.js +22 -3
- package/npm/fleet-cleanup.js +213 -3
- package/npm/fleet-datastore-health.js +107 -0
- package/npm/fleet-destroy-safety.js +182 -13
- package/npm/fleet-job-badge.js +28 -12
- package/npm/fleet-memory.js +306 -8
- package/npm/fleet-probe-transport.js +155 -0
- package/npm/fleet-public-url.js +94 -8
- package/npm/fleet-queue.js +141 -0
- package/npm/fleet-reachability.js +80 -0
- package/npm/fleet-revision.js +212 -0
- package/npm/fleet-run-setup.js +66 -0
- package/npm/fleet-running-plan.js +37 -7
- package/npm/fleet-session-activity.js +162 -0
- package/npm/fleet-vm-access.js +201 -0
- package/npm/fleet-vm-probe-target.js +95 -0
- package/npm/fleet-vm-readiness.js +57 -16
- package/npm/ingress-invoke.js +98 -10
- package/npm/launch-config.js +289 -6
- package/npm/scenario-check.js +760 -53
- package/npm/scenario-handlers.js +18 -0
- package/npm/scenario-interactivity.js +477 -23
- package/npm/scenario-issues.js +72 -0
- package/npm/scenario-mocks.js +245 -39
- package/npm/scenario-playwright.js +181 -36
- package/npm/utils.js +262 -22
- package/package.json +5 -5
- package/ui/dist/assets/{ContainerInspector-qJy9NyEZ.js → ContainerInspector-C7I9wctz.js} +1 -1
- package/ui/dist/assets/EditorShell-9aYMeOdZ.js +330 -0
- package/ui/dist/assets/EditorShell-CE8iG_EM.css +1 -0
- package/ui/dist/assets/ProjectLauncherScreen-E0NYYiwy.js +1 -0
- package/ui/dist/assets/ScenarioDataPanel-CLwkrex0.js +18 -0
- package/ui/dist/assets/{ScenarioDataPanelFullPage-CrNwRIZM.js → ScenarioDataPanelFullPage-DT0TZC4M.js} +1 -1
- package/ui/dist/assets/ScenarioRenderer-Bsarhcrl.js +21 -0
- package/ui/dist/assets/index-BRoAenc1.css +1 -0
- package/ui/dist/assets/index-iMR5siiG.js +27 -0
- package/ui/dist/assets/useServerIdentity-CVuHTzjg.js +1 -0
- package/ui/dist/index.html +2 -2
- package/ui/dist/preview-subdir-demo/index.html +62 -0
- package/ui/dist/preview-subdir-demo/sibling.html +48 -0
- package/ui/dist/assets/EditorShell-BM92Hqc1.js +0 -252
- package/ui/dist/assets/EditorShell-DP5UY20G.css +0 -1
- package/ui/dist/assets/ProjectLauncherScreen-BLHSXp_H.js +0 -1
- package/ui/dist/assets/ScenarioDataPanel-BbkUcHum.js +0 -5
- package/ui/dist/assets/ScenarioRenderer-mB7opGvg.js +0 -21
- package/ui/dist/assets/index-BygCiJSb.css +0 -1
- package/ui/dist/assets/index-Mui6rJUD.js +0 -27
- package/ui/dist/assets/useServerIdentity-COXGxKUT.js +0 -1
package/npm/cloud.js
CHANGED
|
@@ -59,6 +59,7 @@ const {
|
|
|
59
59
|
normalizePrincipal,
|
|
60
60
|
resolveDefaultOperator,
|
|
61
61
|
withDefaultOperator,
|
|
62
|
+
unauthorizableAddRefusal,
|
|
62
63
|
accessFor,
|
|
63
64
|
setAccess,
|
|
64
65
|
addAccess,
|
|
@@ -105,6 +106,10 @@ const {
|
|
|
105
106
|
// edit (the invisible-VM gap improve38 closes). `readRoster` enumerates the
|
|
106
107
|
// fleet for `reconcile-ingress`.
|
|
107
108
|
const { addToRoster, readRoster } = require("./fleet-roster");
|
|
109
|
+
// Tells a docker container-name conflict (a concurrent run already started this
|
|
110
|
+
// container) apart from a real `docker compose up -d` failure, so the abort path
|
|
111
|
+
// can converge instead of escalating to instance deletion.
|
|
112
|
+
const { classifyComposeUpFailure } = require("./compose-conflict");
|
|
108
113
|
|
|
109
114
|
// =============================================================================
|
|
110
115
|
// Pure helpers — every export here is unit-tested in cloud.test.js without
|
|
@@ -509,6 +514,28 @@ function previewSameOriginSafeFromStack(stackJson) {
|
|
|
509
514
|
return typeof flag === "boolean" ? flag : true;
|
|
510
515
|
}
|
|
511
516
|
|
|
517
|
+
/** Resolve the sameOriginSafe flag a seed should ACT on, given the static
|
|
518
|
+
* `stack.json` reading and the running editor's own `/api/config` verdict.
|
|
519
|
+
*
|
|
520
|
+
* Precedence: a BOOLEAN effective verdict wins; anything else (`null`,
|
|
521
|
+
* `undefined`, a non-boolean) defers to the static flag. The type check is the
|
|
522
|
+
* whole rule — `null` means "no server was up to ask", which is the
|
|
523
|
+
* pre-container seed and must leave the static read in charge, NOT "the server
|
|
524
|
+
* said same-origin is fine". Collapsing the two would put the static flag back
|
|
525
|
+
* in control on exactly the path that needs the override.
|
|
526
|
+
*
|
|
527
|
+
* Extracted from `seedPreviewOrigin` because that function's inputs arrive over
|
|
528
|
+
* gcloud SSH, so this rule was only reachable through heavy mocking — while
|
|
529
|
+
* being precisely where the defect lived. `stack.json` cannot express the state
|
|
530
|
+
* that matters here: a project escalates off the same-origin subpath at RUNTIME
|
|
531
|
+
* after a hydration failure, and the adaptive-default stacks that happens to
|
|
532
|
+
* (Next.js) deliberately omit the key. So a seeder that re-derives the flag
|
|
533
|
+
* from the file writes back the same-origin sentinel that just failed, and the
|
|
534
|
+
* project sits at `previewDedicatedOriginUnavailable: true` forever. Pure. */
|
|
535
|
+
function resolveEffectiveSameOriginSafe(staticFlag, effectiveVerdict) {
|
|
536
|
+
return typeof effectiveVerdict === "boolean" ? effectiveVerdict : staticFlag;
|
|
537
|
+
}
|
|
538
|
+
|
|
512
539
|
// Matches an absolute cloudflared quick-tunnel origin
|
|
513
540
|
// (`https://<sub>.trycloudflare.com`, optionally with a port/path/query). These
|
|
514
541
|
// URLs ROTATE on every tunnel respawn, which is exactly why a frozen one rots.
|
|
@@ -845,6 +872,79 @@ function parseEditorComposeState(stdout) {
|
|
|
845
872
|
return null;
|
|
846
873
|
}
|
|
847
874
|
|
|
875
|
+
/** Decide whether `cloud:up`'s abort handler may delete the instance it is
|
|
876
|
+
* unwinding. Pure `{ createdThisRun, containerUp, rostered, probeInconclusive }
|
|
877
|
+
* → 'delete' | 'keep'`, matching the `reapVerdict` / `reattachVerdict` /
|
|
878
|
+
* `reconcileRosteredVm` pattern in npm/fleet-cleanup.js, so the whole matrix is
|
|
879
|
+
* exhaustively testable without a VM.
|
|
880
|
+
*
|
|
881
|
+
* `createdThisRun` stays a NECESSARY condition — never delete an instance this
|
|
882
|
+
* run did not create — but 2026-08-07 proved it is not SUFFICIENT. It answers
|
|
883
|
+
* "did I create this?", not "is it safe to destroy?", and in a concurrent
|
|
884
|
+
* provision race the losing run legitimately owns the flag while the winner has
|
|
885
|
+
* already brought the VM up and rostered it. Deleting there destroyed VM-4 and
|
|
886
|
+
* VM-5. So authorization also requires CURRENT-STATE evidence that the VM is
|
|
887
|
+
* out of service:
|
|
888
|
+
*
|
|
889
|
+
* - `containerUp` — the editor container is running (per the compose-ps probe).
|
|
890
|
+
* A VM serving an editor is in use by definition.
|
|
891
|
+
* - `rostered` — the VM is in the fleet roster, the operator-visible source of
|
|
892
|
+
* truth. Note this is true whether the winning run or THIS run rostered it;
|
|
893
|
+
* either way the VM is visible in the dashboard and removable in one command,
|
|
894
|
+
* so it is not the silent orphan teardown exists to prevent.
|
|
895
|
+
* - `probeInconclusive` — liveness could not be determined. This must fail
|
|
896
|
+
* SAFE: a failing probe is how we reach the abort handler in the first place,
|
|
897
|
+
* so it can never count as evidence the VM is dead. An orphan costs one
|
|
898
|
+
* `npm run cloud:destroy`; a wrong delete costs a 15-40 minute rebuild plus
|
|
899
|
+
* any unpushed work on the VM. */
|
|
900
|
+
function abortTeardownDecision({ createdThisRun, containerUp, rostered, probeInconclusive } = {}) {
|
|
901
|
+
if (!createdThisRun) return "keep";
|
|
902
|
+
if (probeInconclusive) return "keep";
|
|
903
|
+
if (containerUp) return "keep";
|
|
904
|
+
if (rostered) return "keep";
|
|
905
|
+
return "delete";
|
|
906
|
+
}
|
|
907
|
+
|
|
908
|
+
/** Gather the CURRENT-STATE inputs {@link abortTeardownDecision} needs, so the
|
|
909
|
+
* abort handler reads gather → decide → act instead of inlining probes among
|
|
910
|
+
* the delete calls.
|
|
911
|
+
*
|
|
912
|
+
* Only runs the probes when `createdThisRun` is true — the decision is `keep`
|
|
913
|
+
* regardless otherwise, and an abort path should not spend SSH round trips
|
|
914
|
+
* proving something it already knows. In that case it returns the
|
|
915
|
+
* probe-inconclusive shape, which is the safe default at every layer.
|
|
916
|
+
*
|
|
917
|
+
* Both lookups are wrapped: a throwing probe or an unreadable roster must fail
|
|
918
|
+
* SAFE (toward `keep`), never take a second exception path into the delete.
|
|
919
|
+
* `deps` is injectable so the matrix is unit-tested without a VM. */
|
|
920
|
+
function probeAbortTeardownState(cfg, createdThisRun, deps = {}) {
|
|
921
|
+
if (!createdThisRun) {
|
|
922
|
+
return { containerUp: false, rostered: false, probeInconclusive: true };
|
|
923
|
+
}
|
|
924
|
+
const probe = deps.probeEditorComposeState || probeEditorComposeState;
|
|
925
|
+
const roster = deps.readRoster || readRoster;
|
|
926
|
+
const resolveVm = deps.resolveFleetVmNumber || resolveFleetVmNumber;
|
|
927
|
+
const log = deps.log || ((m) => console.error(m));
|
|
928
|
+
|
|
929
|
+
let liveness = { state: null, inconclusive: true };
|
|
930
|
+
try {
|
|
931
|
+
liveness = probe(cfg);
|
|
932
|
+
} catch (e) {
|
|
933
|
+
log(` (liveness probe failed: ${e && e.message})`);
|
|
934
|
+
}
|
|
935
|
+
let rostered = false;
|
|
936
|
+
try {
|
|
937
|
+
const vm = resolveVm(cfg.instanceName);
|
|
938
|
+
rostered = vm != null && (roster() || []).includes(vm);
|
|
939
|
+
} catch { /* unreadable roster — treated as not-rostered; liveness still guards */ }
|
|
940
|
+
|
|
941
|
+
return {
|
|
942
|
+
containerUp: liveness.state === "running",
|
|
943
|
+
rostered,
|
|
944
|
+
probeInconclusive: liveness.inconclusive,
|
|
945
|
+
};
|
|
946
|
+
}
|
|
947
|
+
|
|
848
948
|
/** Translate one of the wrapper's verbs into the exact `gcloud` argv. The
|
|
849
949
|
* switch's job is to keep every per-verb command shape in one reviewable
|
|
850
950
|
* place — tests assert the arrays without ever spawning a process. */
|
|
@@ -880,6 +980,19 @@ function buildGcloudArgs(verb, cfg = {}, extras = {}) {
|
|
|
880
980
|
return ["compute", "instances", "delete", instanceName, "--project", project, "--zone", zone, "--quiet"];
|
|
881
981
|
case "instances-describe":
|
|
882
982
|
return ["compute", "instances", "describe", instanceName, "--project", project, "--zone", zone, "--format=json"];
|
|
983
|
+
// Outstanding (not-yet-DONE) zone operations targeting this instance. Used to
|
|
984
|
+
// tell a normal transient PROVISIONING from a wedged insert: GCE can accept an
|
|
985
|
+
// insert and then never finish it, leaving an operation parked at RUNNING/0%
|
|
986
|
+
// that holds the instance name forever. The status filter is what makes this
|
|
987
|
+
// cheap and unambiguous — a DONE operation is history and says nothing about
|
|
988
|
+
// whether the slot is currently blocked.
|
|
989
|
+
case "operations-list-pending":
|
|
990
|
+
return [
|
|
991
|
+
"compute", "operations", "list",
|
|
992
|
+
"--project", project,
|
|
993
|
+
"--filter", `zone:(${zone}) AND targetLink~${instanceName}$ AND status!=DONE`,
|
|
994
|
+
"--format=json",
|
|
995
|
+
];
|
|
883
996
|
case "instances-list-tagged":
|
|
884
997
|
return [
|
|
885
998
|
"compute", "instances", "list",
|
|
@@ -976,8 +1089,77 @@ function buildGcloudArgs(verb, cfg = {}, extras = {}) {
|
|
|
976
1089
|
// reconcile spawnSync sites (status probe, per-BES timeout converge) so all three
|
|
977
1090
|
// reference one value. 2 min is generous for any single describe/update/SSH probe
|
|
978
1091
|
// yet comfortably under the 600000 ms outer reconcile budget.
|
|
1092
|
+
//
|
|
1093
|
+
// It is a PROBE budget, and provisioning is exempt — see
|
|
1094
|
+
// GCLOUD_PROVISION_TIMEOUT_MS below. Do not re-collapse the two: raising this
|
|
1095
|
+
// constant to accommodate a slow create would un-bound the reconcile sweep,
|
|
1096
|
+
// which is the exact orphan-cascade this value exists to stop.
|
|
979
1097
|
const GCLOUD_CALL_TIMEOUT_MS = 120000;
|
|
980
1098
|
|
|
1099
|
+
// `instances create` / `instances start` are NOT probes: each blocks until GCE's
|
|
1100
|
+
// insert (or start) operation reaches a terminal state, which routinely outruns
|
|
1101
|
+
// the probe budget under zonal contention. Inheriting GCLOUD_CALL_TIMEOUT_MS
|
|
1102
|
+
// SIGTERMd gcloud mid-insert; `spawnSync` then returned `status: null`, which
|
|
1103
|
+
// `!== 0`, so a local kill was indistinguishable from a remote rejection and the
|
|
1104
|
+
// caller threw a bare `Failed to create instance <name>` with no cause in it.
|
|
1105
|
+
// That mattered downstream: every arm of classifyAddFailure keys on a transcript
|
|
1106
|
+
// signature, so a causeless log fell through to the optimistic `verifying`
|
|
1107
|
+
// fallback and told the operator to WAIT on a VM GCE had already wedged
|
|
1108
|
+
// (observed 2026-09-04 on VM-8: insert 14:29:50Z, add errored 14:32:26Z — the
|
|
1109
|
+
// 120s budget plus teardown).
|
|
1110
|
+
//
|
|
1111
|
+
// Deliberately above the 600000 ms outer reconcile budget, on the same reasoning
|
|
1112
|
+
// as PROJECT_BOOTSTRAP_TIMEOUT_MS: ensureInstance is only ever reached from the
|
|
1113
|
+
// add path, never from the reconcile sweep, so the sweep-relative ceiling that
|
|
1114
|
+
// governs GCLOUD_CALL_TIMEOUT_MS does not apply. A create still running at 10
|
|
1115
|
+
// minutes is wedged, not slow.
|
|
1116
|
+
const GCLOUD_PROVISION_TIMEOUT_MS = 10 * 60 * 1000;
|
|
1117
|
+
|
|
1118
|
+
/** The per-call spawn budget a given wrapper verb should run under. Provisioning
|
|
1119
|
+
* verbs get GCLOUD_PROVISION_TIMEOUT_MS; everything else keeps the probe budget.
|
|
1120
|
+
* Pure — the mapping is asserted directly, without spawning a process. */
|
|
1121
|
+
function gcloudCallTimeoutMs(verb) {
|
|
1122
|
+
return verb === "instances-create" || verb === "instances-start"
|
|
1123
|
+
? GCLOUD_PROVISION_TIMEOUT_MS
|
|
1124
|
+
: GCLOUD_CALL_TIMEOUT_MS;
|
|
1125
|
+
}
|
|
1126
|
+
|
|
1127
|
+
// The project bootstrap is NOT a probe: it runs `git clone` + `npm install` in
|
|
1128
|
+
// the repo root and every `apps[].dir`, each package's `postinstall`, then
|
|
1129
|
+
// `npm run setup` — minutes of remote work on a heavy repo. It once ran as ONE
|
|
1130
|
+
// blocking `runGcloud("ssh-script")` call, so it inherited GCLOUD_CALL_TIMEOUT_MS
|
|
1131
|
+
// (a budget whose own comment scopes it to "any single describe/update/SSH
|
|
1132
|
+
// probe"). That SIGTERMd the gcloud child mid-clone; `spawnSync` then returned
|
|
1133
|
+
// `status: null`, which `!== 0`, so the timeout was indistinguishable from a
|
|
1134
|
+
// genuine remote failure and the caller's abort path DELETED A COMPLETELY
|
|
1135
|
+
// HEALTHY VM.
|
|
1136
|
+
//
|
|
1137
|
+
// Observed twice: 2026-07-28 on client repo `lets-go-compost-portal` (a root
|
|
1138
|
+
// postinstall of `playwright install chromium`, ~293 MB), and 2026-07-29
|
|
1139
|
+
// provisioning six self-host `codeyam-editor` VMs, whose clone alone exceeds the
|
|
1140
|
+
// probe budget — all six were built, torn down mid-clone, and reported as
|
|
1141
|
+
// "scripts/gcp-project-bootstrap.sh failed on the VM".
|
|
1142
|
+
//
|
|
1143
|
+
// The install is now DETACHED (projectBootstrapLaunchCommand + a re-attaching
|
|
1144
|
+
// follow loop, outcome read from VM-side state), so no single call carries the
|
|
1145
|
+
// whole install any more. This constant is therefore the OVERALL deadline across
|
|
1146
|
+
// re-attaches — the point at which we stop waiting on a still-healthy VM and say
|
|
1147
|
+
// exactly that. Deliberately well above the outer reconcile budget: a long remote
|
|
1148
|
+
// install is not a step in a sweep, so the sweep-relative ceiling that governs
|
|
1149
|
+
// GCLOUD_CALL_TIMEOUT_MS does not apply to it.
|
|
1150
|
+
const PROJECT_BOOTSTRAP_TIMEOUT_MS = 40 * 60 * 1000;
|
|
1151
|
+
|
|
1152
|
+
/** Whether a `spawnSync` result is a LOCAL timeout kill rather than a remote
|
|
1153
|
+
* failure. `spawnSync` reports a timeout as `status: null` plus an ETIMEDOUT
|
|
1154
|
+
* error, so a bare `status !== 0` check conflates the two — which is what made
|
|
1155
|
+
* a timed-out bootstrap look like a broken one. Pure. */
|
|
1156
|
+
function isSpawnTimeout(result) {
|
|
1157
|
+
if (!result) return false;
|
|
1158
|
+
if (result.status !== null && result.status !== undefined) return false;
|
|
1159
|
+
const code = result.error && result.error.code;
|
|
1160
|
+
return code === "ETIMEDOUT" || result.signal === "SIGTERM";
|
|
1161
|
+
}
|
|
1162
|
+
|
|
981
1163
|
function runGcloud(args, opts = {}) {
|
|
982
1164
|
return spawnSync("gcloud", args, {
|
|
983
1165
|
stdio: opts.stdio || "inherit",
|
|
@@ -1014,6 +1196,66 @@ function formatGcloudFailure(result) {
|
|
|
1014
1196
|
return "(no output captured from gcloud)";
|
|
1015
1197
|
}
|
|
1016
1198
|
|
|
1199
|
+
/** The thrown message for a failed `instances create` / `instances start`.
|
|
1200
|
+
*
|
|
1201
|
+
* Splits the two outcomes the old bare `Failed to create instance <name>`
|
|
1202
|
+
* collapsed, because they are different operator actions: a LOCAL timeout means
|
|
1203
|
+
* GCE may still be working the operation and it should be checked, while a
|
|
1204
|
+
* remote rejection means GCE said no and its own stderr is the answer. Both
|
|
1205
|
+
* arms put a cause in the transcript — classifyAddFailure keys on transcript
|
|
1206
|
+
* text, so a causeless message is what lands an add on the optimistic
|
|
1207
|
+
* `verifying` fallback. Pure: result in, string out.
|
|
1208
|
+
*
|
|
1209
|
+
* `verb` is the operator-facing word ("create" / "start"), not a wrapper verb. */
|
|
1210
|
+
/** Operator-facing copy for a create GCE accepted and then abandoned, kept
|
|
1211
|
+
* separate from the guard that detects it for the same reason
|
|
1212
|
+
* provisionFailureMessage is: the decision of WHEN to refuse and the wording of
|
|
1213
|
+
* WHAT to say are different concerns, and only the wording needs asserting.
|
|
1214
|
+
* Extracting it is what lets the copy be tested without a live gcloud — and on
|
|
1215
|
+
* this feature the copy IS the fix, since the bug was a message that told the
|
|
1216
|
+
* operator to wait for a VM that would never arrive.
|
|
1217
|
+
*
|
|
1218
|
+
* Every field is rendered defensively because this runs on a degraded path: the
|
|
1219
|
+
* operations read that produced `pending` is best-effort, so a partial record
|
|
1220
|
+
* must still yield a message that names the problem rather than printing
|
|
1221
|
+
* "undefined" at the operator. */
|
|
1222
|
+
function wedgedOperationMessage(instanceName, pending = {}) {
|
|
1223
|
+
const detail = [
|
|
1224
|
+
`${pending.name || "unnamed"}: ${pending.operationType || "unknown type"}`,
|
|
1225
|
+
`status ${pending.status || "unknown"}`,
|
|
1226
|
+
`progress ${pending.progress == null ? "unknown" : `${pending.progress}%`}`,
|
|
1227
|
+
`error ${pending.errorCode || "unknown"}${pending.errorMessage ? `: ${pending.errorMessage}` : ""}`,
|
|
1228
|
+
].join(", ");
|
|
1229
|
+
return (
|
|
1230
|
+
`Instance ${instanceName} is stuck in PROVISIONING behind a wedged create operation (${detail}). ` +
|
|
1231
|
+
"GCE accepted the create and never finished it, so this slot cannot be re-added and a delete will queue " +
|
|
1232
|
+
"behind the wedged operation rather than clear it. This needs GCE-side resolution — open a Google Cloud " +
|
|
1233
|
+
"Support case citing that error. Use a different VM number if you need a box now."
|
|
1234
|
+
);
|
|
1235
|
+
}
|
|
1236
|
+
|
|
1237
|
+
function provisionFailureMessage(verb, instanceName, result) {
|
|
1238
|
+
if (isSpawnTimeout(result)) {
|
|
1239
|
+
const minutes = Math.round(GCLOUD_PROVISION_TIMEOUT_MS / 60000);
|
|
1240
|
+
// A SIGTERMd child legitimately often captured nothing, and appending
|
|
1241
|
+
// formatGcloudFailure's "(no output captured)" placeholder to a message that
|
|
1242
|
+
// already states the cause is noise. Append it only when it is real output.
|
|
1243
|
+
const detail = formatGcloudFailure(result);
|
|
1244
|
+
const captured = detail === "(no output captured from gcloud)" ? "" : `\n${detail}`;
|
|
1245
|
+
return (
|
|
1246
|
+
`Failed to ${verb} instance ${instanceName}: the gcloud call exceeded the local ` +
|
|
1247
|
+
`${minutes}-minute provisioning budget and was SIGTERMd locally. GCE may still be ` +
|
|
1248
|
+
`working the operation — check it with \`gcloud compute operations list\` before ` +
|
|
1249
|
+
`retrying, and delete the half-built instance if one exists.` +
|
|
1250
|
+
captured
|
|
1251
|
+
);
|
|
1252
|
+
}
|
|
1253
|
+
return (
|
|
1254
|
+
`Failed to ${verb} instance ${instanceName} (gcloud exit ${result && result.status}).\n` +
|
|
1255
|
+
formatGcloudFailure(result)
|
|
1256
|
+
);
|
|
1257
|
+
}
|
|
1258
|
+
|
|
1017
1259
|
// A bounded, pure classifier that keys on status === 255 (gcloud-ssh's exit
|
|
1018
1260
|
// for an ssh failure) and a signature match against known transient strings.
|
|
1019
1261
|
function isTransientSshFailure(result) {
|
|
@@ -1089,6 +1331,38 @@ function fetchInstanceState(cfg) {
|
|
|
1089
1331
|
}
|
|
1090
1332
|
}
|
|
1091
1333
|
|
|
1334
|
+
/** Read the instance's outstanding (not-DONE) zone operations and summarize the
|
|
1335
|
+
* first one, or null when there are none / the call failed. Shaped as
|
|
1336
|
+
* { name, operationType, status, progress, errorCode, errorMessage } — the fields
|
|
1337
|
+
* the wedged-insert refusal message and the fleet classifier both need.
|
|
1338
|
+
*
|
|
1339
|
+
* Returning null on a failed or unparseable call is deliberate: this is a
|
|
1340
|
+
* DIAGNOSTIC read used to make a refusal more informative, so it must never turn
|
|
1341
|
+
* a working provision into a failing one just because the operations API hiccuped. */
|
|
1342
|
+
function fetchPendingOperation(cfg) {
|
|
1343
|
+
const result = runGcloud(buildGcloudArgs("operations-list-pending", cfg), {
|
|
1344
|
+
stdio: ["ignore", "pipe", "ignore"],
|
|
1345
|
+
});
|
|
1346
|
+
if (result.status !== 0) return null;
|
|
1347
|
+
let ops;
|
|
1348
|
+
try {
|
|
1349
|
+
ops = JSON.parse(result.stdout);
|
|
1350
|
+
} catch {
|
|
1351
|
+
return null;
|
|
1352
|
+
}
|
|
1353
|
+
if (!Array.isArray(ops) || ops.length === 0) return null;
|
|
1354
|
+
const op = ops[0];
|
|
1355
|
+
const err = op && op.error && Array.isArray(op.error.errors) ? op.error.errors[0] : null;
|
|
1356
|
+
return {
|
|
1357
|
+
name: op.name || null,
|
|
1358
|
+
operationType: op.operationType || null,
|
|
1359
|
+
status: op.status || null,
|
|
1360
|
+
progress: typeof op.progress === "number" ? op.progress : null,
|
|
1361
|
+
errorCode: (err && err.code) || null,
|
|
1362
|
+
errorMessage: (err && err.message) || null,
|
|
1363
|
+
};
|
|
1364
|
+
}
|
|
1365
|
+
|
|
1092
1366
|
function loadOrInitCloudConfig(projectDir, flags) {
|
|
1093
1367
|
const existing = loadCloudConfig(projectDir);
|
|
1094
1368
|
// noTunnel is a per-invocation behavior, never persisted — else a later manual
|
|
@@ -1170,7 +1444,11 @@ async function ensureFirewallRule(cfg) {
|
|
|
1170
1444
|
}
|
|
1171
1445
|
}
|
|
1172
1446
|
|
|
1173
|
-
|
|
1447
|
+
// `sleepFn` is injectable so the 60-iteration wait loop can be driven with a
|
|
1448
|
+
// no-op instead of burning 2 real minutes of clock — the same seam
|
|
1449
|
+
// runGcloudSshWithRetry / waitForEditorComposeRunning already expose.
|
|
1450
|
+
async function ensureInstance(cfg, { sleepFn } = {}) {
|
|
1451
|
+
const sleep = sleepFn || ((ms) => new Promise((r) => setTimeout(r, ms)));
|
|
1174
1452
|
console.error(`Checking instance ${cfg.instanceName}...`);
|
|
1175
1453
|
const initialState = fetchInstanceState(cfg);
|
|
1176
1454
|
const createdThisRun = initialState === null;
|
|
@@ -1181,29 +1459,59 @@ async function ensureInstance(cfg) {
|
|
|
1181
1459
|
}
|
|
1182
1460
|
if (initialState === "TERMINATED") {
|
|
1183
1461
|
console.error(` ${cfg.instanceName} is TERMINATED — starting...`);
|
|
1184
|
-
const start = runGcloud(buildGcloudArgs("instances-start", cfg)
|
|
1462
|
+
const start = runGcloud(buildGcloudArgs("instances-start", cfg), {
|
|
1463
|
+
timeout: gcloudCallTimeoutMs("instances-start"),
|
|
1464
|
+
});
|
|
1185
1465
|
if (start.status !== 0) {
|
|
1186
|
-
throw new Error(
|
|
1466
|
+
throw new Error(provisionFailureMessage("start", cfg.instanceName, start));
|
|
1187
1467
|
}
|
|
1188
1468
|
} else if (initialState === null) {
|
|
1189
1469
|
console.error(` ${cfg.instanceName} not found — creating...`);
|
|
1190
|
-
const create = runGcloud(buildGcloudArgs("instances-create", cfg)
|
|
1470
|
+
const create = runGcloud(buildGcloudArgs("instances-create", cfg), {
|
|
1471
|
+
timeout: gcloudCallTimeoutMs("instances-create"),
|
|
1472
|
+
});
|
|
1191
1473
|
if (create.status !== 0) {
|
|
1192
|
-
throw new Error(
|
|
1474
|
+
throw new Error(provisionFailureMessage("create", cfg.instanceName, create));
|
|
1193
1475
|
}
|
|
1194
1476
|
} else {
|
|
1477
|
+
// A slot found in PROVISIONING is ambiguous, and the two readings need
|
|
1478
|
+
// opposite handling. On a normal add it is a transient the poll loop below
|
|
1479
|
+
// resolves in seconds. But GCE can also accept an insert and then never
|
|
1480
|
+
// finish it (2026-09-04, VM-8): the operation parks at RUNNING/0%, eventually
|
|
1481
|
+
// attaches an INTERNAL_ERROR without ever reaching DONE, and holds the
|
|
1482
|
+
// instance name indefinitely — a delete queues behind it rather than clearing
|
|
1483
|
+
// it. Polling that for 2 minutes only rediscovers, slowly and with a message
|
|
1484
|
+
// naming neither the operation nor the wedge, a state the first describe
|
|
1485
|
+
// already showed. Read the outstanding operation instead and fail immediately
|
|
1486
|
+
// when it carries a terminal error, so the transcript carries the signature
|
|
1487
|
+
// fleet-add-failure.js's wedged-insert arm keys on.
|
|
1488
|
+
if (initialState === "PROVISIONING") {
|
|
1489
|
+
const pending = fetchPendingOperation(cfg);
|
|
1490
|
+
if (pending && pending.errorCode) {
|
|
1491
|
+
throw new Error(wedgedOperationMessage(cfg.instanceName, pending));
|
|
1492
|
+
}
|
|
1493
|
+
}
|
|
1195
1494
|
console.error(` ${cfg.instanceName} is ${initialState}; waiting...`);
|
|
1196
1495
|
}
|
|
1197
1496
|
|
|
1497
|
+
let lastState = initialState;
|
|
1198
1498
|
for (let i = 0; i < 60; i++) {
|
|
1199
|
-
|
|
1499
|
+
lastState = fetchInstanceState(cfg);
|
|
1500
|
+
if (lastState === "RUNNING") {
|
|
1200
1501
|
console.error(` ✓ ${cfg.instanceName} is RUNNING`);
|
|
1201
1502
|
await waitForIapReady(cfg);
|
|
1202
1503
|
return { createdThisRun };
|
|
1203
1504
|
}
|
|
1204
|
-
await
|
|
1505
|
+
await sleep(2000);
|
|
1205
1506
|
}
|
|
1206
|
-
|
|
1507
|
+
// Name the state it was actually stuck in. An instance parked in PROVISIONING
|
|
1508
|
+
// is a different operator action from one that stopped answering describe at
|
|
1509
|
+
// all, and the old message ("did not reach RUNNING") distinguished neither.
|
|
1510
|
+
throw new Error(
|
|
1511
|
+
`Instance ${cfg.instanceName} did not reach RUNNING within 2 minutes ` +
|
|
1512
|
+
`(last observed state: ${lastState || "unknown — describe returned no state"}` +
|
|
1513
|
+
`${createdThisRun ? "; a create was issued this run" : ""}).`,
|
|
1514
|
+
);
|
|
1207
1515
|
}
|
|
1208
1516
|
|
|
1209
1517
|
/** GCE reports the instance as RUNNING the moment the boot disk is attached,
|
|
@@ -1586,6 +1894,282 @@ function bootstrapLogTail(cfg, lines = 40) {
|
|
|
1586
1894
|
return String(r.stdout || "").trim() || "(log unavailable)";
|
|
1587
1895
|
}
|
|
1588
1896
|
|
|
1897
|
+
// ─── Detached PROJECT bootstrap ──────────────────────────────────────────────
|
|
1898
|
+
//
|
|
1899
|
+
// The VM bootstrap above solved a problem the PROJECT bootstrap still had: read
|
|
1900
|
+
// success and failure from VM-SIDE STATE, never from the SSH exit code, so a
|
|
1901
|
+
// dropped follow-session over a healthy VM is not misreported as "failed on the
|
|
1902
|
+
// VM". The project bootstrap ran instead as ONE blocking ssh-script call, so a
|
|
1903
|
+
// local budget elapsing (or an SSH pipe breaking) was indistinguishable from a
|
|
1904
|
+
// genuine remote failure and the caller's abort path DELETED A HEALTHY VM.
|
|
1905
|
+
// Raising the budget alone just moves the cliff; these helpers remove it by
|
|
1906
|
+
// mirroring the VM-bootstrap contract — detached launch, re-attaching follow
|
|
1907
|
+
// loop, VM-side outcome probe.
|
|
1908
|
+
//
|
|
1909
|
+
// All VM-side paths are keyed by repo name so two projects on one VM never
|
|
1910
|
+
// collide, and they sit beside the `.done` marker projectBootstrapDoneMarker
|
|
1911
|
+
// already defines. Pure string builders (like their VM-bootstrap counterparts)
|
|
1912
|
+
// so the whole grammar is unit-testable without a live VM.
|
|
1913
|
+
function projectBootstrapScriptPath(repoName) {
|
|
1914
|
+
return `$HOME/.codeyam-project-bootstrap-${repoName}.sh`;
|
|
1915
|
+
}
|
|
1916
|
+
function projectBootstrapRunnerPath(repoName) {
|
|
1917
|
+
return `$HOME/.codeyam-project-bootstrap-${repoName}-run.sh`;
|
|
1918
|
+
}
|
|
1919
|
+
function projectBootstrapLogPath(repoName) {
|
|
1920
|
+
return `$HOME/.codeyam-project-bootstrap-${repoName}.log`;
|
|
1921
|
+
}
|
|
1922
|
+
function projectBootstrapPidPath(repoName) {
|
|
1923
|
+
return `$HOME/.codeyam-project-bootstrap-${repoName}.pid`;
|
|
1924
|
+
}
|
|
1925
|
+
// The runner's recorded EXIT STATUS. Deliberately `.status`, NOT `.done`:
|
|
1926
|
+
// `.done` is already taken by scripts/gcp-project-bootstrap.sh's own terminal
|
|
1927
|
+
// SUCCESS marker (a timestamp, written only after `set -euo pipefail` carried
|
|
1928
|
+
// every step through — see projectBootstrapDoneMarker). Reusing `.done` for an
|
|
1929
|
+
// exit code would overwrite the success signal that both improve40's false-skip
|
|
1930
|
+
// fix and the follow loop below depend on. Two files, two distinct meanings:
|
|
1931
|
+
// `.done` = "the script reached its last line", `.status` = "the runner exited,
|
|
1932
|
+
// here is its code".
|
|
1933
|
+
function projectBootstrapStatusPath(repoName) {
|
|
1934
|
+
return `$HOME/.codeyam-project-bootstrap-${repoName}.status`;
|
|
1935
|
+
}
|
|
1936
|
+
const PROJECT_BOOTSTRAP_LAUNCHED_MARKER = "CODEYAM_PROJECT_BOOTSTRAP_LAUNCHED";
|
|
1937
|
+
const PROJECT_BOOTSTRAP_STATUS_MARKER = "CYPROJ";
|
|
1938
|
+
// One follow session caps at 15 min; the overall wait caps at
|
|
1939
|
+
// PROJECT_BOOTSTRAP_TIMEOUT_MS across re-attaches before we bail loudly with
|
|
1940
|
+
// the VM-side log location. Unlike the VM-bootstrap follow loop — whose follow
|
|
1941
|
+
// calls inherit the 120 s probe budget and so re-attach every two minutes over
|
|
1942
|
+
// a ~14-min build — the project follow call below is given an explicit local
|
|
1943
|
+
// budget just above this remote deadline, so a long install streams in one
|
|
1944
|
+
// session instead of churning re-attaches.
|
|
1945
|
+
const PROJECT_BOOTSTRAP_FOLLOW_SECS = 900;
|
|
1946
|
+
const PROJECT_BOOTSTRAP_FOLLOW_TIMEOUT_MS = (PROJECT_BOOTSTRAP_FOLLOW_SECS + 60) * 1000;
|
|
1947
|
+
|
|
1948
|
+
// The tiny VM-side wrapper the detached launch execs: run the real bootstrap,
|
|
1949
|
+
// redirect all output to the per-repo log, and record the exit status in the
|
|
1950
|
+
// `.status` sentinel — the single source of truth for "did the runner exit, and
|
|
1951
|
+
// with what?" once the SSH follow-session is gone. Contains NO secrets (tokens
|
|
1952
|
+
// arrive via the launch env prefix, never written to disk).
|
|
1953
|
+
function projectBootstrapRunnerScript(repoName) {
|
|
1954
|
+
return [
|
|
1955
|
+
"#!/usr/bin/env bash",
|
|
1956
|
+
"# Detached project-bootstrap runner (written by npm/cloud.js",
|
|
1957
|
+
"# ensureProjectBootstrapped). Survives SSH disconnect: the laptop only",
|
|
1958
|
+
"# *follows* the log below; this process keeps running under its own session",
|
|
1959
|
+
"# if the follow drops. Records its own PID so the status probe can check",
|
|
1960
|
+
"# liveness via `kill -0` (a `pgrep` on the script path would also match the",
|
|
1961
|
+
"# probe command itself).",
|
|
1962
|
+
`echo "$$" > "${projectBootstrapPidPath(repoName)}"`,
|
|
1963
|
+
`bash "${projectBootstrapScriptPath(repoName)}" "$@" > "${projectBootstrapLogPath(repoName)}" 2>&1`,
|
|
1964
|
+
"status=$?",
|
|
1965
|
+
`echo "$status" > "${projectBootstrapStatusPath(repoName)}"`,
|
|
1966
|
+
`rm -f "${projectBootstrapPidPath(repoName)}"`,
|
|
1967
|
+
"",
|
|
1968
|
+
].join("\n");
|
|
1969
|
+
}
|
|
1970
|
+
|
|
1971
|
+
// Build the remote command that (1) materializes the bootstrap script + runner
|
|
1972
|
+
// on the VM, (2) clears the stale sentinels, (3) launches the runner DETACHED
|
|
1973
|
+
// (`nohup setsid`, new session, stdio to /dev/null) so it outlives the SSH
|
|
1974
|
+
// session, and (4) prints a launched marker. The detach is the durable fix: the
|
|
1975
|
+
// install no longer dies when the laptop's SSH pipe breaks, and the VM is no
|
|
1976
|
+
// longer torn down because it did.
|
|
1977
|
+
//
|
|
1978
|
+
// BOTH sentinels are cleared, `.status` and `.done`. Clearing `.done` is what
|
|
1979
|
+
// makes the outcome probe honest: after a launch, a present `.done` can only
|
|
1980
|
+
// mean THIS run's script reached its terminal line, never a leftover from a
|
|
1981
|
+
// previous half-wired attempt.
|
|
1982
|
+
function projectBootstrapLaunchCommand(scriptBody, scriptArgs = [], envVars = {}, repoName, libFiles = {}) {
|
|
1983
|
+
const scriptB64 = Buffer.from(scriptBody, "utf8").toString("base64");
|
|
1984
|
+
const runnerB64 = Buffer.from(projectBootstrapRunnerScript(repoName), "utf8").toString("base64");
|
|
1985
|
+
const argsString = scriptArgs.map(shQuote).join(" ");
|
|
1986
|
+
const envPrefix = envAssignmentPrefix(envVars);
|
|
1987
|
+
// The bootstrap script is written STANDALONE to $HOME, so any `source`/`.` it
|
|
1988
|
+
// does relative to its own directory resolves against $HOME — not against the
|
|
1989
|
+
// repo layout it was authored in. `libFiles` carries those siblings, keyed by
|
|
1990
|
+
// the $HOME-relative path the script will look for them at (e.g.
|
|
1991
|
+
// "lib/npm-pin.sh"). Their directories do not exist on a fresh VM, hence the
|
|
1992
|
+
// mkdir. Omitting a sourced library is not a soft failure: the script aborts
|
|
1993
|
+
// under `set -euo pipefail` and cloud:up tears the new VM down, which is the
|
|
1994
|
+
// 2026-09-04 fleet-wide add outage.
|
|
1995
|
+
const libWrites = Object.entries(libFiles).flatMap(([relPath, body]) => {
|
|
1996
|
+
const b64 = Buffer.from(body, "utf8").toString("base64");
|
|
1997
|
+
const dir = path.posix.dirname(relPath);
|
|
1998
|
+
const mkdir = dir && dir !== "." ? [`mkdir -p $HOME/${dir}`] : [];
|
|
1999
|
+
return [...mkdir, `echo '${b64}' | base64 -d > $HOME/${relPath}`];
|
|
2000
|
+
});
|
|
2001
|
+
// Writes run synchronously (fast, before any backgrounding) so a decode error
|
|
2002
|
+
// aborts before launch; then the runner is detached. The env prefix is on the
|
|
2003
|
+
// setsid command so the tokens are inherited through nohup→setsid→bash but
|
|
2004
|
+
// never appear in any argv. `& echo MARKER` (no `;` before echo) keeps the
|
|
2005
|
+
// launch in the background while still confirming it fired.
|
|
2006
|
+
const writes = [
|
|
2007
|
+
...libWrites,
|
|
2008
|
+
`echo '${scriptB64}' | base64 -d > ${projectBootstrapScriptPath(repoName)}`,
|
|
2009
|
+
`echo '${runnerB64}' | base64 -d > ${projectBootstrapRunnerPath(repoName)}`,
|
|
2010
|
+
`rm -f ${projectBootstrapStatusPath(repoName)} ${projectBootstrapDoneMarker(repoName)}`,
|
|
2011
|
+
].join(" && ");
|
|
2012
|
+
const launch =
|
|
2013
|
+
`${envPrefix}nohup setsid bash ${projectBootstrapRunnerPath(repoName)} ${argsString}`.trimEnd() +
|
|
2014
|
+
` </dev/null >/dev/null 2>&1 &`;
|
|
2015
|
+
return `${writes}; ${launch} echo ${PROJECT_BOOTSTRAP_LAUNCHED_MARKER}`;
|
|
2016
|
+
}
|
|
2017
|
+
|
|
2018
|
+
// Build the remote command that FOLLOWS the detached install: tail the log live
|
|
2019
|
+
// for the operator, polling for the `.status` sentinel, and exit when the runner
|
|
2020
|
+
// finishes OR a per-session deadline elapses. If the SSH session drops
|
|
2021
|
+
// mid-follow this command dies but the detached install keeps going — the caller
|
|
2022
|
+
// then re-attaches.
|
|
2023
|
+
function projectBootstrapFollowCommand(repoName, deadlineSecs = PROJECT_BOOTSTRAP_FOLLOW_SECS) {
|
|
2024
|
+
const log = projectBootstrapLogPath(repoName);
|
|
2025
|
+
const status = projectBootstrapStatusPath(repoName);
|
|
2026
|
+
return [
|
|
2027
|
+
`touch ${log}`,
|
|
2028
|
+
`tail -n +1 -F ${log} 2>/dev/null & TP=$!`,
|
|
2029
|
+
`waited=0`,
|
|
2030
|
+
`while [ ! -f ${status} ]; do ` +
|
|
2031
|
+
`sleep 3; waited=$((waited+3)); ` +
|
|
2032
|
+
`if [ "$waited" -ge ${deadlineSecs} ]; then kill "$TP" 2>/dev/null; exit 0; fi; ` +
|
|
2033
|
+
`done`,
|
|
2034
|
+
`sleep 1`,
|
|
2035
|
+
`kill "$TP" 2>/dev/null`,
|
|
2036
|
+
].join("; ");
|
|
2037
|
+
}
|
|
2038
|
+
|
|
2039
|
+
// Build the remote command that reports terminal VM state in one parseable
|
|
2040
|
+
// line: whether the script's terminal `.done` marker exists, the runner's
|
|
2041
|
+
// recorded exit status (or `none`), and whether the runner is still alive.
|
|
2042
|
+
// Cheap — runs over ssh-probe.
|
|
2043
|
+
function projectBootstrapStatusCommand(repoName) {
|
|
2044
|
+
const pid = projectBootstrapPidPath(repoName);
|
|
2045
|
+
const status = projectBootstrapStatusPath(repoName);
|
|
2046
|
+
// Liveness via the runner's PID file + `kill -0`, NOT pgrep: the status
|
|
2047
|
+
// command's own argv contains the bootstrap path, so a `pgrep -f` on it would
|
|
2048
|
+
// match this very probe and always report running. A stale PID file from a
|
|
2049
|
+
// killed runner reads as not-running because `kill -0` fails.
|
|
2050
|
+
return [
|
|
2051
|
+
`done=no; [ -f ${projectBootstrapDoneMarker(repoName)} ] && done=yes`,
|
|
2052
|
+
`st=none; [ -f ${status} ] && st=$(cat ${status} 2>/dev/null)`,
|
|
2053
|
+
`run=no; if [ -f ${pid} ]; then p=$(cat ${pid} 2>/dev/null); ` +
|
|
2054
|
+
`{ [ -n "$p" ] && kill -0 "$p" 2>/dev/null; } && run=yes; fi`,
|
|
2055
|
+
`echo "${PROJECT_BOOTSTRAP_STATUS_MARKER} done=$done status=$st running=$run"`,
|
|
2056
|
+
].join("; ");
|
|
2057
|
+
}
|
|
2058
|
+
|
|
2059
|
+
// Parse the single status line emitted by projectBootstrapStatusCommand.
|
|
2060
|
+
// Returns `{ done, status, running }` or null when no status line is present
|
|
2061
|
+
// (e.g. the probe ssh itself failed and produced no stdout).
|
|
2062
|
+
function parseProjectBootstrapStatus(stdout) {
|
|
2063
|
+
const m = String(stdout || "").match(
|
|
2064
|
+
new RegExp(`${PROJECT_BOOTSTRAP_STATUS_MARKER} done=(\\S+) status=(\\S+) running=(\\S+)`),
|
|
2065
|
+
);
|
|
2066
|
+
if (!m) return null;
|
|
2067
|
+
return { done: m[1], status: m[2], running: m[3] };
|
|
2068
|
+
}
|
|
2069
|
+
|
|
2070
|
+
// Decide the project-bootstrap outcome from VM-side state alone — never the SSH
|
|
2071
|
+
// exit code. The `.done` marker is the real artifact (the script writes it only
|
|
2072
|
+
// after `set -euo pipefail` carried every step to success), so its presence is
|
|
2073
|
+
// success regardless of how the follow session ended — the same reasoning that
|
|
2074
|
+
// makes the docker image authoritative for the VM bootstrap. A live runner means
|
|
2075
|
+
// "re-attach". A recorded exit status without the marker means the install ran
|
|
2076
|
+
// and failed. Nothing at all means it was never started (or the probe could not
|
|
2077
|
+
// see it).
|
|
2078
|
+
function decideProjectBootstrapOutcome(status) {
|
|
2079
|
+
if (!status) return "absent";
|
|
2080
|
+
const { done, status: exitStatus, running } = status;
|
|
2081
|
+
if (done === "yes") return "success";
|
|
2082
|
+
if (running === "yes") return "running";
|
|
2083
|
+
if (exitStatus && exitStatus !== "none") return "failed";
|
|
2084
|
+
return "absent";
|
|
2085
|
+
}
|
|
2086
|
+
|
|
2087
|
+
// Reduce one follow-loop iteration to an action: "success" | "failed" |
|
|
2088
|
+
// "timeout" | "reattach". Extracted as a pure function because its PRECEDENCE is
|
|
2089
|
+
// load-bearing — it decides whether an operator is told a VM is broken:
|
|
2090
|
+
//
|
|
2091
|
+
// • success and failed outrank the deadline. Otherwise the final poll of a
|
|
2092
|
+
// slow-but-successful bootstrap reports a timeout, and the operator is sent
|
|
2093
|
+
// to reconcile a VM that actually finished.
|
|
2094
|
+
// • a still-running install past the deadline is a TIMEOUT, never a failure.
|
|
2095
|
+
// That is the VM-8 case: healthy VM, still installing, and saying so is what
|
|
2096
|
+
// gets the add classified retryable instead of destroyed.
|
|
2097
|
+
// • an unseen runner ("absent") before the deadline re-attaches rather than
|
|
2098
|
+
// failing — "I could not see it" is not "it broke" — but is still bounded by
|
|
2099
|
+
// the deadline so the loop cannot spin forever.
|
|
2100
|
+
function decideProjectBootstrapFollowAction({ outcome, now, deadline } = {}) {
|
|
2101
|
+
if (outcome === "success") return "success";
|
|
2102
|
+
if (outcome === "failed") return "failed";
|
|
2103
|
+
if (now >= deadline) return "timeout";
|
|
2104
|
+
return "reattach";
|
|
2105
|
+
}
|
|
2106
|
+
|
|
2107
|
+
// Probe the VM for the detached project bootstrap's terminal state and reduce it
|
|
2108
|
+
// to a single outcome string ("success" | "failed" | "running" | "absent").
|
|
2109
|
+
// Used both for the idempotent re-attach decision and after each follow session.
|
|
2110
|
+
function probeProjectBootstrapOutcome(cfg, repoName) {
|
|
2111
|
+
const r = runGcloud(
|
|
2112
|
+
buildGcloudArgs("ssh-probe", cfg, { command: projectBootstrapStatusCommand(repoName) }),
|
|
2113
|
+
{ stdio: ["ignore", "pipe", "pipe"] },
|
|
2114
|
+
);
|
|
2115
|
+
return decideProjectBootstrapOutcome(parseProjectBootstrapStatus(r.stdout));
|
|
2116
|
+
}
|
|
2117
|
+
|
|
2118
|
+
/** Assemble the env forwarded into the project-bootstrap shell on the VM.
|
|
2119
|
+
* Extracted from ensureProjectBootstrapped so its CONDITIONAL keys are testable
|
|
2120
|
+
* without a VM — previously you could not assert "FLEET_DOMAIN is omitted for a
|
|
2121
|
+
* non-fleet project" or "a re-provision without --admin stays non-admin"
|
|
2122
|
+
* without invoking the whole gcloud-touching bootstrap.
|
|
2123
|
+
*
|
|
2124
|
+
* Keys, and why each is here:
|
|
2125
|
+
* GITHUB_TOKEN — clone a private client repo.
|
|
2126
|
+
* CY_GIT_USER_{NAME,EMAIL} — rendered into the compose file, then written
|
|
2127
|
+
* to `git config --global` by the entrypoint so
|
|
2128
|
+
* plan_claim_refs::create_claim's commit-tree
|
|
2129
|
+
* call succeeds. Empty is legitimate (a CI
|
|
2130
|
+
* runner with no author); the entrypoint warns
|
|
2131
|
+
* and proceeds, so the keys stay present.
|
|
2132
|
+
* FLEET_DOMAIN — ONLY for a numbered fleet VM with a domain
|
|
2133
|
+
* configured, so the compose render publishes
|
|
2134
|
+
* on 0.0.0.0 for the LB NEG. Omitted entirely
|
|
2135
|
+
* otherwise, preserving the loopback-only
|
|
2136
|
+
* publish a plain client project expects.
|
|
2137
|
+
* <forwarded launch-form keys> — DATABASE_URL et al, whitelisted by the
|
|
2138
|
+
* dashboard via CY_FORWARD_ENV_KEYS, so the
|
|
2139
|
+
* bootstrap-time `npm run setup` sees them on
|
|
2140
|
+
* the first try. Empty on plain `cloud:up`.
|
|
2141
|
+
* CODEYAM_ADMIN — the Home-tab Setup door. Defaults to the
|
|
2142
|
+
* strict non-admin "".
|
|
2143
|
+
*
|
|
2144
|
+
* Later spreads win, matching the original inline assembly exactly. `deps` lets
|
|
2145
|
+
* a test supply the token and git identity instead of reading the real
|
|
2146
|
+
* environment and shelling out to git. */
|
|
2147
|
+
function projectBootstrapEnv(cfg = {}, { env = process.env, githubToken, gitIdentity } = {}) {
|
|
2148
|
+
const git = gitIdentity || resolveLaptopGitIdentity();
|
|
2149
|
+
const out = {
|
|
2150
|
+
GITHUB_TOKEN: githubToken !== undefined ? githubToken : resolveGithubToken(),
|
|
2151
|
+
CY_GIT_USER_NAME: git.name,
|
|
2152
|
+
CY_GIT_USER_EMAIL: git.email,
|
|
2153
|
+
};
|
|
2154
|
+
const ingressPlan = ingressWiringPlan(cfg, env);
|
|
2155
|
+
if (ingressPlan.wire) out.FLEET_DOMAIN = ingressPlan.domain;
|
|
2156
|
+
Object.assign(out, resolveSetupForwardEnv(env));
|
|
2157
|
+
Object.assign(out, buildProvisionEnv(env, cfg));
|
|
2158
|
+
return out;
|
|
2159
|
+
}
|
|
2160
|
+
|
|
2161
|
+
// Fetch the tail of the VM-side project-bootstrap log for an actionable failure
|
|
2162
|
+
// message — the operator sees WHY the install failed, not a bare exit code.
|
|
2163
|
+
function projectBootstrapLogTail(cfg, repoName, lines = 40) {
|
|
2164
|
+
const r = runGcloud(
|
|
2165
|
+
buildGcloudArgs("ssh-probe", cfg, {
|
|
2166
|
+
command: `tail -n ${lines} ${projectBootstrapLogPath(repoName)} 2>/dev/null`,
|
|
2167
|
+
}),
|
|
2168
|
+
{ stdio: ["ignore", "pipe", "pipe"] },
|
|
2169
|
+
);
|
|
2170
|
+
return String(r.stdout || "").trim() || "(log unavailable)";
|
|
2171
|
+
}
|
|
2172
|
+
|
|
1589
2173
|
/** Assemble the extra provision-time env forwarded into the bootstrap shell,
|
|
1590
2174
|
* where it feeds the compose-render substitution. Currently just the admin
|
|
1591
2175
|
* toggle: CODEYAM_ADMIN gates the Home-tab Setup door and is read from the
|
|
@@ -1623,7 +2207,7 @@ function buildReconcileAdminCommand(repoName, value) {
|
|
|
1623
2207
|
);
|
|
1624
2208
|
}
|
|
1625
2209
|
|
|
1626
|
-
async function ensureProjectBootstrapped(cfg) {
|
|
2210
|
+
async function ensureProjectBootstrapped(cfg, { sleepFn } = {}) {
|
|
1627
2211
|
const repoName = resolveProject(cfg).name;
|
|
1628
2212
|
console.error(`Checking project bootstrap state for ${repoName}...`);
|
|
1629
2213
|
// Gate on the TERMINAL completion marker the script writes as its very last
|
|
@@ -1652,50 +2236,162 @@ async function ensureProjectBootstrapped(cfg) {
|
|
|
1652
2236
|
console.error(` streaming scripts/gcp-project-bootstrap.sh to VM for ${cfg.repo}...`);
|
|
1653
2237
|
const scriptPath = path.join(rootDir, "scripts", "gcp-project-bootstrap.sh");
|
|
1654
2238
|
const scriptBody = readStreamedScript(scriptPath, cfg);
|
|
2239
|
+
// gcp-project-bootstrap.sh sources lib/npm-pin.sh relative to its own
|
|
2240
|
+
// directory. Read it through readStreamedScript too, so the library is subject
|
|
2241
|
+
// to the same drift guard as the script that sources it — streaming a reviewed
|
|
2242
|
+
// remote script beside a stale on-disk library would defeat the guard.
|
|
2243
|
+
const bootstrapLibs = {
|
|
2244
|
+
"lib/npm-pin.sh": readStreamedScript(path.join(rootDir, "scripts", "lib", "npm-pin.sh"), cfg),
|
|
2245
|
+
};
|
|
1655
2246
|
const scriptArgs = clientProjectBootstrapArgs(cfg);
|
|
1656
|
-
//
|
|
1657
|
-
//
|
|
1658
|
-
//
|
|
1659
|
-
|
|
1660
|
-
|
|
1661
|
-
|
|
1662
|
-
//
|
|
1663
|
-
//
|
|
1664
|
-
//
|
|
1665
|
-
//
|
|
1666
|
-
// the
|
|
1667
|
-
//
|
|
1668
|
-
//
|
|
1669
|
-
//
|
|
1670
|
-
|
|
1671
|
-
//
|
|
1672
|
-
//
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
|
|
1676
|
-
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
|
|
1681
|
-
|
|
1682
|
-
|
|
1683
|
-
|
|
1684
|
-
|
|
1685
|
-
|
|
1686
|
-
|
|
1687
|
-
|
|
1688
|
-
|
|
1689
|
-
|
|
1690
|
-
|
|
1691
|
-
|
|
1692
|
-
|
|
1693
|
-
|
|
1694
|
-
|
|
2247
|
+
// Git identity, forwarded launch-form keys, fleet-domain ingress and the admin
|
|
2248
|
+
// toggle — see projectBootstrapEnv for what each key is for and when it is
|
|
2249
|
+
// omitted. Extracted so those conditionals are testable without a VM.
|
|
2250
|
+
const bootstrapEnv = projectBootstrapEnv(cfg);
|
|
2251
|
+
|
|
2252
|
+
// Idempotent re-attach: a prior cloud:up whose follow-session dropped may have
|
|
2253
|
+
// left a detached install still running. Converge on it instead of starting a
|
|
2254
|
+
// second `npm install` on top of the first.
|
|
2255
|
+
//
|
|
2256
|
+
// NOTE the deliberate asymmetry with ensureBootstrapped, which DOES return
|
|
2257
|
+
// early on an outcome of "success". There, the done-probe and the outcome probe
|
|
2258
|
+
// read the same artifact (the docker image), so they always agree. Here the
|
|
2259
|
+
// done-probe above is a strictly STRONGER three-way AND (`.done` marker +
|
|
2260
|
+
// ~/projects/<repo>/.codeyam + docker-compose.cloud.yml) than this probe's
|
|
2261
|
+
// `.done`-alone success, and it has already said no. Honoring a bare `.done`
|
|
2262
|
+
// here would re-open the improve40 VM-7 false-skip from the other side, so only
|
|
2263
|
+
// "running" short-circuits the launch.
|
|
2264
|
+
const initial = probeProjectBootstrapOutcome(cfg, repoName);
|
|
2265
|
+
if (initial === "running") {
|
|
2266
|
+
console.error(" ↺ a detached project bootstrap is already in progress — re-attaching to its log...");
|
|
2267
|
+
} else {
|
|
2268
|
+
// "absent" (never started), "failed" (a prior run died), or "success" (a bare
|
|
2269
|
+
// marker the stricter done-probe rejected) → (re)launch a fresh detached
|
|
2270
|
+
// install that will survive an SSH disconnect. Every step of
|
|
2271
|
+
// gcp-project-bootstrap.sh is skip-on-exists, so re-entering is cheap.
|
|
2272
|
+
console.error(" launching detached project bootstrap on the VM (survives SSH disconnect)...");
|
|
2273
|
+
const launch = projectBootstrapLaunchCommand(scriptBody, scriptArgs, bootstrapEnv, repoName, bootstrapLibs);
|
|
2274
|
+
const launched = await runGcloudSshWithRetry(
|
|
2275
|
+
"ssh-script",
|
|
2276
|
+
cfg,
|
|
2277
|
+
{ command: launch },
|
|
2278
|
+
{ stdio: ["ignore", "pipe", "pipe"], label: "project bootstrap launch", sleepFn },
|
|
2279
|
+
);
|
|
2280
|
+
if (isSpawnTimeout(launched)) {
|
|
2281
|
+
// The LOCAL budget elapsed and we SIGTERMd the gcloud child. Say so:
|
|
2282
|
+
// the launch is a short write-and-detach, so this is an SSH-reachability
|
|
2283
|
+
// problem, NOT a broken bootstrap script — and the VM may be perfectly
|
|
2284
|
+
// healthy. Conflating the two is what destroyed VM-8.
|
|
2285
|
+
throw new Error(
|
|
2286
|
+
`launching the detached project bootstrap exceeded the local gcloud call budget ` +
|
|
2287
|
+
`and was SIGTERMd locally — the VM may be healthy and simply unreachable over SSH ` +
|
|
2288
|
+
`right now. Re-run to re-attach (each bootstrap step is skip-on-exists).`,
|
|
2289
|
+
);
|
|
2290
|
+
}
|
|
2291
|
+
if (launched.status !== 0) {
|
|
2292
|
+
throw new Error(
|
|
2293
|
+
`failed to launch the detached project bootstrap on the VM (gcloud ssh exit ${launched.status}).\n` +
|
|
2294
|
+
formatGcloudFailure(launched),
|
|
2295
|
+
);
|
|
2296
|
+
}
|
|
2297
|
+
}
|
|
2298
|
+
|
|
2299
|
+
// Follow the live log, re-attaching across any follow-session drops, until the
|
|
2300
|
+
// VM reports a terminal state or the overall deadline elapses. Success and
|
|
2301
|
+
// failure are read from VM state (the `.done` marker + the `.status` sentinel),
|
|
2302
|
+
// never the SSH exit code — so "my laptop's SSH dropped" or "a heavy npm
|
|
2303
|
+
// install outran a local budget" no longer masquerades as a broken bootstrap
|
|
2304
|
+
// and no longer gets a healthy VM deleted.
|
|
2305
|
+
const deadline = Date.now() + PROJECT_BOOTSTRAP_TIMEOUT_MS;
|
|
2306
|
+
for (;;) {
|
|
2307
|
+
runGcloud(
|
|
2308
|
+
buildGcloudArgs("ssh-script", cfg, { command: projectBootstrapFollowCommand(repoName) }),
|
|
2309
|
+
{ stdio: "inherit", timeout: PROJECT_BOOTSTRAP_FOLLOW_TIMEOUT_MS },
|
|
2310
|
+
);
|
|
2311
|
+
const outcome = probeProjectBootstrapOutcome(cfg, repoName);
|
|
2312
|
+
const action = decideProjectBootstrapFollowAction({
|
|
2313
|
+
outcome,
|
|
2314
|
+
now: Date.now(),
|
|
2315
|
+
deadline,
|
|
2316
|
+
});
|
|
2317
|
+
if (action === "success") {
|
|
2318
|
+
console.error(` ✓ ${repoName} bootstrap complete`);
|
|
2319
|
+
return;
|
|
2320
|
+
}
|
|
2321
|
+
if (action === "failed") {
|
|
2322
|
+
const tail = projectBootstrapLogTail(cfg, repoName);
|
|
2323
|
+
throw new Error(
|
|
2324
|
+
"scripts/gcp-project-bootstrap.sh failed on the VM. " +
|
|
2325
|
+
`Last lines of ${projectBootstrapLogPath(repoName)}:\n${tail}`,
|
|
2326
|
+
);
|
|
2327
|
+
}
|
|
2328
|
+
if (action === "timeout") {
|
|
2329
|
+
// The install is STILL RUNNING on a healthy VM and we simply stopped
|
|
2330
|
+
// waiting. This message is the signature npm/fleet-add-failure.js keys on
|
|
2331
|
+
// to classify the add as `project-bootstrap-timeout` (retryable) instead of
|
|
2332
|
+
// the non-retryable generic cloud-up-failed that tore down VM-8.
|
|
2333
|
+
throw new Error(
|
|
2334
|
+
`project bootstrap exceeded the local ${Math.round(PROJECT_BOOTSTRAP_TIMEOUT_MS / 60000)}-minute budget ` +
|
|
2335
|
+
`— the VM and its detached bootstrap may be healthy and still running. ` +
|
|
2336
|
+
`Re-run to re-attach (each bootstrap step is skip-on-exists), or inspect ` +
|
|
2337
|
+
`${projectBootstrapLogPath(repoName)} on the VM.`,
|
|
2338
|
+
);
|
|
2339
|
+
}
|
|
2340
|
+
// action === "reattach": the follow session ended but the install is alive
|
|
2341
|
+
// (or the probe could not see it yet). Re-attach to its log.
|
|
2342
|
+
console.error(" ↺ follow session ended but the project bootstrap is still running — re-attaching...");
|
|
2343
|
+
}
|
|
2344
|
+
}
|
|
2345
|
+
|
|
2346
|
+
/** Run the compose-ps liveness probe over SSH and report BOTH the editor
|
|
2347
|
+
* service's state and whether the probe itself could be trusted.
|
|
2348
|
+
*
|
|
2349
|
+
* The `|| true` keeps the remote pipeline's exit status 0 whenever SSH itself
|
|
2350
|
+
* worked, so a non-zero `status` means the transport failed (auth/SSH/IAP) —
|
|
2351
|
+
* not that the container is down. Callers that must fail safe (the abort
|
|
2352
|
+
* teardown) need that distinction; `parseEditorComposeState` alone collapses
|
|
2353
|
+
* "no container" and "could not ask" into the same `null`.
|
|
2354
|
+
*
|
|
2355
|
+
* Returns `{ state, inconclusive }` where `state` is the compose `State` string
|
|
2356
|
+
* (`"running"`, `"exited"`, …) or null when the service is absent, and
|
|
2357
|
+
* `inconclusive` is true only for a transport failure. */
|
|
2358
|
+
function probeEditorComposeState(cfg) {
|
|
2359
|
+
const repoName = resolveProject(cfg).name;
|
|
2360
|
+
const composeCmd = `docker compose -f docker-compose.cloud.yml`;
|
|
2361
|
+
const probe = runGcloud(
|
|
2362
|
+
buildGcloudArgs("ssh-probe", cfg, {
|
|
2363
|
+
command: `cd ~/projects/${repoName} 2>/dev/null && ${composeCmd} ps --format json 2>/dev/null || true`,
|
|
2364
|
+
}),
|
|
2365
|
+
{ stdio: ["ignore", "pipe", "ignore"] },
|
|
1695
2366
|
);
|
|
1696
|
-
if (
|
|
1697
|
-
|
|
2367
|
+
if (probe.status !== 0) {
|
|
2368
|
+
return { state: null, inconclusive: true };
|
|
2369
|
+
}
|
|
2370
|
+
return { state: parseEditorComposeState(probe.stdout), inconclusive: false };
|
|
2371
|
+
}
|
|
2372
|
+
|
|
2373
|
+
/** Poll {@link probeEditorComposeState} until the editor service reads
|
|
2374
|
+
* `running`, returning true as soon as it does. Used to close the TOCTOU window
|
|
2375
|
+
* after a container-name conflict: the winning run may still be inside
|
|
2376
|
+
* `Creating`/`Created` when the conflict surfaces, so a single immediate probe
|
|
2377
|
+
* can under-read and report a healthy container as absent.
|
|
2378
|
+
*
|
|
2379
|
+
* Deliberately short (default ~30s): this is not the multi-minute first-boot
|
|
2380
|
+
* budget `waitForContainerRunning` grants, it is the tail of a start that
|
|
2381
|
+
* another process has already begun. `sleepFn` and `probeFn` are injectable so
|
|
2382
|
+
* a unit test drives the whole loop with neither wall-clock delay nor a VM. */
|
|
2383
|
+
async function waitForEditorComposeRunning(cfg, { attempts = 6, delayMs = 5000, sleepFn, probeFn } = {}) {
|
|
2384
|
+
const sleep = sleepFn || ((ms) => new Promise((r) => setTimeout(r, ms)));
|
|
2385
|
+
const probe = probeFn || probeEditorComposeState;
|
|
2386
|
+
for (let i = 0; i < attempts; i++) {
|
|
2387
|
+
if (probe(cfg).state === "running") {
|
|
2388
|
+
return true;
|
|
2389
|
+
}
|
|
2390
|
+
if (i < attempts - 1) {
|
|
2391
|
+
await sleep(delayMs);
|
|
2392
|
+
}
|
|
1698
2393
|
}
|
|
2394
|
+
return false;
|
|
1699
2395
|
}
|
|
1700
2396
|
|
|
1701
2397
|
async function ensureEditorContainerUp(cfg) {
|
|
@@ -1705,13 +2401,7 @@ async function ensureEditorContainerUp(cfg) {
|
|
|
1705
2401
|
// cloud variant when self-hosting (see gcp-project-bootstrap.sh).
|
|
1706
2402
|
const composeCmd = `docker compose -f docker-compose.cloud.yml`;
|
|
1707
2403
|
console.error("Checking editor container state...");
|
|
1708
|
-
|
|
1709
|
-
buildGcloudArgs("ssh-probe", cfg, {
|
|
1710
|
-
command: `cd ~/projects/${repoName} && ${composeCmd} ps --format json 2>/dev/null || true`,
|
|
1711
|
-
}),
|
|
1712
|
-
{ stdio: ["ignore", "pipe", "ignore"] },
|
|
1713
|
-
);
|
|
1714
|
-
if (probe.status === 0 && parseEditorComposeState(probe.stdout) === "running") {
|
|
2404
|
+
if (probeEditorComposeState(cfg).state === "running") {
|
|
1715
2405
|
console.error(" ✓ editor container is already running");
|
|
1716
2406
|
// containerStarted: the container is confirmed up, so the CLI install runs
|
|
1717
2407
|
// its POST-start path with no wait — see the precondition on ensureAgentCliInstalled.
|
|
@@ -1719,14 +2409,47 @@ async function ensureEditorContainerUp(cfg) {
|
|
|
1719
2409
|
return;
|
|
1720
2410
|
}
|
|
1721
2411
|
console.error(" starting editor container (docker compose up -d)...");
|
|
2412
|
+
// CAPTURE rather than `stdio: "inherit"`: the daemon writes its container-name
|
|
2413
|
+
// conflict to stderr, and with "inherit" that text went straight to our parent
|
|
2414
|
+
// and was never observable here — so the conflict could not be told apart from
|
|
2415
|
+
// a real failure (the 2026-08-07 VM-4/VM-5 delete). spawnSync cannot both
|
|
2416
|
+
// stream and capture, so the operator's view is preserved by echoing the
|
|
2417
|
+
// captured streams immediately below instead of live. Do NOT revert this to
|
|
2418
|
+
// "inherit"; classification depends on it.
|
|
1722
2419
|
const up = runGcloud(
|
|
1723
2420
|
buildGcloudArgs("ssh-probe", cfg, {
|
|
1724
2421
|
command: `cd ~/projects/${repoName} && ${composeCmd} up -d`,
|
|
1725
2422
|
}),
|
|
1726
|
-
{ stdio: "
|
|
2423
|
+
{ stdio: ["ignore", "pipe", "pipe"] },
|
|
1727
2424
|
);
|
|
2425
|
+
if (up.stdout) process.stderr.write(up.stdout);
|
|
2426
|
+
if (up.stderr) process.stderr.write(up.stderr);
|
|
1728
2427
|
if (up.status !== 0) {
|
|
1729
|
-
|
|
2428
|
+
const verdict = classifyComposeUpFailure({ status: up.status, stderr: `${up.stderr || ""}\n${up.stdout || ""}` });
|
|
2429
|
+
if (verdict !== "name-conflict") {
|
|
2430
|
+
throw new Error("docker compose up -d failed on the VM");
|
|
2431
|
+
}
|
|
2432
|
+
// A name conflict means a concurrent run already created this container —
|
|
2433
|
+
// the goal we were about to pursue is being reached by someone else. Confirm
|
|
2434
|
+
// that positively before converging: a conflict whose container never
|
|
2435
|
+
// reaches `running` (e.g. a stale STOPPED container squatting the name) is a
|
|
2436
|
+
// genuine problem and must still fail loudly, or convergence becomes a
|
|
2437
|
+
// catch-all that papers over real breakage.
|
|
2438
|
+
console.error(
|
|
2439
|
+
" a concurrent run already created the editor container — confirming it reaches running...",
|
|
2440
|
+
);
|
|
2441
|
+
if (!(await waitForEditorComposeRunning(cfg))) {
|
|
2442
|
+
throw new Error(
|
|
2443
|
+
"docker compose up -d hit a container-name conflict on the VM, but the conflicting " +
|
|
2444
|
+
"container never reached running (a stale stopped container may be holding the name). " +
|
|
2445
|
+
`Inspect with: gcloud compute ssh ${cfg.instanceName} --command 'cd ~/projects/${repoName} && ${composeCmd} ps -a'`,
|
|
2446
|
+
);
|
|
2447
|
+
}
|
|
2448
|
+
console.error(" ✓ editor container is running (started by a concurrent run)");
|
|
2449
|
+
// containerStarted: the container is confirmed up — same post-start contract
|
|
2450
|
+
// as the already-running short-circuit above.
|
|
2451
|
+
await ensureAgentCliInstalled(cfg, { containerStarted: true });
|
|
2452
|
+
return;
|
|
1730
2453
|
}
|
|
1731
2454
|
// containerStarted: `up -d` returned 0, so the CLI install is legitimately a
|
|
1732
2455
|
// post-start step. A brief wait-for-running here (slow boot settling) is
|
|
@@ -1742,18 +2465,10 @@ async function ensureEditorContainerUp(cfg) {
|
|
|
1742
2465
|
// success. This throws (non-zero exit) when the editor service isn't
|
|
1743
2466
|
// `running`, converting a silent partial provision into a loud, actionable
|
|
1744
2467
|
// failure the operator (and the dashboard's fail-fast-on-nonzero path) can act
|
|
1745
|
-
// on.
|
|
1746
|
-
//
|
|
2468
|
+
// on. An inconclusive probe is not "running" here — a terminal gate must have
|
|
2469
|
+
// positive evidence, and the caller's job is to fail loudly without it.
|
|
1747
2470
|
async function assertEditorContainerRunning(cfg) {
|
|
1748
|
-
|
|
1749
|
-
const composeCmd = `docker compose -f docker-compose.cloud.yml`;
|
|
1750
|
-
const probe = runGcloud(
|
|
1751
|
-
buildGcloudArgs("ssh-probe", cfg, {
|
|
1752
|
-
command: `cd ~/projects/${repoName} 2>/dev/null && ${composeCmd} ps --format json 2>/dev/null || true`,
|
|
1753
|
-
}),
|
|
1754
|
-
{ stdio: ["ignore", "pipe", "ignore"] },
|
|
1755
|
-
);
|
|
1756
|
-
if (!(probe.status === 0 && parseEditorComposeState(probe.stdout) === "running")) {
|
|
2471
|
+
if (probeEditorComposeState(cfg).state !== "running") {
|
|
1757
2472
|
throw new Error(
|
|
1758
2473
|
`provisioning incomplete: editor container is not running on ${cfg.instanceName} ` +
|
|
1759
2474
|
`(no /workspace or container). Re-run: npm run cloud:up -- --instance-name ${cfg.instanceName}`,
|
|
@@ -2306,8 +3021,30 @@ function buildEditorLocalSeedWriteCommand({ repoName, remotePath, b64 }) {
|
|
|
2306
3021
|
* `_managedPreviewOrigin` sentinel). We stamp the sentinel here too so the
|
|
2307
3022
|
* monitor doesn't mistake the cloud:up seed for a user pin. The
|
|
2308
3023
|
* skip-on-already-set guard stays: a non-empty previewOrigin (whether a user
|
|
2309
|
-
* pin or a monitor write) is left untouched on every `cloud:up`.
|
|
2310
|
-
|
|
3024
|
+
* pin or a monitor write) is left untouched on every `cloud:up`.
|
|
3025
|
+
*
|
|
3026
|
+
* `effectiveSameOriginSafe` is the RUNNING editor's own verdict
|
|
3027
|
+
* (`/api/config`'s `previewSameOriginSafe`), supplied by a caller that has
|
|
3028
|
+
* already fetched it. When it is a boolean it WINS over the static
|
|
3029
|
+
* `stack.json` read; `null`/absent leaves the static read in charge.
|
|
3030
|
+
*
|
|
3031
|
+
* That parameter exists because this function had two sources of truth for one
|
|
3032
|
+
* flag, and they disagreed exactly where the adaptive default lives. A
|
|
3033
|
+
* subpath hydration failure escalates a project off `/__codeyam_preview` at
|
|
3034
|
+
* RUNTIME even when `stack.json` never declared `sameOriginSafe: false` — which
|
|
3035
|
+
* is the documented Next.js scaffold shape (a static `false` broke every
|
|
3036
|
+
* SSH-forward VM, so the key is deliberately absent). `healPreviewOriginAtReady`
|
|
3037
|
+
* decided WHETHER to heal from that runtime verdict and was right, then called
|
|
3038
|
+
* this function, which recomputed WHAT to write from the static flag and wrote
|
|
3039
|
+
* the very same-origin sentinel that had just failed. The project then sat
|
|
3040
|
+
* permanently at `previewDedicatedOriginUnavailable: true` — the editor knowing
|
|
3041
|
+
* it must leave the subpath and having nowhere to go — beside a preview host
|
|
3042
|
+
* that was already provisioned, LB-routed, HEALTHY and unused. */
|
|
3043
|
+
async function seedPreviewOrigin(
|
|
3044
|
+
cfg,
|
|
3045
|
+
appPort = VITE_PORT,
|
|
3046
|
+
effectiveSameOriginSafe = null,
|
|
3047
|
+
) {
|
|
2311
3048
|
const repoName = resolveProject(cfg).name;
|
|
2312
3049
|
const remotePath = `~/projects/${repoName}/.codeyam/editor.local.json`;
|
|
2313
3050
|
const stackPath = `~/projects/${repoName}/.codeyam/stack.json`;
|
|
@@ -2343,6 +3080,12 @@ async function seedPreviewOrigin(cfg, appPort = VITE_PORT) {
|
|
|
2343
3080
|
current = parseObj(localJson);
|
|
2344
3081
|
sameOriginSafe = previewSameOriginSafeFromStack(parseObj(stackJson));
|
|
2345
3082
|
}
|
|
3083
|
+
// The running editor's verdict overrides the static file — see
|
|
3084
|
+
// `resolveEffectiveSameOriginSafe` for why only a boolean counts.
|
|
3085
|
+
sameOriginSafe = resolveEffectiveSameOriginSafe(
|
|
3086
|
+
sameOriginSafe,
|
|
3087
|
+
effectiveSameOriginSafe,
|
|
3088
|
+
);
|
|
2346
3089
|
|
|
2347
3090
|
// Resolve this VM's custom slug (if any) so a dedicated-origin seed uses the
|
|
2348
3091
|
// slug host. The slug rides on cfg at Add time (the --vm-slug flag, validated
|
|
@@ -2376,6 +3119,12 @@ async function seedPreviewOrigin(cfg, appPort = VITE_PORT) {
|
|
|
2376
3119
|
// `sameOriginSafe: true`, and seeded `/__codeyam_preview` — which a
|
|
2377
3120
|
// remote browser then loads as `127.0.0.1 refused to connect`). Once
|
|
2378
3121
|
// stack.json exists, heal it forward to the dedicated origin.
|
|
3122
|
+
// `sameOriginSafe` here is the EFFECTIVE verdict, so this exception also
|
|
3123
|
+
// covers the case stack.json can never report: a project whose static
|
|
3124
|
+
// flag is legitimately absent (the Next.js scaffold default) that
|
|
3125
|
+
// escalated off the subpath at runtime after a hydration failure. The
|
|
3126
|
+
// caller passes the editor's own answer; without it this predicate read
|
|
3127
|
+
// `true` and declined to heal the one shape it was needed for.
|
|
2379
3128
|
// Because every dashboard action (Reset / Update / Add / bootstrap-recovery)
|
|
2380
3129
|
// re-runs this seed over cloud:up, either heal reaches every touch-point the
|
|
2381
3130
|
// moment stack.json is present — closing the race without re-ordering
|
|
@@ -2471,11 +3220,14 @@ async function healPreviewOriginAtReady(cfg, appPort = VITE_PORT) {
|
|
|
2471
3220
|
`READY auto-heal: ${cfg.instanceName} reports a not-same-origin-safe stack ` +
|
|
2472
3221
|
`but a wrong-mode previewOrigin (${config.previewOrigin}); re-seeding...`,
|
|
2473
3222
|
);
|
|
2474
|
-
// Re-run the seed
|
|
2475
|
-
//
|
|
2476
|
-
//
|
|
2477
|
-
//
|
|
2478
|
-
|
|
3223
|
+
// Re-run the seed with the verdict we just read, so WHETHER to heal and WHAT
|
|
3224
|
+
// to write come from the same source. Passing it is what makes this heal reach
|
|
3225
|
+
// the adaptive-default stacks it exists for: on a Next.js project whose
|
|
3226
|
+
// `stack.json` omits `sameOriginSafe` (the scaffolded shape), the static read
|
|
3227
|
+
// inside the seed answers `true` and re-writes the same-origin sentinel that
|
|
3228
|
+
// just failed to hydrate. We got here precisely BECAUSE the editor said
|
|
3229
|
+
// `false`, so `false` is the answer the seed must use.
|
|
3230
|
+
await seedPreviewOrigin(cfg, appPort, false);
|
|
2479
3231
|
// Tell the running server to re-read editor.local.json — no restart (the
|
|
2480
3232
|
// footgun that killed VM7's live Claude session).
|
|
2481
3233
|
const reload = runGcloud(
|
|
@@ -2873,7 +3625,16 @@ function promptForInstanceName(expected) {
|
|
|
2873
3625
|
});
|
|
2874
3626
|
}
|
|
2875
3627
|
|
|
2876
|
-
|
|
3628
|
+
// Every file streamed to a VM, and therefore every file the drift guard must
|
|
3629
|
+
// compare against the reviewed remote tip. `scripts/lib/npm-pin.sh` is here
|
|
3630
|
+
// because gcp-project-bootstrap.sh SOURCES it: a library that reaches the VM is
|
|
3631
|
+
// as much streamed content as the script that sources it, and leaving it out
|
|
3632
|
+
// meant the drift guard silently vouched for only half of what was shipped.
|
|
3633
|
+
const STREAMED_SCRIPTS = [
|
|
3634
|
+
"scripts/gcp-bootstrap.sh",
|
|
3635
|
+
"scripts/gcp-project-bootstrap.sh",
|
|
3636
|
+
"scripts/lib/npm-pin.sh",
|
|
3637
|
+
];
|
|
2877
3638
|
|
|
2878
3639
|
/** The configured editor branch to compare/stream streamed scripts against.
|
|
2879
3640
|
* Resolved from config (`resolveEditorConfig` → `editorBranch`, itself derived
|
|
@@ -3110,8 +3871,9 @@ function fleetInstancePrefix() {
|
|
|
3110
3871
|
* scripts/gcp/vm-ingress.sh with the resolved coordinates. Returns the spawnSync
|
|
3111
3872
|
* result so callers decide fatality — Add treats a non-zero as a loud warning
|
|
3112
3873
|
* (the VM is provisioned; ingress can be re-wired), Destroy likewise warns so a
|
|
3113
|
-
* teardown still removes the instance.
|
|
3114
|
-
|
|
3874
|
+
* teardown still removes the instance. `vmAccess` (up only) is the Add's intended
|
|
3875
|
+
* principal set, which the script's grant step uses in place of its sidecar read. */
|
|
3876
|
+
function runVmIngress(cfg, action, vmNumber, domain, appPort, slug, vmAccess) {
|
|
3115
3877
|
const scriptPath = path.join(rootDir, "scripts", "gcp", "vm-ingress.sh");
|
|
3116
3878
|
const inv = vmIngressInvocation({
|
|
3117
3879
|
scriptPath,
|
|
@@ -3123,6 +3885,7 @@ function runVmIngress(cfg, action, vmNumber, domain, appPort, slug) {
|
|
|
3123
3885
|
appPort,
|
|
3124
3886
|
instancePrefix: fleetInstancePrefix(),
|
|
3125
3887
|
slug,
|
|
3888
|
+
vmAccess,
|
|
3126
3889
|
});
|
|
3127
3890
|
return spawnSync(inv.command, inv.args, {
|
|
3128
3891
|
stdio: "inherit",
|
|
@@ -3153,17 +3916,77 @@ function runVmIngressGrantAccess(cfg, vmNumber, domain, slug) {
|
|
|
3153
3916
|
});
|
|
3154
3917
|
}
|
|
3155
3918
|
|
|
3919
|
+
/** THE principal set that must have IAP access to this VM after the Add: the
|
|
3920
|
+
* explicit `cfg.vmAccess` collaborators unioned with the resolved default
|
|
3921
|
+
* operator. Both add-time access paths consume this ONE function —
|
|
3922
|
+
* `wireVmIngressOnAdd` (which hands it to vm-ingress.sh's grant) and
|
|
3923
|
+
* `applyVmAccessOnAdd` (which grants it and subtracts it from the stale-revoke
|
|
3924
|
+
* set). They used to decide independently, and that is precisely how one path
|
|
3925
|
+
* granted `user:jared@codeyam.com` at 00:09:31 and the other revoked the same
|
|
3926
|
+
* principal 18 seconds later as a "stale prior tenant" (VMs 4/5/6, 2026-08-09):
|
|
3927
|
+
* the ingress path read the SIDECAR (still holding the prior tenant) while
|
|
3928
|
+
* `applyVmAccessOnAdd` resolved an empty operator. One source of truth removes
|
|
3929
|
+
* the whole class of disagreement. `env` is injectable for tests; the only I/O
|
|
3930
|
+
* is `resolveDefaultOperator`'s config/gcloud read. */
|
|
3931
|
+
function intendedVmPrincipals(cfg = {}, env = process.env) {
|
|
3932
|
+
return withDefaultOperator(
|
|
3933
|
+
normalizeAccessList(cfg.vmAccess || []),
|
|
3934
|
+
resolveDefaultOperator({ env }),
|
|
3935
|
+
);
|
|
3936
|
+
}
|
|
3937
|
+
|
|
3938
|
+
/** Add preflight: throw — BEFORE any GCE instance exists — when this Add would
|
|
3939
|
+
* come up on an IAP-gated public URL that authorizes NOBODY. The whole condition
|
|
3940
|
+
* is env + config (`intendedVmPrincipals` needs no GCP state), which is why it
|
|
3941
|
+
* belongs beside cmdUp's other cheap boundary validations rather than after a
|
|
3942
|
+
* machine has been provisioned: checking later means a half-provisioned VM to
|
|
3943
|
+
* clean up. On 2026-08-09 the un-refused version of this Add reported `done` for
|
|
3944
|
+
* VMs 4/5/6, left three unreachable URLs, and revoked each slot's prior tenant on
|
|
3945
|
+
* the way.
|
|
3946
|
+
*
|
|
3947
|
+
* Only fires in LB mode on a numbered fleet VM with nothing to grant; an explicit
|
|
3948
|
+
* `--vm-access` satisfies it with no default operator configured, so the
|
|
3949
|
+
* collaborator flow is unaffected, as is every legacy quick-tunnel Add. The
|
|
3950
|
+
* refusal text (and its `Next valid action:` remedy naming all three config
|
|
3951
|
+
* tiers) is the pure `unauthorizableAddRefusal`; this function is the composition
|
|
3952
|
+
* and the throw. */
|
|
3953
|
+
function assertAddCanAuthorizeSomeone(cfg, env = process.env) {
|
|
3954
|
+
let plan;
|
|
3955
|
+
try {
|
|
3956
|
+
plan = ingressWiringPlan(cfg, env);
|
|
3957
|
+
} catch {
|
|
3958
|
+
// A malformed domain is `resolveIngressDomain`'s error to raise on the wiring
|
|
3959
|
+
// path; it must not masquerade here as an authorization refusal.
|
|
3960
|
+
plan = { wire: false };
|
|
3961
|
+
}
|
|
3962
|
+
const refusal = unauthorizableAddRefusal({
|
|
3963
|
+
lbMode: !!plan.wire,
|
|
3964
|
+
vmNumber: plan.wire ? plan.vmNumber : null,
|
|
3965
|
+
intendedCount: plan.wire ? intendedVmPrincipals(cfg, env).length : null,
|
|
3966
|
+
});
|
|
3967
|
+
if (refusal) throw new Error(refusal);
|
|
3968
|
+
}
|
|
3969
|
+
|
|
3156
3970
|
/** Add-time hook: wire VM-N onto the shared LB when a fleet domain is set.
|
|
3157
3971
|
* Best-effort + loud — a wiring hiccup must never abort an otherwise-green Add
|
|
3158
3972
|
* (the VM is reachable operator-side over SSH regardless), so on failure it
|
|
3159
3973
|
* prints the exact re-run command rather than throwing. No-op (and silent) in
|
|
3160
|
-
* legacy quick-tunnel mode or for a non-fleet client project.
|
|
3974
|
+
* legacy quick-tunnel mode or for a non-fleet client project.
|
|
3975
|
+
*
|
|
3976
|
+
* RETURNS the principals this call granted (`intendedVmPrincipals`), or [] when
|
|
3977
|
+
* it did not wire — the explicit dataflow `applyVmAccessOnAdd` needs to know what
|
|
3978
|
+
* the Add has already granted before it computes a stale-revoke. Returned on the
|
|
3979
|
+
* FAILURE path too: a non-zero `vm-ingress.sh up` may still have completed its
|
|
3980
|
+
* grant step, and a principal that was possibly granted must never be revoked. */
|
|
3161
3981
|
function wireVmIngressOnAdd(cfg, appPort, env = process.env) {
|
|
3162
3982
|
const plan = ingressWiringPlan(cfg, env);
|
|
3163
|
-
if (!plan.wire) return;
|
|
3983
|
+
if (!plan.wire) return [];
|
|
3164
3984
|
const label = plan.slug || `vm-${plan.vmNumber}`;
|
|
3985
|
+
// Hand the intended set to the script so its `apply_default_iap_access` grants
|
|
3986
|
+
// THIS Add's principals instead of reading the sidecar's prior tenant.
|
|
3987
|
+
const vmAccess = intendedVmPrincipals(cfg, env);
|
|
3165
3988
|
console.error(` wiring LB ingress for VM-${plan.vmNumber} on ${plan.domain}...`);
|
|
3166
|
-
const res = runVmIngress(cfg, "up", plan.vmNumber, plan.domain, appPort, plan.slug);
|
|
3989
|
+
const res = runVmIngress(cfg, "up", plan.vmNumber, plan.domain, appPort, plan.slug, vmAccess);
|
|
3167
3990
|
if (res.status === 0) {
|
|
3168
3991
|
console.error(` ✓ LB ingress wired: https://${label}.editor.${plan.domain}`);
|
|
3169
3992
|
} else {
|
|
@@ -3185,6 +4008,7 @@ function wireVmIngressOnAdd(cfg, appPort, env = process.env) {
|
|
|
3185
4008
|
`(vm-ingress.sh up exited ${res.status == null ? "abnormally" : res.status}). Re-wire: ${rewire}`,
|
|
3186
4009
|
);
|
|
3187
4010
|
}
|
|
4011
|
+
return vmAccess;
|
|
3188
4012
|
}
|
|
3189
4013
|
|
|
3190
4014
|
/** Destroy-time hook: unwire VM-N's LB backends before the instance is deleted,
|
|
@@ -3253,8 +4077,15 @@ function runIapAccess(cfg, vmNumber, action, principals) {
|
|
|
3253
4077
|
* a revoke/grant hiccup (e.g. the SA missing IAP-admin) prints the exact re-run
|
|
3254
4078
|
* command rather than aborting an otherwise-green Add. `cfg.vmAccess` is the
|
|
3255
4079
|
* array (or comma-separated string) cmdUp validated; absent ⇒ an empty desired
|
|
3256
|
-
* list (the clean-slate case).
|
|
3257
|
-
|
|
4080
|
+
* list (the clean-slate case).
|
|
4081
|
+
*
|
|
4082
|
+
* `grantedThisRun` is what an EARLIER step of this same Add already granted —
|
|
4083
|
+
* `wireVmIngressOnAdd`'s return value. It is subtracted from the stale-revoke
|
|
4084
|
+
* set, so a principal this Add just authorized can never be revoked moments later
|
|
4085
|
+
* as a prior tenant. That is a second, cheaper guard than the shared
|
|
4086
|
+
* `intendedVmPrincipals` resolver above: it holds even if the two paths drift
|
|
4087
|
+
* apart again, which is exactly how VMs 4/5/6 broke on 2026-08-09. */
|
|
4088
|
+
function applyVmAccessOnAdd(cfg, env = process.env, { grantedThisRun = [] } = {}) {
|
|
3258
4089
|
const desired = normalizeAccessList(cfg.vmAccess || []);
|
|
3259
4090
|
let plan;
|
|
3260
4091
|
try {
|
|
@@ -3271,11 +4102,15 @@ function applyVmAccessOnAdd(cfg, env = process.env) {
|
|
|
3271
4102
|
// never inaccessible-by-default (the Error-51 footgun). The default operator is
|
|
3272
4103
|
// NOT persisted to the sidecar (it's a standing grant, not a per-VM
|
|
3273
4104
|
// collaborator), so `desired` alone drives the sidecar + stale-revoke below.
|
|
3274
|
-
const toGrant =
|
|
4105
|
+
const toGrant = intendedVmPrincipals(cfg, env);
|
|
3275
4106
|
const stale = accessFor(vmNumber);
|
|
3276
4107
|
// Never revoke a principal we're about to (re-)grant — a prior tenant who was
|
|
3277
|
-
// also the default operator must not be revoked-then-regranted for nothing
|
|
3278
|
-
|
|
4108
|
+
// also the default operator must not be revoked-then-regranted for nothing —
|
|
4109
|
+
// NOR one an earlier step of this same Add already granted. `toGrant` alone
|
|
4110
|
+
// protected nothing when it was empty: the ingress step had granted the operator
|
|
4111
|
+
// seconds earlier and this revoke took it straight back off (VMs 4/5/6).
|
|
4112
|
+
const grantedAlready = new Set([...toGrant, ...normalizeAccessList(grantedThisRun || [])]);
|
|
4113
|
+
const staleOnly = stale.filter((p) => !grantedAlready.has(p));
|
|
3279
4114
|
|
|
3280
4115
|
if (plan.wire) {
|
|
3281
4116
|
if (staleOnly.length > 0) {
|
|
@@ -3392,12 +4227,23 @@ async function convergeVmIngressOnAdd(cfg, appPort, env = process.env, deps = {}
|
|
|
3392
4227
|
}
|
|
3393
4228
|
if (!plan.wire) {
|
|
3394
4229
|
// Nothing to converge — the no-op wire keeps the clean-slate invariant.
|
|
3395
|
-
wireVmIngressOnAdd(cfg, appPort, env);
|
|
3396
|
-
return { wired: true, reason: "no-wire" };
|
|
3397
|
-
}
|
|
3398
|
-
|
|
4230
|
+
const granted = wireVmIngressOnAdd(cfg, appPort, env);
|
|
4231
|
+
return { wired: true, reason: "no-wire", grantedThisRun: granted };
|
|
4232
|
+
}
|
|
4233
|
+
|
|
4234
|
+
// The intended set, resolved ONCE and handed to every `up` attempt so each
|
|
4235
|
+
// converge iteration grants the same principals — and reported back to the
|
|
4236
|
+
// caller as `grantedThisRun` so the access reconcile that follows cannot revoke
|
|
4237
|
+
// them. Every `up` runs the script's grant step, so these are granted whether
|
|
4238
|
+
// the loop converges or hits its deadline. Injectable via `deps` like the rest
|
|
4239
|
+
// of this loop's slow I/O: the resolve spawns `gcloud config get-value account`
|
|
4240
|
+
// on its last tier, and a unit test must never depend on the gcloud identity
|
|
4241
|
+
// that happens to be active on the machine running it.
|
|
4242
|
+
const grantedThisRun =
|
|
4243
|
+
deps.grantedThisRun != null ? deps.grantedThisRun : intendedVmPrincipals(cfg, env);
|
|
3399
4244
|
const runUp =
|
|
3400
|
-
deps.runUp
|
|
4245
|
+
deps.runUp
|
|
4246
|
+
|| ((p) => runVmIngress(cfg, "up", p.vmNumber, p.domain, appPort, p.slug, grantedThisRun));
|
|
3401
4247
|
const probeVerdict = deps.probeVerdict || ((p) => probeVmIngressVerdict(cfg, p));
|
|
3402
4248
|
const sleep = deps.sleep || ((ms) => new Promise((r) => setTimeout(r, ms)));
|
|
3403
4249
|
const now = deps.now || (() => Date.now());
|
|
@@ -3440,7 +4286,7 @@ async function convergeVmIngressOnAdd(cfg, appPort, env = process.env, deps = {}
|
|
|
3440
4286
|
if (decision.done) {
|
|
3441
4287
|
if (decision.wired) {
|
|
3442
4288
|
console.error(` ✓ LB ingress wired: https://${label}.editor.${plan.domain}`);
|
|
3443
|
-
return { wired: true, reason: "wired", attempts: attempt + 1 };
|
|
4289
|
+
return { wired: true, reason: "wired", attempts: attempt + 1, grantedThisRun };
|
|
3444
4290
|
}
|
|
3445
4291
|
const rewire =
|
|
3446
4292
|
`FLEET_DOMAIN=${plan.domain}${plan.slug ? ` VM_SLUG=${plan.slug}` : ""} ` +
|
|
@@ -3457,7 +4303,7 @@ async function convergeVmIngressOnAdd(cfg, appPort, env = process.env, deps = {}
|
|
|
3457
4303
|
`CY_DEGRADED ingress-wiring-failed: VM-${plan.vmNumber} public URL did not wire ` +
|
|
3458
4304
|
`within the add window (last verdict=${verdict || "unknown"}). Re-wire: ${rewire}`,
|
|
3459
4305
|
);
|
|
3460
|
-
return { wired: false, reason: "deadline", attempts: attempt + 1 };
|
|
4306
|
+
return { wired: false, reason: "deadline", attempts: attempt + 1, grantedThisRun };
|
|
3461
4307
|
}
|
|
3462
4308
|
attempt += 1;
|
|
3463
4309
|
await sleep(decision.delayMs);
|
|
@@ -3486,8 +4332,8 @@ function startDetachedAddWiring(cfg, appPort, env = process.env) {
|
|
|
3486
4332
|
if (!plan.wire) {
|
|
3487
4333
|
// Nothing slow to detach: wiring is a no-op and the access reconcile is a cheap
|
|
3488
4334
|
// sidecar write. Run them inline so the clean-slate invariant still holds.
|
|
3489
|
-
wireVmIngressOnAdd(cfg, appPort, env);
|
|
3490
|
-
applyVmAccessOnAdd(cfg, env);
|
|
4335
|
+
const grantedThisRun = wireVmIngressOnAdd(cfg, appPort, env);
|
|
4336
|
+
applyVmAccessOnAdd(cfg, env, { grantedThisRun });
|
|
3491
4337
|
return;
|
|
3492
4338
|
}
|
|
3493
4339
|
const marker = ingressInflightMarkerPath(INGRESS_INFLIGHT_DIR, plan.vmNumber);
|
|
@@ -3519,8 +4365,8 @@ function startDetachedAddWiring(cfg, appPort, env = process.env) {
|
|
|
3519
4365
|
`running it inline instead.`,
|
|
3520
4366
|
);
|
|
3521
4367
|
try { fs.unlinkSync(marker); } catch { /* marker may not exist */ }
|
|
3522
|
-
wireVmIngressOnAdd(cfg, appPort, env);
|
|
3523
|
-
applyVmAccessOnAdd(cfg, env);
|
|
4368
|
+
const grantedThisRun = wireVmIngressOnAdd(cfg, appPort, env);
|
|
4369
|
+
applyVmAccessOnAdd(cfg, env, { grantedThisRun });
|
|
3524
4370
|
}
|
|
3525
4371
|
}
|
|
3526
4372
|
|
|
@@ -3552,8 +4398,11 @@ async function cmdWireAddTail() {
|
|
|
3552
4398
|
// Converge the URL to `verdict=wired` (retry-until-wired), not a single-shot
|
|
3553
4399
|
// `up` left to the slow reconcile sweep — then the IAP access grant, in the
|
|
3554
4400
|
// SAME order cmdUp ran them inline.
|
|
3555
|
-
|
|
3556
|
-
|
|
4401
|
+
// Carry what the converge granted into the access reconcile, so the revoke
|
|
4402
|
+
// step cannot take back a principal this same Add just authorized — the exact
|
|
4403
|
+
// grant-then-revoke-18s-later sequence `/tmp/cy-wire-add-vm-4.log` recorded.
|
|
4404
|
+
const { grantedThisRun } = await convergeVmIngressOnAdd(cfg, appPort);
|
|
4405
|
+
applyVmAccessOnAdd(cfg, process.env, { grantedThisRun });
|
|
3557
4406
|
} finally {
|
|
3558
4407
|
if (marker) {
|
|
3559
4408
|
try { fs.unlinkSync(marker); } catch { /* already gone — fine */ }
|
|
@@ -3615,6 +4464,75 @@ function releaseVmSlugOnDestroy(cfg) {
|
|
|
3615
4464
|
return vmNumber;
|
|
3616
4465
|
}
|
|
3617
4466
|
|
|
4467
|
+
/** Does `branch` still exist on `repo`? Answered with a single
|
|
4468
|
+
* `git ls-remote --heads` — about a second, and no cloud resource touched.
|
|
4469
|
+
*
|
|
4470
|
+
* Tri-state on purpose, and the third state is the load-bearing one:
|
|
4471
|
+
* true — the ref resolved.
|
|
4472
|
+
* false — the remote answered and the ref is NOT there. Provably gone.
|
|
4473
|
+
* null — we could not get an answer (no network, no credentials, git
|
|
4474
|
+
* missing, timeout). NOT evidence of a deleted branch.
|
|
4475
|
+
* Only `false` refuses an add. Collapsing `null` into `false` would turn every
|
|
4476
|
+
* transient GitHub blip into a fleet-wide add outage, which is a far worse
|
|
4477
|
+
* failure than the one this preflight exists to prevent — and the classifier
|
|
4478
|
+
* arm downstream still catches a genuinely dead ref, just later and dearer.
|
|
4479
|
+
*
|
|
4480
|
+
* A private repo needs credentials, so the token is inlined into the URL the
|
|
4481
|
+
* same way clone_editor_repo does it on the VM side (x-access-token), and for
|
|
4482
|
+
* the same reason: the plain URL 401s. The token is confined to this argv and
|
|
4483
|
+
* never logged — the caller prints `repo` (the un-inlined form) instead. */
|
|
4484
|
+
function gitBranchRefExists({ repo, branch }) {
|
|
4485
|
+
if (!repo || !branch) return null;
|
|
4486
|
+
let url = repo;
|
|
4487
|
+
const token = resolveGithubToken();
|
|
4488
|
+
if (token && /^https:\/\/github\.com\//.test(repo)) {
|
|
4489
|
+
url = repo.replace("https://", `https://x-access-token:${token}@`);
|
|
4490
|
+
}
|
|
4491
|
+
const res = spawnSync("git", ["ls-remote", "--heads", url, branch], {
|
|
4492
|
+
encoding: "utf8",
|
|
4493
|
+
timeout: 30000,
|
|
4494
|
+
});
|
|
4495
|
+
// A non-zero exit is an unreachable/unauthorized remote, not an absent branch:
|
|
4496
|
+
// git exits 0 and prints nothing for "connected fine, no such head".
|
|
4497
|
+
if (res.error || res.status !== 0) return null;
|
|
4498
|
+
return String(res.stdout || "").trim().length > 0;
|
|
4499
|
+
}
|
|
4500
|
+
|
|
4501
|
+
/** Refuse an add whose stored branch has been deleted from origin, BEFORE the
|
|
4502
|
+
* first gcloud call.
|
|
4503
|
+
*
|
|
4504
|
+
* Without this the failure lands as late and as expensively as it possibly can:
|
|
4505
|
+
* on 2026-08-25 four fleet VMs each created a real GCE instance, booted it,
|
|
4506
|
+
* installed npm and cloned 13,226 files before `git fetch` inside the VM said
|
|
4507
|
+
* `fatal: couldn't find remote ref editor-improvements-75` — a branch pruned
|
|
4508
|
+
* from origin hours earlier when it merged. One `git ls-remote` establishes the
|
|
4509
|
+
* same fact in about a second, for free. This repo prunes branches on merge, so
|
|
4510
|
+
* ANY stored feature-branch name expires silently; that makes this a recurring
|
|
4511
|
+
* condition, not a one-off.
|
|
4512
|
+
*
|
|
4513
|
+
* Placed beside checkScriptDrift and assertVariantProvisionable — the other
|
|
4514
|
+
* "fail loud before creating a GCE instance" guards — because it is only worth
|
|
4515
|
+
* anything ahead of ensureInstance. It cannot live in scripts/gcp-bootstrap.sh,
|
|
4516
|
+
* which already runs ON the provisioned box and never sees the client branch.
|
|
4517
|
+
*
|
|
4518
|
+
* Throws with the field, the branch and the repo named. Never throws for a ref
|
|
4519
|
+
* it could not check (see gitBranchRefExists). */
|
|
4520
|
+
function assertBranchRefsExist(cfg, { refExists = gitBranchRefExists } = {}) {
|
|
4521
|
+
// Lazy require: launch-config.js requires THIS module at load time, so a
|
|
4522
|
+
// top-level require here would close the cycle. Same pattern as the
|
|
4523
|
+
// isPlaceholderProject call sites above.
|
|
4524
|
+
const lc = require("./launch-config");
|
|
4525
|
+
const { editorRepo, editorBranch } = resolveEditorConfig(cfg);
|
|
4526
|
+
const targets = lc.branchRefTargets(
|
|
4527
|
+
{ clientRepo: cfg.repo, clientBranch: cfg.branch, editorBranch },
|
|
4528
|
+
{ editorRepo },
|
|
4529
|
+
);
|
|
4530
|
+
if (!targets.length) return;
|
|
4531
|
+
console.error("Checking that the requested branches still exist on origin...");
|
|
4532
|
+
const missing = lc.missingBranchRefs(targets, refExists);
|
|
4533
|
+
if (missing.length) throw new Error(lc.missingBranchRefMessage(missing));
|
|
4534
|
+
}
|
|
4535
|
+
|
|
3618
4536
|
async function cmdUp(args, projectDir = rootDir) {
|
|
3619
4537
|
ensureGcloud();
|
|
3620
4538
|
const flags = parseFlags(args);
|
|
@@ -3665,8 +4583,14 @@ async function cmdUp(args, projectDir = rootDir) {
|
|
|
3665
4583
|
}
|
|
3666
4584
|
}
|
|
3667
4585
|
|
|
4586
|
+
assertAddCanAuthorizeSomeone(cfg);
|
|
4587
|
+
|
|
3668
4588
|
checkScriptDrift(cfg);
|
|
3669
4589
|
|
|
4590
|
+
// Last of the pre-instance guards: a deleted branch ref is checkable in ~1s
|
|
4591
|
+
// and otherwise costs a full instance + boot + 13k-file clone to discover.
|
|
4592
|
+
assertBranchRefsExist(cfg);
|
|
4593
|
+
|
|
3670
4594
|
await ensureFirewallRule(cfg);
|
|
3671
4595
|
const { createdThisRun } = await ensureInstance(cfg);
|
|
3672
4596
|
|
|
@@ -3771,7 +4695,27 @@ async function cmdUp(args, projectDir = rootDir) {
|
|
|
3771
4695
|
);
|
|
3772
4696
|
}
|
|
3773
4697
|
} catch (err) {
|
|
3774
|
-
|
|
4698
|
+
// `createdThisRun` alone MUST NOT authorize deletion. Gather what state the
|
|
4699
|
+
// VM is in right now, then let the pure decision rule — see
|
|
4700
|
+
// abortTeardownDecision's doc comment for why each input is load-bearing.
|
|
4701
|
+
const state = probeAbortTeardownState(cfg, createdThisRun);
|
|
4702
|
+
const verdict = abortTeardownDecision({ createdThisRun, ...state });
|
|
4703
|
+
if (createdThisRun && verdict === "keep") {
|
|
4704
|
+
// Deliberately loud and specific. The old blanket "tearing down the VM
|
|
4705
|
+
// cloud:up created this run (no orphan left behind)" was actively
|
|
4706
|
+
// misleading here — it asserted no orphan while deleting a working VM.
|
|
4707
|
+
const why = state.probeInconclusive
|
|
4708
|
+
? "its liveness could not be confirmed (probe failed), and an unconfirmed VM is never deleted"
|
|
4709
|
+
: state.containerUp
|
|
4710
|
+
? "its editor container is RUNNING — another run brought it up"
|
|
4711
|
+
: "it is in the fleet roster and visible to the dashboard";
|
|
4712
|
+
console.error(
|
|
4713
|
+
`✗ provisioning aborted — the VM ${cfg.instanceName} was LEFT IN PLACE because ${why}.\n` +
|
|
4714
|
+
` This run created the instance, but that alone does not authorize deleting a VM that is in service.\n` +
|
|
4715
|
+
` If it really should go, remove it deliberately: npm run cloud:destroy`,
|
|
4716
|
+
);
|
|
4717
|
+
}
|
|
4718
|
+
if (verdict === "delete") {
|
|
3775
4719
|
console.error(
|
|
3776
4720
|
`✗ provisioning aborted — tearing down the VM cloud:up created this run (no orphan left behind)`,
|
|
3777
4721
|
);
|
|
@@ -3953,14 +4897,7 @@ async function cmdStatus(args, projectDir = rootDir) {
|
|
|
3953
4897
|
const imgStatus = String(imgProbe.stdout || "").trim() || "?";
|
|
3954
4898
|
console.error(` image: ${imgStatus}`);
|
|
3955
4899
|
|
|
3956
|
-
const
|
|
3957
|
-
const containerProbe = runGcloud(
|
|
3958
|
-
buildGcloudArgs("ssh-probe", cfg, {
|
|
3959
|
-
command: `cd ~/projects/${repoName} 2>/dev/null && docker compose -f docker-compose.cloud.yml ps --format json 2>/dev/null || echo`,
|
|
3960
|
-
}),
|
|
3961
|
-
{ stdio: ["ignore", "pipe", "ignore"] },
|
|
3962
|
-
);
|
|
3963
|
-
const containerStatus = parseEditorComposeState(containerProbe.stdout) || "missing";
|
|
4900
|
+
const containerStatus = probeEditorComposeState(cfg).state || "missing";
|
|
3964
4901
|
console.error(` editor container: ${containerStatus}`);
|
|
3965
4902
|
|
|
3966
4903
|
const appPort = await readVmAppPort(cfg);
|
|
@@ -4368,7 +5305,14 @@ function sweepOrphanIngress({ cfg, domain, roster, projectDir = rootDir } = {},
|
|
|
4368
5305
|
});
|
|
4369
5306
|
const res = run(inv.command, inv.args, { encoding: "utf8", env: { ...process.env, ...inv.env } });
|
|
4370
5307
|
const verdict = parseIngressTeardownVerdict((res && res.stdout) || "");
|
|
4371
|
-
|
|
5308
|
+
// Pass `signal` too: a teardown the caller's group-kill reaped has a NULL
|
|
5309
|
+
// status, which the classifier would otherwise report as `spawn-failed`
|
|
5310
|
+
// rather than the truthful `killed=SIGKILL`.
|
|
5311
|
+
const outcome = classifyOrphanDownResult({
|
|
5312
|
+
verdict,
|
|
5313
|
+
status: res ? res.status : null,
|
|
5314
|
+
signal: res ? res.signal : null,
|
|
5315
|
+
});
|
|
4372
5316
|
if (outcome.cleaned) {
|
|
4373
5317
|
summary.cleaned.push(n);
|
|
4374
5318
|
console.error(` VM-${n}: orphaned ingress → auto-cleaned (verified clean)`);
|
|
@@ -4689,6 +5633,10 @@ async function cloudMain() {
|
|
|
4689
5633
|
|
|
4690
5634
|
module.exports = {
|
|
4691
5635
|
// pure helpers
|
|
5636
|
+
isSpawnTimeout,
|
|
5637
|
+
gcloudCallTimeoutMs,
|
|
5638
|
+
provisionFailureMessage,
|
|
5639
|
+
GCLOUD_PROVISION_TIMEOUT_MS,
|
|
4692
5640
|
agentEnvVarName,
|
|
4693
5641
|
agentNpmPackage,
|
|
4694
5642
|
AGENT_ENV_VARS,
|
|
@@ -4699,6 +5647,9 @@ module.exports = {
|
|
|
4699
5647
|
assertAgentCliInstalled,
|
|
4700
5648
|
probeContainerRunning,
|
|
4701
5649
|
probeContainerState,
|
|
5650
|
+
waitForEditorComposeRunning,
|
|
5651
|
+
abortTeardownDecision,
|
|
5652
|
+
probeAbortTeardownState,
|
|
4702
5653
|
probeAgentBinaryInstalled,
|
|
4703
5654
|
waitForContainerRunning,
|
|
4704
5655
|
KNOWN_FLAGS,
|
|
@@ -4721,6 +5672,7 @@ module.exports = {
|
|
|
4721
5672
|
resolveConfiguredAppPort,
|
|
4722
5673
|
resolveSeedPreviewOrigin,
|
|
4723
5674
|
previewSameOriginSafeFromStack,
|
|
5675
|
+
resolveEffectiveSameOriginSafe,
|
|
4724
5676
|
isRottedManagedPreviewOrigin,
|
|
4725
5677
|
isWrongModePreviewOrigin,
|
|
4726
5678
|
isSilentlyBrokenPreview,
|
|
@@ -4758,11 +5710,29 @@ module.exports = {
|
|
|
4758
5710
|
decideBootstrapOutcome,
|
|
4759
5711
|
projectBootstrapDoneMarker,
|
|
4760
5712
|
projectBootstrapDoneProbeCommand,
|
|
5713
|
+
projectBootstrapScriptPath,
|
|
5714
|
+
projectBootstrapRunnerPath,
|
|
5715
|
+
projectBootstrapLogPath,
|
|
5716
|
+
projectBootstrapPidPath,
|
|
5717
|
+
projectBootstrapStatusPath,
|
|
5718
|
+
projectBootstrapRunnerScript,
|
|
5719
|
+
projectBootstrapLaunchCommand,
|
|
5720
|
+
projectBootstrapFollowCommand,
|
|
5721
|
+
projectBootstrapStatusCommand,
|
|
5722
|
+
parseProjectBootstrapStatus,
|
|
5723
|
+
decideProjectBootstrapOutcome,
|
|
5724
|
+
decideProjectBootstrapFollowAction,
|
|
5725
|
+
projectBootstrapEnv,
|
|
5726
|
+
PROJECT_BOOTSTRAP_LAUNCHED_MARKER,
|
|
5727
|
+
PROJECT_BOOTSTRAP_FOLLOW_SECS,
|
|
5728
|
+
ensureProjectBootstrapped,
|
|
4761
5729
|
SSH_KEEPALIVE_FLAGS,
|
|
4762
5730
|
resolveSetupForwardEnv,
|
|
4763
5731
|
FORWARD_ENV_KEYS_ENV,
|
|
4764
5732
|
parseEditorComposeState,
|
|
4765
5733
|
fetchInstanceState,
|
|
5734
|
+
fetchPendingOperation,
|
|
5735
|
+
wedgedOperationMessage,
|
|
4766
5736
|
formatGcloudFailure,
|
|
4767
5737
|
isTransientSshFailure,
|
|
4768
5738
|
runGcloudSshWithRetry,
|
|
@@ -4782,6 +5752,8 @@ module.exports = {
|
|
|
4782
5752
|
ingressWiringPlan,
|
|
4783
5753
|
fleetInstancePrefix,
|
|
4784
5754
|
runVmIngress,
|
|
5755
|
+
intendedVmPrincipals,
|
|
5756
|
+
assertAddCanAuthorizeSomeone,
|
|
4785
5757
|
wireVmIngressOnAdd,
|
|
4786
5758
|
probeVmIngressVerdict,
|
|
4787
5759
|
convergeVmIngressOnAdd,
|
|
@@ -4818,6 +5790,8 @@ module.exports = {
|
|
|
4818
5790
|
readVmAppPort,
|
|
4819
5791
|
buildVmAppPortReadCommand,
|
|
4820
5792
|
resolveLaptopGitIdentity,
|
|
5793
|
+
gitBranchRefExists,
|
|
5794
|
+
assertBranchRefsExist,
|
|
4821
5795
|
};
|
|
4822
5796
|
|
|
4823
5797
|
if (require.main === module) {
|