@codeyam-editor/codeyam-editor 0.1.0-staging.ga747ed2 → 0.1.0-staging.gac53b4e
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/npm/cloud.js +685 -73
- package/npm/compose-conflict.js +59 -0
- package/npm/editor-dev.js +24 -2
- package/npm/editor.js +15 -5
- package/npm/fleet-add-failure.js +520 -30
- package/npm/fleet-auth-account.js +61 -0
- package/npm/fleet-auth.js +69 -1
- package/npm/fleet-branch.js +124 -1
- package/npm/fleet-build-data.js +12 -1
- package/npm/fleet-cleanup.js +213 -3
- package/npm/fleet-datastore-health.js +107 -0
- package/npm/fleet-job-badge.js +28 -12
- package/npm/fleet-memory.js +73 -4
- package/npm/fleet-probe-transport.js +155 -0
- package/npm/fleet-public-url.js +81 -12
- package/npm/fleet-queue.js +141 -0
- package/npm/fleet-reconcile.js +46 -11
- package/npm/fleet-revision.js +212 -0
- package/npm/fleet-run-setup.js +66 -0
- package/npm/fleet-running-plan.js +37 -7
- package/npm/fleet-session-activity.js +162 -0
- package/npm/fleet-vm-access.js +162 -0
- package/npm/fleet-vm-readiness.js +57 -16
- package/npm/group-bounded-spawn.js +111 -0
- package/npm/ingress-invoke.js +46 -2
- package/npm/launch-config.js +379 -27
- package/npm/orphan-ssh-reaper.js +81 -0
- package/npm/scenario-check.js +760 -53
- package/npm/scenario-handlers.js +18 -0
- package/npm/scenario-interactivity.js +477 -23
- package/npm/scenario-issues.js +72 -0
- package/npm/scenario-mocks.js +245 -39
- package/npm/scenario-playwright.js +282 -36
- package/npm/utils.js +262 -22
- package/package.json +5 -5
- package/ui/dist/assets/{ContainerInspector-DgM0pTAx.js → ContainerInspector-DiN91foX.js} +1 -1
- package/ui/dist/assets/EditorShell-BEIdHnHd.css +1 -0
- package/ui/dist/assets/EditorShell-CPV5vhjo.js +330 -0
- package/ui/dist/assets/ProjectLauncherScreen-E0NYYiwy.js +1 -0
- package/ui/dist/assets/ScenarioDataPanel-DHAHyraq.js +18 -0
- package/ui/dist/assets/{ScenarioDataPanelFullPage-BLqRCxQX.js → ScenarioDataPanelFullPage-CHIWip3n.js} +1 -1
- package/ui/dist/assets/ScenarioRenderer-D9a91iwS.js +21 -0
- package/ui/dist/assets/index--cE0hUJw.css +1 -0
- package/ui/dist/assets/index-sW8wcg7v.js +27 -0
- package/ui/dist/assets/useServerIdentity-CVuHTzjg.js +1 -0
- package/ui/dist/index.html +2 -2
- package/ui/dist/preview-subdir-demo/index.html +62 -0
- package/ui/dist/preview-subdir-demo/sibling.html +48 -0
- package/ui/dist/assets/EditorShell-B7Q06uJ_.js +0 -252
- package/ui/dist/assets/EditorShell-DP5UY20G.css +0 -1
- package/ui/dist/assets/ProjectLauncherScreen-BRrEr8Ng.js +0 -1
- package/ui/dist/assets/ScenarioDataPanel-BeQfa28g.js +0 -9
- package/ui/dist/assets/ScenarioRenderer-CIAxfHPh.js +0 -21
- package/ui/dist/assets/index-BygCiJSb.css +0 -1
- package/ui/dist/assets/index-DCHqRrwO.js +0 -27
- package/ui/dist/assets/useServerIdentity-B7rI0u3O.js +0 -1
package/npm/cloud.js
CHANGED
|
@@ -23,6 +23,11 @@ const {
|
|
|
23
23
|
killChildProcess,
|
|
24
24
|
} = require("./utils");
|
|
25
25
|
const { open } = require("./open");
|
|
26
|
+
// Bounded gcloud/ssh spawns run through spawnGroupBoundedSync so a timeout takes
|
|
27
|
+
// gcloud's ssh descendant down with it instead of orphaning it to PID 1; the
|
|
28
|
+
// reaper sweeps sessions already leaked before that existed.
|
|
29
|
+
const { spawnGroupBoundedSync } = require("./group-bounded-spawn");
|
|
30
|
+
const { reapOrphanedFleetSsh } = require("./orphan-ssh-reaper");
|
|
26
31
|
// `FORWARD_ENV_KEYS_ENV` is the env-var name (`CY_FORWARD_ENV_KEYS`) the
|
|
27
32
|
// fleet dashboard sets on `cloud:up`'s child to name the subset of
|
|
28
33
|
// `process.env` keys that came from the operator-supplied launch form.
|
|
@@ -59,6 +64,7 @@ const {
|
|
|
59
64
|
normalizePrincipal,
|
|
60
65
|
resolveDefaultOperator,
|
|
61
66
|
withDefaultOperator,
|
|
67
|
+
unauthorizableAddRefusal,
|
|
62
68
|
accessFor,
|
|
63
69
|
setAccess,
|
|
64
70
|
addAccess,
|
|
@@ -105,6 +111,10 @@ const {
|
|
|
105
111
|
// edit (the invisible-VM gap improve38 closes). `readRoster` enumerates the
|
|
106
112
|
// fleet for `reconcile-ingress`.
|
|
107
113
|
const { addToRoster, readRoster } = require("./fleet-roster");
|
|
114
|
+
// Tells a docker container-name conflict (a concurrent run already started this
|
|
115
|
+
// container) apart from a real `docker compose up -d` failure, so the abort path
|
|
116
|
+
// can converge instead of escalating to instance deletion.
|
|
117
|
+
const { classifyComposeUpFailure } = require("./compose-conflict");
|
|
108
118
|
|
|
109
119
|
// =============================================================================
|
|
110
120
|
// Pure helpers — every export here is unit-tested in cloud.test.js without
|
|
@@ -509,6 +519,28 @@ function previewSameOriginSafeFromStack(stackJson) {
|
|
|
509
519
|
return typeof flag === "boolean" ? flag : true;
|
|
510
520
|
}
|
|
511
521
|
|
|
522
|
+
/** Resolve the sameOriginSafe flag a seed should ACT on, given the static
|
|
523
|
+
* `stack.json` reading and the running editor's own `/api/config` verdict.
|
|
524
|
+
*
|
|
525
|
+
* Precedence: a BOOLEAN effective verdict wins; anything else (`null`,
|
|
526
|
+
* `undefined`, a non-boolean) defers to the static flag. The type check is the
|
|
527
|
+
* whole rule — `null` means "no server was up to ask", which is the
|
|
528
|
+
* pre-container seed and must leave the static read in charge, NOT "the server
|
|
529
|
+
* said same-origin is fine". Collapsing the two would put the static flag back
|
|
530
|
+
* in control on exactly the path that needs the override.
|
|
531
|
+
*
|
|
532
|
+
* Extracted from `seedPreviewOrigin` because that function's inputs arrive over
|
|
533
|
+
* gcloud SSH, so this rule was only reachable through heavy mocking — while
|
|
534
|
+
* being precisely where the defect lived. `stack.json` cannot express the state
|
|
535
|
+
* that matters here: a project escalates off the same-origin subpath at RUNTIME
|
|
536
|
+
* after a hydration failure, and the adaptive-default stacks that happens to
|
|
537
|
+
* (Next.js) deliberately omit the key. So a seeder that re-derives the flag
|
|
538
|
+
* from the file writes back the same-origin sentinel that just failed, and the
|
|
539
|
+
* project sits at `previewDedicatedOriginUnavailable: true` forever. Pure. */
|
|
540
|
+
function resolveEffectiveSameOriginSafe(staticFlag, effectiveVerdict) {
|
|
541
|
+
return typeof effectiveVerdict === "boolean" ? effectiveVerdict : staticFlag;
|
|
542
|
+
}
|
|
543
|
+
|
|
512
544
|
// Matches an absolute cloudflared quick-tunnel origin
|
|
513
545
|
// (`https://<sub>.trycloudflare.com`, optionally with a port/path/query). These
|
|
514
546
|
// URLs ROTATE on every tunnel respawn, which is exactly why a frozen one rots.
|
|
@@ -845,6 +877,79 @@ function parseEditorComposeState(stdout) {
|
|
|
845
877
|
return null;
|
|
846
878
|
}
|
|
847
879
|
|
|
880
|
+
/** Decide whether `cloud:up`'s abort handler may delete the instance it is
|
|
881
|
+
* unwinding. Pure `{ createdThisRun, containerUp, rostered, probeInconclusive }
|
|
882
|
+
* → 'delete' | 'keep'`, matching the `reapVerdict` / `reattachVerdict` /
|
|
883
|
+
* `reconcileRosteredVm` pattern in npm/fleet-cleanup.js, so the whole matrix is
|
|
884
|
+
* exhaustively testable without a VM.
|
|
885
|
+
*
|
|
886
|
+
* `createdThisRun` stays a NECESSARY condition — never delete an instance this
|
|
887
|
+
* run did not create — but 2026-08-07 proved it is not SUFFICIENT. It answers
|
|
888
|
+
* "did I create this?", not "is it safe to destroy?", and in a concurrent
|
|
889
|
+
* provision race the losing run legitimately owns the flag while the winner has
|
|
890
|
+
* already brought the VM up and rostered it. Deleting there destroyed VM-4 and
|
|
891
|
+
* VM-5. So authorization also requires CURRENT-STATE evidence that the VM is
|
|
892
|
+
* out of service:
|
|
893
|
+
*
|
|
894
|
+
* - `containerUp` — the editor container is running (per the compose-ps probe).
|
|
895
|
+
* A VM serving an editor is in use by definition.
|
|
896
|
+
* - `rostered` — the VM is in the fleet roster, the operator-visible source of
|
|
897
|
+
* truth. Note this is true whether the winning run or THIS run rostered it;
|
|
898
|
+
* either way the VM is visible in the dashboard and removable in one command,
|
|
899
|
+
* so it is not the silent orphan teardown exists to prevent.
|
|
900
|
+
* - `probeInconclusive` — liveness could not be determined. This must fail
|
|
901
|
+
* SAFE: a failing probe is how we reach the abort handler in the first place,
|
|
902
|
+
* so it can never count as evidence the VM is dead. An orphan costs one
|
|
903
|
+
* `npm run cloud:destroy`; a wrong delete costs a 15-40 minute rebuild plus
|
|
904
|
+
* any unpushed work on the VM. */
|
|
905
|
+
function abortTeardownDecision({ createdThisRun, containerUp, rostered, probeInconclusive } = {}) {
|
|
906
|
+
if (!createdThisRun) return "keep";
|
|
907
|
+
if (probeInconclusive) return "keep";
|
|
908
|
+
if (containerUp) return "keep";
|
|
909
|
+
if (rostered) return "keep";
|
|
910
|
+
return "delete";
|
|
911
|
+
}
|
|
912
|
+
|
|
913
|
+
/** Gather the CURRENT-STATE inputs {@link abortTeardownDecision} needs, so the
|
|
914
|
+
* abort handler reads gather → decide → act instead of inlining probes among
|
|
915
|
+
* the delete calls.
|
|
916
|
+
*
|
|
917
|
+
* Only runs the probes when `createdThisRun` is true — the decision is `keep`
|
|
918
|
+
* regardless otherwise, and an abort path should not spend SSH round trips
|
|
919
|
+
* proving something it already knows. In that case it returns the
|
|
920
|
+
* probe-inconclusive shape, which is the safe default at every layer.
|
|
921
|
+
*
|
|
922
|
+
* Both lookups are wrapped: a throwing probe or an unreadable roster must fail
|
|
923
|
+
* SAFE (toward `keep`), never take a second exception path into the delete.
|
|
924
|
+
* `deps` is injectable so the matrix is unit-tested without a VM. */
|
|
925
|
+
function probeAbortTeardownState(cfg, createdThisRun, deps = {}) {
|
|
926
|
+
if (!createdThisRun) {
|
|
927
|
+
return { containerUp: false, rostered: false, probeInconclusive: true };
|
|
928
|
+
}
|
|
929
|
+
const probe = deps.probeEditorComposeState || probeEditorComposeState;
|
|
930
|
+
const roster = deps.readRoster || readRoster;
|
|
931
|
+
const resolveVm = deps.resolveFleetVmNumber || resolveFleetVmNumber;
|
|
932
|
+
const log = deps.log || ((m) => console.error(m));
|
|
933
|
+
|
|
934
|
+
let liveness = { state: null, inconclusive: true };
|
|
935
|
+
try {
|
|
936
|
+
liveness = probe(cfg);
|
|
937
|
+
} catch (e) {
|
|
938
|
+
log(` (liveness probe failed: ${e && e.message})`);
|
|
939
|
+
}
|
|
940
|
+
let rostered = false;
|
|
941
|
+
try {
|
|
942
|
+
const vm = resolveVm(cfg.instanceName);
|
|
943
|
+
rostered = vm != null && (roster() || []).includes(vm);
|
|
944
|
+
} catch { /* unreadable roster — treated as not-rostered; liveness still guards */ }
|
|
945
|
+
|
|
946
|
+
return {
|
|
947
|
+
containerUp: liveness.state === "running",
|
|
948
|
+
rostered,
|
|
949
|
+
probeInconclusive: liveness.inconclusive,
|
|
950
|
+
};
|
|
951
|
+
}
|
|
952
|
+
|
|
848
953
|
/** Translate one of the wrapper's verbs into the exact `gcloud` argv. The
|
|
849
954
|
* switch's job is to keep every per-verb command shape in one reviewable
|
|
850
955
|
* place — tests assert the arrays without ever spawning a process. */
|
|
@@ -880,6 +985,19 @@ function buildGcloudArgs(verb, cfg = {}, extras = {}) {
|
|
|
880
985
|
return ["compute", "instances", "delete", instanceName, "--project", project, "--zone", zone, "--quiet"];
|
|
881
986
|
case "instances-describe":
|
|
882
987
|
return ["compute", "instances", "describe", instanceName, "--project", project, "--zone", zone, "--format=json"];
|
|
988
|
+
// Outstanding (not-yet-DONE) zone operations targeting this instance. Used to
|
|
989
|
+
// tell a normal transient PROVISIONING from a wedged insert: GCE can accept an
|
|
990
|
+
// insert and then never finish it, leaving an operation parked at RUNNING/0%
|
|
991
|
+
// that holds the instance name forever. The status filter is what makes this
|
|
992
|
+
// cheap and unambiguous — a DONE operation is history and says nothing about
|
|
993
|
+
// whether the slot is currently blocked.
|
|
994
|
+
case "operations-list-pending":
|
|
995
|
+
return [
|
|
996
|
+
"compute", "operations", "list",
|
|
997
|
+
"--project", project,
|
|
998
|
+
"--filter", `zone:(${zone}) AND targetLink~${instanceName}$ AND status!=DONE`,
|
|
999
|
+
"--format=json",
|
|
1000
|
+
];
|
|
883
1001
|
case "instances-list-tagged":
|
|
884
1002
|
return [
|
|
885
1003
|
"compute", "instances", "list",
|
|
@@ -976,8 +1094,45 @@ function buildGcloudArgs(verb, cfg = {}, extras = {}) {
|
|
|
976
1094
|
// reconcile spawnSync sites (status probe, per-BES timeout converge) so all three
|
|
977
1095
|
// reference one value. 2 min is generous for any single describe/update/SSH probe
|
|
978
1096
|
// yet comfortably under the 600000 ms outer reconcile budget.
|
|
1097
|
+
//
|
|
1098
|
+
// All three spawn through spawnGroupBoundedSync, because a bare spawnSync timeout
|
|
1099
|
+
// signals only the direct child: gcloud died and its `ssh` survived, re-parented
|
|
1100
|
+
// to PID 1 — the budget bounded the wrapper, never the descendant.
|
|
1101
|
+
//
|
|
1102
|
+
// It is a PROBE budget, and provisioning is exempt — see
|
|
1103
|
+
// GCLOUD_PROVISION_TIMEOUT_MS below. Do not re-collapse the two: raising this
|
|
1104
|
+
// constant to accommodate a slow create would un-bound the reconcile sweep,
|
|
1105
|
+
// which is the exact orphan-cascade this value exists to stop.
|
|
979
1106
|
const GCLOUD_CALL_TIMEOUT_MS = 120000;
|
|
980
1107
|
|
|
1108
|
+
// `instances create` / `instances start` are NOT probes: each blocks until GCE's
|
|
1109
|
+
// insert (or start) operation reaches a terminal state, which routinely outruns
|
|
1110
|
+
// the probe budget under zonal contention. Inheriting GCLOUD_CALL_TIMEOUT_MS
|
|
1111
|
+
// SIGTERMd gcloud mid-insert; `spawnSync` then returned `status: null`, which
|
|
1112
|
+
// `!== 0`, so a local kill was indistinguishable from a remote rejection and the
|
|
1113
|
+
// caller threw a bare `Failed to create instance <name>` with no cause in it.
|
|
1114
|
+
// That mattered downstream: every arm of classifyAddFailure keys on a transcript
|
|
1115
|
+
// signature, so a causeless log fell through to the optimistic `verifying`
|
|
1116
|
+
// fallback and told the operator to WAIT on a VM GCE had already wedged
|
|
1117
|
+
// (observed 2026-09-04 on VM-8: insert 14:29:50Z, add errored 14:32:26Z — the
|
|
1118
|
+
// 120s budget plus teardown).
|
|
1119
|
+
//
|
|
1120
|
+
// Deliberately above the 600000 ms outer reconcile budget, on the same reasoning
|
|
1121
|
+
// as PROJECT_BOOTSTRAP_TIMEOUT_MS: ensureInstance is only ever reached from the
|
|
1122
|
+
// add path, never from the reconcile sweep, so the sweep-relative ceiling that
|
|
1123
|
+
// governs GCLOUD_CALL_TIMEOUT_MS does not apply. A create still running at 10
|
|
1124
|
+
// minutes is wedged, not slow.
|
|
1125
|
+
const GCLOUD_PROVISION_TIMEOUT_MS = 10 * 60 * 1000;
|
|
1126
|
+
|
|
1127
|
+
/** The per-call spawn budget a given wrapper verb should run under. Provisioning
|
|
1128
|
+
* verbs get GCLOUD_PROVISION_TIMEOUT_MS; everything else keeps the probe budget.
|
|
1129
|
+
* Pure — the mapping is asserted directly, without spawning a process. */
|
|
1130
|
+
function gcloudCallTimeoutMs(verb) {
|
|
1131
|
+
return verb === "instances-create" || verb === "instances-start"
|
|
1132
|
+
? GCLOUD_PROVISION_TIMEOUT_MS
|
|
1133
|
+
: GCLOUD_CALL_TIMEOUT_MS;
|
|
1134
|
+
}
|
|
1135
|
+
|
|
981
1136
|
// The project bootstrap is NOT a probe: it runs `git clone` + `npm install` in
|
|
982
1137
|
// the repo root and every `apps[].dir`, each package's `postinstall`, then
|
|
983
1138
|
// `npm run setup` — minutes of remote work on a heavy repo. It once ran as ONE
|
|
@@ -1015,7 +1170,7 @@ function isSpawnTimeout(result) {
|
|
|
1015
1170
|
}
|
|
1016
1171
|
|
|
1017
1172
|
function runGcloud(args, opts = {}) {
|
|
1018
|
-
return
|
|
1173
|
+
return spawnGroupBoundedSync("gcloud", args, {
|
|
1019
1174
|
stdio: opts.stdio || "inherit",
|
|
1020
1175
|
input: opts.input,
|
|
1021
1176
|
encoding: opts.encoding || "utf8",
|
|
@@ -1050,6 +1205,66 @@ function formatGcloudFailure(result) {
|
|
|
1050
1205
|
return "(no output captured from gcloud)";
|
|
1051
1206
|
}
|
|
1052
1207
|
|
|
1208
|
+
/** The thrown message for a failed `instances create` / `instances start`.
|
|
1209
|
+
*
|
|
1210
|
+
* Splits the two outcomes the old bare `Failed to create instance <name>`
|
|
1211
|
+
* collapsed, because they are different operator actions: a LOCAL timeout means
|
|
1212
|
+
* GCE may still be working the operation and it should be checked, while a
|
|
1213
|
+
* remote rejection means GCE said no and its own stderr is the answer. Both
|
|
1214
|
+
* arms put a cause in the transcript — classifyAddFailure keys on transcript
|
|
1215
|
+
* text, so a causeless message is what lands an add on the optimistic
|
|
1216
|
+
* `verifying` fallback. Pure: result in, string out.
|
|
1217
|
+
*
|
|
1218
|
+
* `verb` is the operator-facing word ("create" / "start"), not a wrapper verb. */
|
|
1219
|
+
/** Operator-facing copy for a create GCE accepted and then abandoned, kept
|
|
1220
|
+
* separate from the guard that detects it for the same reason
|
|
1221
|
+
* provisionFailureMessage is: the decision of WHEN to refuse and the wording of
|
|
1222
|
+
* WHAT to say are different concerns, and only the wording needs asserting.
|
|
1223
|
+
* Extracting it is what lets the copy be tested without a live gcloud — and on
|
|
1224
|
+
* this feature the copy IS the fix, since the bug was a message that told the
|
|
1225
|
+
* operator to wait for a VM that would never arrive.
|
|
1226
|
+
*
|
|
1227
|
+
* Every field is rendered defensively because this runs on a degraded path: the
|
|
1228
|
+
* operations read that produced `pending` is best-effort, so a partial record
|
|
1229
|
+
* must still yield a message that names the problem rather than printing
|
|
1230
|
+
* "undefined" at the operator. */
|
|
1231
|
+
function wedgedOperationMessage(instanceName, pending = {}) {
|
|
1232
|
+
const detail = [
|
|
1233
|
+
`${pending.name || "unnamed"}: ${pending.operationType || "unknown type"}`,
|
|
1234
|
+
`status ${pending.status || "unknown"}`,
|
|
1235
|
+
`progress ${pending.progress == null ? "unknown" : `${pending.progress}%`}`,
|
|
1236
|
+
`error ${pending.errorCode || "unknown"}${pending.errorMessage ? `: ${pending.errorMessage}` : ""}`,
|
|
1237
|
+
].join(", ");
|
|
1238
|
+
return (
|
|
1239
|
+
`Instance ${instanceName} is stuck in PROVISIONING behind a wedged create operation (${detail}). ` +
|
|
1240
|
+
"GCE accepted the create and never finished it, so this slot cannot be re-added and a delete will queue " +
|
|
1241
|
+
"behind the wedged operation rather than clear it. This needs GCE-side resolution — open a Google Cloud " +
|
|
1242
|
+
"Support case citing that error. Use a different VM number if you need a box now."
|
|
1243
|
+
);
|
|
1244
|
+
}
|
|
1245
|
+
|
|
1246
|
+
function provisionFailureMessage(verb, instanceName, result) {
|
|
1247
|
+
if (isSpawnTimeout(result)) {
|
|
1248
|
+
const minutes = Math.round(GCLOUD_PROVISION_TIMEOUT_MS / 60000);
|
|
1249
|
+
// A SIGTERMd child legitimately often captured nothing, and appending
|
|
1250
|
+
// formatGcloudFailure's "(no output captured)" placeholder to a message that
|
|
1251
|
+
// already states the cause is noise. Append it only when it is real output.
|
|
1252
|
+
const detail = formatGcloudFailure(result);
|
|
1253
|
+
const captured = detail === "(no output captured from gcloud)" ? "" : `\n${detail}`;
|
|
1254
|
+
return (
|
|
1255
|
+
`Failed to ${verb} instance ${instanceName}: the gcloud call exceeded the local ` +
|
|
1256
|
+
`${minutes}-minute provisioning budget and was SIGTERMd locally. GCE may still be ` +
|
|
1257
|
+
`working the operation — check it with \`gcloud compute operations list\` before ` +
|
|
1258
|
+
`retrying, and delete the half-built instance if one exists.` +
|
|
1259
|
+
captured
|
|
1260
|
+
);
|
|
1261
|
+
}
|
|
1262
|
+
return (
|
|
1263
|
+
`Failed to ${verb} instance ${instanceName} (gcloud exit ${result && result.status}).\n` +
|
|
1264
|
+
formatGcloudFailure(result)
|
|
1265
|
+
);
|
|
1266
|
+
}
|
|
1267
|
+
|
|
1053
1268
|
// A bounded, pure classifier that keys on status === 255 (gcloud-ssh's exit
|
|
1054
1269
|
// for an ssh failure) and a signature match against known transient strings.
|
|
1055
1270
|
function isTransientSshFailure(result) {
|
|
@@ -1125,6 +1340,38 @@ function fetchInstanceState(cfg) {
|
|
|
1125
1340
|
}
|
|
1126
1341
|
}
|
|
1127
1342
|
|
|
1343
|
+
/** Read the instance's outstanding (not-DONE) zone operations and summarize the
|
|
1344
|
+
* first one, or null when there are none / the call failed. Shaped as
|
|
1345
|
+
* { name, operationType, status, progress, errorCode, errorMessage } — the fields
|
|
1346
|
+
* the wedged-insert refusal message and the fleet classifier both need.
|
|
1347
|
+
*
|
|
1348
|
+
* Returning null on a failed or unparseable call is deliberate: this is a
|
|
1349
|
+
* DIAGNOSTIC read used to make a refusal more informative, so it must never turn
|
|
1350
|
+
* a working provision into a failing one just because the operations API hiccuped. */
|
|
1351
|
+
function fetchPendingOperation(cfg) {
|
|
1352
|
+
const result = runGcloud(buildGcloudArgs("operations-list-pending", cfg), {
|
|
1353
|
+
stdio: ["ignore", "pipe", "ignore"],
|
|
1354
|
+
});
|
|
1355
|
+
if (result.status !== 0) return null;
|
|
1356
|
+
let ops;
|
|
1357
|
+
try {
|
|
1358
|
+
ops = JSON.parse(result.stdout);
|
|
1359
|
+
} catch {
|
|
1360
|
+
return null;
|
|
1361
|
+
}
|
|
1362
|
+
if (!Array.isArray(ops) || ops.length === 0) return null;
|
|
1363
|
+
const op = ops[0];
|
|
1364
|
+
const err = op && op.error && Array.isArray(op.error.errors) ? op.error.errors[0] : null;
|
|
1365
|
+
return {
|
|
1366
|
+
name: op.name || null,
|
|
1367
|
+
operationType: op.operationType || null,
|
|
1368
|
+
status: op.status || null,
|
|
1369
|
+
progress: typeof op.progress === "number" ? op.progress : null,
|
|
1370
|
+
errorCode: (err && err.code) || null,
|
|
1371
|
+
errorMessage: (err && err.message) || null,
|
|
1372
|
+
};
|
|
1373
|
+
}
|
|
1374
|
+
|
|
1128
1375
|
function loadOrInitCloudConfig(projectDir, flags) {
|
|
1129
1376
|
const existing = loadCloudConfig(projectDir);
|
|
1130
1377
|
// noTunnel is a per-invocation behavior, never persisted — else a later manual
|
|
@@ -1206,7 +1453,11 @@ async function ensureFirewallRule(cfg) {
|
|
|
1206
1453
|
}
|
|
1207
1454
|
}
|
|
1208
1455
|
|
|
1209
|
-
|
|
1456
|
+
// `sleepFn` is injectable so the 60-iteration wait loop can be driven with a
|
|
1457
|
+
// no-op instead of burning 2 real minutes of clock — the same seam
|
|
1458
|
+
// runGcloudSshWithRetry / waitForEditorComposeRunning already expose.
|
|
1459
|
+
async function ensureInstance(cfg, { sleepFn } = {}) {
|
|
1460
|
+
const sleep = sleepFn || ((ms) => new Promise((r) => setTimeout(r, ms)));
|
|
1210
1461
|
console.error(`Checking instance ${cfg.instanceName}...`);
|
|
1211
1462
|
const initialState = fetchInstanceState(cfg);
|
|
1212
1463
|
const createdThisRun = initialState === null;
|
|
@@ -1217,29 +1468,59 @@ async function ensureInstance(cfg) {
|
|
|
1217
1468
|
}
|
|
1218
1469
|
if (initialState === "TERMINATED") {
|
|
1219
1470
|
console.error(` ${cfg.instanceName} is TERMINATED — starting...`);
|
|
1220
|
-
const start = runGcloud(buildGcloudArgs("instances-start", cfg)
|
|
1471
|
+
const start = runGcloud(buildGcloudArgs("instances-start", cfg), {
|
|
1472
|
+
timeout: gcloudCallTimeoutMs("instances-start"),
|
|
1473
|
+
});
|
|
1221
1474
|
if (start.status !== 0) {
|
|
1222
|
-
throw new Error(
|
|
1475
|
+
throw new Error(provisionFailureMessage("start", cfg.instanceName, start));
|
|
1223
1476
|
}
|
|
1224
1477
|
} else if (initialState === null) {
|
|
1225
1478
|
console.error(` ${cfg.instanceName} not found — creating...`);
|
|
1226
|
-
const create = runGcloud(buildGcloudArgs("instances-create", cfg)
|
|
1479
|
+
const create = runGcloud(buildGcloudArgs("instances-create", cfg), {
|
|
1480
|
+
timeout: gcloudCallTimeoutMs("instances-create"),
|
|
1481
|
+
});
|
|
1227
1482
|
if (create.status !== 0) {
|
|
1228
|
-
throw new Error(
|
|
1483
|
+
throw new Error(provisionFailureMessage("create", cfg.instanceName, create));
|
|
1229
1484
|
}
|
|
1230
1485
|
} else {
|
|
1486
|
+
// A slot found in PROVISIONING is ambiguous, and the two readings need
|
|
1487
|
+
// opposite handling. On a normal add it is a transient the poll loop below
|
|
1488
|
+
// resolves in seconds. But GCE can also accept an insert and then never
|
|
1489
|
+
// finish it (2026-09-04, VM-8): the operation parks at RUNNING/0%, eventually
|
|
1490
|
+
// attaches an INTERNAL_ERROR without ever reaching DONE, and holds the
|
|
1491
|
+
// instance name indefinitely — a delete queues behind it rather than clearing
|
|
1492
|
+
// it. Polling that for 2 minutes only rediscovers, slowly and with a message
|
|
1493
|
+
// naming neither the operation nor the wedge, a state the first describe
|
|
1494
|
+
// already showed. Read the outstanding operation instead and fail immediately
|
|
1495
|
+
// when it carries a terminal error, so the transcript carries the signature
|
|
1496
|
+
// fleet-add-failure.js's wedged-insert arm keys on.
|
|
1497
|
+
if (initialState === "PROVISIONING") {
|
|
1498
|
+
const pending = fetchPendingOperation(cfg);
|
|
1499
|
+
if (pending && pending.errorCode) {
|
|
1500
|
+
throw new Error(wedgedOperationMessage(cfg.instanceName, pending));
|
|
1501
|
+
}
|
|
1502
|
+
}
|
|
1231
1503
|
console.error(` ${cfg.instanceName} is ${initialState}; waiting...`);
|
|
1232
1504
|
}
|
|
1233
1505
|
|
|
1506
|
+
let lastState = initialState;
|
|
1234
1507
|
for (let i = 0; i < 60; i++) {
|
|
1235
|
-
|
|
1508
|
+
lastState = fetchInstanceState(cfg);
|
|
1509
|
+
if (lastState === "RUNNING") {
|
|
1236
1510
|
console.error(` ✓ ${cfg.instanceName} is RUNNING`);
|
|
1237
1511
|
await waitForIapReady(cfg);
|
|
1238
1512
|
return { createdThisRun };
|
|
1239
1513
|
}
|
|
1240
|
-
await
|
|
1514
|
+
await sleep(2000);
|
|
1241
1515
|
}
|
|
1242
|
-
|
|
1516
|
+
// Name the state it was actually stuck in. An instance parked in PROVISIONING
|
|
1517
|
+
// is a different operator action from one that stopped answering describe at
|
|
1518
|
+
// all, and the old message ("did not reach RUNNING") distinguished neither.
|
|
1519
|
+
throw new Error(
|
|
1520
|
+
`Instance ${cfg.instanceName} did not reach RUNNING within 2 minutes ` +
|
|
1521
|
+
`(last observed state: ${lastState || "unknown — describe returned no state"}` +
|
|
1522
|
+
`${createdThisRun ? "; a create was issued this run" : ""}).`,
|
|
1523
|
+
);
|
|
1243
1524
|
}
|
|
1244
1525
|
|
|
1245
1526
|
/** GCE reports the instance as RUNNING the moment the boot disk is attached,
|
|
@@ -1707,17 +1988,32 @@ function projectBootstrapRunnerScript(repoName) {
|
|
|
1707
1988
|
// makes the outcome probe honest: after a launch, a present `.done` can only
|
|
1708
1989
|
// mean THIS run's script reached its terminal line, never a leftover from a
|
|
1709
1990
|
// previous half-wired attempt.
|
|
1710
|
-
function projectBootstrapLaunchCommand(scriptBody, scriptArgs = [], envVars = {}, repoName) {
|
|
1991
|
+
function projectBootstrapLaunchCommand(scriptBody, scriptArgs = [], envVars = {}, repoName, libFiles = {}) {
|
|
1711
1992
|
const scriptB64 = Buffer.from(scriptBody, "utf8").toString("base64");
|
|
1712
1993
|
const runnerB64 = Buffer.from(projectBootstrapRunnerScript(repoName), "utf8").toString("base64");
|
|
1713
1994
|
const argsString = scriptArgs.map(shQuote).join(" ");
|
|
1714
1995
|
const envPrefix = envAssignmentPrefix(envVars);
|
|
1996
|
+
// The bootstrap script is written STANDALONE to $HOME, so any `source`/`.` it
|
|
1997
|
+
// does relative to its own directory resolves against $HOME — not against the
|
|
1998
|
+
// repo layout it was authored in. `libFiles` carries those siblings, keyed by
|
|
1999
|
+
// the $HOME-relative path the script will look for them at (e.g.
|
|
2000
|
+
// "lib/npm-pin.sh"). Their directories do not exist on a fresh VM, hence the
|
|
2001
|
+
// mkdir. Omitting a sourced library is not a soft failure: the script aborts
|
|
2002
|
+
// under `set -euo pipefail` and cloud:up tears the new VM down, which is the
|
|
2003
|
+
// 2026-09-04 fleet-wide add outage.
|
|
2004
|
+
const libWrites = Object.entries(libFiles).flatMap(([relPath, body]) => {
|
|
2005
|
+
const b64 = Buffer.from(body, "utf8").toString("base64");
|
|
2006
|
+
const dir = path.posix.dirname(relPath);
|
|
2007
|
+
const mkdir = dir && dir !== "." ? [`mkdir -p $HOME/${dir}`] : [];
|
|
2008
|
+
return [...mkdir, `echo '${b64}' | base64 -d > $HOME/${relPath}`];
|
|
2009
|
+
});
|
|
1715
2010
|
// Writes run synchronously (fast, before any backgrounding) so a decode error
|
|
1716
2011
|
// aborts before launch; then the runner is detached. The env prefix is on the
|
|
1717
2012
|
// setsid command so the tokens are inherited through nohup→setsid→bash but
|
|
1718
2013
|
// never appear in any argv. `& echo MARKER` (no `;` before echo) keeps the
|
|
1719
2014
|
// launch in the background while still confirming it fired.
|
|
1720
2015
|
const writes = [
|
|
2016
|
+
...libWrites,
|
|
1721
2017
|
`echo '${scriptB64}' | base64 -d > ${projectBootstrapScriptPath(repoName)}`,
|
|
1722
2018
|
`echo '${runnerB64}' | base64 -d > ${projectBootstrapRunnerPath(repoName)}`,
|
|
1723
2019
|
`rm -f ${projectBootstrapStatusPath(repoName)} ${projectBootstrapDoneMarker(repoName)}`,
|
|
@@ -1949,6 +2245,13 @@ async function ensureProjectBootstrapped(cfg, { sleepFn } = {}) {
|
|
|
1949
2245
|
console.error(` streaming scripts/gcp-project-bootstrap.sh to VM for ${cfg.repo}...`);
|
|
1950
2246
|
const scriptPath = path.join(rootDir, "scripts", "gcp-project-bootstrap.sh");
|
|
1951
2247
|
const scriptBody = readStreamedScript(scriptPath, cfg);
|
|
2248
|
+
// gcp-project-bootstrap.sh sources lib/npm-pin.sh relative to its own
|
|
2249
|
+
// directory. Read it through readStreamedScript too, so the library is subject
|
|
2250
|
+
// to the same drift guard as the script that sources it — streaming a reviewed
|
|
2251
|
+
// remote script beside a stale on-disk library would defeat the guard.
|
|
2252
|
+
const bootstrapLibs = {
|
|
2253
|
+
"lib/npm-pin.sh": readStreamedScript(path.join(rootDir, "scripts", "lib", "npm-pin.sh"), cfg),
|
|
2254
|
+
};
|
|
1952
2255
|
const scriptArgs = clientProjectBootstrapArgs(cfg);
|
|
1953
2256
|
// Git identity, forwarded launch-form keys, fleet-domain ingress and the admin
|
|
1954
2257
|
// toggle — see projectBootstrapEnv for what each key is for and when it is
|
|
@@ -1976,7 +2279,7 @@ async function ensureProjectBootstrapped(cfg, { sleepFn } = {}) {
|
|
|
1976
2279
|
// install that will survive an SSH disconnect. Every step of
|
|
1977
2280
|
// gcp-project-bootstrap.sh is skip-on-exists, so re-entering is cheap.
|
|
1978
2281
|
console.error(" launching detached project bootstrap on the VM (survives SSH disconnect)...");
|
|
1979
|
-
const launch = projectBootstrapLaunchCommand(scriptBody, scriptArgs, bootstrapEnv, repoName);
|
|
2282
|
+
const launch = projectBootstrapLaunchCommand(scriptBody, scriptArgs, bootstrapEnv, repoName, bootstrapLibs);
|
|
1980
2283
|
const launched = await runGcloudSshWithRetry(
|
|
1981
2284
|
"ssh-script",
|
|
1982
2285
|
cfg,
|
|
@@ -2049,20 +2352,65 @@ async function ensureProjectBootstrapped(cfg, { sleepFn } = {}) {
|
|
|
2049
2352
|
}
|
|
2050
2353
|
}
|
|
2051
2354
|
|
|
2052
|
-
|
|
2355
|
+
/** Run the compose-ps liveness probe over SSH and report BOTH the editor
|
|
2356
|
+
* service's state and whether the probe itself could be trusted.
|
|
2357
|
+
*
|
|
2358
|
+
* The `|| true` keeps the remote pipeline's exit status 0 whenever SSH itself
|
|
2359
|
+
* worked, so a non-zero `status` means the transport failed (auth/SSH/IAP) —
|
|
2360
|
+
* not that the container is down. Callers that must fail safe (the abort
|
|
2361
|
+
* teardown) need that distinction; `parseEditorComposeState` alone collapses
|
|
2362
|
+
* "no container" and "could not ask" into the same `null`.
|
|
2363
|
+
*
|
|
2364
|
+
* Returns `{ state, inconclusive }` where `state` is the compose `State` string
|
|
2365
|
+
* (`"running"`, `"exited"`, …) or null when the service is absent, and
|
|
2366
|
+
* `inconclusive` is true only for a transport failure. */
|
|
2367
|
+
function probeEditorComposeState(cfg) {
|
|
2053
2368
|
const repoName = resolveProject(cfg).name;
|
|
2054
|
-
// -f docker-compose.cloud.yml everywhere: the editor repo's own
|
|
2055
|
-
// docker-compose.yml does `build: .` from source and would shadow the
|
|
2056
|
-
// cloud variant when self-hosting (see gcp-project-bootstrap.sh).
|
|
2057
2369
|
const composeCmd = `docker compose -f docker-compose.cloud.yml`;
|
|
2058
|
-
console.error("Checking editor container state...");
|
|
2059
2370
|
const probe = runGcloud(
|
|
2060
2371
|
buildGcloudArgs("ssh-probe", cfg, {
|
|
2061
|
-
command: `cd ~/projects/${repoName} && ${composeCmd} ps --format json 2>/dev/null || true`,
|
|
2372
|
+
command: `cd ~/projects/${repoName} 2>/dev/null && ${composeCmd} ps --format json 2>/dev/null || true`,
|
|
2062
2373
|
}),
|
|
2063
2374
|
{ stdio: ["ignore", "pipe", "ignore"] },
|
|
2064
2375
|
);
|
|
2065
|
-
if (probe.status
|
|
2376
|
+
if (probe.status !== 0) {
|
|
2377
|
+
return { state: null, inconclusive: true };
|
|
2378
|
+
}
|
|
2379
|
+
return { state: parseEditorComposeState(probe.stdout), inconclusive: false };
|
|
2380
|
+
}
|
|
2381
|
+
|
|
2382
|
+
/** Poll {@link probeEditorComposeState} until the editor service reads
|
|
2383
|
+
* `running`, returning true as soon as it does. Used to close the TOCTOU window
|
|
2384
|
+
* after a container-name conflict: the winning run may still be inside
|
|
2385
|
+
* `Creating`/`Created` when the conflict surfaces, so a single immediate probe
|
|
2386
|
+
* can under-read and report a healthy container as absent.
|
|
2387
|
+
*
|
|
2388
|
+
* Deliberately short (default ~30s): this is not the multi-minute first-boot
|
|
2389
|
+
* budget `waitForContainerRunning` grants, it is the tail of a start that
|
|
2390
|
+
* another process has already begun. `sleepFn` and `probeFn` are injectable so
|
|
2391
|
+
* a unit test drives the whole loop with neither wall-clock delay nor a VM. */
|
|
2392
|
+
async function waitForEditorComposeRunning(cfg, { attempts = 6, delayMs = 5000, sleepFn, probeFn } = {}) {
|
|
2393
|
+
const sleep = sleepFn || ((ms) => new Promise((r) => setTimeout(r, ms)));
|
|
2394
|
+
const probe = probeFn || probeEditorComposeState;
|
|
2395
|
+
for (let i = 0; i < attempts; i++) {
|
|
2396
|
+
if (probe(cfg).state === "running") {
|
|
2397
|
+
return true;
|
|
2398
|
+
}
|
|
2399
|
+
if (i < attempts - 1) {
|
|
2400
|
+
await sleep(delayMs);
|
|
2401
|
+
}
|
|
2402
|
+
}
|
|
2403
|
+
return false;
|
|
2404
|
+
}
|
|
2405
|
+
|
|
2406
|
+
async function ensureEditorContainerUp(cfg) {
|
|
2407
|
+
const repoName = resolveProject(cfg).name;
|
|
2408
|
+
// -f docker-compose.cloud.yml everywhere: the editor repo's own
|
|
2409
|
+
// docker-compose.yml does `build: .` from source and would shadow the
|
|
2410
|
+
// cloud variant when self-hosting (see gcp-project-bootstrap.sh).
|
|
2411
|
+
const composeCmd = `docker compose -f docker-compose.cloud.yml`;
|
|
2412
|
+
console.error("Checking editor container state...");
|
|
2413
|
+
if (probeEditorComposeState(cfg).state === "running") {
|
|
2066
2414
|
console.error(" ✓ editor container is already running");
|
|
2067
2415
|
// containerStarted: the container is confirmed up, so the CLI install runs
|
|
2068
2416
|
// its POST-start path with no wait — see the precondition on ensureAgentCliInstalled.
|
|
@@ -2070,14 +2418,47 @@ async function ensureEditorContainerUp(cfg) {
|
|
|
2070
2418
|
return;
|
|
2071
2419
|
}
|
|
2072
2420
|
console.error(" starting editor container (docker compose up -d)...");
|
|
2421
|
+
// CAPTURE rather than `stdio: "inherit"`: the daemon writes its container-name
|
|
2422
|
+
// conflict to stderr, and with "inherit" that text went straight to our parent
|
|
2423
|
+
// and was never observable here — so the conflict could not be told apart from
|
|
2424
|
+
// a real failure (the 2026-08-07 VM-4/VM-5 delete). spawnSync cannot both
|
|
2425
|
+
// stream and capture, so the operator's view is preserved by echoing the
|
|
2426
|
+
// captured streams immediately below instead of live. Do NOT revert this to
|
|
2427
|
+
// "inherit"; classification depends on it.
|
|
2073
2428
|
const up = runGcloud(
|
|
2074
2429
|
buildGcloudArgs("ssh-probe", cfg, {
|
|
2075
2430
|
command: `cd ~/projects/${repoName} && ${composeCmd} up -d`,
|
|
2076
2431
|
}),
|
|
2077
|
-
{ stdio: "
|
|
2432
|
+
{ stdio: ["ignore", "pipe", "pipe"] },
|
|
2078
2433
|
);
|
|
2434
|
+
if (up.stdout) process.stderr.write(up.stdout);
|
|
2435
|
+
if (up.stderr) process.stderr.write(up.stderr);
|
|
2079
2436
|
if (up.status !== 0) {
|
|
2080
|
-
|
|
2437
|
+
const verdict = classifyComposeUpFailure({ status: up.status, stderr: `${up.stderr || ""}\n${up.stdout || ""}` });
|
|
2438
|
+
if (verdict !== "name-conflict") {
|
|
2439
|
+
throw new Error("docker compose up -d failed on the VM");
|
|
2440
|
+
}
|
|
2441
|
+
// A name conflict means a concurrent run already created this container —
|
|
2442
|
+
// the goal we were about to pursue is being reached by someone else. Confirm
|
|
2443
|
+
// that positively before converging: a conflict whose container never
|
|
2444
|
+
// reaches `running` (e.g. a stale STOPPED container squatting the name) is a
|
|
2445
|
+
// genuine problem and must still fail loudly, or convergence becomes a
|
|
2446
|
+
// catch-all that papers over real breakage.
|
|
2447
|
+
console.error(
|
|
2448
|
+
" a concurrent run already created the editor container — confirming it reaches running...",
|
|
2449
|
+
);
|
|
2450
|
+
if (!(await waitForEditorComposeRunning(cfg))) {
|
|
2451
|
+
throw new Error(
|
|
2452
|
+
"docker compose up -d hit a container-name conflict on the VM, but the conflicting " +
|
|
2453
|
+
"container never reached running (a stale stopped container may be holding the name). " +
|
|
2454
|
+
`Inspect with: gcloud compute ssh ${cfg.instanceName} --command 'cd ~/projects/${repoName} && ${composeCmd} ps -a'`,
|
|
2455
|
+
);
|
|
2456
|
+
}
|
|
2457
|
+
console.error(" ✓ editor container is running (started by a concurrent run)");
|
|
2458
|
+
// containerStarted: the container is confirmed up — same post-start contract
|
|
2459
|
+
// as the already-running short-circuit above.
|
|
2460
|
+
await ensureAgentCliInstalled(cfg, { containerStarted: true });
|
|
2461
|
+
return;
|
|
2081
2462
|
}
|
|
2082
2463
|
// containerStarted: `up -d` returned 0, so the CLI install is legitimately a
|
|
2083
2464
|
// post-start step. A brief wait-for-running here (slow boot settling) is
|
|
@@ -2093,18 +2474,10 @@ async function ensureEditorContainerUp(cfg) {
|
|
|
2093
2474
|
// success. This throws (non-zero exit) when the editor service isn't
|
|
2094
2475
|
// `running`, converting a silent partial provision into a loud, actionable
|
|
2095
2476
|
// failure the operator (and the dashboard's fail-fast-on-nonzero path) can act
|
|
2096
|
-
// on.
|
|
2097
|
-
//
|
|
2477
|
+
// on. An inconclusive probe is not "running" here — a terminal gate must have
|
|
2478
|
+
// positive evidence, and the caller's job is to fail loudly without it.
|
|
2098
2479
|
async function assertEditorContainerRunning(cfg) {
|
|
2099
|
-
|
|
2100
|
-
const composeCmd = `docker compose -f docker-compose.cloud.yml`;
|
|
2101
|
-
const probe = runGcloud(
|
|
2102
|
-
buildGcloudArgs("ssh-probe", cfg, {
|
|
2103
|
-
command: `cd ~/projects/${repoName} 2>/dev/null && ${composeCmd} ps --format json 2>/dev/null || true`,
|
|
2104
|
-
}),
|
|
2105
|
-
{ stdio: ["ignore", "pipe", "ignore"] },
|
|
2106
|
-
);
|
|
2107
|
-
if (!(probe.status === 0 && parseEditorComposeState(probe.stdout) === "running")) {
|
|
2480
|
+
if (probeEditorComposeState(cfg).state !== "running") {
|
|
2108
2481
|
throw new Error(
|
|
2109
2482
|
`provisioning incomplete: editor container is not running on ${cfg.instanceName} ` +
|
|
2110
2483
|
`(no /workspace or container). Re-run: npm run cloud:up -- --instance-name ${cfg.instanceName}`,
|
|
@@ -2657,8 +3030,30 @@ function buildEditorLocalSeedWriteCommand({ repoName, remotePath, b64 }) {
|
|
|
2657
3030
|
* `_managedPreviewOrigin` sentinel). We stamp the sentinel here too so the
|
|
2658
3031
|
* monitor doesn't mistake the cloud:up seed for a user pin. The
|
|
2659
3032
|
* skip-on-already-set guard stays: a non-empty previewOrigin (whether a user
|
|
2660
|
-
* pin or a monitor write) is left untouched on every `cloud:up`.
|
|
2661
|
-
|
|
3033
|
+
* pin or a monitor write) is left untouched on every `cloud:up`.
|
|
3034
|
+
*
|
|
3035
|
+
* `effectiveSameOriginSafe` is the RUNNING editor's own verdict
|
|
3036
|
+
* (`/api/config`'s `previewSameOriginSafe`), supplied by a caller that has
|
|
3037
|
+
* already fetched it. When it is a boolean it WINS over the static
|
|
3038
|
+
* `stack.json` read; `null`/absent leaves the static read in charge.
|
|
3039
|
+
*
|
|
3040
|
+
* That parameter exists because this function had two sources of truth for one
|
|
3041
|
+
* flag, and they disagreed exactly where the adaptive default lives. A
|
|
3042
|
+
* subpath hydration failure escalates a project off `/__codeyam_preview` at
|
|
3043
|
+
* RUNTIME even when `stack.json` never declared `sameOriginSafe: false` — which
|
|
3044
|
+
* is the documented Next.js scaffold shape (a static `false` broke every
|
|
3045
|
+
* SSH-forward VM, so the key is deliberately absent). `healPreviewOriginAtReady`
|
|
3046
|
+
* decided WHETHER to heal from that runtime verdict and was right, then called
|
|
3047
|
+
* this function, which recomputed WHAT to write from the static flag and wrote
|
|
3048
|
+
* the very same-origin sentinel that had just failed. The project then sat
|
|
3049
|
+
* permanently at `previewDedicatedOriginUnavailable: true` — the editor knowing
|
|
3050
|
+
* it must leave the subpath and having nowhere to go — beside a preview host
|
|
3051
|
+
* that was already provisioned, LB-routed, HEALTHY and unused. */
|
|
3052
|
+
async function seedPreviewOrigin(
|
|
3053
|
+
cfg,
|
|
3054
|
+
appPort = VITE_PORT,
|
|
3055
|
+
effectiveSameOriginSafe = null,
|
|
3056
|
+
) {
|
|
2662
3057
|
const repoName = resolveProject(cfg).name;
|
|
2663
3058
|
const remotePath = `~/projects/${repoName}/.codeyam/editor.local.json`;
|
|
2664
3059
|
const stackPath = `~/projects/${repoName}/.codeyam/stack.json`;
|
|
@@ -2694,6 +3089,12 @@ async function seedPreviewOrigin(cfg, appPort = VITE_PORT) {
|
|
|
2694
3089
|
current = parseObj(localJson);
|
|
2695
3090
|
sameOriginSafe = previewSameOriginSafeFromStack(parseObj(stackJson));
|
|
2696
3091
|
}
|
|
3092
|
+
// The running editor's verdict overrides the static file — see
|
|
3093
|
+
// `resolveEffectiveSameOriginSafe` for why only a boolean counts.
|
|
3094
|
+
sameOriginSafe = resolveEffectiveSameOriginSafe(
|
|
3095
|
+
sameOriginSafe,
|
|
3096
|
+
effectiveSameOriginSafe,
|
|
3097
|
+
);
|
|
2697
3098
|
|
|
2698
3099
|
// Resolve this VM's custom slug (if any) so a dedicated-origin seed uses the
|
|
2699
3100
|
// slug host. The slug rides on cfg at Add time (the --vm-slug flag, validated
|
|
@@ -2727,6 +3128,12 @@ async function seedPreviewOrigin(cfg, appPort = VITE_PORT) {
|
|
|
2727
3128
|
// `sameOriginSafe: true`, and seeded `/__codeyam_preview` — which a
|
|
2728
3129
|
// remote browser then loads as `127.0.0.1 refused to connect`). Once
|
|
2729
3130
|
// stack.json exists, heal it forward to the dedicated origin.
|
|
3131
|
+
// `sameOriginSafe` here is the EFFECTIVE verdict, so this exception also
|
|
3132
|
+
// covers the case stack.json can never report: a project whose static
|
|
3133
|
+
// flag is legitimately absent (the Next.js scaffold default) that
|
|
3134
|
+
// escalated off the subpath at runtime after a hydration failure. The
|
|
3135
|
+
// caller passes the editor's own answer; without it this predicate read
|
|
3136
|
+
// `true` and declined to heal the one shape it was needed for.
|
|
2730
3137
|
// Because every dashboard action (Reset / Update / Add / bootstrap-recovery)
|
|
2731
3138
|
// re-runs this seed over cloud:up, either heal reaches every touch-point the
|
|
2732
3139
|
// moment stack.json is present — closing the race without re-ordering
|
|
@@ -2822,11 +3229,14 @@ async function healPreviewOriginAtReady(cfg, appPort = VITE_PORT) {
|
|
|
2822
3229
|
`READY auto-heal: ${cfg.instanceName} reports a not-same-origin-safe stack ` +
|
|
2823
3230
|
`but a wrong-mode previewOrigin (${config.previewOrigin}); re-seeding...`,
|
|
2824
3231
|
);
|
|
2825
|
-
// Re-run the seed
|
|
2826
|
-
//
|
|
2827
|
-
//
|
|
2828
|
-
//
|
|
2829
|
-
|
|
3232
|
+
// Re-run the seed with the verdict we just read, so WHETHER to heal and WHAT
|
|
3233
|
+
// to write come from the same source. Passing it is what makes this heal reach
|
|
3234
|
+
// the adaptive-default stacks it exists for: on a Next.js project whose
|
|
3235
|
+
// `stack.json` omits `sameOriginSafe` (the scaffolded shape), the static read
|
|
3236
|
+
// inside the seed answers `true` and re-writes the same-origin sentinel that
|
|
3237
|
+
// just failed to hydrate. We got here precisely BECAUSE the editor said
|
|
3238
|
+
// `false`, so `false` is the answer the seed must use.
|
|
3239
|
+
await seedPreviewOrigin(cfg, appPort, false);
|
|
2830
3240
|
// Tell the running server to re-read editor.local.json — no restart (the
|
|
2831
3241
|
// footgun that killed VM7's live Claude session).
|
|
2832
3242
|
const reload = runGcloud(
|
|
@@ -3224,7 +3634,16 @@ function promptForInstanceName(expected) {
|
|
|
3224
3634
|
});
|
|
3225
3635
|
}
|
|
3226
3636
|
|
|
3227
|
-
|
|
3637
|
+
// Every file streamed to a VM, and therefore every file the drift guard must
|
|
3638
|
+
// compare against the reviewed remote tip. `scripts/lib/npm-pin.sh` is here
|
|
3639
|
+
// because gcp-project-bootstrap.sh SOURCES it: a library that reaches the VM is
|
|
3640
|
+
// as much streamed content as the script that sources it, and leaving it out
|
|
3641
|
+
// meant the drift guard silently vouched for only half of what was shipped.
|
|
3642
|
+
const STREAMED_SCRIPTS = [
|
|
3643
|
+
"scripts/gcp-bootstrap.sh",
|
|
3644
|
+
"scripts/gcp-project-bootstrap.sh",
|
|
3645
|
+
"scripts/lib/npm-pin.sh",
|
|
3646
|
+
];
|
|
3228
3647
|
|
|
3229
3648
|
/** The configured editor branch to compare/stream streamed scripts against.
|
|
3230
3649
|
* Resolved from config (`resolveEditorConfig` → `editorBranch`, itself derived
|
|
@@ -3461,8 +3880,9 @@ function fleetInstancePrefix() {
|
|
|
3461
3880
|
* scripts/gcp/vm-ingress.sh with the resolved coordinates. Returns the spawnSync
|
|
3462
3881
|
* result so callers decide fatality — Add treats a non-zero as a loud warning
|
|
3463
3882
|
* (the VM is provisioned; ingress can be re-wired), Destroy likewise warns so a
|
|
3464
|
-
* teardown still removes the instance.
|
|
3465
|
-
|
|
3883
|
+
* teardown still removes the instance. `vmAccess` (up only) is the Add's intended
|
|
3884
|
+
* principal set, which the script's grant step uses in place of its sidecar read. */
|
|
3885
|
+
function runVmIngress(cfg, action, vmNumber, domain, appPort, slug, vmAccess) {
|
|
3466
3886
|
const scriptPath = path.join(rootDir, "scripts", "gcp", "vm-ingress.sh");
|
|
3467
3887
|
const inv = vmIngressInvocation({
|
|
3468
3888
|
scriptPath,
|
|
@@ -3474,6 +3894,7 @@ function runVmIngress(cfg, action, vmNumber, domain, appPort, slug) {
|
|
|
3474
3894
|
appPort,
|
|
3475
3895
|
instancePrefix: fleetInstancePrefix(),
|
|
3476
3896
|
slug,
|
|
3897
|
+
vmAccess,
|
|
3477
3898
|
});
|
|
3478
3899
|
return spawnSync(inv.command, inv.args, {
|
|
3479
3900
|
stdio: "inherit",
|
|
@@ -3504,17 +3925,77 @@ function runVmIngressGrantAccess(cfg, vmNumber, domain, slug) {
|
|
|
3504
3925
|
});
|
|
3505
3926
|
}
|
|
3506
3927
|
|
|
3928
|
+
/** THE principal set that must have IAP access to this VM after the Add: the
|
|
3929
|
+
* explicit `cfg.vmAccess` collaborators unioned with the resolved default
|
|
3930
|
+
* operator. Both add-time access paths consume this ONE function —
|
|
3931
|
+
* `wireVmIngressOnAdd` (which hands it to vm-ingress.sh's grant) and
|
|
3932
|
+
* `applyVmAccessOnAdd` (which grants it and subtracts it from the stale-revoke
|
|
3933
|
+
* set). They used to decide independently, and that is precisely how one path
|
|
3934
|
+
* granted `user:jared@codeyam.com` at 00:09:31 and the other revoked the same
|
|
3935
|
+
* principal 18 seconds later as a "stale prior tenant" (VMs 4/5/6, 2026-08-09):
|
|
3936
|
+
* the ingress path read the SIDECAR (still holding the prior tenant) while
|
|
3937
|
+
* `applyVmAccessOnAdd` resolved an empty operator. One source of truth removes
|
|
3938
|
+
* the whole class of disagreement. `env` is injectable for tests; the only I/O
|
|
3939
|
+
* is `resolveDefaultOperator`'s config/gcloud read. */
|
|
3940
|
+
function intendedVmPrincipals(cfg = {}, env = process.env) {
|
|
3941
|
+
return withDefaultOperator(
|
|
3942
|
+
normalizeAccessList(cfg.vmAccess || []),
|
|
3943
|
+
resolveDefaultOperator({ env }),
|
|
3944
|
+
);
|
|
3945
|
+
}
|
|
3946
|
+
|
|
3947
|
+
/** Add preflight: throw — BEFORE any GCE instance exists — when this Add would
|
|
3948
|
+
* come up on an IAP-gated public URL that authorizes NOBODY. The whole condition
|
|
3949
|
+
* is env + config (`intendedVmPrincipals` needs no GCP state), which is why it
|
|
3950
|
+
* belongs beside cmdUp's other cheap boundary validations rather than after a
|
|
3951
|
+
* machine has been provisioned: checking later means a half-provisioned VM to
|
|
3952
|
+
* clean up. On 2026-08-09 the un-refused version of this Add reported `done` for
|
|
3953
|
+
* VMs 4/5/6, left three unreachable URLs, and revoked each slot's prior tenant on
|
|
3954
|
+
* the way.
|
|
3955
|
+
*
|
|
3956
|
+
* Only fires in LB mode on a numbered fleet VM with nothing to grant; an explicit
|
|
3957
|
+
* `--vm-access` satisfies it with no default operator configured, so the
|
|
3958
|
+
* collaborator flow is unaffected, as is every legacy quick-tunnel Add. The
|
|
3959
|
+
* refusal text (and its `Next valid action:` remedy naming all three config
|
|
3960
|
+
* tiers) is the pure `unauthorizableAddRefusal`; this function is the composition
|
|
3961
|
+
* and the throw. */
|
|
3962
|
+
function assertAddCanAuthorizeSomeone(cfg, env = process.env) {
|
|
3963
|
+
let plan;
|
|
3964
|
+
try {
|
|
3965
|
+
plan = ingressWiringPlan(cfg, env);
|
|
3966
|
+
} catch {
|
|
3967
|
+
// A malformed domain is `resolveIngressDomain`'s error to raise on the wiring
|
|
3968
|
+
// path; it must not masquerade here as an authorization refusal.
|
|
3969
|
+
plan = { wire: false };
|
|
3970
|
+
}
|
|
3971
|
+
const refusal = unauthorizableAddRefusal({
|
|
3972
|
+
lbMode: !!plan.wire,
|
|
3973
|
+
vmNumber: plan.wire ? plan.vmNumber : null,
|
|
3974
|
+
intendedCount: plan.wire ? intendedVmPrincipals(cfg, env).length : null,
|
|
3975
|
+
});
|
|
3976
|
+
if (refusal) throw new Error(refusal);
|
|
3977
|
+
}
|
|
3978
|
+
|
|
3507
3979
|
/** Add-time hook: wire VM-N onto the shared LB when a fleet domain is set.
|
|
3508
3980
|
* Best-effort + loud — a wiring hiccup must never abort an otherwise-green Add
|
|
3509
3981
|
* (the VM is reachable operator-side over SSH regardless), so on failure it
|
|
3510
3982
|
* prints the exact re-run command rather than throwing. No-op (and silent) in
|
|
3511
|
-
* legacy quick-tunnel mode or for a non-fleet client project.
|
|
3983
|
+
* legacy quick-tunnel mode or for a non-fleet client project.
|
|
3984
|
+
*
|
|
3985
|
+
* RETURNS the principals this call granted (`intendedVmPrincipals`), or [] when
|
|
3986
|
+
* it did not wire — the explicit dataflow `applyVmAccessOnAdd` needs to know what
|
|
3987
|
+
* the Add has already granted before it computes a stale-revoke. Returned on the
|
|
3988
|
+
* FAILURE path too: a non-zero `vm-ingress.sh up` may still have completed its
|
|
3989
|
+
* grant step, and a principal that was possibly granted must never be revoked. */
|
|
3512
3990
|
function wireVmIngressOnAdd(cfg, appPort, env = process.env) {
|
|
3513
3991
|
const plan = ingressWiringPlan(cfg, env);
|
|
3514
|
-
if (!plan.wire) return;
|
|
3992
|
+
if (!plan.wire) return [];
|
|
3515
3993
|
const label = plan.slug || `vm-${plan.vmNumber}`;
|
|
3994
|
+
// Hand the intended set to the script so its `apply_default_iap_access` grants
|
|
3995
|
+
// THIS Add's principals instead of reading the sidecar's prior tenant.
|
|
3996
|
+
const vmAccess = intendedVmPrincipals(cfg, env);
|
|
3516
3997
|
console.error(` wiring LB ingress for VM-${plan.vmNumber} on ${plan.domain}...`);
|
|
3517
|
-
const res = runVmIngress(cfg, "up", plan.vmNumber, plan.domain, appPort, plan.slug);
|
|
3998
|
+
const res = runVmIngress(cfg, "up", plan.vmNumber, plan.domain, appPort, plan.slug, vmAccess);
|
|
3518
3999
|
if (res.status === 0) {
|
|
3519
4000
|
console.error(` ✓ LB ingress wired: https://${label}.editor.${plan.domain}`);
|
|
3520
4001
|
} else {
|
|
@@ -3536,6 +4017,7 @@ function wireVmIngressOnAdd(cfg, appPort, env = process.env) {
|
|
|
3536
4017
|
`(vm-ingress.sh up exited ${res.status == null ? "abnormally" : res.status}). Re-wire: ${rewire}`,
|
|
3537
4018
|
);
|
|
3538
4019
|
}
|
|
4020
|
+
return vmAccess;
|
|
3539
4021
|
}
|
|
3540
4022
|
|
|
3541
4023
|
/** Destroy-time hook: unwire VM-N's LB backends before the instance is deleted,
|
|
@@ -3604,8 +4086,15 @@ function runIapAccess(cfg, vmNumber, action, principals) {
|
|
|
3604
4086
|
* a revoke/grant hiccup (e.g. the SA missing IAP-admin) prints the exact re-run
|
|
3605
4087
|
* command rather than aborting an otherwise-green Add. `cfg.vmAccess` is the
|
|
3606
4088
|
* array (or comma-separated string) cmdUp validated; absent ⇒ an empty desired
|
|
3607
|
-
* list (the clean-slate case).
|
|
3608
|
-
|
|
4089
|
+
* list (the clean-slate case).
|
|
4090
|
+
*
|
|
4091
|
+
* `grantedThisRun` is what an EARLIER step of this same Add already granted —
|
|
4092
|
+
* `wireVmIngressOnAdd`'s return value. It is subtracted from the stale-revoke
|
|
4093
|
+
* set, so a principal this Add just authorized can never be revoked moments later
|
|
4094
|
+
* as a prior tenant. That is a second, cheaper guard than the shared
|
|
4095
|
+
* `intendedVmPrincipals` resolver above: it holds even if the two paths drift
|
|
4096
|
+
* apart again, which is exactly how VMs 4/5/6 broke on 2026-08-09. */
|
|
4097
|
+
function applyVmAccessOnAdd(cfg, env = process.env, { grantedThisRun = [] } = {}) {
|
|
3609
4098
|
const desired = normalizeAccessList(cfg.vmAccess || []);
|
|
3610
4099
|
let plan;
|
|
3611
4100
|
try {
|
|
@@ -3622,11 +4111,15 @@ function applyVmAccessOnAdd(cfg, env = process.env) {
|
|
|
3622
4111
|
// never inaccessible-by-default (the Error-51 footgun). The default operator is
|
|
3623
4112
|
// NOT persisted to the sidecar (it's a standing grant, not a per-VM
|
|
3624
4113
|
// collaborator), so `desired` alone drives the sidecar + stale-revoke below.
|
|
3625
|
-
const toGrant =
|
|
4114
|
+
const toGrant = intendedVmPrincipals(cfg, env);
|
|
3626
4115
|
const stale = accessFor(vmNumber);
|
|
3627
4116
|
// Never revoke a principal we're about to (re-)grant — a prior tenant who was
|
|
3628
|
-
// also the default operator must not be revoked-then-regranted for nothing
|
|
3629
|
-
|
|
4117
|
+
// also the default operator must not be revoked-then-regranted for nothing —
|
|
4118
|
+
// NOR one an earlier step of this same Add already granted. `toGrant` alone
|
|
4119
|
+
// protected nothing when it was empty: the ingress step had granted the operator
|
|
4120
|
+
// seconds earlier and this revoke took it straight back off (VMs 4/5/6).
|
|
4121
|
+
const grantedAlready = new Set([...toGrant, ...normalizeAccessList(grantedThisRun || [])]);
|
|
4122
|
+
const staleOnly = stale.filter((p) => !grantedAlready.has(p));
|
|
3630
4123
|
|
|
3631
4124
|
if (plan.wire) {
|
|
3632
4125
|
if (staleOnly.length > 0) {
|
|
@@ -3743,12 +4236,23 @@ async function convergeVmIngressOnAdd(cfg, appPort, env = process.env, deps = {}
|
|
|
3743
4236
|
}
|
|
3744
4237
|
if (!plan.wire) {
|
|
3745
4238
|
// Nothing to converge — the no-op wire keeps the clean-slate invariant.
|
|
3746
|
-
wireVmIngressOnAdd(cfg, appPort, env);
|
|
3747
|
-
return { wired: true, reason: "no-wire" };
|
|
3748
|
-
}
|
|
3749
|
-
|
|
4239
|
+
const granted = wireVmIngressOnAdd(cfg, appPort, env);
|
|
4240
|
+
return { wired: true, reason: "no-wire", grantedThisRun: granted };
|
|
4241
|
+
}
|
|
4242
|
+
|
|
4243
|
+
// The intended set, resolved ONCE and handed to every `up` attempt so each
|
|
4244
|
+
// converge iteration grants the same principals — and reported back to the
|
|
4245
|
+
// caller as `grantedThisRun` so the access reconcile that follows cannot revoke
|
|
4246
|
+
// them. Every `up` runs the script's grant step, so these are granted whether
|
|
4247
|
+
// the loop converges or hits its deadline. Injectable via `deps` like the rest
|
|
4248
|
+
// of this loop's slow I/O: the resolve spawns `gcloud config get-value account`
|
|
4249
|
+
// on its last tier, and a unit test must never depend on the gcloud identity
|
|
4250
|
+
// that happens to be active on the machine running it.
|
|
4251
|
+
const grantedThisRun =
|
|
4252
|
+
deps.grantedThisRun != null ? deps.grantedThisRun : intendedVmPrincipals(cfg, env);
|
|
3750
4253
|
const runUp =
|
|
3751
|
-
deps.runUp
|
|
4254
|
+
deps.runUp
|
|
4255
|
+
|| ((p) => runVmIngress(cfg, "up", p.vmNumber, p.domain, appPort, p.slug, grantedThisRun));
|
|
3752
4256
|
const probeVerdict = deps.probeVerdict || ((p) => probeVmIngressVerdict(cfg, p));
|
|
3753
4257
|
const sleep = deps.sleep || ((ms) => new Promise((r) => setTimeout(r, ms)));
|
|
3754
4258
|
const now = deps.now || (() => Date.now());
|
|
@@ -3791,7 +4295,7 @@ async function convergeVmIngressOnAdd(cfg, appPort, env = process.env, deps = {}
|
|
|
3791
4295
|
if (decision.done) {
|
|
3792
4296
|
if (decision.wired) {
|
|
3793
4297
|
console.error(` ✓ LB ingress wired: https://${label}.editor.${plan.domain}`);
|
|
3794
|
-
return { wired: true, reason: "wired", attempts: attempt + 1 };
|
|
4298
|
+
return { wired: true, reason: "wired", attempts: attempt + 1, grantedThisRun };
|
|
3795
4299
|
}
|
|
3796
4300
|
const rewire =
|
|
3797
4301
|
`FLEET_DOMAIN=${plan.domain}${plan.slug ? ` VM_SLUG=${plan.slug}` : ""} ` +
|
|
@@ -3808,7 +4312,7 @@ async function convergeVmIngressOnAdd(cfg, appPort, env = process.env, deps = {}
|
|
|
3808
4312
|
`CY_DEGRADED ingress-wiring-failed: VM-${plan.vmNumber} public URL did not wire ` +
|
|
3809
4313
|
`within the add window (last verdict=${verdict || "unknown"}). Re-wire: ${rewire}`,
|
|
3810
4314
|
);
|
|
3811
|
-
return { wired: false, reason: "deadline", attempts: attempt + 1 };
|
|
4315
|
+
return { wired: false, reason: "deadline", attempts: attempt + 1, grantedThisRun };
|
|
3812
4316
|
}
|
|
3813
4317
|
attempt += 1;
|
|
3814
4318
|
await sleep(decision.delayMs);
|
|
@@ -3837,8 +4341,8 @@ function startDetachedAddWiring(cfg, appPort, env = process.env) {
|
|
|
3837
4341
|
if (!plan.wire) {
|
|
3838
4342
|
// Nothing slow to detach: wiring is a no-op and the access reconcile is a cheap
|
|
3839
4343
|
// sidecar write. Run them inline so the clean-slate invariant still holds.
|
|
3840
|
-
wireVmIngressOnAdd(cfg, appPort, env);
|
|
3841
|
-
applyVmAccessOnAdd(cfg, env);
|
|
4344
|
+
const grantedThisRun = wireVmIngressOnAdd(cfg, appPort, env);
|
|
4345
|
+
applyVmAccessOnAdd(cfg, env, { grantedThisRun });
|
|
3842
4346
|
return;
|
|
3843
4347
|
}
|
|
3844
4348
|
const marker = ingressInflightMarkerPath(INGRESS_INFLIGHT_DIR, plan.vmNumber);
|
|
@@ -3870,8 +4374,8 @@ function startDetachedAddWiring(cfg, appPort, env = process.env) {
|
|
|
3870
4374
|
`running it inline instead.`,
|
|
3871
4375
|
);
|
|
3872
4376
|
try { fs.unlinkSync(marker); } catch { /* marker may not exist */ }
|
|
3873
|
-
wireVmIngressOnAdd(cfg, appPort, env);
|
|
3874
|
-
applyVmAccessOnAdd(cfg, env);
|
|
4377
|
+
const grantedThisRun = wireVmIngressOnAdd(cfg, appPort, env);
|
|
4378
|
+
applyVmAccessOnAdd(cfg, env, { grantedThisRun });
|
|
3875
4379
|
}
|
|
3876
4380
|
}
|
|
3877
4381
|
|
|
@@ -3903,8 +4407,11 @@ async function cmdWireAddTail() {
|
|
|
3903
4407
|
// Converge the URL to `verdict=wired` (retry-until-wired), not a single-shot
|
|
3904
4408
|
// `up` left to the slow reconcile sweep — then the IAP access grant, in the
|
|
3905
4409
|
// SAME order cmdUp ran them inline.
|
|
3906
|
-
|
|
3907
|
-
|
|
4410
|
+
// Carry what the converge granted into the access reconcile, so the revoke
|
|
4411
|
+
// step cannot take back a principal this same Add just authorized — the exact
|
|
4412
|
+
// grant-then-revoke-18s-later sequence `/tmp/cy-wire-add-vm-4.log` recorded.
|
|
4413
|
+
const { grantedThisRun } = await convergeVmIngressOnAdd(cfg, appPort);
|
|
4414
|
+
applyVmAccessOnAdd(cfg, process.env, { grantedThisRun });
|
|
3908
4415
|
} finally {
|
|
3909
4416
|
if (marker) {
|
|
3910
4417
|
try { fs.unlinkSync(marker); } catch { /* already gone — fine */ }
|
|
@@ -3966,6 +4473,75 @@ function releaseVmSlugOnDestroy(cfg) {
|
|
|
3966
4473
|
return vmNumber;
|
|
3967
4474
|
}
|
|
3968
4475
|
|
|
4476
|
+
/** Does `branch` still exist on `repo`? Answered with a single
|
|
4477
|
+
* `git ls-remote --heads` — about a second, and no cloud resource touched.
|
|
4478
|
+
*
|
|
4479
|
+
* Tri-state on purpose, and the third state is the load-bearing one:
|
|
4480
|
+
* true — the ref resolved.
|
|
4481
|
+
* false — the remote answered and the ref is NOT there. Provably gone.
|
|
4482
|
+
* null — we could not get an answer (no network, no credentials, git
|
|
4483
|
+
* missing, timeout). NOT evidence of a deleted branch.
|
|
4484
|
+
* Only `false` refuses an add. Collapsing `null` into `false` would turn every
|
|
4485
|
+
* transient GitHub blip into a fleet-wide add outage, which is a far worse
|
|
4486
|
+
* failure than the one this preflight exists to prevent — and the classifier
|
|
4487
|
+
* arm downstream still catches a genuinely dead ref, just later and dearer.
|
|
4488
|
+
*
|
|
4489
|
+
* A private repo needs credentials, so the token is inlined into the URL the
|
|
4490
|
+
* same way clone_editor_repo does it on the VM side (x-access-token), and for
|
|
4491
|
+
* the same reason: the plain URL 401s. The token is confined to this argv and
|
|
4492
|
+
* never logged — the caller prints `repo` (the un-inlined form) instead. */
|
|
4493
|
+
function gitBranchRefExists({ repo, branch }) {
|
|
4494
|
+
if (!repo || !branch) return null;
|
|
4495
|
+
let url = repo;
|
|
4496
|
+
const token = resolveGithubToken();
|
|
4497
|
+
if (token && /^https:\/\/github\.com\//.test(repo)) {
|
|
4498
|
+
url = repo.replace("https://", `https://x-access-token:${token}@`);
|
|
4499
|
+
}
|
|
4500
|
+
const res = spawnSync("git", ["ls-remote", "--heads", url, branch], {
|
|
4501
|
+
encoding: "utf8",
|
|
4502
|
+
timeout: 30000,
|
|
4503
|
+
});
|
|
4504
|
+
// A non-zero exit is an unreachable/unauthorized remote, not an absent branch:
|
|
4505
|
+
// git exits 0 and prints nothing for "connected fine, no such head".
|
|
4506
|
+
if (res.error || res.status !== 0) return null;
|
|
4507
|
+
return String(res.stdout || "").trim().length > 0;
|
|
4508
|
+
}
|
|
4509
|
+
|
|
4510
|
+
/** Refuse an add whose stored branch has been deleted from origin, BEFORE the
|
|
4511
|
+
* first gcloud call.
|
|
4512
|
+
*
|
|
4513
|
+
* Without this the failure lands as late and as expensively as it possibly can:
|
|
4514
|
+
* on 2026-08-25 four fleet VMs each created a real GCE instance, booted it,
|
|
4515
|
+
* installed npm and cloned 13,226 files before `git fetch` inside the VM said
|
|
4516
|
+
* `fatal: couldn't find remote ref editor-improvements-75` — a branch pruned
|
|
4517
|
+
* from origin hours earlier when it merged. One `git ls-remote` establishes the
|
|
4518
|
+
* same fact in about a second, for free. This repo prunes branches on merge, so
|
|
4519
|
+
* ANY stored feature-branch name expires silently; that makes this a recurring
|
|
4520
|
+
* condition, not a one-off.
|
|
4521
|
+
*
|
|
4522
|
+
* Placed beside checkScriptDrift and assertVariantProvisionable — the other
|
|
4523
|
+
* "fail loud before creating a GCE instance" guards — because it is only worth
|
|
4524
|
+
* anything ahead of ensureInstance. It cannot live in scripts/gcp-bootstrap.sh,
|
|
4525
|
+
* which already runs ON the provisioned box and never sees the client branch.
|
|
4526
|
+
*
|
|
4527
|
+
* Throws with the field, the branch and the repo named. Never throws for a ref
|
|
4528
|
+
* it could not check (see gitBranchRefExists). */
|
|
4529
|
+
function assertBranchRefsExist(cfg, { refExists = gitBranchRefExists } = {}) {
|
|
4530
|
+
// Lazy require: launch-config.js requires THIS module at load time, so a
|
|
4531
|
+
// top-level require here would close the cycle. Same pattern as the
|
|
4532
|
+
// isPlaceholderProject call sites above.
|
|
4533
|
+
const lc = require("./launch-config");
|
|
4534
|
+
const { editorRepo, editorBranch } = resolveEditorConfig(cfg);
|
|
4535
|
+
const targets = lc.branchRefTargets(
|
|
4536
|
+
{ clientRepo: cfg.repo, clientBranch: cfg.branch, editorBranch },
|
|
4537
|
+
{ editorRepo },
|
|
4538
|
+
);
|
|
4539
|
+
if (!targets.length) return;
|
|
4540
|
+
console.error("Checking that the requested branches still exist on origin...");
|
|
4541
|
+
const missing = lc.missingBranchRefs(targets, refExists);
|
|
4542
|
+
if (missing.length) throw new Error(lc.missingBranchRefMessage(missing));
|
|
4543
|
+
}
|
|
4544
|
+
|
|
3969
4545
|
async function cmdUp(args, projectDir = rootDir) {
|
|
3970
4546
|
ensureGcloud();
|
|
3971
4547
|
const flags = parseFlags(args);
|
|
@@ -4016,8 +4592,14 @@ async function cmdUp(args, projectDir = rootDir) {
|
|
|
4016
4592
|
}
|
|
4017
4593
|
}
|
|
4018
4594
|
|
|
4595
|
+
assertAddCanAuthorizeSomeone(cfg);
|
|
4596
|
+
|
|
4019
4597
|
checkScriptDrift(cfg);
|
|
4020
4598
|
|
|
4599
|
+
// Last of the pre-instance guards: a deleted branch ref is checkable in ~1s
|
|
4600
|
+
// and otherwise costs a full instance + boot + 13k-file clone to discover.
|
|
4601
|
+
assertBranchRefsExist(cfg);
|
|
4602
|
+
|
|
4021
4603
|
await ensureFirewallRule(cfg);
|
|
4022
4604
|
const { createdThisRun } = await ensureInstance(cfg);
|
|
4023
4605
|
|
|
@@ -4122,7 +4704,27 @@ async function cmdUp(args, projectDir = rootDir) {
|
|
|
4122
4704
|
);
|
|
4123
4705
|
}
|
|
4124
4706
|
} catch (err) {
|
|
4125
|
-
|
|
4707
|
+
// `createdThisRun` alone MUST NOT authorize deletion. Gather what state the
|
|
4708
|
+
// VM is in right now, then let the pure decision rule — see
|
|
4709
|
+
// abortTeardownDecision's doc comment for why each input is load-bearing.
|
|
4710
|
+
const state = probeAbortTeardownState(cfg, createdThisRun);
|
|
4711
|
+
const verdict = abortTeardownDecision({ createdThisRun, ...state });
|
|
4712
|
+
if (createdThisRun && verdict === "keep") {
|
|
4713
|
+
// Deliberately loud and specific. The old blanket "tearing down the VM
|
|
4714
|
+
// cloud:up created this run (no orphan left behind)" was actively
|
|
4715
|
+
// misleading here — it asserted no orphan while deleting a working VM.
|
|
4716
|
+
const why = state.probeInconclusive
|
|
4717
|
+
? "its liveness could not be confirmed (probe failed), and an unconfirmed VM is never deleted"
|
|
4718
|
+
: state.containerUp
|
|
4719
|
+
? "its editor container is RUNNING — another run brought it up"
|
|
4720
|
+
: "it is in the fleet roster and visible to the dashboard";
|
|
4721
|
+
console.error(
|
|
4722
|
+
`✗ provisioning aborted — the VM ${cfg.instanceName} was LEFT IN PLACE because ${why}.\n` +
|
|
4723
|
+
` This run created the instance, but that alone does not authorize deleting a VM that is in service.\n` +
|
|
4724
|
+
` If it really should go, remove it deliberately: npm run cloud:destroy`,
|
|
4725
|
+
);
|
|
4726
|
+
}
|
|
4727
|
+
if (verdict === "delete") {
|
|
4126
4728
|
console.error(
|
|
4127
4729
|
`✗ provisioning aborted — tearing down the VM cloud:up created this run (no orphan left behind)`,
|
|
4128
4730
|
);
|
|
@@ -4304,14 +4906,7 @@ async function cmdStatus(args, projectDir = rootDir) {
|
|
|
4304
4906
|
const imgStatus = String(imgProbe.stdout || "").trim() || "?";
|
|
4305
4907
|
console.error(` image: ${imgStatus}`);
|
|
4306
4908
|
|
|
4307
|
-
const
|
|
4308
|
-
const containerProbe = runGcloud(
|
|
4309
|
-
buildGcloudArgs("ssh-probe", cfg, {
|
|
4310
|
-
command: `cd ~/projects/${repoName} 2>/dev/null && docker compose -f docker-compose.cloud.yml ps --format json 2>/dev/null || echo`,
|
|
4311
|
-
}),
|
|
4312
|
-
{ stdio: ["ignore", "pipe", "ignore"] },
|
|
4313
|
-
);
|
|
4314
|
-
const containerStatus = parseEditorComposeState(containerProbe.stdout) || "missing";
|
|
4909
|
+
const containerStatus = probeEditorComposeState(cfg).state || "missing";
|
|
4315
4910
|
console.error(` editor container: ${containerStatus}`);
|
|
4316
4911
|
|
|
4317
4912
|
const appPort = await readVmAppPort(cfg);
|
|
@@ -4380,6 +4975,10 @@ async function cmdIngressUp(args, projectDir = rootDir) {
|
|
|
4380
4975
|
async function cmdReconcileIngress(args, projectDir = rootDir) {
|
|
4381
4976
|
ensureGcloud();
|
|
4382
4977
|
parseFlags(args);
|
|
4978
|
+
// Clear ssh sessions earlier timed-out probes leaked before this sweep can add
|
|
4979
|
+
// load of its own. Only PID-1 orphans with the fleet signature — never a live
|
|
4980
|
+
// sweep's in-flight children.
|
|
4981
|
+
reapOrphanedFleetSsh();
|
|
4383
4982
|
const cfg = loadCloudConfig(projectDir) || {};
|
|
4384
4983
|
const domain = resolveIngressDomain(cfg, process.env);
|
|
4385
4984
|
if (!domain) {
|
|
@@ -4410,7 +5009,7 @@ async function cmdReconcileIngress(args, projectDir = rootDir) {
|
|
|
4410
5009
|
// whole sweep. On timeout res.stdout is empty, so parseIngressStatusVerdict
|
|
4411
5010
|
// returns null → an `unknown` verdict → selectIngressReconcileTargets SKIPS
|
|
4412
5011
|
// this VM and the sweep continues to the next one.
|
|
4413
|
-
const res =
|
|
5012
|
+
const res = spawnGroupBoundedSync(inv.command, inv.args, {
|
|
4414
5013
|
env: { ...process.env, ...inv.env },
|
|
4415
5014
|
encoding: "utf8",
|
|
4416
5015
|
timeout: GCLOUD_CALL_TIMEOUT_MS,
|
|
@@ -4471,7 +5070,7 @@ async function cmdReconcileIngress(args, projectDir = rootDir) {
|
|
|
4471
5070
|
// Bound the per-BES converge too: a hung `gcloud backend-services update`
|
|
4472
5071
|
// returns a non-zero (timed-out) result, so it falls into the skip branch
|
|
4473
5072
|
// below and the sweep moves on to the next BES rather than stalling.
|
|
4474
|
-
const res =
|
|
5073
|
+
const res = spawnGroupBoundedSync(inv.command, inv.args, {
|
|
4475
5074
|
encoding: "utf8",
|
|
4476
5075
|
timeout: GCLOUD_CALL_TIMEOUT_MS,
|
|
4477
5076
|
});
|
|
@@ -5048,6 +5647,9 @@ async function cloudMain() {
|
|
|
5048
5647
|
module.exports = {
|
|
5049
5648
|
// pure helpers
|
|
5050
5649
|
isSpawnTimeout,
|
|
5650
|
+
gcloudCallTimeoutMs,
|
|
5651
|
+
provisionFailureMessage,
|
|
5652
|
+
GCLOUD_PROVISION_TIMEOUT_MS,
|
|
5051
5653
|
agentEnvVarName,
|
|
5052
5654
|
agentNpmPackage,
|
|
5053
5655
|
AGENT_ENV_VARS,
|
|
@@ -5058,6 +5660,9 @@ module.exports = {
|
|
|
5058
5660
|
assertAgentCliInstalled,
|
|
5059
5661
|
probeContainerRunning,
|
|
5060
5662
|
probeContainerState,
|
|
5663
|
+
waitForEditorComposeRunning,
|
|
5664
|
+
abortTeardownDecision,
|
|
5665
|
+
probeAbortTeardownState,
|
|
5061
5666
|
probeAgentBinaryInstalled,
|
|
5062
5667
|
waitForContainerRunning,
|
|
5063
5668
|
KNOWN_FLAGS,
|
|
@@ -5080,6 +5685,7 @@ module.exports = {
|
|
|
5080
5685
|
resolveConfiguredAppPort,
|
|
5081
5686
|
resolveSeedPreviewOrigin,
|
|
5082
5687
|
previewSameOriginSafeFromStack,
|
|
5688
|
+
resolveEffectiveSameOriginSafe,
|
|
5083
5689
|
isRottedManagedPreviewOrigin,
|
|
5084
5690
|
isWrongModePreviewOrigin,
|
|
5085
5691
|
isSilentlyBrokenPreview,
|
|
@@ -5138,6 +5744,8 @@ module.exports = {
|
|
|
5138
5744
|
FORWARD_ENV_KEYS_ENV,
|
|
5139
5745
|
parseEditorComposeState,
|
|
5140
5746
|
fetchInstanceState,
|
|
5747
|
+
fetchPendingOperation,
|
|
5748
|
+
wedgedOperationMessage,
|
|
5141
5749
|
formatGcloudFailure,
|
|
5142
5750
|
isTransientSshFailure,
|
|
5143
5751
|
runGcloudSshWithRetry,
|
|
@@ -5157,6 +5765,8 @@ module.exports = {
|
|
|
5157
5765
|
ingressWiringPlan,
|
|
5158
5766
|
fleetInstancePrefix,
|
|
5159
5767
|
runVmIngress,
|
|
5768
|
+
intendedVmPrincipals,
|
|
5769
|
+
assertAddCanAuthorizeSomeone,
|
|
5160
5770
|
wireVmIngressOnAdd,
|
|
5161
5771
|
probeVmIngressVerdict,
|
|
5162
5772
|
convergeVmIngressOnAdd,
|
|
@@ -5193,6 +5803,8 @@ module.exports = {
|
|
|
5193
5803
|
readVmAppPort,
|
|
5194
5804
|
buildVmAppPortReadCommand,
|
|
5195
5805
|
resolveLaptopGitIdentity,
|
|
5806
|
+
gitBranchRefExists,
|
|
5807
|
+
assertBranchRefsExist,
|
|
5196
5808
|
};
|
|
5197
5809
|
|
|
5198
5810
|
if (require.main === module) {
|