@codeyam-editor/codeyam-editor 0.1.0-staging.g3858572 → 0.1.0-staging.g52ee0eb
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/npm/cloud.js +75 -0
- package/npm/editor-dev.js +24 -2
- package/npm/editor.js +15 -5
- package/npm/fleet-add-failure.js +235 -21
- package/npm/fleet-branch.js +124 -1
- package/npm/fleet-build-data.js +12 -1
- package/npm/fleet-cleanup.js +84 -3
- package/npm/fleet-datastore-health.js +107 -0
- package/npm/fleet-revision.js +108 -0
- package/npm/fleet-run-setup.js +66 -0
- package/npm/fleet-session-activity.js +162 -0
- package/npm/fleet-vm-readiness.js +57 -16
- package/npm/launch-config.js +126 -2
- package/npm/scenario-check.js +607 -32
- package/npm/scenario-interactivity.js +467 -23
- package/npm/scenario-issues.js +8 -0
- package/npm/scenario-playwright.js +67 -14
- package/npm/utils.js +262 -22
- package/package.json +5 -5
- package/ui/dist/assets/{ContainerInspector-DSVZvEkB.js → ContainerInspector-B_1ncmBb.js} +1 -1
- package/ui/dist/assets/EditorShell-BscDOu9z.js +326 -0
- package/ui/dist/assets/EditorShell-CE8iG_EM.css +1 -0
- package/ui/dist/assets/ScenarioDataPanel-BJd8d23d.js +15 -0
- package/ui/dist/assets/{ScenarioDataPanelFullPage-Cyk6efiI.js → ScenarioDataPanelFullPage-tiaa3re-.js} +1 -1
- package/ui/dist/assets/ScenarioRenderer-Dptkb4Di.js +21 -0
- package/ui/dist/assets/{index-BPLKgwon.js → index-BG9MM5pc.js} +9 -9
- package/ui/dist/assets/index-DAvNv22V.css +1 -0
- package/ui/dist/index.html +2 -2
- package/ui/dist/preview-subdir-demo/index.html +62 -0
- package/ui/dist/preview-subdir-demo/sibling.html +48 -0
- package/ui/dist/assets/EditorShell-DnXdHrar.js +0 -256
- package/ui/dist/assets/EditorShell-FvbXyqqD.css +0 -1
- package/ui/dist/assets/ScenarioDataPanel-6qJE-0lA.js +0 -15
- package/ui/dist/assets/ScenarioRenderer-Uyt54vHI.js +0 -21
- package/ui/dist/assets/index-DKpgJLo3.css +0 -1
package/npm/cloud.js
CHANGED
|
@@ -4202,6 +4202,75 @@ function releaseVmSlugOnDestroy(cfg) {
|
|
|
4202
4202
|
return vmNumber;
|
|
4203
4203
|
}
|
|
4204
4204
|
|
|
4205
|
+
/** Does `branch` still exist on `repo`? Answered with a single
|
|
4206
|
+
* `git ls-remote --heads` — about a second, and no cloud resource touched.
|
|
4207
|
+
*
|
|
4208
|
+
* Tri-state on purpose, and the third state is the load-bearing one:
|
|
4209
|
+
* true — the ref resolved.
|
|
4210
|
+
* false — the remote answered and the ref is NOT there. Provably gone.
|
|
4211
|
+
* null — we could not get an answer (no network, no credentials, git
|
|
4212
|
+
* missing, timeout). NOT evidence of a deleted branch.
|
|
4213
|
+
* Only `false` refuses an add. Collapsing `null` into `false` would turn every
|
|
4214
|
+
* transient GitHub blip into a fleet-wide add outage, which is a far worse
|
|
4215
|
+
* failure than the one this preflight exists to prevent — and the classifier
|
|
4216
|
+
* arm downstream still catches a genuinely dead ref, just later and dearer.
|
|
4217
|
+
*
|
|
4218
|
+
* A private repo needs credentials, so the token is inlined into the URL the
|
|
4219
|
+
* same way clone_editor_repo does it on the VM side (x-access-token), and for
|
|
4220
|
+
* the same reason: the plain URL 401s. The token is confined to this argv and
|
|
4221
|
+
* never logged — the caller prints `repo` (the un-inlined form) instead. */
|
|
4222
|
+
function gitBranchRefExists({ repo, branch }) {
|
|
4223
|
+
if (!repo || !branch) return null;
|
|
4224
|
+
let url = repo;
|
|
4225
|
+
const token = resolveGithubToken();
|
|
4226
|
+
if (token && /^https:\/\/github\.com\//.test(repo)) {
|
|
4227
|
+
url = repo.replace("https://", `https://x-access-token:${token}@`);
|
|
4228
|
+
}
|
|
4229
|
+
const res = spawnSync("git", ["ls-remote", "--heads", url, branch], {
|
|
4230
|
+
encoding: "utf8",
|
|
4231
|
+
timeout: 30000,
|
|
4232
|
+
});
|
|
4233
|
+
// A non-zero exit is an unreachable/unauthorized remote, not an absent branch:
|
|
4234
|
+
// git exits 0 and prints nothing for "connected fine, no such head".
|
|
4235
|
+
if (res.error || res.status !== 0) return null;
|
|
4236
|
+
return String(res.stdout || "").trim().length > 0;
|
|
4237
|
+
}
|
|
4238
|
+
|
|
4239
|
+
/** Refuse an add whose stored branch has been deleted from origin, BEFORE the
|
|
4240
|
+
* first gcloud call.
|
|
4241
|
+
*
|
|
4242
|
+
* Without this the failure lands as late and as expensively as it possibly can:
|
|
4243
|
+
* on 2026-08-25 four fleet VMs each created a real GCE instance, booted it,
|
|
4244
|
+
* installed npm and cloned 13,226 files before `git fetch` inside the VM said
|
|
4245
|
+
* `fatal: couldn't find remote ref editor-improvements-75` — a branch pruned
|
|
4246
|
+
* from origin hours earlier when it merged. One `git ls-remote` establishes the
|
|
4247
|
+
* same fact in about a second, for free. This repo prunes branches on merge, so
|
|
4248
|
+
* ANY stored feature-branch name expires silently; that makes this a recurring
|
|
4249
|
+
* condition, not a one-off.
|
|
4250
|
+
*
|
|
4251
|
+
* Placed beside checkScriptDrift and assertVariantProvisionable — the other
|
|
4252
|
+
* "fail loud before creating a GCE instance" guards — because it is only worth
|
|
4253
|
+
* anything ahead of ensureInstance. It cannot live in scripts/gcp-bootstrap.sh,
|
|
4254
|
+
* which already runs ON the provisioned box and never sees the client branch.
|
|
4255
|
+
*
|
|
4256
|
+
* Throws with the field, the branch and the repo named. Never throws for a ref
|
|
4257
|
+
* it could not check (see gitBranchRefExists). */
|
|
4258
|
+
function assertBranchRefsExist(cfg, { refExists = gitBranchRefExists } = {}) {
|
|
4259
|
+
// Lazy require: launch-config.js requires THIS module at load time, so a
|
|
4260
|
+
// top-level require here would close the cycle. Same pattern as the
|
|
4261
|
+
// isPlaceholderProject call sites above.
|
|
4262
|
+
const lc = require("./launch-config");
|
|
4263
|
+
const { editorRepo, editorBranch } = resolveEditorConfig(cfg);
|
|
4264
|
+
const targets = lc.branchRefTargets(
|
|
4265
|
+
{ clientRepo: cfg.repo, clientBranch: cfg.branch, editorBranch },
|
|
4266
|
+
{ editorRepo },
|
|
4267
|
+
);
|
|
4268
|
+
if (!targets.length) return;
|
|
4269
|
+
console.error("Checking that the requested branches still exist on origin...");
|
|
4270
|
+
const missing = lc.missingBranchRefs(targets, refExists);
|
|
4271
|
+
if (missing.length) throw new Error(lc.missingBranchRefMessage(missing));
|
|
4272
|
+
}
|
|
4273
|
+
|
|
4205
4274
|
async function cmdUp(args, projectDir = rootDir) {
|
|
4206
4275
|
ensureGcloud();
|
|
4207
4276
|
const flags = parseFlags(args);
|
|
@@ -4256,6 +4325,10 @@ async function cmdUp(args, projectDir = rootDir) {
|
|
|
4256
4325
|
|
|
4257
4326
|
checkScriptDrift(cfg);
|
|
4258
4327
|
|
|
4328
|
+
// Last of the pre-instance guards: a deleted branch ref is checkable in ~1s
|
|
4329
|
+
// and otherwise costs a full instance + boot + 13k-file clone to discover.
|
|
4330
|
+
assertBranchRefsExist(cfg);
|
|
4331
|
+
|
|
4259
4332
|
await ensureFirewallRule(cfg);
|
|
4260
4333
|
const { createdThisRun } = await ensureInstance(cfg);
|
|
4261
4334
|
|
|
@@ -5449,6 +5522,8 @@ module.exports = {
|
|
|
5449
5522
|
readVmAppPort,
|
|
5450
5523
|
buildVmAppPortReadCommand,
|
|
5451
5524
|
resolveLaptopGitIdentity,
|
|
5525
|
+
gitBranchRefExists,
|
|
5526
|
+
assertBranchRefsExist,
|
|
5452
5527
|
};
|
|
5453
5528
|
|
|
5454
5529
|
if (require.main === module) {
|
package/npm/editor-dev.js
CHANGED
|
@@ -40,6 +40,8 @@ const {
|
|
|
40
40
|
logLauncherTeardownSignal,
|
|
41
41
|
readMergedEditorConfig,
|
|
42
42
|
resolveLauncherPort,
|
|
43
|
+
readServerOutputLogTail,
|
|
44
|
+
formatLaunchFailureError,
|
|
43
45
|
staleBinaryBanner,
|
|
44
46
|
enforceFreshBinary,
|
|
45
47
|
} = require("./utils");
|
|
@@ -447,9 +449,29 @@ if (require.main === module) {
|
|
|
447
449
|
buildLauncherArtifacts();
|
|
448
450
|
}
|
|
449
451
|
|
|
452
|
+
// Both helpers take the port to BIND plus the TARGET PROJECT dir — the
|
|
453
|
+
// latter is what lets them find the project's `server-state.json`, and so
|
|
454
|
+
// find a running server whose control port is not `SERVER_PORT` at all.
|
|
450
455
|
const serverChild = shouldRestart
|
|
451
|
-
? await restartServer(SERVER_PORT)
|
|
452
|
-
: await ensureServer(SERVER_PORT);
|
|
456
|
+
? await restartServer(SERVER_PORT, projectDir)
|
|
457
|
+
: await ensureServer(SERVER_PORT, projectDir);
|
|
458
|
+
|
|
459
|
+
// Nothing below may claim the editor is up until something is actually
|
|
460
|
+
// listening on the port we just bound. `ensureServer` returns null only
|
|
461
|
+
// when it ATTACHED to a live server, and its spawn path already races the
|
|
462
|
+
// port — but a launcher that prints a URL it never bound turned a
|
|
463
|
+
// recoverable port conflict into a debugging session, so the banner is
|
|
464
|
+
// gated on an independent probe rather than on that internal contract.
|
|
465
|
+
if (!(await waitForPort(SERVER_PORT, 10000, 200))) {
|
|
466
|
+
console.error(
|
|
467
|
+
formatLaunchFailureError(
|
|
468
|
+
SERVER_PORT,
|
|
469
|
+
projectDir,
|
|
470
|
+
readServerOutputLogTail(projectDir),
|
|
471
|
+
),
|
|
472
|
+
);
|
|
473
|
+
process.exit(1);
|
|
474
|
+
}
|
|
453
475
|
|
|
454
476
|
// Re-read the (possibly just-initialized) project config to decide
|
|
455
477
|
// whether to wait on a dev server at all. On a project where init
|
package/npm/editor.js
CHANGED
|
@@ -27,6 +27,7 @@ const {
|
|
|
27
27
|
readMergedEditorConfig,
|
|
28
28
|
resolveLauncherPort,
|
|
29
29
|
resolveWrappedAppPort,
|
|
30
|
+
resolveRunningControlPort,
|
|
30
31
|
classifyPortHolder,
|
|
31
32
|
fetchServerIdentity,
|
|
32
33
|
formatForeignListenerError,
|
|
@@ -247,13 +248,22 @@ async function main() {
|
|
|
247
248
|
// possible outcome of a port collision — only stop the listener if
|
|
248
249
|
// the identity probe confirms it's this project's server.
|
|
249
250
|
if (shouldRestart) {
|
|
250
|
-
|
|
251
|
+
// The port to TEAR DOWN is not necessarily SERVER_PORT: under the Project
|
|
252
|
+
// Launcher or the desktop supervisor the running server took a dynamic
|
|
253
|
+
// control port and recorded it in `server-state.json`, while
|
|
254
|
+
// `proxy.controlPort` is only the static default. `resolveRunningControlPort`
|
|
255
|
+
// is the one resolver for that question (it mirrors `resolve_control_port`
|
|
256
|
+
// in `crates/codeyam-editor/src/commands/editor/control_port.rs`); gating
|
|
257
|
+
// teardown on the configured port alone is what let a stale server survive a
|
|
258
|
+
// restart. SERVER_PORT stays the port to BIND.
|
|
259
|
+
const runningPort = resolveRunningControlPort(process.cwd(), SERVER_PORT);
|
|
260
|
+
const holder = await classifyPortHolder(runningPort);
|
|
251
261
|
if (holder.kind === "foreign") {
|
|
252
|
-
console.error(formatForeignListenerError(
|
|
262
|
+
console.error(formatForeignListenerError(runningPort));
|
|
253
263
|
process.exit(1);
|
|
254
264
|
}
|
|
255
265
|
if (holder.kind === "cross-project") {
|
|
256
|
-
const outcome = await resolveCrossProjectCollision(
|
|
266
|
+
const outcome = await resolveCrossProjectCollision(runningPort, holder, "restart");
|
|
257
267
|
if (outcome.action !== "took-over") {
|
|
258
268
|
// "abort" (non-TTY) or "declined": resolver already printed the message.
|
|
259
269
|
process.exit(1);
|
|
@@ -261,8 +271,8 @@ async function main() {
|
|
|
261
271
|
// took-over: stopServer freed the port; fall through to the fresh-boot path.
|
|
262
272
|
}
|
|
263
273
|
if (holder.kind === "same-project") {
|
|
264
|
-
console.error(`Stopping existing server on port ${
|
|
265
|
-
await stopServer(
|
|
274
|
+
console.error(`Stopping existing server on port ${runningPort}...`);
|
|
275
|
+
await stopServer(runningPort);
|
|
266
276
|
}
|
|
267
277
|
// holder.kind === "free": nothing to stop, fall through to ensureServer.
|
|
268
278
|
}
|
package/npm/fleet-add-failure.js
CHANGED
|
@@ -33,7 +33,11 @@ const { parseRosterMarkers } = require('./fleet-early-roster');
|
|
|
33
33
|
//
|
|
34
34
|
// 1. NOTHING WAS PROVISIONED. The classifier can prove no cloud resource was
|
|
35
35
|
// created (`preflight-drift` aborts before the first gcloud call;
|
|
36
|
-
// `instance-vanished` confirms the instance is gone
|
|
36
|
+
// `instance-vanished` confirms the instance is gone; `capacity-stockout`
|
|
37
|
+
// pairs a GCE stockout signature with the `Failed to create instance` line
|
|
38
|
+
// that `ensureInstance` throws from exactly one place — the branch where the
|
|
39
|
+
// instance was not found AND the create call failed, so no instance, disk,
|
|
40
|
+
// NEG or backend service can exist). Retry starts clean.
|
|
37
41
|
//
|
|
38
42
|
// 2. THE REMAINING WORK IS RE-ENTRANT. A real VM exists, but every step the
|
|
39
43
|
// retry would re-run is skip-on-exists or re-attaching, so the retry
|
|
@@ -50,6 +54,7 @@ const { parseRosterMarkers } = require('./fleet-early-roster');
|
|
|
50
54
|
const ADD_FAILURE_RETRYABLE = new Set([
|
|
51
55
|
'preflight-drift',
|
|
52
56
|
'instance-vanished',
|
|
57
|
+
'capacity-stockout',
|
|
53
58
|
'project-bootstrap-timeout',
|
|
54
59
|
]);
|
|
55
60
|
|
|
@@ -171,6 +176,113 @@ function nextValidActionRemedy(text) {
|
|
|
171
176
|
return i === -1 ? '' : lines.slice(i).join('\n').trim();
|
|
172
177
|
}
|
|
173
178
|
|
|
179
|
+
// Name the capacity GCE was short of, from whatever the stockout transcript
|
|
180
|
+
// happens to expose — for the capacity-stockout recovery line. GCE reports this
|
|
181
|
+
// inconsistently: the 2026-08-20 VM-11 transcript carried `vmType:` / `zone:`
|
|
182
|
+
// bullets under errorInfo.metadatas, while the 2026-08-18 VM-13 spelling
|
|
183
|
+
// (ZONE_RESOURCE_POOL_EXHAUSTED_WITH_DETAILS) carried neither. So every part is
|
|
184
|
+
// optional and the phrase degrades to a generic-but-true sentence rather than
|
|
185
|
+
// interpolating `undefined` into operator-facing copy. Extracted alongside
|
|
186
|
+
// nextValidActionRemedy (the same shape: a pure transcript reader that enriches
|
|
187
|
+
// one arm's recovery text) so the degradation is asserted directly.
|
|
188
|
+
//
|
|
189
|
+
// `zone:` is anchored with ^\s* so the neighbouring `zonesAvailable:` bullet —
|
|
190
|
+
// which lists the zones that DID have capacity — cannot be mistaken for the zone
|
|
191
|
+
// that failed. Never throws; returns the generic phrase for empty input.
|
|
192
|
+
function stockoutCapacityPhrase(logTail) {
|
|
193
|
+
const log = String(logTail || '');
|
|
194
|
+
const vmType = (log.match(/^\s*vmType:\s*(\S+)/m) || [])[1];
|
|
195
|
+
const zone = (log.match(/^\s*zone:\s*(\S+)/m) || [])[1];
|
|
196
|
+
return [vmType, 'capacity', zone ? `in ${zone}` : 'in the target zone']
|
|
197
|
+
.filter(Boolean)
|
|
198
|
+
.join(' ');
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
// The spellings git uses for "that branch is not on the remote", each capturing
|
|
202
|
+
// the ref name. The union is not defensive padding — the two bootstrap scripts
|
|
203
|
+
// reach the remote with DIFFERENT git commands, and git words the same condition
|
|
204
|
+
// differently for each:
|
|
205
|
+
// `git fetch <url> <branch>` (clone_editor_repo, scripts/gcp-bootstrap.sh)
|
|
206
|
+
// → fatal: couldn't find remote ref <ref>
|
|
207
|
+
// `git clone --branch <b>` (clone_project, scripts/gcp-project-bootstrap.sh)
|
|
208
|
+
// → fatal: Remote branch <b> not found in upstream origin
|
|
209
|
+
// Keying on only the first — the one the 2026-08-25 transcripts happened to
|
|
210
|
+
// carry — would miss every client-side clone failure. Same union-of-spellings
|
|
211
|
+
// lesson capacity-stockout learned from GCE.
|
|
212
|
+
//
|
|
213
|
+
// Deliberately NOT included: git's `warning: Could not find remote branch …`,
|
|
214
|
+
// which is a submodule-update warning that accompanies a SUCCEEDING command.
|
|
215
|
+
// Matching it here would let an add that failed for an unrelated reason claim
|
|
216
|
+
// this category — and this arm sits above the generic one, so a false positive
|
|
217
|
+
// would suppress the true cause rather than merely add noise.
|
|
218
|
+
const MISSING_REF_PATTERNS = [
|
|
219
|
+
/couldn't find remote ref (\S+)/i,
|
|
220
|
+
/Remote branch (\S+) not found in upstream/i,
|
|
221
|
+
];
|
|
222
|
+
|
|
223
|
+
// Which branch died, and which config field carried it — read from the transcript.
|
|
224
|
+
//
|
|
225
|
+
// The field matters because clientBranch and editorBranch are configured in
|
|
226
|
+
// different places: an operator who corrects only one re-issues a request that
|
|
227
|
+
// fails identically. It is derived from the log prefix of the nearest preceding
|
|
228
|
+
// marker line, because the two bootstrap scripts brand their output distinctly
|
|
229
|
+
// (`[gcp-bootstrap]` builds the editor image from the editor branch;
|
|
230
|
+
// `[project-bootstrap]` clones the client repo at the client branch). When the
|
|
231
|
+
// transcript carries neither marker the field is left null and the copy names
|
|
232
|
+
// both fields rather than guessing — the same degrade-to-true-but-generic
|
|
233
|
+
// discipline stockoutCapacityPhrase uses. Never throws; returns
|
|
234
|
+
// { ref: null, field: null } for input with no missing-ref signature.
|
|
235
|
+
function missingBranchRefFacts(logTail) {
|
|
236
|
+
const lines = String(logTail || '').split('\n');
|
|
237
|
+
for (let i = 0; i < lines.length; i += 1) {
|
|
238
|
+
let ref = null;
|
|
239
|
+
for (const re of MISSING_REF_PATTERNS) {
|
|
240
|
+
const m = lines[i].match(re);
|
|
241
|
+
if (m) { ref = m[1]; break; }
|
|
242
|
+
}
|
|
243
|
+
if (!ref) continue;
|
|
244
|
+
let field = null;
|
|
245
|
+
for (let j = i; j >= 0; j -= 1) {
|
|
246
|
+
if (/\[project-bootstrap\]/.test(lines[j])) { field = 'clientBranch'; break; }
|
|
247
|
+
if (/\[gcp-bootstrap\]/.test(lines[j])) { field = 'editorBranch'; break; }
|
|
248
|
+
}
|
|
249
|
+
return { ref, field };
|
|
250
|
+
}
|
|
251
|
+
return { ref: null, field: null };
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
// Operator-facing recovery copy for the missing-branch-ref arm.
|
|
255
|
+
//
|
|
256
|
+
// Extracted from the arm for the same reason stockoutCapacityPhrase was — it is
|
|
257
|
+
// the one arm whose copy VARIES with what the transcript happened to expose, so
|
|
258
|
+
// pulling it out lets the wording be asserted directly instead of only through
|
|
259
|
+
// classifyAddFailure, and keeps the arm itself a two-line decision.
|
|
260
|
+
//
|
|
261
|
+
// Two properties are load-bearing and are what the tests pin:
|
|
262
|
+
// - It states the outcome FIRST and never borrows the generic arm's "a partial
|
|
263
|
+
// instance may exist — reconcile or destroy it" language. At this point git
|
|
264
|
+
// has aborted before any image build or container work, so that advice sends
|
|
265
|
+
// the operator hunting for a box that was never created — the inversion that
|
|
266
|
+
// cost the 2026-08-25 recovery its first hour.
|
|
267
|
+
// - The remedy is a DIFFERENT action, never "retry". Re-issuing the same
|
|
268
|
+
// config against a deleted ref fails identically forever, which is exactly
|
|
269
|
+
// why this category is kept out of ADD_FAILURE_RETRYABLE.
|
|
270
|
+
//
|
|
271
|
+
// `field` is optional: when the transcript carries no bootstrap marker the copy
|
|
272
|
+
// names both fields rather than guessing which one carried the dead ref.
|
|
273
|
+
// Pure; no I/O.
|
|
274
|
+
function missingBranchRefRecovery({ ref, field } = {}) {
|
|
275
|
+
const carried = field
|
|
276
|
+
? `Its ${field} still points at it`
|
|
277
|
+
: 'Its clientBranch and editorBranch both need checking';
|
|
278
|
+
return (
|
|
279
|
+
`Branch '${ref}' is not on origin any more — it was almost certainly pruned when it merged. ` +
|
|
280
|
+
'Nothing was provisioned: git aborted before any image build or container work, so there is no instance, ' +
|
|
281
|
+
`disk, NEG or backend service to clean up, and nothing to destroy. ${carried}, so retrying as-is will fail ` +
|
|
282
|
+
'identically every time. Re-add on a branch that still exists (the current fleet branch is the usual answer).'
|
|
283
|
+
);
|
|
284
|
+
}
|
|
285
|
+
|
|
174
286
|
// Build a verdict, stamping `retryable` from the single-source-of-truth Set so it
|
|
175
287
|
// can never drift from the category.
|
|
176
288
|
function buildFailureVerdict(category, headline, recovery, extra) {
|
|
@@ -200,11 +312,17 @@ function buildFailureVerdict(category, headline, recovery, extra) {
|
|
|
200
312
|
// Returns { category, headline, recovery, retryable, files? }.
|
|
201
313
|
//
|
|
202
314
|
// Detection is ORDER-SENSITIVE. The provision shell echoes cloud:up's tail, so a
|
|
203
|
-
// drift / instance-vanished / container-stuck
|
|
204
|
-
// generic `cloud:up FAILED
|
|
205
|
-
//
|
|
206
|
-
//
|
|
207
|
-
// operator would lose the
|
|
315
|
+
// drift / instance-vanished / container-stuck / capacity-stockout /
|
|
316
|
+
// missing-branch-ref transcript ALSO contains the generic `cloud:up FAILED
|
|
317
|
+
// (exit N)` line. Each specific (and every retryable) signature is therefore
|
|
318
|
+
// tested BEFORE the generic cloud-up-failed signature, or a retryable drift would
|
|
319
|
+
// misclassify as a non-retryable cloud:up failure and the operator would lose the
|
|
320
|
+
// safe one-click recovery. That is not a hypothetical: it is exactly the bug
|
|
321
|
+
// `capacity-stockout` (arm 6) was added to fix, and its placement immediately
|
|
322
|
+
// above cloud-up-failed is load-bearing. `missing-branch-ref` (arm 7) has the
|
|
323
|
+
// same property for the same reason — it also sits immediately above the generic
|
|
324
|
+
// arm, and its ordering is what stops a dead ref reporting as a half-provisioned
|
|
325
|
+
// instance.
|
|
208
326
|
function classifyAddFailure({ logTail, exitCode, instanceAlive, containerHealthy, wiringDeadlineBlown, wiringInFlight } = {}) {
|
|
209
327
|
const log = String(logTail || '');
|
|
210
328
|
|
|
@@ -272,33 +390,97 @@ function classifyAddFailure({ logTail, exitCode, instanceAlive, containerHealthy
|
|
|
272
390
|
'The container came up but the project was never scaffolded (no .codeyam/editor.json on the VM), so provisioning did not finish onboarding. Reconcile this VM to complete the scaffold, then re-add.');
|
|
273
391
|
}
|
|
274
392
|
|
|
275
|
-
// 6.
|
|
276
|
-
//
|
|
277
|
-
//
|
|
393
|
+
// 6. capacity-stockout — GCE had no capacity for the machine type in the target
|
|
394
|
+
// zone, so `ensureInstance` (npm/cloud.js) threw from its create branch. This
|
|
395
|
+
// fired twice in three days — VM-13 / payper on 2026-08-18 and VM-11 /
|
|
396
|
+
// codeyam-cms on 2026-08-20 — and both times classified as the generic
|
|
397
|
+
// cloud-up-failed below, whose copy wrongly warned that a partial instance may
|
|
398
|
+
// exist and whose non-retryable verdict refused the one failure that needs
|
|
399
|
+
// nothing but a retry. Each incident cost manual state surgery on the operator
|
|
400
|
+
// host.
|
|
401
|
+
//
|
|
402
|
+
// Retryable on ground 1 (NOTHING WAS PROVISIONED), proven STRUCTURALLY rather
|
|
403
|
+
// than by trusting the error prose: the arm fires only on a stockout signature
|
|
404
|
+
// AND the `Failed to create instance` line, which ensureInstance throws from
|
|
405
|
+
// exactly one place — the branch where the instance was not found and the
|
|
406
|
+
// create call then failed. That conjunction is positive proof no cloud
|
|
407
|
+
// resource exists, the same standing preflight-drift and instance-vanished
|
|
408
|
+
// already have; both incidents were verified by hand to have left no instance,
|
|
409
|
+
// disk, NEG or backend service behind. The not-rostered guard is belt-and-
|
|
410
|
+
// braces: if a future transcript stockouts on one attempt and then creates a
|
|
411
|
+
// box, a roster marker means a real VM exists and this arm must not fire.
|
|
412
|
+
//
|
|
413
|
+
// The signature is a UNION because GCE spells this more than one way: the
|
|
414
|
+
// 2026-08-18 transcript carried `ZONE_RESOURCE_POOL_EXHAUSTED_WITH_DETAILS`
|
|
415
|
+
// with `reason: stockout`, while 2026-08-20 carried `reason:
|
|
416
|
+
// resource_availability` with `state:STOCKOUT, sub-state:STOCKOUT` and the
|
|
417
|
+
// prose below. Keying on only the first would have missed the second. The
|
|
418
|
+
// prose alternative uses `\s+` because the message wraps mid-sentence.
|
|
419
|
+
//
|
|
420
|
+
// Tested BEFORE cloud-up-failed because the provision shell echoes cloud:up's
|
|
421
|
+
// tail, so a stockout transcript ALSO carries `cloud:up FAILED (exit N)` —
|
|
422
|
+
// matching that first is precisely the bug being fixed here.
|
|
423
|
+
if (/ZONE_RESOURCE_POOL_EXHAUSTED|\bSTOCKOUT\b|reason:\s*resource_availability|does not have\s+enough resources available/i.test(log)
|
|
424
|
+
&& /Failed to create instance/i.test(log)
|
|
425
|
+
&& !transcriptRostered(log)) {
|
|
426
|
+
return buildFailureVerdict('capacity-stockout', 'No capacity in the zone',
|
|
427
|
+
`GCE had no ${stockoutCapacityPhrase(log)} at that moment, so the create call failed before anything existed — there is no instance, disk, NEG or backend service to clean up, and nothing to undo. Capacity is transient but its return is not scheduled: wait a few minutes, then Retry add. Do NOT destroy anything.`);
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
// 7. missing-branch-ref — the requested branch no longer exists on origin, so
|
|
431
|
+
// git aborted the clone/fetch. Observed live on 2026-08-25: VMs 2, 4, 5 and
|
|
432
|
+
// 7 each died on `fatal: couldn't find remote ref editor-improvements-75`,
|
|
433
|
+
// a branch pruned from origin earlier that day when it merged. This repo
|
|
434
|
+
// prunes on merge, so ANY stored feature-branch name expires silently —
|
|
435
|
+
// this is a recurring condition, not a one-off.
|
|
436
|
+
//
|
|
437
|
+
// NOT retryable, and deliberately NOT a member of ADD_FAILURE_RETRYABLE
|
|
438
|
+
// even though it satisfies ground 1 (nothing was provisioned). Membership
|
|
439
|
+
// means "re-issuing the SAME request can succeed"; re-issuing an add
|
|
440
|
+
// against a deleted ref fails identically forever. Admitting it would
|
|
441
|
+
// automate the guaranteed-fail loop the operator was stuck in by hand. The
|
|
442
|
+
// honest remedy is a DIFFERENT request — re-add on a live branch — which is
|
|
443
|
+
// what the recovery copy names.
|
|
444
|
+
//
|
|
445
|
+
// Tested BEFORE cloud-up-failed for the reason arm 6 documents: the
|
|
446
|
+
// provision shell echoes cloud:up's tail, so this transcript ALSO carries
|
|
447
|
+
// the generic `cloud:up FAILED (exit N)` line. Matching that first is the
|
|
448
|
+
// bug being fixed — it is what made the real incident report the OPPOSITE
|
|
449
|
+
// of the truth, warning that "a partial instance may exist — reconcile or
|
|
450
|
+
// destroy it" when nothing had been provisioned at all.
|
|
451
|
+
const missingRef = missingBranchRefFacts(log);
|
|
452
|
+
if (missingRef.ref) {
|
|
453
|
+
return buildFailureVerdict('missing-branch-ref', 'Branch no longer exists',
|
|
454
|
+
missingBranchRefRecovery(missingRef));
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
// 8. cloud-up-failed — a generic cloud:up failure, reached ONLY when the drift /
|
|
458
|
+
// vanished / stuck / stockout / missing-ref signatures above did not match. A
|
|
459
|
+
// partial instance may exist, so not retryable.
|
|
278
460
|
if (/cloud:up FAILED \(exit/i.test(log)) {
|
|
279
461
|
return buildFailureVerdict('cloud-up-failed', 'Provisioning failed',
|
|
280
462
|
'cloud:up failed during provisioning, so a partial instance may exist. Reconcile or destroy this VM before re-adding.');
|
|
281
463
|
}
|
|
282
464
|
|
|
283
|
-
//
|
|
465
|
+
// 9. interrupted — a signal kill (OOM 137, SIGTERM 143, …) or an explicit
|
|
284
466
|
// Killed / out-of-memory marker. The real VM state is unknown. Not retryable.
|
|
285
467
|
if (fc.isSignalKillExit(Number(exitCode)) || /\bKilled\b/.test(log) || /Out of memory/i.test(log)) {
|
|
286
468
|
return buildFailureVerdict('interrupted', 'Provisioning interrupted',
|
|
287
469
|
'The provision was interrupted (killed or out of memory), so the VM state is unknown. Reconcile or destroy this VM before re-adding.');
|
|
288
470
|
}
|
|
289
471
|
|
|
290
|
-
//
|
|
472
|
+
// 10. The post-add IAP read-back verdicts. Unlike every signature above, this one
|
|
291
473
|
// is not produced by the provision transcript at all — it is the BLOCKED
|
|
292
474
|
// reason iapReadbackOutcome hands to annotateAddFailure, which is what
|
|
293
475
|
// stops a read-back failure from being the one add failure that reaches the
|
|
294
476
|
// card with no category at all (and so no differentiated headline, no
|
|
295
477
|
// recovery, no guarded Retry, and no chance for holdConvergingBadge to hold it
|
|
296
|
-
// calm). Tested AFTER all
|
|
478
|
+
// calm). Tested AFTER all eight hard-failure signatures so a proven failure
|
|
297
479
|
// always wins if both texts are ever present, and BEFORE converging/verifying
|
|
298
480
|
// because it is strictly more specific than either.
|
|
299
481
|
if (/IAP policy authorizes no human/i.test(log)) {
|
|
300
482
|
const remedy = nextValidActionRemedy(log);
|
|
301
|
-
//
|
|
483
|
+
// 9a. access-pending — no accessors YET. The grant is the LAST step of the
|
|
302
484
|
// detached wire-add-tail, so while that tail is alive an empty policy is
|
|
303
485
|
// the expected mid-flight reading, not a verdict. Calm: this is the state
|
|
304
486
|
// that used to render a red "Add failed" on healthy VM-4/VM-7 minutes
|
|
@@ -307,7 +489,7 @@ function classifyAddFailure({ logTail, exitCode, instanceAlive, containerHealthy
|
|
|
307
489
|
return buildFailureVerdict('access-pending', 'Access not granted yet',
|
|
308
490
|
'The VM is up and rostered, but its IAP policy has no human accessors yet — the add\'s background wiring applies the grant as its last step and is still running. This resolves on its own once that tail finishes; no action is needed unless it persists past the wiring deadline.');
|
|
309
491
|
}
|
|
310
|
-
//
|
|
492
|
+
// 9b. iap-unauthorized — the wiring is done and the policy STILL authorizes
|
|
311
493
|
// nobody, so the public URL returns "Error code 51" for every human. This
|
|
312
494
|
// is the 2026-08-09 case the read-back exists to catch (one step granted
|
|
313
495
|
// the operator, the next revoked him) and it is a genuine hard failure.
|
|
@@ -317,12 +499,12 @@ function classifyAddFailure({ logTail, exitCode, instanceAlive, containerHealthy
|
|
|
317
499
|
`The VM is up but its IAP policy authorizes no human, so the public URL returns "Error code 51" for everyone. Its background wiring is no longer running, so this will not resolve on its own — grant access to fix it. Do NOT destroy or re-add this VM.${remedy ? `\n\n${remedy}` : ''}`);
|
|
318
500
|
}
|
|
319
501
|
|
|
320
|
-
//
|
|
502
|
+
// 11. converging — the add reached the early-roster marker (`CY_ROSTER`): the
|
|
321
503
|
// container gate passed, the instance is real, and the VM is rostered, yet it
|
|
322
504
|
// exited non-zero WITHOUT matching any specific hard-failure signature above.
|
|
323
505
|
// By construction of the provision script, every genuine hard failure (drift,
|
|
324
|
-
// instance-vanished, container-stuck, incomplete-workspace,
|
|
325
|
-
// interrupted) is printed BEFORE `CY_ROSTER`, so a rostered transcript that
|
|
506
|
+
// instance-vanished, container-stuck, incomplete-workspace, capacity-stockout,
|
|
507
|
+
// cloud-up-failed, interrupted) is printed BEFORE `CY_ROSTER`, so a rostered transcript that
|
|
326
508
|
// fell through to here is a still-wiring box owned by the detached
|
|
327
509
|
// wire-add-tail / reconcile-ingress — NOT an add failure. Surface it as an
|
|
328
510
|
// honest in-flight state instead of the scary red `unknown`. Not retryable:
|
|
@@ -335,7 +517,7 @@ function classifyAddFailure({ logTail, exitCode, instanceAlive, containerHealthy
|
|
|
335
517
|
'The container is up and the VM is rostered; the LB ingress is still wiring in the background. This resolves on its own once the editor is reachable over the LB — no action needed unless it persists past the wiring deadline.');
|
|
336
518
|
}
|
|
337
519
|
|
|
338
|
-
//
|
|
520
|
+
// 12. The residue: no failure signature above matched. The polarity here is
|
|
339
521
|
// INVERTED from the original design. It used to read "no recognized success
|
|
340
522
|
// signature ⇒ failed", which made red the DEFAULT for "we don't know" — so a
|
|
341
523
|
// VM whose container was Up and whose editor answered /api/health 200 got a
|
|
@@ -345,7 +527,7 @@ function classifyAddFailure({ logTail, exitCode, instanceAlive, containerHealthy
|
|
|
345
527
|
//
|
|
346
528
|
// Red now requires positive proof of death, and everything else is in-flight:
|
|
347
529
|
//
|
|
348
|
-
//
|
|
530
|
+
// 11a. unknown — a non-zero exit CORROBORATED by a durable fact (see
|
|
349
531
|
// addFailureCorroborated). The box is provably dead or provably never
|
|
350
532
|
// came up, so the loud, terminal escalation is warranted and unchanged.
|
|
351
533
|
// Still never invites an unproven retry.
|
|
@@ -354,7 +536,7 @@ function classifyAddFailure({ logTail, exitCode, instanceAlive, containerHealthy
|
|
|
354
536
|
'The add failed for an unclassified reason and the VM is confirmed dead or never came up — inspect the transcript below, then reconcile or destroy this VM before re-adding.');
|
|
355
537
|
}
|
|
356
538
|
|
|
357
|
-
//
|
|
539
|
+
// 11b. verifying — the honest default. No failure signature, and nothing
|
|
358
540
|
// durable proves this VM is dead: it may still be provisioning or
|
|
359
541
|
// wiring, and the ingress can keep propagating for minutes after the
|
|
360
542
|
// wrapper is gone. Calm copy, transcript still one click away. NOT
|
|
@@ -405,4 +587,36 @@ function holdConvergingBadge({ category, wiringDeadlineBlown } = {}) {
|
|
|
405
587
|
return CALM_ADD_CATEGORIES.has(category) && !wiringDeadlineBlown;
|
|
406
588
|
}
|
|
407
589
|
|
|
408
|
-
|
|
590
|
+
// Refusal copy for a Retry on a card whose branch was deleted from origin.
|
|
591
|
+
//
|
|
592
|
+
// The generic non-retryable refusal says the add "may have left a
|
|
593
|
+
// half-provisioned instance — reconcile or destroy it". For this category that
|
|
594
|
+
// is the OPPOSITE of the truth: git aborts before any image build or container
|
|
595
|
+
// work, so nothing exists. Sending an operator to hunt for a box that was never
|
|
596
|
+
// created is what cost the 2026-08-25 recovery its first hour, so this copy
|
|
597
|
+
// leads with the outcome (nothing to clean up) before naming the mechanism.
|
|
598
|
+
//
|
|
599
|
+
// The remedy is deliberately not "retry". Re-issuing the same config against a
|
|
600
|
+
// deleted ref fails identically forever — which is exactly why the category is
|
|
601
|
+
// kept OUT of ADD_FAILURE_RETRYABLE — so the honest advice is a different
|
|
602
|
+
// action. For a self-host VM the fleet pin names the live branch and a re-add
|
|
603
|
+
// now picks it up, so the copy can be specific; for a client VM the branch is a
|
|
604
|
+
// per-project choice only the operator can make, so it asks instead of guessing.
|
|
605
|
+
//
|
|
606
|
+
// `ref` / `field` come from missingBranchRefFacts and are both optional: the copy
|
|
607
|
+
// degrades to true-but-generic rather than interpolating `undefined`, the same
|
|
608
|
+
// discipline stockoutCapacityPhrase uses. Pure; no I/O.
|
|
609
|
+
function deadRefRetryMessage({ n, ref, field, selfHost = false, pin } = {}) {
|
|
610
|
+
const which = ref ? `'${ref}'` : 'the requested branch';
|
|
611
|
+
const carried = field ? ` (${field})` : '';
|
|
612
|
+
const remedy = selfHost && pin
|
|
613
|
+
? `Add VM again — it now picks up the current fleet branch (${pin}) instead of the stored one.`
|
|
614
|
+
: 'Add VM again with a branch that still exists on origin.';
|
|
615
|
+
return (
|
|
616
|
+
`VM-${n}: branch ${which}${carried} no longer exists on origin, so nothing was provisioned — ` +
|
|
617
|
+
'there is no instance, disk or backend service to clean up. Retrying as-is re-targets ' +
|
|
618
|
+
`the same dead branch and fails identically every time. ${remedy}`
|
|
619
|
+
);
|
|
620
|
+
}
|
|
621
|
+
|
|
622
|
+
module.exports = { classifyAddFailure, extractDriftFiles, lastNonEmptyLines, canRetryAdd, transcriptRostered, holdConvergingBadge, addFailureCorroborated, containerHealthWitness, stockoutCapacityPhrase, missingBranchRefFacts, missingBranchRefRecovery, deadRefRetryMessage, CALM_ADD_CATEGORIES, ADD_FAILURE_RETRYABLE };
|