@codeyam-editor/codeyam-editor 0.1.15-staging.g8a9ebd4 → 0.1.16-staging.ge120017

package/npm/app-port-drift.js

@@ -59,6 +59,27 @@ function parsePublishedAppPort(stdout) {
   return isValidPort(port) ? port : null;
 }
 
+const PUBLISHED_BEGIN = "<<<PUBLISHED_PORTS>>>";
+const PUBLISHED_END = "<<<END_PUBLISHED_PORTS>>>";
+
+/** The published-port read wrapped in begin/end markers, so it can ride an SSH
+ *  round-trip that also prints other output (the dashboard folds it into its
+ *  existing per-VM fact gather rather than paying a second SSH). */
+function buildPublishedAppPortMarkedCommand() {
+  return `echo '${PUBLISHED_BEGIN}'; ${buildPublishedAppPortReadCommand()}; echo '${PUBLISHED_END}'`;
+}
+
+/** Pull the published app port out of that marked section, or null when the
+ *  section is missing or truncated — never a guess. */
+function extractPublishedAppPort(stdout) {
+  if (typeof stdout !== "string") return null;
+  const begin = stdout.indexOf(PUBLISHED_BEGIN);
+  if (begin === -1) return null;
+  const end = stdout.indexOf(PUBLISHED_END, begin);
+  if (end === -1) return null;
+  return parsePublishedAppPort(stdout.slice(begin + PUBLISHED_BEGIN.length, end));
+}
+
 /** Parse `cloud-session-idle.sh` stdout into `idle` | `active` | null. Null
  *  (unknown) is deliberately NOT idle — the same asymmetry that script and
  *  `converge-vm.sh --converge-env` honour. */
@@ -161,6 +182,8 @@ module.exports = {
   EDITOR_CONTROL_PORT,
   buildPublishedAppPortReadCommand,
   parsePublishedAppPort,
+  buildPublishedAppPortMarkedCommand,
+  extractPublishedAppPort,
   parseIdleVerdict,
   appPortDriftRemediations,
   collectAppPortDrift,

package/npm/fleet-add-failure-evidence.js

@@ -0,0 +1,309 @@
+'use strict';
+
+// npm/fleet-add-failure-evidence.js — the per-arm transcript readers and
+// operator-facing recovery copy behind classifyAddFailure (fleet-add-failure.js).
+// Each arm that needs more than a regex reads its facts out of the provision
+// transcript here, and each arm whose wording varies with those facts builds its
+// recovery line here, so the copy is asserted directly rather than only through
+// the classifier. Pure: no I/O, never throws on empty input.
+
+// Name the capacity GCE was short of, from whatever the stockout transcript
+// happens to expose — for the capacity-stockout recovery line. GCE reports this
+// inconsistently: the 2026-08-20 VM-11 transcript carried `vmType:` / `zone:`
+// bullets under errorInfo.metadatas, while the 2026-08-18 VM-13 spelling
+// (ZONE_RESOURCE_POOL_EXHAUSTED_WITH_DETAILS) carried neither. So every part is
+// optional and the phrase degrades to a generic-but-true sentence rather than
+// interpolating `undefined` into operator-facing copy. Extracted alongside
+// nextValidActionRemedy (the same shape: a pure transcript reader that enriches
+// one arm's recovery text) so the degradation is asserted directly.
+//
+// `zone:` is anchored with ^\s* so the neighbouring `zonesAvailable:` bullet —
+// which lists the zones that DID have capacity — cannot be mistaken for the zone
+// that failed. Never throws; returns the generic phrase for empty input.
+function stockoutCapacityPhrase(logTail) {
+  const log = String(logTail || '');
+  const vmType = (log.match(/^\s*vmType:\s*(\S+)/m) || [])[1];
+  const zone = (log.match(/^\s*zone:\s*(\S+)/m) || [])[1];
+  return [vmType, 'capacity', zone ? `in ${zone}` : 'in the target zone']
+    .filter(Boolean)
+    .join(' ');
+}
+
+// The spellings git uses for "that branch is not on the remote", each capturing
+// the ref name. The union is not defensive padding — the two bootstrap scripts
+// reach the remote with DIFFERENT git commands, and git words the same condition
+// differently for each:
+//   `git fetch <url> <branch>`  (clone_editor_repo, scripts/gcp-bootstrap.sh)
+//        → fatal: couldn't find remote ref <ref>
+//   `git clone --branch <b>`    (clone_project, scripts/gcp-project-bootstrap.sh)
+//        → fatal: Remote branch <b> not found in upstream origin
+// Keying on only the first — the one the 2026-08-25 transcripts happened to
+// carry — would miss every client-side clone failure. Same union-of-spellings
+// lesson capacity-stockout learned from GCE.
+//
+// Deliberately NOT included: git's `warning: Could not find remote branch …`,
+// which is a submodule-update warning that accompanies a SUCCEEDING command.
+// Matching it here would let an add that failed for an unrelated reason claim
+// this category — and this arm sits above the generic one, so a false positive
+// would suppress the true cause rather than merely add noise.
+const MISSING_REF_PATTERNS = [
+  /couldn't find remote ref (\S+)/i,
+  /Remote branch (\S+) not found in upstream/i,
+];
+
+// Which branch died, and which config field carried it — read from the transcript.
+//
+// The field matters because clientBranch and editorBranch are configured in
+// different places: an operator who corrects only one re-issues a request that
+// fails identically. It is derived from the log prefix of the nearest preceding
+// marker line, because the two bootstrap scripts brand their output distinctly
+// (`[gcp-bootstrap]` builds the editor image from the editor branch;
+// `[project-bootstrap]` clones the client repo at the client branch). When the
+// transcript carries neither marker the field is left null and the copy names
+// both fields rather than guessing — the same degrade-to-true-but-generic
+// discipline stockoutCapacityPhrase uses. Never throws; returns
+// { ref: null, field: null } for input with no missing-ref signature.
+function missingBranchRefFacts(logTail) {
+  const lines = String(logTail || '').split('\n');
+  for (let i = 0; i < lines.length; i += 1) {
+    let ref = null;
+    for (const re of MISSING_REF_PATTERNS) {
+      const m = lines[i].match(re);
+      if (m) { ref = m[1]; break; }
+    }
+    if (!ref) continue;
+    let field = null;
+    for (let j = i; j >= 0; j -= 1) {
+      if (/\[project-bootstrap\]/.test(lines[j])) { field = 'clientBranch'; break; }
+      if (/\[gcp-bootstrap\]/.test(lines[j])) { field = 'editorBranch'; break; }
+    }
+    return { ref, field };
+  }
+  return { ref: null, field: null };
+}
+
+// Operator-facing recovery copy for the missing-branch-ref arm.
+//
+// Extracted from the arm for the same reason stockoutCapacityPhrase was — it is
+// the one arm whose copy VARIES with what the transcript happened to expose, so
+// pulling it out lets the wording be asserted directly instead of only through
+// classifyAddFailure, and keeps the arm itself a two-line decision.
+//
+// Two properties are load-bearing and are what the tests pin:
+//   - It states the outcome FIRST and never borrows the generic arm's "a partial
+//     instance may exist — reconcile or destroy it" language. At this point git
+//     has aborted before any image build or container work, so that advice sends
+//     the operator hunting for a box that was never created — the inversion that
+//     cost the 2026-08-25 recovery its first hour.
+//   - The remedy is a DIFFERENT action, never "retry". Re-issuing the same
+//     config against a deleted ref fails identically forever, which is exactly
+//     why this category is kept out of ADD_FAILURE_RETRYABLE.
+//
+// `field` is optional: when the transcript carries no bootstrap marker the copy
+// names both fields rather than guessing which one carried the dead ref.
+// Pure; no I/O.
+function missingBranchRefRecovery({ ref, field } = {}) {
+  const carried = field
+    ? `Its ${field} still points at it`
+    : 'Its clientBranch and editorBranch both need checking';
+  return (
+    `Branch '${ref}' is not on origin any more — it was almost certainly pruned when it merged. ` +
+    'Nothing was provisioned: git aborted before any image build or container work, so there is no instance, ' +
+    `disk, NEG or backend service to clean up, and nothing to destroy. ${carried}, so retrying as-is will fail ` +
+    'identically every time. Re-add on a branch that still exists (the current fleet branch is the usual answer).'
+  );
+}
+
+// Facts for the wedged-insert arm: a GCE instance insert that was ACCEPTED and
+// then never finished. Returns { wedged, code, instance } — `code` is the GCE
+// support code when the transcript carried one, so the recovery copy can hand the
+// operator the exact string a Support case needs instead of making them dig it out
+// of the transcript by hand.
+//
+// Two spellings reach here and both are proof, so the match is a UNION:
+//   A. The raw GCE prose, which is what the 2026-09-04 VM-8 transcript carried:
+//      `Internal error. Please try again or contact Google Support. (Code: '…')`
+//      PAIRED with evidence this run actually attempted a create. The pairing is
+//      what keeps the arm narrow — the same prose surfacing from some other
+//      gcloud call is not an insert that wedged, and must not claim to be one.
+//   B. ensureInstance's own fast-fail line (npm/cloud.js), which fires when a
+//      re-add finds the slot still parked in PROVISIONING behind an operation
+//      that never reached DONE. That path exists precisely so the transcript
+//      carries a signature instead of an uninformative 2-minute timeout, so this
+//      arm keys on it directly.
+//
+// Deliberately NOT keyed on elapsed time. A duration heuristic ("PROVISIONING for
+// too long") would misfire on a legitimately slow create; an INTERNAL_ERROR on an
+// accepted insert is provable evidence, which is the same signature-over-duration
+// discipline the capacity-stockout arm uses.
+const WEDGED_INSERT_PROSE = /Internal error\. Please try again or contact Google Support/i;
+const WEDGED_INSERT_FASTFAIL = /stuck in PROVISIONING behind a wedged create operation/i;
+const WEDGED_INSERT_CREATE_ATTEMPTED = /not found\s*—\s*creating|Failed to create instance|did not reach RUNNING/i;
+
+function wedgedInsertFacts(logTail) {
+  const log = String(logTail || '');
+  const wedged = WEDGED_INSERT_FASTFAIL.test(log)
+    || (WEDGED_INSERT_PROSE.test(log) && WEDGED_INSERT_CREATE_ATTEMPTED.test(log));
+  if (!wedged) return { wedged: false, code: null, instance: null };
+  return {
+    wedged: true,
+    code: (log.match(/\(Code:\s*'(-?\d+)'\)/) || [])[1] || null,
+    instance: (log.match(/Checking instance\s+(\S+?)\.\.\./) || [])[1] || null,
+  };
+}
+
+// Operator-facing recovery copy for the wedged-insert arm, extracted for the same
+// reason stockoutCapacityPhrase and missingBranchRefRecovery were: its wording
+// varies with what the transcript exposed, so pulling it out lets the copy be
+// asserted directly rather than only through classifyAddFailure.
+//
+// Three properties are load-bearing and are what the tests pin:
+//   - It never borrows the generic arm's "reconcile or destroy it" line. A delete
+//     against a wedged slot is ACCEPTED and then queues behind the wedged insert
+//     rather than clearing it, so that advice sends the operator to an action that
+//     provably does not work — the same inversion that cost the 2026-08-25
+//     missing-branch-ref recovery its first hour.
+//   - It says plainly that this is not the operator's fault and not their fix.
+//     The only thing that resolves it is GCE-side.
+//   - It names the Support code when one was captured, because that string is
+//     what a Google Cloud Support case is keyed on.
+function wedgedInsertRecovery({ code, instance } = {}) {
+  const named = instance || 'the instance';
+  const cite = code ? ` citing operation error code ${code}` : '';
+  return (
+    `GCE accepted the create for ${named} and then never finished it: the insert stayed at status RUNNING, progress 0 ` +
+    'and eventually attached an INTERNAL_ERROR without ever reaching DONE, leaving the instance in PROVISIONING and its ' +
+    'boot disk in CREATING. Nothing on our side failed and there is nothing here to reconcile. Do NOT retry — the ' +
+    'instance name is held by the wedged operation, so a re-add cannot succeed while it is outstanding. Do NOT try to ' +
+    'destroy it either — a delete is accepted but queues behind the wedged insert instead of clearing it. This needs ' +
+    `GCE-side resolution: open a Google Cloud Support case${cite}. If you need the slot before Google releases it, add ` +
+    'on a different VM number.'
+  );
+}
+
+// Facts for the iap-not-ready arm: waitForIapReady (npm/cloud.js) threw its
+// readiness timeout. Returns { timedOut, instance, budgetSecs, cause } — `cause` is
+// the label waitForIapReady derived from its probe stderr (null when none of its
+// signatures matched, or on a transcript from before the label existed), so the
+// recovery copy can name WHICH wait was blown instead of only `return code [255]`.
+//
+// The signature is emitted from exactly one place, and that place runs before any
+// bootstrap work, which is what lets this arm assert the instance is bare.
+const IAP_NOT_READY = /IAP did not become ready for (\S+) within (\d+)s/;
+
+function iapNotReadyFacts(logTail) {
+  const m = String(logTail || '').match(IAP_NOT_READY);
+  if (!m) return { timedOut: false, instance: null, budgetSecs: null, cause: null };
+  const cause = (String(logTail).match(/likely cause: ([^;)]+)/) || [])[1] || null;
+  return { timedOut: true, instance: m[1], budgetSecs: Number(m[2]), cause: cause && cause.trim() };
+}
+
+// Operator-facing recovery copy for the iap-not-ready arm. It states what the
+// generic arm cannot: the instance EXISTS and is BARE. "A partial instance may
+// exist" is what steered VM-4's operator toward destroy + re-add (15-25 minutes)
+// when Retry resumes on the same box in seconds — and it left nothing on the card
+// saying a RUNNING, billing instance was sitting there.
+function iapNotReadyRecovery({ instance, budgetSecs, cause } = {}) {
+  const named = instance || 'The instance';
+  const within = budgetSecs ? ` within ${budgetSecs}s` : '';
+  const why = cause ? ` (likely cause: ${cause})` : '';
+  return (
+    `${named} was created and is RUNNING, but SSH over IAP did not answer${within}${why}. This is usually a transient ` +
+    'control-plane race. Nothing was installed on it — the wait runs before any bootstrap step — so it is a bare box and ' +
+    'there is nothing partial to reconcile. Retry add resumes on this same instance rather than creating a second one. ' +
+    'Until you retry or destroy it, the instance keeps running and billing.'
+  );
+}
+
+// Facts for the streamed-library-missing arm: a streamed bootstrap script sourced
+// a `$HOME`-rooted `lib/…` file that was never shipped to the VM, and bash
+// reported `<script>: line N: <path>: No such file or directory`. This took ALL
+// fleet provisioning down twice (lib/npm-pin.sh 2026-09-04, lib/render-cloud-
+// compose.sh 2026-09-18). Returns { missing, script, library } — `script` is the
+// basename of the aborting script, `library` the `lib/…` path relative to $HOME,
+// which is how PROJECT_BOOTSTRAP_LIBS (npm/cloud.js) names its entries.
+const STREAMED_LIBRARY_MISSING = /(\S+): line \d+: \/(?:home\/[^/\s]+|root)\/(lib\/[^\s:]+): No such file or directory/;
+
+function streamedLibraryMissingFacts(logTail) {
+  const m = String(logTail || '').match(STREAMED_LIBRARY_MISSING);
+  if (!m) return { missing: false, script: null, library: null };
+  return { missing: true, script: m[1].split('/').pop(), library: m[2] };
+}
+
+// Did cloud:up report that it deleted the instance it created this run? The abort
+// path (npm/cloud.js) prints the teardown marker BEFORE it issues the delete, and
+// prints a `during abort teardown` warning when that delete fails — so the marker
+// alone is a claim of intent, and only the marker WITHOUT the warning is evidence
+// that no orphan was left behind.
+function cleanTeardownReported(logTail) {
+  const log = String(logTail || '');
+  return /tearing down the VM cloud:up created this run \(no orphan left behind\)/.test(log)
+    && !/during abort teardown/.test(log);
+}
+
+// Operator-facing recovery copy for the streamed-library-missing arm. It names
+// what to fix — register the library so it is streamed — and, when the transcript
+// proves a clean teardown, says there is no orphan to hunt for. Every add on this
+// branch fails identically until the registry is fixed, so it never offers Retry.
+function streamedLibraryMissingRecovery({ script, library, cleanTeardown } = {}) {
+  const lib = library || 'a lib/ file';
+  const from = script ? ` sourced by ${script}` : '';
+  const orphan = cleanTeardown
+    ? 'cloud:up already deleted the VM it created, so there is nothing to reconcile or destroy.'
+    : 'The transcript does not confirm the VM was deleted — check whether a partial instance exists before re-adding.';
+  return (
+    `${lib}${from} was never shipped to the VM, so the bootstrap aborted. Add it to PROJECT_BOOTSTRAP_LIBS in npm/cloud.js, ` +
+    `push, and re-add — every add from this branch fails the same way until then. ${orphan}`
+  );
+}
+
+// Did the OPERATOR HOST run out of filesystem? Node spells it `ENOSPC: no space
+// left on device`, the shell `No space left on device`, gcloud `OSError: [Errno
+// 28] No space left on device` — matched on the message, never on the errno
+// number, which means something else on other platforms. A transcript that says a
+// script "failed on the VM" is quoting the VM's own log tail (npm/cloud.js prints
+// that header above it), so an ENOSPC there is the VM's disk, not the host's, and
+// stays with the generic arm.
+const HOST_DISK_EXHAUSTED = /ENOSPC: no space left on device|No space left on device/i;
+
+function hostDiskExhausted(logTail) {
+  const log = String(logTail || '');
+  return HOST_DISK_EXHAUSTED.test(log) && !/failed on the VM/i.test(log);
+}
+
+// Operator-facing recovery copy for the host-disk-exhausted arm. The ENOSPC fires
+// on the operator host before `instances create`, so nothing exists on GCP; the
+// fix is freeing the host's filesystem, which every add needs first.
+function hostDiskExhaustedRecovery() {
+  return (
+    'The operator host ran out of filesystem space (bytes or inodes) before anything was created, so there is no VM to ' +
+    'reconcile or destroy. Prune old date directories under /root/.config/gcloud/logs — gcloud writes one log per ' +
+    'invocation and is the usual cause — and confirm with `df -i` and `df -h`. Every add fails the same way until then.'
+  );
+}
+
+// Operator-facing recovery copy for the generic cloud-up-failed arm. "A partial
+// instance may exist" is only true when the transcript does not show cloud:up
+// deleting what it created; saying it regardless sends the operator hunting for
+// an orphan that is not there.
+function cloudUpFailedRecovery({ cleanTeardown } = {}) {
+  return cleanTeardown
+    ? 'cloud:up failed during provisioning and deleted the VM it created, so nothing was left behind. Read the transcript for the cause before re-adding.'
+    : 'cloud:up failed during provisioning, so a partial instance may exist. Reconcile or destroy this VM before re-adding.';
+}
+
+module.exports = {
+  stockoutCapacityPhrase,
+  missingBranchRefFacts,
+  missingBranchRefRecovery,
+  wedgedInsertFacts,
+  wedgedInsertRecovery,
+  iapNotReadyFacts,
+  iapNotReadyRecovery,
+  streamedLibraryMissingFacts,
+  cleanTeardownReported,
+  streamedLibraryMissingRecovery,
+  hostDiskExhausted,
+  hostDiskExhaustedRecovery,
+  cloudUpFailedRecovery,
+};