@codeyam-editor/codeyam-editor 0.1.12 → 0.1.13-staging.g1b6c798

package/npm/cloud.js

@@ -114,7 +114,7 @@ const { addToRoster, readRoster } = require("./fleet-roster");
 // Tells a docker container-name conflict (a concurrent run already started this
 // container) apart from a real `docker compose up -d` failure, so the abort path
 // can converge instead of escalating to instance deletion.
-const { classifyComposeUpFailure } = require("./compose-conflict");
+const { classifyComposeUpFailure, isGcloudTransportFailure } = require("./compose-conflict");
 // The two streamed bootstrap scripts, named once for this file's STREAMED_SCRIPTS
 // and for the advance's provisioning pre-flight, which checks the same set.
 const { BOOTSTRAP_SCRIPT, PROJECT_BOOTSTRAP_SCRIPT } = require("./provisioning-contract");
@@ -881,8 +881,8 @@ function parseEditorComposeState(stdout) {
 }
 
 /** Decide whether `cloud:up`'s abort handler may delete the instance it is
- *  unwinding. Pure `{ createdThisRun, containerUp, rostered, probeInconclusive }
- *  → 'delete' | 'keep'`, matching the `reapVerdict` / `reattachVerdict` /
+ *  unwinding. Pure `{ createdThisRun, containerUp, rostered, probeInconclusive,
+ *  remoteCommandNeverRan } → 'delete' | 'keep'`, matching the `reapVerdict` / `reattachVerdict` /
  *  `reconcileRosteredVm` pattern in npm/fleet-cleanup.js, so the whole matrix is
  *  exhaustively testable without a VM.
  *
@@ -904,9 +904,20 @@ function parseEditorComposeState(stdout) {
  *    SAFE: a failing probe is how we reach the abort handler in the first place,
  *    so it can never count as evidence the VM is dead. An orphan costs one
  *    `npm run cloud:destroy`; a wrong delete costs a 15-40 minute rebuild plus
- *    any unpushed work on the VM. */
-function abortTeardownDecision({ createdThisRun, containerUp, rostered, probeInconclusive } = {}) {
+ *    any unpushed work on the VM.
+ *  - `remoteCommandNeverRan` — the error that brought us here was a transport
+ *    failure: the gcloud client never reached the VM (an OS Login quota 429 on
+ *    2026-10-01), so the step it was running never executed. Same class as
+ *    `probeInconclusive` — absence of evidence — and it must fail SAFE the same
+ *    way. On that date the probes truthfully reported "container down, not
+ *    rostered", but only BECAUSE compose-up never ran; deleting on that reading
+ *    destroyed VM-8 after ~19 minutes of finished build. A run that never managed
+ *    to execute its step has learned nothing about the VM, so it has not earned
+ *    the right to delete one. Defense in depth: the compose-up retry should keep
+ *    this from being reached at all. */
+function abortTeardownDecision({ createdThisRun, containerUp, rostered, probeInconclusive, remoteCommandNeverRan } = {}) {
   if (!createdThisRun) return "keep";
+  if (remoteCommandNeverRan) return "keep";
   if (probeInconclusive) return "keep";
   if (containerUp) return "keep";
   if (rostered) return "keep";
@@ -1268,13 +1279,24 @@ function provisionFailureMessage(verb, instanceName, result) {
   );
 }
 
-// A bounded, pure classifier that keys on status === 255 (gcloud-ssh's exit
-// for an ssh failure) and a signature match against known transient strings.
+// A bounded, pure classifier with two independent branches:
+//  - status === 255 (gcloud-ssh's exit for an ssh failure) plus a signature
+//    match against known transient SSH-transport strings;
+//  - a gcloud CLIENT-SIDE API failure (an OS Login quota 429), at any status.
+//    gcloud does not exit 255 for these, so the status code is evidence for the
+//    first branch, never a precondition for reading the text — gating on it is
+//    what kept the 2026-10-01 VM-8 quota 429 out of the retry path.
 function isTransientSshFailure(result) {
-  if (!result || result.status !== 255) {
+  if (!result || result.status === 0 || result.status == null) {
     return false;
   }
   const output = String(result.stderr || "") + String(result.stdout || "");
+  if (isGcloudTransportFailure(output)) {
+    return true;
+  }
+  if (result.status !== 255) {
+    return false;
+  }
   const transientRegex = /kex_exchange_identification|Connection closed|failed to connect to backend|Failed to connect to port 22|Connection refused|Connection reset|Connection timed out|4003|4047/i;
   return transientRegex.test(output);
 }
@@ -2471,16 +2493,34 @@ async function ensureEditorContainerUp(cfg) {
   // stream and capture, so the operator's view is preserved by echoing the
   // captured streams immediately below instead of live. Do NOT revert this to
   // "inherit"; classification depends on it.
-  const up = runGcloud(
-    buildGcloudArgs("ssh-probe", cfg, {
-      command: `cd ~/projects/${repoName} && ${composeCmd} up -d`,
-    }),
-    { stdio: ["ignore", "pipe", "pipe"] },
+  //
+  // Through the retry wrapper: a transport failure (an OS Login quota 429 on
+  // 2026-10-01) means `up -d` never ran, so the honest response is to try again.
+  // Retrying is safe without a did-it-run probe — `up -d` is idempotent here, and
+  // a retry that races a container into existence lands on the name-conflict
+  // verdict below, which converges.
+  const up = await runGcloudSshWithRetry(
+    "ssh-probe",
+    cfg,
+    { command: `cd ~/projects/${repoName} && ${composeCmd} up -d` },
+    { stdio: ["ignore", "pipe", "pipe"], label: "docker compose up -d" },
   );
   if (up.stdout) process.stderr.write(up.stdout);
   if (up.stderr) process.stderr.write(up.stderr);
   if (up.status !== 0) {
     const verdict = classifyComposeUpFailure({ status: up.status, stderr: `${up.stderr || ""}\n${up.stdout || ""}` });
+    if (verdict === "transport-error") {
+      // Retries exhausted and gcloud still never reached the VM. Say so, and mark
+      // the error so the abort handler leaves the VM alone: nothing about the VM
+      // has been learned, so it cannot be judged broken.
+      const err = new Error(
+        "could not reach the VM to run docker compose up -d: gcloud failed before connecting " +
+          "(an OS Login API quota 429 — too many concurrent fleet ssh calls). The remote command " +
+          "never ran; the VM itself was not shown to be broken.",
+      );
+      err.remoteCommandNeverRan = true;
+      throw err;
+    }
     if (verdict !== "name-conflict") {
       throw new Error("docker compose up -d failed on the VM");
     }
@@ -4773,12 +4813,18 @@ async function cmdUp(args, projectDir = rootDir) {
     // VM is in right now, then let the pure decision rule — see
     // abortTeardownDecision's doc comment for why each input is load-bearing.
     const state = probeAbortTeardownState(cfg, createdThisRun);
-    const verdict = abortTeardownDecision({ createdThisRun, ...state });
+    // An error that never reached the VM proves nothing about it — see the
+    // `remoteCommandNeverRan` paragraph of abortTeardownDecision.
+    const remoteCommandNeverRan =
+      Boolean(err && err.remoteCommandNeverRan) || isGcloudTransportFailure(err && err.message);
+    const verdict = abortTeardownDecision({ createdThisRun, ...state, remoteCommandNeverRan });
     if (createdThisRun && verdict === "keep") {
       // Deliberately loud and specific. The old blanket "tearing down the VM
       // cloud:up created this run (no orphan left behind)" was actively
       // misleading here — it asserted no orphan while deleting a working VM.
-      const why = state.probeInconclusive
+      const why = remoteCommandNeverRan
+        ? "the failing step never reached the VM (a gcloud transport/quota error), so nothing shows the VM is broken"
+        : state.probeInconclusive
         ? "its liveness could not be confirmed (probe failed), and an unconfirmed VM is never deleted"
         : state.containerUp
           ? "its editor container is RUNNING — another run brought it up"

package/npm/compose-conflict.js

@@ -31,12 +31,44 @@
 // whole captured stream rather than line- or anchor-based.
 const NAME_CONFLICT_FRAGMENTS = [/\bConflict\./i, /container name/i, /is already in use/i];
 
+// The stable fragments of a gcloud CLIENT-SIDE API failure — gcloud's own HTTP
+// client failed before any ssh was attempted, so the remote command never ran.
+// Observed verbatim in the 2026-10-01 VM-8 add log, when an OS Login
+// loginProfile read hit the `ReadRequestsPerMinutePerUser` quota (limit 60):
+//
+//   ERROR: (gcloud.compute.ssh) HttpError accessing <https://oslogin.googleapis.com/…
+//   /loginProfile?…>: response: <{'status': 429}>, content <{"error": {"code": 429,
+//   "message": "Quota exceeded for quota metric …", "status": "RESOURCE_EXHAUSTED",
+//   "details": [{"reason": "RATE_LIMIT_EXCEEDED"}]}}
+//
+// Same discipline as the conflict fragments, for the same reason — Google
+// rephrases these: require gcloud's `HttpError accessing` wrapper AND at least
+// one quota token, never a single token, so an app-level message that merely
+// contains `429` is not swept in.
+const GCLOUD_HTTP_ERROR_FRAGMENT = /HttpError accessing/;
+const GCLOUD_QUOTA_FRAGMENTS = [/'status':\s*429\b/, /RESOURCE_EXHAUSTED/, /RATE_LIMIT_EXCEEDED/];
+
+/** True when `text` carries a gcloud client-side API quota failure, i.e. the
+ *  gcloud client never reached the VM. Shared by {@link classifyComposeUpFailure}
+ *  and `cloud.js`'s transient-ssh predicate and abort handler, so the three
+ *  cannot drift apart on what counts as "never reached the VM". */
+function isGcloudTransportFailure(text) {
+  const s = String(text == null ? '' : text);
+  return GCLOUD_HTTP_ERROR_FRAGMENT.test(s) && GCLOUD_QUOTA_FRAGMENTS.some((re) => re.test(s));
+}
+
 /** Classify the outcome of `docker compose up -d`.
  *
  *  Returns one of:
  *  - `'ok'` — the command succeeded; there is no failure to classify. Guards
  *    against a caller that classifies unconditionally, so a zero status can
  *    never be read as a conflict.
+ *  - `'transport-error'` — the gcloud ssh client never reached the VM (an OS
+ *    Login quota 429 on 2026-10-01), so `up -d` did not run and nothing has been
+ *    learned about the box. Reading this as `'failed'` is what drove the abort
+ *    handler into deleting VM-8 after it had already built its editor image.
+ *    The caller retries; it must never treat this as a provisioning failure.
+ *    Checked first so a transport failure can never be read as a daemon message.
  *  - `'name-conflict'` — the daemon refused because a container of that name
  *    already exists, i.e. a concurrent run already started it. The caller
  *    re-probes for a running container and converges instead of throwing.
@@ -50,10 +82,11 @@ const NAME_CONFLICT_FRAGMENTS = [/\bConflict\./i, /container name/i, /is already
 function classifyComposeUpFailure({ status, stderr } = {}) {
   if (status === 0) return 'ok';
   const text = String(stderr == null ? '' : stderr);
+  if (isGcloudTransportFailure(text)) return 'transport-error';
   if (NAME_CONFLICT_FRAGMENTS.every((re) => re.test(text))) {
     return 'name-conflict';
   }
   return 'failed';
 }
 
-module.exports = { classifyComposeUpFailure };
+module.exports = { classifyComposeUpFailure, isGcloudTransportFailure };

package/npm/fleet-add-failure.js

@@ -99,6 +99,15 @@ function lastNonEmptyLines(text, maxLines = 50) {
   return lines.join('\n');
 }
 
+// How many trailing lines the CLASSIFIER reads — deliberately wider than the
+// 50-line tail the card displays. A verbose error body can crowd the signature
+// that explains it out of a narrow window: VM-8's 2026-10-01 add died on an
+// OS Login 429 whose JSON error body alone ran to 35 lines, pushing cloud:up's
+// own teardown line to the edge of the 50-line tail. Reading more costs nothing
+// and has no false-negative mode, unlike filtering "noisy" lines, which would
+// have to know every error shape in advance.
+const CLASSIFY_TAIL_LINES = 200;
+
 // True iff the provision transcript reached the early-roster marker (`CY_ROSTER
 // <n>`) — i.e. the instance exists and the container gate passed, so the VM is
 // rostered. Reuses parseRosterMarkers (npm/fleet-early-roster.js) so the
@@ -139,14 +148,17 @@ const CALM_ADD_CATEGORIES = new Set(['converging', 'verifying', 'access-pending'
 //          answering — the box exists but never came up.
 //        - `wiringDeadlineBlown`       the wire-add-tail gave up (the failed
 //          marker is the tail's own honest deadline signal).
+//        - `teardownReported`          the transcript carries cloud:up's own
+//          clean-teardown report (cleanTeardownReported): it deleted the VM it
+//          created, so the instance is gone by the provisioner's own account.
 //
 // Every fact is TRI-STATE on purpose: `undefined`/`null` means "not probed" and
 // corroborates nothing. Only an explicit `false` counts, so an unprobed VM can
 // never be escalated to red by absence of evidence.
-function addFailureCorroborated({ exitCode, instanceAlive, containerHealthy, wiringDeadlineBlown } = {}) {
+function addFailureCorroborated({ exitCode, instanceAlive, containerHealthy, wiringDeadlineBlown, teardownReported } = {}) {
   const code = exitCode == null ? NaN : Number(exitCode);
   if (!Number.isInteger(code) || code === 0) return false;
-  return instanceAlive === false || containerHealthy === false || !!wiringDeadlineBlown;
+  return instanceAlive === false || containerHealthy === false || !!wiringDeadlineBlown || !!teardownReported;
 }
 
 // Resolve the two independent health signals into the single tri-state
@@ -697,6 +709,26 @@ function classifyAddFailure({ logTail, exitCode, instanceAlive, containerHealthy
       cloudUpFailedRecovery({ cleanTeardown: cleanTeardownReported(log) }));
   }
 
+  // 11b. torn-down — cloud:up reported, in its own words, that it deleted the VM it
+  //    created this run, but no named cause above matched. That report is positive
+  //    proof of death — the standard addFailureCorroborated demands — so the outcome
+  //    is PROVEN, not unconfirmed. VM-8, 2026-10-01: an OS Login 429 killed the
+  //    container step, cloud:up tore the instance down, and the transcript said so
+  //    plainly; with no arm reading that line as evidence it fell to the `verifying`
+  //    residue, whose calm hold could never release for a VM that never rostered,
+  //    and the card spun "Add outcome unconfirmed" for 49 minutes and counting.
+  //
+  //    Terminal and NOT retryable: nothing is left to reconcile or destroy, but the
+  //    cause may still be present, so re-adding is the operator's call, not a
+  //    one-click action. Tested after every named-cause arm so a specific cause
+  //    always wins and keeps its actionable recovery (cloud-up-failed already reads
+  //    the same teardown fact into its copy), and BEFORE interrupted / converging /
+  //    verifying, none of which may describe a VM the provisioner reports deleted.
+  if (cleanTeardownReported(log)) {
+    return buildFailureVerdict('torn-down', 'Provisioning failed — VM deleted',
+      cloudUpFailedRecovery({ cleanTeardown: true }));
+  }
+
   // 12. interrupted — a signal kill (OOM 137, SIGTERM 143, …) or an explicit
   //    Killed / out-of-memory marker. The real VM state is unknown. Not retryable.
   if (fc.isSignalKillExit(Number(exitCode)) || /\bKilled\b/.test(log) || /Out of memory/i.test(log)) {
@@ -766,7 +798,8 @@ function classifyAddFailure({ logTail, exitCode, instanceAlive, containerHealthy
   //        addFailureCorroborated). The box is provably dead or provably never
   //        came up, so the loud, terminal escalation is warranted and unchanged.
   //        Still never invites an unproven retry.
-  if (addFailureCorroborated({ exitCode, instanceAlive, containerHealthy, wiringDeadlineBlown })) {
+  if (addFailureCorroborated({ exitCode, instanceAlive, containerHealthy, wiringDeadlineBlown,
+                               teardownReported: cleanTeardownReported(log) })) {
     return buildFailureVerdict('unknown', 'Add failed',
       'The add failed for an unclassified reason and the VM is confirmed dead or never came up — inspect the transcript below, then reconcile or destroy this VM before re-adding.');
   }
@@ -818,8 +851,56 @@ function canRetryAdd({ n, job } = {}) {
 // escalation, which this predicate narrows but never softens. Kept pure (like
 // canRetryAdd / canClearBadge) so the badge decision is unit-tested without a
 // live dashboard.
-function holdConvergingBadge({ category, wiringDeadlineBlown } = {}) {
-  return CALM_ADD_CATEGORIES.has(category) && !wiringDeadlineBlown;
+//
+// The hold also has an ABSOLUTE AGE CEILING, because `wiringDeadlineBlown` alone is
+// not a reachable release for every add: the failed marker is dropped by the
+// add's wire tail, and an add that never rostered never had one. VM-8
+// (2026-10-01) sat in `verifying` for 49 minutes with no possible exit. So:
+//
+//   ageMs     ms since the add settled (job.finishedAt). Past the ceiling the hold
+//             releases. Missing / non-finite HOLDS — an unknown age may never
+//             escalate a card on its own, the same tri-state rule
+//             addFailureCorroborated applies to every fact.
+//   rostered  is the VM in the roster? An explicit `false` means the add never
+//             reached CY_ROSTER, so there is no ingress convergence left to wait
+//             for and the shorter ceiling applies. Unknown takes the longer one.
+//
+// `wiringDeadlineBlown` stays the fast path: a genuinely failed wiring still
+// releases immediately rather than waiting out the clock.
+const CALM_HOLD_CEILING_MS = 30 * 60 * 1000;          // well past the 8-minute wiring deadline plus LB propagation
+const NEVER_ROSTERED_HOLD_CEILING_MS = 10 * 60 * 1000; // no wiring to wait for — only a late transcript flush
+
+function holdConvergingBadge({ category, wiringDeadlineBlown, ageMs, rostered } = {}) {
+  if (!CALM_ADD_CATEGORIES.has(category) || wiringDeadlineBlown) return false;
+  if (typeof ageMs !== 'number' || !Number.isFinite(ageMs)) return true;
+  const ceiling = rostered === false ? NEVER_ROSTERED_HOLD_CEILING_MS : CALM_HOLD_CEILING_MS;
+  return ageMs < ceiling;
+}
+
+// Age in ms of a settled job, from its `finishedAt` ISO stamp, for
+// holdConvergingBadge's ceiling. Returns undefined when the stamp is missing or
+// unparseable, which holds — never a fabricated age.
+function settledAgeMs(job, nowMs = Date.now()) {
+  const t = job && job.finishedAt ? Date.parse(job.finishedAt) : NaN;
+  return Number.isFinite(t) ? Math.max(0, nowMs - t) : undefined;
+}
+
+// The `instanceAlive` fact for addFailureCorroborated, read from a RECORDED GCP
+// existence verdict ({ exists, at } — what the vanish reconcile's describe
+// returned, and when) rather than a fresh probe, so the failure path stays
+// synchronous. Two rules keep it from escalating a live VM:
+//
+//   - only a boolean verdict counts; the describe's inconclusive `null`, or no
+//     record at all, returns null (corroborates nothing);
+//   - a verdict taken BEFORE `sinceIso` (the add's startedAt) describes a
+//     previous instance under the same number, not this add's, so it returns
+//     null too. An unparseable or missing `sinceIso` cannot date the record
+//     against anything, and the verdict is used as-is.
+function instanceAliveSince(record, sinceIso) {
+  if (!record || typeof record.exists !== 'boolean') return null;
+  const since = sinceIso ? Date.parse(sinceIso) : NaN;
+  if (Number.isFinite(since) && !(record.at >= since)) return null;
+  return record.exists;
 }
 
 // Refusal copy for a Retry on a card whose branch was deleted from origin.
@@ -884,4 +965,16 @@ function streamedLibraryRetryMessage(n, recovery) {
   return `VM-${n} cannot be retried until the code is fixed: ${recovery || 'a streamed bootstrap library was never shipped to the VM.'}`;
 }
 
-module.exports = { classifyAddFailure, extractDriftFiles, lastNonEmptyLines, canRetryAdd, transcriptRostered, holdConvergingBadge, addFailureCorroborated, containerHealthWitness, stockoutCapacityPhrase, missingBranchRefFacts, missingBranchRefRecovery, deadRefRetryMessage, wedgedInsertFacts, wedgedInsertRecovery, wedgedRetryMessage, iapNotReadyFacts, iapNotReadyRecovery, streamedLibraryMissingFacts, cleanTeardownReported, streamedLibraryMissingRecovery, cloudUpFailedRecovery, streamedLibraryRetryMessage, CALM_ADD_CATEGORIES, ADD_FAILURE_RETRYABLE };
+// Refusal copy for a one-click "Retry add" on a `torn-down` card. The generic
+// refusal's "may have left a half-provisioned instance — reconcile or destroy it"
+// is false here: cloud:up reported deleting the VM it created. The retry is
+// refused only because the unclassified cause may still be present, so the copy
+// says nothing is left behind and hands the re-add to the operator. Pure; no I/O.
+function tornDownRetryMessage(n) {
+  return (
+    `VM-${n} was deleted by cloud:up when its add failed, so there is nothing to reconcile or destroy. ` +
+    'Read the transcript for the cause, then use Add VM again once it is resolved.'
+  );
+}
+
+module.exports = { classifyAddFailure, extractDriftFiles, lastNonEmptyLines, canRetryAdd, transcriptRostered, holdConvergingBadge, addFailureCorroborated, containerHealthWitness, stockoutCapacityPhrase, missingBranchRefFacts, missingBranchRefRecovery, deadRefRetryMessage, wedgedInsertFacts, wedgedInsertRecovery, wedgedRetryMessage, iapNotReadyFacts, iapNotReadyRecovery, streamedLibraryMissingFacts, cleanTeardownReported, streamedLibraryMissingRecovery, cloudUpFailedRecovery, streamedLibraryRetryMessage, tornDownRetryMessage, settledAgeMs, instanceAliveSince, CALM_ADD_CATEGORIES, ADD_FAILURE_RETRYABLE, CLASSIFY_TAIL_LINES };

package/npm/fleet-api-budget.js

@@ -0,0 +1,105 @@
+'use strict';
+
+// npm/fleet-api-budget.js — the RATE dimension of heavy-job admission: how many
+// Google API calls per minute the fleet's in-flight work is projected to spend,
+// and whether admitting one more add would push that past the per-user budget.
+//
+// Background (fleet--a-batch-of-adds-blows-a-per-user-google-api-quota): every
+// `gcloud compute ssh` against an OS-Login VM spends at least one
+// `oslogin.googleapis.com` loginProfile read; the limit is 60 per minute per
+// USER, and all fleet automation runs as one service account — so the whole
+// fleet shares one 60/min budget. On 2026-10-01 it ran out while the job cap of
+// 4 held the entire time: a cap counts jobs, and a quota is a rate. A 429 landed
+// mid-provision and tore down an add that had already built its editor image.
+//
+// This module is the accountant for that rate. It is a PROJECTION from known
+// per-add shapes, not a measured counter: counting real calls would mean
+// intercepting every gcloud invocation across detached shells — the same
+// invisibility that produced this class of bug twice. Every function is pure.
+
+/** Defaults for the API-rate dimension of add admission.
+ *
+ *  - `budgetPerMin` — the real limit: `oslogin.googleapis.com` read requests per
+ *    minute per user.
+ *  - `reservedPerMin` — headroom kept back for traffic admission cannot see: the
+ *    dashboard's own monitoring gathers (bounded in concurrency by hostOpLimiter,
+ *    not in rate) and the tunnel watchdog.
+ *  - `perAddCallsPerMin` — what one in-flight add is charged. Deliberately its
+ *    PEAK, not its average: the fast first minute of the container gate (6 reads)
+ *    overlapping cloud:up's own ssh steps or the post-roster burst. Charging the
+ *    peak is wrong in the safe direction — it queues a little early rather than
+ *    429ing a provision.
+ *
+ *  At these defaults (60 - 20) / 10 = 4 concurrent adds, the same as the job cap,
+ *  so an operator who raises MAX_JOB_CONCURRENCY meets this budget instead of the
+ *  quota. */
+const API_BUDGET_DEFAULTS = Object.freeze({ budgetPerMin: 60, reservedPerMin: 20, perAddCallsPerMin: 10 });
+
+// A positive finite number, else 0 — so garbage degrades toward admitting work,
+// never toward a phantom load that would freeze admission forever.
+function nonNegative(value) {
+  const n = Number(value);
+  return Number.isFinite(n) && n > 0 ? n : 0;
+}
+
+/** Project the fleet's Google API call rate (calls/min) from the heavy work in
+ *  flight. `inFlight` is `heavyChargeCount`'s answer — running adds PLUS detached
+ *  wiring tails — so this dimension inherits the tail visibility the job cap
+ *  already has instead of re-deriving it. */
+function projectApiCallsPerMin({
+  inFlight = 0,
+  perAddCallsPerMin = API_BUDGET_DEFAULTS.perAddCallsPerMin,
+  reservedPerMin = API_BUDGET_DEFAULTS.reservedPerMin,
+} = {}) {
+  return nonNegative(reservedPerMin) + nonNegative(inFlight) * nonNegative(perAddCallsPerMin);
+}
+
+/** True when admitting one more add (costing `apiCallsPerAdd`) would project past
+ *  the budget. False whenever the budget or the projection is absent or
+ *  non-numeric, or the budget is non-positive — an unset or misconfigured budget
+ *  degrades to "no rate gate", the same rule `decideAddSlot` applies to a bad
+ *  cap. */
+function wouldExceedApiBudget({
+  apiCallBudgetPerMin,
+  projectedApiCallsPerMin,
+  apiCallsPerAdd = API_BUDGET_DEFAULTS.perAddCallsPerMin,
+} = {}) {
+  if (apiCallBudgetPerMin == null || projectedApiCallsPerMin == null) return false;
+  const budget = Number(apiCallBudgetPerMin);
+  const projected = Number(projectedApiCallsPerMin);
+  if (!Number.isFinite(budget) || budget <= 0 || !Number.isFinite(projected)) return false;
+  return projected + nonNegative(apiCallsPerAdd) > budget;
+}
+
+/** Whether the RATE dimension, not the job cap, is what holds the next add back
+ *  — so a queue paced by the API budget can say so instead of reading as an
+ *  unexplained stall. Takes the same inputs as `decideAddSlot`. A non-positive
+ *  cap means "no job cap", so the slot is free and only the rate can hold. */
+function heldByApiBudget(inputs = {}) {
+  const cap = Number(inputs.maxConcurrency);
+  const capFree = !(cap > 0) || Number(inputs.activeCount) < cap;
+  return capFree && wouldExceedApiBudget(inputs);
+}
+
+/** The text Google returns when a per-user API quota is spent. `RESOURCE_EXHAUSTED`
+ *  is the gRPC status name, `Quota exceeded` / `rateLimitExceeded` the REST
+ *  wording. A bare "429" is deliberately NOT matched alone, since port numbers
+ *  and VM indexes contain it — only an HTTP-status shape of it is. */
+const API_QUOTA_PATTERN = /RESOURCE_EXHAUSTED|Quota exceeded|rateLimitExceeded|RATE_LIMIT_EXCEEDED|HTTP(?:Error)? 429|status(?:Code)?[ =:]+429/i;
+
+/** First line of `text` naming a Google API quota exhaustion, trimmed and capped
+ *  at 300 chars, or null. Matched against job output so a 429 reads as a named
+ *  condition instead of an "unconfirmed" card with the cause in a transcript. */
+function detectApiQuotaExhaustion(text) {
+  if (typeof text !== 'string' || !text) return null;
+  const line = text.split(/\r?\n/).find((l) => API_QUOTA_PATTERN.test(l));
+  return line ? line.trim().slice(0, 300) : null;
+}
+
+module.exports = {
+  API_BUDGET_DEFAULTS,
+  projectApiCallsPerMin,
+  wouldExceedApiBudget,
+  heldByApiBudget,
+  detectApiQuotaExhaustion,
+};

package/npm/fleet-job-badge.js

@@ -117,10 +117,13 @@ function canClearBadge({ rostered, job } = {}) {
 //
 // Only the LABEL changes. `job.state` stays 'error', so canRetryAdd,
 // canClearBadge and shouldReconcileFailedBadge keep gating on it unchanged.
-function jobBadgeLabel({ job, wiringDeadlineBlown } = {}) {
+//
+// `ageMs` / `rostered` are holdConvergingBadge's age-ceiling inputs, passed
+// through unchanged for the same reason.
+function jobBadgeLabel({ job, wiringDeadlineBlown, ageMs, rostered } = {}) {
   if (!job) return '';
   if (job.action === 'add' && job.state === 'error') {
-    return faf.holdConvergingBadge({ category: job.category, wiringDeadlineBlown })
+    return faf.holdConvergingBadge({ category: job.category, wiringDeadlineBlown, ageMs, rostered })
       ? 'add unconfirmed'
       : 'add failed';
   }

package/npm/launch-config.js

@@ -12,6 +12,7 @@ const { CANONICAL_EDITOR_REPO, EDITOR_VARIANTS, repoBasename, AGENT_ENV_VARS } =
 const { normalizeVmSlug } = require("./vm-urls");
 const { normalizeAccessList } = require("./fleet-vm-access");
 const { normalizeRepoUrl } = require("./fleet-env");
+const { API_BUDGET_DEFAULTS, wouldExceedApiBudget } = require("./fleet-api-budget");
 const { posix: posixPath } = require("node:path");
 
 /** The build agents a VM's editor terminal can run, as the keys of cloud.js's
@@ -396,11 +397,26 @@ function classifyContainerWait({ everSeenUp = false, instanceGone = false, elaps
  *  queue + posting a `queued` job entry on "queue", or starting addVm on
  *  "start". Decoupling the decision from the I/O makes it unit-testable
  *  against arbitrary (cap, active) tuples — the test signal the OOM bug
- *  needed and the queue never had. */
-function decideAddSlot({ maxConcurrency, activeCount = 0 } = {}) {
+ *  needed and the queue never had.
+ *
+ *  A second, optional dimension is a RATE, not a count: the per-user Google API
+ *  budget (oslogin loginProfile reads, 60/min for the one service account the
+ *  whole fleet runs as). When `apiCallBudgetPerMin` AND `projectedApiCallsPerMin`
+ *  are both supplied, an add that would push the projection past the budget by
+ *  its own `apiCallsPerAdd` is queued even with job slots free — the 2026-10-01
+ *  shape, where a cap of 4 held while the fleet 429'd a provision mid-flight.
+ *  Omit either and the decision is exactly the count-only one above. */
+function decideAddSlot({
+  maxConcurrency,
+  activeCount = 0,
+  apiCallBudgetPerMin,
+  projectedApiCallsPerMin,
+  apiCallsPerAdd = API_BUDGET_DEFAULTS.perAddCallsPerMin,
+} = {}) {
   const cap = Number(maxConcurrency);
-  if (!Number.isFinite(cap) || cap <= 0) return "start";
-  return Number(activeCount) < cap ? "start" : "queue";
+  const underCap = !Number.isFinite(cap) || cap <= 0 || Number(activeCount) < cap;
+  if (!underCap) return "queue";
+  return wouldExceedApiBudget({ apiCallBudgetPerMin, projectedApiCallsPerMin, apiCallsPerAdd }) ? "queue" : "start";
 }
 
 /** How many concurrency slots the fleet's heavy work is ACTUALLY occupying —
@@ -1479,6 +1495,109 @@ function localVmReapCmd({ port, previewPort } = {}) {
   ].join("; ");
 }
 
+/** Poll cadence for the Add's container gate, as `[{ untilSec, everySec }, …]`
+ *  bands over elapsed time; the last band (no `untilSec`) runs to the deadline.
+ *  A cold image build takes 15-25 min, so the old flat 10s cadence asked "is it
+ *  up yet?" far finer than the event it waits for — and each ask is a
+ *  `gcloud compute ssh`, i.e. an oslogin read against the fleet-wide 60/min
+ *  per-user quota. Fast for the first minute (a warm image is up in seconds),
+ *  then widening: ~6 reads in minute one, 2/min to minute five, 1/min after —
+ *  roughly 32 reads over a full 28-min wait instead of 168. */
+const CONTAINER_GATE_SCHEDULE = Object.freeze([
+  Object.freeze({ untilSec: 60, everySec: 10 }),
+  Object.freeze({ untilSec: 300, everySec: 30 }),
+  Object.freeze({ everySec: 60 }),
+]);
+
+/** Build the Add script's container gate: wait for the editor container to report
+ *  `Up`, up to `deadlineSec` of REAL elapsed time (default 1680s, the historical
+ *  ~28 min), mirroring `classifyContainerWait`:
+ *   - any single `Up` is terminal success (`st=1`);
+ *   - a clean cloud:up exit is the normal `--no-tunnel` success path — keep
+ *     probing; a NON-ZERO exit gets one final container re-check and otherwise
+ *     fails fast with the cloud:up log tail instead of probing a dead provision;
+ *   - an empty/errored probe (transient gcloud-ssh blip) is retried, never read
+ *     as "not up";
+ *   - past the deadline: a final re-check, then a vanished instance is told
+ *     apart from a build that never came up, and cloud:up is reaped.
+ *
+ *  Only the POLLING COST differs from the inline loop this replaced: probes follow
+ *  `schedule`, and the waits between them are 5s `kill -0` slices — free, no API
+ *  call — so a cloud:up exit (success under `--no-tunnel`, or a failure) is still
+ *  noticed within ~5s rather than a full band interval later. Elapsed time is read
+ *  from the clock, not counted in ticks, so a wider cadence cannot move the
+ *  deadline.
+ *
+ *  Each probe's stderr is kept (it used to be discarded) and scanned for a Google
+ *  API quota refusal; a hit prints one `CY_API_QUOTA` line on the job's stdout so
+ *  the dashboard can name it. `probeCmd` must already be shell-quoted. Pure. */
+function containerGateCmd({
+  gcloud = "gcloud",
+  inst,
+  zone,
+  probeCmd,
+  logPath,
+  deadlineSec = 1680,
+  schedule = CONTAINER_GATE_SCHEDULE,
+  pidVar = "UP_PID",
+} = {}) {
+  const errPath = `${logPath}.gate-err`;
+  const bands = (Array.isArray(schedule) && schedule.length ? schedule : CONTAINER_GATE_SCHEDULE);
+  // The last band is the fallback whatever its `untilSec` says, so the chain
+  // always ends in an `else` and every elapsed value gets a cadence.
+  const cadence = bands.map((b, i) => {
+    if (i === bands.length - 1) return i === 0 ? `  __cy_gate_every=${b.everySec}` : `  else __cy_gate_every=${b.everySec}`;
+    return `  ${i === 0 ? "if" : "elif"} [ "$__cy_gate_elapsed" -lt ${b.untilSec} ]; then __cy_gate_every=${b.everySec}`;
+  });
+  if (bands.length > 1) cadence.push(`  fi`);
+  const minutes = Math.round(deadlineSec / 60);
+  // cloud:up's own ssh steps can be the call that 429s; its log is private, so on
+  // every failure exit republish the first quota line onto the job's stdout.
+  const quotaFromLog = `grep -m1 -iE 'RESOURCE_EXHAUSTED|Quota exceeded|rateLimitExceeded' ${logPath} 2>/dev/null | sed 's/^/CY_API_QUOTA cloud:up: /' || true`;
+  return [
+    `__cy_gate_start=$(date +%s); st=0; cloud_up_exited=0`,
+    `__cy_gate_probe() {`,
+    `  up=$(${gcloud} compute ssh ${inst} --tunnel-through-iap --zone=${zone} --command=${probeCmd} 2>${errPath} | grep -c Up || true)`,
+    `  __cy_quota=$(grep -m1 -iE 'RESOURCE_EXHAUSTED|Quota exceeded|rateLimitExceeded' ${errPath} 2>/dev/null || true)`,
+    `  if [ -n "$__cy_quota" ]; then echo "CY_API_QUOTA container gate probe for ${inst}: $__cy_quota"; fi`,
+    `}`,
+    `while :; do`,
+    `  __cy_gate_probe`,
+    `  if [ "$up" = "1" ]; then st=1; break; fi`,
+    `  if [ "$cloud_up_exited" = "0" ] && ! kill -0 $${pidVar} 2>/dev/null; then`,
+    `    cloud_up_exited=1; wait $${pidVar} 2>/dev/null; up_rc=$?`,
+    `    if [ "$up_rc" != "0" ]; then`,
+    `      __cy_gate_probe`,
+    `      if [ "$up" != "1" ]; then`,
+    `        echo "cloud:up FAILED (exit $up_rc) before ${inst}'s container came up — provisioning aborted (see ${logPath}):"`,
+    `        tail -20 ${logPath} 2>/dev/null | grep -vE 'WARNING|NumPy|cloud\\.google\\.com/iap' || true`,
+    `        ${quotaFromLog}`,
+    `        exit 1`,
+    `      fi`,
+    `      st=1; break`,
+    `    fi`,
+    `    echo "cloud:up exited cleanly (expected under --no-tunnel once the container is up); continuing to probe ${inst} until it reports Up or the deadline..."`,
+    `  fi`,
+    `  __cy_gate_elapsed=$(( $(date +%s) - __cy_gate_start ))`,
+    `  if [ "$__cy_gate_elapsed" -ge ${deadlineSec} ]; then break; fi`,
+    ...cadence,
+    `  __cy_gate_left=$(( ${deadlineSec} - __cy_gate_elapsed ))`,
+    `  if [ "$__cy_gate_every" -gt "$__cy_gate_left" ]; then __cy_gate_every=$__cy_gate_left; fi`,
+    `  __cy_gate_slept=0`,
+    `  while [ "$__cy_gate_slept" -lt "$__cy_gate_every" ]; do`,
+    `    if [ "$cloud_up_exited" = "0" ] && ! kill -0 $${pidVar} 2>/dev/null; then break; fi`,
+    `    sleep 5; __cy_gate_slept=$((__cy_gate_slept + 5))`,
+    `  done`,
+    `done`,
+    `if [ "$st" != "1" ]; then __cy_gate_probe; [ "$up" = "1" ] && st=1; fi`,
+    `if [ "$st" != "1" ]; then`,
+    `  if ! ${gcloud} compute instances describe ${inst} --zone=${zone} --format='value(name)' >/dev/null 2>&1; then echo "instance ${inst} is gone — provisioning failed before the container started (see ${logPath})"; else echo "container never came up for ${inst} within ~${minutes} min (see ${logPath})"; fi`,
+    `  ${quotaFromLog}`,
+    `  kill $${pidVar} 2>/dev/null; ${cloudUpSweepCmd({ inst })}; exit 1`,
+    `fi`,
+  ].join("\n");
+}
+
 /** Build the shell snippet the Add script runs to AWAIT cloud:up's clean exit at
  *  container-healthy instead of reaping it mid-flight.
  *
@@ -1568,6 +1687,8 @@ module.exports = {
   classifyContainerWait,
   decideAddSlot,
   heavyChargeCount,
+  CONTAINER_GATE_SCHEDULE,
+  containerGateCmd,
   HEAVY_ACTIONS,
   queuedStatusForAction,
   pushHistory,

package/npm/scenario-playwright.js

@@ -98,6 +98,9 @@ function escapeHtmlAttribute(value) {
 // Preview. Callers (via scenario-check.js) pass a concrete color when the
 // UI has detected a background it wants the capture to paint behind the
 // iframe, e.g. `var(--bg-deep)` from the editor shell.
+// The iframe is named `codeyam-preview` (CODEYAM_PREVIEW_FRAME_NAME in
+// ui/src/helpers/previewMockPolicy.ts) so a framed editor page knows it is
+// inside codeyam's own capture, not a hosting IDE's pane.
 function buildIframeHarness(url, { background = "transparent" } = {}) {
   const escapedUrl = escapeHtmlAttribute(url);
   const bg = String(background);
@@ -124,7 +127,7 @@ function buildIframeHarness(url, { background = "transparent" } = {}) {
     </style>
   </head>
   <body>
-    <iframe id="scenario-frame" title="Scenario Preview" src="${escapedUrl}"></iframe>
+    <iframe id="scenario-frame" name="codeyam-preview" title="Scenario Preview" src="${escapedUrl}"></iframe>
   </body>
 </html>`;
 }