@codeyam-editor/codeyam-editor 0.1.9 → 0.1.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/npm/cloud.js +16 -3
- package/npm/fleet-auth-account.js +61 -0
- package/npm/fleet-auth.js +69 -1
- package/npm/fleet-public-url.js +81 -12
- package/npm/fleet-reconcile.js +46 -11
- package/npm/group-bounded-spawn.js +111 -0
- package/npm/launch-config.js +90 -21
- package/npm/orphan-ssh-reaper.js +81 -0
- package/npm/scenario-playwright.js +107 -6
- package/package.json +5 -5
- package/ui/dist/assets/{ContainerInspector-C7I9wctz.js → ContainerInspector-DiN91foX.js} +1 -1
- package/ui/dist/assets/{EditorShell-CE8iG_EM.css → EditorShell-BEIdHnHd.css} +1 -1
- package/ui/dist/assets/{EditorShell-9aYMeOdZ.js → EditorShell-CPV5vhjo.js} +70 -70
- package/ui/dist/assets/ScenarioDataPanel-DHAHyraq.js +18 -0
- package/ui/dist/assets/{ScenarioDataPanelFullPage-DT0TZC4M.js → ScenarioDataPanelFullPage-CHIWip3n.js} +1 -1
- package/ui/dist/assets/ScenarioRenderer-D9a91iwS.js +21 -0
- package/ui/dist/assets/{index-BRoAenc1.css → index--cE0hUJw.css} +1 -1
- package/ui/dist/assets/{index-iMR5siiG.js → index-sW8wcg7v.js} +2 -2
- package/ui/dist/index.html +2 -2
- package/ui/dist/assets/ScenarioDataPanel-CLwkrex0.js +0 -18
- package/ui/dist/assets/ScenarioRenderer-Bsarhcrl.js +0 -21
package/npm/cloud.js
CHANGED
|
@@ -23,6 +23,11 @@ const {
|
|
|
23
23
|
killChildProcess,
|
|
24
24
|
} = require("./utils");
|
|
25
25
|
const { open } = require("./open");
|
|
26
|
+
// Bounded gcloud/ssh spawns run through spawnGroupBoundedSync so a timeout takes
|
|
27
|
+
// gcloud's ssh descendant down with it instead of orphaning it to PID 1; the
|
|
28
|
+
// reaper sweeps sessions already leaked before that existed.
|
|
29
|
+
const { spawnGroupBoundedSync } = require("./group-bounded-spawn");
|
|
30
|
+
const { reapOrphanedFleetSsh } = require("./orphan-ssh-reaper");
|
|
26
31
|
// `FORWARD_ENV_KEYS_ENV` is the env-var name (`CY_FORWARD_ENV_KEYS`) the
|
|
27
32
|
// fleet dashboard sets on `cloud:up`'s child to name the subset of
|
|
28
33
|
// `process.env` keys that came from the operator-supplied launch form.
|
|
@@ -1090,6 +1095,10 @@ function buildGcloudArgs(verb, cfg = {}, extras = {}) {
|
|
|
1090
1095
|
// reference one value. 2 min is generous for any single describe/update/SSH probe
|
|
1091
1096
|
// yet comfortably under the 600000 ms outer reconcile budget.
|
|
1092
1097
|
//
|
|
1098
|
+
// All three spawn through spawnGroupBoundedSync, because a bare spawnSync timeout
|
|
1099
|
+
// signals only the direct child: gcloud died and its `ssh` survived, re-parented
|
|
1100
|
+
// to PID 1 — the budget bounded the wrapper, never the descendant.
|
|
1101
|
+
//
|
|
1093
1102
|
// It is a PROBE budget, and provisioning is exempt — see
|
|
1094
1103
|
// GCLOUD_PROVISION_TIMEOUT_MS below. Do not re-collapse the two: raising this
|
|
1095
1104
|
// constant to accommodate a slow create would un-bound the reconcile sweep,
|
|
@@ -1161,7 +1170,7 @@ function isSpawnTimeout(result) {
|
|
|
1161
1170
|
}
|
|
1162
1171
|
|
|
1163
1172
|
function runGcloud(args, opts = {}) {
|
|
1164
|
-
return
|
|
1173
|
+
return spawnGroupBoundedSync("gcloud", args, {
|
|
1165
1174
|
stdio: opts.stdio || "inherit",
|
|
1166
1175
|
input: opts.input,
|
|
1167
1176
|
encoding: opts.encoding || "utf8",
|
|
@@ -4966,6 +4975,10 @@ async function cmdIngressUp(args, projectDir = rootDir) {
|
|
|
4966
4975
|
async function cmdReconcileIngress(args, projectDir = rootDir) {
|
|
4967
4976
|
ensureGcloud();
|
|
4968
4977
|
parseFlags(args);
|
|
4978
|
+
// Clear ssh sessions earlier timed-out probes leaked before this sweep can add
|
|
4979
|
+
// load of its own. Only PID-1 orphans with the fleet signature — never a live
|
|
4980
|
+
// sweep's in-flight children.
|
|
4981
|
+
reapOrphanedFleetSsh();
|
|
4969
4982
|
const cfg = loadCloudConfig(projectDir) || {};
|
|
4970
4983
|
const domain = resolveIngressDomain(cfg, process.env);
|
|
4971
4984
|
if (!domain) {
|
|
@@ -4996,7 +5009,7 @@ async function cmdReconcileIngress(args, projectDir = rootDir) {
|
|
|
4996
5009
|
// whole sweep. On timeout res.stdout is empty, so parseIngressStatusVerdict
|
|
4997
5010
|
// returns null → an `unknown` verdict → selectIngressReconcileTargets SKIPS
|
|
4998
5011
|
// this VM and the sweep continues to the next one.
|
|
4999
|
-
const res =
|
|
5012
|
+
const res = spawnGroupBoundedSync(inv.command, inv.args, {
|
|
5000
5013
|
env: { ...process.env, ...inv.env },
|
|
5001
5014
|
encoding: "utf8",
|
|
5002
5015
|
timeout: GCLOUD_CALL_TIMEOUT_MS,
|
|
@@ -5057,7 +5070,7 @@ async function cmdReconcileIngress(args, projectDir = rootDir) {
|
|
|
5057
5070
|
// Bound the per-BES converge too: a hung `gcloud backend-services update`
|
|
5058
5071
|
// returns a non-zero (timed-out) result, so it falls into the skip branch
|
|
5059
5072
|
// below and the sweep moves on to the next BES rather than stalling.
|
|
5060
|
-
const res =
|
|
5073
|
+
const res = spawnGroupBoundedSync(inv.command, inv.args, {
|
|
5061
5074
|
encoding: "utf8",
|
|
5062
5075
|
timeout: GCLOUD_CALL_TIMEOUT_MS,
|
|
5063
5076
|
});
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// npm/fleet-auth-account.js — the operator's chosen fleet Claude account
|
|
4
|
+
// (`.codeyam/logs/fleet-auth-account.json`). This is the account a newly added
|
|
5
|
+
// VM authenticates as: the Add path picks a credential donor holding it, and the
|
|
6
|
+
// Add dialog preselects it. Persisted so the choice survives a dashboard restart,
|
|
7
|
+
// and explicit so a background token refresh on some other VM can never re-point
|
|
8
|
+
// it (the drift the derived default in fleet-auth.js is subject to).
|
|
9
|
+
//
|
|
10
|
+
// Shape on disk: { "email": "<account email>", "updatedAt": "<ISO>" }. A seam
|
|
11
|
+
// modeled on fleet-vm-slugs.js: atomic temp-file + rename, tolerant of a
|
|
12
|
+
// missing/malformed file (both read as "never chosen" → null).
|
|
13
|
+
//
|
|
14
|
+
// Env override: `FLEET_AUTH_ACCOUNT_FILE` relocates the file (mirrors
|
|
15
|
+
// fleet-roster.js's `VMS_STATE_FILE`); resolved once at module load.
|
|
16
|
+
|
|
17
|
+
const fs = require('fs');
|
|
18
|
+
const path = require('path');
|
|
19
|
+
|
|
20
|
+
const { rootDir } = require('./utils');
|
|
21
|
+
const { isAccountEmail } = require('./fleet-auth');
|
|
22
|
+
|
|
23
|
+
const FLEET_AUTH_ACCOUNT_FILE = process.env.FLEET_AUTH_ACCOUNT_FILE
|
|
24
|
+
|| path.join(rootDir, '.codeyam', 'logs', 'fleet-auth-account.json');
|
|
25
|
+
|
|
26
|
+
/** The chosen account email, or null when none has been chosen (file missing,
|
|
27
|
+
* malformed, or carrying no email). */
|
|
28
|
+
function readAuthAccount() {
|
|
29
|
+
try {
|
|
30
|
+
const data = JSON.parse(fs.readFileSync(FLEET_AUTH_ACCOUNT_FILE, 'utf8'));
|
|
31
|
+
if (data && typeof data.email === 'string' && data.email.trim()) return data.email.trim();
|
|
32
|
+
} catch { /* missing or malformed */ }
|
|
33
|
+
return null;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/** Persist `email` as the fleet's chosen account, atomically: write a temp file
|
|
37
|
+
* then `rename` it over the target so a concurrent reader never observes a
|
|
38
|
+
* half-written file. Returns the email written. A value that is not a
|
|
39
|
+
* plausible email is rejected (nothing is written) — forgetting the choice is
|
|
40
|
+
* `clearAuthAccount`. */
|
|
41
|
+
function writeAuthAccount(email) {
|
|
42
|
+
const e = typeof email === 'string' ? email.trim() : '';
|
|
43
|
+
if (!isAccountEmail(e)) throw new Error(`not a valid account email: "${e}"`);
|
|
44
|
+
fs.mkdirSync(path.dirname(FLEET_AUTH_ACCOUNT_FILE), { recursive: true });
|
|
45
|
+
const tmp = `${FLEET_AUTH_ACCOUNT_FILE}.${process.pid}.tmp`;
|
|
46
|
+
fs.writeFileSync(tmp, JSON.stringify({ email: e, updatedAt: new Date().toISOString() }));
|
|
47
|
+
fs.renameSync(tmp, FLEET_AUTH_ACCOUNT_FILE); // atomic replace
|
|
48
|
+
return e;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/** Forget the chosen account, so the fleet falls back to the derived default.
|
|
52
|
+
* Removing an absent file is a no-op. */
|
|
53
|
+
function clearAuthAccount() {
|
|
54
|
+
try {
|
|
55
|
+
fs.unlinkSync(FLEET_AUTH_ACCOUNT_FILE);
|
|
56
|
+
} catch (e) {
|
|
57
|
+
if (e.code !== 'ENOENT') throw e;
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
module.exports = { FLEET_AUTH_ACCOUNT_FILE, readAuthAccount, writeAuthAccount, clearAuthAccount };
|
package/npm/fleet-auth.js
CHANGED
|
@@ -108,9 +108,77 @@ function deriveDefaultAuthAccount(vms, lastAuthAt) {
|
|
|
108
108
|
return best ? best.account : null;
|
|
109
109
|
}
|
|
110
110
|
|
|
111
|
+
// Enumerate the Claude accounts the fleet can seed a new VM from: one row per
|
|
112
|
+
// distinct account email held by an authenticated VM, carrying the account
|
|
113
|
+
// object and the VM numbers holding it (so the Add dialog can show
|
|
114
|
+
// "jared@codeyam.com — VM-14"). Pure over the polled VM snapshot — the same
|
|
115
|
+
// array deriveDefaultAuthAccount reads. Emails compare case-insensitively; rows
|
|
116
|
+
// sort by email and each row's VM numbers ascend, so the result is deterministic.
|
|
117
|
+
function listAvailableAccounts(vms) {
|
|
118
|
+
const byEmail = new Map();
|
|
119
|
+
for (const v of vms || []) {
|
|
120
|
+
const pa = v && v.providerAuth;
|
|
121
|
+
const acct = pa && pa.account;
|
|
122
|
+
if (!acct || !acct.email || pa.state !== 'authenticated') continue;
|
|
123
|
+
const key = String(acct.email).toLowerCase();
|
|
124
|
+
const row = byEmail.get(key) || { account: acct, vms: [] };
|
|
125
|
+
if (!row.vms.includes(v.n)) row.vms.push(v.n);
|
|
126
|
+
byEmail.set(key, row);
|
|
127
|
+
}
|
|
128
|
+
return [...byEmail.values()]
|
|
129
|
+
.map((r) => ({ account: r.account, vms: [...r.vms].sort((a, b) => a - b) }))
|
|
130
|
+
.sort((a, b) => String(a.account.email).localeCompare(String(b.account.email)));
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
// Pick the credential donor for a chosen account: the lowest-numbered VM in
|
|
134
|
+
// `reachableDonors` whose authenticated account is `email`, or null when no
|
|
135
|
+
// reachable VM holds it (or no email was asked for). `vms` is the per-VM poll
|
|
136
|
+
// state keyed by VM number (`state.vms`). Pure.
|
|
137
|
+
function selectDonorForAccount(reachableDonors, vms, email) {
|
|
138
|
+
if (!email) return null;
|
|
139
|
+
const want = String(email).toLowerCase();
|
|
140
|
+
for (const n of [...(reachableDonors || [])].sort((a, b) => a - b)) {
|
|
141
|
+
const pa = vms && vms[n] && vms[n].providerAuth;
|
|
142
|
+
const got = pa && pa.state === 'authenticated' && pa.account && pa.account.email;
|
|
143
|
+
if (got && String(got).toLowerCase() === want) return n;
|
|
144
|
+
}
|
|
145
|
+
return null;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
// A plausible Claude account email: one "@", a dotted domain, no whitespace or
|
|
149
|
+
// shell-significant characters. The shared guard for every place an operator
|
|
150
|
+
// can name an account (the Add form's per-launch override, the set-default
|
|
151
|
+
// action, the persisted sidecar). Deliberately NOT checked against the fleet,
|
|
152
|
+
// whose roster can change between naming an account and provisioning with it.
|
|
153
|
+
const ACCOUNT_EMAIL_RE = /^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}$/;
|
|
154
|
+
function isAccountEmail(s) {
|
|
155
|
+
return typeof s === 'string' && ACCOUNT_EMAIL_RE.test(s.trim());
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
// Resolve the fleet's chosen Claude account email: the operator's persisted
|
|
159
|
+
// choice (`saved`, read from npm/fleet-auth-account.js by the caller) when it is
|
|
160
|
+
// a plausible email, else the derived default — the most recently auth-updated
|
|
161
|
+
// VM's account — so a fleet that has never chosen behaves as it did before the
|
|
162
|
+
// picker existed. Pure. Returns an email string, or null when neither exists.
|
|
163
|
+
function resolveChosenAccount(saved, vms, lastAuthAt) {
|
|
164
|
+
if (isAccountEmail(saved)) return saved.trim();
|
|
165
|
+
const derived = deriveDefaultAuthAccount(vms, lastAuthAt);
|
|
166
|
+
return (derived && derived.email) || null;
|
|
167
|
+
}
|
|
168
|
+
|
|
111
169
|
// Return true if this one VM is eligible for an auth-start action now.
|
|
112
170
|
function isAuthStartable(v, job) {
|
|
113
171
|
return classifyAuthVm(v, job).eligible;
|
|
114
172
|
}
|
|
115
173
|
|
|
116
|
-
module.exports = {
|
|
174
|
+
module.exports = {
|
|
175
|
+
classifyAuthVm,
|
|
176
|
+
selectAuthTargets,
|
|
177
|
+
summarizeAuthBatch,
|
|
178
|
+
deriveDefaultAuthAccount,
|
|
179
|
+
listAvailableAccounts,
|
|
180
|
+
selectDonorForAccount,
|
|
181
|
+
isAccountEmail,
|
|
182
|
+
resolveChosenAccount,
|
|
183
|
+
isAuthStartable,
|
|
184
|
+
};
|
package/npm/fleet-public-url.js
CHANGED
|
@@ -302,10 +302,32 @@ function ingressUnwiredReason({ url, domain, slug, n } = {}) {
|
|
|
302
302
|
* to all GFEs (VM5's CORRECT rule 503'd for ~minutes before flipping), so a
|
|
303
303
|
* per-cycle re-fire would storm the LB while a good change is still
|
|
304
304
|
* propagating. A missing/0 `lastRewireAt` means "never re-wired" → the window
|
|
305
|
-
* is satisfied.
|
|
305
|
+
* is satisfied.
|
|
306
|
+
*
|
|
307
|
+
* FOURTH CASE — `routed === 'unrouted'` BYPASSES both `reachable` and the
|
|
308
|
+
* `rewirable` backend test above, and that bypass is the whole point. A VM the
|
|
309
|
+
* url-map carries no host rule for has no route from its public hostname to
|
|
310
|
+
* anything, so it is unreachable BY CONSTRUCTION — gating its self-heal on
|
|
311
|
+
* `reachable` gates it on the absence of the very symptom it exists to repair,
|
|
312
|
+
* and it can therefore never fire. Observed 2026-09-08 on 3 of 3 freshly-added
|
|
313
|
+
* self-host VMs: NEGs created, backend services created, NEGs ATTACHED, and no
|
|
314
|
+
* host rule — the add's wiring tail died between the NEG attach and
|
|
315
|
+
* `reconcile_host_rule_once`. The backend test misses it for an independent
|
|
316
|
+
* reason: the backends WERE attached, so this reads as a genuinely-converging
|
|
317
|
+
* 'unhealthy' that the gate deliberately excludes. Both exclusions are correct
|
|
318
|
+
* for a VM that answers and is merely mis-routed, and both are exactly inverted
|
|
319
|
+
* for a VM with no route at all — hence a distinct entry condition rather than
|
|
320
|
+
* a relaxation of the existing one. Do NOT "restore" the `reachable`
|
|
321
|
+
* precondition on this branch as an obvious safety check; it is what made three
|
|
322
|
+
* plainly-broken VMs invisible to the automation for a day.
|
|
323
|
+
* 'unknown' (the url-map read errored) is NOT 'unrouted' — absence of evidence
|
|
324
|
+
* never authorizes a re-wire, matching classifyBackendAttachment's error case.
|
|
325
|
+
* The add's own wiring tail is still excluded here by `inFlight`: a VM mid-
|
|
326
|
+
* wiring is unrouted-so-far, not broken. */
|
|
306
327
|
function shouldAutoRewireIngress({
|
|
307
328
|
verdict,
|
|
308
329
|
reachable,
|
|
330
|
+
routed,
|
|
309
331
|
backendEditorHealth,
|
|
310
332
|
backendEditorBackendless,
|
|
311
333
|
inFlight,
|
|
@@ -314,21 +336,67 @@ function shouldAutoRewireIngress({
|
|
|
314
336
|
debounceMs,
|
|
315
337
|
} = {}) {
|
|
316
338
|
if (verdict !== 'unwired') return false;
|
|
317
|
-
|
|
318
|
-
//
|
|
319
|
-
//
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
339
|
+
// An unrouted VM skips the reachability + backend-health tests below (see the
|
|
340
|
+
// FOURTH CASE note above), but never the in-flight and debounce guards — those
|
|
341
|
+
// are what keep one re-wire per VM per propagation window.
|
|
342
|
+
const unrouted = routed === 'unrouted';
|
|
343
|
+
if (!unrouted) {
|
|
344
|
+
if (!reachable) return false;
|
|
345
|
+
// 'healthy' = host-rule drift (VM5); 'absent' = backend never created
|
|
346
|
+
// (interrupted-add VM-4); 'unhealthy' + backendless = BES exists with zero NEG
|
|
347
|
+
// backends (concurrent-add editor-band race) — it can't converge unaided, so it
|
|
348
|
+
// re-wires too. A genuinely-converging 'unhealthy' (backend attached, health
|
|
349
|
+
// flipping → backendless false) and 'unknown' stay excluded.
|
|
350
|
+
const rewirable =
|
|
351
|
+
backendEditorHealth === 'healthy' ||
|
|
352
|
+
backendEditorHealth === 'absent' ||
|
|
353
|
+
(backendEditorHealth === 'unhealthy' && backendEditorBackendless === true);
|
|
354
|
+
if (!rewirable) return false;
|
|
355
|
+
}
|
|
328
356
|
if (inFlight) return false;
|
|
329
357
|
return (Number(now) - (Number(lastRewireAt) || 0)) >= Number(debounceMs);
|
|
330
358
|
}
|
|
331
359
|
|
|
360
|
+
/** Classify whether the shared fleet url-map routes `host` at all — the input
|
|
361
|
+
* `shouldAutoRewireIngress`'s unrouted branch turns on, and a signal no backend
|
|
362
|
+
* probe can give. `backendEditorHealth` describes the backend SERVICE; none of
|
|
363
|
+
* its four values can express "nothing routes here", which is why all three of
|
|
364
|
+
* the 2026-09-08 stranded VMs classified as a converging 'unhealthy' while
|
|
365
|
+
* carrying no host rule at all. Reads:
|
|
366
|
+
*
|
|
367
|
+
* 'routed' — `host` appears verbatim among the url-map's host rules.
|
|
368
|
+
* 'unrouted' — the read SUCCEEDED, listed at least one host rule, and `host`
|
|
369
|
+
* is not among them. This is the only reading that authorizes a
|
|
370
|
+
* re-wire, and it requires positive evidence of a populated map.
|
|
371
|
+
* 'unknown' — the read errored, produced nothing, or no host was asked for.
|
|
372
|
+
*
|
|
373
|
+
* EMPTY OUTPUT IS 'unknown', NOT 'unrouted', and the distinction is load-
|
|
374
|
+
* bearing. A url-map that lists zero host rules is indistinguishable from a
|
|
375
|
+
* read that silently returned nothing, and reading it as 'unrouted' would
|
|
376
|
+
* authorize a simultaneous re-wire for EVERY VM in the fleet off a single
|
|
377
|
+
* unreliable read — the re-wire storm classifyBackendAttachment's error case
|
|
378
|
+
* exists to prevent, at fleet scale. Absence of evidence never manufactures a
|
|
379
|
+
* fault.
|
|
380
|
+
*
|
|
381
|
+
* Matching is by exact token, never substring: gcloud flattens
|
|
382
|
+
* `value(hostRules.hosts)` with whitespace and `;` separators, and a substring
|
|
383
|
+
* test would read `vm-2.editor.<domain>` as routed on the strength of an
|
|
384
|
+
* unrelated `staging.vm-2.editor.<domain>` rule.
|
|
385
|
+
*
|
|
386
|
+
* Takes the raw (err, stdout) pair rather than a VM number so it stays PURE and
|
|
387
|
+
* needs no domain/slug derivation — the same I/O-edge / pure-half split
|
|
388
|
+
* classifyBackendAttachment follows. The caller derives `host` from
|
|
389
|
+
* google-ingress's `urlMapHostRules` / vmUrls and does the gcloud read. */
|
|
390
|
+
function classifyUrlMapRouting(err, stdout, host) {
|
|
391
|
+
if (err) return 'unknown';
|
|
392
|
+
if (!host) return 'unknown';
|
|
393
|
+
const hosts = String(stdout == null ? '' : stdout)
|
|
394
|
+
.split(/[\s;,]+/)
|
|
395
|
+
.filter(Boolean);
|
|
396
|
+
if (hosts.length === 0) return 'unknown';
|
|
397
|
+
return hosts.includes(String(host)) ? 'routed' : 'unrouted';
|
|
398
|
+
}
|
|
399
|
+
|
|
332
400
|
/** Classify a `gcloud compute backend-services describe --format=value(backends[].group)`
|
|
333
401
|
* result into the attachment vocabulary — the signal `get-health` cannot give,
|
|
334
402
|
* because an empty health result is ambiguous between "no backend attached" and
|
|
@@ -390,5 +458,6 @@ module.exports = {
|
|
|
390
458
|
ingressUnwiredReason,
|
|
391
459
|
shouldAutoRewireIngress,
|
|
392
460
|
classifyBackendAttachment,
|
|
461
|
+
classifyUrlMapRouting,
|
|
393
462
|
classifyUrlWiringState,
|
|
394
463
|
};
|
package/npm/fleet-reconcile.js
CHANGED
|
@@ -59,16 +59,33 @@ function strandedAdoptDecision({ stranded, streaks, threshold } = {}) {
|
|
|
59
59
|
return { streaks: next, ready };
|
|
60
60
|
}
|
|
61
61
|
|
|
62
|
+
// Each entry maps a degraded-reason shape to the live witness that disconfirms
|
|
63
|
+
// it. Extend the table as new degraded reasons gain a corresponding signal.
|
|
64
|
+
//
|
|
65
|
+
// TWO SHAPES PER CLASS, ON PURPOSE. The dashboard writes its auth reason under
|
|
66
|
+
// the structured `claude-auth-not-seeded:` prefix (server.js, and the prefix
|
|
67
|
+
// two other call sites key on), but the original free-text pattern needs a
|
|
68
|
+
// SPACE — `/claude auth/` never matches the hyphenated prefix. Such reasons
|
|
69
|
+
// cleared only because the canned message happens to contain the word "login";
|
|
70
|
+
// phrased any other way they were retained forever, which is the reported bug
|
|
71
|
+
// surviving in the one form the reconcile was aimed at. Matching the prefix
|
|
72
|
+
// only would be the opposite mistake: reasons like `claude auth copy failed on
|
|
73
|
+
// VM-2 (…)` are already recorded in live snapshots and carry no prefix at all,
|
|
74
|
+
// so both shapes have to match.
|
|
75
|
+
const REASON_WITNESSES = [
|
|
76
|
+
{ match: /^\s*claude-auth-not-seeded\b/i, witness: 'authed' },
|
|
77
|
+
{ match: /claude auth|claude\.json|login/i, witness: 'authed' },
|
|
78
|
+
{ match: /container|never came up|provisioning failed/i, witness: 'reachableServing' },
|
|
79
|
+
];
|
|
80
|
+
|
|
62
81
|
// Map a single degraded reason to the live health signal that resolves it.
|
|
63
82
|
// Returns true only when that signal now passes; a reason with no mapped signal
|
|
64
83
|
// returns false (retained — a genuine config problem, never a false-green).
|
|
65
|
-
// Start with the two observed classes (improve41); extend the table as new
|
|
66
|
-
// degraded reasons gain a corresponding live signal.
|
|
67
84
|
function reasonClearedByHealth(reason, { reachableServing, authed }) {
|
|
68
85
|
const r = String(reason || '');
|
|
69
|
-
|
|
70
|
-
if (
|
|
71
|
-
return
|
|
86
|
+
const entry = REASON_WITNESSES.find((e) => e.match.test(r));
|
|
87
|
+
if (!entry) return false;
|
|
88
|
+
return entry.witness === 'authed' ? !!authed : !!reachableServing;
|
|
72
89
|
}
|
|
73
90
|
|
|
74
91
|
// Given a terminal add-job badge and the VM's live health, return the corrected
|
|
@@ -79,7 +96,14 @@ function reasonClearedByHealth(reason, { reachableServing, authed }) {
|
|
|
79
96
|
// facts (I/O at the edge) and applies the result; the verdict itself is pure.
|
|
80
97
|
// job: { state: 'done'|'error', degraded, degradedReasons }
|
|
81
98
|
// health: { reachable, editorHttpOk, providerAuthState }
|
|
82
|
-
// → null | { state, degraded, degradedReasons, transition }
|
|
99
|
+
// → null | { state, degraded, degradedReasons, retiredReasons, transition }
|
|
100
|
+
//
|
|
101
|
+
// `retiredReasons` names the reasons this verdict just dropped. Without it a
|
|
102
|
+
// retired reason simply vanishes from the card, so an operator who saw amber
|
|
103
|
+
// never learns it resolved — the shell renders these as the same muted
|
|
104
|
+
// "recovered (was: …)" trace the reconciled error badge already leaves. What
|
|
105
|
+
// that trace should SAY is the shell's call; naming what was retired is this
|
|
106
|
+
// module's.
|
|
83
107
|
function reconcileTerminalAddBadge({ job, health } = {}) {
|
|
84
108
|
if (!job || !health) return null;
|
|
85
109
|
const reachableServing = !!(health.reachable && health.editorHttpOk);
|
|
@@ -91,7 +115,13 @@ function reconcileTerminalAddBadge({ job, health } = {}) {
|
|
|
91
115
|
if (job.state === 'error') {
|
|
92
116
|
const reasons = job.degradedReasons || [];
|
|
93
117
|
if (reasons.length === 0 && reachableServing) {
|
|
94
|
-
return {
|
|
118
|
+
return {
|
|
119
|
+
state: 'done',
|
|
120
|
+
degraded: false,
|
|
121
|
+
degradedReasons: [],
|
|
122
|
+
retiredReasons: [],
|
|
123
|
+
transition: 'error→done (editor 200)',
|
|
124
|
+
};
|
|
95
125
|
}
|
|
96
126
|
return null;
|
|
97
127
|
}
|
|
@@ -100,15 +130,20 @@ function reconcileTerminalAddBadge({ job, health } = {}) {
|
|
|
100
130
|
// passes; retain the rest. Drop `degraded` only when no reasons remain.
|
|
101
131
|
if (job.state === 'done' && job.degraded) {
|
|
102
132
|
const reasons = job.degradedReasons || [];
|
|
103
|
-
const remaining =
|
|
104
|
-
|
|
105
|
-
const
|
|
133
|
+
const remaining = [];
|
|
134
|
+
const retired = [];
|
|
135
|
+
for (const reason of reasons) {
|
|
136
|
+
const bucket = reasonClearedByHealth(reason, { reachableServing, authed }) ? retired : remaining;
|
|
137
|
+
bucket.push(reason);
|
|
138
|
+
}
|
|
139
|
+
if (retired.length === 0) return null; // nothing cleared → no-op
|
|
106
140
|
return {
|
|
107
141
|
state: 'done',
|
|
108
142
|
degraded: remaining.length > 0,
|
|
109
143
|
degradedReasons: remaining,
|
|
144
|
+
retiredReasons: retired,
|
|
110
145
|
transition: remaining.length > 0
|
|
111
|
-
? `degraded: cleared ${
|
|
146
|
+
? `degraded: cleared ${retired.length} reason(s), ${remaining.length} remain`
|
|
112
147
|
: 'degraded→clear (health restored)',
|
|
113
148
|
};
|
|
114
149
|
}
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* group-bounded-spawn.js — run a command synchronously under a wall-clock budget
|
|
5
|
+
* so that when the budget trips, the command's WHOLE process tree dies with it.
|
|
6
|
+
*
|
|
7
|
+
* `spawnSync`'s `timeout` signals only the direct child. `gcloud compute ssh` is a
|
|
8
|
+
* Python wrapper that runs `/usr/bin/ssh` as its own child, so a timed-out probe
|
|
9
|
+
* killed the wrapper and orphaned the ssh: re-parented to PID 1 and alive for tens
|
|
10
|
+
* of minutes against a 2-minute budget (twelve of them on one laptop, driving load
|
|
11
|
+
* to 14.8 — plan fleet--a-timed-out-gcloud-probe-leaks-its-ssh-session).
|
|
12
|
+
*
|
|
13
|
+
* The fix routes the spawn through this same file run as a tiny runner. The runner
|
|
14
|
+
* stays in the caller's process group, so a terminal Ctrl-C still reaches it; it
|
|
15
|
+
* starts the real command as the leader of a NEW process group, and on any
|
|
16
|
+
* terminating signal — including the SIGTERM spawnSync's own timeout sends — takes
|
|
17
|
+
* that entire group down before exiting with the same signal. The caller therefore
|
|
18
|
+
* sees exactly the result a plain spawnSync timeout produces (`status: null`,
|
|
19
|
+
* `signal: "SIGTERM"`, `error.code: "ETIMEDOUT"`), so isSpawnTimeout and
|
|
20
|
+
* formatGcloudFailure read it unchanged.
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
const childProcess = require("child_process");
|
|
24
|
+
const { reconcileGroupKillTarget } = require("./ingress-invoke");
|
|
25
|
+
|
|
26
|
+
// Between the polite group SIGTERM and the SIGKILL that guarantees nothing
|
|
27
|
+
// survives. Long enough for ssh to restore a raw-mode terminal on SIGTERM.
|
|
28
|
+
const GROUP_KILL_GRACE_MS = 500;
|
|
29
|
+
const TERMINATING_SIGNALS = ["SIGTERM", "SIGINT", "SIGHUP"];
|
|
30
|
+
|
|
31
|
+
/** Drop-in for `spawnSync(command, args, opts)` whose `timeout` (and any
|
|
32
|
+
* terminating signal) reaches every descendant, not only the direct child. */
|
|
33
|
+
function spawnGroupBoundedSync(command, args, opts = {}) {
|
|
34
|
+
// `childProcess.spawnSync` is resolved at call time, not destructured at load,
|
|
35
|
+
// so a caller that swaps it (cloud.test.js does) sees every spawn.
|
|
36
|
+
// Platform assumption: process groups are POSIX. Windows has no group to
|
|
37
|
+
// signal, so it keeps plain spawnSync semantics (direct child only).
|
|
38
|
+
if (process.platform === "win32") return childProcess.spawnSync(command, args, opts);
|
|
39
|
+
return childProcess.spawnSync(process.execPath, [__filename, command, ...args], opts);
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function killGroup(leaderPid, signal) {
|
|
43
|
+
// The shared target derivation refuses pid <= 1, where a negative kill would
|
|
44
|
+
// signal every process this user owns.
|
|
45
|
+
const target = reconcileGroupKillTarget(leaderPid);
|
|
46
|
+
if (target == null) return;
|
|
47
|
+
try {
|
|
48
|
+
process.kill(target, signal);
|
|
49
|
+
} catch {
|
|
50
|
+
// The group is already gone — nothing left to reap.
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/** Exit the runner the way the wrapped command ended, so spawnSync's result
|
|
55
|
+
* mirrors a direct spawn. A signal is re-raised on ourselves with its handler
|
|
56
|
+
* removed, which is what makes `result.signal` read true. */
|
|
57
|
+
function exitLike(code, signal) {
|
|
58
|
+
if (signal) {
|
|
59
|
+
for (const s of TERMINATING_SIGNALS) process.removeAllListeners(s);
|
|
60
|
+
process.kill(process.pid, signal);
|
|
61
|
+
return;
|
|
62
|
+
}
|
|
63
|
+
process.exit(code == null ? 1 : code);
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/** Take the whole group down: a polite SIGTERM so ssh can restore a raw-mode
|
|
67
|
+
* terminal, a SIGKILL after the grace so nothing survives, then exit with the
|
|
68
|
+
* signal that started it. */
|
|
69
|
+
function beginGroupTermination(child, signal) {
|
|
70
|
+
killGroup(child.pid, "SIGTERM");
|
|
71
|
+
setTimeout(() => {
|
|
72
|
+
killGroup(child.pid, "SIGKILL");
|
|
73
|
+
exitLike(null, signal);
|
|
74
|
+
}, GROUP_KILL_GRACE_MS);
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/** Runner body: `node group-bounded-spawn.js <command> [args...]`. */
|
|
78
|
+
function runAsGroupLeader(command, args) {
|
|
79
|
+
const child = childProcess.spawn(command, args, { stdio: "inherit", detached: true });
|
|
80
|
+
let terminating = false;
|
|
81
|
+
|
|
82
|
+
for (const sig of TERMINATING_SIGNALS) {
|
|
83
|
+
process.on(sig, () => {
|
|
84
|
+
if (terminating) return;
|
|
85
|
+
terminating = true;
|
|
86
|
+
beginGroupTermination(child, sig);
|
|
87
|
+
});
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
child.on("error", (err) => {
|
|
91
|
+
// ENOENT and friends: the command never started. 127 is the shell's
|
|
92
|
+
// "command not found", and the message lands on stderr for the caller.
|
|
93
|
+
process.stderr.write(`${err.message}\n`);
|
|
94
|
+
process.exit(127);
|
|
95
|
+
});
|
|
96
|
+
child.on("exit", (code, signal) => {
|
|
97
|
+
// A termination in flight finishes on its own grace timer.
|
|
98
|
+
if (terminating) return;
|
|
99
|
+
// The leader died by a signal nobody sent through us (OOM, an external
|
|
100
|
+
// kill): its descendants are the same orphans, so take them down too.
|
|
101
|
+
if (signal) killGroup(child.pid, "SIGKILL");
|
|
102
|
+
exitLike(code, signal);
|
|
103
|
+
});
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
if (require.main === module) {
|
|
107
|
+
const [command, ...args] = process.argv.slice(2);
|
|
108
|
+
runAsGroupLeader(command, args);
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
module.exports = { spawnGroupBoundedSync };
|