@codeyam-editor/codeyam-editor 0.1.3 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/npm/cloud-branch-switch.js +1 -1
- package/npm/cloud.js +1693 -96
- package/npm/container.js +14 -0
- package/npm/disk-pressure.js +167 -0
- package/npm/editor-dev.js +107 -15
- package/npm/editor.js +97 -37
- package/npm/fleet-auth-health.js +75 -0
- package/npm/fleet-auth.js +6 -1
- package/npm/fleet-build-data.js +130 -0
- package/npm/fleet-cleanup.js +145 -7
- package/npm/fleet-destroy-safety.js +55 -0
- package/npm/fleet-effect-status.js +33 -32
- package/npm/fleet-host.js +74 -0
- package/npm/fleet-job-badge.js +46 -2
- package/npm/fleet-launchd.js +171 -0
- package/npm/fleet-live-status.js +123 -0
- package/npm/fleet-plans.js +231 -0
- package/npm/fleet-prewarm.js +165 -0
- package/npm/fleet-public-url.js +160 -0
- package/npm/fleet-queue.js +38 -1
- package/npm/fleet-reconcile.js +100 -1
- package/npm/fleet-reset.js +298 -13
- package/npm/fleet-roster.js +6 -8
- package/npm/fleet-self-update.js +142 -0
- package/npm/fleet-setup-steps.js +105 -0
- package/npm/fleet-sleep-status.js +33 -0
- package/npm/fleet-strand.js +150 -0
- package/npm/fleet-stuck-status.js +111 -0
- package/npm/fleet-vm-access.js +234 -0
- package/npm/fleet-vm-readiness.js +104 -0
- package/npm/fleet-vm-slugs.js +116 -0
- package/npm/google-ingress.js +211 -0
- package/npm/ingress-invoke.js +292 -0
- package/npm/launch-config.js +310 -4
- package/npm/scenario-check.js +684 -33
- package/npm/scenario-handlers.js +94 -12
- package/npm/scenario-metrics.js +116 -1
- package/npm/scenario-mocks.js +20 -0
- package/npm/scenario-playwright.js +366 -29
- package/npm/script-drift.js +81 -0
- package/npm/utils.js +453 -43
- package/npm/vm-urls.js +69 -8
- package/package.json +5 -5
- package/ui/dist/assets/ContainerInspector-D7VZkkaj.js +2 -0
- package/ui/dist/assets/EditorShell-DHhIaotd.css +1 -0
- package/ui/dist/assets/EditorShell-r-796N_W.js +191 -0
- package/ui/dist/assets/{Markdown-BHBqz-Mj.js → Markdown-BoLHHFXH.js} +1 -1
- package/ui/dist/assets/ProjectLauncherScreen-Ck-XWkQN.js +1 -0
- package/ui/dist/assets/ScenarioDataPanel-B9iDSUpA.js +5 -0
- package/ui/dist/assets/ScenarioDataPanelFullPage-CBLTGC4-.js +1 -0
- package/ui/dist/assets/ScenarioRenderer-BhLfT-nX.js +21 -0
- package/ui/dist/assets/StepsSimulator-BFldWyoX.js +1 -0
- package/ui/dist/assets/index-CIXZkwfB.css +1 -0
- package/ui/dist/assets/{index-KDFAgt2e.js → index-QsOiOG9x.js} +9 -9
- package/ui/dist/assets/useEvents-DHkLJZR4.js +1 -0
- package/ui/dist/codeyam-mark.svg +4 -0
- package/ui/dist/codeyam-wordmark.svg +5 -0
- package/ui/dist/index.html +2 -2
- package/ui/dist/assets/ContainerInspector-C8EkH4PP.js +0 -2
- package/ui/dist/assets/EditorShell-CGsBSA6u.js +0 -186
- package/ui/dist/assets/ScenarioDataPanel-nuLXJzLf.js +0 -5
- package/ui/dist/assets/ScenarioDataPanelFullPage-Bnghtj8a.js +0 -1
- package/ui/dist/assets/ScenarioRenderer-B_Za9jL8.js +0 -1
- package/ui/dist/assets/StepsSimulator-CRtmlyNZ.js +0 -1
- package/ui/dist/assets/index-Bih6HJ_u.css +0 -1
- package/ui/dist/assets/useEvents-DggWrtHe.js +0 -1
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
// npm/fleet-prewarm.js — pure verdicts for the fleet dashboard's concurrent
|
|
4
|
+
// cold-add pre-warm (vm--concurrent-add-prewarm-branch-image).
|
|
5
|
+
//
|
|
6
|
+
// When several VMs are added at once on a branch whose prebuilt image is not yet
|
|
7
|
+
// published, EVERY VM compiles the editor from source (~10 min each) in
|
|
8
|
+
// parallel — none has finished compiling-and-publishing before its siblings
|
|
9
|
+
// start, so the self-feeding fast path (publish_compiled_binary in
|
|
10
|
+
// scripts/gcp-bootstrap.sh) never gets a chance to prime. The fix is to bring up
|
|
11
|
+
// ONE "pacer" VM first, wait until it has published the branch binary (or
|
|
12
|
+
// settled on a definite won't-publish), then release the rest so they pull it.
|
|
13
|
+
//
|
|
14
|
+
// This module holds the *decisions* — pace-or-fan-out, which tags constitute a
|
|
15
|
+
// usable image, parsing the bootstrap's machine markers, and the push-token
|
|
16
|
+
// diagnostic — so each case is unit-tested without a live VM or registry. The
|
|
17
|
+
// I/O (the manifest probe, reading the add log, threading the held queue) lives
|
|
18
|
+
// in the server / cloud.js shells that call these.
|
|
19
|
+
|
|
20
|
+
// Machine-detectable markers gcp-bootstrap.sh emits into the cloud:up add log
|
|
21
|
+
// (streamed to /tmp/dash-add-vm-<n>.log). They are stable contracts the
|
|
22
|
+
// dashboard greps for — keep them in sync with scripts/gcp-bootstrap.sh.
|
|
23
|
+
const MARKER_PUBLISHED = 'CY_BINARY_PUBLISHED'; // pacer published the branch binary; held adds can pull it
|
|
24
|
+
const MARKER_PUBLISH_SKIPPED = 'CY_BINARY_PUBLISH_SKIPPED'; // pacer will NOT publish (no token / no branch / failure)
|
|
25
|
+
const MARKER_PATH_PULL = 'CY_PROVISION_PATH pull'; // this VM pulled a prebuilt image (~2 min fast path)
|
|
26
|
+
const MARKER_PATH_COMPILE = 'CY_PROVISION_PATH compile'; // this VM compiled from source (~10 min slow path)
|
|
27
|
+
|
|
28
|
+
// Mirror scripts/gcp-bootstrap.sh's BRANCH_SLUG: '/' → '-', lowercased, capped
|
|
29
|
+
// at 128 chars. The dashboard and the VM MUST agree on the slug or the probe
|
|
30
|
+
// checks different tags than the bootstrap pulls. Empty / missing branch → the
|
|
31
|
+
// universal "main" slug, matching the bootstrap's detached-HEAD fallback.
|
|
32
|
+
function branchSlug(branch) {
|
|
33
|
+
if (!branch || typeof branch !== 'string') return 'main';
|
|
34
|
+
const slug = branch.replace(/\//g, '-').toLowerCase().slice(0, 128);
|
|
35
|
+
return slug || 'main';
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
// The tag chain that constitutes a "usable" prebuilt editor image, in the same
|
|
39
|
+
// precedence the dev bootstrap pulls (build_editor_image in gcp-bootstrap.sh):
|
|
40
|
+
// 1. cloud-smoke:<slug>-latest — the branch base image
|
|
41
|
+
// 2. cloud-smoke:main-latest — the universal CI fallback (skipped when
|
|
42
|
+
// the branch already IS main, no duplicate)
|
|
43
|
+
// 3. binary:<slug>-latest — the branch binary overlay
|
|
44
|
+
// If a manifest HEAD on ANY of these succeeds, a cold add can pull instead of
|
|
45
|
+
// compiling, so no pacer is needed. Registry defaults to the same
|
|
46
|
+
// CODEYAM_REGISTRY the bootstrap reads.
|
|
47
|
+
function usableImageTags({ registry, branch } = {}) {
|
|
48
|
+
const reg = registry || 'ghcr.io/codeyam-ai';
|
|
49
|
+
const slug = branchSlug(branch);
|
|
50
|
+
const tags = [`${reg}/codeyam-editor-cloud-smoke:${slug}-latest`];
|
|
51
|
+
if (slug !== 'main') tags.push(`${reg}/codeyam-editor-cloud-smoke:main-latest`);
|
|
52
|
+
tags.push(`${reg}/codeyam-editor-binary:${slug}-latest`);
|
|
53
|
+
return tags;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
// The core verdict: should a batch add pace one VM ahead of the rest?
|
|
57
|
+
//
|
|
58
|
+
// count how many VMs the operator asked to add
|
|
59
|
+
// hasUsableImage the probe result — true (image present), false (definitely
|
|
60
|
+
// absent), or null (couldn't determine: no token, probe error)
|
|
61
|
+
// hasPushToken a write:packages token is present, so the pacer CAN publish
|
|
62
|
+
//
|
|
63
|
+
// Pace ONLY when it strictly pays AND can work:
|
|
64
|
+
// - count > 1 — a single add is never serialized (never slower)
|
|
65
|
+
// - hasUsableImage === false — known-absent; an existing image means all N can
|
|
66
|
+
// pull concurrently, and `null` (unknown) falls
|
|
67
|
+
// back to today's fan-out rather than risk a
|
|
68
|
+
// 10-min serialization that buys nothing
|
|
69
|
+
// - hasPushToken — without it the pacer can't publish, so pacing would
|
|
70
|
+
// just serialize N source compiles for no gain
|
|
71
|
+
// Any miss → fan out all N immediately (today's behavior, strictly additive).
|
|
72
|
+
function decideBatchPrewarm({ count, hasUsableImage, hasPushToken } = {}) {
|
|
73
|
+
const k = Math.round(Number(count) || 0);
|
|
74
|
+
if (k <= 1) return { pace: false, reason: 'single add — never serialized' };
|
|
75
|
+
if (hasUsableImage === true)
|
|
76
|
+
return { pace: false, reason: 'usable prebuilt image exists — all pull concurrently' };
|
|
77
|
+
if (hasUsableImage !== false)
|
|
78
|
+
return { pace: false, reason: 'image presence undetermined — fan out (no worse than today)' };
|
|
79
|
+
if (!hasPushToken)
|
|
80
|
+
return { pace: false, reason: 'no push token — pacer could not publish, fan out' };
|
|
81
|
+
return { pace: true, reason: `pacing 1 of ${k} — branch image absent, will publish for the rest` };
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
// Has the pacer's add log settled to a definite publish outcome the held batch
|
|
85
|
+
// can act on? Returns 'published' (binary is now pullable — release and pull),
|
|
86
|
+
// 'skipped' (pacer will NOT publish — release; they recompile, today's
|
|
87
|
+
// behavior), or null (still in flight — keep waiting). A SKIPPED that is later
|
|
88
|
+
// followed by a PUBLISHED still reports published (the success wins), so marker
|
|
89
|
+
// ordering can't strand the batch on a transient earlier skip.
|
|
90
|
+
function detectPublishSettled(logText) {
|
|
91
|
+
if (!logText || typeof logText !== 'string') return null;
|
|
92
|
+
if (logText.includes(MARKER_PUBLISHED)) return 'published';
|
|
93
|
+
if (logText.includes(MARKER_PUBLISH_SKIPPED)) return 'skipped';
|
|
94
|
+
return null;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
// The pure release decision for the pre-warm hold (startPrewarmWatch's per-tick
|
|
98
|
+
// verdict): given the pacer's current signals, should the held batch be released
|
|
99
|
+
// now, and why? Returns a human reason string to release, or null to keep
|
|
100
|
+
// waiting. Releases on the FIRST of: a settled publish marker, the pacer job
|
|
101
|
+
// terminating without one, or the bounded timeout — so a publish failure or a
|
|
102
|
+
// stuck pacer can never strand the batch.
|
|
103
|
+
//
|
|
104
|
+
// settled detectPublishSettled() result: 'published' | 'skipped' | null
|
|
105
|
+
// pacerJob the pacer add job record ({ state, ... }) or null/undefined
|
|
106
|
+
// elapsedMs ms since the watch started
|
|
107
|
+
// timeoutMs the bounded fallback after which the batch releases regardless
|
|
108
|
+
function prewarmReleaseDecision({ settled, pacerJob, elapsedMs, timeoutMs } = {}) {
|
|
109
|
+
if (settled) return `pacer ${settled}`;
|
|
110
|
+
const st = pacerJob && pacerJob.state;
|
|
111
|
+
if (st === 'done' || st === 'error') return `pacer ${st}`;
|
|
112
|
+
if (Number(elapsedMs) > Number(timeoutMs)) return 'timeout';
|
|
113
|
+
return null;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
// Which provision path did a VM take, parsed from its add log's path marker?
|
|
117
|
+
// Returns 'pull' (~2 min prebuilt pull), 'compile' (~10 min source compile), or
|
|
118
|
+
// null (not yet determined). The LAST marker wins — a VM that tries a pull,
|
|
119
|
+
// misses, then falls back to a compile correctly reads 'compile'.
|
|
120
|
+
function provisionPath(logText) {
|
|
121
|
+
if (!logText || typeof logText !== 'string') return null;
|
|
122
|
+
const pull = logText.lastIndexOf(MARKER_PATH_PULL);
|
|
123
|
+
const compile = logText.lastIndexOf(MARKER_PATH_COMPILE);
|
|
124
|
+
if (pull < 0 && compile < 0) return null;
|
|
125
|
+
return compile > pull ? 'compile' : 'pull';
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
// Operator-facing one-liner for a provision path, for the add card sub-status.
|
|
129
|
+
// null path → null (render nothing).
|
|
130
|
+
function provisionPathLabel(path) {
|
|
131
|
+
if (path === 'pull') return 'pulling prebuilt image (~2 min)';
|
|
132
|
+
if (path === 'compile') return 'compiling from source (~10 min)';
|
|
133
|
+
return null;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
// The push-token diagnostic surfaced on the dashboard. Without a write:packages
|
|
137
|
+
// token the self-feeding publish is silently off, so EVERY cold add recompiles
|
|
138
|
+
// (~10 min) and the pacer can't help either — a state the operator must see, not
|
|
139
|
+
// discover from a buried per-VM log line.
|
|
140
|
+
function pushTokenDiagnostic({ hasPushToken } = {}) {
|
|
141
|
+
if (hasPushToken) return { warn: false, message: null };
|
|
142
|
+
return {
|
|
143
|
+
warn: true,
|
|
144
|
+
message:
|
|
145
|
+
'Self-feeding disabled: no CODEYAM_GHCR_PUSH_TOKEN (write:packages). ' +
|
|
146
|
+
'Cold adds will always recompile the editor from source (~10 min each) ' +
|
|
147
|
+
'and concurrent batches cannot pre-warm. Install a write-scoped token and ' +
|
|
148
|
+
'reinstall the fleet agents to enable the fast path.',
|
|
149
|
+
};
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
module.exports = {
|
|
153
|
+
MARKER_PUBLISHED,
|
|
154
|
+
MARKER_PUBLISH_SKIPPED,
|
|
155
|
+
MARKER_PATH_PULL,
|
|
156
|
+
MARKER_PATH_COMPILE,
|
|
157
|
+
branchSlug,
|
|
158
|
+
usableImageTags,
|
|
159
|
+
decideBatchPrewarm,
|
|
160
|
+
detectPublishSettled,
|
|
161
|
+
prewarmReleaseDecision,
|
|
162
|
+
provisionPath,
|
|
163
|
+
provisionPathLabel,
|
|
164
|
+
pushTokenDiagnostic,
|
|
165
|
+
};
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
// Pure classification + reason-string derivation for the dashboard's public-URL
|
|
3
|
+
// reachability probe (vm: Dashboard validates the public ingress URL, not just
|
|
4
|
+
// the tunnel).
|
|
5
|
+
//
|
|
6
|
+
// The fleet dashboard probes each VM's editor through the local SSH tunnel and,
|
|
7
|
+
// in Google-LB mode, the per-VM backend SERVICE health (gcloud get-health) —
|
|
8
|
+
// but never the public `<slug>.editor.<domain>` URL the card advertises. VM5
|
|
9
|
+
// (2026-06-19) sat "healthy" on every one of those signals while the advertised
|
|
10
|
+
// URL returned 503 "no healthy upstream" for hours: a reused-slug Add aborted on
|
|
11
|
+
// a `Duplicate path matcher` collision before binding the host rule, so the
|
|
12
|
+
// hostname fell through to the default backend. A backend-health probe is
|
|
13
|
+
// structurally blind to a missing host rule; only an end-to-end GET of the
|
|
14
|
+
// advertised hostname catches it.
|
|
15
|
+
//
|
|
16
|
+
// This module is the PURE half — parsing + string derivation, no network —
|
|
17
|
+
// mirroring classifyBackendHealth's `absent|unknown|unhealthy|healthy`
|
|
18
|
+
// contract. The HTTPS GET I/O lives in server.js's gatherPublicUrlHealth, the
|
|
19
|
+
// same split gatherBackendHealth / classifyBackendHealth already follow. Unit-
|
|
20
|
+
// tested in npm/fleet-public-url.test.js.
|
|
21
|
+
//
|
|
22
|
+
// Stack note: this is fleet-operator infrastructure (the GCE fleet that runs
|
|
23
|
+
// codeyam-editor itself), GCP / HTTPS-LB-specific by design — the same posture
|
|
24
|
+
// as the scripts it backstops. A project that never joins the fleet never sets
|
|
25
|
+
// FLEET_DOMAIN and never reaches this probe.
|
|
26
|
+
|
|
27
|
+
// The Google HTTPS-LB "no backend answered" body signature: GFE emits this when
|
|
28
|
+
// a host rule routes nowhere (or to a backendless default). A single source so
|
|
29
|
+
// the classifier and any future log line agree on the string.
|
|
30
|
+
const NO_HEALTHY_UPSTREAM_RE = /no healthy upstream/i;
|
|
31
|
+
|
|
32
|
+
/** Classify a public-URL probe result into the ingress-wiring vocabulary,
|
|
33
|
+
* following classifyBackendHealth's contract — absence of evidence is never a
|
|
34
|
+
* problem state:
|
|
35
|
+
*
|
|
36
|
+
* 'unknown' — a network / TLS / timeout / DNS error (no statusCode): the
|
|
37
|
+
* probe couldn't reach a verdict. NEVER a degraded state, exactly
|
|
38
|
+
* like classifyBackendHealth's 'unknown'. Also covers an
|
|
39
|
+
* unexpected non-503 HTTP error (a 404/500 from the editor
|
|
40
|
+
* itself) — that means the host rule DID route, just not to a
|
|
41
|
+
* happy response, so it is neither 'unwired' nor a clean 'wired'.
|
|
42
|
+
* 'unwired' — the LB answered but no backend is bound to the advertised host:
|
|
43
|
+
* a 503, or a recognizable "no healthy upstream" default-backend
|
|
44
|
+
* body. THIS is the missing-host-rule signal a backend-health
|
|
45
|
+
* probe cannot see (the VM5 case).
|
|
46
|
+
* 'wired' — any 2xx / 3xx: the editor answered (a 200, or its login 302
|
|
47
|
+
* redirect — the real VM5-healthy shape). The host rule routes.
|
|
48
|
+
*/
|
|
49
|
+
function classifyPublicUrlHealth({ statusCode, body, error } = {}) {
|
|
50
|
+
if (error || statusCode == null) return 'unknown';
|
|
51
|
+
const code = Number(statusCode);
|
|
52
|
+
if (code === 503) return 'unwired';
|
|
53
|
+
if (NO_HEALTHY_UPSTREAM_RE.test(String(body || ''))) return 'unwired';
|
|
54
|
+
if (code >= 200 && code < 400) return 'wired';
|
|
55
|
+
// Any other 4xx/5xx is the editor answering with an error of its own — the
|
|
56
|
+
// host rule routes, so it is NOT 'unwired'; but it is not a clean success
|
|
57
|
+
// either. Treat as 'unknown' (absence of the unwired signal) so we never raise
|
|
58
|
+
// a false ingress-unwired reason on it.
|
|
59
|
+
return 'unknown';
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/** The sticky `ingress-unwired` degraded-reason string — the single source for
|
|
63
|
+
* both the dashboard card reason and any log line. Embeds the advertised URL
|
|
64
|
+
* and the EXACT re-wire command an operator runs to fix it, so the amber card
|
|
65
|
+
* is self-documenting (the surface VM5 should have shown for hours instead of
|
|
66
|
+
* reading healthy). */
|
|
67
|
+
function ingressUnwiredReason({ url, domain, slug, n } = {}) {
|
|
68
|
+
const target = url || `(vm-${n} editor URL)`;
|
|
69
|
+
const slugEnv = slug ? ` VM_SLUG=${slug}` : '';
|
|
70
|
+
const rewire = `FLEET_DOMAIN=${domain}${slugEnv} bash scripts/gcp/vm-ingress.sh ${n} up`;
|
|
71
|
+
return `ingress-unwired: ${target} returns 503 (host rule missing). Re-wire: ${rewire}`;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/** Decide whether a just-landed public-URL verdict warrants an auto-rewire
|
|
75
|
+
* self-heal. PURE — the dashboard reads the live VM state (I/O at the edge) and
|
|
76
|
+
* passes the facts in; the gate logic itself is unit-testable without network
|
|
77
|
+
* or server state. Fires only for a GENUINE missing-host-rule case:
|
|
78
|
+
* - `verdict` is 'unwired' (the URL 503s), AND
|
|
79
|
+
* - `reachable` (the tunnel is up — a down VM is not a missing-host-rule
|
|
80
|
+
* case; re-wiring it accomplishes nothing), AND
|
|
81
|
+
* - `backendEditorHealth` is 'healthy' (the backend SERVICE is fine, so only
|
|
82
|
+
* the host rule is missing — the VM5 host-rule-drift case) OR 'absent' (the
|
|
83
|
+
* per-VM backend service was NEVER created — the interrupted-add VM-4 case,
|
|
84
|
+
* where a SIGTERM killed `backend-services create` mid-wiring) OR
|
|
85
|
+
* 'unhealthy' *with* `backendEditorBackendless` true (the BES EXISTS but has
|
|
86
|
+
* ZERO NEG backends attached — the concurrent-add editor-band race, where the
|
|
87
|
+
* backend slipped its NEG attachment but the BES object landed). All three
|
|
88
|
+
* are recoverable by a single `vm-ingress.sh up`, which is
|
|
89
|
+
* describe-before-create idempotent and creates/attaches only the missing
|
|
90
|
+
* pieces. A backendless BES classifies as 'unhealthy' (get-health returns
|
|
91
|
+
* empty, indistinguishable from a still-cold backend) yet will NEVER converge
|
|
92
|
+
* on its own — it has no endpoint to become healthy — so it MUST re-wire, not
|
|
93
|
+
* wait. 'unknown' (a transient gcloud/auth error — absence of evidence must
|
|
94
|
+
* never trigger a re-wire storm) and a GENUINELY-converging 'unhealthy' (a
|
|
95
|
+
* backend IS attached, health still flipping — `backendEditorBackendless`
|
|
96
|
+
* false: let it converge, don't churn the LB) stay EXCLUDED, AND
|
|
97
|
+
* - no re-wire is already `inFlight` (one per VM), AND
|
|
98
|
+
* - the debounce window has elapsed since `lastRewireAt`.
|
|
99
|
+
* The debounce is load-bearing: GCP host-rule changes take minutes to roll out
|
|
100
|
+
* to all GFEs (VM5's CORRECT rule 503'd for ~minutes before flipping), so a
|
|
101
|
+
* per-cycle re-fire would storm the LB while a good change is still
|
|
102
|
+
* propagating. A missing/0 `lastRewireAt` means "never re-wired" → the window
|
|
103
|
+
* is satisfied. */
|
|
104
|
+
function shouldAutoRewireIngress({
|
|
105
|
+
verdict,
|
|
106
|
+
reachable,
|
|
107
|
+
backendEditorHealth,
|
|
108
|
+
backendEditorBackendless,
|
|
109
|
+
inFlight,
|
|
110
|
+
lastRewireAt,
|
|
111
|
+
now,
|
|
112
|
+
debounceMs,
|
|
113
|
+
} = {}) {
|
|
114
|
+
if (verdict !== 'unwired') return false;
|
|
115
|
+
if (!reachable) return false;
|
|
116
|
+
// 'healthy' = host-rule drift (VM5); 'absent' = backend never created
|
|
117
|
+
// (interrupted-add VM-4); 'unhealthy' + backendless = BES exists with zero NEG
|
|
118
|
+
// backends (concurrent-add editor-band race) — it can't converge unaided, so it
|
|
119
|
+
// re-wires too. A genuinely-converging 'unhealthy' (backend attached, health
|
|
120
|
+
// flipping → backendless false) and 'unknown' stay excluded.
|
|
121
|
+
const rewirable =
|
|
122
|
+
backendEditorHealth === 'healthy' ||
|
|
123
|
+
backendEditorHealth === 'absent' ||
|
|
124
|
+
(backendEditorHealth === 'unhealthy' && backendEditorBackendless === true);
|
|
125
|
+
if (!rewirable) return false;
|
|
126
|
+
if (inFlight) return false;
|
|
127
|
+
return (Number(now) - (Number(lastRewireAt) || 0)) >= Number(debounceMs);
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/** Classify a `gcloud compute backend-services describe --format=value(backends[].group)`
|
|
131
|
+
* result into the attachment vocabulary — the signal `get-health` cannot give,
|
|
132
|
+
* because an empty health result is ambiguous between "no backend attached" and
|
|
133
|
+
* "a backend attached but still health-pending":
|
|
134
|
+
*
|
|
135
|
+
* 'backendless' — the BES EXISTS but has ZERO NEG backends attached (the
|
|
136
|
+
* describe printed empty / whitespace-only output). This BES
|
|
137
|
+
* will NEVER converge on its own — there is no endpoint to
|
|
138
|
+
* become healthy — so it is re-wirable, feeding
|
|
139
|
+
* shouldAutoRewireIngress's `backendEditorBackendless` branch
|
|
140
|
+
* (the concurrent-add editor-band slip).
|
|
141
|
+
* 'attached' — the describe printed at least one backend group, OR it
|
|
142
|
+
* ERRORED. An error is treated as 'attached' (absence of
|
|
143
|
+
* evidence): a NotFound is the 'absent' case classifyBackendHealth
|
|
144
|
+
* already owns, and a transient gcloud/auth failure must NEVER
|
|
145
|
+
* manufacture a 'backendless' verdict that storms the LB with a
|
|
146
|
+
* re-wire. Pure (no I/O), so the empty-backends classification is
|
|
147
|
+
* unit-testable without gcloud — the I/O lives in server.js's
|
|
148
|
+
* gatherEditorBackendless, the same split classifyPublicUrlHealth
|
|
149
|
+
* follows. */
|
|
150
|
+
function classifyBackendAttachment(err, stdout) {
|
|
151
|
+
if (err) return 'attached';
|
|
152
|
+
return String(stdout == null ? '' : stdout).trim() === '' ? 'backendless' : 'attached';
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
module.exports = {
|
|
156
|
+
classifyPublicUrlHealth,
|
|
157
|
+
ingressUnwiredReason,
|
|
158
|
+
shouldAutoRewireIngress,
|
|
159
|
+
classifyBackendAttachment,
|
|
160
|
+
};
|
package/npm/fleet-queue.js
CHANGED
|
@@ -37,4 +37,41 @@ function decideAutoReleaseGoneHead({ queue, vms, jobs, orphans }) {
|
|
|
37
37
|
return { release: true, uuid: head.uuid, vm: n, reason: `owning VM-${n} left the fleet` };
|
|
38
38
|
}
|
|
39
39
|
|
|
40
|
-
|
|
40
|
+
/** Compute the set-priority writes a reorder requires.
|
|
41
|
+
*
|
|
42
|
+
* A reorder POSTs a desired uuid order; the queue's priority convention is
|
|
43
|
+
* first=N … last=1 (higher priority sorts earlier). This maps the desired
|
|
44
|
+
* order onto target priorities, drops the entries already at their target
|
|
45
|
+
* priority (no-ops — no `set-priority` shell-out needed), and sorts the
|
|
46
|
+
* remaining writes intended-HEAD-first so the head lands first even if a
|
|
47
|
+
* later write fails. `current` is the authoritative queue ([{uuid,priority}]);
|
|
48
|
+
* `order` is the desired uuid sequence.
|
|
49
|
+
*
|
|
50
|
+
* Returns one of:
|
|
51
|
+
* { ok: false, reason: 'empty' } — no order supplied
|
|
52
|
+
* { ok: false, reason: 'stale' } — order doesn't match current set
|
|
53
|
+
* { ok: true, targets: [{uuid, prio}] } — writes to apply (possibly [])
|
|
54
|
+
* Pure; no I/O.
|
|
55
|
+
*/
|
|
56
|
+
function computeReorderTargets({ order, current }) {
|
|
57
|
+
const ord = (order || []).map((s) => String(s).trim()).filter(Boolean);
|
|
58
|
+
const cur = current || [];
|
|
59
|
+
const curUuids = cur.map((e) => e.uuid);
|
|
60
|
+
if (!ord.length) return { ok: false, reason: 'empty' };
|
|
61
|
+
// Same membership, same length — a reorder, not an add/remove. Anything else
|
|
62
|
+
// means the queue changed underneath the operator since they last saw it.
|
|
63
|
+
if (ord.length !== curUuids.length || !ord.every((u) => curUuids.includes(u))) {
|
|
64
|
+
return { ok: false, reason: 'stale' };
|
|
65
|
+
}
|
|
66
|
+
const N = ord.length;
|
|
67
|
+
const targets = ord
|
|
68
|
+
.map((uuid, i) => ({ uuid, prio: N - i })) // first=N … last=1
|
|
69
|
+
.filter(({ uuid, prio }) => {
|
|
70
|
+
const e = cur.find((x) => x.uuid === uuid);
|
|
71
|
+
return !e || e.priority !== prio; // skip no-ops
|
|
72
|
+
})
|
|
73
|
+
.sort((a, b) => b.prio - a.prio); // intended HEAD first
|
|
74
|
+
return { ok: true, targets };
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
module.exports = { decideAutoReleaseGoneHead, computeReorderTargets };
|
package/npm/fleet-reconcile.js
CHANGED
|
@@ -33,4 +33,103 @@ function computeUnrostered(running, shown) {
|
|
|
33
33
|
.sort((a, b) => a - b);
|
|
34
34
|
}
|
|
35
35
|
|
|
36
|
-
|
|
36
|
+
// Streak bookkeeping for the adopt-direction stranded-instance reconcile — the
|
|
37
|
+
// mirror of pollVanishedInstances' `unreachableStreak` gate, in the opposite
|
|
38
|
+
// (adopt) direction. Given the currently-stranded VM numbers (running in GCP
|
|
39
|
+
// but absent from roster / provisioning / orphan), the prior streak map, and
|
|
40
|
+
// the threshold, return the NEXT streak map (incremented for each still-stranded
|
|
41
|
+
// number; entries for no-longer-stranded numbers dropped so a transient blip
|
|
42
|
+
// never leaves a stale count) plus `ready` — the numbers whose streak has
|
|
43
|
+
// reached the threshold and may now be auto-finished. A single stale
|
|
44
|
+
// `runningFleetVms` snapshot therefore can't trigger a recovery: the instance
|
|
45
|
+
// must read stranded for `threshold` consecutive polls first. Pure, so the gate
|
|
46
|
+
// is unit-tested without gcloud or a running server.
|
|
47
|
+
function strandedAdoptDecision({ stranded, streaks, threshold } = {}) {
|
|
48
|
+
const prev = streaks || {};
|
|
49
|
+
const t = Number(threshold) > 0 ? Number(threshold) : 1;
|
|
50
|
+
const present = [...new Set((stranded || []).filter((n) => Number.isInteger(n) && n > 0))];
|
|
51
|
+
const next = {};
|
|
52
|
+
const ready = [];
|
|
53
|
+
for (const n of present) {
|
|
54
|
+
const count = (Number(prev[n]) || 0) + 1;
|
|
55
|
+
next[n] = count;
|
|
56
|
+
if (count >= t) ready.push(n);
|
|
57
|
+
}
|
|
58
|
+
ready.sort((a, b) => a - b);
|
|
59
|
+
return { streaks: next, ready };
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
// Map a single degraded reason to the live health signal that resolves it.
|
|
63
|
+
// Returns true only when that signal now passes; a reason with no mapped signal
|
|
64
|
+
// returns false (retained — a genuine config problem, never a false-green).
|
|
65
|
+
// Start with the two observed classes (improve41); extend the table as new
|
|
66
|
+
// degraded reasons gain a corresponding live signal.
|
|
67
|
+
function reasonClearedByHealth(reason, { reachableServing, authed }) {
|
|
68
|
+
const r = String(reason || '');
|
|
69
|
+
if (/claude auth|claude\.json|login/i.test(r)) return authed;
|
|
70
|
+
if (/container|never came up|provisioning failed/i.test(r)) return reachableServing;
|
|
71
|
+
return false;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
// Given a terminal add-job badge and the VM's live health, return the corrected
|
|
75
|
+
// badge — or null when nothing changes. Conservative by construction: clears a
|
|
76
|
+
// degraded reason only when its live health signal now passes; flips a bare
|
|
77
|
+
// `error` add to `done` when the VM is reachable + serving; leaves any reason
|
|
78
|
+
// with no health signal intact (no false-green). The poll loop supplies the live
|
|
79
|
+
// facts (I/O at the edge) and applies the result; the verdict itself is pure.
|
|
80
|
+
// job: { state: 'done'|'error', degraded, degradedReasons }
|
|
81
|
+
// health: { reachable, editorHttpOk, providerAuthState }
|
|
82
|
+
// → null | { state, degraded, degradedReasons, transition }
|
|
83
|
+
function reconcileTerminalAddBadge({ job, health } = {}) {
|
|
84
|
+
if (!job || !health) return null;
|
|
85
|
+
const reachableServing = !!(health.reachable && health.editorHttpOk);
|
|
86
|
+
const authed = health.providerAuthState === 'authenticated';
|
|
87
|
+
|
|
88
|
+
// Bare `error` add (no degraded reasons): flip to `done` once the VM is
|
|
89
|
+
// reachable + serving — the VM3 case (a manual `cloud:up` recovered the editor
|
|
90
|
+
// outside the dashboard's view).
|
|
91
|
+
if (job.state === 'error') {
|
|
92
|
+
const reasons = job.degradedReasons || [];
|
|
93
|
+
if (reasons.length === 0 && reachableServing) {
|
|
94
|
+
return { state: 'done', degraded: false, degradedReasons: [], transition: 'error→done (editor 200)' };
|
|
95
|
+
}
|
|
96
|
+
return null;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
// Terminal `done` that is degraded: clear each reason whose live signal now
|
|
100
|
+
// passes; retain the rest. Drop `degraded` only when no reasons remain.
|
|
101
|
+
if (job.state === 'done' && job.degraded) {
|
|
102
|
+
const reasons = job.degradedReasons || [];
|
|
103
|
+
const remaining = reasons.filter((reason) => !reasonClearedByHealth(reason, { reachableServing, authed }));
|
|
104
|
+
if (remaining.length === reasons.length) return null; // nothing cleared → no-op
|
|
105
|
+
const cleared = reasons.length - remaining.length;
|
|
106
|
+
return {
|
|
107
|
+
state: 'done',
|
|
108
|
+
degraded: remaining.length > 0,
|
|
109
|
+
degradedReasons: remaining,
|
|
110
|
+
transition: remaining.length > 0
|
|
111
|
+
? `degraded: cleared ${cleared} reason(s), ${remaining.length} remain`
|
|
112
|
+
: 'degraded→clear (health restored)',
|
|
113
|
+
};
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
return null;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
// Self-heal for the slug sidecar: given the slug map (`{ "<N>": "<slug>" }`) and
|
|
120
|
+
// the roster (the live VM numbers), return the sorted, de-duped VM numbers whose
|
|
121
|
+
// slug entry should be released because the VM is no longer rostered. A
|
|
122
|
+
// destroyed VM is removed from the roster, so any slug still keyed to a
|
|
123
|
+
// non-rostered number is stranded (e.g. a destroy path that failed to call
|
|
124
|
+
// `removeSlug`) and would otherwise block re-using that name forever. Pure: the
|
|
125
|
+
// caller reads the sidecar + roster (I/O at the edge) and applies `removeSlug`
|
|
126
|
+
// to each returned number. Tolerates a null/absent slug map and an empty roster.
|
|
127
|
+
function pruneOrphanSlugs({ slugs, roster } = {}) {
|
|
128
|
+
const rostered = new Set((roster || []).filter(Number.isInteger));
|
|
129
|
+
const orphans = Object.keys(slugs || {})
|
|
130
|
+
.map((k) => Number(k))
|
|
131
|
+
.filter((n) => Number.isInteger(n) && n > 0 && !rostered.has(n));
|
|
132
|
+
return [...new Set(orphans)].sort((a, b) => a - b);
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
module.exports = { parseRunningFleetVms, computeUnrostered, strandedAdoptDecision, reconcileTerminalAddBadge, pruneOrphanSlugs };
|