@codeyam-editor/codeyam-editor 0.1.4 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/npm/cli.js +5 -1
- package/npm/cloud-branch-switch.js +1 -1
- package/npm/cloud.js +3166 -150
- package/npm/codesign-plan.js +68 -0
- package/npm/container.js +3 -0
- package/npm/disk-pressure.js +78 -0
- package/npm/editor-dev.js +96 -15
- package/npm/editor.js +85 -37
- package/npm/empty-project-dir.js +92 -7
- package/npm/fleet-add-failure.js +252 -0
- package/npm/fleet-asset-version.js +37 -0
- package/npm/fleet-auth-health.js +213 -0
- package/npm/fleet-auth.js +6 -1
- package/npm/fleet-build-data.js +109 -5
- package/npm/fleet-cleanup.js +232 -16
- package/npm/fleet-destroy-safety.js +272 -0
- package/npm/fleet-early-roster.js +87 -0
- package/npm/fleet-effect-poll.js +110 -0
- package/npm/fleet-effect-status.js +33 -32
- package/npm/fleet-env.js +86 -0
- package/npm/fleet-ghcr-secret.js +129 -0
- package/npm/fleet-git-maintenance.js +142 -0
- package/npm/fleet-host-op-limiter.js +68 -0
- package/npm/fleet-host.js +74 -0
- package/npm/fleet-iap-token.js +165 -0
- package/npm/fleet-job-badge.js +1 -1
- package/npm/fleet-launchd.js +171 -0
- package/npm/fleet-live-status.js +155 -0
- package/npm/fleet-memory.js +245 -9
- package/npm/fleet-plans.js +233 -4
- package/npm/fleet-pool.js +75 -16
- package/npm/fleet-prewarm.js +241 -0
- package/npm/fleet-project-drift.js +58 -0
- package/npm/fleet-public-url.js +394 -0
- package/npm/fleet-queue.js +95 -1
- package/npm/fleet-reachability-debounce.js +69 -0
- package/npm/fleet-reachability.js +80 -0
- package/npm/fleet-reconcile.js +43 -1
- package/npm/fleet-reset.js +330 -13
- package/npm/fleet-roster.js +6 -8
- package/npm/fleet-running-plan.js +85 -0
- package/npm/fleet-self-update.js +142 -0
- package/npm/fleet-session-token-cache.js +65 -0
- package/npm/fleet-setup-steps.js +112 -0
- package/npm/fleet-sleep-status.js +33 -0
- package/npm/fleet-status-refs.js +96 -0
- package/npm/fleet-strand.js +181 -0
- package/npm/fleet-stuck-status.js +117 -0
- package/npm/fleet-vm-access.js +404 -0
- package/npm/fleet-vm-probe-target.js +282 -0
- package/npm/fleet-vm-readiness.js +237 -0
- package/npm/fleet-vm-slugs.js +116 -0
- package/npm/google-ingress.js +226 -0
- package/npm/ingress-invoke.js +770 -0
- package/npm/launch-config.js +386 -9
- package/npm/scenario-check.js +880 -37
- package/npm/scenario-handlers.js +154 -30
- package/npm/scenario-interactivity.js +145 -6
- package/npm/scenario-issues.js +13 -1
- package/npm/scenario-metrics.js +116 -1
- package/npm/scenario-mocks.js +144 -21
- package/npm/scenario-playwright.js +649 -38
- package/npm/script-drift.js +152 -0
- package/npm/utils.js +365 -52
- package/npm/vm-urls.js +69 -8
- package/package.json +5 -5
- package/ui/dist/assets/ContainerInspector-DgM0pTAx.js +2 -0
- package/ui/dist/assets/EditorShell-B7Q06uJ_.js +252 -0
- package/ui/dist/assets/EditorShell-DP5UY20G.css +1 -0
- package/ui/dist/assets/{Markdown-BHBqz-Mj.js → Markdown-Chgdf3PW.js} +1 -1
- package/ui/dist/assets/ProjectLauncherScreen-BRrEr8Ng.js +1 -0
- package/ui/dist/assets/ScenarioDataPanel-BeQfa28g.js +9 -0
- package/ui/dist/assets/{ScenarioDataPanel-DYP7pi_n.css → ScenarioDataPanel-C1LukYI6.css} +1 -1
- package/ui/dist/assets/ScenarioDataPanelFullPage-BLqRCxQX.js +1 -0
- package/ui/dist/assets/ScenarioRenderer-CIAxfHPh.js +21 -0
- package/ui/dist/assets/StepsSimulator-CymGgfbu.js +1 -0
- package/ui/dist/assets/index-BygCiJSb.css +1 -0
- package/ui/dist/assets/{index-BuK0lizt.js → index-DCHqRrwO.js} +9 -9
- package/ui/dist/assets/{markdown-v0F0UEt3.js → markdown-C9tbsVHX.js} +1 -1
- package/ui/dist/assets/react-nrLBr15I.js +28 -0
- package/ui/dist/assets/useEvents-S3Qk1Y_d.js +1 -0
- package/ui/dist/assets/useServerIdentity-B7rI0u3O.js +1 -0
- package/ui/dist/codeyam-mark.svg +4 -0
- package/ui/dist/codeyam-wordmark.svg +5 -0
- package/ui/dist/index.html +4 -4
- package/ui/dist/open-source/codeyam-counter.png +0 -0
- package/ui/dist/open-source/el-carot.png +0 -0
- package/ui/dist/open-source/tabcommand.png +0 -0
- package/ui/dist/assets/ContainerInspector-C8EkH4PP.js +0 -2
- package/ui/dist/assets/EditorShell-tMMdarZD.js +0 -187
- package/ui/dist/assets/ScenarioDataPanel-xWYGZOpo.js +0 -5
- package/ui/dist/assets/ScenarioDataPanelFullPage-5G6KuZgw.js +0 -1
- package/ui/dist/assets/ScenarioRenderer-CSdhX1Ak.js +0 -1
- package/ui/dist/assets/StepsSimulator-CRtmlyNZ.js +0 -1
- package/ui/dist/assets/index-BGqairFV.css +0 -1
- package/ui/dist/assets/react-CSS0HapR.js +0 -28
- package/ui/dist/assets/useEvents-DggWrtHe.js +0 -1
package/npm/fleet-memory.js
CHANGED
|
@@ -20,38 +20,167 @@ const MEMORY_DEFAULTS = Object.freeze({
|
|
|
20
20
|
drainCheckIntervalMs: 5000,
|
|
21
21
|
snapshotIntervalMs: 30_000,
|
|
22
22
|
snapshotMaxAgeMs: 5 * 60 * 1000,
|
|
23
|
+
// Fraction of the cgroup's own memory.high at which the drain engages. A
|
|
24
|
+
// FRACTION, not an absolute MB, because the absolute number silently
|
|
25
|
+
// desynchronizes from the unit file: codeyam-dashboard.service was raised
|
|
26
|
+
// from MemoryHigh=1.5G to 2.5G on 2026-07-28, while the operator host kept
|
|
27
|
+
// throttling at 1536MB — a hardcoded threshold cannot track that, a fraction
|
|
28
|
+
// of the value the kernel actually reports does. 0.80 of the 1536MB limit in
|
|
29
|
+
// force during the 2026-07-30 wedge is 1229MB, comfortably ahead of the
|
|
30
|
+
// 1309MB (85%) geometry at which the event loop had already stalled.
|
|
31
|
+
drainHighFraction: 0.80,
|
|
32
|
+
// Fraction of memory.high at which the drain is no longer enough and the
|
|
33
|
+
// process restarts itself to actually reclaim. Sits in a deliberately narrow
|
|
34
|
+
// band ABOVE the drain and BELOW the measured stall: 0.83 of 1536MB is
|
|
35
|
+
// 1275MB, ~34MB before the 1309MB geometry that wedged the dashboard. The
|
|
36
|
+
// band is tight because the evidence leaves little room — reclaim was already
|
|
37
|
+
// continuous at 85%.
|
|
38
|
+
restartHighFraction: 0.83,
|
|
39
|
+
// Crash-loop guard for the self-restart remedy. Counted over a rolling hour
|
|
40
|
+
// and persisted across restarts by the caller (an in-memory counter would
|
|
41
|
+
// reset on every restart and so could never limit anything).
|
|
42
|
+
maxRestartsPerHour: 3,
|
|
43
|
+
restartWindowMs: 60 * 60 * 1000,
|
|
23
44
|
});
|
|
24
45
|
|
|
25
46
|
/** Should the dashboard refuse to admit new heavy jobs?
|
|
26
47
|
*
|
|
27
|
-
*
|
|
28
|
-
*
|
|
29
|
-
* misconfigured env var degrades to "no drain" instead of silently rejecting
|
|
30
|
-
* every heavy job. Bytes on both sides so the caller doesn't have to pick a
|
|
31
|
-
* unit; server.js passes MB→bytes once at module init.
|
|
48
|
+
* Decides from the CGROUP total when a cgroup reading is available, and only
|
|
49
|
+
* falls back to this process's RSS when it is not.
|
|
32
50
|
*
|
|
33
|
-
*
|
|
34
|
-
|
|
35
|
-
|
|
51
|
+
* Why the cgroup and not RSS: RSS is the wrong layer. The kernel throttles on
|
|
52
|
+
* the cgroup's memory.current against memory.high, and the cgroup runs
|
|
53
|
+
* 100-230MB ABOVE this process's RSS (109MB fresh, 226MB wedged, measured
|
|
54
|
+
* 2026-07-30). Sampling RSS therefore under-reads the condition by design —
|
|
55
|
+
* readCgroupMemory() in server.js has said so in a comment since improve37,
|
|
56
|
+
* but the lesson was only ever applied to operatorFaultVerdict, never carried
|
|
57
|
+
* back here. During the 2026-07-30 wedge RSS read 1083MB against a 1500MB
|
|
58
|
+
* threshold while the cgroup sat at 1309MB of a 1536MB limit, already emitting
|
|
59
|
+
* 42546 reclaim events: the drain could not fire before the stall, and no
|
|
60
|
+
* [memory] line was ever logged across the whole leak-to-wedge cycle. No RSS
|
|
61
|
+
* number fixes that, because the RSS-to-cgroup gap is itself variable.
|
|
62
|
+
*
|
|
63
|
+
* Contracts preserved exactly from the RSS-only version:
|
|
64
|
+
* • strict greater-than at the threshold, so a reading pinned at the limit
|
|
65
|
+
* cannot oscillate the drain on and off each tick;
|
|
66
|
+
* • a non-positive or non-finite drainThresholdBytes disables the drain
|
|
67
|
+
* entirely and returns false. That is the documented operator escape
|
|
68
|
+
* hatch (MEMORY_DRAIN_THRESHOLD_MB=0) and it is checked FIRST, so it
|
|
69
|
+
* disables the cgroup path too, not just the RSS one;
|
|
70
|
+
* • any unusable input degrades to "no drain", never to "always drain" — a
|
|
71
|
+
* dev laptop with no cgroup must not self-throttle.
|
|
72
|
+
*
|
|
73
|
+
* memoryHigh of null/0 means an unlimited cgroup (memory.high == 'max') or no
|
|
74
|
+
* cgroup at all (macOS, cgroup v1, a container without cgroup v2). Either way
|
|
75
|
+
* it contributes no pressure signal and the RSS fallback decides.
|
|
76
|
+
*
|
|
77
|
+
* Pure; no I/O — the caller supplies the numbers, as it always has. */
|
|
78
|
+
function decideMemoryDrain({
|
|
79
|
+
rssBytes,
|
|
80
|
+
drainThresholdBytes,
|
|
81
|
+
memoryCurrent,
|
|
82
|
+
memoryHigh,
|
|
83
|
+
drainHighFraction,
|
|
84
|
+
} = {}) {
|
|
36
85
|
const cap = Number(drainThresholdBytes);
|
|
37
86
|
if (!Number.isFinite(cap) || cap <= 0) return false;
|
|
87
|
+
|
|
88
|
+
// Preferred path: the quantity the kernel actually limits.
|
|
89
|
+
const current = Number(memoryCurrent);
|
|
90
|
+
const high = Number(memoryHigh);
|
|
91
|
+
const asked = Number(drainHighFraction);
|
|
92
|
+
const fraction = (Number.isFinite(asked) && asked > 0)
|
|
93
|
+
? asked
|
|
94
|
+
: MEMORY_DEFAULTS.drainHighFraction;
|
|
95
|
+
if (Number.isFinite(current) && current >= 0 && Number.isFinite(high) && high > 0) {
|
|
96
|
+
return current > high * fraction;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
// Documented fallback: no cgroup reading available, so keep the pre-existing
|
|
100
|
+
// RSS behavior rather than losing the drain entirely off-cgroup.
|
|
101
|
+
const rss = Number(rssBytes);
|
|
38
102
|
if (!Number.isFinite(rss) || rss < 0) return false;
|
|
39
103
|
return rss > cap;
|
|
40
104
|
}
|
|
41
105
|
|
|
106
|
+
/** Should the dashboard restart itself to reclaim memory?
|
|
107
|
+
*
|
|
108
|
+
* The drain is an admission brake: it refuses NEW heavy jobs. On an idle-but-
|
|
109
|
+
* leaking process it changes nothing — during the 2026-07-30 wedge no job had
|
|
110
|
+
* run since Jul 28, so even a correctly-firing drain would not have helped.
|
|
111
|
+
* Something has to actually reclaim, and a restart is cheap here: systemd's
|
|
112
|
+
* Restart=always brings the process straight back and flushStateSnapshot has
|
|
113
|
+
* already persisted {jobs, vms, orphans} within the last 30s for
|
|
114
|
+
* restoreFromSnapshot to rehydrate.
|
|
115
|
+
*
|
|
116
|
+
* Returns { restart, reason } rather than a bare boolean so the caller can log
|
|
117
|
+
* WHY it declined — silence was the original failure mode, where the
|
|
118
|
+
* operator's first signal was an apparently-dead fleet.
|
|
119
|
+
*
|
|
120
|
+
* Refuses, in order:
|
|
121
|
+
* • the drain escape hatch is set (drainThresholdBytes <= 0) — disabling the
|
|
122
|
+
* drain disables the whole memory remedy, including this;
|
|
123
|
+
* • no usable cgroup reading — an off-cgroup dev laptop never self-restarts;
|
|
124
|
+
* • below the high-water fraction;
|
|
125
|
+
* • a job is in flight — a restart mid-provision would strand it;
|
|
126
|
+
* • the rolling-window restart budget is spent, which is the crash-loop guard.
|
|
127
|
+
*
|
|
128
|
+
* restartsInWindow is supplied by the caller because it must be PERSISTED: an
|
|
129
|
+
* in-memory counter resets on the very restart it is meant to be limiting.
|
|
130
|
+
*
|
|
131
|
+
* Pure; no I/O. */
|
|
132
|
+
function decideMemoryRestart({
|
|
133
|
+
memoryCurrent,
|
|
134
|
+
memoryHigh,
|
|
135
|
+
restartHighFraction,
|
|
136
|
+
drainThresholdBytes,
|
|
137
|
+
jobInFlight,
|
|
138
|
+
restartsInWindow,
|
|
139
|
+
maxRestartsPerWindow,
|
|
140
|
+
} = {}) {
|
|
141
|
+
const cap = Number(drainThresholdBytes);
|
|
142
|
+
if (!Number.isFinite(cap) || cap <= 0) return { restart: false, reason: 'memory-remedy-disabled' };
|
|
143
|
+
|
|
144
|
+
const current = Number(memoryCurrent);
|
|
145
|
+
const high = Number(memoryHigh);
|
|
146
|
+
if (!Number.isFinite(current) || current < 0) return { restart: false, reason: 'no-cgroup-reading' };
|
|
147
|
+
if (!Number.isFinite(high) || high <= 0) return { restart: false, reason: 'no-cgroup-limit' };
|
|
148
|
+
|
|
149
|
+
const asked = Number(restartHighFraction);
|
|
150
|
+
const fraction = (Number.isFinite(asked) && asked > 0)
|
|
151
|
+
? asked
|
|
152
|
+
: MEMORY_DEFAULTS.restartHighFraction;
|
|
153
|
+
if (!(current > high * fraction)) return { restart: false, reason: 'below-high-water' };
|
|
154
|
+
|
|
155
|
+
if (jobInFlight) return { restart: false, reason: 'job-in-flight' };
|
|
156
|
+
|
|
157
|
+
const askedMax = Number(maxRestartsPerWindow);
|
|
158
|
+
const max = Number.isFinite(askedMax) ? askedMax : MEMORY_DEFAULTS.maxRestartsPerHour;
|
|
159
|
+
if (max <= 0) return { restart: false, reason: 'self-restart-disabled' };
|
|
160
|
+
const used = Number(restartsInWindow);
|
|
161
|
+
if (Number.isFinite(used) && used >= max) return { restart: false, reason: 'rate-limited' };
|
|
162
|
+
|
|
163
|
+
return { restart: true, reason: 'over-high-water' };
|
|
164
|
+
}
|
|
165
|
+
|
|
42
166
|
/** Build the serialized state snapshot envelope.
|
|
43
167
|
*
|
|
44
168
|
* Keeps only the fields the dashboard needs to recover after a launchd
|
|
45
169
|
* respawn: in-flight jobs, last VM poll merge, operator-visible orphans.
|
|
46
170
|
* Times come from the caller so the same input yields the same output
|
|
47
171
|
* (snapshot equality testable). Pure; no I/O. */
|
|
48
|
-
function buildStateSnapshot({ jobs, vms, orphans, nowIso }) {
|
|
172
|
+
function buildStateSnapshot({ jobs, vms, orphans, orphanedIngress, nowIso }) {
|
|
49
173
|
return {
|
|
50
174
|
version: 1,
|
|
51
175
|
timestamp: nowIso,
|
|
52
176
|
jobs: jobs || {},
|
|
53
177
|
vms: vms || {},
|
|
54
178
|
orphans: orphans || {},
|
|
179
|
+
// Fleet-level orphaned-LB-ingress attention items keyed by destroyed VM
|
|
180
|
+
// number (vm--destroy-fully-tears-down-lb-ingress). Persisted so a genuine
|
|
181
|
+
// orphan survives a dashboard restart — the VM is gone, so it can't be
|
|
182
|
+
// re-derived from live poll state the way jobs/vms are.
|
|
183
|
+
orphanedIngress: orphanedIngress || {},
|
|
55
184
|
};
|
|
56
185
|
}
|
|
57
186
|
|
|
@@ -145,12 +274,119 @@ function restoreFromSnapshot({ snapshot, maxAgeMs, nowIso, sidecarKeys } = {}) {
|
|
|
145
274
|
jobs,
|
|
146
275
|
vms: (snapshot.vms && typeof snapshot.vms === 'object') ? snapshot.vms : {},
|
|
147
276
|
orphans: (snapshot.orphans && typeof snapshot.orphans === 'object') ? snapshot.orphans : {},
|
|
277
|
+
orphanedIngress: (snapshot.orphanedIngress && typeof snapshot.orphanedIngress === 'object')
|
|
278
|
+
? snapshot.orphanedIngress : {},
|
|
148
279
|
};
|
|
149
280
|
}
|
|
150
281
|
|
|
282
|
+
/** Render the three memory layers as one line.
|
|
283
|
+
*
|
|
284
|
+
* Deliberately reports RSS, the cgroup total, AND the gap between them
|
|
285
|
+
* together. Reading any one of those alone is how the drain sat on the wrong
|
|
286
|
+
* layer for so long: RSS said 1083MB and looked healthy against a 1500MB
|
|
287
|
+
* threshold while the cgroup was at 1309MB of 1536MB and already in continuous
|
|
288
|
+
* reclaim. The gap is not noise, it is the diagnostic.
|
|
289
|
+
*
|
|
290
|
+
* Degrades to 'cgroup=unavailable' rather than printing zeros off-cgroup, so a
|
|
291
|
+
* laptop log cannot be misread as a cgroup at 0MB.
|
|
292
|
+
*
|
|
293
|
+
* Pure; no I/O. */
|
|
294
|
+
function formatMemoryText({ rssBytes, memoryCurrent, memoryHigh } = {}) {
|
|
295
|
+
const mb = (b) => (Number(b) / 1024 / 1024).toFixed(0);
|
|
296
|
+
const current = Number(memoryCurrent);
|
|
297
|
+
const high = Number(memoryHigh);
|
|
298
|
+
const parts = [`RSS=${Number.isFinite(Number(rssBytes)) ? mb(rssBytes) : '?'}MB`];
|
|
299
|
+
if (Number.isFinite(current) && current > 0) {
|
|
300
|
+
parts.push(`cgroup=${mb(current)}MB`);
|
|
301
|
+
if (Number.isFinite(high) && high > 0) {
|
|
302
|
+
parts.push(`high=${mb(high)}MB`);
|
|
303
|
+
parts.push(`used=${((current / high) * 100).toFixed(0)}%`);
|
|
304
|
+
if (Number.isFinite(Number(rssBytes))) parts.push(`gap=${mb(current - Number(rssBytes))}MB`);
|
|
305
|
+
} else {
|
|
306
|
+
parts.push('high=unlimited');
|
|
307
|
+
}
|
|
308
|
+
} else {
|
|
309
|
+
parts.push('cgroup=unavailable');
|
|
310
|
+
}
|
|
311
|
+
return parts.join(' ');
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
/** Is any job doing real work right now?
|
|
315
|
+
*
|
|
316
|
+
* Gates the self-restart: reclaiming memory by exiting is cheap when the fleet
|
|
317
|
+
* is quiet and expensive mid-provision, where it would strand the job.
|
|
318
|
+
*
|
|
319
|
+
* `recovering` counts as in flight even though nothing is executing — the
|
|
320
|
+
* startup reap has not yet resolved it against gcloud, and restarting again
|
|
321
|
+
* would only defer it a second time. `queued` does NOT count: restoreFromSnapshot
|
|
322
|
+
* drops queued jobs on restore anyway, and a queued job has done no work worth
|
|
323
|
+
* protecting. The dashboard's own self-update counts, being a heavy job by any
|
|
324
|
+
* other name.
|
|
325
|
+
*
|
|
326
|
+
* Pure; no I/O — the caller passes state.jobs / state.dashboardUpdate. */
|
|
327
|
+
function hasJobInFlight({ jobs, dashboardUpdate } = {}) {
|
|
328
|
+
const busy = (j) => !!j && (j.state === 'running' || j.state === 'recovering');
|
|
329
|
+
const anyJob = Object.values((jobs && typeof jobs === 'object') ? jobs : {}).some(busy);
|
|
330
|
+
return anyJob || !!(dashboardUpdate && dashboardUpdate.state === 'running');
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
/** Should this escalation level be announced to the journal?
|
|
334
|
+
*
|
|
335
|
+
* Announce on a genuine level TRANSITION only. The check runs every 5s, so
|
|
336
|
+
* announcing unconditionally would bury the journal, while announcing nothing
|
|
337
|
+
* is what let the original leak run to a wedge with no warning — no [memory]
|
|
338
|
+
* line was emitted even once across the whole leak-to-wedge cycle.
|
|
339
|
+
*
|
|
340
|
+
* The quiet levels are the steady states that carry no news: healthy operation
|
|
341
|
+
* (below-high-water) and the structural "this host has no cgroup / the remedy
|
|
342
|
+
* is switched off" conditions, which would otherwise announce once per boot on
|
|
343
|
+
* every dev laptop for no reason.
|
|
344
|
+
*
|
|
345
|
+
* Pure; no I/O. */
|
|
346
|
+
const QUIET_MEMORY_LEVELS = Object.freeze([
|
|
347
|
+
'below-high-water',
|
|
348
|
+
'no-cgroup-reading',
|
|
349
|
+
'no-cgroup-limit',
|
|
350
|
+
'memory-remedy-disabled',
|
|
351
|
+
]);
|
|
352
|
+
|
|
353
|
+
function shouldAnnounceMemoryLevel({ level, lastLevel } = {}) {
|
|
354
|
+
if (!level) return false;
|
|
355
|
+
if (level === lastLevel) return false;
|
|
356
|
+
return !QUIET_MEMORY_LEVELS.includes(level);
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
/** Keep only the restart timestamps still inside the rolling window.
|
|
360
|
+
*
|
|
361
|
+
* Split out from decideMemoryRestart so the "how many restarts have I already
|
|
362
|
+
* spent" bookkeeping is testable on its own, and so the caller can prune the
|
|
363
|
+
* persisted file with the same rule the decision uses. Non-parseable entries
|
|
364
|
+
* are dropped rather than throwing — a hand-edited or partially-written file
|
|
365
|
+
* degrades to "fewer restarts counted", which fails toward allowing a restart
|
|
366
|
+
* that reclaims memory rather than wedging.
|
|
367
|
+
*
|
|
368
|
+
* Pure; no I/O. */
|
|
369
|
+
function pruneRestartWindow({ timestamps, nowIso, windowMs } = {}) {
|
|
370
|
+
const now = Date.parse(nowIso);
|
|
371
|
+
const span = Number(windowMs);
|
|
372
|
+
if (!Number.isFinite(now)) return [];
|
|
373
|
+
const cap = (Number.isFinite(span) && span > 0) ? span : MEMORY_DEFAULTS.restartWindowMs;
|
|
374
|
+
return (Array.isArray(timestamps) ? timestamps : [])
|
|
375
|
+
.filter((t) => {
|
|
376
|
+
const ms = Date.parse(t);
|
|
377
|
+
return Number.isFinite(ms) && now - ms <= cap && ms <= now;
|
|
378
|
+
});
|
|
379
|
+
}
|
|
380
|
+
|
|
151
381
|
module.exports = {
|
|
152
382
|
MEMORY_DEFAULTS,
|
|
383
|
+
QUIET_MEMORY_LEVELS,
|
|
153
384
|
decideMemoryDrain,
|
|
385
|
+
decideMemoryRestart,
|
|
386
|
+
formatMemoryText,
|
|
387
|
+
hasJobInFlight,
|
|
388
|
+
shouldAnnounceMemoryLevel,
|
|
389
|
+
pruneRestartWindow,
|
|
154
390
|
buildStateSnapshot,
|
|
155
391
|
decodeStateSnapshot,
|
|
156
392
|
restoreFromSnapshot,
|
package/npm/fleet-plans.js
CHANGED
|
@@ -1,7 +1,16 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
|
-
* Indexes the
|
|
4
|
+
* Indexes the reachable VMs by every session id that identifies them: their
|
|
5
|
+
* live `sessionId` (from /api/session-info) AND their workflow-lock
|
|
6
|
+
* `holderSessionId` (from /api/step's editor-step.json). A claim maps to a VM
|
|
7
|
+
* through this index, so including the holder is what lets a claim attribute to
|
|
8
|
+
* the executing VM even after an editor restart drops the live sessionId (the
|
|
9
|
+
* VM reports sessionId:null but editor-step.json still names the holder).
|
|
10
|
+
*
|
|
11
|
+
* A live `sessionId` always wins a key collision over a `holderSessionId`: the
|
|
12
|
+
* live registration is the authoritative current identity, so it is indexed in
|
|
13
|
+
* a first pass and the holder pass only fills in ids not already claimed.
|
|
5
14
|
* @param {Array|Object} vms The merged VM states on the dashboard.
|
|
6
15
|
* @returns {Object} A map of sessionId -> VM number (n).
|
|
7
16
|
*/
|
|
@@ -14,6 +23,39 @@ function indexVmsBySessionId(vms) {
|
|
|
14
23
|
map[vm.sessionId] = vm.n;
|
|
15
24
|
}
|
|
16
25
|
}
|
|
26
|
+
// Second pass: holderSessionId fills in only ids a live sessionId didn't
|
|
27
|
+
// already claim, so a live registration never loses to a stale holder.
|
|
28
|
+
for (const vm of list) {
|
|
29
|
+
if (vm && vm.holderSessionId && vm.n !== undefined && map[vm.holderSessionId] === undefined) {
|
|
30
|
+
map[vm.holderSessionId] = vm.n;
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
return map;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Indexes the reachable VMs by their fleet machine name, so a claim that only
|
|
38
|
+
* carries `claimedBy.machine` (and no resolvable sessionId) can still attribute
|
|
39
|
+
* to a VM. A VM's machine name is `<instancePrefix><n>` (the GCE instance name
|
|
40
|
+
* the editor stamps into CODEYAM_MACHINE), so the map is derived, not probed.
|
|
41
|
+
*
|
|
42
|
+
* This is the fallback for backend/HTTP-"Run" claims: the editor stamps
|
|
43
|
+
* `claimedBy.sessionId` only for a PTY-attached `plan-select`, so a
|
|
44
|
+
* server-driven autonomous claim has an empty sessionId and never resolves via
|
|
45
|
+
* {@link indexVmsBySessionId}. Its machine name always resolves.
|
|
46
|
+
* @param {Array|Object} vms The merged VM states on the dashboard.
|
|
47
|
+
* @param {string} instancePrefix The fleet instance-name prefix (INSTANCE_PREFIX).
|
|
48
|
+
* @returns {Object} A map of machine name -> VM number (n).
|
|
49
|
+
*/
|
|
50
|
+
function indexVmsByMachine(vms, instancePrefix) {
|
|
51
|
+
const map = {};
|
|
52
|
+
if (!vms || !instancePrefix) return map;
|
|
53
|
+
const list = Array.isArray(vms) ? vms : Object.values(vms);
|
|
54
|
+
for (const vm of list) {
|
|
55
|
+
if (vm && vm.n !== undefined) {
|
|
56
|
+
map[`${instancePrefix}${vm.n}`] = vm.n;
|
|
57
|
+
}
|
|
58
|
+
}
|
|
17
59
|
return map;
|
|
18
60
|
}
|
|
19
61
|
|
|
@@ -36,9 +78,11 @@ function unsatisfiedDeps(dependsOn, completedSlugs) {
|
|
|
36
78
|
* @param {Object[]} params.claims Array of PlanClaim objects from /api/plan-claims.
|
|
37
79
|
* @param {Set|string[]} params.completedSlugs Completed plan slugs.
|
|
38
80
|
* @param {Object} params.vmsBySessionId Map of sessionId -> VM number.
|
|
81
|
+
* @param {Object} [params.vmsByMachine] Map of machine name -> VM number, the
|
|
82
|
+
* fallback for backend claims whose sessionId is empty.
|
|
39
83
|
* @returns {Object[]} Annotated plan rows.
|
|
40
84
|
*/
|
|
41
|
-
function resolvePlanRows({ plans, claims, completedSlugs, vmsBySessionId }) {
|
|
85
|
+
function resolvePlanRows({ plans, claims, completedSlugs, vmsBySessionId, vmsByMachine }) {
|
|
42
86
|
const list = Array.isArray(plans) ? plans : [];
|
|
43
87
|
const claimsList = Array.isArray(claims) ? claims : [];
|
|
44
88
|
|
|
@@ -60,8 +104,14 @@ function resolvePlanRows({ plans, claims, completedSlugs, vmsBySessionId }) {
|
|
|
60
104
|
if (claim) {
|
|
61
105
|
const claimedBy = claim.claimedBy || claim.claimed_by || {};
|
|
62
106
|
const sessionId = claimedBy.sessionId || claimedBy.session_id || null;
|
|
63
|
-
|
|
64
|
-
|
|
107
|
+
// Prefer the authoritative live sessionId match; fall back to the claim's
|
|
108
|
+
// machine name (backend claims have an empty sessionId, so this is the
|
|
109
|
+
// only key that resolves them to a VM).
|
|
110
|
+
let vm = (sessionId && vmsBySessionId && vmsBySessionId[sessionId] !== undefined) ? vmsBySessionId[sessionId] : null;
|
|
111
|
+
if (vm === null && claimedBy.machine && vmsByMachine && vmsByMachine[claimedBy.machine] !== undefined) {
|
|
112
|
+
vm = vmsByMachine[claimedBy.machine];
|
|
113
|
+
}
|
|
114
|
+
|
|
65
115
|
claimInfo = {
|
|
66
116
|
vm,
|
|
67
117
|
machine: claimedBy.machine || null,
|
|
@@ -88,9 +138,188 @@ const REQUIRED_PLAN_ROW_FIELDS = [
|
|
|
88
138
|
'slug', 'title', 'mode', 'order', 'prefix', 'blockedBy', 'claim'
|
|
89
139
|
];
|
|
90
140
|
|
|
141
|
+
/**
|
|
142
|
+
* Decides which data source a project's Queued Plans panel should read from.
|
|
143
|
+
* The dashboard's own project (codeyam-editor) reads plans fresh from origin
|
|
144
|
+
* via the laptop checkout; every other project keeps the VM probe.
|
|
145
|
+
* @param {Object} params
|
|
146
|
+
* @param {string} params.project The project name for the panel.
|
|
147
|
+
* @param {?string} params.selfHostProject The dashboard's own project name.
|
|
148
|
+
* @returns {'origin'|'vm'} The source to use.
|
|
149
|
+
*/
|
|
150
|
+
function planSourceForProject({ project, selfHostProject }) {
|
|
151
|
+
return selfHostProject && project === selfHostProject ? 'origin' : 'vm';
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/**
|
|
155
|
+
* Builds the static (non-ticking) part of a Queued Plans panel header label so
|
|
156
|
+
* the data source and its staleness risk are always visible. Origin-sourced
|
|
157
|
+
* panels name the branch + short sha; VM-sourced panels carry the "may lag
|
|
158
|
+
* origin" caveat. The relative age ("· 8s ago") is appended by the frontend's
|
|
159
|
+
* own ago ticker, so this returns the static prefix only.
|
|
160
|
+
* @param {Object} params
|
|
161
|
+
* @param {'origin'|'vm'} params.source Where the panel's data came from.
|
|
162
|
+
* @param {?string} params.ref The git ref (e.g. "origin/<branch>") for origin.
|
|
163
|
+
* @param {?string} params.sha The full commit sha for origin.
|
|
164
|
+
* @param {?number} params.fromVm The representative VM number for vm source.
|
|
165
|
+
* @param {?('stale-binary'|'fetch-failed'|'parse-error'|'unknown')} params.originError
|
|
166
|
+
* When the self-host origin path was *attempted* and failed, the classified
|
|
167
|
+
* reason it fell back to the VM probe. Absent/null for ordinary VM-sourced
|
|
168
|
+
* projects (which were never meant to read origin).
|
|
169
|
+
* @returns {string} The static header annotation.
|
|
170
|
+
*/
|
|
171
|
+
function planPanelSourceLabel({ source, ref, sha, fromVm, originError }) {
|
|
172
|
+
if (source === 'origin') {
|
|
173
|
+
const shortSha = sha ? String(sha).slice(0, 7) : '?';
|
|
174
|
+
const where = ref || 'origin';
|
|
175
|
+
return `${where} @ ${shortSha}`;
|
|
176
|
+
}
|
|
177
|
+
// VM-sourced: surface the residual lag risk the origin path eliminates.
|
|
178
|
+
const vm = (fromVm !== null && fromVm !== undefined) ? `VM-${fromVm}` : 'VM';
|
|
179
|
+
if (originError) {
|
|
180
|
+
// The origin path was attempted and failed: this is a *degraded* panel, not
|
|
181
|
+
// a normal VM-sourced one. Name the failure class so it is self-diagnosing.
|
|
182
|
+
return `via ${vm} · origin unavailable: ${originErrorPhrase(originError)}`;
|
|
183
|
+
}
|
|
184
|
+
return `via ${vm} · may lag origin`;
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
/**
|
|
188
|
+
* Maps a classified origin-failure bucket to a short operator-facing phrase for
|
|
189
|
+
* the panel header. Mirrors the bucket set of classifyOriginError; an unknown
|
|
190
|
+
* bucket degrades to the generic "origin read failed" phrase.
|
|
191
|
+
* @param {string} originError The classified bucket.
|
|
192
|
+
* @returns {string} A short human-readable phrase.
|
|
193
|
+
*/
|
|
194
|
+
function originErrorPhrase(originError) {
|
|
195
|
+
switch (originError) {
|
|
196
|
+
case 'stale-binary':
|
|
197
|
+
return 'stale editor binary';
|
|
198
|
+
case 'fetch-failed':
|
|
199
|
+
return 'git fetch failed';
|
|
200
|
+
case 'parse-error':
|
|
201
|
+
return 'bad snapshot output';
|
|
202
|
+
default:
|
|
203
|
+
return 'origin read failed';
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
/**
|
|
208
|
+
* Classifies a caught origin-path error message into a small, stable set of
|
|
209
|
+
* buckets the operator can act on, rather than surfacing a noisy raw error
|
|
210
|
+
* (which can carry paths/branch names) in the panel header. Pure so it is
|
|
211
|
+
* unit-testable.
|
|
212
|
+
* - `unrecognized subcommand` → 'stale-binary' (CLI predates the command)
|
|
213
|
+
* - leading `git fetch origin` → 'fetch-failed' (couldn't reach origin)
|
|
214
|
+
* - no-JSON / malformed snapshot → 'parse-error' (plans-snapshot output)
|
|
215
|
+
* - anything else → 'unknown'
|
|
216
|
+
* @param {?string} message The caught error's message text.
|
|
217
|
+
* @returns {'stale-binary'|'fetch-failed'|'parse-error'|'unknown'}
|
|
218
|
+
*/
|
|
219
|
+
function classifyOriginError(message) {
|
|
220
|
+
const msg = typeof message === 'string' ? message : '';
|
|
221
|
+
if (msg.includes('unrecognized subcommand')) return 'stale-binary';
|
|
222
|
+
if (/^git fetch origin\b/.test(msg)) return 'fetch-failed';
|
|
223
|
+
if (msg.includes('plans-snapshot produced no JSON') ||
|
|
224
|
+
msg.includes('plans-snapshot JSON parse')) return 'parse-error';
|
|
225
|
+
return 'unknown';
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
/**
|
|
229
|
+
* True when an origin-fetch error means the branch no longer exists on the
|
|
230
|
+
* remote — it was merged and deleted, so `git fetch origin <branch>` fails with
|
|
231
|
+
* git's `couldn't find remote ref <branch>`. This is NOT a transient fetch
|
|
232
|
+
* failure (network / auth): the branch is *gone*, so re-fetching it every poll
|
|
233
|
+
* cycle is pure noise. The dashboard uses this to record the branch as gone,
|
|
234
|
+
* fall back to the VM probe quietly, and stop logging an error every cycle —
|
|
235
|
+
* the difference between a healthy fleet that *reads* broken in the log and one
|
|
236
|
+
* that reads calm. Pure so it is unit-testable.
|
|
237
|
+
* @param {?string} message The caught fetch-error message text.
|
|
238
|
+
* @returns {boolean}
|
|
239
|
+
*/
|
|
240
|
+
function isRemoteRefGone(message) {
|
|
241
|
+
const msg = typeof message === 'string' ? message : '';
|
|
242
|
+
return /couldn't find remote ref/i.test(msg);
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
/**
|
|
246
|
+
* Parses the stdout of `editor plans-snapshot --format json` into the snapshot
|
|
247
|
+
* envelope the dashboard consumes. The pure half of snapshotPlansFromOrigin:
|
|
248
|
+
* extracts the JSON object (the CLI may prefix a dev-build banner), parses it,
|
|
249
|
+
* and normalizes missing arrays to []. Throws a descriptive error on no-JSON
|
|
250
|
+
* or malformed-JSON so the caller can fall back to the VM probe.
|
|
251
|
+
* @param {string} stdout The plans-snapshot CLI stdout.
|
|
252
|
+
* @returns {{ref: string, sha: string, plans: Object[], completedSlugs: string[]}}
|
|
253
|
+
*/
|
|
254
|
+
function parsePlansSnapshotStdout(stdout) {
|
|
255
|
+
const m = stdout && stdout.match(/\{[\s\S]*\}/);
|
|
256
|
+
if (!m) throw new Error('plans-snapshot produced no JSON');
|
|
257
|
+
let data;
|
|
258
|
+
try {
|
|
259
|
+
data = JSON.parse(m[0]);
|
|
260
|
+
} catch (e) {
|
|
261
|
+
throw new Error(`plans-snapshot JSON parse: ${e.message}`);
|
|
262
|
+
}
|
|
263
|
+
return {
|
|
264
|
+
ref: data.ref,
|
|
265
|
+
sha: data.sha,
|
|
266
|
+
plans: data.plans || [],
|
|
267
|
+
completedSlugs: data.completedSlugs || [],
|
|
268
|
+
};
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
/**
|
|
272
|
+
* Unions the `/api/plan-claims` responses of several VMs in a group into one
|
|
273
|
+
* claim list, deduped by `(slug, uuid)`.
|
|
274
|
+
*
|
|
275
|
+
* Sourcing a group's claims from a single representative VM is fragile: that VM
|
|
276
|
+
* may lag, be unreachable, or — the shape this fixes — be the very VM holding
|
|
277
|
+
* the claim, whose own display filter is the one place the claim can be
|
|
278
|
+
* suppressed. Every *other* VM in the group carries the same ref as a plain
|
|
279
|
+
* `remote` claim, so unioning surfaces it regardless.
|
|
280
|
+
*
|
|
281
|
+
* When the same `(slug, uuid)` arrives from several VMs, the most-informed copy
|
|
282
|
+
* wins: a same-machine (`local`) view beats a `remote` one (only the holder
|
|
283
|
+
* knows its own session liveness), and among equals a live claim beats a
|
|
284
|
+
* not-live one. `fromVm` is stamped from the VM that supplied the winner so the
|
|
285
|
+
* panel's provenance stays meaningful.
|
|
286
|
+
* @param {Array<{vm: number, claims: Object[]}>} claimLists Per-VM claim responses.
|
|
287
|
+
* @returns {Object[]} The deduped union, each claim carrying its winning `fromVm`.
|
|
288
|
+
*/
|
|
289
|
+
function unionClaims(claimLists) {
|
|
290
|
+
const lists = Array.isArray(claimLists) ? claimLists : [];
|
|
291
|
+
// Rank a claim by how authoritative its view is. Higher wins a tie on key.
|
|
292
|
+
const rank = claim => {
|
|
293
|
+
const local = claim && claim.source === 'local' ? 2 : 0;
|
|
294
|
+
const alive = claim && claim.sessionAlive === true ? 1 : 0;
|
|
295
|
+
return local + alive;
|
|
296
|
+
};
|
|
297
|
+
const byKey = new Map();
|
|
298
|
+
for (const entry of lists) {
|
|
299
|
+
if (!entry || !Array.isArray(entry.claims)) continue;
|
|
300
|
+
for (const claim of entry.claims) {
|
|
301
|
+
if (!claim || !claim.slug) continue;
|
|
302
|
+
const key = `${claim.slug}${claim.uuid || ''}`;
|
|
303
|
+
const candidate = { ...claim, fromVm: entry.vm };
|
|
304
|
+
const held = byKey.get(key);
|
|
305
|
+
if (!held || rank(candidate) > rank(held)) {
|
|
306
|
+
byKey.set(key, candidate);
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
return Array.from(byKey.values());
|
|
311
|
+
}
|
|
312
|
+
|
|
91
313
|
module.exports = {
|
|
92
314
|
indexVmsBySessionId,
|
|
315
|
+
indexVmsByMachine,
|
|
316
|
+
unionClaims,
|
|
93
317
|
unsatisfiedDeps,
|
|
94
318
|
resolvePlanRows,
|
|
319
|
+
planSourceForProject,
|
|
320
|
+
planPanelSourceLabel,
|
|
321
|
+
classifyOriginError,
|
|
322
|
+
isRemoteRefGone,
|
|
323
|
+
parsePlansSnapshotStdout,
|
|
95
324
|
REQUIRED_PLAN_ROW_FIELDS
|
|
96
325
|
};
|