mandrel-platform 0.21.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/scripts/check-runner-health.mjs +469 -0
- package/scripts/check-runner-health.test.mjs +389 -0
- package/scripts/platform-sync.test.mjs +36 -9
- package/scripts/runner-fleet-consumers.json +20 -0
- package/templates/runbooks/runner-fleet-health.md +150 -0
- package/templates/workflows/deploy-staging-run.yml +77 -0
- package/templates/workflows/deploy-staging.yml +66 -64
package/package.json
CHANGED
|
@@ -0,0 +1,469 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* check-runner-health.mjs
|
|
4
|
+
*
|
|
5
|
+
* Scheduled runner-fleet health monitor for mandrel-platform (Story #258).
|
|
6
|
+
*
|
|
7
|
+
* All nine self-hosted runners across the fleet (domio, athportal, swarm-os)
|
|
8
|
+
* are co-resident on ONE operator Mac (2026-07-03 runner audit, repo-ops
|
|
9
|
+
* matrix §1a). If that host sleeps, reboots for an OS update, fills its disk,
|
|
10
|
+
* or a launchd service dies, every consumer's CI and deploy-trigger jobs
|
|
11
|
+
* silently queue ("waiting for a runner") with no alert — nothing watches the
|
|
12
|
+
* fleet today. This script is the standing check that closes that gap.
|
|
13
|
+
*
|
|
14
|
+
* Modeled on `check-pin-drift.mjs` (same GitHub-hosted, config-driven,
|
|
15
|
+
* `GITHUB_STEP_SUMMARY`-rendering shape via the `scripts/lib/gh-json.mjs`
|
|
16
|
+
* seam) with two differences: it must run FREQUENTLY (not weekly) and it must
|
|
17
|
+
* ALERT (not just render a dashboard), because the whole point is catching an
|
|
18
|
+
* offline fleet fast.
|
|
19
|
+
*
|
|
20
|
+
* For each repo in `scripts/runner-fleet-consumers.json` it:
|
|
21
|
+
* 1. Calls `GET /repos/{owner}/{repo}/actions/runners` and flags any runner
|
|
22
|
+
* whose `status !== "online"`.
|
|
23
|
+
* 2. Flags a count shortfall: fewer online runners matching the expected
|
|
24
|
+
* `labels` set than `expectedCount`.
|
|
25
|
+
* 3. Flags queue-staleness: a `queued`/`waiting` workflow run older than
|
|
26
|
+
* `staleQueuedMinutes` with no online runner matching its labels
|
|
27
|
+
* (a wedged fleet accepts jobs into the queue but never claims them).
|
|
28
|
+
* 4. Renders a per-repo dashboard to `GITHUB_STEP_SUMMARY`.
|
|
29
|
+
*
|
|
30
|
+
* Alerting (no external dependency, on-ethos default): the CLI exits
|
|
31
|
+
* non-zero when any repo is unhealthy, so GitHub's native failed-workflow
|
|
32
|
+
* notification fires. That is the only alert channel — no tracking issues
|
|
33
|
+
* are filed.
|
|
34
|
+
*
|
|
35
|
+
* GitHub access is via the `gh` CLI (`gh api`), through the same injectable
|
|
36
|
+
* `runGh` seam `check-pin-drift.mjs` uses (`scripts/lib/gh-json.mjs`), so the
|
|
37
|
+
* whole pipeline is exercised offline in tests with canned responses.
|
|
38
|
+
*
|
|
39
|
+
* Usage:
|
|
40
|
+
* node scripts/check-runner-health.mjs
|
|
41
|
+
* node scripts/check-runner-health.mjs --config scripts/runner-fleet-consumers.json
|
|
42
|
+
* node scripts/check-runner-health.mjs --json # machine-readable envelope
|
|
43
|
+
*
|
|
44
|
+
* Exit codes:
|
|
45
|
+
* 0 — every repo healthy.
|
|
46
|
+
* 1 — at least one repo unhealthy (offline runner, count shortfall, stale
|
|
47
|
+
* queued run, or a fetch error), OR a fatal error (bad config).
|
|
48
|
+
*
|
|
49
|
+
* GitHub Actions: when GITHUB_STEP_SUMMARY is set, the human-readable report
|
|
50
|
+
* is also appended there so it renders on the job summary page.
|
|
51
|
+
*/
|
|
52
|
+
|
|
53
|
+
import { readFileSync, appendFileSync } from "node:fs";
|
|
54
|
+
import { resolve } from "node:path";
|
|
55
|
+
|
|
56
|
+
import { defaultGhRunner, ghApiJson, isNotFound } from "./lib/gh-json.mjs";
|
|
57
|
+
|
|
58
|
+
// ---------------------------------------------------------------------------
|
|
59
|
+
// Arg parsing
|
|
60
|
+
// ---------------------------------------------------------------------------
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* @param {string[]} argv
|
|
64
|
+
* @returns {{ config: string, json: boolean }}
|
|
65
|
+
*/
|
|
66
|
+
export function parseArgv(argv = []) {
|
|
67
|
+
let config = "scripts/runner-fleet-consumers.json";
|
|
68
|
+
let json = false;
|
|
69
|
+
for (let i = 0; i < argv.length; i += 1) {
|
|
70
|
+
const a = argv[i];
|
|
71
|
+
if (a === "--config") {
|
|
72
|
+
const next = argv[i + 1];
|
|
73
|
+
if (next && !next.startsWith("--")) {
|
|
74
|
+
config = next;
|
|
75
|
+
i += 1;
|
|
76
|
+
}
|
|
77
|
+
} else if (a === "--json") {
|
|
78
|
+
json = true;
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
return { config, json };
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
// ---------------------------------------------------------------------------
|
|
85
|
+
// Pure helpers (exported for unit-style probing without GitHub access)
|
|
86
|
+
// ---------------------------------------------------------------------------
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* Does a runner's label set satisfy the expected label roster? A runner
|
|
90
|
+
* "matches" when every expected label is present among the runner's labels
|
|
91
|
+
* (order-independent, extra labels on the runner are fine).
|
|
92
|
+
*
|
|
93
|
+
* @param {string[]} runnerLabels
|
|
94
|
+
* @param {string[]} expectedLabels
|
|
95
|
+
* @returns {boolean}
|
|
96
|
+
*/
|
|
97
|
+
export function runnerMatchesLabels(runnerLabels, expectedLabels) {
|
|
98
|
+
const have = new Set((runnerLabels || []).map((l) => l.toLowerCase()));
|
|
99
|
+
return (expectedLabels || []).every((l) => have.has(l.toLowerCase()));
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* Classify one repo's live runner list against its expected roster.
|
|
104
|
+
*
|
|
105
|
+
* @param {Array<{ id: number, name: string, status: string, busy?: boolean, labels: Array<{ name: string }> }>} runners
|
|
106
|
+
* @param {{ expectedCount: number, labels: string[] }} expected
|
|
107
|
+
* @returns {{
|
|
108
|
+
* total: number,
|
|
109
|
+
* online: number,
|
|
110
|
+
* offline: Array<{ id: number, name: string, status: string }>,
|
|
111
|
+
* matchingOnline: number,
|
|
112
|
+
* shortfall: number,
|
|
113
|
+
* hasShortfall: boolean,
|
|
114
|
+
* hasOffline: boolean,
|
|
115
|
+
* }}
|
|
116
|
+
*/
|
|
117
|
+
export function classifyRunners(runners, expected) {
|
|
118
|
+
const list = Array.isArray(runners) ? runners : [];
|
|
119
|
+
const offline = list
|
|
120
|
+
.filter((r) => r.status !== "online")
|
|
121
|
+
.map((r) => ({ id: r.id, name: r.name, status: r.status }));
|
|
122
|
+
const online = list.filter((r) => r.status === "online");
|
|
123
|
+
const matchingOnline = online.filter((r) =>
|
|
124
|
+
runnerMatchesLabels(
|
|
125
|
+
(r.labels || []).map((l) => (typeof l === "string" ? l : l.name)),
|
|
126
|
+
expected.labels,
|
|
127
|
+
),
|
|
128
|
+
).length;
|
|
129
|
+
const expectedCount =
|
|
130
|
+
typeof expected.expectedCount === "number" ? expected.expectedCount : 0;
|
|
131
|
+
const shortfall = Math.max(0, expectedCount - matchingOnline);
|
|
132
|
+
return {
|
|
133
|
+
total: list.length,
|
|
134
|
+
online: online.length,
|
|
135
|
+
offline,
|
|
136
|
+
matchingOnline,
|
|
137
|
+
shortfall,
|
|
138
|
+
hasShortfall: shortfall > 0,
|
|
139
|
+
hasOffline: offline.length > 0,
|
|
140
|
+
};
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/**
|
|
144
|
+
* Is a workflow run "stuck in queue"? True when its status is `queued` or
|
|
145
|
+
* `waiting`, it is older than `staleMinutes`, and no online runner in
|
|
146
|
+
* `onlineLabelSets` matches every one of its `labels` (i.e. nothing could ever
|
|
147
|
+
* pick it up).
|
|
148
|
+
*
|
|
149
|
+
* @param {{ status: string, created_at: string, labels?: string[] }} run A workflow run summary (labels resolved from its job requirements when available).
|
|
150
|
+
* @param {string[][]} onlineLabelSets Each online runner's label list.
|
|
151
|
+
* @param {number} staleMinutes
|
|
152
|
+
* @param {number} [nowMs] Injectable current epoch ms (for tests).
|
|
153
|
+
* @returns {boolean}
|
|
154
|
+
*/
|
|
155
|
+
export function isStaleQueuedRun(run, onlineLabelSets, staleMinutes, nowMs = Date.now()) {
|
|
156
|
+
if (run.status !== "queued" && run.status !== "waiting") return false;
|
|
157
|
+
const createdMs = Date.parse(run.created_at);
|
|
158
|
+
if (Number.isNaN(createdMs)) return false;
|
|
159
|
+
const ageMinutes = (nowMs - createdMs) / 60000;
|
|
160
|
+
if (ageMinutes < staleMinutes) return false;
|
|
161
|
+
const runLabels = Array.isArray(run.labels) ? run.labels : [];
|
|
162
|
+
if (runLabels.length === 0) {
|
|
163
|
+
// No label info to match against — can't assert "nothing could pick this
|
|
164
|
+
// up" without over-claiming, so don't flag it as stale from labels alone.
|
|
165
|
+
return false;
|
|
166
|
+
}
|
|
167
|
+
const anyOnlineMatches = onlineLabelSets.some((labels) =>
|
|
168
|
+
runnerMatchesLabels(labels, runLabels),
|
|
169
|
+
);
|
|
170
|
+
return !anyOnlineMatches;
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
/**
|
|
174
|
+
* Combine the three health signals into a single per-repo health verdict.
|
|
175
|
+
*
|
|
176
|
+
* @param {ReturnType<typeof classifyRunners>} runnerVerdict
|
|
177
|
+
* @param {Array<object>} staleRuns
|
|
178
|
+
* @returns {boolean} true when the repo is healthy (no offline, no shortfall, no stale queued runs).
|
|
179
|
+
*/
|
|
180
|
+
export function isRepoHealthy(runnerVerdict, staleRuns) {
|
|
181
|
+
return (
|
|
182
|
+
!runnerVerdict.hasOffline &&
|
|
183
|
+
!runnerVerdict.hasShortfall &&
|
|
184
|
+
(!Array.isArray(staleRuns) || staleRuns.length === 0)
|
|
185
|
+
);
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
/**
|
|
189
|
+
* Render the human-readable dashboard report.
|
|
190
|
+
*
|
|
191
|
+
* @param {{
|
|
192
|
+
* results: Array<{
|
|
193
|
+
* name: string,
|
|
194
|
+
* repo: string,
|
|
195
|
+
* error?: string,
|
|
196
|
+
* verdict?: ReturnType<typeof classifyRunners>,
|
|
197
|
+
* staleRuns?: Array<{ id: number, html_url?: string, created_at: string }>,
|
|
198
|
+
* healthy?: boolean,
|
|
199
|
+
* }>,
|
|
200
|
+
* }} report
|
|
201
|
+
* @returns {string}
|
|
202
|
+
*/
|
|
203
|
+
export function renderReport(report) {
|
|
204
|
+
const out = [];
|
|
205
|
+
out.push("## Runner-fleet health dashboard");
|
|
206
|
+
out.push("");
|
|
207
|
+
out.push(
|
|
208
|
+
"All self-hosted runners are co-resident on one operator Mac — a wedged " +
|
|
209
|
+
"host silently stalls every listed repo's CI with no other alert.",
|
|
210
|
+
);
|
|
211
|
+
out.push("");
|
|
212
|
+
out.push("| Repo | Online / Expected | Offline | Stale queued | Status |");
|
|
213
|
+
out.push("| ---- | ------------------ | ------- | ------------- | ------ |");
|
|
214
|
+
|
|
215
|
+
const problemLines = [];
|
|
216
|
+
for (const r of report.results) {
|
|
217
|
+
if (r.error) {
|
|
218
|
+
out.push(`| \`${r.name}\` | — | — | — | ⚠️ error |`);
|
|
219
|
+
problemLines.push(`- \`${r.name}\` (${r.repo}): error — ${r.error}`);
|
|
220
|
+
continue;
|
|
221
|
+
}
|
|
222
|
+
const v = r.verdict;
|
|
223
|
+
const stale = Array.isArray(r.staleRuns) ? r.staleRuns : [];
|
|
224
|
+
const healthy = r.healthy === true;
|
|
225
|
+
const status = healthy ? "✅ healthy" : "❌ degraded";
|
|
226
|
+
out.push(
|
|
227
|
+
`| \`${r.name}\` | ${v.matchingOnline}/${v.matchingOnline + v.shortfall} | ${v.offline.length} | ${stale.length} | ${status} |`,
|
|
228
|
+
);
|
|
229
|
+
if (!healthy) {
|
|
230
|
+
if (v.hasOffline) {
|
|
231
|
+
const names = v.offline.map((o) => `\`${o.name}\` (${o.status})`).join(", ");
|
|
232
|
+
problemLines.push(
|
|
233
|
+
`- \`${r.name}\` (${r.repo}): OFFLINE runner(s) — ${names}.`,
|
|
234
|
+
);
|
|
235
|
+
}
|
|
236
|
+
if (v.hasShortfall) {
|
|
237
|
+
problemLines.push(
|
|
238
|
+
`- \`${r.name}\` (${r.repo}): SHORTFALL — ${v.matchingOnline} online runner(s) matching labels, expected at least ${v.matchingOnline + v.shortfall}.`,
|
|
239
|
+
);
|
|
240
|
+
}
|
|
241
|
+
if (stale.length > 0) {
|
|
242
|
+
const runs = stale
|
|
243
|
+
.map((run) => (run.html_url ? `[#${run.id}](${run.html_url})` : `#${run.id}`))
|
|
244
|
+
.join(", ");
|
|
245
|
+
problemLines.push(
|
|
246
|
+
`- \`${r.name}\` (${r.repo}): STALE QUEUED RUN(S) — ${runs} queued with no matching online runner.`,
|
|
247
|
+
);
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
out.push("");
|
|
253
|
+
if (problemLines.length > 0) {
|
|
254
|
+
out.push("### Degraded");
|
|
255
|
+
out.push("");
|
|
256
|
+
out.push(...problemLines);
|
|
257
|
+
out.push("");
|
|
258
|
+
out.push(
|
|
259
|
+
"**Operator response:** wake/reboot the Mac, check disk space, and " +
|
|
260
|
+
"restart the launchd runner services. See " +
|
|
261
|
+
"`templates/runbooks/runner-fleet-health.md`.",
|
|
262
|
+
);
|
|
263
|
+
} else {
|
|
264
|
+
out.push("### ✅ Fleet healthy");
|
|
265
|
+
out.push("");
|
|
266
|
+
out.push("Every configured repo has its expected online runner count and no stale queued runs.");
|
|
267
|
+
}
|
|
268
|
+
out.push("");
|
|
269
|
+
return out.join("\n");
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
// ---------------------------------------------------------------------------
|
|
273
|
+
// GitHub access — runners + workflow runs.
|
|
274
|
+
// ---------------------------------------------------------------------------
|
|
275
|
+
|
|
276
|
+
/**
|
|
277
|
+
* Fetch a repo's live self-hosted runner list.
|
|
278
|
+
*
|
|
279
|
+
* @param {string} repo "owner/name".
|
|
280
|
+
* @param {(args: string[]) => string} runGh
|
|
281
|
+
* @returns {Array<{ id: number, name: string, status: string, busy?: boolean, labels: Array<{ name: string }> }>}
|
|
282
|
+
*/
|
|
283
|
+
export function fetchRunners(repo, runGh) {
|
|
284
|
+
let obj;
|
|
285
|
+
try {
|
|
286
|
+
obj = ghApiJson(`repos/${repo}/actions/runners?per_page=100`, runGh);
|
|
287
|
+
} catch (err) {
|
|
288
|
+
if (isNotFound(err)) return [];
|
|
289
|
+
throw err;
|
|
290
|
+
}
|
|
291
|
+
return Array.isArray(obj?.runners) ? obj.runners : [];
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
/**
|
|
295
|
+
* Fetch a repo's currently queued/waiting workflow runs (for stale-queue
|
|
296
|
+
* detection). Returns [] on a 404 (repo unreadable, or genuinely no runs);
|
|
297
|
+
* any other error propagates so the caller records an `error` row.
|
|
298
|
+
*
|
|
299
|
+
* @param {string} repo
|
|
300
|
+
* @param {(args: string[]) => string} runGh
|
|
301
|
+
* @returns {Array<{ id: number, status: string, created_at: string, html_url?: string, labels?: string[] }>}
|
|
302
|
+
*/
|
|
303
|
+
export function fetchQueuedRuns(repo, runGh) {
|
|
304
|
+
let obj;
|
|
305
|
+
try {
|
|
306
|
+
obj = ghApiJson(
|
|
307
|
+
`repos/${repo}/actions/runs?status=queued&per_page=50`,
|
|
308
|
+
runGh,
|
|
309
|
+
);
|
|
310
|
+
} catch (err) {
|
|
311
|
+
if (isNotFound(err)) return [];
|
|
312
|
+
throw err;
|
|
313
|
+
}
|
|
314
|
+
const queued = Array.isArray(obj?.workflow_runs) ? obj.workflow_runs : [];
|
|
315
|
+
let waitingObj;
|
|
316
|
+
try {
|
|
317
|
+
waitingObj = ghApiJson(
|
|
318
|
+
`repos/${repo}/actions/runs?status=waiting&per_page=50`,
|
|
319
|
+
runGh,
|
|
320
|
+
);
|
|
321
|
+
} catch (err) {
|
|
322
|
+
if (isNotFound(err)) return queued;
|
|
323
|
+
throw err;
|
|
324
|
+
}
|
|
325
|
+
const waiting = Array.isArray(waitingObj?.workflow_runs)
|
|
326
|
+
? waitingObj.workflow_runs
|
|
327
|
+
: [];
|
|
328
|
+
return [...queued, ...waiting];
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
// ---------------------------------------------------------------------------
|
|
332
|
+
// Orchestration
|
|
333
|
+
// ---------------------------------------------------------------------------
|
|
334
|
+
|
|
335
|
+
/**
|
|
336
|
+
* Build the full health report for the configured repos.
|
|
337
|
+
*
|
|
338
|
+
* @param {{
|
|
339
|
+
* repos: Array<{ name: string, repo: string, expectedCount: number, labels: string[], staleQueuedMinutes?: number }>,
|
|
340
|
+
* defaultStaleQueuedMinutes?: number,
|
|
341
|
+
* }} config
|
|
342
|
+
* @param {(args: string[]) => string} runGh
|
|
343
|
+
* @param {number} [nowMs]
|
|
344
|
+
* @returns {{ results: Array<object> }}
|
|
345
|
+
*/
|
|
346
|
+
export function buildReport(config, runGh, nowMs = Date.now()) {
|
|
347
|
+
const defaultStale =
|
|
348
|
+
typeof config.defaultStaleQueuedMinutes === "number"
|
|
349
|
+
? config.defaultStaleQueuedMinutes
|
|
350
|
+
: 20;
|
|
351
|
+
const results = [];
|
|
352
|
+
for (const entry of config.repos) {
|
|
353
|
+
try {
|
|
354
|
+
const runners = fetchRunners(entry.repo, runGh);
|
|
355
|
+
const verdict = classifyRunners(runners, entry);
|
|
356
|
+
const onlineLabelSets = runners
|
|
357
|
+
.filter((r) => r.status === "online")
|
|
358
|
+
.map((r) => (r.labels || []).map((l) => (typeof l === "string" ? l : l.name)));
|
|
359
|
+
const staleMinutes =
|
|
360
|
+
typeof entry.staleQueuedMinutes === "number"
|
|
361
|
+
? entry.staleQueuedMinutes
|
|
362
|
+
: defaultStale;
|
|
363
|
+
const queuedRuns = fetchQueuedRuns(entry.repo, runGh);
|
|
364
|
+
const staleRuns = queuedRuns.filter((run) =>
|
|
365
|
+
isStaleQueuedRun(run, onlineLabelSets, staleMinutes, nowMs),
|
|
366
|
+
);
|
|
367
|
+
results.push({
|
|
368
|
+
name: entry.name,
|
|
369
|
+
repo: entry.repo,
|
|
370
|
+
verdict,
|
|
371
|
+
staleRuns,
|
|
372
|
+
healthy: isRepoHealthy(verdict, staleRuns),
|
|
373
|
+
});
|
|
374
|
+
} catch (err) {
|
|
375
|
+
results.push({
|
|
376
|
+
name: entry.name,
|
|
377
|
+
repo: entry.repo,
|
|
378
|
+
error: err instanceof Error ? err.message : String(err),
|
|
379
|
+
healthy: false,
|
|
380
|
+
});
|
|
381
|
+
}
|
|
382
|
+
}
|
|
383
|
+
return { results };
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
/**
|
|
387
|
+
* @param {{ results: Array<{ error?: string, healthy?: boolean }> }} report
|
|
388
|
+
* @returns {boolean} true when any repo is unhealthy or errored.
|
|
389
|
+
*/
|
|
390
|
+
export function hasUnhealthy(report) {
|
|
391
|
+
return report.results.some((r) => r.error || r.healthy === false);
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
// ---------------------------------------------------------------------------
|
|
395
|
+
// CLI entry
|
|
396
|
+
// ---------------------------------------------------------------------------
|
|
397
|
+
|
|
398
|
+
/**
|
|
399
|
+
* @param {{
|
|
400
|
+
* argv?: string[],
|
|
401
|
+
* cwd?: string,
|
|
402
|
+
* stdout?: { write: (s: string) => void },
|
|
403
|
+
* stderr?: { write: (s: string) => void },
|
|
404
|
+
* runGh?: (args: string[]) => string,
|
|
405
|
+
* summaryPath?: string | undefined,
|
|
406
|
+
* nowMs?: number,
|
|
407
|
+
* }} [opts]
|
|
408
|
+
* @returns {number} exit code
|
|
409
|
+
*/
|
|
410
|
+
export function runCli({
|
|
411
|
+
argv = process.argv.slice(2),
|
|
412
|
+
cwd = process.cwd(),
|
|
413
|
+
stdout = process.stdout,
|
|
414
|
+
stderr = process.stderr,
|
|
415
|
+
runGh = defaultGhRunner,
|
|
416
|
+
summaryPath = process.env.GITHUB_STEP_SUMMARY,
|
|
417
|
+
nowMs = Date.now(),
|
|
418
|
+
} = {}) {
|
|
419
|
+
const { config: configRel, json } = parseArgv(argv);
|
|
420
|
+
const configPath = resolve(cwd, configRel);
|
|
421
|
+
|
|
422
|
+
let config;
|
|
423
|
+
try {
|
|
424
|
+
config = JSON.parse(readFileSync(configPath, "utf-8"));
|
|
425
|
+
} catch (err) {
|
|
426
|
+
stderr.write(
|
|
427
|
+
`[runner-health] ❌ failed to read config ${configPath}: ${err instanceof Error ? err.message : String(err)}\n`,
|
|
428
|
+
);
|
|
429
|
+
return 1;
|
|
430
|
+
}
|
|
431
|
+
if (!Array.isArray(config.repos)) {
|
|
432
|
+
stderr.write(`[runner-health] ❌ config must define { repos: [] }\n`);
|
|
433
|
+
return 1;
|
|
434
|
+
}
|
|
435
|
+
|
|
436
|
+
const report = buildReport(config, runGh, nowMs);
|
|
437
|
+
const unhealthy = hasUnhealthy(report);
|
|
438
|
+
|
|
439
|
+
if (json) {
|
|
440
|
+
stdout.write(
|
|
441
|
+
`${JSON.stringify({ kind: "runner-fleet-health-report", unhealthy, ...report }, null, 2)}\n`,
|
|
442
|
+
);
|
|
443
|
+
} else {
|
|
444
|
+
const text = renderReport(report);
|
|
445
|
+
stdout.write(`${text}\n`);
|
|
446
|
+
if (summaryPath) {
|
|
447
|
+
try {
|
|
448
|
+
appendFileSync(summaryPath, `${text}\n`);
|
|
449
|
+
} catch (err) {
|
|
450
|
+
stderr.write(
|
|
451
|
+
`[runner-health] ⚠ could not write job summary: ${err instanceof Error ? err.message : String(err)}\n`,
|
|
452
|
+
);
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
if (unhealthy) {
|
|
458
|
+
stderr.write(`[runner-health] ❌ fleet degraded\n`);
|
|
459
|
+
return 1;
|
|
460
|
+
}
|
|
461
|
+
return 0;
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
// Direct-invocation guard (matches the repo's other scripts/*.mjs entry style).
|
|
465
|
+
const invokedDirectly =
|
|
466
|
+
process.argv[1] && resolve(process.argv[1]) === resolve(new URL(import.meta.url).pathname);
|
|
467
|
+
if (invokedDirectly) {
|
|
468
|
+
process.exit(runCli());
|
|
469
|
+
}
|
|
@@ -0,0 +1,389 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* check-runner-health.test.mjs — node:test suite for the scheduled
|
|
4
|
+
* runner-fleet health monitor (Story #258).
|
|
5
|
+
*
|
|
6
|
+
* The checker exposes pure helpers plus an injectable `runGh` seam, so the
|
|
7
|
+
* whole pipeline is exercised offline with canned GitHub responses — no
|
|
8
|
+
* network, no `gh` auth.
|
|
9
|
+
*
|
|
10
|
+
* Run: node scripts/check-runner-health.test.mjs (or `node --test scripts/`)
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import assert from "node:assert/strict";
|
|
14
|
+
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
|
15
|
+
import { tmpdir } from "node:os";
|
|
16
|
+
import { join } from "node:path";
|
|
17
|
+
import { test } from "node:test";
|
|
18
|
+
|
|
19
|
+
import {
|
|
20
|
+
buildReport,
|
|
21
|
+
classifyRunners,
|
|
22
|
+
fetchQueuedRuns,
|
|
23
|
+
fetchRunners,
|
|
24
|
+
hasUnhealthy,
|
|
25
|
+
isRepoHealthy,
|
|
26
|
+
isStaleQueuedRun,
|
|
27
|
+
parseArgv,
|
|
28
|
+
renderReport,
|
|
29
|
+
runCli,
|
|
30
|
+
runnerMatchesLabels,
|
|
31
|
+
} from "./check-runner-health.mjs";
|
|
32
|
+
|
|
33
|
+
// ---------------------------------------------------------------------------
|
|
34
|
+
// parseArgv
|
|
35
|
+
// ---------------------------------------------------------------------------
|
|
36
|
+
|
|
37
|
+
test("parseArgv defaults to the fleet consumer config", () => {
|
|
38
|
+
const opts = parseArgv([]);
|
|
39
|
+
assert.equal(opts.config, "scripts/runner-fleet-consumers.json");
|
|
40
|
+
assert.equal(opts.json, false);
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
test("parseArgv parses --config and --json", () => {
|
|
44
|
+
const opts = parseArgv(["--config", "custom.json", "--json"]);
|
|
45
|
+
assert.equal(opts.config, "custom.json");
|
|
46
|
+
assert.equal(opts.json, true);
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
// ---------------------------------------------------------------------------
|
|
50
|
+
// runnerMatchesLabels
|
|
51
|
+
// ---------------------------------------------------------------------------
|
|
52
|
+
|
|
53
|
+
test("runnerMatchesLabels is case-insensitive and order-independent", () => {
|
|
54
|
+
assert.equal(
|
|
55
|
+
runnerMatchesLabels(["Self-Hosted", "macOS", "ARM64", "domio-runner"], [
|
|
56
|
+
"self-hosted",
|
|
57
|
+
"arm64",
|
|
58
|
+
]),
|
|
59
|
+
true,
|
|
60
|
+
);
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
test("runnerMatchesLabels fails when an expected label is missing", () => {
|
|
64
|
+
assert.equal(
|
|
65
|
+
runnerMatchesLabels(["self-hosted", "macOS"], ["self-hosted", "ARM64"]),
|
|
66
|
+
false,
|
|
67
|
+
);
|
|
68
|
+
});
|
|
69
|
+
|
|
70
|
+
// ---------------------------------------------------------------------------
|
|
71
|
+
// classifyRunners
|
|
72
|
+
// ---------------------------------------------------------------------------
|
|
73
|
+
|
|
74
|
+
const EXPECTED = { expectedCount: 3, labels: ["self-hosted", "macOS", "ARM64", "domio-runner"] };
|
|
75
|
+
|
|
76
|
+
test("classifyRunners reports a fully healthy fleet", () => {
|
|
77
|
+
const runners = [1, 2, 3].map((n) => ({
|
|
78
|
+
id: n,
|
|
79
|
+
name: `domio-runner-${n}`,
|
|
80
|
+
status: "online",
|
|
81
|
+
labels: [{ name: "self-hosted" }, { name: "macOS" }, { name: "ARM64" }, { name: "domio-runner" }],
|
|
82
|
+
}));
|
|
83
|
+
const v = classifyRunners(runners, EXPECTED);
|
|
84
|
+
assert.equal(v.total, 3);
|
|
85
|
+
assert.equal(v.online, 3);
|
|
86
|
+
assert.equal(v.matchingOnline, 3);
|
|
87
|
+
assert.equal(v.shortfall, 0);
|
|
88
|
+
assert.equal(v.hasShortfall, false);
|
|
89
|
+
assert.equal(v.hasOffline, false);
|
|
90
|
+
});
|
|
91
|
+
|
|
92
|
+
test("classifyRunners flags an offline runner", () => {
|
|
93
|
+
const runners = [
|
|
94
|
+
{ id: 1, name: "domio-runner-1", status: "online", labels: [{ name: "self-hosted" }, { name: "macOS" }, { name: "ARM64" }, { name: "domio-runner" }] },
|
|
95
|
+
{ id: 2, name: "domio-runner-2", status: "offline", labels: [{ name: "self-hosted" }, { name: "macOS" }, { name: "ARM64" }, { name: "domio-runner" }] },
|
|
96
|
+
];
|
|
97
|
+
const v = classifyRunners(runners, EXPECTED);
|
|
98
|
+
assert.equal(v.hasOffline, true);
|
|
99
|
+
assert.equal(v.offline.length, 1);
|
|
100
|
+
assert.equal(v.offline[0].name, "domio-runner-2");
|
|
101
|
+
assert.equal(v.matchingOnline, 1);
|
|
102
|
+
assert.equal(v.hasShortfall, true);
|
|
103
|
+
assert.equal(v.shortfall, 2);
|
|
104
|
+
});
|
|
105
|
+
|
|
106
|
+
test("classifyRunners flags a count shortfall even with all-online but wrong labels", () => {
|
|
107
|
+
const runners = [
|
|
108
|
+
{ id: 1, name: "other-runner", status: "online", labels: [{ name: "self-hosted" }, { name: "linux" }] },
|
|
109
|
+
];
|
|
110
|
+
const v = classifyRunners(runners, EXPECTED);
|
|
111
|
+
assert.equal(v.hasOffline, false);
|
|
112
|
+
assert.equal(v.matchingOnline, 0);
|
|
113
|
+
assert.equal(v.hasShortfall, true);
|
|
114
|
+
assert.equal(v.shortfall, 3);
|
|
115
|
+
});
|
|
116
|
+
|
|
117
|
+
test("classifyRunners handles an empty runner list", () => {
|
|
118
|
+
const v = classifyRunners([], EXPECTED);
|
|
119
|
+
assert.equal(v.total, 0);
|
|
120
|
+
assert.equal(v.hasShortfall, true);
|
|
121
|
+
assert.equal(v.shortfall, 3);
|
|
122
|
+
});
|
|
123
|
+
|
|
124
|
+
// ---------------------------------------------------------------------------
|
|
125
|
+
// isStaleQueuedRun
|
|
126
|
+
// ---------------------------------------------------------------------------
|
|
127
|
+
|
|
128
|
+
const NOW = Date.parse("2026-07-03T12:00:00Z");
|
|
129
|
+
|
|
130
|
+
test("isStaleQueuedRun flags an old queued run with no matching online runner", () => {
|
|
131
|
+
const run = {
|
|
132
|
+
status: "queued",
|
|
133
|
+
created_at: "2026-07-03T11:00:00Z", // 60 min ago
|
|
134
|
+
labels: ["self-hosted", "macOS", "ARM64", "domio-runner"],
|
|
135
|
+
};
|
|
136
|
+
assert.equal(isStaleQueuedRun(run, [], 20, NOW), true);
|
|
137
|
+
});
|
|
138
|
+
|
|
139
|
+
test("isStaleQueuedRun does not flag a recently queued run", () => {
|
|
140
|
+
const run = {
|
|
141
|
+
status: "queued",
|
|
142
|
+
created_at: "2026-07-03T11:55:00Z", // 5 min ago
|
|
143
|
+
labels: ["self-hosted", "macOS", "ARM64", "domio-runner"],
|
|
144
|
+
};
|
|
145
|
+
assert.equal(isStaleQueuedRun(run, [], 20, NOW), false);
|
|
146
|
+
});
|
|
147
|
+
|
|
148
|
+
test("isStaleQueuedRun does not flag when a matching online runner exists", () => {
|
|
149
|
+
const run = {
|
|
150
|
+
status: "queued",
|
|
151
|
+
created_at: "2026-07-03T11:00:00Z",
|
|
152
|
+
labels: ["self-hosted", "macOS", "ARM64", "domio-runner"],
|
|
153
|
+
};
|
|
154
|
+
const onlineLabelSets = [["self-hosted", "macOS", "ARM64", "domio-runner"]];
|
|
155
|
+
assert.equal(isStaleQueuedRun(run, onlineLabelSets, 20, NOW), false);
|
|
156
|
+
});
|
|
157
|
+
|
|
158
|
+
test("isStaleQueuedRun ignores non-queued/waiting runs", () => {
|
|
159
|
+
const run = { status: "completed", created_at: "2026-07-03T09:00:00Z" };
|
|
160
|
+
assert.equal(isStaleQueuedRun(run, [], 20, NOW), false);
|
|
161
|
+
});
|
|
162
|
+
|
|
163
|
+
test("isStaleQueuedRun does not flag when no label info is available", () => {
|
|
164
|
+
const run = { status: "queued", created_at: "2026-07-03T09:00:00Z", labels: [] };
|
|
165
|
+
assert.equal(isStaleQueuedRun(run, [], 20, NOW), false);
|
|
166
|
+
});
|
|
167
|
+
|
|
168
|
+
// ---------------------------------------------------------------------------
|
|
169
|
+
// isRepoHealthy
|
|
170
|
+
// ---------------------------------------------------------------------------
|
|
171
|
+
|
|
172
|
+
test("isRepoHealthy is true only with no offline/shortfall/stale", () => {
|
|
173
|
+
const healthyVerdict = { hasOffline: false, hasShortfall: false };
|
|
174
|
+
assert.equal(isRepoHealthy(healthyVerdict, []), true);
|
|
175
|
+
assert.equal(isRepoHealthy({ ...healthyVerdict, hasOffline: true }, []), false);
|
|
176
|
+
assert.equal(isRepoHealthy({ ...healthyVerdict, hasShortfall: true }, []), false);
|
|
177
|
+
assert.equal(isRepoHealthy(healthyVerdict, [{ id: 1 }]), false);
|
|
178
|
+
});
|
|
179
|
+
|
|
180
|
+
// ---------------------------------------------------------------------------
|
|
181
|
+
// renderReport
|
|
182
|
+
// ---------------------------------------------------------------------------
|
|
183
|
+
|
|
184
|
+
test("renderReport renders a healthy fleet with no Degraded section", () => {
|
|
185
|
+
const report = {
|
|
186
|
+
results: [
|
|
187
|
+
{
|
|
188
|
+
name: "domio",
|
|
189
|
+
repo: "dsj1984/domio",
|
|
190
|
+
verdict: { matchingOnline: 3, shortfall: 0, offline: [], hasOffline: false, hasShortfall: false },
|
|
191
|
+
staleRuns: [],
|
|
192
|
+
healthy: true,
|
|
193
|
+
},
|
|
194
|
+
],
|
|
195
|
+
};
|
|
196
|
+
const text = renderReport(report);
|
|
197
|
+
assert.match(text, /✅ healthy/);
|
|
198
|
+
assert.match(text, /✅ Fleet healthy/);
|
|
199
|
+
assert.doesNotMatch(text, /### Degraded/);
|
|
200
|
+
});
|
|
201
|
+
|
|
202
|
+
test("renderReport surfaces offline / shortfall / stale-run detail lines", () => {
|
|
203
|
+
const report = {
|
|
204
|
+
results: [
|
|
205
|
+
{
|
|
206
|
+
name: "domio",
|
|
207
|
+
repo: "dsj1984/domio",
|
|
208
|
+
verdict: {
|
|
209
|
+
matchingOnline: 1,
|
|
210
|
+
shortfall: 2,
|
|
211
|
+
offline: [{ id: 2, name: "domio-runner-2", status: "offline" }],
|
|
212
|
+
hasOffline: true,
|
|
213
|
+
hasShortfall: true,
|
|
214
|
+
},
|
|
215
|
+
staleRuns: [{ id: 42, html_url: "https://github.com/dsj1984/domio/actions/runs/42", created_at: "x" }],
|
|
216
|
+
healthy: false,
|
|
217
|
+
},
|
|
218
|
+
],
|
|
219
|
+
};
|
|
220
|
+
const text = renderReport(report);
|
|
221
|
+
assert.match(text, /❌ degraded/);
|
|
222
|
+
assert.match(text, /OFFLINE runner\(s\)/);
|
|
223
|
+
assert.match(text, /SHORTFALL/);
|
|
224
|
+
assert.match(text, /STALE QUEUED RUN/);
|
|
225
|
+
assert.match(text, /### Degraded/);
|
|
226
|
+
assert.match(text, /templates\/runbooks\/runner-fleet-health\.md/);
|
|
227
|
+
});
|
|
228
|
+
|
|
229
|
+
test("renderReport surfaces a fetch error row", () => {
|
|
230
|
+
const report = { results: [{ name: "domio", repo: "dsj1984/domio", error: "boom" }] };
|
|
231
|
+
const text = renderReport(report);
|
|
232
|
+
assert.match(text, /⚠️ error/);
|
|
233
|
+
assert.match(text, /error — boom/);
|
|
234
|
+
});
|
|
235
|
+
|
|
236
|
+
// ---------------------------------------------------------------------------
|
|
237
|
+
// fetchRunners / fetchQueuedRuns (injectable runGh)
|
|
238
|
+
// ---------------------------------------------------------------------------
|
|
239
|
+
|
|
240
|
+
test("fetchRunners returns the runners array from the API response", () => {
|
|
241
|
+
const runGh = () =>
|
|
242
|
+
JSON.stringify({ total_count: 1, runners: [{ id: 1, name: "r1", status: "online", labels: [] }] });
|
|
243
|
+
const runners = fetchRunners("dsj1984/domio", runGh);
|
|
244
|
+
assert.equal(runners.length, 1);
|
|
245
|
+
assert.equal(runners[0].name, "r1");
|
|
246
|
+
});
|
|
247
|
+
|
|
248
|
+
test("fetchRunners returns [] on a 404", () => {
|
|
249
|
+
const runGh = () => {
|
|
250
|
+
const err = new Error("gh: Not Found (HTTP 404)");
|
|
251
|
+
throw err;
|
|
252
|
+
};
|
|
253
|
+
assert.deepEqual(fetchRunners("dsj1984/domio", runGh), []);
|
|
254
|
+
});
|
|
255
|
+
|
|
256
|
+
test("fetchRunners propagates a non-404 error", () => {
|
|
257
|
+
const runGh = () => {
|
|
258
|
+
throw new Error("gh: Forbidden (HTTP 403)");
|
|
259
|
+
};
|
|
260
|
+
assert.throws(() => fetchRunners("dsj1984/domio", runGh));
|
|
261
|
+
});
|
|
262
|
+
|
|
263
|
+
test("fetchQueuedRuns merges queued and waiting runs", () => {
|
|
264
|
+
const runGh = (args) => {
|
|
265
|
+
const path = args[1];
|
|
266
|
+
if (path.includes("status=queued")) {
|
|
267
|
+
return JSON.stringify({ workflow_runs: [{ id: 1, status: "queued", created_at: "x" }] });
|
|
268
|
+
}
|
|
269
|
+
if (path.includes("status=waiting")) {
|
|
270
|
+
return JSON.stringify({ workflow_runs: [{ id: 2, status: "waiting", created_at: "y" }] });
|
|
271
|
+
}
|
|
272
|
+
throw new Error(`unexpected path ${path}`);
|
|
273
|
+
};
|
|
274
|
+
const runs = fetchQueuedRuns("dsj1984/domio", runGh);
|
|
275
|
+
assert.equal(runs.length, 2);
|
|
276
|
+
assert.deepEqual(runs.map((r) => r.id).sort(), [1, 2]);
|
|
277
|
+
});
|
|
278
|
+
|
|
279
|
+
test("fetchQueuedRuns returns [] on a 404", () => {
|
|
280
|
+
const runGh = () => {
|
|
281
|
+
throw new Error("gh: Not Found (HTTP 404)");
|
|
282
|
+
};
|
|
283
|
+
assert.deepEqual(fetchQueuedRuns("dsj1984/domio", runGh), []);
|
|
284
|
+
});
|
|
285
|
+
|
|
286
|
+
// ---------------------------------------------------------------------------
|
|
287
|
+
// buildReport / hasUnhealthy
|
|
288
|
+
// ---------------------------------------------------------------------------
|
|
289
|
+
|
|
290
|
+
const CONFIG = {
|
|
291
|
+
defaultStaleQueuedMinutes: 20,
|
|
292
|
+
repos: [
|
|
293
|
+
{ name: "domio", repo: "dsj1984/domio", expectedCount: 2, labels: ["self-hosted", "domio-runner"] },
|
|
294
|
+
],
|
|
295
|
+
};
|
|
296
|
+
|
|
297
|
+
test("buildReport marks a repo healthy when runners are all online and matching", () => {
|
|
298
|
+
const runGh = (args) => {
|
|
299
|
+
const path = args[1];
|
|
300
|
+
if (path.includes("actions/runners")) {
|
|
301
|
+
return JSON.stringify({
|
|
302
|
+
runners: [
|
|
303
|
+
{ id: 1, name: "r1", status: "online", labels: [{ name: "self-hosted" }, { name: "domio-runner" }] },
|
|
304
|
+
{ id: 2, name: "r2", status: "online", labels: [{ name: "self-hosted" }, { name: "domio-runner" }] },
|
|
305
|
+
],
|
|
306
|
+
});
|
|
307
|
+
}
|
|
308
|
+
if (path.includes("actions/runs")) {
|
|
309
|
+
return JSON.stringify({ workflow_runs: [] });
|
|
310
|
+
}
|
|
311
|
+
throw new Error(`unexpected ${path}`);
|
|
312
|
+
};
|
|
313
|
+
const report = buildReport(CONFIG, runGh, NOW);
|
|
314
|
+
assert.equal(report.results[0].healthy, true);
|
|
315
|
+
assert.equal(hasUnhealthy(report), false);
|
|
316
|
+
});
|
|
317
|
+
|
|
318
|
+
test("buildReport marks a repo unhealthy and records a fetch error as unhealthy", () => {
|
|
319
|
+
const runGh = (args) => {
|
|
320
|
+
const path = args[1];
|
|
321
|
+
if (path.includes("actions/runners")) throw new Error("gh: Service Unavailable (HTTP 503)");
|
|
322
|
+
return "{}";
|
|
323
|
+
};
|
|
324
|
+
const report = buildReport(CONFIG, runGh, NOW);
|
|
325
|
+
assert.equal(report.results[0].error !== undefined, true);
|
|
326
|
+
assert.equal(hasUnhealthy(report), true);
|
|
327
|
+
});
|
|
328
|
+
|
|
329
|
+
// ---------------------------------------------------------------------------
|
|
330
|
+
// runCli (end-to-end against a temp config + injected runGh)
|
|
331
|
+
// ---------------------------------------------------------------------------
|
|
332
|
+
|
|
333
|
+
test("runCli exits 0 for a healthy fleet and 1 for a degraded one", () => {
|
|
334
|
+
const dir = mkdtempSync(join(tmpdir(), "runner-health-cli-"));
|
|
335
|
+
try {
|
|
336
|
+
const configPath = join(dir, "runner-fleet-consumers.json");
|
|
337
|
+
writeFileSync(
|
|
338
|
+
configPath,
|
|
339
|
+
JSON.stringify({
|
|
340
|
+
defaultStaleQueuedMinutes: 20,
|
|
341
|
+
repos: [
|
|
342
|
+
{ name: "domio", repo: "dsj1984/domio", expectedCount: 2, labels: ["self-hosted", "domio-runner"] },
|
|
343
|
+
],
|
|
344
|
+
}),
|
|
345
|
+
);
|
|
346
|
+
const stdout = { buf: "", write(s) { this.buf += s; } };
|
|
347
|
+
const stderr = { buf: "", write(s) { this.buf += s; } };
|
|
348
|
+
const healthyRunGh = (args) => {
|
|
349
|
+
const path = args[1];
|
|
350
|
+
if (path.includes("actions/runners")) {
|
|
351
|
+
return JSON.stringify({
|
|
352
|
+
runners: [
|
|
353
|
+
{ id: 1, name: "r1", status: "online", labels: [{ name: "self-hosted" }, { name: "domio-runner" }] },
|
|
354
|
+
{ id: 2, name: "r2", status: "online", labels: [{ name: "self-hosted" }, { name: "domio-runner" }] },
|
|
355
|
+
],
|
|
356
|
+
});
|
|
357
|
+
}
|
|
358
|
+
if (path.includes("actions/runs")) return JSON.stringify({ workflow_runs: [] });
|
|
359
|
+
throw new Error(`unexpected ${path}`);
|
|
360
|
+
};
|
|
361
|
+
const code = runCli({
|
|
362
|
+
argv: ["--config", configPath],
|
|
363
|
+
cwd: process.cwd(),
|
|
364
|
+
stdout,
|
|
365
|
+
stderr,
|
|
366
|
+
runGh: healthyRunGh,
|
|
367
|
+
summaryPath: undefined,
|
|
368
|
+
nowMs: NOW,
|
|
369
|
+
});
|
|
370
|
+
assert.equal(code, 0);
|
|
371
|
+
assert.match(stdout.buf, /Fleet healthy|Runner-fleet health dashboard/);
|
|
372
|
+
} finally {
|
|
373
|
+
rmSync(dir, { recursive: true, force: true });
|
|
374
|
+
}
|
|
375
|
+
});
|
|
376
|
+
|
|
377
|
+
test("runCli returns 1 on a bad config path", () => {
|
|
378
|
+
const stdout = { buf: "", write(s) { this.buf += s; } };
|
|
379
|
+
const stderr = { buf: "", write(s) { this.buf += s; } };
|
|
380
|
+
const code = runCli({
|
|
381
|
+
argv: ["--config", "scripts/does-not-exist.json"],
|
|
382
|
+
cwd: process.cwd(),
|
|
383
|
+
stdout,
|
|
384
|
+
stderr,
|
|
385
|
+
runGh: () => "{}",
|
|
386
|
+
});
|
|
387
|
+
assert.equal(code, 1);
|
|
388
|
+
assert.match(stderr.buf, /failed to read config/);
|
|
389
|
+
});
|
|
@@ -570,22 +570,49 @@ test("--check-ruleset requires --consumer-repo", () => {
|
|
|
570
570
|
}, /consumer-repo/);
|
|
571
571
|
});
|
|
572
572
|
|
|
573
|
-
test("apply materializes the canonical deploy-staging
|
|
573
|
+
test("apply materializes the canonical deploy-staging dispatcher + run templates (Story #272)", () => {
|
|
574
574
|
const out = JSON.parse(run([]));
|
|
575
|
-
const
|
|
576
|
-
|
|
575
|
+
const dispatcher = join(consumer, ".github", "workflows", "deploy-staging.yml");
|
|
576
|
+
const runner = join(consumer, ".github", "workflows", "deploy-staging-run.yml");
|
|
577
|
+
assert.ok(existsSync(dispatcher), "deploy-staging.yml (dispatcher) materialized");
|
|
578
|
+
assert.ok(existsSync(runner), "deploy-staging-run.yml (deploy) materialized");
|
|
577
579
|
assert.ok(
|
|
578
580
|
out.workflowStubs.created.some((f) => f.endsWith("deploy-staging.yml")),
|
|
579
|
-
"deploy-staging.yml reported as created"
|
|
581
|
+
"deploy-staging.yml (dispatcher) reported as created"
|
|
582
|
+
);
|
|
583
|
+
assert.ok(
|
|
584
|
+
out.workflowStubs.created.some((f) => f.endsWith("deploy-staging-run.yml")),
|
|
585
|
+
"deploy-staging-run.yml (deploy) reported as created"
|
|
586
|
+
);
|
|
587
|
+
const dispatcherBody = readFileSync(dispatcher, "utf8");
|
|
588
|
+
const runnerBody = readFileSync(runner, "utf8");
|
|
589
|
+
// Both halves carry the never-clobber template marker.
|
|
590
|
+
assert.ok(
|
|
591
|
+
dispatcherBody.includes("Canonical staging-deploy caller template"),
|
|
592
|
+
"dispatcher carries the template marker"
|
|
593
|
+
);
|
|
594
|
+
assert.ok(
|
|
595
|
+
runnerBody.includes("Canonical staging-deploy caller template"),
|
|
596
|
+
"run template carries the template marker"
|
|
597
|
+
);
|
|
598
|
+
// The dispatcher fires on CI-green (workflow_run) and DISPATCHES the run
|
|
599
|
+
// workflow — it must NOT call the deploy directly, since a workflow_run
|
|
600
|
+
// deploy skips every environment: job (Story #272).
|
|
601
|
+
assert.ok(
|
|
602
|
+
dispatcherBody.includes("workflow_run") &&
|
|
603
|
+
dispatcherBody.includes("gh workflow run deploy-staging-run.yml"),
|
|
604
|
+
"dispatcher fires on workflow_run and dispatches deploy-staging-run.yml"
|
|
580
605
|
);
|
|
581
|
-
const body = readFileSync(stub, "utf8");
|
|
582
606
|
assert.ok(
|
|
583
|
-
|
|
584
|
-
"
|
|
607
|
+
!dispatcherBody.includes("dsj1984/mandrel-platform/.github/workflows/deploy-cloudflare.yml"),
|
|
608
|
+
"dispatcher does NOT uses: deploy-cloudflare.yml directly (that would skip environment: jobs on workflow_run)"
|
|
585
609
|
);
|
|
610
|
+
// The deploy half runs on workflow_dispatch (where environment: jobs execute)
|
|
611
|
+
// and uses the shared reusable workflow.
|
|
586
612
|
assert.ok(
|
|
587
|
-
|
|
588
|
-
|
|
613
|
+
runnerBody.includes("workflow_dispatch") &&
|
|
614
|
+
runnerBody.includes("dsj1984/mandrel-platform/.github/workflows/deploy-cloudflare.yml"),
|
|
615
|
+
"run template deploys on workflow_dispatch via the shared deploy-cloudflare.yml"
|
|
589
616
|
);
|
|
590
617
|
});
|
|
591
618
|
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$comment": "Data-driven expected roster for scripts/check-runner-health.mjs (Story #258). Each entry is one repo whose self-hosted runner fleet the scheduled runner-fleet-health.yml workflow monitors. Adding/removing a runner needs only an edit here — the checker reads GET /repos/{owner}/{repo}/actions/runners and compares live status against `expectedCount` + `labels`. All nine runners (including Beestera/swarm-os's three) are co-resident on one operator Mac (2026-07-03 runner audit, repo-ops matrix §1a); if that host sleeps, reboots, fills its disk, or a launchd service dies, every listed repo's CI silently queues with no alert until this monitor catches it.",
|
|
3
|
+
"$comment_swarm_os": "Beestera/swarm-os is deliberately NOT listed: no dsj1984-owned PAT can read another org's runner API (fine-grained PATs are bound to one resource owner; the Beestera org rejects classic PATs), so its row would permanently false-positive as 0/3 degraded. Host-level coverage is retained regardless — its runners share the Mac with the rows below, so a wedged host still trips domio/athportal. What is NOT covered: swarm-os's individual launchd services dying while the host stays healthy, and its stale-queue check. Re-add the entry if a Beestera-owned credential (fine-grained PAT or GitHub App) plus per-repo token support ever lands.",
|
|
4
|
+
"$comment_staleQueuedMinutes": "A queued/waiting workflow run older than this many minutes with no online runner matching its labels is flagged as a queue-staleness signal (optional per-repo override via `staleQueuedMinutes`).",
|
|
5
|
+
"defaultStaleQueuedMinutes": 20,
|
|
6
|
+
"repos": [
|
|
7
|
+
{
|
|
8
|
+
"name": "domio",
|
|
9
|
+
"repo": "dsj1984/domio",
|
|
10
|
+
"expectedCount": 3,
|
|
11
|
+
"labels": ["self-hosted", "macOS", "ARM64", "domio-runner"]
|
|
12
|
+
},
|
|
13
|
+
{
|
|
14
|
+
"name": "athportal",
|
|
15
|
+
"repo": "dsj1984/athportal",
|
|
16
|
+
"expectedCount": 3,
|
|
17
|
+
"labels": ["self-hosted", "macOS", "ARM64", "athportal-runner"]
|
|
18
|
+
}
|
|
19
|
+
]
|
|
20
|
+
}
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
# Runner Fleet Health Monitor
|
|
2
|
+
|
|
3
|
+
> **Self-contained runbook** (not a thin stub). Unlike most templates in this
|
|
4
|
+
> directory, there is no canonical `docs/runbooks/` counterpart to link — this
|
|
5
|
+
> file IS the process, mirroring `runner-provisioning.md`. It documents the
|
|
6
|
+
> scheduled `.github/workflows/runner-fleet-health.yml` monitor (Story #258):
|
|
7
|
+
> what it checks, the token scope it needs, the alert semantics, and the
|
|
8
|
+
> operator response when it fires.
|
|
9
|
+
|
|
10
|
+
---
|
|
11
|
+
|
|
12
|
+
## Why this exists
|
|
13
|
+
|
|
14
|
+
All self-hosted runners across the fleet (`domio`, `athportal`, `swarm-os`)
|
|
15
|
+
are **co-resident on one operator Mac**. If that host sleeps, reboots for an
|
|
16
|
+
OS update, fills its disk, or a launchd runner service dies, **every
|
|
17
|
+
consumer's CI and deploy-trigger jobs silently queue** ("waiting for a
|
|
18
|
+
runner") with no alert. Nothing else watches this:
|
|
19
|
+
|
|
20
|
+
- The Better Stack uptime unit (`uptime-apply.yml`) monitors the **deployed
|
|
21
|
+
apps**, not the runners.
|
|
22
|
+
- Deploy pipelines are `workflow_run`-gated, so a wedged runner can silently
|
|
23
|
+
stall staging indefinitely — no failed job, no notification, just a queue
|
|
24
|
+
that never drains.
|
|
25
|
+
|
|
26
|
+
> **Roster note:** `Beestera/swarm-os` is monitored only *indirectly*. No
|
|
27
|
+
> `dsj1984`-owned token can read another org's runner API (fine-grained PATs
|
|
28
|
+
> are bound to one resource owner; the Beestera org rejects classic PATs), so
|
|
29
|
+
> its roster entry would permanently false-positive as `0/3` degraded.
|
|
30
|
+
> Because its runners share the Mac with the rostered repos, a wedged
|
|
31
|
+
> **host** still trips the `domio`/`athportal` rows; what goes unwatched is a
|
|
32
|
+
> swarm-os-only launchd service death and its stale queue. See
|
|
33
|
+
> `$comment_swarm_os` in `scripts/runner-fleet-consumers.json`.
|
|
34
|
+
|
|
35
|
+
`runner-fleet-health.yml` is the standing check that catches this fast.
|
|
36
|
+
|
|
37
|
+
## What it checks
|
|
38
|
+
|
|
39
|
+
Runs on a schedule (~every 15 minutes) plus `workflow_dispatch`, on
|
|
40
|
+
`ubuntu-latest` (deliberately GitHub-hosted so it keeps running when the Mac
|
|
41
|
+
is down). For each repo in `scripts/runner-fleet-consumers.json` it calls
|
|
42
|
+
`GET /repos/{owner}/{repo}/actions/runners` and:
|
|
43
|
+
|
|
44
|
+
1. **Offline runners** — flags any runner whose `status != online`.
|
|
45
|
+
2. **Count shortfall** — flags fewer online runners matching the repo's
|
|
46
|
+
expected `labels` set than its configured `expectedCount`.
|
|
47
|
+
3. **Stale queued runs** (optional signal) — a `queued`/`waiting` workflow run
|
|
48
|
+
older than `staleQueuedMinutes` (default 20) with no online runner matching
|
|
49
|
+
its labels. This catches the case where the runner *looks* present in the
|
|
50
|
+
roster count but is actually wedged and not claiming jobs.
|
|
51
|
+
|
|
52
|
+
It renders a per-repo dashboard on `GITHUB_STEP_SUMMARY`.
|
|
53
|
+
|
|
54
|
+
## Config-driven roster
|
|
55
|
+
|
|
56
|
+
Adding, removing, or resizing a runner needs **only a config edit** —
|
|
57
|
+
`scripts/runner-fleet-consumers.json`:
|
|
58
|
+
|
|
59
|
+
```jsonc
|
|
60
|
+
{
|
|
61
|
+
"defaultStaleQueuedMinutes": 20,
|
|
62
|
+
"repos": [
|
|
63
|
+
{ "name": "domio", "repo": "dsj1984/domio", "expectedCount": 3, "labels": ["self-hosted", "macOS", "ARM64", "domio-runner"] },
|
|
64
|
+
// ... one object per repo
|
|
65
|
+
],
|
|
66
|
+
}
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
## Token scope: `PIN_DRIFT_TOKEN`
|
|
70
|
+
|
|
71
|
+
The monitor reuses the same fine-grained PAT `pin-drift.yml` already
|
|
72
|
+
provisions (`secrets.PIN_DRIFT_TOKEN`), falling back to the built-in
|
|
73
|
+
`github.token` when the secret is absent (the built-in token only grants read
|
|
74
|
+
access to the workflow's own repo — cross-repo rows then surface as `⚠️
|
|
75
|
+
error` rather than hard-failing this repo's own row).
|
|
76
|
+
|
|
77
|
+
For the runner reads, `PIN_DRIFT_TOKEN` must carry, on every rostered repo:
|
|
78
|
+
|
|
79
|
+
- **Administration: read** — required by `GET .../actions/runners` (the
|
|
80
|
+
self-hosted runner list is an admin-surface endpoint; `actions:read` is
|
|
81
|
+
NOT sufficient for it).
|
|
82
|
+
- **Actions: read** — required by `GET .../actions/runs` (the stale-queue
|
|
83
|
+
check).
|
|
84
|
+
|
|
85
|
+
Resource-owner caveat: a fine-grained PAT is bound to a **single** resource
|
|
86
|
+
owner, and the Beestera org rejects classic PATs — which is exactly why
|
|
87
|
+
`Beestera/swarm-os` is off the roster (see the roster note above). Every
|
|
88
|
+
rostered repo must be readable by the ONE token this workflow gets; a repo
|
|
89
|
+
the token cannot see 404s and false-positives as degraded, so extend the
|
|
90
|
+
roster only together with a credential that covers the new repo.
|
|
91
|
+
|
|
92
|
+
When the token lacks visibility, GitHub returns **404** (not 403) and the
|
|
93
|
+
script treats the empty runner list as a real shortfall — the repo's row
|
|
94
|
+
reads `❌ degraded` with `0/N` online even when the runners are healthy. A
|
|
95
|
+
fleet-wide `0/N` across every repo is the token-misconfiguration signature;
|
|
96
|
+
check the secret before touching the runner host.
|
|
97
|
+
|
|
98
|
+
## Alert semantics
|
|
99
|
+
|
|
100
|
+
Alert-only by design (no host-side remediation) — the monitor never touches
|
|
101
|
+
the runner host itself. One channel fires on an unhealthy repo, deliberately
|
|
102
|
+
without adding a new external dependency:
|
|
103
|
+
|
|
104
|
+
- **Native GitHub failed-workflow notification.** The job script exits
|
|
105
|
+
non-zero when any repo is unhealthy, so GitHub's own email/notification
|
|
106
|
+
settings fire the standard "workflow run failed" alert to whoever
|
|
107
|
+
watches this repo. No tracking issues are filed — the dashboard detail
|
|
108
|
+
lives on the failed run's job summary.
|
|
109
|
+
|
|
110
|
+
A future Slack/PagerDuty push could layer on top of this later — deliberately
|
|
111
|
+
deferred (see the Story's Out of Scope) to avoid a new external dependency for
|
|
112
|
+
the initial alert-only default.
|
|
113
|
+
|
|
114
|
+
## Operator response
|
|
115
|
+
|
|
116
|
+
When the scheduled workflow run fails:
|
|
117
|
+
|
|
118
|
+
1. **Read the dashboard** on the workflow run's job summary — it names which
|
|
119
|
+
signal fired (offline runner, count shortfall, or stale queued run) and
|
|
120
|
+
for which repo.
|
|
121
|
+
2. **Wake or reboot the Mac** if it's asleep, powered off, or unresponsive
|
|
122
|
+
over SSH.
|
|
123
|
+
3. **Check disk space** (`df -h`) — a full disk is a common launchd-runner
|
|
124
|
+
death cause; free space and restart the affected runner service(s).
|
|
125
|
+
4. **Restart the launchd runner service(s)** for the affected repo:
|
|
126
|
+
```bash
|
|
127
|
+
cd <RUNNER_DIR> # see templates/runbooks/runner-provisioning.md
|
|
128
|
+
./svc.sh stop && ./svc.sh start
|
|
129
|
+
./svc.sh status # expect: Started · running
|
|
130
|
+
```
|
|
131
|
+
5. **Re-run the monitor** (`workflow_dispatch` from the Actions tab, or wait
|
|
132
|
+
for the next 15-minute tick) to confirm recovery — a green run means the
|
|
133
|
+
fleet reports healthy again.
|
|
134
|
+
|
|
135
|
+
## Out of scope (Story #258)
|
|
136
|
+
|
|
137
|
+
- Host-side remediation / auto-recovery (waking the Mac, restarting services)
|
|
138
|
+
— this monitor is alert-only; the operator performs the response above by
|
|
139
|
+
hand.
|
|
140
|
+
- Host disk-usage monitoring — not exposable via the runners API anyway, and
|
|
141
|
+
tracked separately.
|
|
142
|
+
- External paging integrations beyond the native failed-workflow
|
|
143
|
+
notification.
|
|
144
|
+
- Cross-repo runner isolation / ephemeral-runner questions — explicitly
|
|
145
|
+
deferred.
|
|
146
|
+
|
|
147
|
+
## Project-Specific Notes
|
|
148
|
+
|
|
149
|
+
<!-- Record host quirks, roster changes, or false-positive tuning
|
|
150
|
+
(staleQueuedMinutes overrides) specific to this fleet. -->
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
name: deploy-staging-run
|
|
2
|
+
|
|
3
|
+
# Canonical staging-deploy caller template — DEPLOY half (Story #272).
|
|
4
|
+
#
|
|
5
|
+
# > **Why this file runs on `workflow_dispatch`.** Its sibling
|
|
6
|
+
# > `deploy-staging.yml` fires on CI-green (`workflow_run`) and DISPATCHES this
|
|
7
|
+
# > workflow. The deploy lives here, on `workflow_dispatch`, because the shared
|
|
8
|
+
# > `deploy-cloudflare.yml`'s `environment:`-gated jobs (`check-env` /
|
|
9
|
+
# > `migration` / `deploy` / `boot-smoke`) are silently SKIPPED on a
|
|
10
|
+
# > `workflow_run` event but run normally on `workflow_dispatch` (Story #272).
|
|
11
|
+
# > Adopt this file together with `deploy-staging.yml`; `platform-sync`
|
|
12
|
+
# > materializes both.
|
|
13
|
+
#
|
|
14
|
+
# > **Thin local caller.** The defence-in-depth deploy core lives in the shared
|
|
15
|
+
# > `dsj1984/mandrel-platform` `deploy-cloudflare.yml` reusable workflow — see
|
|
16
|
+
# > https://github.com/dsj1984/mandrel-platform/blob/main/docs/reusable-workflows.md#deploy-cloudflareyml.
|
|
17
|
+
# > This file only holds <PROJECT_NAME>-specific values (worker names, build
|
|
18
|
+
# > step, secret mapping). When the deploy PROCESS changes, that change lands
|
|
19
|
+
# > upstream in mandrel-platform — not here.
|
|
20
|
+
#
|
|
21
|
+
# Replace every <PLACEHOLDER> with your project's real values:
|
|
22
|
+
# <MANDREL_PLATFORM_SHA> the pinned mandrel-platform commit SHA (resolve via
|
|
23
|
+
# `node scripts/platform-sync.mjs --ref <release-tag>`
|
|
24
|
+
# from the consumer repo root, or `git ls-remote`).
|
|
25
|
+
# <MANDREL_PLATFORM_TAG> the human-readable release tag matching the SHA
|
|
26
|
+
# above (trailing `# <tag>` comment).
|
|
27
|
+
# <WORKERS_CSV> comma-separated Worker names for this env, e.g.
|
|
28
|
+
# "api,web".
|
|
29
|
+
# <BUILD_COMMAND> optional build command (omit build-command /
|
|
30
|
+
# build-artifact entirely if the deploy job's default
|
|
31
|
+
# checkout is build-ready).
|
|
32
|
+
#
|
|
33
|
+
# See the full input/secret contract:
|
|
34
|
+
# https://github.com/dsj1984/mandrel-platform/blob/main/docs/reusable-workflows.md#deploy-cloudflareyml
|
|
35
|
+
|
|
36
|
+
on:
|
|
37
|
+
# Dispatched by deploy-staging.yml on CI-green, and available for manual
|
|
38
|
+
# on-demand deploys (UI "Run workflow" + `gh workflow run`). The optional sha
|
|
39
|
+
# input records the CI-verified commit; the deploy itself runs against the
|
|
40
|
+
# main tip (`--ref main`).
|
|
41
|
+
workflow_dispatch:
|
|
42
|
+
inputs:
|
|
43
|
+
sha:
|
|
44
|
+
description: >
|
|
45
|
+
Commit SHA that passed CI (informational — the deploy runs against
|
|
46
|
+
the current main tip). Populated automatically when dispatched by
|
|
47
|
+
deploy-staging.yml.
|
|
48
|
+
required: false
|
|
49
|
+
type: string
|
|
50
|
+
|
|
51
|
+
permissions:
|
|
52
|
+
contents: read
|
|
53
|
+
|
|
54
|
+
# Serialize staging deploys: only the freshest dispatch should reach the
|
|
55
|
+
# staging surfaces. The shared deploy-cloudflare.yml additionally serializes
|
|
56
|
+
# per-environment.
|
|
57
|
+
concurrency:
|
|
58
|
+
group: deploy-staging-run
|
|
59
|
+
cancel-in-progress: true
|
|
60
|
+
|
|
61
|
+
jobs:
|
|
62
|
+
deploy:
|
|
63
|
+
name: Staging deploy (shared deploy-cloudflare.yml)
|
|
64
|
+
uses: dsj1984/mandrel-platform/.github/workflows/deploy-cloudflare.yml@<MANDREL_PLATFORM_SHA> # <MANDREL_PLATFORM_TAG>
|
|
65
|
+
with:
|
|
66
|
+
environment: staging
|
|
67
|
+
gh-environment: staging
|
|
68
|
+
workers: <WORKERS_CSV>
|
|
69
|
+
migrate: true
|
|
70
|
+
# db-engine defaults to 'd1'. Set db-engine + migrate-command +
|
|
71
|
+
# snapshot-command for a non-D1 engine (e.g. Turso) — see the contract
|
|
72
|
+
# doc's "command seams" section.
|
|
73
|
+
# Frozen secret allowlist: only {CLOUDFLARE_*, TURSO_*} cross into the
|
|
74
|
+
# shared workflow. Map your project's secret NAMES onto these slots.
|
|
75
|
+
secrets:
|
|
76
|
+
CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
|
|
77
|
+
CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
|
|
@@ -1,89 +1,91 @@
|
|
|
1
1
|
name: deploy-staging
|
|
2
2
|
|
|
3
|
-
# Canonical staging-deploy caller template (Story #175
|
|
3
|
+
# Canonical staging-deploy caller template — DISPATCHER half (Story #175,
|
|
4
|
+
# reworked for Story #272).
|
|
5
|
+
#
|
|
6
|
+
# > **Why two files (Story #272).** A reusable workflow's `environment:`-gated
|
|
7
|
+
# > jobs (`deploy-cloudflare.yml`'s `check-env` / `migration` / `deploy` /
|
|
8
|
+
# > `boot-smoke`) are **silently skipped on a `workflow_run` event** — a
|
|
9
|
+
# > documented GitHub limitation. The previous single-file template called
|
|
10
|
+
# > `deploy-cloudflare.yml` DIRECTLY from `on: workflow_run`, so those jobs
|
|
11
|
+
# > skipped and **0 workers deployed while the run reported green** (all-skipped,
|
|
12
|
+
# > none failed). One consumer hid 40+ consecutive non-deploys this way.
|
|
13
|
+
# >
|
|
14
|
+
# > The fix: this file is now a thin **dispatcher** that fires on CI-green and
|
|
15
|
+
# > re-launches the deploy via **`workflow_dispatch`** — the event on which the
|
|
16
|
+
# > `environment:` jobs DO run — against its sibling `deploy-staging-run.yml`.
|
|
17
|
+
# > `workflow_dispatch` and `repository_dispatch` are the two events that
|
|
18
|
+
# > "always create workflow runs" even when triggered with the built-in
|
|
19
|
+
# > `GITHUB_TOKEN`, so **no PAT is required** for this same-repo dispatch — just
|
|
20
|
+
# > `permissions: actions: write` below. (Cross-repo dispatch, like
|
|
21
|
+
# > `smoke-dispatch.yml`, still needs a PAT; same repo does not.)
|
|
4
22
|
#
|
|
5
23
|
# > **Thin local caller.** The defence-in-depth deploy core (secret-isolation
|
|
6
24
|
# > audit -> CF env gate -> migration (snapshot + apply) -> deploy ->
|
|
7
|
-
# > boot-smoke + auto-rollback)
|
|
25
|
+
# > boot-smoke + auto-rollback) lives in the shared
|
|
8
26
|
# > `dsj1984/mandrel-platform` `deploy-cloudflare.yml` reusable workflow — see
|
|
9
27
|
# > https://github.com/dsj1984/mandrel-platform/blob/main/docs/reusable-workflows.md#deploy-cloudflareyml.
|
|
10
|
-
# >
|
|
11
|
-
# >
|
|
12
|
-
# > upstream in mandrel-platform — not here.
|
|
13
|
-
#
|
|
14
|
-
# One paved road (operator decision 2026-07-01, D4): every consumer triggers
|
|
15
|
-
# staging deploy via `workflow_run` on its own CI workflow, gated on
|
|
16
|
-
# `conclusion == 'success'`. `workflow_run` fires on BOTH a successful AND a
|
|
17
|
-
# failed upstream run, so a caller-side guard against a red run used to be
|
|
18
|
-
# REQUIRED here — every consumer hand-copied the same `preflight` job (see
|
|
19
|
-
# mandrel-platform Story #175 context). That guard now lives INSIDE
|
|
20
|
-
# `deploy-cloudflare.yml` itself as a job-level `if:` on its entry jobs
|
|
21
|
-
# (`github.event` inside a reusable workflow is the CALLER's event, so the
|
|
22
|
-
# shared workflow can see and gate on the `workflow_run` conclusion even
|
|
23
|
-
# though it cannot own this file's `on:` block); a red upstream run skips
|
|
24
|
-
# the whole chain with zero runner spin-ups (mandrel-platform Story #237).
|
|
25
|
-
# This template needs NO caller-side preflight guard as a result —
|
|
26
|
-
# copy it as-is and fill in the placeholders below.
|
|
28
|
+
# > The <PROJECT_NAME>-specific values (worker names, build step, secret
|
|
29
|
+
# > mapping) live in the sibling `deploy-staging-run.yml`. When the deploy
|
|
30
|
+
# > PROCESS changes, that change lands upstream in mandrel-platform — not here.
|
|
27
31
|
#
|
|
28
|
-
#
|
|
29
|
-
#
|
|
30
|
-
#
|
|
31
|
-
#
|
|
32
|
-
#
|
|
33
|
-
#
|
|
34
|
-
# <MANDREL_PLATFORM_SHA> the pinned mandrel-platform commit SHA (resolve
|
|
35
|
-
# via `node scripts/platform-sync.mjs --ref
|
|
36
|
-
# <release-tag>` from the consumer repo root, or
|
|
37
|
-
# hand-resolve via `git ls-remote`).
|
|
38
|
-
# <MANDREL_PLATFORM_TAG> the human-readable release tag matching the SHA
|
|
39
|
-
# above (trailing `# <tag>` comment).
|
|
40
|
-
# <WORKERS_CSV> comma-separated Worker names for this env, e.g.
|
|
41
|
-
# "api,web".
|
|
42
|
-
# <BUILD_COMMAND> optional build command (omit build-command /
|
|
43
|
-
# build-artifact entirely if the deploy job's
|
|
44
|
-
# default checkout is build-ready).
|
|
32
|
+
# Adopt BOTH files together: this `deploy-staging.yml` (dispatcher) and
|
|
33
|
+
# `deploy-staging-run.yml` (the actual deploy). `platform-sync` materializes
|
|
34
|
+
# both. Replace every <PLACEHOLDER>:
|
|
35
|
+
# <CI_WORKFLOW_NAME> the `name:` of the workflow this deploy gates on (e.g.
|
|
36
|
+
# "quality", "CI"). Must match EXACTLY — GitHub matches
|
|
37
|
+
# `workflow_run.workflows` by workflow name, not path.
|
|
45
38
|
#
|
|
46
39
|
# See the full input/secret contract:
|
|
47
40
|
# https://github.com/dsj1984/mandrel-platform/blob/main/docs/reusable-workflows.md#deploy-cloudflareyml
|
|
48
41
|
|
|
49
42
|
on:
|
|
50
|
-
# CI-green gate: fires when <CI_WORKFLOW_NAME>
|
|
51
|
-
#
|
|
52
|
-
#
|
|
53
|
-
#
|
|
43
|
+
# CI-green gate: fires when <CI_WORKFLOW_NAME> completes on main. Unlike the
|
|
44
|
+
# old shape, this does NOT call the deploy directly — a `workflow_run` deploy
|
|
45
|
+
# would skip every `environment:` job. It only DISPATCHES the deploy (below)
|
|
46
|
+
# on success, so the actual deploy runs on `workflow_dispatch` where those
|
|
47
|
+
# jobs execute. A red upstream run simply does not dispatch — there is no
|
|
48
|
+
# green-but-didn't-deploy run at all.
|
|
54
49
|
workflow_run:
|
|
55
50
|
workflows: [<CI_WORKFLOW_NAME>]
|
|
56
51
|
branches: [main]
|
|
57
52
|
types: [completed]
|
|
58
|
-
# Manual on-demand trigger (UI "Run workflow" + `gh workflow run`).
|
|
59
|
-
# workflow_dispatch always passes the shared workflow's CI-green guard
|
|
60
|
-
# (operator-intentional, no upstream conclusion to gate on).
|
|
61
|
-
workflow_dispatch:
|
|
62
53
|
|
|
54
|
+
# actions:write lets the built-in GITHUB_TOKEN dispatch deploy-staging-run.yml
|
|
55
|
+
# via the workflow_dispatch API. No PAT needed for a same-repo dispatch —
|
|
56
|
+
# workflow_dispatch always creates a run even from GITHUB_TOKEN.
|
|
63
57
|
permissions:
|
|
64
58
|
contents: read
|
|
59
|
+
actions: write
|
|
65
60
|
|
|
66
|
-
#
|
|
67
|
-
#
|
|
68
|
-
# deploy-
|
|
61
|
+
# Only the freshest tip of main should reach staging: cancel an in-flight
|
|
62
|
+
# DISPATCH when a newer CI run completes. (The deploy itself is serialized
|
|
63
|
+
# separately in deploy-staging-run.yml and per-environment inside the shared
|
|
64
|
+
# deploy-cloudflare.yml.)
|
|
69
65
|
concurrency:
|
|
70
|
-
group: deploy-staging
|
|
66
|
+
group: deploy-staging-dispatch
|
|
71
67
|
cancel-in-progress: true
|
|
72
68
|
|
|
73
69
|
jobs:
|
|
74
|
-
|
|
75
|
-
name:
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
70
|
+
dispatch:
|
|
71
|
+
name: Dispatch staging deploy on CI-green
|
|
72
|
+
runs-on: ubuntu-latest
|
|
73
|
+
timeout-minutes: 5
|
|
74
|
+
# CI-green gate: dispatch the deploy ONLY when the upstream CI run
|
|
75
|
+
# concluded 'success'. `workflow_run` fires on both success and failure, so
|
|
76
|
+
# this guard is load-bearing — without it a red main would still deploy.
|
|
77
|
+
if: ${{ github.event.workflow_run.conclusion == 'success' }}
|
|
78
|
+
steps:
|
|
79
|
+
- name: Dispatch deploy-staging-run.yml (workflow_dispatch)
|
|
80
|
+
env:
|
|
81
|
+
GH_TOKEN: ${{ github.token }}
|
|
82
|
+
REPO: ${{ github.repository }}
|
|
83
|
+
SHA: ${{ github.event.workflow_run.head_sha }}
|
|
84
|
+
shell: bash
|
|
85
|
+
run: |
|
|
86
|
+
set -euo pipefail
|
|
87
|
+
gh workflow run deploy-staging-run.yml \
|
|
88
|
+
--repo "${REPO}" \
|
|
89
|
+
--ref main \
|
|
90
|
+
-f sha="${SHA}"
|
|
91
|
+
echo "Dispatched staging deploy for ${SHA} (deploy runs on workflow_dispatch so environment: jobs execute)."
|