mandrel-platform 0.21.0 → 0.24.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "mandrel-platform",
3
- "version": "0.21.0",
3
+ "version": "0.24.0",
4
4
  "description": "Shared CI/deploy workflows, composite toolchain action, npm config package, Renovate preset, and operator runbook templates.",
5
5
  "license": "MIT",
6
6
  "repository": {
@@ -0,0 +1,469 @@
1
+ #!/usr/bin/env node
2
+ /**
3
+ * check-runner-health.mjs
4
+ *
5
+ * Scheduled runner-fleet health monitor for mandrel-platform (Story #258).
6
+ *
7
+ * All nine self-hosted runners across the fleet (domio, athportal, swarm-os)
8
+ * are co-resident on ONE operator Mac (2026-07-03 runner audit, repo-ops
9
+ * matrix §1a). If that host sleeps, reboots for an OS update, fills its disk,
10
+ * or a launchd service dies, every consumer's CI and deploy-trigger jobs
11
+ * silently queue ("waiting for a runner") with no alert — nothing watches the
12
+ * fleet today. This script is the standing check that closes that gap.
13
+ *
14
+ * Modeled on `check-pin-drift.mjs` (same GitHub-hosted, config-driven,
15
+ * `GITHUB_STEP_SUMMARY`-rendering shape via the `scripts/lib/gh-json.mjs`
16
+ * seam) with two differences: it must run FREQUENTLY (not weekly) and it must
17
+ * ALERT (not just render a dashboard), because the whole point is catching an
18
+ * offline fleet fast.
19
+ *
20
+ * For each repo in `scripts/runner-fleet-consumers.json` it:
21
+ * 1. Calls `GET /repos/{owner}/{repo}/actions/runners` and flags any runner
22
+ * whose `status !== "online"`.
23
+ * 2. Flags a count shortfall: fewer online runners matching the expected
24
+ * `labels` set than `expectedCount`.
25
+ * 3. Flags queue-staleness: a `queued`/`waiting` workflow run older than
26
+ * `staleQueuedMinutes` with no online runner matching its labels
27
+ * (a wedged fleet accepts jobs into the queue but never claims them).
28
+ * 4. Renders a per-repo dashboard to `GITHUB_STEP_SUMMARY`.
29
+ *
30
+ * Alerting (no external dependency, on-ethos default): the CLI exits
31
+ * non-zero when any repo is unhealthy, so GitHub's native failed-workflow
32
+ * notification fires. That is the only alert channel — no tracking issues
33
+ * are filed.
34
+ *
35
+ * GitHub access is via the `gh` CLI (`gh api`), through the same injectable
36
+ * `runGh` seam `check-pin-drift.mjs` uses (`scripts/lib/gh-json.mjs`), so the
37
+ * whole pipeline is exercised offline in tests with canned responses.
38
+ *
39
+ * Usage:
40
+ * node scripts/check-runner-health.mjs
41
+ * node scripts/check-runner-health.mjs --config scripts/runner-fleet-consumers.json
42
+ * node scripts/check-runner-health.mjs --json # machine-readable envelope
43
+ *
44
+ * Exit codes:
45
+ * 0 — every repo healthy.
46
+ * 1 — at least one repo unhealthy (offline runner, count shortfall, stale
47
+ * queued run, or a fetch error), OR a fatal error (bad config).
48
+ *
49
+ * GitHub Actions: when GITHUB_STEP_SUMMARY is set, the human-readable report
50
+ * is also appended there so it renders on the job summary page.
51
+ */
52
+
53
+ import { readFileSync, appendFileSync } from "node:fs";
54
+ import { resolve } from "node:path";
55
+
56
+ import { defaultGhRunner, ghApiJson, isNotFound } from "./lib/gh-json.mjs";
57
+
58
+ // ---------------------------------------------------------------------------
59
+ // Arg parsing
60
+ // ---------------------------------------------------------------------------
61
+
62
+ /**
63
+ * @param {string[]} argv
64
+ * @returns {{ config: string, json: boolean }}
65
+ */
66
+ export function parseArgv(argv = []) {
67
+ let config = "scripts/runner-fleet-consumers.json";
68
+ let json = false;
69
+ for (let i = 0; i < argv.length; i += 1) {
70
+ const a = argv[i];
71
+ if (a === "--config") {
72
+ const next = argv[i + 1];
73
+ if (next && !next.startsWith("--")) {
74
+ config = next;
75
+ i += 1;
76
+ }
77
+ } else if (a === "--json") {
78
+ json = true;
79
+ }
80
+ }
81
+ return { config, json };
82
+ }
83
+
84
+ // ---------------------------------------------------------------------------
85
+ // Pure helpers (exported for unit-style probing without GitHub access)
86
+ // ---------------------------------------------------------------------------
87
+
88
+ /**
89
+ * Does a runner's label set satisfy the expected label roster? A runner
90
+ * "matches" when every expected label is present among the runner's labels
91
+ * (order-independent, extra labels on the runner are fine).
92
+ *
93
+ * @param {string[]} runnerLabels
94
+ * @param {string[]} expectedLabels
95
+ * @returns {boolean}
96
+ */
97
+ export function runnerMatchesLabels(runnerLabels, expectedLabels) {
98
+ const have = new Set((runnerLabels || []).map((l) => l.toLowerCase()));
99
+ return (expectedLabels || []).every((l) => have.has(l.toLowerCase()));
100
+ }
101
+
102
+ /**
103
+ * Classify one repo's live runner list against its expected roster.
104
+ *
105
+ * @param {Array<{ id: number, name: string, status: string, busy?: boolean, labels: Array<{ name: string }> }>} runners
106
+ * @param {{ expectedCount: number, labels: string[] }} expected
107
+ * @returns {{
108
+ * total: number,
109
+ * online: number,
110
+ * offline: Array<{ id: number, name: string, status: string }>,
111
+ * matchingOnline: number,
112
+ * shortfall: number,
113
+ * hasShortfall: boolean,
114
+ * hasOffline: boolean,
115
+ * }}
116
+ */
117
+ export function classifyRunners(runners, expected) {
118
+ const list = Array.isArray(runners) ? runners : [];
119
+ const offline = list
120
+ .filter((r) => r.status !== "online")
121
+ .map((r) => ({ id: r.id, name: r.name, status: r.status }));
122
+ const online = list.filter((r) => r.status === "online");
123
+ const matchingOnline = online.filter((r) =>
124
+ runnerMatchesLabels(
125
+ (r.labels || []).map((l) => (typeof l === "string" ? l : l.name)),
126
+ expected.labels,
127
+ ),
128
+ ).length;
129
+ const expectedCount =
130
+ typeof expected.expectedCount === "number" ? expected.expectedCount : 0;
131
+ const shortfall = Math.max(0, expectedCount - matchingOnline);
132
+ return {
133
+ total: list.length,
134
+ online: online.length,
135
+ offline,
136
+ matchingOnline,
137
+ shortfall,
138
+ hasShortfall: shortfall > 0,
139
+ hasOffline: offline.length > 0,
140
+ };
141
+ }
142
+
143
+ /**
144
+ * Is a workflow run "stuck in queue"? True when its status is `queued` or
145
+ * `waiting`, it is older than `staleMinutes`, and no online runner in
146
+ * `onlineLabelSets` matches every one of its `labels` (i.e. nothing could ever
147
+ * pick it up).
148
+ *
149
+ * @param {{ status: string, created_at: string, labels?: string[] }} run A workflow run summary (labels resolved from its job requirements when available).
150
+ * @param {string[][]} onlineLabelSets Each online runner's label list.
151
+ * @param {number} staleMinutes
152
+ * @param {number} [nowMs] Injectable current epoch ms (for tests).
153
+ * @returns {boolean}
154
+ */
155
+ export function isStaleQueuedRun(run, onlineLabelSets, staleMinutes, nowMs = Date.now()) {
156
+ if (run.status !== "queued" && run.status !== "waiting") return false;
157
+ const createdMs = Date.parse(run.created_at);
158
+ if (Number.isNaN(createdMs)) return false;
159
+ const ageMinutes = (nowMs - createdMs) / 60000;
160
+ if (ageMinutes < staleMinutes) return false;
161
+ const runLabels = Array.isArray(run.labels) ? run.labels : [];
162
+ if (runLabels.length === 0) {
163
+ // No label info to match against — can't assert "nothing could pick this
164
+ // up" without over-claiming, so don't flag it as stale from labels alone.
165
+ return false;
166
+ }
167
+ const anyOnlineMatches = onlineLabelSets.some((labels) =>
168
+ runnerMatchesLabels(labels, runLabels),
169
+ );
170
+ return !anyOnlineMatches;
171
+ }
172
+
173
+ /**
174
+ * Combine the three health signals into a single per-repo health verdict.
175
+ *
176
+ * @param {ReturnType<typeof classifyRunners>} runnerVerdict
177
+ * @param {Array<object>} staleRuns
178
+ * @returns {boolean} true when the repo is healthy (no offline, no shortfall, no stale queued runs).
179
+ */
180
+ export function isRepoHealthy(runnerVerdict, staleRuns) {
181
+ return (
182
+ !runnerVerdict.hasOffline &&
183
+ !runnerVerdict.hasShortfall &&
184
+ (!Array.isArray(staleRuns) || staleRuns.length === 0)
185
+ );
186
+ }
187
+
188
+ /**
189
+ * Render the human-readable dashboard report.
190
+ *
191
+ * @param {{
192
+ * results: Array<{
193
+ * name: string,
194
+ * repo: string,
195
+ * error?: string,
196
+ * verdict?: ReturnType<typeof classifyRunners>,
197
+ * staleRuns?: Array<{ id: number, html_url?: string, created_at: string }>,
198
+ * healthy?: boolean,
199
+ * }>,
200
+ * }} report
201
+ * @returns {string}
202
+ */
203
+ export function renderReport(report) {
204
+ const out = [];
205
+ out.push("## Runner-fleet health dashboard");
206
+ out.push("");
207
+ out.push(
208
+ "All self-hosted runners are co-resident on one operator Mac — a wedged " +
209
+ "host silently stalls every listed repo's CI with no other alert.",
210
+ );
211
+ out.push("");
212
+ out.push("| Repo | Online / Expected | Offline | Stale queued | Status |");
213
+ out.push("| ---- | ------------------ | ------- | ------------- | ------ |");
214
+
215
+ const problemLines = [];
216
+ for (const r of report.results) {
217
+ if (r.error) {
218
+ out.push(`| \`${r.name}\` | — | — | — | ⚠️ error |`);
219
+ problemLines.push(`- \`${r.name}\` (${r.repo}): error — ${r.error}`);
220
+ continue;
221
+ }
222
+ const v = r.verdict;
223
+ const stale = Array.isArray(r.staleRuns) ? r.staleRuns : [];
224
+ const healthy = r.healthy === true;
225
+ const status = healthy ? "✅ healthy" : "❌ degraded";
226
+ out.push(
227
+ `| \`${r.name}\` | ${v.matchingOnline}/${v.matchingOnline + v.shortfall} | ${v.offline.length} | ${stale.length} | ${status} |`,
228
+ );
229
+ if (!healthy) {
230
+ if (v.hasOffline) {
231
+ const names = v.offline.map((o) => `\`${o.name}\` (${o.status})`).join(", ");
232
+ problemLines.push(
233
+ `- \`${r.name}\` (${r.repo}): OFFLINE runner(s) — ${names}.`,
234
+ );
235
+ }
236
+ if (v.hasShortfall) {
237
+ problemLines.push(
238
+ `- \`${r.name}\` (${r.repo}): SHORTFALL — ${v.matchingOnline} online runner(s) matching labels, expected at least ${v.matchingOnline + v.shortfall}.`,
239
+ );
240
+ }
241
+ if (stale.length > 0) {
242
+ const runs = stale
243
+ .map((run) => (run.html_url ? `[#${run.id}](${run.html_url})` : `#${run.id}`))
244
+ .join(", ");
245
+ problemLines.push(
246
+ `- \`${r.name}\` (${r.repo}): STALE QUEUED RUN(S) — ${runs} queued with no matching online runner.`,
247
+ );
248
+ }
249
+ }
250
+ }
251
+
252
+ out.push("");
253
+ if (problemLines.length > 0) {
254
+ out.push("### Degraded");
255
+ out.push("");
256
+ out.push(...problemLines);
257
+ out.push("");
258
+ out.push(
259
+ "**Operator response:** wake/reboot the Mac, check disk space, and " +
260
+ "restart the launchd runner services. See " +
261
+ "`templates/runbooks/runner-fleet-health.md`.",
262
+ );
263
+ } else {
264
+ out.push("### ✅ Fleet healthy");
265
+ out.push("");
266
+ out.push("Every configured repo has its expected online runner count and no stale queued runs.");
267
+ }
268
+ out.push("");
269
+ return out.join("\n");
270
+ }
271
+
272
+ // ---------------------------------------------------------------------------
273
+ // GitHub access — runners + workflow runs.
274
+ // ---------------------------------------------------------------------------
275
+
276
+ /**
277
+ * Fetch a repo's live self-hosted runner list.
278
+ *
279
+ * @param {string} repo "owner/name".
280
+ * @param {(args: string[]) => string} runGh
281
+ * @returns {Array<{ id: number, name: string, status: string, busy?: boolean, labels: Array<{ name: string }> }>}
282
+ */
283
+ export function fetchRunners(repo, runGh) {
284
+ let obj;
285
+ try {
286
+ obj = ghApiJson(`repos/${repo}/actions/runners?per_page=100`, runGh);
287
+ } catch (err) {
288
+ if (isNotFound(err)) return [];
289
+ throw err;
290
+ }
291
+ return Array.isArray(obj?.runners) ? obj.runners : [];
292
+ }
293
+
294
+ /**
295
+ * Fetch a repo's currently queued/waiting workflow runs (for stale-queue
296
+ * detection). Returns [] on a 404 (repo unreadable, or genuinely no runs);
297
+ * any other error propagates so the caller records an `error` row.
298
+ *
299
+ * @param {string} repo
300
+ * @param {(args: string[]) => string} runGh
301
+ * @returns {Array<{ id: number, status: string, created_at: string, html_url?: string, labels?: string[] }>}
302
+ */
303
+ export function fetchQueuedRuns(repo, runGh) {
304
+ let obj;
305
+ try {
306
+ obj = ghApiJson(
307
+ `repos/${repo}/actions/runs?status=queued&per_page=50`,
308
+ runGh,
309
+ );
310
+ } catch (err) {
311
+ if (isNotFound(err)) return [];
312
+ throw err;
313
+ }
314
+ const queued = Array.isArray(obj?.workflow_runs) ? obj.workflow_runs : [];
315
+ let waitingObj;
316
+ try {
317
+ waitingObj = ghApiJson(
318
+ `repos/${repo}/actions/runs?status=waiting&per_page=50`,
319
+ runGh,
320
+ );
321
+ } catch (err) {
322
+ if (isNotFound(err)) return queued;
323
+ throw err;
324
+ }
325
+ const waiting = Array.isArray(waitingObj?.workflow_runs)
326
+ ? waitingObj.workflow_runs
327
+ : [];
328
+ return [...queued, ...waiting];
329
+ }
330
+
331
+ // ---------------------------------------------------------------------------
332
+ // Orchestration
333
+ // ---------------------------------------------------------------------------
334
+
335
+ /**
336
+ * Build the full health report for the configured repos.
337
+ *
338
+ * @param {{
339
+ * repos: Array<{ name: string, repo: string, expectedCount: number, labels: string[], staleQueuedMinutes?: number }>,
340
+ * defaultStaleQueuedMinutes?: number,
341
+ * }} config
342
+ * @param {(args: string[]) => string} runGh
343
+ * @param {number} [nowMs]
344
+ * @returns {{ results: Array<object> }}
345
+ */
346
+ export function buildReport(config, runGh, nowMs = Date.now()) {
347
+ const defaultStale =
348
+ typeof config.defaultStaleQueuedMinutes === "number"
349
+ ? config.defaultStaleQueuedMinutes
350
+ : 20;
351
+ const results = [];
352
+ for (const entry of config.repos) {
353
+ try {
354
+ const runners = fetchRunners(entry.repo, runGh);
355
+ const verdict = classifyRunners(runners, entry);
356
+ const onlineLabelSets = runners
357
+ .filter((r) => r.status === "online")
358
+ .map((r) => (r.labels || []).map((l) => (typeof l === "string" ? l : l.name)));
359
+ const staleMinutes =
360
+ typeof entry.staleQueuedMinutes === "number"
361
+ ? entry.staleQueuedMinutes
362
+ : defaultStale;
363
+ const queuedRuns = fetchQueuedRuns(entry.repo, runGh);
364
+ const staleRuns = queuedRuns.filter((run) =>
365
+ isStaleQueuedRun(run, onlineLabelSets, staleMinutes, nowMs),
366
+ );
367
+ results.push({
368
+ name: entry.name,
369
+ repo: entry.repo,
370
+ verdict,
371
+ staleRuns,
372
+ healthy: isRepoHealthy(verdict, staleRuns),
373
+ });
374
+ } catch (err) {
375
+ results.push({
376
+ name: entry.name,
377
+ repo: entry.repo,
378
+ error: err instanceof Error ? err.message : String(err),
379
+ healthy: false,
380
+ });
381
+ }
382
+ }
383
+ return { results };
384
+ }
385
+
386
+ /**
387
+ * @param {{ results: Array<{ error?: string, healthy?: boolean }> }} report
388
+ * @returns {boolean} true when any repo is unhealthy or errored.
389
+ */
390
+ export function hasUnhealthy(report) {
391
+ return report.results.some((r) => r.error || r.healthy === false);
392
+ }
393
+
394
+ // ---------------------------------------------------------------------------
395
+ // CLI entry
396
+ // ---------------------------------------------------------------------------
397
+
398
+ /**
399
+ * @param {{
400
+ * argv?: string[],
401
+ * cwd?: string,
402
+ * stdout?: { write: (s: string) => void },
403
+ * stderr?: { write: (s: string) => void },
404
+ * runGh?: (args: string[]) => string,
405
+ * summaryPath?: string | undefined,
406
+ * nowMs?: number,
407
+ * }} [opts]
408
+ * @returns {number} exit code
409
+ */
410
+ export function runCli({
411
+ argv = process.argv.slice(2),
412
+ cwd = process.cwd(),
413
+ stdout = process.stdout,
414
+ stderr = process.stderr,
415
+ runGh = defaultGhRunner,
416
+ summaryPath = process.env.GITHUB_STEP_SUMMARY,
417
+ nowMs = Date.now(),
418
+ } = {}) {
419
+ const { config: configRel, json } = parseArgv(argv);
420
+ const configPath = resolve(cwd, configRel);
421
+
422
+ let config;
423
+ try {
424
+ config = JSON.parse(readFileSync(configPath, "utf-8"));
425
+ } catch (err) {
426
+ stderr.write(
427
+ `[runner-health] ❌ failed to read config ${configPath}: ${err instanceof Error ? err.message : String(err)}\n`,
428
+ );
429
+ return 1;
430
+ }
431
+ if (!Array.isArray(config.repos)) {
432
+ stderr.write(`[runner-health] ❌ config must define { repos: [] }\n`);
433
+ return 1;
434
+ }
435
+
436
+ const report = buildReport(config, runGh, nowMs);
437
+ const unhealthy = hasUnhealthy(report);
438
+
439
+ if (json) {
440
+ stdout.write(
441
+ `${JSON.stringify({ kind: "runner-fleet-health-report", unhealthy, ...report }, null, 2)}\n`,
442
+ );
443
+ } else {
444
+ const text = renderReport(report);
445
+ stdout.write(`${text}\n`);
446
+ if (summaryPath) {
447
+ try {
448
+ appendFileSync(summaryPath, `${text}\n`);
449
+ } catch (err) {
450
+ stderr.write(
451
+ `[runner-health] ⚠ could not write job summary: ${err instanceof Error ? err.message : String(err)}\n`,
452
+ );
453
+ }
454
+ }
455
+ }
456
+
457
+ if (unhealthy) {
458
+ stderr.write(`[runner-health] ❌ fleet degraded\n`);
459
+ return 1;
460
+ }
461
+ return 0;
462
+ }
463
+
464
+ // Direct-invocation guard (matches the repo's other scripts/*.mjs entry style).
465
+ const invokedDirectly =
466
+ process.argv[1] && resolve(process.argv[1]) === resolve(new URL(import.meta.url).pathname);
467
+ if (invokedDirectly) {
468
+ process.exit(runCli());
469
+ }
@@ -0,0 +1,389 @@
1
+ #!/usr/bin/env node
2
+ /**
3
+ * check-runner-health.test.mjs — node:test suite for the scheduled
4
+ * runner-fleet health monitor (Story #258).
5
+ *
6
+ * The checker exposes pure helpers plus an injectable `runGh` seam, so the
7
+ * whole pipeline is exercised offline with canned GitHub responses — no
8
+ * network, no `gh` auth.
9
+ *
10
+ * Run: node scripts/check-runner-health.test.mjs (or `node --test scripts/`)
11
+ */
12
+
13
+ import assert from "node:assert/strict";
14
+ import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
15
+ import { tmpdir } from "node:os";
16
+ import { join } from "node:path";
17
+ import { test } from "node:test";
18
+
19
+ import {
20
+ buildReport,
21
+ classifyRunners,
22
+ fetchQueuedRuns,
23
+ fetchRunners,
24
+ hasUnhealthy,
25
+ isRepoHealthy,
26
+ isStaleQueuedRun,
27
+ parseArgv,
28
+ renderReport,
29
+ runCli,
30
+ runnerMatchesLabels,
31
+ } from "./check-runner-health.mjs";
32
+
33
+ // ---------------------------------------------------------------------------
34
+ // parseArgv
35
+ // ---------------------------------------------------------------------------
36
+
37
+ test("parseArgv defaults to the fleet consumer config", () => {
38
+ const opts = parseArgv([]);
39
+ assert.equal(opts.config, "scripts/runner-fleet-consumers.json");
40
+ assert.equal(opts.json, false);
41
+ });
42
+
43
+ test("parseArgv parses --config and --json", () => {
44
+ const opts = parseArgv(["--config", "custom.json", "--json"]);
45
+ assert.equal(opts.config, "custom.json");
46
+ assert.equal(opts.json, true);
47
+ });
48
+
49
+ // ---------------------------------------------------------------------------
50
+ // runnerMatchesLabels
51
+ // ---------------------------------------------------------------------------
52
+
53
+ test("runnerMatchesLabels is case-insensitive and order-independent", () => {
54
+ assert.equal(
55
+ runnerMatchesLabels(["Self-Hosted", "macOS", "ARM64", "domio-runner"], [
56
+ "self-hosted",
57
+ "arm64",
58
+ ]),
59
+ true,
60
+ );
61
+ });
62
+
63
+ test("runnerMatchesLabels fails when an expected label is missing", () => {
64
+ assert.equal(
65
+ runnerMatchesLabels(["self-hosted", "macOS"], ["self-hosted", "ARM64"]),
66
+ false,
67
+ );
68
+ });
69
+
70
+ // ---------------------------------------------------------------------------
71
+ // classifyRunners
72
+ // ---------------------------------------------------------------------------
73
+
74
+ const EXPECTED = { expectedCount: 3, labels: ["self-hosted", "macOS", "ARM64", "domio-runner"] };
75
+
76
+ test("classifyRunners reports a fully healthy fleet", () => {
77
+ const runners = [1, 2, 3].map((n) => ({
78
+ id: n,
79
+ name: `domio-runner-${n}`,
80
+ status: "online",
81
+ labels: [{ name: "self-hosted" }, { name: "macOS" }, { name: "ARM64" }, { name: "domio-runner" }],
82
+ }));
83
+ const v = classifyRunners(runners, EXPECTED);
84
+ assert.equal(v.total, 3);
85
+ assert.equal(v.online, 3);
86
+ assert.equal(v.matchingOnline, 3);
87
+ assert.equal(v.shortfall, 0);
88
+ assert.equal(v.hasShortfall, false);
89
+ assert.equal(v.hasOffline, false);
90
+ });
91
+
92
+ test("classifyRunners flags an offline runner", () => {
93
+ const runners = [
94
+ { id: 1, name: "domio-runner-1", status: "online", labels: [{ name: "self-hosted" }, { name: "macOS" }, { name: "ARM64" }, { name: "domio-runner" }] },
95
+ { id: 2, name: "domio-runner-2", status: "offline", labels: [{ name: "self-hosted" }, { name: "macOS" }, { name: "ARM64" }, { name: "domio-runner" }] },
96
+ ];
97
+ const v = classifyRunners(runners, EXPECTED);
98
+ assert.equal(v.hasOffline, true);
99
+ assert.equal(v.offline.length, 1);
100
+ assert.equal(v.offline[0].name, "domio-runner-2");
101
+ assert.equal(v.matchingOnline, 1);
102
+ assert.equal(v.hasShortfall, true);
103
+ assert.equal(v.shortfall, 2);
104
+ });
105
+
106
+ test("classifyRunners flags a count shortfall even with all-online but wrong labels", () => {
107
+ const runners = [
108
+ { id: 1, name: "other-runner", status: "online", labels: [{ name: "self-hosted" }, { name: "linux" }] },
109
+ ];
110
+ const v = classifyRunners(runners, EXPECTED);
111
+ assert.equal(v.hasOffline, false);
112
+ assert.equal(v.matchingOnline, 0);
113
+ assert.equal(v.hasShortfall, true);
114
+ assert.equal(v.shortfall, 3);
115
+ });
116
+
117
+ test("classifyRunners handles an empty runner list", () => {
118
+ const v = classifyRunners([], EXPECTED);
119
+ assert.equal(v.total, 0);
120
+ assert.equal(v.hasShortfall, true);
121
+ assert.equal(v.shortfall, 3);
122
+ });
123
+
124
+ // ---------------------------------------------------------------------------
125
+ // isStaleQueuedRun
126
+ // ---------------------------------------------------------------------------
127
+
128
+ const NOW = Date.parse("2026-07-03T12:00:00Z");
129
+
130
+ test("isStaleQueuedRun flags an old queued run with no matching online runner", () => {
131
+ const run = {
132
+ status: "queued",
133
+ created_at: "2026-07-03T11:00:00Z", // 60 min ago
134
+ labels: ["self-hosted", "macOS", "ARM64", "domio-runner"],
135
+ };
136
+ assert.equal(isStaleQueuedRun(run, [], 20, NOW), true);
137
+ });
138
+
139
+ test("isStaleQueuedRun does not flag a recently queued run", () => {
140
+ const run = {
141
+ status: "queued",
142
+ created_at: "2026-07-03T11:55:00Z", // 5 min ago
143
+ labels: ["self-hosted", "macOS", "ARM64", "domio-runner"],
144
+ };
145
+ assert.equal(isStaleQueuedRun(run, [], 20, NOW), false);
146
+ });
147
+
148
+ test("isStaleQueuedRun does not flag when a matching online runner exists", () => {
149
+ const run = {
150
+ status: "queued",
151
+ created_at: "2026-07-03T11:00:00Z",
152
+ labels: ["self-hosted", "macOS", "ARM64", "domio-runner"],
153
+ };
154
+ const onlineLabelSets = [["self-hosted", "macOS", "ARM64", "domio-runner"]];
155
+ assert.equal(isStaleQueuedRun(run, onlineLabelSets, 20, NOW), false);
156
+ });
157
+
158
+ test("isStaleQueuedRun ignores non-queued/waiting runs", () => {
159
+ const run = { status: "completed", created_at: "2026-07-03T09:00:00Z" };
160
+ assert.equal(isStaleQueuedRun(run, [], 20, NOW), false);
161
+ });
162
+
163
+ test("isStaleQueuedRun does not flag when no label info is available", () => {
164
+ const run = { status: "queued", created_at: "2026-07-03T09:00:00Z", labels: [] };
165
+ assert.equal(isStaleQueuedRun(run, [], 20, NOW), false);
166
+ });
167
+
168
+ // ---------------------------------------------------------------------------
169
+ // isRepoHealthy
170
+ // ---------------------------------------------------------------------------
171
+
172
+ test("isRepoHealthy is true only with no offline/shortfall/stale", () => {
173
+ const healthyVerdict = { hasOffline: false, hasShortfall: false };
174
+ assert.equal(isRepoHealthy(healthyVerdict, []), true);
175
+ assert.equal(isRepoHealthy({ ...healthyVerdict, hasOffline: true }, []), false);
176
+ assert.equal(isRepoHealthy({ ...healthyVerdict, hasShortfall: true }, []), false);
177
+ assert.equal(isRepoHealthy(healthyVerdict, [{ id: 1 }]), false);
178
+ });
179
+
180
+ // ---------------------------------------------------------------------------
181
+ // renderReport
182
+ // ---------------------------------------------------------------------------
183
+
184
+ test("renderReport renders a healthy fleet with no Degraded section", () => {
185
+ const report = {
186
+ results: [
187
+ {
188
+ name: "domio",
189
+ repo: "dsj1984/domio",
190
+ verdict: { matchingOnline: 3, shortfall: 0, offline: [], hasOffline: false, hasShortfall: false },
191
+ staleRuns: [],
192
+ healthy: true,
193
+ },
194
+ ],
195
+ };
196
+ const text = renderReport(report);
197
+ assert.match(text, /✅ healthy/);
198
+ assert.match(text, /✅ Fleet healthy/);
199
+ assert.doesNotMatch(text, /### Degraded/);
200
+ });
201
+
202
+ test("renderReport surfaces offline / shortfall / stale-run detail lines", () => {
203
+ const report = {
204
+ results: [
205
+ {
206
+ name: "domio",
207
+ repo: "dsj1984/domio",
208
+ verdict: {
209
+ matchingOnline: 1,
210
+ shortfall: 2,
211
+ offline: [{ id: 2, name: "domio-runner-2", status: "offline" }],
212
+ hasOffline: true,
213
+ hasShortfall: true,
214
+ },
215
+ staleRuns: [{ id: 42, html_url: "https://github.com/dsj1984/domio/actions/runs/42", created_at: "x" }],
216
+ healthy: false,
217
+ },
218
+ ],
219
+ };
220
+ const text = renderReport(report);
221
+ assert.match(text, /❌ degraded/);
222
+ assert.match(text, /OFFLINE runner\(s\)/);
223
+ assert.match(text, /SHORTFALL/);
224
+ assert.match(text, /STALE QUEUED RUN/);
225
+ assert.match(text, /### Degraded/);
226
+ assert.match(text, /templates\/runbooks\/runner-fleet-health\.md/);
227
+ });
228
+
229
+ test("renderReport surfaces a fetch error row", () => {
230
+ const report = { results: [{ name: "domio", repo: "dsj1984/domio", error: "boom" }] };
231
+ const text = renderReport(report);
232
+ assert.match(text, /⚠️ error/);
233
+ assert.match(text, /error — boom/);
234
+ });
235
+
236
+ // ---------------------------------------------------------------------------
237
+ // fetchRunners / fetchQueuedRuns (injectable runGh)
238
+ // ---------------------------------------------------------------------------
239
+
240
+ test("fetchRunners returns the runners array from the API response", () => {
241
+ const runGh = () =>
242
+ JSON.stringify({ total_count: 1, runners: [{ id: 1, name: "r1", status: "online", labels: [] }] });
243
+ const runners = fetchRunners("dsj1984/domio", runGh);
244
+ assert.equal(runners.length, 1);
245
+ assert.equal(runners[0].name, "r1");
246
+ });
247
+
248
+ test("fetchRunners returns [] on a 404", () => {
249
+ const runGh = () => {
250
+ const err = new Error("gh: Not Found (HTTP 404)");
251
+ throw err;
252
+ };
253
+ assert.deepEqual(fetchRunners("dsj1984/domio", runGh), []);
254
+ });
255
+
256
+ test("fetchRunners propagates a non-404 error", () => {
257
+ const runGh = () => {
258
+ throw new Error("gh: Forbidden (HTTP 403)");
259
+ };
260
+ assert.throws(() => fetchRunners("dsj1984/domio", runGh));
261
+ });
262
+
263
+ test("fetchQueuedRuns merges queued and waiting runs", () => {
264
+ const runGh = (args) => {
265
+ const path = args[1];
266
+ if (path.includes("status=queued")) {
267
+ return JSON.stringify({ workflow_runs: [{ id: 1, status: "queued", created_at: "x" }] });
268
+ }
269
+ if (path.includes("status=waiting")) {
270
+ return JSON.stringify({ workflow_runs: [{ id: 2, status: "waiting", created_at: "y" }] });
271
+ }
272
+ throw new Error(`unexpected path ${path}`);
273
+ };
274
+ const runs = fetchQueuedRuns("dsj1984/domio", runGh);
275
+ assert.equal(runs.length, 2);
276
+ assert.deepEqual(runs.map((r) => r.id).sort(), [1, 2]);
277
+ });
278
+
279
+ test("fetchQueuedRuns returns [] on a 404", () => {
280
+ const runGh = () => {
281
+ throw new Error("gh: Not Found (HTTP 404)");
282
+ };
283
+ assert.deepEqual(fetchQueuedRuns("dsj1984/domio", runGh), []);
284
+ });
285
+
286
+ // ---------------------------------------------------------------------------
287
+ // buildReport / hasUnhealthy
288
+ // ---------------------------------------------------------------------------
289
+
290
+ const CONFIG = {
291
+ defaultStaleQueuedMinutes: 20,
292
+ repos: [
293
+ { name: "domio", repo: "dsj1984/domio", expectedCount: 2, labels: ["self-hosted", "domio-runner"] },
294
+ ],
295
+ };
296
+
297
+ test("buildReport marks a repo healthy when runners are all online and matching", () => {
298
+ const runGh = (args) => {
299
+ const path = args[1];
300
+ if (path.includes("actions/runners")) {
301
+ return JSON.stringify({
302
+ runners: [
303
+ { id: 1, name: "r1", status: "online", labels: [{ name: "self-hosted" }, { name: "domio-runner" }] },
304
+ { id: 2, name: "r2", status: "online", labels: [{ name: "self-hosted" }, { name: "domio-runner" }] },
305
+ ],
306
+ });
307
+ }
308
+ if (path.includes("actions/runs")) {
309
+ return JSON.stringify({ workflow_runs: [] });
310
+ }
311
+ throw new Error(`unexpected ${path}`);
312
+ };
313
+ const report = buildReport(CONFIG, runGh, NOW);
314
+ assert.equal(report.results[0].healthy, true);
315
+ assert.equal(hasUnhealthy(report), false);
316
+ });
317
+
318
+ test("buildReport marks a repo unhealthy and records a fetch error as unhealthy", () => {
319
+ const runGh = (args) => {
320
+ const path = args[1];
321
+ if (path.includes("actions/runners")) throw new Error("gh: Service Unavailable (HTTP 503)");
322
+ return "{}";
323
+ };
324
+ const report = buildReport(CONFIG, runGh, NOW);
325
+ assert.equal(report.results[0].error !== undefined, true);
326
+ assert.equal(hasUnhealthy(report), true);
327
+ });
328
+
329
+ // ---------------------------------------------------------------------------
330
+ // runCli (end-to-end against a temp config + injected runGh)
331
+ // ---------------------------------------------------------------------------
332
+
333
+ test("runCli exits 0 for a healthy fleet and 1 for a degraded one", () => {
334
+ const dir = mkdtempSync(join(tmpdir(), "runner-health-cli-"));
335
+ try {
336
+ const configPath = join(dir, "runner-fleet-consumers.json");
337
+ writeFileSync(
338
+ configPath,
339
+ JSON.stringify({
340
+ defaultStaleQueuedMinutes: 20,
341
+ repos: [
342
+ { name: "domio", repo: "dsj1984/domio", expectedCount: 2, labels: ["self-hosted", "domio-runner"] },
343
+ ],
344
+ }),
345
+ );
346
+ const stdout = { buf: "", write(s) { this.buf += s; } };
347
+ const stderr = { buf: "", write(s) { this.buf += s; } };
348
+ const healthyRunGh = (args) => {
349
+ const path = args[1];
350
+ if (path.includes("actions/runners")) {
351
+ return JSON.stringify({
352
+ runners: [
353
+ { id: 1, name: "r1", status: "online", labels: [{ name: "self-hosted" }, { name: "domio-runner" }] },
354
+ { id: 2, name: "r2", status: "online", labels: [{ name: "self-hosted" }, { name: "domio-runner" }] },
355
+ ],
356
+ });
357
+ }
358
+ if (path.includes("actions/runs")) return JSON.stringify({ workflow_runs: [] });
359
+ throw new Error(`unexpected ${path}`);
360
+ };
361
+ const code = runCli({
362
+ argv: ["--config", configPath],
363
+ cwd: process.cwd(),
364
+ stdout,
365
+ stderr,
366
+ runGh: healthyRunGh,
367
+ summaryPath: undefined,
368
+ nowMs: NOW,
369
+ });
370
+ assert.equal(code, 0);
371
+ assert.match(stdout.buf, /Fleet healthy|Runner-fleet health dashboard/);
372
+ } finally {
373
+ rmSync(dir, { recursive: true, force: true });
374
+ }
375
+ });
376
+
377
+ test("runCli returns 1 on a bad config path", () => {
378
+ const stdout = { buf: "", write(s) { this.buf += s; } };
379
+ const stderr = { buf: "", write(s) { this.buf += s; } };
380
+ const code = runCli({
381
+ argv: ["--config", "scripts/does-not-exist.json"],
382
+ cwd: process.cwd(),
383
+ stdout,
384
+ stderr,
385
+ runGh: () => "{}",
386
+ });
387
+ assert.equal(code, 1);
388
+ assert.match(stderr.buf, /failed to read config/);
389
+ });
@@ -570,22 +570,49 @@ test("--check-ruleset requires --consumer-repo", () => {
570
570
  }, /consumer-repo/);
571
571
  });
572
572
 
573
- test("apply materializes the canonical deploy-staging.yml workflow caller template", () => {
573
+ test("apply materializes the canonical deploy-staging dispatcher + run templates (Story #272)", () => {
574
574
  const out = JSON.parse(run([]));
575
- const stub = join(consumer, ".github", "workflows", "deploy-staging.yml");
576
- assert.ok(existsSync(stub));
575
+ const dispatcher = join(consumer, ".github", "workflows", "deploy-staging.yml");
576
+ const runner = join(consumer, ".github", "workflows", "deploy-staging-run.yml");
577
+ assert.ok(existsSync(dispatcher), "deploy-staging.yml (dispatcher) materialized");
578
+ assert.ok(existsSync(runner), "deploy-staging-run.yml (deploy) materialized");
577
579
  assert.ok(
578
580
  out.workflowStubs.created.some((f) => f.endsWith("deploy-staging.yml")),
579
- "deploy-staging.yml reported as created"
581
+ "deploy-staging.yml (dispatcher) reported as created"
582
+ );
583
+ assert.ok(
584
+ out.workflowStubs.created.some((f) => f.endsWith("deploy-staging-run.yml")),
585
+ "deploy-staging-run.yml (deploy) reported as created"
586
+ );
587
+ const dispatcherBody = readFileSync(dispatcher, "utf8");
588
+ const runnerBody = readFileSync(runner, "utf8");
589
+ // Both halves carry the never-clobber template marker.
590
+ assert.ok(
591
+ dispatcherBody.includes("Canonical staging-deploy caller template"),
592
+ "dispatcher carries the template marker"
593
+ );
594
+ assert.ok(
595
+ runnerBody.includes("Canonical staging-deploy caller template"),
596
+ "run template carries the template marker"
597
+ );
598
+ // The dispatcher fires on CI-green (workflow_run) and DISPATCHES the run
599
+ // workflow — it must NOT call the deploy directly, since a workflow_run
600
+ // deploy skips every environment: job (Story #272).
601
+ assert.ok(
602
+ dispatcherBody.includes("workflow_run") &&
603
+ dispatcherBody.includes("gh workflow run deploy-staging-run.yml"),
604
+ "dispatcher fires on workflow_run and dispatches deploy-staging-run.yml"
580
605
  );
581
- const body = readFileSync(stub, "utf8");
582
606
  assert.ok(
583
- body.includes("Canonical staging-deploy caller template"),
584
- "materialized workflow carries the template marker"
607
+ !dispatcherBody.includes("dsj1984/mandrel-platform/.github/workflows/deploy-cloudflare.yml"),
608
+ "dispatcher does NOT uses: deploy-cloudflare.yml directly (that would skip environment: jobs on workflow_run)"
585
609
  );
610
+ // The deploy half runs on workflow_dispatch (where environment: jobs execute)
611
+ // and uses the shared reusable workflow.
586
612
  assert.ok(
587
- body.includes("dsj1984/mandrel-platform/.github/workflows/deploy-cloudflare.yml"),
588
- "template calls the shared deploy-cloudflare.yml reusable workflow"
613
+ runnerBody.includes("workflow_dispatch") &&
614
+ runnerBody.includes("dsj1984/mandrel-platform/.github/workflows/deploy-cloudflare.yml"),
615
+ "run template deploys on workflow_dispatch via the shared deploy-cloudflare.yml"
589
616
  );
590
617
  });
591
618
 
@@ -0,0 +1,20 @@
1
+ {
2
+ "$comment": "Data-driven expected roster for scripts/check-runner-health.mjs (Story #258). Each entry is one repo whose self-hosted runner fleet the scheduled runner-fleet-health.yml workflow monitors. Adding/removing a runner needs only an edit here — the checker reads GET /repos/{owner}/{repo}/actions/runners and compares live status against `expectedCount` + `labels`. All nine runners (including Beestera/swarm-os's three) are co-resident on one operator Mac (2026-07-03 runner audit, repo-ops matrix §1a); if that host sleeps, reboots, fills its disk, or a launchd service dies, every listed repo's CI silently queues with no alert until this monitor catches it.",
3
+ "$comment_swarm_os": "Beestera/swarm-os is deliberately NOT listed: no dsj1984-owned PAT can read another org's runner API (fine-grained PATs are bound to one resource owner; the Beestera org rejects classic PATs), so its row would permanently false-positive as 0/3 degraded. Host-level coverage is retained regardless — its runners share the Mac with the rows below, so a wedged host still trips domio/athportal. What is NOT covered: swarm-os's individual launchd services dying while the host stays healthy, and its stale-queue check. Re-add the entry if a Beestera-owned credential (fine-grained PAT or GitHub App) plus per-repo token support ever lands.",
4
+ "$comment_staleQueuedMinutes": "A queued/waiting workflow run older than this many minutes with no online runner matching its labels is flagged as a queue-staleness signal (optional per-repo override via `staleQueuedMinutes`).",
5
+ "defaultStaleQueuedMinutes": 20,
6
+ "repos": [
7
+ {
8
+ "name": "domio",
9
+ "repo": "dsj1984/domio",
10
+ "expectedCount": 3,
11
+ "labels": ["self-hosted", "macOS", "ARM64", "domio-runner"]
12
+ },
13
+ {
14
+ "name": "athportal",
15
+ "repo": "dsj1984/athportal",
16
+ "expectedCount": 3,
17
+ "labels": ["self-hosted", "macOS", "ARM64", "athportal-runner"]
18
+ }
19
+ ]
20
+ }
@@ -0,0 +1,150 @@
1
+ # Runner Fleet Health Monitor
2
+
3
+ > **Self-contained runbook** (not a thin stub). Unlike most templates in this
4
+ > directory, there is no canonical `docs/runbooks/` counterpart to link — this
5
+ > file IS the process, mirroring `runner-provisioning.md`. It documents the
6
+ > scheduled `.github/workflows/runner-fleet-health.yml` monitor (Story #258):
7
+ > what it checks, the token scope it needs, the alert semantics, and the
8
+ > operator response when it fires.
9
+
10
+ ---
11
+
12
+ ## Why this exists
13
+
14
+ All self-hosted runners across the fleet (`domio`, `athportal`, `swarm-os`)
15
+ are **co-resident on one operator Mac**. If that host sleeps, reboots for an
16
+ OS update, fills its disk, or a launchd runner service dies, **every
17
+ consumer's CI and deploy-trigger jobs silently queue** ("waiting for a
18
+ runner") with no alert. Nothing else watches this:
19
+
20
+ - The Better Stack uptime unit (`uptime-apply.yml`) monitors the **deployed
21
+ apps**, not the runners.
22
+ - Deploy pipelines are `workflow_run`-gated, so a wedged runner can silently
23
+ stall staging indefinitely — no failed job, no notification, just a queue
24
+ that never drains.
25
+
26
+ > **Roster note:** `Beestera/swarm-os` is monitored only *indirectly*. No
27
+ > `dsj1984`-owned token can read another org's runner API (fine-grained PATs
28
+ > are bound to one resource owner; the Beestera org rejects classic PATs), so
29
+ > its roster entry would permanently false-positive as `0/3` degraded.
30
+ > Because its runners share the Mac with the rostered repos, a wedged
31
+ > **host** still trips the `domio`/`athportal` rows; what goes unwatched is a
32
+ > swarm-os-only launchd service death and its stale queue. See
33
+ > `$comment_swarm_os` in `scripts/runner-fleet-consumers.json`.
34
+
35
+ `runner-fleet-health.yml` is the standing check that catches this fast.
36
+
37
+ ## What it checks
38
+
39
+ Runs on a schedule (~every 15 minutes) plus `workflow_dispatch`, on
40
+ `ubuntu-latest` (deliberately GitHub-hosted so it keeps running when the Mac
41
+ is down). For each repo in `scripts/runner-fleet-consumers.json` it calls
42
+ `GET /repos/{owner}/{repo}/actions/runners` and:
43
+
44
+ 1. **Offline runners** — flags any runner whose `status != online`.
45
+ 2. **Count shortfall** — flags fewer online runners matching the repo's
46
+ expected `labels` set than its configured `expectedCount`.
47
+ 3. **Stale queued runs** (optional signal) — a `queued`/`waiting` workflow run
48
+ older than `staleQueuedMinutes` (default 20) with no online runner matching
49
+ its labels. This catches the case where the runner *looks* present in the
50
+ roster count but is actually wedged and not claiming jobs.
51
+
52
+ It renders a per-repo dashboard on `GITHUB_STEP_SUMMARY`.
53
+
54
+ ## Config-driven roster
55
+
56
+ Adding, removing, or resizing a runner needs **only a config edit** —
57
+ `scripts/runner-fleet-consumers.json`:
58
+
59
+ ```jsonc
60
+ {
61
+ "defaultStaleQueuedMinutes": 20,
62
+ "repos": [
63
+ { "name": "domio", "repo": "dsj1984/domio", "expectedCount": 3, "labels": ["self-hosted", "macOS", "ARM64", "domio-runner"] },
64
+ // ... one object per repo
65
+ ],
66
+ }
67
+ ```
68
+
69
+ ## Token scope: `PIN_DRIFT_TOKEN`
70
+
71
+ The monitor reuses the same fine-grained PAT `pin-drift.yml` already
72
+ provisions (`secrets.PIN_DRIFT_TOKEN`), falling back to the built-in
73
+ `github.token` when the secret is absent (the built-in token only grants read
74
+ access to the workflow's own repo — cross-repo rows then surface as `⚠️
75
+ error` rather than hard-failing this repo's own row).
76
+
77
+ For the runner reads, `PIN_DRIFT_TOKEN` must carry, on every rostered repo:
78
+
79
+ - **Administration: read** — required by `GET .../actions/runners` (the
80
+ self-hosted runner list is an admin-surface endpoint; `actions:read` is
81
+ NOT sufficient for it).
82
+ - **Actions: read** — required by `GET .../actions/runs` (the stale-queue
83
+ check).
84
+
85
+ Resource-owner caveat: a fine-grained PAT is bound to a **single** resource
86
+ owner, and the Beestera org rejects classic PATs — which is exactly why
87
+ `Beestera/swarm-os` is off the roster (see the roster note above). Every
88
+ rostered repo must be readable by the ONE token this workflow gets; a repo
89
+ the token cannot see 404s and false-positives as degraded, so extend the
90
+ roster only together with a credential that covers the new repo.
91
+
92
+ When the token lacks visibility, GitHub returns **404** (not 403) and the
93
+ script treats the empty runner list as a real shortfall — the repo's row
94
+ reads `❌ degraded` with `0/N` online even when the runners are healthy. A
95
+ fleet-wide `0/N` across every repo is the token-misconfiguration signature;
96
+ check the secret before touching the runner host.
97
+
98
+ ## Alert semantics
99
+
100
+ Alert-only by design (no host-side remediation) — the monitor never touches
101
+ the runner host itself. One channel fires on an unhealthy repo, deliberately
102
+ without adding a new external dependency:
103
+
104
+ - **Native GitHub failed-workflow notification.** The job script exits
105
+ non-zero when any repo is unhealthy, so GitHub's own email/notification
106
+ settings fire the standard "workflow run failed" alert to whoever
107
+ watches this repo. No tracking issues are filed — the dashboard detail
108
+ lives on the failed run's job summary.
109
+
110
+ A future Slack/PagerDuty push could layer on top of this later — deliberately
111
+ deferred (see the Story's Out of Scope) to avoid a new external dependency for
112
+ the initial alert-only default.
113
+
114
+ ## Operator response
115
+
116
+ When the scheduled workflow run fails:
117
+
118
+ 1. **Read the dashboard** on the workflow run's job summary — it names which
119
+ signal fired (offline runner, count shortfall, or stale queued run) and
120
+ for which repo.
121
+ 2. **Wake or reboot the Mac** if it's asleep, powered off, or unresponsive
122
+ over SSH.
123
+ 3. **Check disk space** (`df -h`) — a full disk is a common launchd-runner
124
+ death cause; free space and restart the affected runner service(s).
125
+ 4. **Restart the launchd runner service(s)** for the affected repo:
126
+ ```bash
127
+ cd <RUNNER_DIR> # see templates/runbooks/runner-provisioning.md
128
+ ./svc.sh stop && ./svc.sh start
129
+ ./svc.sh status # expect: Started · running
130
+ ```
131
+ 5. **Re-run the monitor** (`workflow_dispatch` from the Actions tab, or wait
132
+ for the next 15-minute tick) to confirm recovery — a green run means the
133
+ fleet reports healthy again.
134
+
135
+ ## Out of scope (Story #258)
136
+
137
+ - Host-side remediation / auto-recovery (waking the Mac, restarting services)
138
+ — this monitor is alert-only; the operator performs the response above by
139
+ hand.
140
+ - Host disk-usage monitoring — not exposable via the runners API anyway, and
141
+ tracked separately.
142
+ - External paging integrations beyond the native failed-workflow
143
+ notification.
144
+ - Cross-repo runner isolation / ephemeral-runner questions — explicitly
145
+ deferred.
146
+
147
+ ## Project-Specific Notes
148
+
149
+ <!-- Record host quirks, roster changes, or false-positive tuning
150
+ (staleQueuedMinutes overrides) specific to this fleet. -->
@@ -0,0 +1,77 @@
1
+ name: deploy-staging-run
2
+
3
+ # Canonical staging-deploy caller template — DEPLOY half (Story #272).
4
+ #
5
+ # > **Why this file runs on `workflow_dispatch`.** Its sibling
6
+ # > `deploy-staging.yml` fires on CI-green (`workflow_run`) and DISPATCHES this
7
+ # > workflow. The deploy lives here, on `workflow_dispatch`, because the shared
8
+ # > `deploy-cloudflare.yml`'s `environment:`-gated jobs (`check-env` /
9
+ # > `migration` / `deploy` / `boot-smoke`) are silently SKIPPED on a
10
+ # > `workflow_run` event but run normally on `workflow_dispatch` (Story #272).
11
+ # > Adopt this file together with `deploy-staging.yml`; `platform-sync`
12
+ # > materializes both.
13
+ #
14
+ # > **Thin local caller.** The defence-in-depth deploy core lives in the shared
15
+ # > `dsj1984/mandrel-platform` `deploy-cloudflare.yml` reusable workflow — see
16
+ # > https://github.com/dsj1984/mandrel-platform/blob/main/docs/reusable-workflows.md#deploy-cloudflareyml.
17
+ # > This file only holds <PROJECT_NAME>-specific values (worker names, build
18
+ # > step, secret mapping). When the deploy PROCESS changes, that change lands
19
+ # > upstream in mandrel-platform — not here.
20
+ #
21
+ # Replace every <PLACEHOLDER> with your project's real values:
22
+ # <MANDREL_PLATFORM_SHA> the pinned mandrel-platform commit SHA (resolve via
23
+ # `node scripts/platform-sync.mjs --ref <release-tag>`
24
+ # from the consumer repo root, or `git ls-remote`).
25
+ # <MANDREL_PLATFORM_TAG> the human-readable release tag matching the SHA
26
+ # above (trailing `# <tag>` comment).
27
+ # <WORKERS_CSV> comma-separated Worker names for this env, e.g.
28
+ # "api,web".
29
+ # <BUILD_COMMAND> optional build command (omit build-command /
30
+ # build-artifact entirely if the deploy job's default
31
+ # checkout is build-ready).
32
+ #
33
+ # See the full input/secret contract:
34
+ # https://github.com/dsj1984/mandrel-platform/blob/main/docs/reusable-workflows.md#deploy-cloudflareyml
35
+
36
+ on:
37
+ # Dispatched by deploy-staging.yml on CI-green, and available for manual
38
+ # on-demand deploys (UI "Run workflow" + `gh workflow run`). The optional sha
39
+ # input records the CI-verified commit; the deploy itself runs against the
40
+ # main tip (`--ref main`).
41
+ workflow_dispatch:
42
+ inputs:
43
+ sha:
44
+ description: >
45
+ Commit SHA that passed CI (informational — the deploy runs against
46
+ the current main tip). Populated automatically when dispatched by
47
+ deploy-staging.yml.
48
+ required: false
49
+ type: string
50
+
51
+ permissions:
52
+ contents: read
53
+
54
+ # Serialize staging deploys: only the freshest dispatch should reach the
55
+ # staging surfaces. The shared deploy-cloudflare.yml additionally serializes
56
+ # per-environment.
57
+ concurrency:
58
+ group: deploy-staging-run
59
+ cancel-in-progress: true
60
+
61
+ jobs:
62
+ deploy:
63
+ name: Staging deploy (shared deploy-cloudflare.yml)
64
+ uses: dsj1984/mandrel-platform/.github/workflows/deploy-cloudflare.yml@<MANDREL_PLATFORM_SHA> # <MANDREL_PLATFORM_TAG>
65
+ with:
66
+ environment: staging
67
+ gh-environment: staging
68
+ workers: <WORKERS_CSV>
69
+ migrate: true
70
+ # db-engine defaults to 'd1'. Set db-engine + migrate-command +
71
+ # snapshot-command for a non-D1 engine (e.g. Turso) — see the contract
72
+ # doc's "command seams" section.
73
+ # Frozen secret allowlist: only {CLOUDFLARE_*, TURSO_*} cross into the
74
+ # shared workflow. Map your project's secret NAMES onto these slots.
75
+ secrets:
76
+ CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
77
+ CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
@@ -1,89 +1,91 @@
1
1
  name: deploy-staging
2
2
 
3
- # Canonical staging-deploy caller template (Story #175).
3
+ # Canonical staging-deploy caller template — DISPATCHER half (Story #175,
4
+ # reworked for Story #272).
5
+ #
6
+ # > **Why two files (Story #272).** A reusable workflow's `environment:`-gated
7
+ # > jobs (`deploy-cloudflare.yml`'s `check-env` / `migration` / `deploy` /
8
+ # > `boot-smoke`) are **silently skipped on a `workflow_run` event** — a
9
+ # > documented GitHub limitation. The previous single-file template called
10
+ # > `deploy-cloudflare.yml` DIRECTLY from `on: workflow_run`, so those jobs
11
+ # > skipped and **0 workers deployed while the run reported green** (all-skipped,
12
+ # > none failed). One consumer hid 40+ consecutive non-deploys this way.
13
+ # >
14
+ # > The fix: this file is now a thin **dispatcher** that fires on CI-green and
15
+ # > re-launches the deploy via **`workflow_dispatch`** — the event on which the
16
+ # > `environment:` jobs DO run — against its sibling `deploy-staging-run.yml`.
17
+ # > `workflow_dispatch` and `repository_dispatch` are the two events that
18
+ # > "always create workflow runs" even when triggered with the built-in
19
+ # > `GITHUB_TOKEN`, so **no PAT is required** for this same-repo dispatch — just
20
+ # > `permissions: actions: write` below. (Cross-repo dispatch, like
21
+ # > `smoke-dispatch.yml`, still needs a PAT; same repo does not.)
4
22
  #
5
23
  # > **Thin local caller.** The defence-in-depth deploy core (secret-isolation
6
24
  # > audit -> CF env gate -> migration (snapshot + apply) -> deploy ->
7
- # > boot-smoke + auto-rollback) AND the CI-green guard both live in the shared
25
+ # > boot-smoke + auto-rollback) lives in the shared
8
26
  # > `dsj1984/mandrel-platform` `deploy-cloudflare.yml` reusable workflow — see
9
27
  # > https://github.com/dsj1984/mandrel-platform/blob/main/docs/reusable-workflows.md#deploy-cloudflareyml.
10
- # > This file only holds <PROJECT_NAME>-specific values (worker names, build
11
- # > step, secret mapping). When the deploy PROCESS changes, that change lands
12
- # > upstream in mandrel-platform — not here.
13
- #
14
- # One paved road (operator decision 2026-07-01, D4): every consumer triggers
15
- # staging deploy via `workflow_run` on its own CI workflow, gated on
16
- # `conclusion == 'success'`. `workflow_run` fires on BOTH a successful AND a
17
- # failed upstream run, so a caller-side guard against a red run used to be
18
- # REQUIRED here — every consumer hand-copied the same `preflight` job (see
19
- # mandrel-platform Story #175 context). That guard now lives INSIDE
20
- # `deploy-cloudflare.yml` itself as a job-level `if:` on its entry jobs
21
- # (`github.event` inside a reusable workflow is the CALLER's event, so the
22
- # shared workflow can see and gate on the `workflow_run` conclusion even
23
- # though it cannot own this file's `on:` block); a red upstream run skips
24
- # the whole chain with zero runner spin-ups (mandrel-platform Story #237).
25
- # This template needs NO caller-side preflight guard as a result —
26
- # copy it as-is and fill in the placeholders below.
28
+ # > The <PROJECT_NAME>-specific values (worker names, build step, secret
29
+ # > mapping) live in the sibling `deploy-staging-run.yml`. When the deploy
30
+ # > PROCESS changes, that change lands upstream in mandrel-platform — not here.
27
31
  #
28
- # Replace every <PLACEHOLDER> with your project's real values:
29
- # <CI_WORKFLOW_NAME> the `name:` of the workflow this deploy should
30
- # gate on (e.g. "quality", "CI", "PR Quality").
31
- # Must match EXACTLY GitHub matches
32
- # `workflow_run.workflows` by workflow name, not
33
- # file path.
34
- # <MANDREL_PLATFORM_SHA> the pinned mandrel-platform commit SHA (resolve
35
- # via `node scripts/platform-sync.mjs --ref
36
- # <release-tag>` from the consumer repo root, or
37
- # hand-resolve via `git ls-remote`).
38
- # <MANDREL_PLATFORM_TAG> the human-readable release tag matching the SHA
39
- # above (trailing `# <tag>` comment).
40
- # <WORKERS_CSV> comma-separated Worker names for this env, e.g.
41
- # "api,web".
42
- # <BUILD_COMMAND> optional build command (omit build-command /
43
- # build-artifact entirely if the deploy job's
44
- # default checkout is build-ready).
32
+ # Adopt BOTH files together: this `deploy-staging.yml` (dispatcher) and
33
+ # `deploy-staging-run.yml` (the actual deploy). `platform-sync` materializes
34
+ # both. Replace every <PLACEHOLDER>:
35
+ # <CI_WORKFLOW_NAME> the `name:` of the workflow this deploy gates on (e.g.
36
+ # "quality", "CI"). Must match EXACTLY — GitHub matches
37
+ # `workflow_run.workflows` by workflow name, not path.
45
38
  #
46
39
  # See the full input/secret contract:
47
40
  # https://github.com/dsj1984/mandrel-platform/blob/main/docs/reusable-workflows.md#deploy-cloudflareyml
48
41
 
49
42
  on:
50
- # CI-green gate: fires when <CI_WORKFLOW_NAME> finishes on main. The shared
51
- # deploy-cloudflare.yml's job-level CI-green gate skips the entire job
52
- # chain (zero runners) unless the upstream conclusion was 'success' no
53
- # caller-side guard needed.
43
+ # CI-green gate: fires when <CI_WORKFLOW_NAME> completes on main. Unlike the
44
+ # old shape, this does NOT call the deploy directly — a `workflow_run` deploy
45
+ # would skip every `environment:` job. It only DISPATCHES the deploy (below)
46
+ # on success, so the actual deploy runs on `workflow_dispatch` where those
47
+ # jobs execute. A red upstream run simply does not dispatch — there is no
48
+ # green-but-didn't-deploy run at all.
54
49
  workflow_run:
55
50
  workflows: [<CI_WORKFLOW_NAME>]
56
51
  branches: [main]
57
52
  types: [completed]
58
- # Manual on-demand trigger (UI "Run workflow" + `gh workflow run`).
59
- # workflow_dispatch always passes the shared workflow's CI-green guard
60
- # (operator-intentional, no upstream conclusion to gate on).
61
- workflow_dispatch:
62
53
 
54
+ # actions:write lets the built-in GITHUB_TOKEN dispatch deploy-staging-run.yml
55
+ # via the workflow_dispatch API. No PAT needed for a same-repo dispatch —
56
+ # workflow_dispatch always creates a run even from GITHUB_TOKEN.
63
57
  permissions:
64
58
  contents: read
59
+ actions: write
65
60
 
66
- # Cancel an in-flight staging deploy when a newer commit lands on main — only
67
- # the freshest tip of main should reach the staging surfaces. The shared
68
- # deploy-cloudflare.yml additionally serializes per-environment.
61
+ # Only the freshest tip of main should reach staging: cancel an in-flight
62
+ # DISPATCH when a newer CI run completes. (The deploy itself is serialized
63
+ # separately in deploy-staging-run.yml and per-environment inside the shared
64
+ # deploy-cloudflare.yml.)
69
65
  concurrency:
70
- group: deploy-staging
66
+ group: deploy-staging-dispatch
71
67
  cancel-in-progress: true
72
68
 
73
69
  jobs:
74
- deploy:
75
- name: Staging deploy (shared deploy-cloudflare.yml)
76
- uses: dsj1984/mandrel-platform/.github/workflows/deploy-cloudflare.yml@<MANDREL_PLATFORM_SHA> # <MANDREL_PLATFORM_TAG>
77
- with:
78
- environment: staging
79
- gh-environment: staging
80
- workers: <WORKERS_CSV>
81
- migrate: true
82
- # db-engine defaults to 'd1'. Set db-engine + migrate-command +
83
- # snapshot-command for a non-D1 engine (e.g. Turso) — see the contract
84
- # doc's "command seams" section.
85
- # Frozen secret allowlist: only {CLOUDFLARE_*, TURSO_*} cross into the
86
- # shared workflow. Map your project's secret NAMES onto these slots.
87
- secrets:
88
- CLOUDFLARE_API_TOKEN: ${{ secrets.CLOUDFLARE_API_TOKEN }}
89
- CLOUDFLARE_ACCOUNT_ID: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }}
70
+ dispatch:
71
+ name: Dispatch staging deploy on CI-green
72
+ runs-on: ubuntu-latest
73
+ timeout-minutes: 5
74
+ # CI-green gate: dispatch the deploy ONLY when the upstream CI run
75
+ # concluded 'success'. `workflow_run` fires on both success and failure, so
76
+ # this guard is load-bearing — without it a red main would still deploy.
77
+ if: ${{ github.event.workflow_run.conclusion == 'success' }}
78
+ steps:
79
+ - name: Dispatch deploy-staging-run.yml (workflow_dispatch)
80
+ env:
81
+ GH_TOKEN: ${{ github.token }}
82
+ REPO: ${{ github.repository }}
83
+ SHA: ${{ github.event.workflow_run.head_sha }}
84
+ shell: bash
85
+ run: |
86
+ set -euo pipefail
87
+ gh workflow run deploy-staging-run.yml \
88
+ --repo "${REPO}" \
89
+ --ref main \
90
+ -f sha="${SHA}"
91
+ echo "Dispatched staging deploy for ${SHA} (deploy runs on workflow_dispatch so environment: jobs execute)."