mandrel-platform 0.20.1 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/scripts/check-ci-required-aggregator.test.mjs +227 -0
- package/scripts/check-coverage-threshold.test.mjs +56 -51
- package/scripts/check-destructive-migration.mjs +68 -6
- package/scripts/check-destructive-migration.test.mjs +146 -0
- package/scripts/check-runner-health.mjs +469 -0
- package/scripts/check-runner-health.test.mjs +389 -0
- package/scripts/deploy-boot-smoke.mjs +364 -0
- package/scripts/deploy-boot-smoke.test.mjs +381 -0
- package/scripts/deploy-worker-secrets.mjs +190 -0
- package/scripts/deploy-worker-secrets.test.mjs +136 -0
- package/scripts/platform-sync.test.mjs +36 -9
- package/scripts/runner-fleet-consumers.json +20 -0
- package/templates/runbooks/README.md +13 -0
- package/templates/runbooks/runner-fleet-health.md +150 -0
- package/templates/runbooks/runner-provisioning.md +187 -0
- package/templates/runner/.env.example +45 -0
- package/templates/runner/job-cleanup.sh +97 -0
- package/templates/workflows/deploy-staging-run.yml +77 -0
- package/templates/workflows/deploy-staging.yml +67 -62
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
import { test } from "node:test";
|
|
2
|
+
import assert from "node:assert/strict";
|
|
3
|
+
|
|
4
|
+
import {
|
|
5
|
+
parseCsv,
|
|
6
|
+
parseSecretNames,
|
|
7
|
+
resolveSecretValue,
|
|
8
|
+
provisionWorkerSecrets,
|
|
9
|
+
} from "./deploy-worker-secrets.mjs";
|
|
10
|
+
|
|
11
|
+
// ---------------------------------------------------------------------------
|
|
12
|
+
// Pure helpers
|
|
13
|
+
// ---------------------------------------------------------------------------
|
|
14
|
+
|
|
15
|
+
test("parseCsv trims entries and drops empties", () => {
|
|
16
|
+
assert.deepEqual(parseCsv(" api , worker-cron ,, "), ["api", "worker-cron"]);
|
|
17
|
+
assert.deepEqual(parseCsv(undefined), []);
|
|
18
|
+
});
|
|
19
|
+
|
|
20
|
+
test("parseSecretNames trims, skips blanks, and skips '#' comment lines", () => {
|
|
21
|
+
const raw = ["# deploy-critical only", "TURSO_AUTH_TOKEN", "", " UPSTREAM_API_KEY ", " # trailing note"].join("\n");
|
|
22
|
+
assert.deepEqual(parseSecretNames(raw), ["TURSO_AUTH_TOKEN", "UPSTREAM_API_KEY"]);
|
|
23
|
+
assert.deepEqual(parseSecretNames(""), []);
|
|
24
|
+
});
|
|
25
|
+
|
|
26
|
+
test("resolveSecretValue returns the value only for present, non-empty string entries", () => {
|
|
27
|
+
const ctx = { A: "value", B: "", C: 42 };
|
|
28
|
+
assert.equal(resolveSecretValue(ctx, "A"), "value");
|
|
29
|
+
assert.equal(resolveSecretValue(ctx, "B"), null);
|
|
30
|
+
assert.equal(resolveSecretValue(ctx, "C"), null);
|
|
31
|
+
assert.equal(resolveSecretValue(ctx, "MISSING"), null);
|
|
32
|
+
assert.equal(resolveSecretValue(null, "A"), null);
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
// ---------------------------------------------------------------------------
|
|
36
|
+
// provisionWorkerSecrets — orchestration with an injected wrangler runner
|
|
37
|
+
// ---------------------------------------------------------------------------
|
|
38
|
+
|
|
39
|
+
function collect() {
|
|
40
|
+
const lines = [];
|
|
41
|
+
const calls = [];
|
|
42
|
+
return {
|
|
43
|
+
lines,
|
|
44
|
+
calls,
|
|
45
|
+
log: (line) => lines.push(line),
|
|
46
|
+
runWrangler: (args, stdinValue) => {
|
|
47
|
+
calls.push({ args, stdinValue });
|
|
48
|
+
return 0;
|
|
49
|
+
},
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
const BASE_ENV = {
|
|
54
|
+
DEPLOY_ENV: "production",
|
|
55
|
+
DEPLOYED_WORKERS: "api,worker-cron",
|
|
56
|
+
WORKER_SECRETS: "TURSO_AUTH_TOKEN",
|
|
57
|
+
SECRETS_CONTEXT: JSON.stringify({ TURSO_AUTH_TOKEN: "s3cret" }),
|
|
58
|
+
};
|
|
59
|
+
|
|
60
|
+
test("provisions each name onto each worker via versions secret put, then promotes with versions deploy -y", () => {
|
|
61
|
+
const { log, runWrangler, calls } = collect();
|
|
62
|
+
const code = provisionWorkerSecrets(BASE_ENV, { log, runWrangler });
|
|
63
|
+
assert.equal(code, 0);
|
|
64
|
+
|
|
65
|
+
assert.deepEqual(
|
|
66
|
+
calls.map((c) => c.args.slice(0, 3).join(" ")),
|
|
67
|
+
[
|
|
68
|
+
"versions secret put", // api
|
|
69
|
+
"versions deploy --name", // api promote
|
|
70
|
+
"versions secret put", // worker-cron
|
|
71
|
+
"versions deploy --name", // worker-cron promote
|
|
72
|
+
]
|
|
73
|
+
);
|
|
74
|
+
|
|
75
|
+
// put: value travels over stdin, never in argv.
|
|
76
|
+
const put = calls[0];
|
|
77
|
+
assert.deepEqual(put.args, ["versions", "secret", "put", "TURSO_AUTH_TOKEN", "--name", "api", "--env", "production"]);
|
|
78
|
+
assert.equal(put.stdinValue, "s3cret");
|
|
79
|
+
|
|
80
|
+
// promote: non-interactive -y, message names the Story.
|
|
81
|
+
const promote = calls[1];
|
|
82
|
+
assert.ok(promote.args.includes("-y"));
|
|
83
|
+
assert.ok(promote.args.includes("--message"));
|
|
84
|
+
assert.equal(promote.stdinValue, undefined);
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
test("secret values never appear in log output", () => {
|
|
88
|
+
const { log, runWrangler, lines } = collect();
|
|
89
|
+
provisionWorkerSecrets(BASE_ENV, { log, runWrangler });
|
|
90
|
+
assert.ok(lines.every((l) => !l.includes("s3cret")));
|
|
91
|
+
});
|
|
92
|
+
|
|
93
|
+
test("zero resolved names is a notice + exit 0 (nothing to provision)", () => {
|
|
94
|
+
const { log, runWrangler, calls, lines } = collect();
|
|
95
|
+
const code = provisionWorkerSecrets(
|
|
96
|
+
{ ...BASE_ENV, WORKER_SECRETS: "\n# only comments\n\n" },
|
|
97
|
+
{ log, runWrangler }
|
|
98
|
+
);
|
|
99
|
+
assert.equal(code, 0);
|
|
100
|
+
assert.equal(calls.length, 0);
|
|
101
|
+
assert.ok(lines.some((l) => l.includes("resolved to zero names")));
|
|
102
|
+
});
|
|
103
|
+
|
|
104
|
+
test("a listed name absent (or empty) in the inherited context is a hard error", () => {
|
|
105
|
+
const { log, runWrangler, lines } = collect();
|
|
106
|
+
const code = provisionWorkerSecrets(
|
|
107
|
+
{ ...BASE_ENV, WORKER_SECRETS: "MISSING_SECRET" },
|
|
108
|
+
{ log, runWrangler }
|
|
109
|
+
);
|
|
110
|
+
assert.equal(code, 1);
|
|
111
|
+
assert.ok(lines.some((l) => l.includes("'MISSING_SECRET' is not present (or empty)")));
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
test("malformed SECRETS_CONTEXT JSON is a hard error", () => {
|
|
115
|
+
const { log, runWrangler } = collect();
|
|
116
|
+
const code = provisionWorkerSecrets({ ...BASE_ENV, SECRETS_CONTEXT: "{not json" }, { log, runWrangler });
|
|
117
|
+
assert.equal(code, 1);
|
|
118
|
+
});
|
|
119
|
+
|
|
120
|
+
test("a failing wrangler versions secret put aborts with exit 1", () => {
|
|
121
|
+
const { log } = collect();
|
|
122
|
+
const code = provisionWorkerSecrets(BASE_ENV, {
|
|
123
|
+
log,
|
|
124
|
+
runWrangler: (args) => (args[1] === "secret" ? 1 : 0),
|
|
125
|
+
});
|
|
126
|
+
assert.equal(code, 1);
|
|
127
|
+
});
|
|
128
|
+
|
|
129
|
+
test("a failing wrangler versions deploy aborts with exit 1", () => {
|
|
130
|
+
const { log } = collect();
|
|
131
|
+
const code = provisionWorkerSecrets(BASE_ENV, {
|
|
132
|
+
log,
|
|
133
|
+
runWrangler: (args) => (args[0] === "versions" && args[1] === "deploy" ? 1 : 0),
|
|
134
|
+
});
|
|
135
|
+
assert.equal(code, 1);
|
|
136
|
+
});
|
|
@@ -570,22 +570,49 @@ test("--check-ruleset requires --consumer-repo", () => {
|
|
|
570
570
|
}, /consumer-repo/);
|
|
571
571
|
});
|
|
572
572
|
|
|
573
|
-
test("apply materializes the canonical deploy-staging
|
|
573
|
+
test("apply materializes the canonical deploy-staging dispatcher + run templates (Story #272)", () => {
|
|
574
574
|
const out = JSON.parse(run([]));
|
|
575
|
-
const
|
|
576
|
-
|
|
575
|
+
const dispatcher = join(consumer, ".github", "workflows", "deploy-staging.yml");
|
|
576
|
+
const runner = join(consumer, ".github", "workflows", "deploy-staging-run.yml");
|
|
577
|
+
assert.ok(existsSync(dispatcher), "deploy-staging.yml (dispatcher) materialized");
|
|
578
|
+
assert.ok(existsSync(runner), "deploy-staging-run.yml (deploy) materialized");
|
|
577
579
|
assert.ok(
|
|
578
580
|
out.workflowStubs.created.some((f) => f.endsWith("deploy-staging.yml")),
|
|
579
|
-
"deploy-staging.yml reported as created"
|
|
581
|
+
"deploy-staging.yml (dispatcher) reported as created"
|
|
582
|
+
);
|
|
583
|
+
assert.ok(
|
|
584
|
+
out.workflowStubs.created.some((f) => f.endsWith("deploy-staging-run.yml")),
|
|
585
|
+
"deploy-staging-run.yml (deploy) reported as created"
|
|
586
|
+
);
|
|
587
|
+
const dispatcherBody = readFileSync(dispatcher, "utf8");
|
|
588
|
+
const runnerBody = readFileSync(runner, "utf8");
|
|
589
|
+
// Both halves carry the never-clobber template marker.
|
|
590
|
+
assert.ok(
|
|
591
|
+
dispatcherBody.includes("Canonical staging-deploy caller template"),
|
|
592
|
+
"dispatcher carries the template marker"
|
|
593
|
+
);
|
|
594
|
+
assert.ok(
|
|
595
|
+
runnerBody.includes("Canonical staging-deploy caller template"),
|
|
596
|
+
"run template carries the template marker"
|
|
597
|
+
);
|
|
598
|
+
// The dispatcher fires on CI-green (workflow_run) and DISPATCHES the run
|
|
599
|
+
// workflow — it must NOT call the deploy directly, since a workflow_run
|
|
600
|
+
// deploy skips every environment: job (Story #272).
|
|
601
|
+
assert.ok(
|
|
602
|
+
dispatcherBody.includes("workflow_run") &&
|
|
603
|
+
dispatcherBody.includes("gh workflow run deploy-staging-run.yml"),
|
|
604
|
+
"dispatcher fires on workflow_run and dispatches deploy-staging-run.yml"
|
|
580
605
|
);
|
|
581
|
-
const body = readFileSync(stub, "utf8");
|
|
582
606
|
assert.ok(
|
|
583
|
-
|
|
584
|
-
"
|
|
607
|
+
!dispatcherBody.includes("dsj1984/mandrel-platform/.github/workflows/deploy-cloudflare.yml"),
|
|
608
|
+
"dispatcher does NOT uses: deploy-cloudflare.yml directly (that would skip environment: jobs on workflow_run)"
|
|
585
609
|
);
|
|
610
|
+
// The deploy half runs on workflow_dispatch (where environment: jobs execute)
|
|
611
|
+
// and uses the shared reusable workflow.
|
|
586
612
|
assert.ok(
|
|
587
|
-
|
|
588
|
-
|
|
613
|
+
runnerBody.includes("workflow_dispatch") &&
|
|
614
|
+
runnerBody.includes("dsj1984/mandrel-platform/.github/workflows/deploy-cloudflare.yml"),
|
|
615
|
+
"run template deploys on workflow_dispatch via the shared deploy-cloudflare.yml"
|
|
589
616
|
);
|
|
590
617
|
});
|
|
591
618
|
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$comment": "Data-driven expected roster for scripts/check-runner-health.mjs (Story #258). Each entry is one repo whose self-hosted runner fleet the scheduled runner-fleet-health.yml workflow monitors. Adding/removing a runner needs only an edit here — the checker reads GET /repos/{owner}/{repo}/actions/runners and compares live status against `expectedCount` + `labels`. All nine runners (including Beestera/swarm-os's three) are co-resident on one operator Mac (2026-07-03 runner audit, repo-ops matrix §1a); if that host sleeps, reboots, fills its disk, or a launchd service dies, every listed repo's CI silently queues with no alert until this monitor catches it.",
|
|
3
|
+
"$comment_swarm_os": "Beestera/swarm-os is deliberately NOT listed: no dsj1984-owned PAT can read another org's runner API (fine-grained PATs are bound to one resource owner; the Beestera org rejects classic PATs), so its row would permanently false-positive as 0/3 degraded. Host-level coverage is retained regardless — its runners share the Mac with the rows below, so a wedged host still trips domio/athportal. What is NOT covered: swarm-os's individual launchd services dying while the host stays healthy, and its stale-queue check. Re-add the entry if a Beestera-owned credential (fine-grained PAT or GitHub App) plus per-repo token support ever lands.",
|
|
4
|
+
"$comment_staleQueuedMinutes": "A queued/waiting workflow run older than this many minutes with no online runner matching its labels is flagged as a queue-staleness signal (optional per-repo override via `staleQueuedMinutes`).",
|
|
5
|
+
"defaultStaleQueuedMinutes": 20,
|
|
6
|
+
"repos": [
|
|
7
|
+
{
|
|
8
|
+
"name": "domio",
|
|
9
|
+
"repo": "dsj1984/domio",
|
|
10
|
+
"expectedCount": 3,
|
|
11
|
+
"labels": ["self-hosted", "macOS", "ARM64", "domio-runner"]
|
|
12
|
+
},
|
|
13
|
+
{
|
|
14
|
+
"name": "athportal",
|
|
15
|
+
"repo": "dsj1984/athportal",
|
|
16
|
+
"expectedCount": 3,
|
|
17
|
+
"labels": ["self-hosted", "macOS", "ARM64", "athportal-runner"]
|
|
18
|
+
}
|
|
19
|
+
]
|
|
20
|
+
}
|
|
@@ -54,3 +54,16 @@ node node_modules/mandrel-platform/scripts/platform-sync.mjs --ref mandrel-platf
|
|
|
54
54
|
|
|
55
55
|
> Placeholder convention: `<UPPER_SNAKE>` between angle brackets. Search for
|
|
56
56
|
> `<` after copying to find everything that still needs a value.
|
|
57
|
+
|
|
58
|
+
## Self-contained runbooks (not stubs)
|
|
59
|
+
|
|
60
|
+
One template in this directory is a **full runbook**, not a thin stub — it
|
|
61
|
+
has no canonical `docs/runbooks/` counterpart to link and carries the entire
|
|
62
|
+
process inline (placeholders included):
|
|
63
|
+
|
|
64
|
+
| Runbook | Scope |
|
|
65
|
+
|---------|-------|
|
|
66
|
+
| `runner-provisioning.md` | provision a persistent self-hosted runner with the [`templates/runner/`](../runner/) hygiene kit (job-start cleanup hook + runner-scoped `.env`) |
|
|
67
|
+
|
|
68
|
+
Adopt it the same way as a stub (copy into `docs/runbooks/`, fill the
|
|
69
|
+
placeholders); `platform-sync.mjs` materializes it alongside the stubs.
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
# Runner Fleet Health Monitor
|
|
2
|
+
|
|
3
|
+
> **Self-contained runbook** (not a thin stub). Unlike most templates in this
|
|
4
|
+
> directory, there is no canonical `docs/runbooks/` counterpart to link — this
|
|
5
|
+
> file IS the process, mirroring `runner-provisioning.md`. It documents the
|
|
6
|
+
> scheduled `.github/workflows/runner-fleet-health.yml` monitor (Story #258):
|
|
7
|
+
> what it checks, the token scope it needs, the alert semantics, and the
|
|
8
|
+
> operator response when it fires.
|
|
9
|
+
|
|
10
|
+
---
|
|
11
|
+
|
|
12
|
+
## Why this exists
|
|
13
|
+
|
|
14
|
+
All self-hosted runners across the fleet (`domio`, `athportal`, `swarm-os`)
|
|
15
|
+
are **co-resident on one operator Mac**. If that host sleeps, reboots for an
|
|
16
|
+
OS update, fills its disk, or a launchd runner service dies, **every
|
|
17
|
+
consumer's CI and deploy-trigger jobs silently queue** ("waiting for a
|
|
18
|
+
runner") with no alert. Nothing else watches this:
|
|
19
|
+
|
|
20
|
+
- The Better Stack uptime unit (`uptime-apply.yml`) monitors the **deployed
|
|
21
|
+
apps**, not the runners.
|
|
22
|
+
- Deploy pipelines are `workflow_run`-gated, so a wedged runner can silently
|
|
23
|
+
stall staging indefinitely — no failed job, no notification, just a queue
|
|
24
|
+
that never drains.
|
|
25
|
+
|
|
26
|
+
> **Roster note:** `Beestera/swarm-os` is monitored only *indirectly*. No
|
|
27
|
+
> `dsj1984`-owned token can read another org's runner API (fine-grained PATs
|
|
28
|
+
> are bound to one resource owner; the Beestera org rejects classic PATs), so
|
|
29
|
+
> its roster entry would permanently false-positive as `0/3` degraded.
|
|
30
|
+
> Because its runners share the Mac with the rostered repos, a wedged
|
|
31
|
+
> **host** still trips the `domio`/`athportal` rows; what goes unwatched is a
|
|
32
|
+
> swarm-os-only launchd service death and its stale queue. See
|
|
33
|
+
> `$comment_swarm_os` in `scripts/runner-fleet-consumers.json`.
|
|
34
|
+
|
|
35
|
+
`runner-fleet-health.yml` is the standing check that catches this fast.
|
|
36
|
+
|
|
37
|
+
## What it checks
|
|
38
|
+
|
|
39
|
+
Runs on a schedule (~every 15 minutes) plus `workflow_dispatch`, on
|
|
40
|
+
`ubuntu-latest` (deliberately GitHub-hosted so it keeps running when the Mac
|
|
41
|
+
is down). For each repo in `scripts/runner-fleet-consumers.json` it calls
|
|
42
|
+
`GET /repos/{owner}/{repo}/actions/runners` and:
|
|
43
|
+
|
|
44
|
+
1. **Offline runners** — flags any runner whose `status != online`.
|
|
45
|
+
2. **Count shortfall** — flags fewer online runners matching the repo's
|
|
46
|
+
expected `labels` set than its configured `expectedCount`.
|
|
47
|
+
3. **Stale queued runs** (optional signal) — a `queued`/`waiting` workflow run
|
|
48
|
+
older than `staleQueuedMinutes` (default 20) with no online runner matching
|
|
49
|
+
its labels. This catches the case where the runner *looks* present in the
|
|
50
|
+
roster count but is actually wedged and not claiming jobs.
|
|
51
|
+
|
|
52
|
+
It renders a per-repo dashboard on `GITHUB_STEP_SUMMARY`.
|
|
53
|
+
|
|
54
|
+
## Config-driven roster
|
|
55
|
+
|
|
56
|
+
Adding, removing, or resizing a runner needs **only a config edit** —
|
|
57
|
+
`scripts/runner-fleet-consumers.json`:
|
|
58
|
+
|
|
59
|
+
```jsonc
|
|
60
|
+
{
|
|
61
|
+
"defaultStaleQueuedMinutes": 20,
|
|
62
|
+
"repos": [
|
|
63
|
+
{ "name": "domio", "repo": "dsj1984/domio", "expectedCount": 3, "labels": ["self-hosted", "macOS", "ARM64", "domio-runner"] },
|
|
64
|
+
// ... one object per repo
|
|
65
|
+
],
|
|
66
|
+
}
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
## Token scope: `PIN_DRIFT_TOKEN`
|
|
70
|
+
|
|
71
|
+
The monitor reuses the same fine-grained PAT `pin-drift.yml` already
|
|
72
|
+
provisions (`secrets.PIN_DRIFT_TOKEN`), falling back to the built-in
|
|
73
|
+
`github.token` when the secret is absent (the built-in token only grants read
|
|
74
|
+
access to the workflow's own repo — cross-repo rows then surface as `⚠️
|
|
75
|
+
error` rather than hard-failing this repo's own row).
|
|
76
|
+
|
|
77
|
+
For the runner reads, `PIN_DRIFT_TOKEN` must carry, on every rostered repo:
|
|
78
|
+
|
|
79
|
+
- **Administration: read** — required by `GET .../actions/runners` (the
|
|
80
|
+
self-hosted runner list is an admin-surface endpoint; `actions:read` is
|
|
81
|
+
NOT sufficient for it).
|
|
82
|
+
- **Actions: read** — required by `GET .../actions/runs` (the stale-queue
|
|
83
|
+
check).
|
|
84
|
+
|
|
85
|
+
Resource-owner caveat: a fine-grained PAT is bound to a **single** resource
|
|
86
|
+
owner, and the Beestera org rejects classic PATs — which is exactly why
|
|
87
|
+
`Beestera/swarm-os` is off the roster (see the roster note above). Every
|
|
88
|
+
rostered repo must be readable by the ONE token this workflow gets; a repo
|
|
89
|
+
the token cannot see 404s and false-positives as degraded, so extend the
|
|
90
|
+
roster only together with a credential that covers the new repo.
|
|
91
|
+
|
|
92
|
+
When the token lacks visibility, GitHub returns **404** (not 403) and the
|
|
93
|
+
script treats the empty runner list as a real shortfall — the repo's row
|
|
94
|
+
reads `❌ degraded` with `0/N` online even when the runners are healthy. A
|
|
95
|
+
fleet-wide `0/N` across every repo is the token-misconfiguration signature;
|
|
96
|
+
check the secret before touching the runner host.
|
|
97
|
+
|
|
98
|
+
## Alert semantics
|
|
99
|
+
|
|
100
|
+
Alert-only by design (no host-side remediation) — the monitor never touches
|
|
101
|
+
the runner host itself. One channel fires on an unhealthy repo, deliberately
|
|
102
|
+
without adding a new external dependency:
|
|
103
|
+
|
|
104
|
+
- **Native GitHub failed-workflow notification.** The job script exits
|
|
105
|
+
non-zero when any repo is unhealthy, so GitHub's own email/notification
|
|
106
|
+
settings fire the standard "workflow run failed" alert to whoever
|
|
107
|
+
watches this repo. No tracking issues are filed — the dashboard detail
|
|
108
|
+
lives on the failed run's job summary.
|
|
109
|
+
|
|
110
|
+
A future Slack/PagerDuty push could layer on top of this later — deliberately
|
|
111
|
+
deferred (see the Story's Out of Scope) to avoid a new external dependency for
|
|
112
|
+
the initial alert-only default.
|
|
113
|
+
|
|
114
|
+
## Operator response
|
|
115
|
+
|
|
116
|
+
When the scheduled workflow run fails:
|
|
117
|
+
|
|
118
|
+
1. **Read the dashboard** on the workflow run's job summary — it names which
|
|
119
|
+
signal fired (offline runner, count shortfall, or stale queued run) and
|
|
120
|
+
for which repo.
|
|
121
|
+
2. **Wake or reboot the Mac** if it's asleep, powered off, or unresponsive
|
|
122
|
+
over SSH.
|
|
123
|
+
3. **Check disk space** (`df -h`) — a full disk is a common launchd-runner
|
|
124
|
+
death cause; free space and restart the affected runner service(s).
|
|
125
|
+
4. **Restart the launchd runner service(s)** for the affected repo:
|
|
126
|
+
```bash
|
|
127
|
+
cd <RUNNER_DIR> # see templates/runbooks/runner-provisioning.md
|
|
128
|
+
./svc.sh stop && ./svc.sh start
|
|
129
|
+
./svc.sh status # expect: Started · running
|
|
130
|
+
```
|
|
131
|
+
5. **Re-run the monitor** (`workflow_dispatch` from the Actions tab, or wait
|
|
132
|
+
for the next 15-minute tick) to confirm recovery — a green run means the
|
|
133
|
+
fleet reports healthy again.
|
|
134
|
+
|
|
135
|
+
## Out of scope (Story #258)
|
|
136
|
+
|
|
137
|
+
- Host-side remediation / auto-recovery (waking the Mac, restarting services)
|
|
138
|
+
— this monitor is alert-only; the operator performs the response above by
|
|
139
|
+
hand.
|
|
140
|
+
- Host disk-usage monitoring — not exposable via the runners API anyway, and
|
|
141
|
+
tracked separately.
|
|
142
|
+
- External paging integrations beyond the native failed-workflow
|
|
143
|
+
notification.
|
|
144
|
+
- Cross-repo runner isolation / ephemeral-runner questions — explicitly
|
|
145
|
+
deferred.
|
|
146
|
+
|
|
147
|
+
## Project-Specific Notes
|
|
148
|
+
|
|
149
|
+
<!-- Record host quirks, roster changes, or false-positive tuning
|
|
150
|
+
(staleQueuedMinutes overrides) specific to this fleet. -->
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
# Self-Hosted Runner Provisioning — <PROJECT_NAME>
|
|
2
|
+
|
|
3
|
+
> **Self-contained runbook** (not a thin stub). Unlike the other templates in
|
|
4
|
+
> this directory, there is no canonical `docs/runbooks/` counterpart to link —
|
|
5
|
+
> this file IS the process. It provisions a **persistent** (non-ephemeral)
|
|
6
|
+
> GitHub Actions runner on a macOS host using the mandrel-platform runner kit
|
|
7
|
+
> (`templates/runner/`), which ships the job-start hygiene hook
|
|
8
|
+
> (`job-cleanup.sh`) and the per-runner `.env` (`.env.example`).
|
|
9
|
+
>
|
|
10
|
+
> Placeholder convention: `<UPPER_SNAKE>` between angle brackets — search for
|
|
11
|
+
> `<` after copying to find everything that still needs a value.
|
|
12
|
+
|
|
13
|
+
---
|
|
14
|
+
|
|
15
|
+
## ⚠️ Read first: the shared-`$HOME` concurrency hazard
|
|
16
|
+
|
|
17
|
+
Every runner on a host typically runs as the **same OS user**, so anything
|
|
18
|
+
resolved against `$HOME` is **shared across all co-resident runners** — it is
|
|
19
|
+
*not* scoped to "this runner", no matter what a comment claims. The two
|
|
20
|
+
hazards this kit exists to close:
|
|
21
|
+
|
|
22
|
+
1. **`~/setup-pnpm` is shared.** `pnpm/action-setup`'s `dest` input defaults
|
|
23
|
+
to `~/setup-pnpm`. With N runners on one host, N concurrent jobs race on
|
|
24
|
+
one pnpm shim install. Worse, a cleanup hook that `pkill`s processes
|
|
25
|
+
matching `~/setup-pnpm` or `rm -rf`s it at job start will **destroy a pnpm
|
|
26
|
+
install a concurrent runner is mid-flight on**. The fix is ordering:
|
|
27
|
+
**first** make the pnpm shim location per-runner (workflow-side, see
|
|
28
|
+
[pnpm scoping](#3-pnpm-scoping-workflow-side-prerequisite) below), and only
|
|
29
|
+
**then** is a reap/delete hook safe — and even then it must target only the
|
|
30
|
+
runner-scoped path. The kit's `job-cleanup.sh` never touches
|
|
31
|
+
`~/setup-pnpm`.
|
|
32
|
+
2. **The default tool cache is shared.** Without a runner-scoped
|
|
33
|
+
`RUNNER_TOOL_CACHE`, toolchain actions extract into a host-shared cache
|
|
34
|
+
and co-resident runners race on it. The kit's `.env` scopes it to the
|
|
35
|
+
runner's own `_work/_tool`.
|
|
36
|
+
|
|
37
|
+
Do not roll the hook out to a host until every workflow that job's runner
|
|
38
|
+
serves installs pnpm to a runner-scoped `dest`. Rolling out the hook without
|
|
39
|
+
the pnpm-scoping prerequisite reintroduces the exact corruption it guards
|
|
40
|
+
against.
|
|
41
|
+
|
|
42
|
+
## Host Values
|
|
43
|
+
|
|
44
|
+
| Value | Setting |
|
|
45
|
+
|-------|---------|
|
|
46
|
+
| Runner host | `<RUNNER_HOST>` |
|
|
47
|
+
| Runner OS user | `<RUNNER_USER>` |
|
|
48
|
+
| Runner root dir | `<RUNNER_DIR>` (e.g. `~/Development/github-runners/<REPO>`) |
|
|
49
|
+
| Repository | `<OWNER>/<REPO>` |
|
|
50
|
+
| Runner name | `<REPO>-runner` (or `<REPO>-runner-<N>` for a pool) |
|
|
51
|
+
| Labels | `self-hosted, macOS, ARM64, <REPO>-runner` |
|
|
52
|
+
| Runner version | `<RUNNER_VERSION>` (latest from [actions/runner releases](https://github.com/actions/runner/releases)) |
|
|
53
|
+
|
|
54
|
+
## 1. Download and unpack the runner
|
|
55
|
+
|
|
56
|
+
One directory per runner — never share a runner root between registrations.
|
|
57
|
+
|
|
58
|
+
```bash
|
|
59
|
+
mkdir -p <RUNNER_DIR> && cd <RUNNER_DIR>
|
|
60
|
+
curl -o actions-runner-osx-arm64-<RUNNER_VERSION>.tar.gz -L \
|
|
61
|
+
https://github.com/actions/runner/releases/download/v<RUNNER_VERSION>/actions-runner-osx-arm64-<RUNNER_VERSION>.tar.gz
|
|
62
|
+
# Verify the SHA-256 against the checksum published on the release page
|
|
63
|
+
# before unpacking — same download-and-verify posture as the platform's
|
|
64
|
+
# pinned gitleaks/actionlint installs.
|
|
65
|
+
shasum -a 256 actions-runner-osx-arm64-<RUNNER_VERSION>.tar.gz
|
|
66
|
+
tar xzf actions-runner-osx-arm64-<RUNNER_VERSION>.tar.gz
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
## 2. Register with `config.sh` (repo-level)
|
|
70
|
+
|
|
71
|
+
Registration is **repo-level** (the fleet's standing model), not org-level.
|
|
72
|
+
Mint a short-lived registration token via the repo UI
|
|
73
|
+
(*Settings → Actions → Runners → New self-hosted runner*) or:
|
|
74
|
+
|
|
75
|
+
```bash
|
|
76
|
+
gh api -X POST repos/<OWNER>/<REPO>/actions/runners/registration-token --jq .token
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
Then configure:
|
|
80
|
+
|
|
81
|
+
```bash
|
|
82
|
+
cd <RUNNER_DIR>
|
|
83
|
+
./config.sh \
|
|
84
|
+
--url https://github.com/<OWNER>/<REPO> \
|
|
85
|
+
--token <REGISTRATION_TOKEN> \
|
|
86
|
+
--name <REPO>-runner \
|
|
87
|
+
--labels self-hosted,macOS,ARM64,<REPO>-runner \
|
|
88
|
+
--work _work \
|
|
89
|
+
--unattended
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
- `--labels` — the four-label contract the fleet's workflows target
|
|
93
|
+
(`self-hosted, macOS, ARM64, <REPO>-runner`). The `<REPO>-runner` label is
|
|
94
|
+
the routing key; keep it unique per repo.
|
|
95
|
+
- `--work _work` — keeps the work tree inside `<RUNNER_DIR>`, which is what
|
|
96
|
+
makes every path in the hygiene kit runner-scoped.
|
|
97
|
+
- Registration tokens expire after ~1 hour; mint a fresh one per runner.
|
|
98
|
+
|
|
99
|
+
## 3. pnpm scoping (workflow-side prerequisite)
|
|
100
|
+
|
|
101
|
+
Before installing the hook, confirm every workflow this runner serves
|
|
102
|
+
installs the pnpm shim to a **runner-scoped** destination:
|
|
103
|
+
|
|
104
|
+
- Workflows using the platform's `setup-toolchain` composite action (all
|
|
105
|
+
`pr-quality.yml` tiers) are already safe: it defaults `pnpm/action-setup`'s
|
|
106
|
+
`dest` to `${{ runner.temp }}/pnpm`, i.e. `<RUNNER_DIR>/_work/_temp/pnpm`,
|
|
107
|
+
unique per runner. `pr-quality.yml` also exposes a `pnpm-dest` input for
|
|
108
|
+
explicit overrides.
|
|
109
|
+
- Workflows calling `pnpm/action-setup` directly MUST pass
|
|
110
|
+
`dest: ${{ runner.temp }}/pnpm`. The action's default (`~/setup-pnpm`) is
|
|
111
|
+
host-shared and unsafe under runner concurrency (see the hazard header).
|
|
112
|
+
|
|
113
|
+
There is **no runner-side override** for the pnpm `dest` — it is a workflow
|
|
114
|
+
input — which is why this step is a rollout gate, not an `.env` line.
|
|
115
|
+
|
|
116
|
+
## 4. Install the hygiene kit (hook + `.env`)
|
|
117
|
+
|
|
118
|
+
Copy the kit from the platform payload into the runner root:
|
|
119
|
+
|
|
120
|
+
```bash
|
|
121
|
+
cp node_modules/mandrel-platform/templates/runner/job-cleanup.sh <RUNNER_DIR>/job-cleanup.sh
|
|
122
|
+
chmod +x <RUNNER_DIR>/job-cleanup.sh
|
|
123
|
+
cp node_modules/mandrel-platform/templates/runner/.env.example <RUNNER_DIR>/.env
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
Then edit `<RUNNER_DIR>/.env` and replace every `<RUNNER_DIR>` placeholder
|
|
127
|
+
with the runner root's absolute path. The resulting file wires:
|
|
128
|
+
|
|
129
|
+
- `ACTIONS_RUNNER_HOOK_JOB_STARTED=<RUNNER_DIR>/job-cleanup.sh` — the
|
|
130
|
+
job-start hook. It reaps orphaned pnpm/node processes parented to **this**
|
|
131
|
+
runner's work tree, clears stale runner-scoped pnpm installs, and
|
|
132
|
+
age-gate-sweeps shared-`$TMPDIR` gitleaks leftovers. It never fails a job
|
|
133
|
+
(always exits 0) and never touches another runner's state.
|
|
134
|
+
- `RUNNER_TOOL_CACHE=<RUNNER_DIR>/_work/_tool` and
|
|
135
|
+
`AGENT_TOOLSDIRECTORY=<RUNNER_DIR>/_work/_tool` — runner-scoped tool cache
|
|
136
|
+
(two env names, one dir; some actions read the legacy name).
|
|
137
|
+
- `LANG=en_US.UTF-8`.
|
|
138
|
+
|
|
139
|
+
The hook needs no per-runner editing: it derives `RUNNER_DIR` from its own
|
|
140
|
+
location, so the same file works verbatim on every runner.
|
|
141
|
+
|
|
142
|
+
## 5. Install as a launchd service (`svc.sh`)
|
|
143
|
+
|
|
144
|
+
```bash
|
|
145
|
+
cd <RUNNER_DIR>
|
|
146
|
+
./svc.sh install # generates the launchd plist for the current user
|
|
147
|
+
./svc.sh start
|
|
148
|
+
./svc.sh status # expect: Started · running
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
Verify end-to-end: push a trivial workflow run targeting
|
|
152
|
+
`runs-on: [self-hosted, macOS, ARM64, <REPO>-runner]` and confirm (a) the job
|
|
153
|
+
is picked up and (b) the job log shows the `Set up runner` hook phase running
|
|
154
|
+
`job-cleanup.sh` before the first step.
|
|
155
|
+
|
|
156
|
+
The runner loads `.env` at service start — after any `.env` change, restart:
|
|
157
|
+
|
|
158
|
+
```bash
|
|
159
|
+
./svc.sh stop && ./svc.sh start
|
|
160
|
+
```
|
|
161
|
+
|
|
162
|
+
## 6. Update / rotation guidance
|
|
163
|
+
|
|
164
|
+
- **Runner version updates.** Persistent runners self-update by default when
|
|
165
|
+
GitHub releases a new runner version; no action needed. If a runner is
|
|
166
|
+
pinned or the self-update wedges, stop the service, download/unpack the new
|
|
167
|
+
tarball over `<RUNNER_DIR>` (config and `.env` survive), and restart via
|
|
168
|
+
`svc.sh`.
|
|
169
|
+
- **Kit updates.** The hook and `.env.example` are versioned in
|
|
170
|
+
mandrel-platform. On a platform release that touches `templates/runner/`,
|
|
171
|
+
re-copy `job-cleanup.sh` (verbatim — it is parameterized) and diff
|
|
172
|
+
`.env.example` against the live `.env`, then `./svc.sh stop && ./svc.sh
|
|
173
|
+
start`. There is no `mandrel sync` equivalent for a runner host's
|
|
174
|
+
filesystem — this is an operator-applied step.
|
|
175
|
+
- **Token/registration rotation.** Registration tokens are one-shot at
|
|
176
|
+
config time; nothing persists to rotate. To move a runner between repos or
|
|
177
|
+
rename it: `./svc.sh stop && ./svc.sh uninstall && ./config.sh remove
|
|
178
|
+
--token <REMOVAL_TOKEN>`, then re-register (§2) and reinstall the service
|
|
179
|
+
(§5). Mint the removal token via
|
|
180
|
+
`gh api -X POST repos/<OWNER>/<REPO>/actions/runners/remove-token --jq .token`.
|
|
181
|
+
- **Decommission.** Same removal sequence, then delete `<RUNNER_DIR>`.
|
|
182
|
+
Confirm the runner disappeared from *Settings → Actions → Runners*.
|
|
183
|
+
|
|
184
|
+
## Project-Specific Notes
|
|
185
|
+
|
|
186
|
+
<!-- Record host quirks: co-resident runner inventory for this host, Xcode /
|
|
187
|
+
toolchain versions the workloads assume, monitoring hooks, etc. -->
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# .env — per-runner environment for a PERSISTENT self-hosted runner
|
|
2
|
+
# (mandrel-platform runner kit).
|
|
3
|
+
#
|
|
4
|
+
# Copy this file to `<RUNNER_DIR>/.env` (the runner root, next to config.sh)
|
|
5
|
+
# and replace every `<RUNNER_DIR>` placeholder with the runner's ABSOLUTE
|
|
6
|
+
# root path (e.g. /Users/ci/Development/github-runners/myrepo). The runner
|
|
7
|
+
# loads this file at service start and injects the variables into every job.
|
|
8
|
+
#
|
|
9
|
+
# Placeholder convention: `<UPPER_SNAKE>` between angle brackets — search for
|
|
10
|
+
# `<` after copying to find everything that still needs a value.
|
|
11
|
+
#
|
|
12
|
+
# See templates/runbooks/runner-provisioning.md for the full provisioning
|
|
13
|
+
# procedure, including WHY every value here must be runner-scoped (multiple
|
|
14
|
+
# runners on one host share an OS user, so anything resolved against $HOME is
|
|
15
|
+
# a cross-runner concurrency hazard).
|
|
16
|
+
|
|
17
|
+
# Locale — pnpm/git/node emit UTF-8; a C locale garbles their output.
|
|
18
|
+
LANG=en_US.UTF-8
|
|
19
|
+
|
|
20
|
+
# Job-start hygiene hook. Runs templates/runner/job-cleanup.sh (installed
|
|
21
|
+
# into the runner root) before every job: reaps orphaned pnpm/node processes
|
|
22
|
+
# from THIS runner's work tree and clears stale install/temp artifacts.
|
|
23
|
+
# The hook is runner-scoped and never fails the job (always exits 0).
|
|
24
|
+
ACTIONS_RUNNER_HOOK_JOB_STARTED=<RUNNER_DIR>/job-cleanup.sh
|
|
25
|
+
|
|
26
|
+
# Runner-scoped tool cache. Without this, actions/setup-node & friends
|
|
27
|
+
# default the tool cache to a host-shared location and co-resident runners
|
|
28
|
+
# race on extraction. `_work/_tool` is inside this runner's own work tree,
|
|
29
|
+
# so each runner gets an isolated cache.
|
|
30
|
+
RUNNER_TOOL_CACHE=<RUNNER_DIR>/_work/_tool
|
|
31
|
+
|
|
32
|
+
# Same value, second consumer: some toolchain actions read
|
|
33
|
+
# AGENT_TOOLSDIRECTORY (the Azure Pipelines-era name) instead of
|
|
34
|
+
# RUNNER_TOOL_CACHE. Keep both pointing at the same runner-scoped dir.
|
|
35
|
+
AGENT_TOOLSDIRECTORY=<RUNNER_DIR>/_work/_tool
|
|
36
|
+
|
|
37
|
+
# ── pnpm scoping (read this before enabling the hook) ──────────────────────
|
|
38
|
+
# The pnpm shim install location is a WORKFLOW-side setting, not a runner-side
|
|
39
|
+
# one: pnpm/action-setup's `dest` input defaults to `~/setup-pnpm`, which is
|
|
40
|
+
# SHARED across every runner on the host. The platform's setup-toolchain
|
|
41
|
+
# composite action already re-defaults `dest` to `${{ runner.temp }}/pnpm`
|
|
42
|
+
# (runner-scoped); workflows that call pnpm/action-setup directly MUST pass
|
|
43
|
+
# `dest: ${{ runner.temp }}/pnpm` (or the pr-quality.yml `pnpm-dest` input).
|
|
44
|
+
# The job-cleanup.sh hook only reaps the runner-scoped locations — it
|
|
45
|
+
# deliberately never touches `~/setup-pnpm`.
|