mandrel-platform 1.0.1 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/scripts/audit-check.mjs +112 -12
- package/scripts/audit-check.test.mjs +195 -0
- package/scripts/check-cancelled-provenance.test.mjs +493 -1
- package/scripts/check-docs-staleness.mjs +15 -2
- package/scripts/check-docs-staleness.test.mjs +114 -9
- package/scripts/job-cleanup-hook.test.mjs +234 -0
- package/templates/runbooks/runner-provisioning.md +12 -3
- package/templates/runner/.env.example +8 -2
- package/templates/runner/job-cleanup.sh +49 -20
|
@@ -29,6 +29,24 @@
|
|
|
29
29
|
* cancel is likewise never neutralized under any policy: that tier produced no
|
|
30
30
|
* signal, and passing on a tier that never ran is a vacuous pass.
|
|
31
31
|
*
|
|
32
|
+
* TIMED-OUT + TIMEOUT HEADROOM (Story #342)
|
|
33
|
+
* -----------------------------------------
|
|
34
|
+
* A fifth cause was missing: GitHub killing a job that exceeded its own
|
|
35
|
+
* `timeout-minutes`. It reported as `stopped-mid-step`, or as `never-started`
|
|
36
|
+
* when the kill landed before a step completed — both of which point triage at
|
|
37
|
+
* the runner fleet when the fix is a number in the workflow. swarm-os run
|
|
38
|
+
* 30179418666 cost a full forensic misdiagnosis to exactly that.
|
|
39
|
+
*
|
|
40
|
+
* The ceilings themselves were also unreachable by callers, which is what made
|
|
41
|
+
* the class necessary: GitHub charges the pre-job `Set up runner` wait against
|
|
42
|
+
* the same clock, so on a saturated self-hosted pool a hardcoded 5-minute
|
|
43
|
+
* aggregator budget passes by luck. This suite therefore pins BOTH halves —
|
|
44
|
+
* the `tier-timeouts` override surface and the `timed-out` inference —
|
|
45
|
+
* plus the negative that makes them shippable: pr-quality.yml declares no new
|
|
46
|
+
* permission scope, because a reusable workflow's permissions are validated
|
|
47
|
+
* against the caller's grant at compile time and a new scope breaks every
|
|
48
|
+
* consumer (Story #292's `pull-requests: read` is the precedent).
|
|
49
|
+
*
|
|
32
50
|
* Run: node --test scripts/check-cancelled-provenance.test.mjs
|
|
33
51
|
*/
|
|
34
52
|
|
|
@@ -104,6 +122,28 @@ function runFixture(jobs, { createdAt = "2026-07-25T10:00:00Z" } = {}) {
|
|
|
104
122
|
};
|
|
105
123
|
}
|
|
106
124
|
|
|
125
|
+
// pr-quality.yml's own base budgets — what the workflow passes as
|
|
126
|
+
// TIER_TIMEOUT_BASES. The caller's override map (TIER_TIMEOUT_OVERRIDES) is
|
|
127
|
+
// unioned in by the script, so tests that care about an override state it.
|
|
128
|
+
const CEILINGS = "[5, 10, 15, 20, 45]";
|
|
129
|
+
|
|
130
|
+
/**
|
|
131
|
+
* A job with a wall duration, in seconds. Duration is the ONLY signal that
|
|
132
|
+
* separates a timeout from any other cancel, so every timed-out fixture is
|
|
133
|
+
* built by stating it explicitly rather than by hand-writing timestamps.
|
|
134
|
+
*/
|
|
135
|
+
function timedJob(name, conclusion, steps, durationSeconds) {
|
|
136
|
+
const startedAt = "2026-07-25T10:00:00Z";
|
|
137
|
+
// Whole-second ISO-8601 with no fractional part — the exact shape
|
|
138
|
+
// `gh run view --json jobs` emits for startedAt/completedAt. jq's
|
|
139
|
+
// `fromdateiso8601` rejects a `.000` fraction, so a fixture carrying one
|
|
140
|
+
// would exercise the degradation path instead of the classifier.
|
|
141
|
+
const completedAt = new Date(Date.parse(startedAt) + durationSeconds * 1000)
|
|
142
|
+
.toISOString()
|
|
143
|
+
.replace(/\.\d+Z$/, "Z");
|
|
144
|
+
return { name, conclusion, steps, startedAt, completedAt };
|
|
145
|
+
}
|
|
146
|
+
|
|
107
147
|
const FIXTURES = {
|
|
108
148
|
// A sibling genuinely failed; the rest are fail-fast collateral.
|
|
109
149
|
failFast: runFixture([
|
|
@@ -156,7 +196,14 @@ const semantics = {
|
|
|
156
196
|
|
|
157
197
|
function runAggregator(
|
|
158
198
|
needsResults,
|
|
159
|
-
{
|
|
199
|
+
{
|
|
200
|
+
policy = "strict",
|
|
201
|
+
runJson = null,
|
|
202
|
+
runList = null,
|
|
203
|
+
ghFails = false,
|
|
204
|
+
ceilings = CEILINGS,
|
|
205
|
+
overrides = "{}",
|
|
206
|
+
} = {}
|
|
160
207
|
) {
|
|
161
208
|
const dir = mkdtempSync(join(tmpdir(), "cancelled-provenance-"));
|
|
162
209
|
try {
|
|
@@ -208,6 +255,8 @@ function runAggregator(
|
|
|
208
255
|
PATH: `${bin}:${process.env.PATH}`,
|
|
209
256
|
NEEDS_JSON: JSON.stringify(needsJson),
|
|
210
257
|
CANCELLED_POLICY: policy,
|
|
258
|
+
TIER_TIMEOUT_BASES: ceilings,
|
|
259
|
+
TIER_TIMEOUT_OVERRIDES: overrides,
|
|
211
260
|
GH_TOKEN: "stub-token",
|
|
212
261
|
RUN_ID,
|
|
213
262
|
REPO: "Beestera/swarm-os",
|
|
@@ -277,12 +326,219 @@ test("a failed sibling outranks a never-started job", semantics, () => {
|
|
|
277
326
|
assert.doesNotMatch(r.stderr, /provenance: never-started/);
|
|
278
327
|
});
|
|
279
328
|
|
|
329
|
+
test(
|
|
330
|
+
"classifies a job killed at its ceiling as timed-out, not stopped-mid-step",
|
|
331
|
+
semantics,
|
|
332
|
+
() => {
|
|
333
|
+
// It ran steps, so the old classifier called this `stopped-mid-step` —
|
|
334
|
+
// "something external stopped a running job", which is not what happened.
|
|
335
|
+
const r = runAggregator(ONE_CANCELLED, {
|
|
336
|
+
runJson: runFixture([
|
|
337
|
+
timedJob("Unit (1/2)", "success", ranSteps, 90),
|
|
338
|
+
timedJob("E2E / Smoke (1/1)", "cancelled", ranSteps, 45 * 60),
|
|
339
|
+
]),
|
|
340
|
+
});
|
|
341
|
+
assert.equal(r.status, 1);
|
|
342
|
+
assert.match(r.stderr, /Cancelled provenance: timed-out/);
|
|
343
|
+
// The ceiling is named, so the operator knows WHICH number to raise.
|
|
344
|
+
assert.match(r.stderr, /E2E \/ Smoke \(1\/1\): timed-out \(hit its 45m ceiling\)/);
|
|
345
|
+
}
|
|
346
|
+
);
|
|
347
|
+
|
|
348
|
+
test(
|
|
349
|
+
"classifies a job killed at its ceiling before any step as timed-out, not never-started",
|
|
350
|
+
semantics,
|
|
351
|
+
() => {
|
|
352
|
+
// The swarm-os run 30179418666 shape: `Set up runner` ate the whole budget,
|
|
353
|
+
// so the job was killed with every step still SKIPPED. Reporting that as
|
|
354
|
+
// `never-started` asserts an infra provisioning hang and is what sent the
|
|
355
|
+
// consumer to file a runner investigation for a config fault.
|
|
356
|
+
const r = runAggregator(ONE_CANCELLED, {
|
|
357
|
+
runJson: runFixture([
|
|
358
|
+
timedJob("Unit (1/2)", "success", ranSteps, 90),
|
|
359
|
+
timedJob("Typecheck", "cancelled", neverStartedSteps, 15 * 60),
|
|
360
|
+
]),
|
|
361
|
+
});
|
|
362
|
+
assert.equal(r.status, 1);
|
|
363
|
+
assert.match(r.stderr, /Cancelled provenance: timed-out/);
|
|
364
|
+
assert.match(r.stderr, /Typecheck: timed-out \(hit its 15m ceiling\)/);
|
|
365
|
+
assert.doesNotMatch(r.stderr, /provenance: never-started/);
|
|
366
|
+
}
|
|
367
|
+
);
|
|
368
|
+
|
|
369
|
+
test("a failed sibling outranks a timed-out job", semantics, () => {
|
|
370
|
+
// A fail-fast cancel can race a ceiling. The failing sibling is still the
|
|
371
|
+
// thing to triage, so `fail-fast` stays first in the precedence order.
|
|
372
|
+
const r = runAggregator(ONE_CANCELLED, {
|
|
373
|
+
runJson: runFixture([
|
|
374
|
+
timedJob("Accessibility (2/3)", "failure", ranSteps, 120),
|
|
375
|
+
timedJob("E2E / Smoke (1/1)", "cancelled", ranSteps, 45 * 60),
|
|
376
|
+
]),
|
|
377
|
+
});
|
|
378
|
+
assert.match(r.stderr, /Cancelled provenance: fail-fast/);
|
|
379
|
+
assert.doesNotMatch(r.stderr, /provenance: timed-out/);
|
|
380
|
+
});
|
|
381
|
+
|
|
382
|
+
test(
|
|
383
|
+
"a cancel far from every ceiling keeps its existing classification",
|
|
384
|
+
semantics,
|
|
385
|
+
() => {
|
|
386
|
+
// Positive evidence only: a duration that matches no ceiling must not be
|
|
387
|
+
// relabelled. 30 minutes sits between the 20m and 45m tiers.
|
|
388
|
+
const r = runAggregator(ONE_CANCELLED, {
|
|
389
|
+
runJson: runFixture([
|
|
390
|
+
timedJob("Unit (1/2)", "success", ranSteps, 90),
|
|
391
|
+
timedJob("E2E / Smoke (1/1)", "cancelled", ranSteps, 30 * 60),
|
|
392
|
+
]),
|
|
393
|
+
runList: NO_NEWER_RUNS,
|
|
394
|
+
});
|
|
395
|
+
assert.equal(r.status, 1);
|
|
396
|
+
assert.match(r.stderr, /E2E \/ Smoke \(1\/1\): stopped-mid-step/);
|
|
397
|
+
assert.doesNotMatch(r.stderr, /provenance: timed-out/);
|
|
398
|
+
}
|
|
399
|
+
);
|
|
400
|
+
|
|
401
|
+
test(
|
|
402
|
+
"a cancel just UNDER a ceiling is not claimed as a timeout",
|
|
403
|
+
semantics,
|
|
404
|
+
() => {
|
|
405
|
+
// GitHub never kills a job early, so a duration below the ceiling is
|
|
406
|
+
// positive evidence AGAINST a timeout — and the seconds just under one are
|
|
407
|
+
// exactly where an ordinary fail-fast or superseded cancel of a
|
|
408
|
+
// long-running job lands. Matching there would be a confident wrong answer;
|
|
409
|
+
// the classifier must prefer a missing label to a false one.
|
|
410
|
+
const r = runAggregator(ONE_CANCELLED, {
|
|
411
|
+
runJson: runFixture([
|
|
412
|
+
timedJob("Unit (1/2)", "success", ranSteps, 90),
|
|
413
|
+
timedJob("E2E / Smoke (1/1)", "cancelled", ranSteps, 45 * 60 - 20),
|
|
414
|
+
]),
|
|
415
|
+
runList: NO_NEWER_RUNS,
|
|
416
|
+
});
|
|
417
|
+
assert.equal(r.status, 1);
|
|
418
|
+
assert.match(r.stderr, /E2E \/ Smoke \(1\/1\): stopped-mid-step/);
|
|
419
|
+
assert.doesNotMatch(r.stderr, /provenance: timed-out/);
|
|
420
|
+
}
|
|
421
|
+
);
|
|
422
|
+
|
|
423
|
+
test(
|
|
424
|
+
"a caller's override value is detectable as a ceiling",
|
|
425
|
+
semantics,
|
|
426
|
+
() => {
|
|
427
|
+
// The whole point of the override map: a caller that raises `e2e` to 57m
|
|
428
|
+
// must still get `timed-out` when that tier is killed at 57m. If only the
|
|
429
|
+
// platform's base list were consulted, every override would silently make
|
|
430
|
+
// its own tier undetectable — the failure mode would arrive exactly for the
|
|
431
|
+
// fleets that needed the input in the first place.
|
|
432
|
+
const jobs = [
|
|
433
|
+
timedJob("Unit (1/2)", "success", ranSteps, 90),
|
|
434
|
+
timedJob("E2E / Smoke (1/1)", "cancelled", ranSteps, 57 * 60),
|
|
435
|
+
];
|
|
436
|
+
const overridden = runAggregator(ONE_CANCELLED, {
|
|
437
|
+
overrides: '{"e2e": 57}',
|
|
438
|
+
runJson: runFixture(jobs),
|
|
439
|
+
runList: NO_NEWER_RUNS,
|
|
440
|
+
});
|
|
441
|
+
assert.equal(overridden.status, 1);
|
|
442
|
+
assert.match(overridden.stderr, /Cancelled provenance: timed-out/);
|
|
443
|
+
assert.match(overridden.stderr, /E2E \/ Smoke \(1\/1\): timed-out \(hit its 57m ceiling\)/);
|
|
444
|
+
|
|
445
|
+
// Same run without the override — 57m matches no base budget, so the label
|
|
446
|
+
// is not claimed. This is what proves the override is what carried it.
|
|
447
|
+
const plain = runAggregator(ONE_CANCELLED, {
|
|
448
|
+
runJson: runFixture(jobs),
|
|
449
|
+
runList: NO_NEWER_RUNS,
|
|
450
|
+
});
|
|
451
|
+
assert.match(plain.stderr, /E2E \/ Smoke \(1\/1\): stopped-mid-step/);
|
|
452
|
+
}
|
|
453
|
+
);
|
|
454
|
+
|
|
455
|
+
test(
|
|
456
|
+
"a ceiling the set omits under-detects rather than mis-detects",
|
|
457
|
+
semantics,
|
|
458
|
+
() => {
|
|
459
|
+
// The set is per-WORKFLOW, so a run can contain a job governed by a
|
|
460
|
+
// ceiling the set does not list — most concretely ci.yml, whose `[5, 10]`
|
|
461
|
+
// omits the nested pr-quality tiers' 15/20/45m budgets. The direction of
|
|
462
|
+
// that gap is the load-bearing part: a job killed at an unlisted ceiling
|
|
463
|
+
// keeps its previous label (a lost `timed-out`, still red), never a
|
|
464
|
+
// confident wrong one. Both halves are asserted here, since only the pair
|
|
465
|
+
// rules out the mirror-image bug where a narrow set relabels everything.
|
|
466
|
+
const jobs = [
|
|
467
|
+
timedJob("Node-script checks", "success", ranSteps, 90),
|
|
468
|
+
timedJob("security / E2E / Smoke (1/1)", "cancelled", ranSteps, 45 * 60),
|
|
469
|
+
];
|
|
470
|
+
const narrow = runAggregator(ONE_CANCELLED, {
|
|
471
|
+
ceilings: "[5, 10]",
|
|
472
|
+
runJson: runFixture(jobs),
|
|
473
|
+
runList: NO_NEWER_RUNS,
|
|
474
|
+
});
|
|
475
|
+
assert.equal(narrow.status, 1);
|
|
476
|
+
assert.match(narrow.stderr, /security \/ E2E \/ Smoke \(1\/1\): stopped-mid-step/);
|
|
477
|
+
assert.doesNotMatch(narrow.stderr, /provenance: timed-out/);
|
|
478
|
+
|
|
479
|
+
// Same job, same duration, against a set that DOES list 45 — proving the
|
|
480
|
+
// assertion above turns on the omission and not on the fixture.
|
|
481
|
+
const wide = runAggregator(ONE_CANCELLED, {
|
|
482
|
+
runJson: runFixture(jobs),
|
|
483
|
+
runList: NO_NEWER_RUNS,
|
|
484
|
+
});
|
|
485
|
+
assert.equal(wide.status, 1);
|
|
486
|
+
assert.match(wide.stderr, /Cancelled provenance: timed-out/);
|
|
487
|
+
}
|
|
488
|
+
);
|
|
489
|
+
|
|
490
|
+
test(
|
|
491
|
+
"an unusable ceiling set degrades to the pre-existing classification",
|
|
492
|
+
semantics,
|
|
493
|
+
() => {
|
|
494
|
+
// An absent or malformed TIER_TIMEOUT_MINUTES costs the label and nothing
|
|
495
|
+
// else — never a misclassification, and never a pass.
|
|
496
|
+
for (const ceilings of ["", "not json", '{"Unit": 20}']) {
|
|
497
|
+
const r = runAggregator(ONE_CANCELLED, {
|
|
498
|
+
ceilings,
|
|
499
|
+
runJson: runFixture([
|
|
500
|
+
timedJob("Unit (1/2)", "success", ranSteps, 90),
|
|
501
|
+
timedJob("Typecheck", "cancelled", neverStartedSteps, 15 * 60),
|
|
502
|
+
]),
|
|
503
|
+
});
|
|
504
|
+
assert.equal(r.status, 1, `ceilings=${JSON.stringify(ceilings)}`);
|
|
505
|
+
assert.match(
|
|
506
|
+
r.stderr,
|
|
507
|
+
/Cancelled provenance: never-started/,
|
|
508
|
+
`ceilings=${JSON.stringify(ceilings)} should fall back, not throw`
|
|
509
|
+
);
|
|
510
|
+
}
|
|
511
|
+
}
|
|
512
|
+
);
|
|
513
|
+
|
|
280
514
|
test("classification is reported on the job summary too", semantics, () => {
|
|
281
515
|
const r = runAggregator(ONE_CANCELLED, { runJson: FIXTURES.neverStarted });
|
|
282
516
|
assert.match(r.summary, /Provenance: `never-started`/);
|
|
283
517
|
assert.match(r.summary, /INFRA fault/);
|
|
284
518
|
});
|
|
285
519
|
|
|
520
|
+
test(
|
|
521
|
+
"a timed-out summary names the tier, its ceiling, and the remediation input",
|
|
522
|
+
semantics,
|
|
523
|
+
() => {
|
|
524
|
+
// The whole value of the class is legibility: the operator must be able to
|
|
525
|
+
// read the summary and know to edit a number, not to open a runner ticket.
|
|
526
|
+
const r = runAggregator(ONE_CANCELLED, {
|
|
527
|
+
runJson: runFixture([
|
|
528
|
+
timedJob("Unit (1/2)", "success", ranSteps, 90),
|
|
529
|
+
timedJob("Typecheck", "cancelled", neverStartedSteps, 15 * 60),
|
|
530
|
+
]),
|
|
531
|
+
});
|
|
532
|
+
assert.match(r.summary, /Provenance: `timed-out`/);
|
|
533
|
+
assert.match(r.summary, /Typecheck: timed-out \(hit its 15m ceiling\)/);
|
|
534
|
+
assert.match(r.summary, /CONFIG fault/);
|
|
535
|
+
assert.match(r.summary, /tier-timeouts/);
|
|
536
|
+
// It must not send the reader at the runner fleet — the misdiagnosis this
|
|
537
|
+
// class exists to prevent.
|
|
538
|
+
assert.match(r.summary, /do NOT escalate/);
|
|
539
|
+
}
|
|
540
|
+
);
|
|
541
|
+
|
|
286
542
|
test("a green run performs no provenance lookup at all", semantics, () => {
|
|
287
543
|
const r = runAggregator({ unit: "success", e2e: "success" });
|
|
288
544
|
assert.equal(r.status, 0, r.stderr);
|
|
@@ -344,6 +600,28 @@ test(
|
|
|
344
600
|
}
|
|
345
601
|
);
|
|
346
602
|
|
|
603
|
+
test(
|
|
604
|
+
"provenance-aware keeps a timed-out cancel red even when a newer run exists",
|
|
605
|
+
semantics,
|
|
606
|
+
() => {
|
|
607
|
+
// `timed-out` carries the same verdict semantics as `never-started`: the
|
|
608
|
+
// tier produced no complete signal. A newer sibling run is present here
|
|
609
|
+
// precisely so the test proves the timeout is what holds the gate red —
|
|
610
|
+
// the superseded probe never gets to run.
|
|
611
|
+
const r = runAggregator(ONE_CANCELLED, {
|
|
612
|
+
policy: "provenance-aware",
|
|
613
|
+
runJson: runFixture([
|
|
614
|
+
timedJob("Unit (1/2)", "success", ranSteps, 90),
|
|
615
|
+
timedJob("E2E / Smoke (1/1)", "cancelled", ranSteps, 45 * 60),
|
|
616
|
+
]),
|
|
617
|
+
runList: NEWER_RUNS,
|
|
618
|
+
});
|
|
619
|
+
assert.equal(r.status, 1);
|
|
620
|
+
assert.match(r.summary, /Provenance: `timed-out`/);
|
|
621
|
+
assert.doesNotMatch(r.summary, /neutral \(superseded run\)/);
|
|
622
|
+
}
|
|
623
|
+
);
|
|
624
|
+
|
|
347
625
|
test("provenance-aware keeps a fail-fast collateral cancel red", semantics, () => {
|
|
348
626
|
const r = runAggregator(ONE_CANCELLED, {
|
|
349
627
|
policy: "provenance-aware",
|
|
@@ -492,3 +770,217 @@ test("cancelled-policy is declared with a strict default", () => {
|
|
|
492
770
|
`declared instead: ${JSON.stringify(block)}`
|
|
493
771
|
);
|
|
494
772
|
});
|
|
773
|
+
|
|
774
|
+
test("tier-timeouts is declared with an empty-map default", () => {
|
|
775
|
+
// Same indentation walk as the sibling above, and the same reason for the
|
|
776
|
+
// default: an empty override map is byte-for-byte the behaviour every
|
|
777
|
+
// existing caller already gets, so adopting the input is opt-in.
|
|
778
|
+
const lines = readFileSync(join(repoRoot, WORKFLOW), "utf8").split("\n");
|
|
779
|
+
const start = lines.findIndex((l) => l === " tier-timeouts:");
|
|
780
|
+
assert.notEqual(start, -1, "`tier-timeouts:` input not declared");
|
|
781
|
+
|
|
782
|
+
const block = [];
|
|
783
|
+
for (let i = start + 1; i < lines.length; i++) {
|
|
784
|
+
if (/^\s*$/.test(lines[i])) continue;
|
|
785
|
+
if (lines[i].match(/^(\s*)/)[1].length <= 6) break;
|
|
786
|
+
block.push(lines[i].trim());
|
|
787
|
+
}
|
|
788
|
+
|
|
789
|
+
assert.ok(
|
|
790
|
+
block.includes("type: string"),
|
|
791
|
+
`\`tier-timeouts\` carries a JSON object, so it must be a string input; declared: ${JSON.stringify(block)}`
|
|
792
|
+
);
|
|
793
|
+
assert.ok(
|
|
794
|
+
block.includes("default: '{}'"),
|
|
795
|
+
"`tier-timeouts` must default to an empty map so existing consumers are unaffected; " +
|
|
796
|
+
`declared instead: ${JSON.stringify(block)}`
|
|
797
|
+
);
|
|
798
|
+
});
|
|
799
|
+
|
|
800
|
+
// ---------------------------------------------------------------------------
|
|
801
|
+
// 5. TIMEOUT SURFACE — every tier's ceiling is caller-tunable (Story #342).
|
|
802
|
+
//
|
|
803
|
+
// GitHub charges the pre-job `Set up runner` wait against the job's own
|
|
804
|
+
// `timeout-minutes`, so on a self-hosted pool a tier's real budget is
|
|
805
|
+
// `work + queue wait` and only the caller knows its own queue distribution. A
|
|
806
|
+
// literal ceiling left behind here is a tier the caller cannot reach — which is
|
|
807
|
+
// exactly the gap #340 filed, so it is asserted rather than trusted to review.
|
|
808
|
+
//
|
|
809
|
+
// The override is a `fromJSON(...)['<tier>'] || <base>` lookup rather than the
|
|
810
|
+
// single headroom addend #340 preferred, because GitHub Actions expressions
|
|
811
|
+
// have no arithmetic operators at all — `${{ 15 + inputs.x }}` fails to LEX,
|
|
812
|
+
// taking the whole workflow down with a "workflow file issue" and zero jobs.
|
|
813
|
+
// The `|| <base>` tail is what keeps a partial override map working, so it is
|
|
814
|
+
// asserted too: without it an unlisted tier would resolve to null.
|
|
815
|
+
// ---------------------------------------------------------------------------
|
|
816
|
+
|
|
817
|
+
/** Every `timeout-minutes:` declaration, tagged with its nesting depth. */
|
|
818
|
+
function timeoutDeclarations(rel) {
|
|
819
|
+
const lines = readFileSync(join(repoRoot, rel), "utf8").split("\n");
|
|
820
|
+
const found = [];
|
|
821
|
+
for (const [i, line] of lines.entries()) {
|
|
822
|
+
const m = line.match(/^(\s*)timeout-minutes:\s*(.+?)\s*$/);
|
|
823
|
+
if (m) found.push({ line: i + 1, indent: m[1].length, value: m[2] });
|
|
824
|
+
}
|
|
825
|
+
return found;
|
|
826
|
+
}
|
|
827
|
+
|
|
828
|
+
test("every job-level ceiling in pr-quality.yml is caller-overridable", () => {
|
|
829
|
+
// Job-level keys sit at 4 spaces (`jobs:` → `<job>:` → key); anything deeper
|
|
830
|
+
// is a step-level budget, covered by the next test.
|
|
831
|
+
const jobLevel = timeoutDeclarations(WORKFLOW).filter((d) => d.indent === 4);
|
|
832
|
+
assert.ok(
|
|
833
|
+
jobLevel.length >= 8,
|
|
834
|
+
`expected every tier plus the aggregator to declare a ceiling; found ${jobLevel.length}`
|
|
835
|
+
);
|
|
836
|
+
for (const d of jobLevel) {
|
|
837
|
+
assert.match(
|
|
838
|
+
d.value,
|
|
839
|
+
/^\$\{\{ fromJSON\(inputs\.tier-timeouts\)\['[a-z0-9-]+'\] \|\| \d+ \}\}$/,
|
|
840
|
+
`${WORKFLOW}:${d.line} declares a job-level ceiling the caller cannot reach ` +
|
|
841
|
+
`(${d.value}) — its budget is then work + queue wait with no way to raise ` +
|
|
842
|
+
"the clock, and without the `|| <base>` tail a partial override map would " +
|
|
843
|
+
"resolve an unlisted tier to null"
|
|
844
|
+
);
|
|
845
|
+
}
|
|
846
|
+
});
|
|
847
|
+
|
|
848
|
+
test("each tier's override key matches its own job id", () => {
|
|
849
|
+
// A copy-paste that points two tiers at one key silently makes one of them
|
|
850
|
+
// unreachable — the caller sets `e2e` and the e2e job keeps its default.
|
|
851
|
+
const lines = readFileSync(join(repoRoot, WORKFLOW), "utf8").split("\n");
|
|
852
|
+
let job = null;
|
|
853
|
+
const seen = new Set();
|
|
854
|
+
for (const line of lines) {
|
|
855
|
+
const j = line.match(/^ ([a-z][a-z0-9-]*):$/);
|
|
856
|
+
if (j) job = j[1];
|
|
857
|
+
const t = line.match(
|
|
858
|
+
/^ timeout-minutes: \$\{\{ fromJSON\(inputs\.tier-timeouts\)\['([a-z0-9-]+)'\]/
|
|
859
|
+
);
|
|
860
|
+
if (!t) continue;
|
|
861
|
+
assert.equal(
|
|
862
|
+
t[1],
|
|
863
|
+
job,
|
|
864
|
+
`job \`${job}\` reads override key \`${t[1]}\` — the key must be the job id, ` +
|
|
865
|
+
"or the caller's override for that tier lands on the wrong job (or nowhere)"
|
|
866
|
+
);
|
|
867
|
+
assert.ok(!seen.has(t[1]), `override key \`${t[1]}\` is claimed by two jobs`);
|
|
868
|
+
seen.add(t[1]);
|
|
869
|
+
}
|
|
870
|
+
assert.ok(seen.size >= 8, `expected every tier to be overridable; found ${seen.size}`);
|
|
871
|
+
});
|
|
872
|
+
|
|
873
|
+
test("the fail-fast cancel step's own budget carries no headroom", () => {
|
|
874
|
+
// `timeout-minutes: 1` on that step bounds a single API call inside an
|
|
875
|
+
// ALREADY-RUNNING job, so no queue wait is charged against it. Adding
|
|
876
|
+
// headroom there would slow a cancelled run down for no reason.
|
|
877
|
+
const stepLevel = timeoutDeclarations(WORKFLOW).filter((d) => d.indent > 4);
|
|
878
|
+
assert.ok(stepLevel.length > 0, "expected the fail-fast cancel step budget");
|
|
879
|
+
for (const d of stepLevel) {
|
|
880
|
+
assert.match(
|
|
881
|
+
d.value,
|
|
882
|
+
/^\d+$/,
|
|
883
|
+
`${WORKFLOW}:${d.line} is a step-level budget and must stay a literal; got ${d.value}`
|
|
884
|
+
);
|
|
885
|
+
}
|
|
886
|
+
});
|
|
887
|
+
|
|
888
|
+
test("the aggregator's base list carries every tier's default budget", () => {
|
|
889
|
+
// TIER_TIMEOUT_BASES is what makes `timed-out` detectable for a tier the
|
|
890
|
+
// caller did NOT override. If a tier's default budget changes and the list
|
|
891
|
+
// does not, that tier's timeouts silently fall back to `stopped-mid-step` —
|
|
892
|
+
// the misdiagnosis this Story removed. The caller's override values are
|
|
893
|
+
// unioned in at runtime, so they need no counterpart here.
|
|
894
|
+
const raw = readFileSync(join(repoRoot, WORKFLOW), "utf8");
|
|
895
|
+
const onJobs = new Set(
|
|
896
|
+
[
|
|
897
|
+
...raw.matchAll(
|
|
898
|
+
/^ {4}timeout-minutes: \$\{\{ fromJSON\(inputs\.tier-timeouts\)\['[a-z0-9-]+'\] \|\| (\d+) \}\}$/gm
|
|
899
|
+
),
|
|
900
|
+
].map((m) => m[1])
|
|
901
|
+
);
|
|
902
|
+
assert.ok(onJobs.size > 0, "no job-level ceilings found");
|
|
903
|
+
|
|
904
|
+
const decl = raw.match(/^\s*TIER_TIMEOUT_BASES: '(\[[^\]]*\])'$/m);
|
|
905
|
+
assert.ok(decl, "`TIER_TIMEOUT_BASES` not declared as a JSON array literal");
|
|
906
|
+
const declared = new Set(JSON.parse(decl[1]).map(String));
|
|
907
|
+
|
|
908
|
+
for (const base of onJobs) {
|
|
909
|
+
assert.ok(
|
|
910
|
+
declared.has(base),
|
|
911
|
+
`a tier defaults to a ${base}m budget but TIER_TIMEOUT_BASES does not list ` +
|
|
912
|
+
`it (${decl[1]}) — a timeout on that tier would be misreported as ` +
|
|
913
|
+
"stopped-mid-step"
|
|
914
|
+
);
|
|
915
|
+
}
|
|
916
|
+
});
|
|
917
|
+
|
|
918
|
+
test("the aggregator env carries no job name (self-maintaining)", () => {
|
|
919
|
+
// The reason the caller's override map is threaded through RAW rather than
|
|
920
|
+
// resolved per tier: a per-tier map in this env block would name every job,
|
|
921
|
+
// which is exactly the bookkeeping check-ci-required-aggregator.test.mjs
|
|
922
|
+
// forbids. That suite checks `needs:` ids; this one pins the narrower rule
|
|
923
|
+
// that made the design what it is, next to the code it constrains.
|
|
924
|
+
const raw = readFileSync(join(repoRoot, WORKFLOW), "utf8");
|
|
925
|
+
const start = raw.indexOf("TIER_TIMEOUT_BASES:");
|
|
926
|
+
const end = raw.indexOf("GH_TOKEN:", start);
|
|
927
|
+
assert.ok(start !== -1 && end > start, "ceiling env block not found");
|
|
928
|
+
const block = raw.slice(start, end);
|
|
929
|
+
assert.doesNotMatch(
|
|
930
|
+
block,
|
|
931
|
+
/fromJSON\(inputs\.tier-timeouts\)\[/,
|
|
932
|
+
"the ceiling env block resolves per-tier overrides by name — thread " +
|
|
933
|
+
"`inputs.tier-timeouts` through raw and union it in the script instead, so " +
|
|
934
|
+
"adding a job to `needs:` stays the only edit"
|
|
935
|
+
);
|
|
936
|
+
});
|
|
937
|
+
|
|
938
|
+
// ---------------------------------------------------------------------------
|
|
939
|
+
// 6. PERMISSION RATCHET — the negative that makes this shippable (Story #342).
|
|
940
|
+
//
|
|
941
|
+
// #341 proposed reading GitHub's own timeout annotation via
|
|
942
|
+
// `GET /repos/{owner}/{repo}/check-runs/{id}/annotations`, which needs
|
|
943
|
+
// `checks: read`. GitHub validates a called reusable workflow's declared
|
|
944
|
+
// permissions against the CALLER's grant at compile time, ignoring every job's
|
|
945
|
+
// `if:` gate — so a new scope fails the entire call with `startup_failure` for
|
|
946
|
+
// any consumer that has not granted it. Story #292 added `pull-requests: read`
|
|
947
|
+
// to one job and broke ci.yml, the cross-repo smoke consumer, and a release.
|
|
948
|
+
// This is why the classifier infers from duration instead, and the allowlist
|
|
949
|
+
// below is what keeps a future change from quietly reintroducing the break.
|
|
950
|
+
// ---------------------------------------------------------------------------
|
|
951
|
+
|
|
952
|
+
test("pr-quality.yml declares no permission grant outside the allowlist", () => {
|
|
953
|
+
// Scope AND level, not scope alone: escalating an existing `contents: read`
|
|
954
|
+
// to `contents: write` is the same class of compile-time consumer break as
|
|
955
|
+
// adding a brand-new scope, and a name-only allowlist would wave it through.
|
|
956
|
+
const ALLOWED = new Set([
|
|
957
|
+
"contents:read",
|
|
958
|
+
"actions:write",
|
|
959
|
+
"pull-requests:read",
|
|
960
|
+
]);
|
|
961
|
+
const lines = readFileSync(join(repoRoot, WORKFLOW), "utf8").split("\n");
|
|
962
|
+
const grants = new Set();
|
|
963
|
+
|
|
964
|
+
for (const [i, line] of lines.entries()) {
|
|
965
|
+
if (!/^\s*permissions:\s*$/.test(line)) continue;
|
|
966
|
+
const blockIndent = line.match(/^(\s*)/)[1].length;
|
|
967
|
+
for (let j = i + 1; j < lines.length; j++) {
|
|
968
|
+
if (/^\s*$/.test(lines[j]) || /^\s*#/.test(lines[j])) continue;
|
|
969
|
+
if (lines[j].match(/^(\s*)/)[1].length <= blockIndent) break;
|
|
970
|
+
const m = lines[j].match(/^\s*([a-z-]+):\s*(read|write|none)\s*$/);
|
|
971
|
+
if (m) grants.add(`${m[1]}:${m[2]}`);
|
|
972
|
+
}
|
|
973
|
+
}
|
|
974
|
+
|
|
975
|
+
assert.ok(grants.size > 0, "no `permissions:` block found to check");
|
|
976
|
+
for (const grant of grants) {
|
|
977
|
+
assert.ok(
|
|
978
|
+
ALLOWED.has(grant),
|
|
979
|
+
`pr-quality.yml declares \`${grant.replace(":", ": ")}\` — a reusable workflow's ` +
|
|
980
|
+
"permissions are validated against the caller's grant at COMPILE time regardless " +
|
|
981
|
+
"of any `if:` gate, so a new or escalated scope breaks every consumer that has " +
|
|
982
|
+
"not granted it (startup_failure, zero jobs). Widen this allowlist only alongside " +
|
|
983
|
+
"a lockstep update to ci.yml, the smoke consumer, and docs/reusable-workflows.md."
|
|
984
|
+
);
|
|
985
|
+
}
|
|
986
|
+
});
|
|
@@ -90,7 +90,14 @@ export const RULES = [
|
|
|
90
90
|
{
|
|
91
91
|
id: 'quality-yml-ref',
|
|
92
92
|
description: 'References `quality.yml` — verify this file exists in the project (swarm-os ships `ci.yml` instead)',
|
|
93
|
-
|
|
93
|
+
// The bare filename only — the lookbehind stops the platform's OWN
|
|
94
|
+
// `pr-quality.yml` (and any other `<prefix>-quality.yml`) from matching as
|
|
95
|
+
// a substring. Without it this rule produced 58 findings against
|
|
96
|
+
// mandrel-platform's docs and 1 was a real bare reference, drowning a
|
|
97
|
+
// genuine `expired-placeholder` error in known-benign warnings. Guarding on
|
|
98
|
+
// `[-\w]` rather than spelling out `pr-` keeps it correct for a consumer
|
|
99
|
+
// that names its own caller `ci-quality.yml`.
|
|
100
|
+
pattern: /(?<![-\w])quality\.yml/g,
|
|
94
101
|
severity: 'warning',
|
|
95
102
|
},
|
|
96
103
|
{
|
|
@@ -102,7 +109,13 @@ export const RULES = [
|
|
|
102
109
|
// 4-digit 20xx year and defer the "is it actually in the past?" decision
|
|
103
110
|
// to `matchFilter`, so the rule stays correct as the calendar advances and
|
|
104
111
|
// never flags a still-valid FUTURE expiry.
|
|
105
|
-
|
|
112
|
+
// The optional quotes either side of the separator are load-bearing: the
|
|
113
|
+
// CVE allowlist's own shape is JSON (`"expires": "2026-12-31"` — see
|
|
114
|
+
// audit-check.mjs), and `expires"` is neither `:` nor whitespace, so the
|
|
115
|
+
// unquoted-only form skipped every documented allowlist entry. That is the
|
|
116
|
+
// same fail-open class as the 202[0-4] year window this rule already fixed;
|
|
117
|
+
// it was hiding a second lapsed date in docs/runbooks/dependency-update.md.
|
|
118
|
+
pattern: /expires['"]?[:\s]+['"]?(20\d{2}-\d{2}-\d{2})/gi,
|
|
106
119
|
severity: 'error',
|
|
107
120
|
// Only flag when the captured date is strictly before today (UTC). Future
|
|
108
121
|
// expiries are still valid and must not be reported.
|