@codyswann/lisa 2.348.0 → 2.349.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/core/upstream-evidence-manifest.d.ts.map +1 -1
- package/dist/core/upstream-evidence-manifest.js +26 -3
- package/dist/core/upstream-evidence-manifest.js.map +1 -1
- package/expo/create-only/.github/required-checks.json +52 -0
- package/expo/create-only/.github/workflows/nightly-e2e-bypass-reaper.yml +83 -0
- package/expo/create-only/.github/workflows/nightly-e2e-health.yml +140 -0
- package/expo/github-rulesets/{playwright.json → nightly-e2e-health.json} +3 -8
- package/package.json +1 -1
- package/plugins/lisa/.claude-plugin/plugin.json +1 -1
- package/plugins/lisa/.codex-plugin/plugin.json +1 -1
- package/plugins/lisa-agy/plugin.json +1 -1
- package/plugins/lisa-cdk/.claude-plugin/plugin.json +1 -1
- package/plugins/lisa-cdk/.codex-plugin/plugin.json +1 -1
- package/plugins/lisa-cdk-agy/plugin.json +1 -1
- package/plugins/lisa-cdk-copilot/.claude-plugin/plugin.json +1 -1
- package/plugins/lisa-cdk-cursor/.claude-plugin/plugin.json +1 -1
- package/plugins/lisa-copilot/.claude-plugin/plugin.json +1 -1
- package/plugins/lisa-cursor/.claude-plugin/plugin.json +1 -1
- package/plugins/lisa-expo/.claude-plugin/plugin.json +1 -1
- package/plugins/lisa-expo/.codex-plugin/plugin.json +1 -1
- package/plugins/lisa-expo-agy/plugin.json +1 -1
- package/plugins/lisa-expo-copilot/.claude-plugin/plugin.json +1 -1
- package/plugins/lisa-expo-cursor/.claude-plugin/plugin.json +1 -1
- package/plugins/lisa-harper-fabric/.claude-plugin/plugin.json +1 -1
- package/plugins/lisa-harper-fabric/.codex-plugin/plugin.json +1 -1
- package/plugins/lisa-harper-fabric-agy/plugin.json +1 -1
- package/plugins/lisa-harper-fabric-copilot/.claude-plugin/plugin.json +1 -1
- package/plugins/lisa-harper-fabric-cursor/.claude-plugin/plugin.json +1 -1
- package/plugins/lisa-nestjs/.claude-plugin/plugin.json +1 -1
- package/plugins/lisa-nestjs/.codex-plugin/plugin.json +1 -1
- package/plugins/lisa-nestjs-agy/plugin.json +1 -1
- package/plugins/lisa-nestjs-copilot/.claude-plugin/plugin.json +1 -1
- package/plugins/lisa-nestjs-cursor/.claude-plugin/plugin.json +1 -1
- package/plugins/lisa-openclaw/.claude-plugin/plugin.json +1 -1
- package/plugins/lisa-openclaw/.codex-plugin/plugin.json +1 -1
- package/plugins/lisa-openclaw-agy/plugin.json +1 -1
- package/plugins/lisa-openclaw-copilot/.claude-plugin/plugin.json +1 -1
- package/plugins/lisa-openclaw-cursor/.claude-plugin/plugin.json +1 -1
- package/plugins/lisa-phaser/.claude-plugin/plugin.json +1 -1
- package/plugins/lisa-phaser/.codex-plugin/plugin.json +1 -1
- package/plugins/lisa-phaser-agy/plugin.json +1 -1
- package/plugins/lisa-phaser-copilot/.claude-plugin/plugin.json +1 -1
- package/plugins/lisa-phaser-cursor/.claude-plugin/plugin.json +1 -1
- package/plugins/lisa-rails/.claude-plugin/plugin.json +1 -1
- package/plugins/lisa-rails/.codex-plugin/plugin.json +1 -1
- package/plugins/lisa-rails-agy/plugin.json +1 -1
- package/plugins/lisa-rails-copilot/.claude-plugin/plugin.json +1 -1
- package/plugins/lisa-rails-cursor/.claude-plugin/plugin.json +1 -1
- package/plugins/lisa-typescript/.claude-plugin/plugin.json +1 -1
- package/plugins/lisa-typescript/.codex-plugin/plugin.json +1 -1
- package/plugins/lisa-typescript-agy/plugin.json +1 -1
- package/plugins/lisa-typescript-copilot/.claude-plugin/plugin.json +1 -1
- package/plugins/lisa-typescript-cursor/.claude-plugin/plugin.json +1 -1
- package/plugins/lisa-wiki/.claude-plugin/plugin.json +1 -1
- package/plugins/lisa-wiki/.codex-plugin/plugin.json +1 -1
- package/plugins/lisa-wiki-agy/plugin.json +1 -1
- package/plugins/lisa-wiki-copilot/.claude-plugin/plugin.json +1 -1
- package/plugins/lisa-wiki-cursor/.claude-plugin/plugin.json +1 -1
- package/typescript/copy-overwrite/scripts/check-nightly-e2e-health.mjs +1516 -0
- package/typescript/copy-overwrite/scripts/check-skipped-required-checks.mjs +580 -0
- package/typescript/copy-overwrite/scripts/nightly-e2e-suites.schema.json +85 -0
- package/typescript/create-only/.github/required-checks.json +28 -0
- package/typescript/package-lisa/package.lisa.json +4 -1
|
@@ -0,0 +1,1516 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* check-nightly-e2e-health — the fail-closed nightly e2e merge gate.
|
|
4
|
+
*
|
|
5
|
+
* Shipped by Lisa (copy-overwrite). The reusable workflow that drives it is
|
|
6
|
+
* `CodySwannGT/lisa/.github/workflows/nightly-e2e-health.yml`; the contract both
|
|
7
|
+
* halves implement is `docs/nightly-e2e-gate.md` in Lisa, whose §2 truth table
|
|
8
|
+
* is proven row-by-row by `tests/unit/scripts/nightly-e2e-health*.test.ts`
|
|
9
|
+
* (rows 1-16, `-api` rows 17-20, `-bypass` rows 21-25).
|
|
10
|
+
*
|
|
11
|
+
* Usage:
|
|
12
|
+
* node scripts/check-nightly-e2e-health.mjs # human report, exit 1 when blocked
|
|
13
|
+
* node scripts/check-nightly-e2e-health.mjs --json # machine report, always exit 0
|
|
14
|
+
* node scripts/check-nightly-e2e-health.mjs --contract-version
|
|
15
|
+
*
|
|
16
|
+
* Zero dependencies and no install step, on purpose: a gate that sits on every
|
|
17
|
+
* pull request has to be cheap enough to stay uncontroversial, and a gate that
|
|
18
|
+
* needs a lockfile resolved is a gate that flakes.
|
|
19
|
+
*
|
|
20
|
+
* ## What it queries
|
|
21
|
+
*
|
|
22
|
+
* GitHub Actions RUN HISTORY. It runs no tests, dispatches nothing, and reads no
|
|
23
|
+
* artifacts. A reusable workflow's `uses:` must be a static literal, so it
|
|
24
|
+
* CANNOT dynamically call a list of workflow filenames supplied in an input —
|
|
25
|
+
* any mental model of "the gate runs the suites in the table" is impossible on
|
|
26
|
+
* this platform. Artifacts are excluded deliberately: they are zip archives
|
|
27
|
+
* (Node ships no zip reader) and they expire, which would turn "the evidence
|
|
28
|
+
* aged out" into an unreadable verdict.
|
|
29
|
+
*
|
|
30
|
+
* ## Fail-closed, in one paragraph
|
|
31
|
+
*
|
|
32
|
+
* Only a fresh `success` on the required branch passes. Failed, timed out and
|
|
33
|
+
* action-required all block. So do `cancelled` and `skipped` — if cancelling a
|
|
34
|
+
* run read as inconclusive-and-passing, cancelling the suite would be a
|
|
35
|
+
* one-click way to clear the gate, the same false-green shape
|
|
36
|
+
* `check-skipped-required-checks.mjs` exists to refuse. So does a run that is
|
|
37
|
+
* missing, stale, on the wrong branch, or whose job the matcher no longer finds.
|
|
38
|
+
* An unreachable or rate-limited API is a HARD failure after a bounded retry;
|
|
39
|
+
* there is no configuration that renders "we could not check" as "it is fine".
|
|
40
|
+
*
|
|
41
|
+
* The one forgiving state is the TIME-BOXED bootstrap window (`bootstrap_until`),
|
|
42
|
+
* during which MISSING evidence is reported but does not block, always with its
|
|
43
|
+
* expiry timestamp on screen. Bootstrap forgives absence of evidence, never
|
|
44
|
+
* evidence of failure — a red run is red inside the window too.
|
|
45
|
+
*
|
|
46
|
+
* ## Inherited from three implementations, with one path closed
|
|
47
|
+
*
|
|
48
|
+
* `DECISIVE_CONCLUSIONS` comes from gemini's `check-nightly-e2e.mjs` and is kept
|
|
49
|
+
* because it is the right vocabulary. What is NOT kept is gemini's
|
|
50
|
+
* `unknown`-passes-with-a-warning: that is a fail-open path, and here `unknown`
|
|
51
|
+
* fails once bootstrap closes. The bypass model and the context-pinning
|
|
52
|
+
* discipline come from tunnl (TUN-525 / TUN-402). The job-name filter comes from
|
|
53
|
+
* propswap's `nightly-e2e-lib.sh`, whose unbounded bootstrap is what §4 of the
|
|
54
|
+
* contract time-boxes.
|
|
55
|
+
*
|
|
56
|
+
* @module scripts/check-nightly-e2e-health
|
|
57
|
+
*/
|
|
58
|
+
|
|
59
|
+
import { pathToFileURL } from "node:url";
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Contract version of the gate, asserted by the reusable workflow against its
|
|
63
|
+
* own expectation. The workflow travels by git ref and this script travels by
|
|
64
|
+
* `lisa apply`, so the two halves WILL drift; a MAJOR mismatch fails closed
|
|
65
|
+
* rather than running a contract neither half agrees on. See §8 of
|
|
66
|
+
* `docs/nightly-e2e-gate.md` for what counts as major / minor / patch.
|
|
67
|
+
*/
|
|
68
|
+
export const NIGHTLY_E2E_CONTRACT_VERSION = "1.0.0";
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* The conclusions that constitute a verdict about the code.
|
|
72
|
+
*
|
|
73
|
+
* Closed on purpose. `cancelled` / `skipped` / `neutral` / `stale` / `null` are
|
|
74
|
+
* operator or plumbing outcomes that say nothing about health, and a conclusion
|
|
75
|
+
* GitHub introduces after this file was written is unknown to us — an unknown
|
|
76
|
+
* conclusion is not evidence of health. Everything outside this set resolves to
|
|
77
|
+
* state `unknown`, which blocks once bootstrap closes.
|
|
78
|
+
*/
|
|
79
|
+
export const DECISIVE_CONCLUSIONS = Object.freeze(
|
|
80
|
+
new Set([
|
|
81
|
+
"success",
|
|
82
|
+
"failure",
|
|
83
|
+
"timed_out",
|
|
84
|
+
"action_required",
|
|
85
|
+
"startup_failure",
|
|
86
|
+
])
|
|
87
|
+
);
|
|
88
|
+
|
|
89
|
+
/** The one conclusion that clears the gate. */
|
|
90
|
+
export const GREEN_CONCLUSION = "success";
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* Run events that count as a nightly verdict.
|
|
94
|
+
*
|
|
95
|
+
* `workflow_dispatch` is the unblock path, not a convenience: it is what makes
|
|
96
|
+
* the gate escapable by FIXING rather than by waiting for tomorrow's cron. It
|
|
97
|
+
* stays branch-filtered — a dispatch from someone's feature branch must never
|
|
98
|
+
* clear the gate for everybody.
|
|
99
|
+
*/
|
|
100
|
+
export const COUNTED_EVENTS = Object.freeze(["schedule", "workflow_dispatch"]);
|
|
101
|
+
|
|
102
|
+
/** Suite states. `unknown` is "no readable verdict", which is not a pass. */
|
|
103
|
+
export const SUITE_STATES = Object.freeze({
|
|
104
|
+
pass: "pass",
|
|
105
|
+
fail: "fail",
|
|
106
|
+
unknown: "unknown",
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
// ---------------------------------------------------------------------------
|
|
110
|
+
// SECURITY LIMITS — source constants, never env-readable (portfolio doctrine)
|
|
111
|
+
// ---------------------------------------------------------------------------
|
|
112
|
+
//
|
|
113
|
+
// Every value in this block is a SOURCE CONSTANT and is deliberately not
|
|
114
|
+
// overridable from the environment. The doctrine comes from WS-0a: a security
|
|
115
|
+
// gate is an ALLOWLIST, never a denylist, and its limits live in code.
|
|
116
|
+
//
|
|
117
|
+
// The reason is that an env-readable limit fails OPEN on exactly the inputs
|
|
118
|
+
// nobody tests — an unset variable, a typo'd name, a new deployment that forgot
|
|
119
|
+
// to set it. `NIGHTLY_BOOTSTRAP_MAX_DAYS=100000` would have restored the
|
|
120
|
+
// forever-bootstrap this gate exists to delete, and a caller-supplied
|
|
121
|
+
// `bypass_reason_pattern` of `.*` would have satisfied the "reason required"
|
|
122
|
+
// rule with an empty PR body. Both were real holes in this file before the
|
|
123
|
+
// doctrine was applied to it.
|
|
124
|
+
//
|
|
125
|
+
// Callers may still TIGHTEN any of these through the workflow inputs. What they
|
|
126
|
+
// cannot do is loosen one, and `resolveSecurityLimits` below is the single
|
|
127
|
+
// place that is enforced — so fail-closed only has to be right once.
|
|
128
|
+
|
|
129
|
+
/** Hard ceiling on a bypass's lifetime, whatever the caller asks for. */
|
|
130
|
+
export const BYPASS_ABSOLUTE_MAX_HOURS = 72;
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* Repository permissions that may grant a bypass — an ALLOWLIST.
|
|
134
|
+
*
|
|
135
|
+
* Never expressed as "anything except read/triage": a denylist of the roles we
|
|
136
|
+
* happen to know about today silently admits any role GitHub adds tomorrow.
|
|
137
|
+
*/
|
|
138
|
+
export const BYPASS_PERMISSIONS = Object.freeze(new Set(["admin", "maintain"]));
|
|
139
|
+
|
|
140
|
+
/**
|
|
141
|
+
* The reason line the PR body must carry. ALWAYS enforced.
|
|
142
|
+
*
|
|
143
|
+
* A reason AND a tracker reference, in the artefact reviewers already read.
|
|
144
|
+
* Multiline so it can match one line of a body; not global, so `.exec` has no
|
|
145
|
+
* sticky `lastIndex` to alternate on.
|
|
146
|
+
*
|
|
147
|
+
* A caller-supplied `bypass_reason_pattern` is an ADDITIONAL requirement that
|
|
148
|
+
* must ALSO match — never a replacement for this one. That asymmetry is the
|
|
149
|
+
* whole point: an override can only narrow what qualifies as a valid bypass.
|
|
150
|
+
*/
|
|
151
|
+
export const REQUIRED_BYPASS_REASON_PATTERN =
|
|
152
|
+
"^Nightly-E2E-Bypass:\\s*(?<ticket>[A-Z][A-Z0-9]+-\\d+|#\\d+)\\s+(?<reason>\\S.*)$";
|
|
153
|
+
|
|
154
|
+
/**
|
|
155
|
+
* Back-compatible alias.
|
|
156
|
+
*
|
|
157
|
+
* @deprecated Use `REQUIRED_BYPASS_REASON_PATTERN`. Kept so the name in the
|
|
158
|
+
* published contract keeps resolving during the deprecation window (§8).
|
|
159
|
+
*/
|
|
160
|
+
export const DEFAULT_BYPASS_REASON_PATTERN = REQUIRED_BYPASS_REASON_PATTERN;
|
|
161
|
+
|
|
162
|
+
/** Hard ceiling on how far out a bootstrap window may sit. */
|
|
163
|
+
export const BOOTSTRAP_ABSOLUTE_MAX_DAYS = 30;
|
|
164
|
+
|
|
165
|
+
/** Hard ceiling on how stale a run may be and still speak for the branch. */
|
|
166
|
+
export const ABSOLUTE_MAX_FRESHNESS_HOURS = 720;
|
|
167
|
+
|
|
168
|
+
/** Hard ceiling on retry attempts, so a "bounded retry" stays bounded. */
|
|
169
|
+
export const ABSOLUTE_MAX_API_ATTEMPTS = 5;
|
|
170
|
+
|
|
171
|
+
/**
|
|
172
|
+
* Hard ceiling on job-list pages one run may consume.
|
|
173
|
+
*
|
|
174
|
+
* Unbounded, `NIGHTLY_API_MAX_PAGES` would remove the very bound that makes a
|
|
175
|
+
* truncated job list detectable.
|
|
176
|
+
*/
|
|
177
|
+
export const ABSOLUTE_MAX_API_PAGES = 20;
|
|
178
|
+
|
|
179
|
+
/**
|
|
180
|
+
* Hard ceiling on a single retry wait.
|
|
181
|
+
*
|
|
182
|
+
* Unbounded, `NIGHTLY_API_RETRY_MAX_SECONDS=86400` would park the gate until the
|
|
183
|
+
* runner timeout — a check that never reports, which on a required context
|
|
184
|
+
* blocks every PR just as effectively as a red one but with nothing to read.
|
|
185
|
+
*/
|
|
186
|
+
export const ABSOLUTE_MAX_RETRY_SECONDS = 120;
|
|
187
|
+
|
|
188
|
+
/**
|
|
189
|
+
* Applies every source-constant ceiling, in one place.
|
|
190
|
+
*
|
|
191
|
+
* Numeric ceilings CLAMP DOWN rather than fail, because clamping toward
|
|
192
|
+
* strictness cannot fail open and a caller who asked for a looser gate than
|
|
193
|
+
* policy allows should still get the policy gate rather than a broken one. The
|
|
194
|
+
* clamp is reported so it is never silent.
|
|
195
|
+
*
|
|
196
|
+
* `bootstrap_until` is the deliberate exception and FAILS instead of clamping
|
|
197
|
+
* (see `resolveBootstrap`): it is a date somebody chose, and quietly pulling it
|
|
198
|
+
* closer would make the gate arm on a day nobody expected.
|
|
199
|
+
*
|
|
200
|
+
* @param {{bypassMaxHours: number, bootstrapMaxDays: number, freshnessHours: number, apiMaxAttempts: number, apiMaxPages: number, apiRetryMaxSeconds: number}} requested - What the caller asked for
|
|
201
|
+
* @returns {{limits: object, clamped: ReadonlyArray<string>}} The effective limits and what was reduced
|
|
202
|
+
*/
|
|
203
|
+
export function resolveSecurityLimits(requested) {
|
|
204
|
+
const clamped = [];
|
|
205
|
+
/**
|
|
206
|
+
* Clamps one value to its source-constant ceiling, recording any reduction.
|
|
207
|
+
*
|
|
208
|
+
* @param {string} name - Caller-facing input name
|
|
209
|
+
* @param {number} asked - Requested value
|
|
210
|
+
* @param {number} ceiling - Source-constant ceiling
|
|
211
|
+
* @returns {number} The effective value
|
|
212
|
+
*/
|
|
213
|
+
const cap = (name, asked, ceiling) => {
|
|
214
|
+
if (asked > ceiling) {
|
|
215
|
+
clamped.push(
|
|
216
|
+
`\`${name}\` was ${asked}, above the policy ceiling of ${ceiling}; using ${ceiling}. This limit is a source constant and cannot be raised from a workflow input or the environment.`
|
|
217
|
+
);
|
|
218
|
+
return ceiling;
|
|
219
|
+
}
|
|
220
|
+
return asked;
|
|
221
|
+
};
|
|
222
|
+
return {
|
|
223
|
+
limits: Object.freeze({
|
|
224
|
+
bypassMaxHours: cap(
|
|
225
|
+
"bypass_max_hours",
|
|
226
|
+
requested.bypassMaxHours,
|
|
227
|
+
BYPASS_ABSOLUTE_MAX_HOURS
|
|
228
|
+
),
|
|
229
|
+
bootstrapMaxDays: cap(
|
|
230
|
+
"bootstrap_max_days",
|
|
231
|
+
requested.bootstrapMaxDays,
|
|
232
|
+
BOOTSTRAP_ABSOLUTE_MAX_DAYS
|
|
233
|
+
),
|
|
234
|
+
freshnessHours: cap(
|
|
235
|
+
"freshness_hours",
|
|
236
|
+
requested.freshnessHours,
|
|
237
|
+
ABSOLUTE_MAX_FRESHNESS_HOURS
|
|
238
|
+
),
|
|
239
|
+
apiMaxAttempts: cap(
|
|
240
|
+
"api_max_attempts",
|
|
241
|
+
requested.apiMaxAttempts,
|
|
242
|
+
ABSOLUTE_MAX_API_ATTEMPTS
|
|
243
|
+
),
|
|
244
|
+
apiMaxPages: cap(
|
|
245
|
+
"api_max_pages",
|
|
246
|
+
requested.apiMaxPages,
|
|
247
|
+
ABSOLUTE_MAX_API_PAGES
|
|
248
|
+
),
|
|
249
|
+
apiRetryMaxSeconds: cap(
|
|
250
|
+
"api_retry_max_seconds",
|
|
251
|
+
requested.apiRetryMaxSeconds,
|
|
252
|
+
ABSOLUTE_MAX_RETRY_SECONDS
|
|
253
|
+
),
|
|
254
|
+
}),
|
|
255
|
+
clamped: Object.freeze(clamped),
|
|
256
|
+
};
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
/** Raised for anything that makes the gate's own configuration unreadable. */
|
|
260
|
+
export class GateConfigError extends Error {
|
|
261
|
+
/**
|
|
262
|
+
* @param {string} message - What is wrong and how to fix it
|
|
263
|
+
*/
|
|
264
|
+
constructor(message) {
|
|
265
|
+
super(message);
|
|
266
|
+
this.name = "GateConfigError";
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
/** Raised when the Actions API could not be read after bounded retries. */
|
|
271
|
+
export class GateApiError extends Error {
|
|
272
|
+
/**
|
|
273
|
+
* @param {string} message - What failed
|
|
274
|
+
*/
|
|
275
|
+
constructor(message) {
|
|
276
|
+
super(message);
|
|
277
|
+
this.name = "GateApiError";
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
// ---------------------------------------------------------------------------
|
|
282
|
+
// 1. Configuration — the `suites` table
|
|
283
|
+
// ---------------------------------------------------------------------------
|
|
284
|
+
|
|
285
|
+
/** Keys a suite entry may carry. Anything else is a typo, and typos fail. */
|
|
286
|
+
const SUITE_KEYS = Object.freeze(
|
|
287
|
+
new Set(["label", "workflow", "match", "freshness_hours", "required_sha"])
|
|
288
|
+
);
|
|
289
|
+
|
|
290
|
+
/** Keys each match mode may carry. */
|
|
291
|
+
const MATCH_KEYS = Object.freeze({
|
|
292
|
+
run: new Set(["mode"]),
|
|
293
|
+
job: new Set(["mode", "name"]),
|
|
294
|
+
job_pattern: new Set(["mode", "pattern"]),
|
|
295
|
+
});
|
|
296
|
+
|
|
297
|
+
/**
|
|
298
|
+
* Rejects a value that is not a plain object.
|
|
299
|
+
*
|
|
300
|
+
* @param {unknown} value - Candidate
|
|
301
|
+
* @returns {boolean} True when the value is a non-array object
|
|
302
|
+
*/
|
|
303
|
+
function isPlainObject(value) {
|
|
304
|
+
return (
|
|
305
|
+
typeof value === "object" &&
|
|
306
|
+
value !== null &&
|
|
307
|
+
Array.isArray(value) === false
|
|
308
|
+
);
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
/**
|
|
312
|
+
* Validates one suite's `match` block.
|
|
313
|
+
*
|
|
314
|
+
* @param {unknown} match - Candidate match block
|
|
315
|
+
* @param {string} where - Human location for error messages
|
|
316
|
+
* @returns {{mode: string, name?: string, pattern?: string}} The validated block
|
|
317
|
+
* @throws {GateConfigError} When the block is unusable
|
|
318
|
+
*/
|
|
319
|
+
function validateMatch(match, where) {
|
|
320
|
+
if (!isPlainObject(match)) {
|
|
321
|
+
throw new GateConfigError(`${where}: \`match\` must be an object.`);
|
|
322
|
+
}
|
|
323
|
+
const mode = match.mode;
|
|
324
|
+
if (typeof mode !== "string" || !(mode in MATCH_KEYS)) {
|
|
325
|
+
throw new GateConfigError(
|
|
326
|
+
`${where}: \`match.mode\` must be one of "run", "job", "job_pattern" (got ${JSON.stringify(mode)}).`
|
|
327
|
+
);
|
|
328
|
+
}
|
|
329
|
+
for (const key of Object.keys(match)) {
|
|
330
|
+
if (!MATCH_KEYS[mode].has(key)) {
|
|
331
|
+
throw new GateConfigError(
|
|
332
|
+
`${where}: \`match\` carries unknown key \`${key}\` for mode "${mode}". An ignored key is a gate configured differently than you believe.`
|
|
333
|
+
);
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
if (mode === "job") {
|
|
337
|
+
if (typeof match.name !== "string" || match.name.length === 0) {
|
|
338
|
+
throw new GateConfigError(
|
|
339
|
+
`${where}: \`match.name\` is required and must be a non-empty string for mode "job".`
|
|
340
|
+
);
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
if (mode === "job_pattern") {
|
|
344
|
+
const { pattern } = match;
|
|
345
|
+
if (typeof pattern !== "string" || pattern.length === 0) {
|
|
346
|
+
throw new GateConfigError(
|
|
347
|
+
`${where}: \`match.pattern\` is required and must be a non-empty string for mode "job_pattern".`
|
|
348
|
+
);
|
|
349
|
+
}
|
|
350
|
+
if (!pattern.startsWith("^") || !pattern.endsWith("$")) {
|
|
351
|
+
throw new GateConfigError(
|
|
352
|
+
`${where}: \`match.pattern\` must be anchored at both ends (^…$). An unanchored regex is a substring test wearing a regex's clothes — "Playwright" would match "Playwright (skipped placeholder)". Got ${JSON.stringify(pattern)}.`
|
|
353
|
+
);
|
|
354
|
+
}
|
|
355
|
+
try {
|
|
356
|
+
// No `g` flag, ever: a sticky lastIndex makes repeated .test() calls
|
|
357
|
+
// return alternating answers, which reads as an intermittent gate.
|
|
358
|
+
void new RegExp(pattern);
|
|
359
|
+
} catch (error) {
|
|
360
|
+
throw new GateConfigError(
|
|
361
|
+
`${where}: \`match.pattern\` does not compile: ${error instanceof Error ? error.message : String(error)}`
|
|
362
|
+
);
|
|
363
|
+
}
|
|
364
|
+
}
|
|
365
|
+
return match;
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
/**
|
|
369
|
+
* Parses and validates the `suites` table.
|
|
370
|
+
*
|
|
371
|
+
* Every failure here is a HARD failure of the gate, never a warning: a table the
|
|
372
|
+
* gate cannot read is a gate that measures nothing, and a gate that measures
|
|
373
|
+
* nothing must not report success.
|
|
374
|
+
*
|
|
375
|
+
* @param {string} raw - The raw JSON string from the workflow input
|
|
376
|
+
* @returns {ReadonlyArray<object>} Validated, frozen suite entries
|
|
377
|
+
* @throws {GateConfigError} When the table is absent, malformed or ambiguous
|
|
378
|
+
*/
|
|
379
|
+
export function validateSuites(raw) {
|
|
380
|
+
if (typeof raw !== "string" || raw.trim().length === 0) {
|
|
381
|
+
throw new GateConfigError(
|
|
382
|
+
"The `suites` input is empty. The gate has nothing to check, and a gate with nothing to check must not report success."
|
|
383
|
+
);
|
|
384
|
+
}
|
|
385
|
+
let parsed;
|
|
386
|
+
try {
|
|
387
|
+
parsed = JSON.parse(raw);
|
|
388
|
+
} catch (error) {
|
|
389
|
+
throw new GateConfigError(
|
|
390
|
+
`The \`suites\` input is not valid JSON: ${error instanceof Error ? error.message : String(error)}`
|
|
391
|
+
);
|
|
392
|
+
}
|
|
393
|
+
if (!Array.isArray(parsed) || parsed.length === 0) {
|
|
394
|
+
throw new GateConfigError(
|
|
395
|
+
"The `suites` input must be a JSON array with at least one entry."
|
|
396
|
+
);
|
|
397
|
+
}
|
|
398
|
+
|
|
399
|
+
const labels = new Set();
|
|
400
|
+
const identities = new Set();
|
|
401
|
+
const suites = parsed.map((entry, index) => {
|
|
402
|
+
const where = `suites[${index}]`;
|
|
403
|
+
if (!isPlainObject(entry)) {
|
|
404
|
+
throw new GateConfigError(`${where}: each suite must be an object.`);
|
|
405
|
+
}
|
|
406
|
+
for (const key of Object.keys(entry)) {
|
|
407
|
+
if (!SUITE_KEYS.has(key)) {
|
|
408
|
+
throw new GateConfigError(
|
|
409
|
+
`${where}: unknown key \`${key}\`. Did you mean one of ${[...SUITE_KEYS].join(", ")}? A typo'd key silently takes the default, which is a looser gate than you wrote.`
|
|
410
|
+
);
|
|
411
|
+
}
|
|
412
|
+
}
|
|
413
|
+
if (typeof entry.label !== "string" || entry.label.length === 0) {
|
|
414
|
+
throw new GateConfigError(
|
|
415
|
+
`${where}: \`label\` is required and must be a non-empty string.`
|
|
416
|
+
);
|
|
417
|
+
}
|
|
418
|
+
if (typeof entry.workflow !== "string" || entry.workflow.length === 0) {
|
|
419
|
+
throw new GateConfigError(
|
|
420
|
+
`${where}: \`workflow\` is required and must be a non-empty workflow FILE name (e.g. "maestro-e2e.yml").`
|
|
421
|
+
);
|
|
422
|
+
}
|
|
423
|
+
if (labels.has(entry.label)) {
|
|
424
|
+
throw new GateConfigError(
|
|
425
|
+
`${where}: duplicate label ${JSON.stringify(entry.label)}. Two suites sharing a label produce one report line for two verdicts.`
|
|
426
|
+
);
|
|
427
|
+
}
|
|
428
|
+
labels.add(entry.label);
|
|
429
|
+
|
|
430
|
+
const match = validateMatch(entry.match, where);
|
|
431
|
+
const identity = `${entry.workflow}${match.mode}${match.name ?? match.pattern ?? ""}`;
|
|
432
|
+
if (identities.has(identity)) {
|
|
433
|
+
throw new GateConfigError(
|
|
434
|
+
`${where}: duplicate workflow+match (${entry.workflow}, ${match.mode}). The same suite declared twice is a copy-paste error, and the duplicate can mask a typo in the one you meant.`
|
|
435
|
+
);
|
|
436
|
+
}
|
|
437
|
+
identities.add(identity);
|
|
438
|
+
|
|
439
|
+
if (entry.freshness_hours !== undefined) {
|
|
440
|
+
const hours = entry.freshness_hours;
|
|
441
|
+
if (
|
|
442
|
+
typeof hours !== "number" ||
|
|
443
|
+
!Number.isFinite(hours) ||
|
|
444
|
+
hours <= 0 ||
|
|
445
|
+
hours > ABSOLUTE_MAX_FRESHNESS_HOURS
|
|
446
|
+
) {
|
|
447
|
+
throw new GateConfigError(
|
|
448
|
+
`${where}: \`freshness_hours\` must be a number in (0, ${ABSOLUTE_MAX_FRESHNESS_HOURS}].`
|
|
449
|
+
);
|
|
450
|
+
}
|
|
451
|
+
}
|
|
452
|
+
if (entry.required_sha !== undefined) {
|
|
453
|
+
if (
|
|
454
|
+
typeof entry.required_sha !== "string" ||
|
|
455
|
+
!/^[0-9a-f]{40}$/.test(entry.required_sha)
|
|
456
|
+
) {
|
|
457
|
+
throw new GateConfigError(
|
|
458
|
+
`${where}: \`required_sha\` must be a full 40-character lowercase commit SHA.`
|
|
459
|
+
);
|
|
460
|
+
}
|
|
461
|
+
}
|
|
462
|
+
return Object.freeze({ ...entry, match: Object.freeze({ ...match }) });
|
|
463
|
+
});
|
|
464
|
+
|
|
465
|
+
return Object.freeze(suites);
|
|
466
|
+
}
|
|
467
|
+
|
|
468
|
+
// ---------------------------------------------------------------------------
|
|
469
|
+
// 2. Classification — the truth table, as pure functions
|
|
470
|
+
// ---------------------------------------------------------------------------
|
|
471
|
+
|
|
472
|
+
/**
|
|
473
|
+
* Whether a run is inside the freshness window.
|
|
474
|
+
*
|
|
475
|
+
* @param {object} run - An Actions run
|
|
476
|
+
* @param {number} freshnessHours - Window size in hours
|
|
477
|
+
* @param {Date} now - Evaluation instant
|
|
478
|
+
* @returns {boolean} True when the run is fresh enough to speak for the branch
|
|
479
|
+
*/
|
|
480
|
+
export function isFresh(run, freshnessHours, now) {
|
|
481
|
+
const created = Date.parse(run?.created_at ?? "");
|
|
482
|
+
if (Number.isNaN(created)) return false;
|
|
483
|
+
return now.getTime() - created <= freshnessHours * 3_600_000;
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
/**
|
|
487
|
+
* Turns one conclusion into a suite state.
|
|
488
|
+
*
|
|
489
|
+
* @param {string|null|undefined} conclusion - The API's conclusion value
|
|
490
|
+
* @returns {"pass"|"fail"|"unknown"} The state
|
|
491
|
+
*/
|
|
492
|
+
export function stateForConclusion(conclusion) {
|
|
493
|
+
if (conclusion === GREEN_CONCLUSION) return SUITE_STATES.pass;
|
|
494
|
+
if (DECISIVE_CONCLUSIONS.has(conclusion)) return SUITE_STATES.fail;
|
|
495
|
+
return SUITE_STATES.unknown;
|
|
496
|
+
}
|
|
497
|
+
|
|
498
|
+
/**
|
|
499
|
+
* Assesses one suite from what was observed for it.
|
|
500
|
+
*
|
|
501
|
+
* `reason` is a stable machine token (the truth-table row), so the tests can
|
|
502
|
+
* assert the ROW rather than the prose, and the prose can be reworded without
|
|
503
|
+
* anyone claiming the contract changed.
|
|
504
|
+
*
|
|
505
|
+
* @param {object} suite - A validated suite entry
|
|
506
|
+
* @param {object} observation - `{ run, jobs, workflowMissing }`
|
|
507
|
+
* @param {{branch: string, freshnessHours: number, now: Date}} context - Evaluation context
|
|
508
|
+
* @returns {{label: string, state: string, reason: string, conclusion: string|null, url: string|null, createdAt: string|null, event: string|null}} The finding
|
|
509
|
+
*/
|
|
510
|
+
export function assessSuite(suite, observation, context) {
|
|
511
|
+
const base = { label: suite.label, workflow: suite.workflow };
|
|
512
|
+
const blank = { conclusion: null, url: null, createdAt: null, event: null };
|
|
513
|
+
|
|
514
|
+
// Row 11 — the workflow file the table names no longer exists. That is not
|
|
515
|
+
// missing evidence, it is a broken gate: someone renamed or deleted the suite
|
|
516
|
+
// out from under it, and bootstrap must not forgive that.
|
|
517
|
+
if (observation.workflowMissing) {
|
|
518
|
+
return {
|
|
519
|
+
...base,
|
|
520
|
+
...blank,
|
|
521
|
+
state: SUITE_STATES.fail,
|
|
522
|
+
reason: "workflow_not_found",
|
|
523
|
+
};
|
|
524
|
+
}
|
|
525
|
+
|
|
526
|
+
const run = observation.run ?? null;
|
|
527
|
+
if (!run) {
|
|
528
|
+
return { ...base, ...blank, state: SUITE_STATES.unknown, reason: "no_run" };
|
|
529
|
+
}
|
|
530
|
+
|
|
531
|
+
const seen = {
|
|
532
|
+
conclusion: run.conclusion ?? null,
|
|
533
|
+
url: run.html_url ?? null,
|
|
534
|
+
createdAt: run.created_at ?? null,
|
|
535
|
+
event: run.event ?? null,
|
|
536
|
+
};
|
|
537
|
+
|
|
538
|
+
// Row 15 — the API was asked for one branch; verifying head_branch anyway is
|
|
539
|
+
// defence against a filter that silently stops filtering.
|
|
540
|
+
if (run.head_branch && run.head_branch !== context.branch) {
|
|
541
|
+
return {
|
|
542
|
+
...base,
|
|
543
|
+
...seen,
|
|
544
|
+
state: SUITE_STATES.unknown,
|
|
545
|
+
reason: "wrong_branch",
|
|
546
|
+
};
|
|
547
|
+
}
|
|
548
|
+
// Row 16 — stale SHA.
|
|
549
|
+
if (suite.required_sha && run.head_sha !== suite.required_sha) {
|
|
550
|
+
return {
|
|
551
|
+
...base,
|
|
552
|
+
...seen,
|
|
553
|
+
state: SUITE_STATES.unknown,
|
|
554
|
+
reason: "stale_sha",
|
|
555
|
+
};
|
|
556
|
+
}
|
|
557
|
+
// Row 10 — runs exist, but none recent enough to speak for the branch.
|
|
558
|
+
const freshnessHours = suite.freshness_hours ?? context.freshnessHours;
|
|
559
|
+
if (!isFresh(run, freshnessHours, context.now)) {
|
|
560
|
+
return {
|
|
561
|
+
...base,
|
|
562
|
+
...seen,
|
|
563
|
+
state: SUITE_STATES.unknown,
|
|
564
|
+
reason: "stale_run",
|
|
565
|
+
};
|
|
566
|
+
}
|
|
567
|
+
|
|
568
|
+
if (suite.match.mode === "run") {
|
|
569
|
+
const state = stateForConclusion(run.conclusion);
|
|
570
|
+
return {
|
|
571
|
+
...base,
|
|
572
|
+
...seen,
|
|
573
|
+
state,
|
|
574
|
+
reason:
|
|
575
|
+
state === SUITE_STATES.unknown
|
|
576
|
+
? "indecisive_conclusion"
|
|
577
|
+
: "run_conclusion",
|
|
578
|
+
};
|
|
579
|
+
}
|
|
580
|
+
|
|
581
|
+
const jobs = observation.jobs ?? [];
|
|
582
|
+
const matches =
|
|
583
|
+
suite.match.mode === "job"
|
|
584
|
+
? jobs.filter(job => job.name === suite.match.name)
|
|
585
|
+
: jobs.filter(job => new RegExp(suite.match.pattern).test(job.name));
|
|
586
|
+
|
|
587
|
+
// Rows 12 and 13 — the matcher found nothing. A renamed job and a regex that
|
|
588
|
+
// no longer matches are the same defect, and both are how a gate stops gating
|
|
589
|
+
// with nothing to see. Never "nothing to report".
|
|
590
|
+
//
|
|
591
|
+
// `conclusion: null` is load-bearing here, not tidiness. `seen.conclusion` is
|
|
592
|
+
// the RUN's conclusion, which for a job-scoped suite is a different question
|
|
593
|
+
// from the one being answered — a workflow that also carries lint can conclude
|
|
594
|
+
// `failure` while the watched job is green, and vice versa. Reporting the run's
|
|
595
|
+
// value beside a job-derived state prints "❌ … [failure]" for a suite whose
|
|
596
|
+
// job was never found, or worse "✅ … [failure]" for one that passed. Both read
|
|
597
|
+
// as the gate contradicting itself, which is how a reader learns to stop
|
|
598
|
+
// trusting it.
|
|
599
|
+
if (matches.length === 0) {
|
|
600
|
+
return {
|
|
601
|
+
...base,
|
|
602
|
+
...seen,
|
|
603
|
+
conclusion: null,
|
|
604
|
+
state: SUITE_STATES.unknown,
|
|
605
|
+
reason:
|
|
606
|
+
suite.match.mode === "job"
|
|
607
|
+
? "job_not_found"
|
|
608
|
+
: "pattern_matched_nothing",
|
|
609
|
+
};
|
|
610
|
+
}
|
|
611
|
+
|
|
612
|
+
// Row 14 — any non-success match decides, and we report THAT job's URL so the
|
|
613
|
+
// message points at the thing that failed rather than at the run's summary.
|
|
614
|
+
const offender = matches.find(job => job.conclusion !== GREEN_CONCLUSION);
|
|
615
|
+
if (!offender) {
|
|
616
|
+
return {
|
|
617
|
+
...base,
|
|
618
|
+
...seen,
|
|
619
|
+
conclusion: GREEN_CONCLUSION,
|
|
620
|
+
state: SUITE_STATES.pass,
|
|
621
|
+
reason: "job_conclusion",
|
|
622
|
+
};
|
|
623
|
+
}
|
|
624
|
+
const state = stateForConclusion(offender.conclusion);
|
|
625
|
+
return {
|
|
626
|
+
...base,
|
|
627
|
+
...seen,
|
|
628
|
+
conclusion: offender.conclusion ?? null,
|
|
629
|
+
url: offender.html_url ?? seen.url,
|
|
630
|
+
state,
|
|
631
|
+
reason:
|
|
632
|
+
state === SUITE_STATES.unknown
|
|
633
|
+
? "indecisive_conclusion"
|
|
634
|
+
: "job_conclusion",
|
|
635
|
+
};
|
|
636
|
+
}
|
|
637
|
+
|
|
638
|
+
// ---------------------------------------------------------------------------
|
|
639
|
+
// 3. Bootstrap — time-boxed, with a visible expiry
|
|
640
|
+
// ---------------------------------------------------------------------------
|
|
641
|
+
|
|
642
|
+
/**
|
|
643
|
+
* Resolves the bootstrap window.
|
|
644
|
+
*
|
|
645
|
+
* A window further out than `maxDays` is INVALID CONFIGURATION and fails the
|
|
646
|
+
* gate (row 24), rather than being clamped. Clamping would let the window be
|
|
647
|
+
* extended forever by editing one string; failing makes extension require
|
|
648
|
+
* changing the cap too, which is a reviewable act.
|
|
649
|
+
*
|
|
650
|
+
* `maxDays` reaches here already clamped to `BOOTSTRAP_ABSOLUTE_MAX_DAYS` by
|
|
651
|
+
* `resolveSecurityLimits`, so a caller cannot raise the ceiling it is checked
|
|
652
|
+
* against — which is what stops this row from being defeated by one input.
|
|
653
|
+
*
|
|
654
|
+
* @param {string} until - ISO-8601 UTC timestamp, or "" for no window
|
|
655
|
+
* @param {number} maxDays - Ceiling on how far out the window may sit
|
|
656
|
+
* @param {Date} now - Evaluation instant
|
|
657
|
+
* @returns {{active: boolean, until: string|null, expiresInDays: number|null}} The window
|
|
658
|
+
* @throws {GateConfigError} When the timestamp is unparseable or beyond the cap
|
|
659
|
+
*/
|
|
660
|
+
export function resolveBootstrap(until, maxDays, now) {
|
|
661
|
+
if (typeof until !== "string" || until.trim().length === 0) {
|
|
662
|
+
return Object.freeze({ active: false, until: null, expiresInDays: null });
|
|
663
|
+
}
|
|
664
|
+
const parsed = Date.parse(until.trim());
|
|
665
|
+
if (Number.isNaN(parsed)) {
|
|
666
|
+
throw new GateConfigError(
|
|
667
|
+
`\`bootstrap_until\` is not an ISO-8601 timestamp: ${JSON.stringify(until)}. Use e.g. "2026-09-15T00:00:00Z".`
|
|
668
|
+
);
|
|
669
|
+
}
|
|
670
|
+
const days = (parsed - now.getTime()) / 86_400_000;
|
|
671
|
+
if (days > maxDays) {
|
|
672
|
+
throw new GateConfigError(
|
|
673
|
+
`\`bootstrap_until\` (${until}) is ${Math.ceil(days)} days out, beyond \`bootstrap_max_days\` (${maxDays}). A bootstrap window that can be extended by editing one string is propswap's forever-bootstrap: a suite that never runs passes forever. Raise the cap deliberately, in the same review, or bring the date in.`
|
|
674
|
+
);
|
|
675
|
+
}
|
|
676
|
+
return Object.freeze({
|
|
677
|
+
active: parsed > now.getTime(),
|
|
678
|
+
until: new Date(parsed).toISOString(),
|
|
679
|
+
expiresInDays: Math.max(0, Math.ceil(days)),
|
|
680
|
+
});
|
|
681
|
+
}
|
|
682
|
+
|
|
683
|
+
// ---------------------------------------------------------------------------
|
|
684
|
+
// 4. Bypass — maintainers only, no self-bypass, reason required, auto-expiring
|
|
685
|
+
// ---------------------------------------------------------------------------
|
|
686
|
+
|
|
687
|
+
/**
|
|
688
|
+
* Decides whether a bypass request is valid, as a pure function of the facts
|
|
689
|
+
* the caller gathered.
|
|
690
|
+
*
|
|
691
|
+
* Every rejection carries a stable `reason` token so the audit says WHICH
|
|
692
|
+
* condition failed. A bypass one person can both request and grant is not a
|
|
693
|
+
* control, which is why `self_bypass` exists as its own rejection.
|
|
694
|
+
*
|
|
695
|
+
* `extraReasonPattern` is an ADDITIONAL requirement, never a replacement for
|
|
696
|
+
* `REQUIRED_BYPASS_REASON_PATTERN`. An override that could replace the built-in
|
|
697
|
+
* rule would let `.*` satisfy "a reason and a ticket are required" with an empty
|
|
698
|
+
* PR body — a security limit a caller can relax is not a limit.
|
|
699
|
+
*
|
|
700
|
+
* @param {object} request - `{ labelEvent, prAuthor, prNumber, label, prBody, actorPermission, maxHours, extraReasonPattern, now }`
|
|
701
|
+
* @returns {{valid: boolean, reason: string, actor: string|null, appliedAt: string|null, expiresAt: string|null, ticket: string|null, detail: string|null}} The decision
|
|
702
|
+
*/
|
|
703
|
+
export function evaluateBypass(request) {
|
|
704
|
+
const {
|
|
705
|
+
labelEvent,
|
|
706
|
+
prAuthor,
|
|
707
|
+
prNumber = null,
|
|
708
|
+
label = null,
|
|
709
|
+
prBody,
|
|
710
|
+
actorPermission,
|
|
711
|
+
maxHours,
|
|
712
|
+
extraReasonPattern,
|
|
713
|
+
now,
|
|
714
|
+
} = request;
|
|
715
|
+
|
|
716
|
+
/** Identity of the request, carried onto every outcome for the audit. */
|
|
717
|
+
const subject = {
|
|
718
|
+
label,
|
|
719
|
+
prAuthor: prAuthor ?? null,
|
|
720
|
+
prNumber,
|
|
721
|
+
actorPermission: actorPermission ?? null,
|
|
722
|
+
};
|
|
723
|
+
|
|
724
|
+
const reject = (reason, extra = {}) =>
|
|
725
|
+
Object.freeze({
|
|
726
|
+
...subject,
|
|
727
|
+
valid: false,
|
|
728
|
+
reason,
|
|
729
|
+
actor: labelEvent?.actor ?? null,
|
|
730
|
+
appliedAt: labelEvent?.createdAt ?? null,
|
|
731
|
+
expiresAt: null,
|
|
732
|
+
ticket: null,
|
|
733
|
+
detail: null,
|
|
734
|
+
...extra,
|
|
735
|
+
});
|
|
736
|
+
|
|
737
|
+
if (!labelEvent || !labelEvent.actor || !labelEvent.createdAt) {
|
|
738
|
+
// The label is present but nobody can be shown to have applied it — on a
|
|
739
|
+
// fork PR the timeline is unreadable. An unattributable bypass is not an
|
|
740
|
+
// audited bypass.
|
|
741
|
+
return reject("no_attributable_actor");
|
|
742
|
+
}
|
|
743
|
+
if (!BYPASS_PERMISSIONS.has(actorPermission)) {
|
|
744
|
+
return reject("actor_not_maintainer", {
|
|
745
|
+
detail: actorPermission ?? "unknown",
|
|
746
|
+
});
|
|
747
|
+
}
|
|
748
|
+
if (prAuthor && labelEvent.actor.toLowerCase() === prAuthor.toLowerCase()) {
|
|
749
|
+
return reject("self_bypass");
|
|
750
|
+
}
|
|
751
|
+
|
|
752
|
+
const cappedHours = Math.min(maxHours, BYPASS_ABSOLUTE_MAX_HOURS);
|
|
753
|
+
const appliedMs = Date.parse(labelEvent.createdAt);
|
|
754
|
+
if (Number.isNaN(appliedMs)) return reject("no_attributable_actor");
|
|
755
|
+
const expiresMs = appliedMs + cappedHours * 3_600_000;
|
|
756
|
+
if (now.getTime() > expiresMs) {
|
|
757
|
+
return reject("bypass_expired", {
|
|
758
|
+
expiresAt: new Date(expiresMs).toISOString(),
|
|
759
|
+
});
|
|
760
|
+
}
|
|
761
|
+
|
|
762
|
+
// The built-in rule ALWAYS applies. It is checked first and on its own, so no
|
|
763
|
+
// caller-supplied pattern can stand in for it.
|
|
764
|
+
const found = new RegExp(REQUIRED_BYPASS_REASON_PATTERN, "m").exec(
|
|
765
|
+
prBody ?? ""
|
|
766
|
+
);
|
|
767
|
+
if (!found) {
|
|
768
|
+
return reject("no_reason_or_ticket", {
|
|
769
|
+
expiresAt: new Date(expiresMs).toISOString(),
|
|
770
|
+
});
|
|
771
|
+
}
|
|
772
|
+
|
|
773
|
+
// An optional project rule can only NARROW what qualifies — e.g. a repo that
|
|
774
|
+
// wants its own ticket prefix, or a second required line. It is an AND.
|
|
775
|
+
if (typeof extraReasonPattern === "string" && extraReasonPattern.length > 0) {
|
|
776
|
+
let extra;
|
|
777
|
+
try {
|
|
778
|
+
extra = new RegExp(extraReasonPattern, "m");
|
|
779
|
+
} catch (error) {
|
|
780
|
+
throw new GateConfigError(
|
|
781
|
+
`\`bypass_reason_pattern\` does not compile: ${error instanceof Error ? error.message : String(error)}`
|
|
782
|
+
);
|
|
783
|
+
}
|
|
784
|
+
if (!extra.test(prBody ?? "")) {
|
|
785
|
+
return reject("no_reason_or_ticket", {
|
|
786
|
+
expiresAt: new Date(expiresMs).toISOString(),
|
|
787
|
+
});
|
|
788
|
+
}
|
|
789
|
+
}
|
|
790
|
+
|
|
791
|
+
return Object.freeze({
|
|
792
|
+
...subject,
|
|
793
|
+
valid: true,
|
|
794
|
+
reason: "valid",
|
|
795
|
+
actor: labelEvent.actor,
|
|
796
|
+
appliedAt: new Date(appliedMs).toISOString(),
|
|
797
|
+
expiresAt: new Date(expiresMs).toISOString(),
|
|
798
|
+
ticket: found.groups?.ticket ?? null,
|
|
799
|
+
detail: (found.groups?.reason ?? "").trim() || null,
|
|
800
|
+
});
|
|
801
|
+
}
|
|
802
|
+
|
|
803
|
+
/** Human wording for each bypass rejection token. */
|
|
804
|
+
const BYPASS_REJECTIONS = Object.freeze({
|
|
805
|
+
no_attributable_actor:
|
|
806
|
+
"nobody can be shown to have applied it (the PR timeline was unreadable — this is normal on a fork PR). An unattributable bypass is not an audited bypass.",
|
|
807
|
+
actor_not_maintainer:
|
|
808
|
+
"the person who applied it does not have `admin` or `maintain` on this repository. Bypasses are maintainers only.",
|
|
809
|
+
self_bypass:
|
|
810
|
+
"it was applied by the PR's own author. A bypass one person can both request and grant is not a control.",
|
|
811
|
+
bypass_expired:
|
|
812
|
+
"it was applied longer ago than `bypass_max_hours` allows. Bypasses auto-expire so a label nobody removes cannot become a permanent hole.",
|
|
813
|
+
no_reason_or_ticket:
|
|
814
|
+
"the PR body carries no `Nightly-E2E-Bypass: <TICKET> <reason>` line. A bypass without a reason and a ticket is not auditable.",
|
|
815
|
+
});
|
|
816
|
+
|
|
817
|
+
// ---------------------------------------------------------------------------
|
|
818
|
+
// 5. Verdict
|
|
819
|
+
// ---------------------------------------------------------------------------
|
|
820
|
+
|
|
821
|
+
/**
|
|
822
|
+
* Combines suite findings, the bootstrap window and the bypass decision into
|
|
823
|
+
* the one verdict the check reports.
|
|
824
|
+
*
|
|
825
|
+
* Order matters and is the contract: bootstrap softens only `unknown`, and the
|
|
826
|
+
* bypass is applied LAST — so a bypass waives a genuine red, and a green PR
|
|
827
|
+
* carrying a stale label still reports `pass` rather than pretending the label
|
|
828
|
+
* did something.
|
|
829
|
+
*
|
|
830
|
+
* @param {ReadonlyArray<object>} findings - Per-suite findings
|
|
831
|
+
* @param {{bootstrap: object, bypass: object|null}} options - Window and bypass decision
|
|
832
|
+
* @returns {{verdict: string, blocked: boolean, findings: ReadonlyArray<object>, bootstrap: object, bypass: object|null}} The verdict
|
|
833
|
+
*/
|
|
834
|
+
export function decide(findings, { bootstrap, bypass = null }) {
|
|
835
|
+
const rendered = findings.map(finding =>
|
|
836
|
+
finding.state === SUITE_STATES.unknown && bootstrap.active
|
|
837
|
+
? { ...finding, state: "bootstrap" }
|
|
838
|
+
: finding
|
|
839
|
+
);
|
|
840
|
+
const blocking = rendered.filter(
|
|
841
|
+
finding =>
|
|
842
|
+
finding.state === SUITE_STATES.fail ||
|
|
843
|
+
finding.state === SUITE_STATES.unknown
|
|
844
|
+
);
|
|
845
|
+
|
|
846
|
+
if (blocking.length === 0) {
|
|
847
|
+
const anyBootstrap = rendered.some(
|
|
848
|
+
finding => finding.state === "bootstrap"
|
|
849
|
+
);
|
|
850
|
+
return Object.freeze({
|
|
851
|
+
verdict: anyBootstrap ? "bootstrap" : "pass",
|
|
852
|
+
blocked: false,
|
|
853
|
+
findings: Object.freeze(rendered),
|
|
854
|
+
bootstrap,
|
|
855
|
+
// A stale label on a non-red PR waived nothing; say so by reporting no
|
|
856
|
+
// bypass at all rather than an inert one.
|
|
857
|
+
bypass: null,
|
|
858
|
+
});
|
|
859
|
+
}
|
|
860
|
+
|
|
861
|
+
if (bypass?.valid) {
|
|
862
|
+
return Object.freeze({
|
|
863
|
+
verdict: "bypassed",
|
|
864
|
+
blocked: false,
|
|
865
|
+
findings: Object.freeze(rendered),
|
|
866
|
+
bootstrap,
|
|
867
|
+
bypass: Object.freeze({ ...bypass, waived: Object.freeze(blocking) }),
|
|
868
|
+
});
|
|
869
|
+
}
|
|
870
|
+
|
|
871
|
+
return Object.freeze({
|
|
872
|
+
verdict: "fail",
|
|
873
|
+
blocked: true,
|
|
874
|
+
findings: Object.freeze(rendered),
|
|
875
|
+
bootstrap,
|
|
876
|
+
bypass: bypass ?? null,
|
|
877
|
+
});
|
|
878
|
+
}
|
|
879
|
+
|
|
880
|
+
// ---------------------------------------------------------------------------
|
|
881
|
+
// 6. Reporting
|
|
882
|
+
// ---------------------------------------------------------------------------
|
|
883
|
+
|
|
884
|
+
/** Marker per rendered state. */
|
|
885
|
+
const STATE_MARKERS = Object.freeze({
|
|
886
|
+
pass: "✅",
|
|
887
|
+
fail: "❌",
|
|
888
|
+
unknown: "⚪",
|
|
889
|
+
bootstrap: "⚠️",
|
|
890
|
+
});
|
|
891
|
+
|
|
892
|
+
/** One human sentence per truth-table reason. */
|
|
893
|
+
const REASON_TEXT = Object.freeze({
|
|
894
|
+
workflow_not_found:
|
|
895
|
+
"the workflow file this gate watches does not exist any more. Someone renamed or deleted the suite out from under the gate — fix the `suites` table or restore the workflow.",
|
|
896
|
+
no_run: "no completed run on this branch at all.",
|
|
897
|
+
stale_run: "no completed run inside the freshness window.",
|
|
898
|
+
wrong_branch: "the newest run is on a different branch.",
|
|
899
|
+
stale_sha: "the newest run is for a different commit than the one required.",
|
|
900
|
+
indecisive_conclusion:
|
|
901
|
+
"the newest run reached no verdict about the code (cancelled / skipped / neutral). That is not a green — cancelling a run must never be a one-click way to clear a merge gate.",
|
|
902
|
+
job_not_found:
|
|
903
|
+
"the run completed without ever producing the job this gate reads. The job was renamed, which silently disarms the gate.",
|
|
904
|
+
pattern_matched_nothing:
|
|
905
|
+
"the job pattern matched zero jobs in the newest run. Zero matches is the signature of a renamed job.",
|
|
906
|
+
run_conclusion: "",
|
|
907
|
+
job_conclusion: "",
|
|
908
|
+
});
|
|
909
|
+
|
|
910
|
+
/**
|
|
911
|
+
* Renders one finding as a single line.
|
|
912
|
+
*
|
|
913
|
+
* @param {object} finding - A rendered finding
|
|
914
|
+
* @returns {string} One report line
|
|
915
|
+
*/
|
|
916
|
+
export function formatFinding(finding) {
|
|
917
|
+
const marker = STATE_MARKERS[finding.state] ?? "•";
|
|
918
|
+
const when = finding.createdAt
|
|
919
|
+
? ` (${finding.createdAt} via \`${finding.event ?? "?"}\`)`
|
|
920
|
+
: "";
|
|
921
|
+
const link = finding.url ? `: ${finding.url}` : "";
|
|
922
|
+
const detail = REASON_TEXT[finding.reason] || "";
|
|
923
|
+
if (finding.state === "pass") {
|
|
924
|
+
return `${marker} ${finding.label} — green${when}${link}`;
|
|
925
|
+
}
|
|
926
|
+
const verdictWord =
|
|
927
|
+
finding.state === "bootstrap"
|
|
928
|
+
? "not yet blocking"
|
|
929
|
+
: finding.state.toUpperCase();
|
|
930
|
+
const conclusion = finding.conclusion ? ` [${finding.conclusion}]` : "";
|
|
931
|
+
return `${marker} ${finding.label} — ${verdictWord}${conclusion}${when} — ${detail}${link}`;
|
|
932
|
+
}
|
|
933
|
+
|
|
934
|
+
/**
|
|
935
|
+
* Renders the full report.
|
|
936
|
+
*
|
|
937
|
+
* @param {object} verdict - Output of `decide`
|
|
938
|
+
* @param {{branch: string, bypassLabel: string}} context - Report context
|
|
939
|
+
* @returns {string} Markdown report
|
|
940
|
+
*/
|
|
941
|
+
export function formatReport(verdict, context) {
|
|
942
|
+
const lines = ["## 🌙 Nightly E2E Health", ""];
|
|
943
|
+
lines.push(...verdict.findings.map(finding => `- ${formatFinding(finding)}`));
|
|
944
|
+
lines.push("");
|
|
945
|
+
|
|
946
|
+
// A limit the caller asked for and did not get must be visible, or the gate
|
|
947
|
+
// is quietly stricter than its configuration says and the next person debugs
|
|
948
|
+
// the wrong thing.
|
|
949
|
+
for (const note of verdict.clamped ?? []) {
|
|
950
|
+
lines.push(`🔒 **Policy ceiling applied** — ${note}`, "");
|
|
951
|
+
}
|
|
952
|
+
|
|
953
|
+
if (verdict.bootstrap.active) {
|
|
954
|
+
lines.push(
|
|
955
|
+
`⏳ **Bootstrap window active — expires ${verdict.bootstrap.until} (${verdict.bootstrap.expiresInDays} day(s) from now).** Missing evidence is reported but not blocking until then. Evidence of FAILURE still blocks, inside the window as well as outside it. When the window lapses, every ⚠️ above becomes a ❌ with no further action.`,
|
|
956
|
+
""
|
|
957
|
+
);
|
|
958
|
+
}
|
|
959
|
+
|
|
960
|
+
if (verdict.verdict === "bypassed") {
|
|
961
|
+
lines.push(
|
|
962
|
+
`⚠️ **Gate bypassed — audited.** Applied by \`${verdict.bypass.actor}\` at ${verdict.bypass.appliedAt}, expires ${verdict.bypass.expiresAt}. Ticket: \`${verdict.bypass.ticket}\`. Reason: ${verdict.bypass.detail}`,
|
|
963
|
+
"",
|
|
964
|
+
"Waived:",
|
|
965
|
+
...verdict.bypass.waived.map(
|
|
966
|
+
finding =>
|
|
967
|
+
` - ${finding.label} — ${finding.conclusion ?? finding.reason}${finding.url ? `: ${finding.url}` : ""}`
|
|
968
|
+
),
|
|
969
|
+
"",
|
|
970
|
+
"The nightly is still red. This waives the gate for THIS pull request only; the tracking issue stays open until a green run lands."
|
|
971
|
+
);
|
|
972
|
+
} else if (verdict.blocked) {
|
|
973
|
+
if (verdict.bypass && !verdict.bypass.valid) {
|
|
974
|
+
lines.push(
|
|
975
|
+
`⛔ **A \`${context.bypassLabel}\` label is present but was REJECTED** — ${BYPASS_REJECTIONS[verdict.bypass.reason] ?? verdict.bypass.reason}`,
|
|
976
|
+
""
|
|
977
|
+
);
|
|
978
|
+
}
|
|
979
|
+
lines.push(
|
|
980
|
+
`Merges into \`${context.branch}\` are blocked until the nightly e2e suites are green again. To unblock:`,
|
|
981
|
+
" 1. Fix the failure (open the run above — it names the failing spec or flow).",
|
|
982
|
+
` 2. Re-run the suite from the Actions tab against \`${context.branch}\`. A \`workflow_dispatch\` run counts exactly like a scheduled one, so a green dispatch clears this gate immediately — no waiting for tomorrow.`,
|
|
983
|
+
" 3. Re-run this check on your PR.",
|
|
984
|
+
"",
|
|
985
|
+
`If the failure is in the harness rather than the app — or this IS the PR that fixes the red nightly — a maintainer (not you) can apply the \`${context.bypassLabel}\` label after you add a \`Nightly-E2E-Bypass: <TICKET> <reason>\` line to the PR body. There is no admin-merge-past-red: the audited bypass is the only sanctioned path.`
|
|
986
|
+
);
|
|
987
|
+
} else {
|
|
988
|
+
lines.push(
|
|
989
|
+
"Last night's e2e verdict is not red. Nothing here blocks this pull request."
|
|
990
|
+
);
|
|
991
|
+
}
|
|
992
|
+
return `${lines.join("\n")}\n`;
|
|
993
|
+
}
|
|
994
|
+
|
|
995
|
+
// ---------------------------------------------------------------------------
|
|
996
|
+
// 7. The Actions API
|
|
997
|
+
// ---------------------------------------------------------------------------
|
|
998
|
+
|
|
999
|
+
/**
|
|
1000
|
+
* Sleeps.
|
|
1001
|
+
*
|
|
1002
|
+
* @param {number} ms - Milliseconds
|
|
1003
|
+
* @returns {Promise<void>} Resolves after the delay
|
|
1004
|
+
*/
|
|
1005
|
+
function sleep(ms) {
|
|
1006
|
+
return new Promise(resolve => setTimeout(resolve, ms));
|
|
1007
|
+
}
|
|
1008
|
+
|
|
1009
|
+
/**
|
|
1010
|
+
* How long to wait before retrying a throttled response, bounded.
|
|
1011
|
+
*
|
|
1012
|
+
* @param {Response} response - The throttled response
|
|
1013
|
+
* @param {number} attempt - 1-based attempt number
|
|
1014
|
+
* @param {number} maxSeconds - Ceiling on the wait
|
|
1015
|
+
* @returns {number} Milliseconds to wait
|
|
1016
|
+
*/
|
|
1017
|
+
export function retryDelayMs(response, attempt, maxSeconds) {
|
|
1018
|
+
const retryAfter = Number(response.headers?.get?.("retry-after"));
|
|
1019
|
+
if (Number.isFinite(retryAfter) && retryAfter > 0) {
|
|
1020
|
+
return Math.min(retryAfter, maxSeconds) * 1000;
|
|
1021
|
+
}
|
|
1022
|
+
const remaining = response.headers?.get?.("x-ratelimit-remaining");
|
|
1023
|
+
const reset = Number(response.headers?.get?.("x-ratelimit-reset"));
|
|
1024
|
+
if (remaining === "0" && Number.isFinite(reset)) {
|
|
1025
|
+
const seconds = Math.max(0, reset - Math.floor(Date.now() / 1000));
|
|
1026
|
+
return Math.min(seconds, maxSeconds) * 1000;
|
|
1027
|
+
}
|
|
1028
|
+
return Math.min(2 ** attempt, maxSeconds) * 1000;
|
|
1029
|
+
}
|
|
1030
|
+
|
|
1031
|
+
/**
|
|
1032
|
+
* One GET against the Actions API, with bounded retry.
|
|
1033
|
+
*
|
|
1034
|
+
* 404 is returned to the caller as `null` because it is meaningful (a workflow
|
|
1035
|
+
* file that no longer exists). Everything else that is not OK is retried and
|
|
1036
|
+
* then RAISED — "we could not check" must never render as "it is fine".
|
|
1037
|
+
*
|
|
1038
|
+
* @param {object} api - `{ apiUrl, repo, token, maxAttempts, retryMaxSeconds }`
|
|
1039
|
+
* @param {string} path - API path beginning with `/`
|
|
1040
|
+
* @param {(ms: number) => Promise<void>} [wait] - Injectable sleep, for tests
|
|
1041
|
+
* @returns {Promise<{body: object, headers: Headers}|null>} Response, or null on 404
|
|
1042
|
+
* @throws {GateApiError} When the API stayed unreadable
|
|
1043
|
+
*/
|
|
1044
|
+
export async function apiGet(api, path, wait = sleep) {
|
|
1045
|
+
let lastProblem = "unknown";
|
|
1046
|
+
for (let attempt = 1; attempt <= api.maxAttempts; attempt += 1) {
|
|
1047
|
+
let response;
|
|
1048
|
+
try {
|
|
1049
|
+
response = await fetch(`${api.apiUrl}${path}`, {
|
|
1050
|
+
headers: {
|
|
1051
|
+
accept: "application/vnd.github+json",
|
|
1052
|
+
authorization: `Bearer ${api.token}`,
|
|
1053
|
+
"x-github-api-version": "2022-11-28",
|
|
1054
|
+
"user-agent": "lisa-nightly-e2e-health",
|
|
1055
|
+
},
|
|
1056
|
+
});
|
|
1057
|
+
} catch (error) {
|
|
1058
|
+
lastProblem = `network error: ${error instanceof Error ? error.message : String(error)}`;
|
|
1059
|
+
if (attempt < api.maxAttempts)
|
|
1060
|
+
await wait(
|
|
1061
|
+
retryDelayMs({ headers: null }, attempt, api.retryMaxSeconds)
|
|
1062
|
+
);
|
|
1063
|
+
continue;
|
|
1064
|
+
}
|
|
1065
|
+
if (response.status === 404) return null;
|
|
1066
|
+
if (response.ok) {
|
|
1067
|
+
return { body: await response.json(), headers: response.headers };
|
|
1068
|
+
}
|
|
1069
|
+
// 401/403 without rate-limit headers is an auth problem: retrying cannot
|
|
1070
|
+
// fix a token that is not allowed to read run history, so fail immediately
|
|
1071
|
+
// with a message that names the fix.
|
|
1072
|
+
const remaining = response.headers?.get?.("x-ratelimit-remaining");
|
|
1073
|
+
const throttled =
|
|
1074
|
+
response.status === 429 || (response.status === 403 && remaining === "0");
|
|
1075
|
+
if ((response.status === 401 || response.status === 403) && !throttled) {
|
|
1076
|
+
throw new GateApiError(
|
|
1077
|
+
`The Actions API returned ${response.status} for ${path}. The token cannot read run history — the caller job needs \`permissions: actions: read\`. Refusing to read an unreadable API as a green nightly.`
|
|
1078
|
+
);
|
|
1079
|
+
}
|
|
1080
|
+
lastProblem = `HTTP ${response.status}`;
|
|
1081
|
+
if (attempt < api.maxAttempts) {
|
|
1082
|
+
await wait(retryDelayMs(response, attempt, api.retryMaxSeconds));
|
|
1083
|
+
}
|
|
1084
|
+
}
|
|
1085
|
+
throw new GateApiError(
|
|
1086
|
+
`The Actions API stayed unreadable for ${path} after ${api.maxAttempts} attempts (${lastProblem}). Refusing to read an unreachable API as a green nightly — this check is RED, not inconclusive.`
|
|
1087
|
+
);
|
|
1088
|
+
}
|
|
1089
|
+
|
|
1090
|
+
/**
|
|
1091
|
+
* Newest completed run of one workflow on one branch for one event.
|
|
1092
|
+
*
|
|
1093
|
+
* @param {object} api - API coordinates
|
|
1094
|
+
* @param {string} file - Workflow file name
|
|
1095
|
+
* @param {string} branch - Branch to read
|
|
1096
|
+
* @param {string} event - Run event
|
|
1097
|
+
* @param {(ms: number) => Promise<void>} [wait] - Injectable sleep
|
|
1098
|
+
* @returns {Promise<{run: object|null, missing: boolean}>} The newest run, or a 404 marker
|
|
1099
|
+
*/
|
|
1100
|
+
export async function fetchNewestRun(api, file, branch, event, wait) {
|
|
1101
|
+
const query = new URLSearchParams({
|
|
1102
|
+
branch,
|
|
1103
|
+
status: "completed",
|
|
1104
|
+
event,
|
|
1105
|
+
per_page: "1",
|
|
1106
|
+
});
|
|
1107
|
+
const result = await apiGet(
|
|
1108
|
+
api,
|
|
1109
|
+
`/repos/${api.repo}/actions/workflows/${encodeURIComponent(file)}/runs?${query}`,
|
|
1110
|
+
wait
|
|
1111
|
+
);
|
|
1112
|
+
if (result === null) return { run: null, missing: true };
|
|
1113
|
+
return { run: result.body.workflow_runs?.[0] ?? null, missing: false };
|
|
1114
|
+
}
|
|
1115
|
+
|
|
1116
|
+
/**
|
|
1117
|
+
* Every job of a run, paginated to exhaustion.
|
|
1118
|
+
*
|
|
1119
|
+
* Exhaustive on purpose: a matrix suite routinely exceeds one page, and a
|
|
1120
|
+
* truncated job list turns "the failing shard is on page 2" into a false green.
|
|
1121
|
+
* Hitting the page cap while still unread is raised rather than silently
|
|
1122
|
+
* truncated.
|
|
1123
|
+
*
|
|
1124
|
+
* @param {object} api - API coordinates
|
|
1125
|
+
* @param {number|string} runId - The run
|
|
1126
|
+
* @param {(ms: number) => Promise<void>} [wait] - Injectable sleep
|
|
1127
|
+
* @returns {Promise<ReadonlyArray<object>>} All jobs
|
|
1128
|
+
*/
|
|
1129
|
+
export async function fetchAllJobs(api, runId, wait) {
|
|
1130
|
+
const jobs = [];
|
|
1131
|
+
for (let page = 1; page <= api.maxPages; page += 1) {
|
|
1132
|
+
const result = await apiGet(
|
|
1133
|
+
api,
|
|
1134
|
+
`/repos/${api.repo}/actions/runs/${runId}/jobs?per_page=100&page=${page}&filter=latest`,
|
|
1135
|
+
wait
|
|
1136
|
+
);
|
|
1137
|
+
// A 404 means the run's job list stopped being readable mid-walk. Return
|
|
1138
|
+
// what was read; falling through to the page-cap throw below would blame a
|
|
1139
|
+
// pagination limit that had nothing to do with it AND discard every job
|
|
1140
|
+
// already collected.
|
|
1141
|
+
if (result === null) return Object.freeze(jobs);
|
|
1142
|
+
const batch = result.body.jobs ?? [];
|
|
1143
|
+
jobs.push(...batch);
|
|
1144
|
+
if (batch.length < 100) return Object.freeze(jobs);
|
|
1145
|
+
}
|
|
1146
|
+
throw new GateApiError(
|
|
1147
|
+
`Run ${runId} reports more jobs than \`api_max_pages\` (${api.maxPages}) allows this gate to read. A truncated job list can hide the failing shard, so this is RED rather than a partial read.`
|
|
1148
|
+
);
|
|
1149
|
+
}
|
|
1150
|
+
|
|
1151
|
+
/**
|
|
1152
|
+
* Observes every suite.
|
|
1153
|
+
*
|
|
1154
|
+
* @param {object} api - API coordinates
|
|
1155
|
+
* @param {ReadonlyArray<object>} suites - Validated suites
|
|
1156
|
+
* @param {string} branch - Branch to read
|
|
1157
|
+
* @param {(ms: number) => Promise<void>} [wait] - Injectable sleep
|
|
1158
|
+
* @returns {Promise<ReadonlyArray<object>>} One observation per suite
|
|
1159
|
+
*/
|
|
1160
|
+
export async function observe(api, suites, branch, wait) {
|
|
1161
|
+
return await Promise.all(
|
|
1162
|
+
suites.map(async suite => {
|
|
1163
|
+
const perEvent = await Promise.all(
|
|
1164
|
+
COUNTED_EVENTS.map(event =>
|
|
1165
|
+
fetchNewestRun(api, suite.workflow, branch, event, wait)
|
|
1166
|
+
)
|
|
1167
|
+
);
|
|
1168
|
+
if (perEvent.every(result => result.missing)) {
|
|
1169
|
+
return { workflowMissing: true, run: null, jobs: [] };
|
|
1170
|
+
}
|
|
1171
|
+
// Newest by created_at, so a fresh dispatch supersedes an older failed
|
|
1172
|
+
// schedule. ISO-8601 UTC compares correctly as strings.
|
|
1173
|
+
const run = perEvent
|
|
1174
|
+
.map(result => result.run)
|
|
1175
|
+
.filter(
|
|
1176
|
+
candidate => candidate && typeof candidate.created_at === "string"
|
|
1177
|
+
)
|
|
1178
|
+
.reduce(
|
|
1179
|
+
(newest, candidate) =>
|
|
1180
|
+
newest === null || candidate.created_at > newest.created_at
|
|
1181
|
+
? candidate
|
|
1182
|
+
: newest,
|
|
1183
|
+
null
|
|
1184
|
+
);
|
|
1185
|
+
if (!run || suite.match.mode === "run") {
|
|
1186
|
+
return { workflowMissing: false, run, jobs: [] };
|
|
1187
|
+
}
|
|
1188
|
+
return {
|
|
1189
|
+
workflowMissing: false,
|
|
1190
|
+
run,
|
|
1191
|
+
jobs: await fetchAllJobs(api, run.id, wait),
|
|
1192
|
+
};
|
|
1193
|
+
})
|
|
1194
|
+
);
|
|
1195
|
+
}
|
|
1196
|
+
|
|
1197
|
+
/**
|
|
1198
|
+
* Reads who most recently applied the bypass label, from the PR's issue events.
|
|
1199
|
+
*
|
|
1200
|
+
* PAGINATED, and that is not defensive padding. The issue-events API returns
|
|
1201
|
+
* events OLDEST-FIRST, so on a long-lived pull request — a big feature branch,
|
|
1202
|
+
* or exactly the sort of PR that ends up needing a bypass — page 1 holds the
|
|
1203
|
+
* oldest hundred events and the label application is on a later page. Reading
|
|
1204
|
+
* only page 1 would report `no_attributable_actor` for a perfectly valid
|
|
1205
|
+
* maintainer bypass. That fails closed, so it is not a security hole, but it
|
|
1206
|
+
* rejects the legitimate case for the wrong stated reason, which is its own kind
|
|
1207
|
+
* of untrustworthy gate.
|
|
1208
|
+
*
|
|
1209
|
+
* Every page is scanned and the newest match across all of them wins, so a label
|
|
1210
|
+
* removed and re-applied is attributed to the person who applied it LAST.
|
|
1211
|
+
*
|
|
1212
|
+
* @param {object} api - API coordinates
|
|
1213
|
+
* @param {number} prNumber - Pull request number
|
|
1214
|
+
* @param {string} label - The bypass label
|
|
1215
|
+
* @param {(ms: number) => Promise<void>} [wait] - Injectable sleep
|
|
1216
|
+
* @returns {Promise<{actor: string, createdAt: string}|null>} The labelling event
|
|
1217
|
+
*/
|
|
1218
|
+
export async function fetchLabelEvent(api, prNumber, label, wait) {
|
|
1219
|
+
let newest = null;
|
|
1220
|
+
for (let page = 1; page <= api.maxPages; page += 1) {
|
|
1221
|
+
const result = await apiGet(
|
|
1222
|
+
api,
|
|
1223
|
+
`/repos/${api.repo}/issues/${prNumber}/events?per_page=100&page=${page}`,
|
|
1224
|
+
wait
|
|
1225
|
+
).catch(() => null);
|
|
1226
|
+
if (result === null) break;
|
|
1227
|
+
const batch = result.body ?? [];
|
|
1228
|
+
for (const event of batch) {
|
|
1229
|
+
if (event.event !== "labeled" || event.label?.name !== label) continue;
|
|
1230
|
+
if (newest === null || event.created_at > newest.created_at) {
|
|
1231
|
+
newest = event;
|
|
1232
|
+
}
|
|
1233
|
+
}
|
|
1234
|
+
if (batch.length < 100) break;
|
|
1235
|
+
}
|
|
1236
|
+
if (!newest?.actor?.login) return null;
|
|
1237
|
+
return { actor: newest.actor.login, createdAt: newest.created_at };
|
|
1238
|
+
}
|
|
1239
|
+
|
|
1240
|
+
/**
|
|
1241
|
+
* Reads an actor's permission on the repository.
|
|
1242
|
+
*
|
|
1243
|
+
* @param {object} api - API coordinates
|
|
1244
|
+
* @param {string} login - The actor
|
|
1245
|
+
* @param {(ms: number) => Promise<void>} [wait] - Injectable sleep
|
|
1246
|
+
* @returns {Promise<string|null>} `admin` | `maintain` | `write` | `read` | null
|
|
1247
|
+
*/
|
|
1248
|
+
export async function fetchActorPermission(api, login, wait) {
|
|
1249
|
+
const result = await apiGet(
|
|
1250
|
+
api,
|
|
1251
|
+
`/repos/${api.repo}/collaborators/${encodeURIComponent(login)}/permission`,
|
|
1252
|
+
wait
|
|
1253
|
+
).catch(() => null);
|
|
1254
|
+
return result?.body?.role_name ?? result?.body?.permission ?? null;
|
|
1255
|
+
}
|
|
1256
|
+
|
|
1257
|
+
// ---------------------------------------------------------------------------
|
|
1258
|
+
// 8. Entry point
|
|
1259
|
+
// ---------------------------------------------------------------------------
|
|
1260
|
+
|
|
1261
|
+
/**
|
|
1262
|
+
* Resolves everything the gate needs from the environment, failing loudly
|
|
1263
|
+
* rather than degrading into a check that silently measures nothing.
|
|
1264
|
+
*
|
|
1265
|
+
* @param {NodeJS.ProcessEnv} env - The environment
|
|
1266
|
+
* @returns {object} Resolved settings
|
|
1267
|
+
* @throws {GateConfigError} When required settings are absent or unusable
|
|
1268
|
+
*/
|
|
1269
|
+
export function resolveSettings(env) {
|
|
1270
|
+
const token = env.GITHUB_TOKEN || env.GH_TOKEN;
|
|
1271
|
+
if (!token) {
|
|
1272
|
+
throw new GateConfigError(
|
|
1273
|
+
"No GITHUB_TOKEN / GH_TOKEN in the environment. This gate reads the Actions API and cannot report a verdict without one."
|
|
1274
|
+
);
|
|
1275
|
+
}
|
|
1276
|
+
if (!env.GITHUB_REPOSITORY) {
|
|
1277
|
+
throw new GateConfigError(
|
|
1278
|
+
"No GITHUB_REPOSITORY in the environment, so there is no repository to read nightly runs from."
|
|
1279
|
+
);
|
|
1280
|
+
}
|
|
1281
|
+
const branch = (env.NIGHTLY_BRANCH || "").trim();
|
|
1282
|
+
if (!branch) {
|
|
1283
|
+
throw new GateConfigError(
|
|
1284
|
+
"No `branch` input. The gate must know which branch's nightly verdict it speaks for — an unfiltered read would let a dispatch from any feature branch clear the gate for everybody."
|
|
1285
|
+
);
|
|
1286
|
+
}
|
|
1287
|
+
const number = (name, fallback) => {
|
|
1288
|
+
const raw = env[name];
|
|
1289
|
+
if (raw === undefined || String(raw).trim() === "") return fallback;
|
|
1290
|
+
const value = Number(raw);
|
|
1291
|
+
if (!Number.isFinite(value) || value <= 0) {
|
|
1292
|
+
throw new GateConfigError(
|
|
1293
|
+
`\`${name}\` must be a positive number (got ${raw}).`
|
|
1294
|
+
);
|
|
1295
|
+
}
|
|
1296
|
+
return value;
|
|
1297
|
+
};
|
|
1298
|
+
|
|
1299
|
+
// Every security ceiling is applied HERE, through the one shared resolver, so
|
|
1300
|
+
// fail-closed only has to be right once. Resolved at call time from the env
|
|
1301
|
+
// passed in — never captured at module load, so a test or a re-entrant caller
|
|
1302
|
+
// cannot be reading limits some earlier import froze.
|
|
1303
|
+
const { limits, clamped } = resolveSecurityLimits({
|
|
1304
|
+
bypassMaxHours: number("NIGHTLY_BYPASS_MAX_HOURS", 24),
|
|
1305
|
+
bootstrapMaxDays: number("NIGHTLY_BOOTSTRAP_MAX_DAYS", 30),
|
|
1306
|
+
freshnessHours: number("NIGHTLY_FRESHNESS_HOURS", 36),
|
|
1307
|
+
apiMaxAttempts: number("NIGHTLY_API_MAX_ATTEMPTS", 3),
|
|
1308
|
+
apiMaxPages: number("NIGHTLY_API_MAX_PAGES", 5),
|
|
1309
|
+
apiRetryMaxSeconds: number("NIGHTLY_API_RETRY_MAX_SECONDS", 60),
|
|
1310
|
+
});
|
|
1311
|
+
|
|
1312
|
+
return {
|
|
1313
|
+
api: {
|
|
1314
|
+
apiUrl: env.GITHUB_API_URL || "https://api.github.com",
|
|
1315
|
+
repo: env.GITHUB_REPOSITORY,
|
|
1316
|
+
token,
|
|
1317
|
+
maxAttempts: limits.apiMaxAttempts,
|
|
1318
|
+
maxPages: limits.apiMaxPages,
|
|
1319
|
+
retryMaxSeconds: limits.apiRetryMaxSeconds,
|
|
1320
|
+
},
|
|
1321
|
+
branch,
|
|
1322
|
+
suites: validateSuites(env.NIGHTLY_SUITES),
|
|
1323
|
+
freshnessHours: limits.freshnessHours,
|
|
1324
|
+
bootstrapUntil: env.NIGHTLY_BOOTSTRAP_UNTIL || "",
|
|
1325
|
+
bootstrapMaxDays: limits.bootstrapMaxDays,
|
|
1326
|
+
bypassLabel: env.NIGHTLY_BYPASS_LABEL || "nightly-e2e-bypass",
|
|
1327
|
+
bypassMaxHours: limits.bypassMaxHours,
|
|
1328
|
+
// An ADDITIONAL project rule, never a replacement — see `evaluateBypass`.
|
|
1329
|
+
extraBypassReasonPattern: env.NIGHTLY_BYPASS_REASON_PATTERN || "",
|
|
1330
|
+
clamped,
|
|
1331
|
+
pr: {
|
|
1332
|
+
number: Number(env.NIGHTLY_PR_NUMBER) || null,
|
|
1333
|
+
author: env.NIGHTLY_PR_AUTHOR || null,
|
|
1334
|
+
body: env.NIGHTLY_PR_BODY || "",
|
|
1335
|
+
labels: (() => {
|
|
1336
|
+
try {
|
|
1337
|
+
const parsed = JSON.parse(env.NIGHTLY_PR_LABELS || "[]");
|
|
1338
|
+
return Array.isArray(parsed) ? parsed.map(String) : [];
|
|
1339
|
+
} catch {
|
|
1340
|
+
return [];
|
|
1341
|
+
}
|
|
1342
|
+
})(),
|
|
1343
|
+
},
|
|
1344
|
+
};
|
|
1345
|
+
}
|
|
1346
|
+
|
|
1347
|
+
/**
|
|
1348
|
+
* Runs the gate.
|
|
1349
|
+
*
|
|
1350
|
+
* @param {NodeJS.ProcessEnv} env - The environment
|
|
1351
|
+
* @param {(ms: number) => Promise<void>} [wait] - Injectable sleep
|
|
1352
|
+
* @returns {Promise<object>} The verdict
|
|
1353
|
+
*/
|
|
1354
|
+
export async function runGate(env, wait) {
|
|
1355
|
+
const settings = resolveSettings(env);
|
|
1356
|
+
const now = new Date();
|
|
1357
|
+
const bootstrap = resolveBootstrap(
|
|
1358
|
+
settings.bootstrapUntil,
|
|
1359
|
+
settings.bootstrapMaxDays,
|
|
1360
|
+
now
|
|
1361
|
+
);
|
|
1362
|
+
const observations = await observe(
|
|
1363
|
+
settings.api,
|
|
1364
|
+
settings.suites,
|
|
1365
|
+
settings.branch,
|
|
1366
|
+
wait
|
|
1367
|
+
);
|
|
1368
|
+
const findings = settings.suites.map((suite, index) =>
|
|
1369
|
+
assessSuite(suite, observations[index], {
|
|
1370
|
+
branch: settings.branch,
|
|
1371
|
+
freshnessHours: settings.freshnessHours,
|
|
1372
|
+
now,
|
|
1373
|
+
})
|
|
1374
|
+
);
|
|
1375
|
+
|
|
1376
|
+
let bypass = null;
|
|
1377
|
+
if (settings.pr.number && settings.pr.labels.includes(settings.bypassLabel)) {
|
|
1378
|
+
const labelEvent = await fetchLabelEvent(
|
|
1379
|
+
settings.api,
|
|
1380
|
+
settings.pr.number,
|
|
1381
|
+
settings.bypassLabel,
|
|
1382
|
+
wait
|
|
1383
|
+
);
|
|
1384
|
+
const actorPermission = labelEvent
|
|
1385
|
+
? await fetchActorPermission(settings.api, labelEvent.actor, wait)
|
|
1386
|
+
: null;
|
|
1387
|
+
bypass = evaluateBypass({
|
|
1388
|
+
labelEvent,
|
|
1389
|
+
prAuthor: settings.pr.author,
|
|
1390
|
+
prBody: settings.pr.body,
|
|
1391
|
+
actorPermission,
|
|
1392
|
+
prNumber: settings.pr.number,
|
|
1393
|
+
label: settings.bypassLabel,
|
|
1394
|
+
maxHours: settings.bypassMaxHours,
|
|
1395
|
+
extraReasonPattern: settings.extraBypassReasonPattern,
|
|
1396
|
+
now,
|
|
1397
|
+
});
|
|
1398
|
+
}
|
|
1399
|
+
|
|
1400
|
+
return {
|
|
1401
|
+
...decide(findings, { bootstrap, bypass }),
|
|
1402
|
+
clamped: settings.clamped,
|
|
1403
|
+
settings,
|
|
1404
|
+
};
|
|
1405
|
+
}
|
|
1406
|
+
|
|
1407
|
+
/**
|
|
1408
|
+
* CLI.
|
|
1409
|
+
*
|
|
1410
|
+
* @param {ReadonlyArray<string>} argv - Arguments
|
|
1411
|
+
* @returns {Promise<void>} Resolves once the report is written
|
|
1412
|
+
*/
|
|
1413
|
+
async function main(argv) {
|
|
1414
|
+
if (argv.includes("--contract-version")) {
|
|
1415
|
+
process.stdout.write(`${NIGHTLY_E2E_CONTRACT_VERSION}\n`);
|
|
1416
|
+
return;
|
|
1417
|
+
}
|
|
1418
|
+
const asJson = argv.includes("--json");
|
|
1419
|
+
|
|
1420
|
+
/** @type {object} */
|
|
1421
|
+
let verdict;
|
|
1422
|
+
try {
|
|
1423
|
+
verdict = await runGate(process.env);
|
|
1424
|
+
} catch (error) {
|
|
1425
|
+
// Configuration and API failures are the gate's own failure modes, and both
|
|
1426
|
+
// are RED. Never a pass, never a warning.
|
|
1427
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1428
|
+
const kind = error instanceof GateConfigError ? "configuration" : "api";
|
|
1429
|
+
const failure = {
|
|
1430
|
+
verdict: "fail",
|
|
1431
|
+
blocked: true,
|
|
1432
|
+
error: { kind, message },
|
|
1433
|
+
findings: [],
|
|
1434
|
+
};
|
|
1435
|
+
// A downstream job reading `verdict`/`blocked` must see the gate's OWN
|
|
1436
|
+
// failures too. Leaving them unset makes a configuration or API failure
|
|
1437
|
+
// indistinguishable from a job that never ran — the same "absence reads as
|
|
1438
|
+
// fine" shape the whole gate refuses.
|
|
1439
|
+
await writeOutputs(failure);
|
|
1440
|
+
if (asJson) {
|
|
1441
|
+
process.stdout.write(`${JSON.stringify(failure, null, 2)}\n`);
|
|
1442
|
+
return;
|
|
1443
|
+
}
|
|
1444
|
+
const report = `## 🌙 Nightly E2E Health\n\n❌ **The gate could not produce a verdict (${kind}).**\n\n${message}\n\nThis check is RED, not inconclusive: "we could not check" must never render as "it is fine".\n`;
|
|
1445
|
+
process.stdout.write(report);
|
|
1446
|
+
await appendSummary(report);
|
|
1447
|
+
process.stderr.write(
|
|
1448
|
+
`::error title=Nightly E2E gate ${kind} failure::${message.split("\n")[0]}\n`
|
|
1449
|
+
);
|
|
1450
|
+
process.exitCode = 1;
|
|
1451
|
+
return;
|
|
1452
|
+
}
|
|
1453
|
+
|
|
1454
|
+
const { settings, ...machine } = verdict;
|
|
1455
|
+
if (asJson) {
|
|
1456
|
+
process.stdout.write(`${JSON.stringify(machine, null, 2)}\n`);
|
|
1457
|
+
return;
|
|
1458
|
+
}
|
|
1459
|
+
|
|
1460
|
+
const report = formatReport(machine, {
|
|
1461
|
+
branch: settings.branch,
|
|
1462
|
+
bypassLabel: settings.bypassLabel,
|
|
1463
|
+
});
|
|
1464
|
+
process.stdout.write(report);
|
|
1465
|
+
await appendSummary(report);
|
|
1466
|
+
await writeOutputs(machine);
|
|
1467
|
+
|
|
1468
|
+
if (machine.verdict === "bypassed") {
|
|
1469
|
+
process.stderr.write(
|
|
1470
|
+
`::notice title=Nightly E2E gate BYPASSED (audited)::${machine.bypass.actor} waived ${machine.bypass.waived.length} red suite(s) under ticket ${machine.bypass.ticket}; the bypass expires ${machine.bypass.expiresAt}\n`
|
|
1471
|
+
);
|
|
1472
|
+
}
|
|
1473
|
+
if (machine.blocked) {
|
|
1474
|
+
for (const finding of machine.findings.filter(f => f.state !== "pass")) {
|
|
1475
|
+
process.stderr.write(
|
|
1476
|
+
`::error title=Nightly E2E is not green::${finding.label} — ${finding.state}${finding.conclusion ? ` (${finding.conclusion})` : ""} [${finding.reason}]${finding.url ? ` — ${finding.url}` : ""}\n`
|
|
1477
|
+
);
|
|
1478
|
+
}
|
|
1479
|
+
process.exitCode = 1;
|
|
1480
|
+
}
|
|
1481
|
+
}
|
|
1482
|
+
|
|
1483
|
+
/**
|
|
1484
|
+
* Appends a report to the job summary when one exists.
|
|
1485
|
+
*
|
|
1486
|
+
* @param {string} report - Markdown
|
|
1487
|
+
* @returns {Promise<void>} Resolves when written
|
|
1488
|
+
*/
|
|
1489
|
+
async function appendSummary(report) {
|
|
1490
|
+
if (!process.env.GITHUB_STEP_SUMMARY) return;
|
|
1491
|
+
const { appendFileSync } = await import("node:fs");
|
|
1492
|
+
appendFileSync(process.env.GITHUB_STEP_SUMMARY, report);
|
|
1493
|
+
}
|
|
1494
|
+
|
|
1495
|
+
/**
|
|
1496
|
+
* Publishes the machine verdict as step outputs.
|
|
1497
|
+
*
|
|
1498
|
+
* @param {object} machine - The verdict
|
|
1499
|
+
* @returns {Promise<void>} Resolves when written
|
|
1500
|
+
*/
|
|
1501
|
+
async function writeOutputs(machine) {
|
|
1502
|
+
if (!process.env.GITHUB_OUTPUT) return;
|
|
1503
|
+
const { appendFileSync } = await import("node:fs");
|
|
1504
|
+
const audit = JSON.stringify(machine);
|
|
1505
|
+
appendFileSync(
|
|
1506
|
+
process.env.GITHUB_OUTPUT,
|
|
1507
|
+
`verdict=${machine.verdict}\nblocked=${machine.blocked}\naudit_json<<LISA_AUDIT_EOF\n${audit}\nLISA_AUDIT_EOF\n`
|
|
1508
|
+
);
|
|
1509
|
+
}
|
|
1510
|
+
|
|
1511
|
+
if (
|
|
1512
|
+
process.argv[1] &&
|
|
1513
|
+
import.meta.url === pathToFileURL(process.argv[1]).href
|
|
1514
|
+
) {
|
|
1515
|
+
await main(process.argv.slice(2));
|
|
1516
|
+
}
|