@intentius/chant 0.124.69 → 0.124.70
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/main.d.ts.map +1 -1
- package/dist/cli/main.js +6 -0
- package/dist/cli/parse-args.d.ts.map +1 -1
- package/dist/cli/parse-args.js +15 -0
- package/dist/cli/registry.d.ts +4 -0
- package/dist/cli/registry.d.ts.map +1 -1
- package/dist/release-floors.d.ts +1 -1
- package/dist/release-floors.js +1 -1
- package/dist/wave-runtime.mjs +290 -290
- package/dist/workspace/ci-green-cli.d.ts +7 -3
- package/dist/workspace/ci-green-cli.d.ts.map +1 -1
- package/dist/workspace/ci-green-cli.js +29 -8
- package/dist/workspace/ci-green-forge.d.ts +29 -0
- package/dist/workspace/ci-green-forge.d.ts.map +1 -1
- package/dist/workspace/ci-green-forge.js +50 -0
- package/dist/workspace/ci-green-workflow.d.ts +9 -1
- package/dist/workspace/ci-green-workflow.d.ts.map +1 -1
- package/dist/workspace/ci-green-workflow.js +15 -3
- package/dist/workspace/ci-green.d.ts +35 -1
- package/dist/workspace/ci-green.d.ts.map +1 -1
- package/dist/workspace/ci-green.js +123 -4
- package/dist/workspace/declaration.d.ts +14 -2
- package/dist/workspace/declaration.d.ts.map +1 -1
- package/dist/workspace/declaration.js +14 -1
- package/dist/workspace/declaration.schema.json +17 -2
- package/package.json +1 -1
- package/src/cli/main.ts +6 -0
- package/src/cli/parse-args.ts +11 -0
- package/src/cli/registry.ts +4 -0
- package/src/release-floors.ts +1 -1
- package/src/workspace/ci-green-cli.ts +27 -8
- package/src/workspace/ci-green-forge.ts +70 -0
- package/src/workspace/ci-green-workflow.ts +15 -3
- package/src/workspace/ci-green.ts +131 -5
- package/src/workspace/ci-verdict-retries.test.ts +376 -0
- package/src/workspace/declaration.schema.json +17 -2
- package/src/workspace/declaration.ts +28 -4
|
@@ -58,7 +58,8 @@ import { execFileSync } from "node:child_process";
|
|
|
58
58
|
import { greenBranches, laneOf, laneTagPrefixes, readDeclaration, readerVersion, WORKSPACE_ERROR_CODES, WorkspaceReadError, type CiGreen, type CiPhase, type CiSource } from "./declaration";
|
|
59
59
|
import { makeNotifyEvent, sendNotify, type NotifyFields } from "../notify";
|
|
60
60
|
import { lifecycleRemoteOverride } from "../lifecycle/ledger-remote";
|
|
61
|
-
import type
|
|
61
|
+
import { isTransientForgeError, type CheckRun, type CiForge, type OpRunRef, type WorkflowRun } from "./ci-green-forge";
|
|
62
|
+
import { NEXT_RELEASE } from "../release-floors";
|
|
62
63
|
import type { OpRunRecord } from "../op/runtime";
|
|
63
64
|
import { readRunLedger } from "../lifecycle/run-ledger";
|
|
64
65
|
import { fetchLifecycleStatus, listFilesInDir, listLedgerTrees, pushLifecycleStatus, writeLedgerFiles } from "../lifecycle/git";
|
|
@@ -90,6 +91,8 @@ export const CI_STALE_NOTIFY_FLOOR = "0.124.63";
|
|
|
90
91
|
export const CI_STALE_WALK = 200;
|
|
91
92
|
/** The first chant that writes a merge queue's trigger and verdict job, and reads `chant ci verdict` (#4644). */
|
|
92
93
|
export const CI_MERGE_QUEUE_FLOOR = "0.124.61";
|
|
94
|
+
/** The first chant that reads `ci.green.mergeQueue: { retries }` and whose `ci verdict` re-runs a failed job and looks again after a forge 5xx (#4729). */
|
|
95
|
+
export const CI_VERDICT_RETRIES_FLOOR = NEXT_RELEASE;
|
|
93
96
|
|
|
94
97
|
export const CI_LAST_GREEN_CONTRACT_VERSION = 1;
|
|
95
98
|
export const CI_LAST_GREEN_SCHEMA_ID = "https://intentius.io/chant/schemas/workspace/ci-last-green/v1/ci-last-green.schema.json";
|
|
@@ -1550,26 +1553,149 @@ export function lastGreen(req: { cwd: string; remote?: string; from?: string; br
|
|
|
1550
1553
|
|
|
1551
1554
|
/** How long `ci verdict --wait` waits between reads of the check runs. */
|
|
1552
1555
|
export const CI_VERDICT_INTERVAL_MS = 30_000;
|
|
1556
|
+
/** How many forge errors in a row (5xx, 429, no answer) `ci verdict --wait` looks past before it fails (#4729). */
|
|
1557
|
+
export const CI_VERDICT_FORGE_ERRORS = 8;
|
|
1558
|
+
/** The wait after the first forge error in a row (#4729); it doubles with each one after, up to {@link CI_VERDICT_BACKOFF_MAX_MS}. */
|
|
1559
|
+
export const CI_VERDICT_BACKOFF_MS = 15_000;
|
|
1560
|
+
export const CI_VERDICT_BACKOFF_MAX_MS = 120_000;
|
|
1561
|
+
|
|
1562
|
+
/** `ci verdict --wait` stopped without a verdict (#4729): out of time, or the forge kept failing. */
|
|
1563
|
+
export class CiVerdictGaveUpError extends Error {
|
|
1564
|
+
constructor(message: string) {
|
|
1565
|
+
super(message);
|
|
1566
|
+
this.name = "CiVerdictGaveUpError";
|
|
1567
|
+
}
|
|
1568
|
+
}
|
|
1569
|
+
|
|
1570
|
+
/** A failed job whose workflow run failed again on its last allowed attempt (#4729). */
|
|
1571
|
+
export interface VerdictRepeat {
|
|
1572
|
+
phase: string;
|
|
1573
|
+
run: RunVerdict;
|
|
1574
|
+
workflowRun: WorkflowRun;
|
|
1575
|
+
}
|
|
1553
1576
|
|
|
1554
1577
|
/**
|
|
1555
1578
|
* `chant ci verdict` (#4644): judge one commit by ci.green's required phases
|
|
1556
1579
|
* from its check runs, as the tick judges a commit, leaving out the
|
|
1557
1580
|
* verdict's own check run (`chant-verdict`). With `wait`, read again until no
|
|
1558
1581
|
* phase is pending. A merge queue's verdict job runs it on the group's commit.
|
|
1582
|
+
*
|
|
1583
|
+
* With `wait`, two more things (#4729). A forge error that says to look again
|
|
1584
|
+
* (5xx, 429, no answer) is looked past, with a backoff, up to
|
|
1585
|
+
* {@link CI_VERDICT_FORGE_ERRORS} in a row. And with `retries` above 0 (from
|
|
1586
|
+
* the call, else `ci.green.mergeQueue.retries`), a failed job doesn't fail
|
|
1587
|
+
* the commit at once: once the job's workflow run has finished (a run can be
|
|
1588
|
+
* re-run only then), its failed jobs are re-run while its attempt is no more
|
|
1589
|
+
* than `retries`, and the commit fails when the run fails on attempt
|
|
1590
|
+
* `retries + 1`, returned as `repeated`. With retries 0 the first failed job
|
|
1591
|
+
* fails it, as before. `timeoutMs` bounds the wait: past it, the verdict
|
|
1592
|
+
* throws {@link CiVerdictGaveUpError} saying what it was waiting for.
|
|
1559
1593
|
*/
|
|
1560
|
-
export async function ciVerdict(req: {
|
|
1594
|
+
export async function ciVerdict(req: {
|
|
1595
|
+
cwd: string;
|
|
1596
|
+
sha: string;
|
|
1597
|
+
forge: CiForge;
|
|
1598
|
+
wait?: boolean;
|
|
1599
|
+
branch?: string;
|
|
1600
|
+
retries?: number;
|
|
1601
|
+
timeoutMs?: number;
|
|
1602
|
+
sleep?: (ms: number) => Promise<void>;
|
|
1603
|
+
now?: () => number;
|
|
1604
|
+
onPending?: (v: CommitVerdict) => void;
|
|
1605
|
+
/** A line about a re-run, a run it waits on, or a forge error it looks past. */
|
|
1606
|
+
onNote?: (line: string) => void;
|
|
1607
|
+
}): Promise<{ sha: string; verdict: CommitVerdict; repeated?: VerdictRepeat }> {
|
|
1561
1608
|
const read = readGreen(req.cwd);
|
|
1562
1609
|
const green = greenOn(read.green, req.branch);
|
|
1563
1610
|
const sleep = req.sleep ?? ((ms: number) => new Promise<void>((r) => setTimeout(r, ms)));
|
|
1611
|
+
const now = req.now ?? Date.now;
|
|
1612
|
+
const retries = req.wait ? (req.retries ?? green.mergeQueue?.retries ?? 0) : 0;
|
|
1613
|
+
const deadline = req.wait && req.timeoutMs !== undefined ? now() + req.timeoutMs : Infinity;
|
|
1614
|
+
const note = req.onNote ?? (() => {});
|
|
1615
|
+
let errors = 0;
|
|
1616
|
+
let waitingOn = "the first read of the check runs";
|
|
1617
|
+
// Each workflow run re-run, at the attempt it was on when asked: until GitHub starts the new attempt, it isn't asked again.
|
|
1618
|
+
const asked = new Map<number, number>();
|
|
1619
|
+
const told = new Set<string>();
|
|
1620
|
+
const once = (line: string) => {
|
|
1621
|
+
if (told.has(line)) return;
|
|
1622
|
+
told.add(line);
|
|
1623
|
+
note(line);
|
|
1624
|
+
};
|
|
1625
|
+
const pause = async (ms: number) => {
|
|
1626
|
+
const left = deadline - now();
|
|
1627
|
+
if (left <= 0) throw new CiVerdictGaveUpError(`no verdict after ${minutes(req.timeoutMs!)}: ${waitingOn}`);
|
|
1628
|
+
await sleep(Math.min(ms, left));
|
|
1629
|
+
};
|
|
1564
1630
|
for (;;) {
|
|
1631
|
+
let step: { done: { verdict: CommitVerdict; repeated?: VerdictRepeat } } | { verdict: CommitVerdict; waiting: boolean };
|
|
1632
|
+
try {
|
|
1633
|
+
step = await look();
|
|
1634
|
+
} catch (err) {
|
|
1635
|
+
if (!req.wait || !isTransientForgeError(err)) throw err;
|
|
1636
|
+
errors++;
|
|
1637
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
1638
|
+
if (errors >= CI_VERDICT_FORGE_ERRORS) throw new CiVerdictGaveUpError(`the forge failed ${errors} times in a row, the last: ${message}`);
|
|
1639
|
+
const backoff = Math.min(CI_VERDICT_BACKOFF_MS * 2 ** (errors - 1), CI_VERDICT_BACKOFF_MAX_MS);
|
|
1640
|
+
note(`the forge failed (${message}); looking again in ${Math.round(backoff / 1000)}s (${errors} of ${CI_VERDICT_FORGE_ERRORS - 1} looked past)`);
|
|
1641
|
+
waitingOn = `the forge failing: ${message}`;
|
|
1642
|
+
await pause(backoff);
|
|
1643
|
+
continue;
|
|
1644
|
+
}
|
|
1645
|
+
errors = 0;
|
|
1646
|
+
if ("done" in step) return { sha: req.sha, ...step.done };
|
|
1647
|
+
if (!step.waiting) req.onPending?.(step.verdict);
|
|
1648
|
+
await pause(CI_VERDICT_INTERVAL_MS);
|
|
1649
|
+
}
|
|
1650
|
+
|
|
1651
|
+
async function look(): Promise<{ done: { verdict: CommitVerdict; repeated?: VerdictRepeat } } | { verdict: CommitVerdict; waiting: boolean }> {
|
|
1565
1652
|
const runs = (await req.forge.checkRuns(req.sha)).filter((r) => r.name !== MERGE_QUEUE_VERDICT_JOB);
|
|
1566
1653
|
const verdict = evaluateCommit(green, runs);
|
|
1567
|
-
if (!req.wait
|
|
1568
|
-
|
|
1569
|
-
|
|
1654
|
+
if (!req.wait) return { done: { verdict } };
|
|
1655
|
+
if (verdict.verdict === "pending") {
|
|
1656
|
+
waitingOn = `still waiting on ${verdict.phases.filter((p) => p.verdict === "pending").map((p) => p.phase).join(", ")}`;
|
|
1657
|
+
return { verdict, waiting: false };
|
|
1658
|
+
}
|
|
1659
|
+
if (verdict.verdict !== "fail" || retries === 0 || !req.forge.workflowRun || !req.forge.rerunFailedJobs) return { done: { verdict } };
|
|
1660
|
+
// #4729: every failed run of a required phase, each job once.
|
|
1661
|
+
const failed: { phase: string; run: RunVerdict }[] = [];
|
|
1662
|
+
for (const p of verdict.phases) for (const r of p.runs) if (r.verdict === "fail" && !failed.some((f) => f.run.id === r.id)) failed.push({ phase: p.phase, run: r });
|
|
1663
|
+
const waits: string[] = [];
|
|
1664
|
+
for (const f of failed) {
|
|
1665
|
+
const wr = await req.forge.workflowRun(f.run.id);
|
|
1666
|
+
// Another app's check run: nothing chant can re-run.
|
|
1667
|
+
if (wr === null) return { done: { verdict } };
|
|
1668
|
+
const what = `${f.run.name} (phase ${f.phase}) in workflow run ${wr.name}${wr.url ? ` ${wr.url}` : ""}`;
|
|
1669
|
+
if (wr.status !== "completed") {
|
|
1670
|
+
once(`${f.run.name} failed (phase ${f.phase}); waiting for workflow run ${wr.name} to finish before re-running it`);
|
|
1671
|
+
waits.push(`workflow run ${wr.name} to finish`);
|
|
1672
|
+
continue;
|
|
1673
|
+
}
|
|
1674
|
+
if (wr.attempt > retries) return { done: { verdict, repeated: { phase: f.phase, run: f.run, workflowRun: wr } } };
|
|
1675
|
+
if (asked.get(wr.id) === wr.attempt) {
|
|
1676
|
+
waits.push(`the re-run of workflow run ${wr.name} to start`);
|
|
1677
|
+
continue;
|
|
1678
|
+
}
|
|
1679
|
+
note(`${what} failed on attempt ${wr.attempt} of ${retries + 1}; re-running its failed jobs`);
|
|
1680
|
+
try {
|
|
1681
|
+
await req.forge.rerunFailedJobs(wr.id);
|
|
1682
|
+
} catch (err) {
|
|
1683
|
+
if (isTransientForgeError(err)) throw err;
|
|
1684
|
+
throw new Error(`could not re-run the failed jobs of workflow run ${wr.name} (${wr.id}): ${err instanceof Error ? err.message : String(err)}. The verdict's job needs actions: write, which chant ci workflow gives it when ci.green.mergeQueue.retries is above 0`);
|
|
1685
|
+
}
|
|
1686
|
+
asked.set(wr.id, wr.attempt);
|
|
1687
|
+
waits.push(`the re-run of workflow run ${wr.name}`);
|
|
1688
|
+
}
|
|
1689
|
+
waitingOn = `waiting for ${[...new Set(waits)].join(", ")}`;
|
|
1690
|
+
return { verdict, waiting: true };
|
|
1570
1691
|
}
|
|
1571
1692
|
}
|
|
1572
1693
|
|
|
1694
|
+
function minutes(ms: number): string {
|
|
1695
|
+
const m = ms / 60_000;
|
|
1696
|
+
return `${Number.isInteger(m) ? m : m.toFixed(1)} minute${m === 1 ? "" : "s"}`;
|
|
1697
|
+
}
|
|
1698
|
+
|
|
1573
1699
|
// ── base ─────────────────────────────────────────────────────────────────────
|
|
1574
1700
|
|
|
1575
1701
|
/**
|
|
@@ -0,0 +1,376 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* #4729 — `chant ci verdict --wait` re-runs a failed job's workflow run
|
|
3
|
+
* before it fails a merge group, looks past a forge 5xx, and gives up before
|
|
4
|
+
* the job's own timeout. With retries 0 it fails at the first failed job, as
|
|
5
|
+
* #4644 did.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import { afterAll, afterEach, describe, expect, test, vi } from "vitest";
|
|
9
|
+
import { ForgeApiError, type ForgeFetch } from "../pr-forge";
|
|
10
|
+
import { parseArgs } from "../cli/parse-args";
|
|
11
|
+
import { parseYAMLDocument } from "../yaml";
|
|
12
|
+
import { cleanScratch, declaration, git, scratchDir, writeFiles } from "./__fixtures__/contract-repo";
|
|
13
|
+
import { CI_VERDICT_BACKOFF_MS, CI_VERDICT_FORGE_ERRORS, CI_VERDICT_INTERVAL_MS, ciVerdict, CiVerdictGaveUpError } from "./ci-green";
|
|
14
|
+
import { runCiVerdict } from "./ci-green-cli";
|
|
15
|
+
import { githubCiForge, isTransientForgeError, type CheckRun, type CiForge, type WorkflowRun } from "./ci-green-forge";
|
|
16
|
+
import { MERGE_QUEUE_VERDICT_JOB, MERGE_QUEUE_VERDICT_TIMEOUT_MINUTES, renderMergeQueueWorkflow } from "./ci-green-workflow";
|
|
17
|
+
import { parseDeclaration } from "./declaration";
|
|
18
|
+
|
|
19
|
+
afterAll(cleanScratch);
|
|
20
|
+
afterEach(() => {
|
|
21
|
+
vi.restoreAllMocks();
|
|
22
|
+
vi.unstubAllEnvs();
|
|
23
|
+
vi.unstubAllGlobals();
|
|
24
|
+
vi.useRealTimers();
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
const SHA = "a".repeat(40);
|
|
28
|
+
const run = (id: number, name: string, conclusion: string | null, status = "completed"): CheckRun => ({ id, name, status, conclusion: status === "completed" ? conclusion : null, completedAt: null, url: null });
|
|
29
|
+
const wfRun = (attempt: number, status = "completed", conclusion: string | null = status === "completed" ? "failure" : null): WorkflowRun => ({ id: 70, name: "check", attempt, status, conclusion, url: "https://github.com/acme/app/actions/runs/70" });
|
|
30
|
+
const http = (status: number) => new ForgeApiError("GET", `https://api.github.com/repos/acme/app/commits/${SHA}/check-runs`, status, "");
|
|
31
|
+
|
|
32
|
+
function workspace(mergeQueue: unknown): string {
|
|
33
|
+
const root = scratchDir("chant-verdict-retries-");
|
|
34
|
+
git(root, "init", "-q");
|
|
35
|
+
writeFiles(root, { "chant.workspace.json": declaration([], { ci: { green: { branch: "main", phases: { lint: ["lint"], test: ["test"] }, require: ["lint", "test"], mergeQueue } } }) });
|
|
36
|
+
return root;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/** A forge that answers each read from a script (the last answer repeats), and a clock that only `sleep` moves. */
|
|
40
|
+
function scripted(checks: (CheckRun[] | Error)[], runs: (WorkflowRun | null | Error)[] = []) {
|
|
41
|
+
let t = 0;
|
|
42
|
+
const log = { reads: 0, runReads: 0, reruns: [] as number[], sleeps: [] as number[], notes: [] as string[] };
|
|
43
|
+
const next = <T>(list: T[], i: number): T => list[Math.min(i, list.length - 1)]!;
|
|
44
|
+
const forge: CiForge = {
|
|
45
|
+
kind: "github",
|
|
46
|
+
async checkRuns() {
|
|
47
|
+
const a = next(checks, log.reads++);
|
|
48
|
+
if (a instanceof Error) throw a;
|
|
49
|
+
return a;
|
|
50
|
+
},
|
|
51
|
+
async workflowRun() {
|
|
52
|
+
const a = next(runs, log.runReads++);
|
|
53
|
+
if (a instanceof Error) throw a;
|
|
54
|
+
return a;
|
|
55
|
+
},
|
|
56
|
+
async rerunFailedJobs(id) {
|
|
57
|
+
log.reruns.push(id);
|
|
58
|
+
},
|
|
59
|
+
};
|
|
60
|
+
const clock = {
|
|
61
|
+
now: () => t,
|
|
62
|
+
sleep: async (ms: number) => {
|
|
63
|
+
log.sleeps.push(ms);
|
|
64
|
+
t += ms;
|
|
65
|
+
},
|
|
66
|
+
onNote: (line: string) => void log.notes.push(line),
|
|
67
|
+
};
|
|
68
|
+
return { forge, clock, log };
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
const LINT = run(1, "lint", "success");
|
|
72
|
+
|
|
73
|
+
describe("ci verdict --wait with retries (#4729)", () => {
|
|
74
|
+
test("a flake that passes on its re-run passes, and its run is re-run once", async () => {
|
|
75
|
+
const { forge, clock, log } = scripted(
|
|
76
|
+
[[LINT, run(2, "test", "failure")], [LINT, run(2, "test", "failure")], [LINT, run(2, "test", "failure"), run(3, "test", null, "in_progress")], [LINT, run(2, "test", "failure"), run(3, "test", "success")]],
|
|
77
|
+
[wfRun(1), wfRun(2, "queued")],
|
|
78
|
+
);
|
|
79
|
+
const out = await ciVerdict({ cwd: workspace({ retries: 1 }), sha: SHA, forge, wait: true, ...clock });
|
|
80
|
+
expect(out.verdict.verdict).toBe("pass");
|
|
81
|
+
expect(out.repeated).toBeUndefined();
|
|
82
|
+
expect(log.reruns).toEqual([70]);
|
|
83
|
+
expect(log.notes[0]).toBe("test (phase test) in workflow run check https://github.com/acme/app/actions/runs/70 failed on attempt 1 of 2; re-running its failed jobs");
|
|
84
|
+
});
|
|
85
|
+
|
|
86
|
+
test("a failure that repeats fails, naming the phase, the job and the workflow run", async () => {
|
|
87
|
+
const { forge, clock, log } = scripted([[LINT, run(2, "test", "failure")], [LINT, run(2, "test", "failure"), run(3, "test", "failure")]], [wfRun(1), wfRun(2)]);
|
|
88
|
+
const out = await ciVerdict({ cwd: workspace({ retries: 1 }), sha: SHA, forge, wait: true, ...clock });
|
|
89
|
+
expect(out.verdict.verdict).toBe("fail");
|
|
90
|
+
expect(out.repeated).toMatchObject({ phase: "test", run: { id: 3, name: "test" }, workflowRun: { id: 70, name: "check", attempt: 2 } });
|
|
91
|
+
expect(log.reruns).toEqual([70]);
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
test("retries 2 re-runs twice and fails on the third attempt", async () => {
|
|
95
|
+
const failed = [LINT, run(2, "test", "failure")];
|
|
96
|
+
const { forge, clock, log } = scripted([failed, failed, failed], [wfRun(1), wfRun(2), wfRun(3)]);
|
|
97
|
+
const out = await ciVerdict({ cwd: workspace({ retries: 2 }), sha: SHA, forge, wait: true, ...clock });
|
|
98
|
+
expect(out.repeated?.workflowRun.attempt).toBe(3);
|
|
99
|
+
expect(log.reruns).toEqual([70, 70]);
|
|
100
|
+
});
|
|
101
|
+
|
|
102
|
+
test("a failed job waits for its workflow run to finish before the re-run", async () => {
|
|
103
|
+
const failed = [LINT, run(2, "test", "failure")];
|
|
104
|
+
const { forge, clock, log } = scripted([failed, failed, failed, [LINT, run(3, "test", "success")]], [wfRun(1, "in_progress"), wfRun(1, "in_progress"), wfRun(1)]);
|
|
105
|
+
const out = await ciVerdict({ cwd: workspace({ retries: 1 }), sha: SHA, forge, wait: true, ...clock });
|
|
106
|
+
expect(out.verdict.verdict).toBe("pass");
|
|
107
|
+
expect(log.reruns).toEqual([70]);
|
|
108
|
+
// Said once, however long the run takes.
|
|
109
|
+
expect(log.notes.filter((n) => n.includes("waiting for workflow run check to finish"))).toHaveLength(1);
|
|
110
|
+
});
|
|
111
|
+
|
|
112
|
+
test("a re-run GitHub hasn't started yet isn't asked for twice", async () => {
|
|
113
|
+
const failed = [LINT, run(2, "test", "failure")];
|
|
114
|
+
const { forge, clock, log } = scripted([failed, failed, failed, [LINT, run(3, "test", "success")]], [wfRun(1), wfRun(1), wfRun(2, "queued")]);
|
|
115
|
+
expect((await ciVerdict({ cwd: workspace({ retries: 1 }), sha: SHA, forge, wait: true, ...clock })).verdict.verdict).toBe("pass");
|
|
116
|
+
expect(log.reruns).toEqual([70]);
|
|
117
|
+
});
|
|
118
|
+
|
|
119
|
+
test("--retries overrides the setting", async () => {
|
|
120
|
+
const { forge, clock, log } = scripted([[LINT, run(2, "test", "failure")], [LINT, run(3, "test", "success")]], [wfRun(1)]);
|
|
121
|
+
expect((await ciVerdict({ cwd: workspace(true), sha: SHA, forge, wait: true, retries: 1, ...clock })).verdict.verdict).toBe("pass");
|
|
122
|
+
expect(log.reruns).toEqual([70]);
|
|
123
|
+
});
|
|
124
|
+
|
|
125
|
+
test("retries 0 fails at the first failed job on the first read and asks nothing more, as before", async () => {
|
|
126
|
+
for (const mq of [true, { retries: 0 }]) {
|
|
127
|
+
const { forge, clock, log } = scripted([[LINT, run(2, "test", "failure")]], [wfRun(1)]);
|
|
128
|
+
const out = await ciVerdict({ cwd: workspace(mq), sha: SHA, forge, wait: true, ...clock });
|
|
129
|
+
expect(out.verdict).toMatchObject({ verdict: "fail", failure: { phase: "test", run: { name: "test" } } });
|
|
130
|
+
expect(out.repeated).toBeUndefined();
|
|
131
|
+
expect(log).toMatchObject({ reads: 1, runReads: 0, reruns: [], sleeps: [], notes: [] });
|
|
132
|
+
}
|
|
133
|
+
// And --retries 0 over a setting of 1.
|
|
134
|
+
const { forge, clock, log } = scripted([[LINT, run(2, "test", "failure")]], [wfRun(1)]);
|
|
135
|
+
expect((await ciVerdict({ cwd: workspace({ retries: 1 }), sha: SHA, forge, wait: true, retries: 0, ...clock })).verdict.verdict).toBe("fail");
|
|
136
|
+
expect(log.reruns).toEqual([]);
|
|
137
|
+
});
|
|
138
|
+
|
|
139
|
+
test("without --wait it reads once and re-runs nothing", async () => {
|
|
140
|
+
const { forge, log } = scripted([[LINT, run(2, "test", "failure")]], [wfRun(1)]);
|
|
141
|
+
expect((await ciVerdict({ cwd: workspace({ retries: 1 }), sha: SHA, forge })).verdict.verdict).toBe("fail");
|
|
142
|
+
expect(log).toMatchObject({ reads: 1, runReads: 0, reruns: [] });
|
|
143
|
+
});
|
|
144
|
+
|
|
145
|
+
test("a check run that is no Actions job fails at once: there is nothing to re-run", async () => {
|
|
146
|
+
const { forge, clock, log } = scripted([[LINT, run(2, "test", "failure")]], [null]);
|
|
147
|
+
const out = await ciVerdict({ cwd: workspace({ retries: 1 }), sha: SHA, forge, wait: true, ...clock });
|
|
148
|
+
expect(out.verdict.verdict).toBe("fail");
|
|
149
|
+
expect(out.repeated).toBeUndefined();
|
|
150
|
+
expect(log.reruns).toEqual([]);
|
|
151
|
+
});
|
|
152
|
+
|
|
153
|
+
test("a refused re-run says the job needs actions: write", async () => {
|
|
154
|
+
const { forge, clock } = scripted([[LINT, run(2, "test", "failure")]], [wfRun(1)]);
|
|
155
|
+
forge.rerunFailedJobs = async () => {
|
|
156
|
+
throw new ForgeApiError("POST", "https://api.github.com/repos/acme/app/actions/runs/70/rerun-failed-jobs", 403, "Resource not accessible by integration");
|
|
157
|
+
};
|
|
158
|
+
await expect(ciVerdict({ cwd: workspace({ retries: 1 }), sha: SHA, forge, wait: true, ...clock })).rejects.toThrow(/could not re-run the failed jobs of workflow run check \(70\).*needs actions: write/);
|
|
159
|
+
});
|
|
160
|
+
});
|
|
161
|
+
|
|
162
|
+
describe("ci verdict --wait past forge errors (#4729)", () => {
|
|
163
|
+
test("a 5xx that clears passes, after a backoff that doubles", async () => {
|
|
164
|
+
const { forge, clock, log } = scripted([http(502), http(503), [LINT, run(2, "test", "success")]]);
|
|
165
|
+
const out = await ciVerdict({ cwd: workspace(true), sha: SHA, forge, wait: true, ...clock });
|
|
166
|
+
expect(out.verdict.verdict).toBe("pass");
|
|
167
|
+
expect(log.sleeps).toEqual([CI_VERDICT_BACKOFF_MS, 2 * CI_VERDICT_BACKOFF_MS]);
|
|
168
|
+
expect(log.notes[0]).toMatch(/^the forge failed \(GET .* answered 502\); looking again in 15s \(1 of 7 looked past\)$/);
|
|
169
|
+
});
|
|
170
|
+
|
|
171
|
+
test("a request with no answer is looked past too, and a 5xx while reading a workflow run", async () => {
|
|
172
|
+
const failed = [LINT, run(2, "test", "failure")];
|
|
173
|
+
const { forge, clock, log } = scripted([new TypeError("fetch failed"), failed, failed, [LINT, run(3, "test", "success")]], [http(500), wfRun(1)]);
|
|
174
|
+
expect((await ciVerdict({ cwd: workspace({ retries: 1 }), sha: SHA, forge, wait: true, ...clock })).verdict.verdict).toBe("pass");
|
|
175
|
+
expect(log.reruns).toEqual([70]);
|
|
176
|
+
});
|
|
177
|
+
|
|
178
|
+
test("errors that are not in a row each start the count again", async () => {
|
|
179
|
+
const pending = [LINT, run(2, "test", null, "in_progress")];
|
|
180
|
+
const answers: (CheckRun[] | Error)[] = [];
|
|
181
|
+
for (let i = 0; i < CI_VERDICT_FORGE_ERRORS + 2; i++) answers.push(http(502), pending);
|
|
182
|
+
answers.push([LINT, run(2, "test", "success")]);
|
|
183
|
+
const { forge, clock } = scripted(answers);
|
|
184
|
+
expect((await ciVerdict({ cwd: workspace(true), sha: SHA, forge, wait: true, ...clock })).verdict.verdict).toBe("pass");
|
|
185
|
+
});
|
|
186
|
+
|
|
187
|
+
test("a 5xx that doesn't clear fails at the bound, saying so", async () => {
|
|
188
|
+
const { forge, clock, log } = scripted([http(502)]);
|
|
189
|
+
const err = await ciVerdict({ cwd: workspace(true), sha: SHA, forge, wait: true, ...clock }).catch((e: unknown) => e);
|
|
190
|
+
expect(err).toBeInstanceOf(CiVerdictGaveUpError);
|
|
191
|
+
expect((err as Error).message).toMatch(new RegExp(`^the forge failed ${CI_VERDICT_FORGE_ERRORS} times in a row, the last: GET .* answered 502$`));
|
|
192
|
+
expect(log.reads).toBe(CI_VERDICT_FORGE_ERRORS);
|
|
193
|
+
expect(log.sleeps).toEqual([15_000, 30_000, 60_000, 120_000, 120_000, 120_000, 120_000]);
|
|
194
|
+
});
|
|
195
|
+
|
|
196
|
+
test("a 4xx is an answer, not a blip: it fails at once, as before", async () => {
|
|
197
|
+
const { forge, clock, log } = scripted([http(404)]);
|
|
198
|
+
await expect(ciVerdict({ cwd: workspace(true), sha: SHA, forge, wait: true, ...clock })).rejects.toBeInstanceOf(ForgeApiError);
|
|
199
|
+
expect(log.reads).toBe(1);
|
|
200
|
+
});
|
|
201
|
+
|
|
202
|
+
test("without --wait a 5xx fails at once, as before", async () => {
|
|
203
|
+
const { forge, log } = scripted([http(502)]);
|
|
204
|
+
await expect(ciVerdict({ cwd: workspace(true), sha: SHA, forge })).rejects.toBeInstanceOf(ForgeApiError);
|
|
205
|
+
expect(log.reads).toBe(1);
|
|
206
|
+
});
|
|
207
|
+
});
|
|
208
|
+
|
|
209
|
+
describe("ci verdict --wait --timeout-minutes (#4729)", () => {
|
|
210
|
+
test("a run that never finishes gives up before the timeout, saying what it waited on", async () => {
|
|
211
|
+
const { forge, clock, log } = scripted([[LINT, run(2, "test", null, "in_progress")]]);
|
|
212
|
+
const err = await ciVerdict({ cwd: workspace(true), sha: SHA, forge, wait: true, timeoutMs: 10 * 60_000, ...clock }).catch((e: unknown) => e);
|
|
213
|
+
expect(err).toBeInstanceOf(CiVerdictGaveUpError);
|
|
214
|
+
expect((err as Error).message).toBe("no verdict after 10 minutes: still waiting on test");
|
|
215
|
+
expect(clock.now()).toBe(10 * 60_000);
|
|
216
|
+
expect(log.sleeps.every((ms) => ms <= CI_VERDICT_INTERVAL_MS)).toBe(true);
|
|
217
|
+
// It looked once more at the deadline before giving up.
|
|
218
|
+
expect(log.reads).toBe(10 * 60_000 / CI_VERDICT_INTERVAL_MS + 1);
|
|
219
|
+
});
|
|
220
|
+
|
|
221
|
+
test("a re-run that never finishes gives up too, naming the run", async () => {
|
|
222
|
+
const failed = [LINT, run(2, "test", "failure")];
|
|
223
|
+
const { forge, clock } = scripted([failed], [wfRun(1), wfRun(2, "in_progress")]);
|
|
224
|
+
await expect(ciVerdict({ cwd: workspace({ retries: 1 }), sha: SHA, forge, wait: true, timeoutMs: 5 * 60_000, ...clock })).rejects.toThrow("no verdict after 5 minutes: waiting for workflow run check to finish");
|
|
225
|
+
});
|
|
226
|
+
|
|
227
|
+
test("a forge that keeps failing gives up at the deadline when that comes first", async () => {
|
|
228
|
+
const { forge, clock } = scripted([http(502)]);
|
|
229
|
+
await expect(ciVerdict({ cwd: workspace(true), sha: SHA, forge, wait: true, timeoutMs: 60_000, ...clock })).rejects.toThrow(/^no verdict after 1 minute: the forge failing: GET .* answered 502$/);
|
|
230
|
+
expect(clock.now()).toBe(60_000);
|
|
231
|
+
});
|
|
232
|
+
});
|
|
233
|
+
|
|
234
|
+
describe("ci.green.mergeQueue's retries (#4729)", () => {
|
|
235
|
+
const green = (mergeQueue: unknown) => parseDeclaration(JSON.stringify({ name: "w", schema: 1, members: [], ci: { green: { branch: "main", phases: { test: ["test"] }, require: ["test"], mergeQueue } } }), "chant.workspace.json").ci!.green!;
|
|
236
|
+
|
|
237
|
+
test("true reads as no retries; the object as its retries; false as no merge queue", () => {
|
|
238
|
+
expect(green(true).mergeQueue).toEqual({ retries: 0 });
|
|
239
|
+
expect(green({ retries: 0 }).mergeQueue).toEqual({ retries: 0 });
|
|
240
|
+
expect(green({ retries: 2 }).mergeQueue).toEqual({ retries: 2 });
|
|
241
|
+
expect(green(false).mergeQueue).toBeUndefined();
|
|
242
|
+
});
|
|
243
|
+
|
|
244
|
+
test("retries is a whole number from 0 to 5, and nothing else is in the object", () => {
|
|
245
|
+
for (const bad of [{ retries: -1 }, { retries: 6 }, { retries: 1.5 }, { retries: "1" }, {}, { retries: 1, extra: true }]) {
|
|
246
|
+
expect(() => green(bad), JSON.stringify(bad)).toThrow();
|
|
247
|
+
}
|
|
248
|
+
});
|
|
249
|
+
|
|
250
|
+
test("ci workflow without retries writes the verdict workflow as before", () => {
|
|
251
|
+
const yaml = renderMergeQueueWorkflow({ green: green(true), chantVersion: "0.124.70", workspaceRoot: "." });
|
|
252
|
+
expect(yaml).toBe(renderMergeQueueWorkflow({ green: green({ retries: 0 }), chantVersion: "0.124.70", workspaceRoot: "." }));
|
|
253
|
+
const q = parseYAMLDocument(yaml) as { permissions: unknown; jobs: Record<string, { "timeout-minutes": number; steps: { run?: string }[] }> };
|
|
254
|
+
expect(q.permissions).toEqual({ contents: "read", checks: "read" });
|
|
255
|
+
expect(q.jobs[MERGE_QUEUE_VERDICT_JOB]!["timeout-minutes"]).toBe(120);
|
|
256
|
+
expect(q.jobs[MERGE_QUEUE_VERDICT_JOB]!.steps.at(-1)!.run).toBe("npx --yes @intentius/chant@0.124.70 ci verdict --wait");
|
|
257
|
+
expect(yaml).not.toContain("actions:");
|
|
258
|
+
expect(yaml).not.toContain("re-run");
|
|
259
|
+
});
|
|
260
|
+
|
|
261
|
+
test("ci workflow with retries gives the job actions: write and a time budget under its timeout", () => {
|
|
262
|
+
const yaml = renderMergeQueueWorkflow({ green: green({ retries: 1 }), chantVersion: "0.124.70", workspaceRoot: "." });
|
|
263
|
+
const q = parseYAMLDocument(yaml) as { permissions: unknown; jobs: Record<string, { "timeout-minutes": number; steps: { run?: string }[] }> };
|
|
264
|
+
expect(q.permissions).toEqual({ contents: "read", checks: "read", actions: "write" });
|
|
265
|
+
const job = q.jobs[MERGE_QUEUE_VERDICT_JOB]!;
|
|
266
|
+
expect(job.steps.at(-1)!.run).toBe(`npx --yes @intentius/chant@0.124.70 ci verdict --wait --timeout-minutes ${MERGE_QUEUE_VERDICT_TIMEOUT_MINUTES}`);
|
|
267
|
+
expect(MERGE_QUEUE_VERDICT_TIMEOUT_MINUTES).toBeLessThan(job["timeout-minutes"]);
|
|
268
|
+
expect(yaml).toContain("# A failed job's workflow run is re-run up to 1 time first (ci.green.mergeQueue.retries).");
|
|
269
|
+
// --chant's command takes the same flag.
|
|
270
|
+
const own = parseYAMLDocument(renderMergeQueueWorkflow({ green: green({ retries: 2 }), chantVersion: "0.124.70", workspaceRoot: ".", chant: "npx chant" })) as typeof q;
|
|
271
|
+
expect(own.jobs[MERGE_QUEUE_VERDICT_JOB]!.steps.at(-1)!.run).toBe(`npx chant ci verdict --wait --timeout-minutes ${MERGE_QUEUE_VERDICT_TIMEOUT_MINUTES}`);
|
|
272
|
+
});
|
|
273
|
+
});
|
|
274
|
+
|
|
275
|
+
describe("the GitHub client's workflow runs (#4729)", () => {
|
|
276
|
+
function github(routes: Record<string, { status: number; body?: unknown }>) {
|
|
277
|
+
const calls: string[] = [];
|
|
278
|
+
const fetch: ForgeFetch = async (url, init) => {
|
|
279
|
+
calls.push(`${init.method} ${url}`);
|
|
280
|
+
const r = routes[`${init.method} ${url}`] ?? { status: 404, body: { message: "Not Found" } };
|
|
281
|
+
return { ok: r.status >= 200 && r.status < 300, status: r.status, text: async () => (r.body === undefined ? "" : JSON.stringify(r.body)) };
|
|
282
|
+
};
|
|
283
|
+
return { forge: githubCiForge({ apiBase: "https://api.github.com", repo: "acme/app", token: "t", fetch }), calls };
|
|
284
|
+
}
|
|
285
|
+
const API = "https://api.github.com/repos/acme/app";
|
|
286
|
+
|
|
287
|
+
test("a job's workflow run, its job read once and the run each time", async () => {
|
|
288
|
+
const { forge, calls } = github({
|
|
289
|
+
[`GET ${API}/actions/jobs/2`]: { status: 200, body: { id: 2, run_id: 70 } },
|
|
290
|
+
[`GET ${API}/actions/runs/70`]: { status: 200, body: { id: 70, name: "check", run_attempt: 2, status: "completed", conclusion: "failure", html_url: "https://github.com/acme/app/actions/runs/70" } },
|
|
291
|
+
});
|
|
292
|
+
const expected = { id: 70, name: "check", attempt: 2, status: "completed", conclusion: "failure", url: "https://github.com/acme/app/actions/runs/70" };
|
|
293
|
+
expect(await forge.workflowRun!(2)).toEqual(expected);
|
|
294
|
+
expect(await forge.workflowRun!(2)).toEqual(expected);
|
|
295
|
+
expect(calls).toEqual([`GET ${API}/actions/jobs/2`, `GET ${API}/actions/runs/70`, `GET ${API}/actions/runs/70`]);
|
|
296
|
+
});
|
|
297
|
+
|
|
298
|
+
test("another app's check run has no workflow run", async () => {
|
|
299
|
+
expect(await github({}).forge.workflowRun!(5)).toBeNull();
|
|
300
|
+
});
|
|
301
|
+
|
|
302
|
+
test("a 5xx is thrown for the verdict to look past", async () => {
|
|
303
|
+
const { forge } = github({ [`GET ${API}/actions/jobs/2`]: { status: 502 } });
|
|
304
|
+
const err = await forge.workflowRun!(2).catch((e: unknown) => e);
|
|
305
|
+
expect(err).toBeInstanceOf(ForgeApiError);
|
|
306
|
+
expect(isTransientForgeError(err)).toBe(true);
|
|
307
|
+
});
|
|
308
|
+
|
|
309
|
+
test("re-running posts rerun-failed-jobs every time it's asked", async () => {
|
|
310
|
+
const { forge, calls } = github({ [`POST ${API}/actions/runs/70/rerun-failed-jobs`]: { status: 201 } });
|
|
311
|
+
await forge.rerunFailedJobs!(70);
|
|
312
|
+
await forge.rerunFailedJobs!(70);
|
|
313
|
+
expect(calls).toEqual([`POST ${API}/actions/runs/70/rerun-failed-jobs`, `POST ${API}/actions/runs/70/rerun-failed-jobs`]);
|
|
314
|
+
});
|
|
315
|
+
|
|
316
|
+
test("which errors say to look again", () => {
|
|
317
|
+
expect([500, 502, 503, 504, 429].map((s) => isTransientForgeError(http(s)))).toEqual([true, true, true, true, true]);
|
|
318
|
+
expect([400, 401, 403, 404, 422].map((s) => isTransientForgeError(http(s)))).toEqual([false, false, false, false, false]);
|
|
319
|
+
expect(isTransientForgeError(new TypeError("fetch failed"))).toBe(true);
|
|
320
|
+
expect(isTransientForgeError(new Error("something else"))).toBe(false);
|
|
321
|
+
});
|
|
322
|
+
});
|
|
323
|
+
|
|
324
|
+
describe("chant ci verdict's flags (#4729)", () => {
|
|
325
|
+
test("--retries and --timeout-minutes parse, and refuse what isn't a number", () => {
|
|
326
|
+
expect(parseArgs(["ci", "verdict", "--wait", "--retries", "2", "--timeout-minutes", "85"])).toMatchObject({ command: "ci", path: "verdict", ciWait: true, ciRetries: 2, ciTimeoutMinutes: 85 });
|
|
327
|
+
expect(() => parseArgs(["ci", "verdict", "--wait", "--retries", "-1"])).toThrow("--retries needs a whole number");
|
|
328
|
+
expect(() => parseArgs(["ci", "verdict", "--wait", "--retries"])).toThrow("--retries needs a whole number");
|
|
329
|
+
expect(() => parseArgs(["ci", "verdict", "--wait", "--timeout-minutes", "0"])).toThrow("--timeout-minutes needs a number of minutes above 0");
|
|
330
|
+
expect(() => parseArgs(["ci", "verdict", "--wait", "--timeout-minutes", "soon"])).toThrow("--timeout-minutes needs a number of minutes above 0");
|
|
331
|
+
});
|
|
332
|
+
|
|
333
|
+
test("they go with --wait", async () => {
|
|
334
|
+
vi.spyOn(process, "cwd").mockReturnValue(workspace(true));
|
|
335
|
+
const errors: string[] = [];
|
|
336
|
+
vi.spyOn(console, "error").mockImplementation((...m: unknown[]) => void errors.push(m.map(String).join(" ")));
|
|
337
|
+
expect(await runCiVerdict({ args: { command: "ci", path: "verdict", ciRetries: 1 } } as never)).toBe(1);
|
|
338
|
+
expect(errors.join("\n")).toContain("--retries and --timeout-minutes go with --wait");
|
|
339
|
+
});
|
|
340
|
+
|
|
341
|
+
test("end to end against a fake GitHub: a failure that repeats exits 1 naming the phase, the job and the run", async () => {
|
|
342
|
+
vi.useFakeTimers({ toFake: ["setTimeout", "Date"] });
|
|
343
|
+
vi.spyOn(process, "cwd").mockReturnValue(workspace({ retries: 1 }));
|
|
344
|
+
vi.stubEnv("GITHUB_REPOSITORY", "acme/app");
|
|
345
|
+
vi.stubEnv("GITHUB_TOKEN", "t");
|
|
346
|
+
vi.stubEnv("CHANT_FORGE_TOKEN", "t");
|
|
347
|
+
vi.stubEnv("GITHUB_API_URL", "https://api.github.com");
|
|
348
|
+
const API = "https://api.github.com/repos/acme/app";
|
|
349
|
+
// GitHub: test fails on attempt 1; after the re-run, attempt 2's test fails too.
|
|
350
|
+
let attempt = 1;
|
|
351
|
+
const posts: string[] = [];
|
|
352
|
+
const checkRun = (id: number, name: string, conclusion: string) => ({ id, name, status: "completed", conclusion, completed_at: "2026-11-06T10:00:00Z", html_url: `https://github.com/acme/app/runs/${id}` });
|
|
353
|
+
const answer = (status: number, body: unknown) => ({ ok: status < 300, status, text: async () => JSON.stringify(body) });
|
|
354
|
+
vi.stubGlobal("fetch", (async (url: string, init: { method: string }) => {
|
|
355
|
+
if (init.method === "POST") {
|
|
356
|
+
posts.push(url);
|
|
357
|
+
attempt = 2;
|
|
358
|
+
return answer(201, {});
|
|
359
|
+
}
|
|
360
|
+
if (url.includes("/check-runs")) {
|
|
361
|
+
const runs = [checkRun(1, "lint", "success"), checkRun(2, "test", "failure"), ...(attempt === 2 ? [checkRun(3, "test", "failure")] : [])];
|
|
362
|
+
return answer(200, { total_count: runs.length, check_runs: runs });
|
|
363
|
+
}
|
|
364
|
+
if (url.startsWith(`${API}/actions/jobs/`)) return answer(200, { run_id: 70 });
|
|
365
|
+
if (url === `${API}/actions/runs/70`) return answer(200, { id: 70, name: "check", run_attempt: attempt, status: "completed", conclusion: "failure", html_url: "https://github.com/acme/app/actions/runs/70" });
|
|
366
|
+
return answer(404, { message: "Not Found" });
|
|
367
|
+
}) as ForgeFetch);
|
|
368
|
+
const errors: string[] = [];
|
|
369
|
+
vi.spyOn(console, "error").mockImplementation((...m: unknown[]) => void errors.push(m.map(String).join(" ")));
|
|
370
|
+
const exit = runCiVerdict({ args: { command: "ci", path: "verdict", commit: SHA, ciWait: true, ciTimeoutMinutes: 115 } } as never);
|
|
371
|
+
await vi.runAllTimersAsync();
|
|
372
|
+
expect(await exit).toBe(1);
|
|
373
|
+
expect(posts).toEqual([`${API}/actions/runs/70/rerun-failed-jobs`]);
|
|
374
|
+
expect(errors.at(-1)).toBe(`${SHA.slice(0, 12)}: phase test failed in test (failure) again on attempt 2 of workflow run check (https://github.com/acme/app/actions/runs/70)`);
|
|
375
|
+
});
|
|
376
|
+
});
|
|
@@ -1902,8 +1902,23 @@
|
|
|
1902
1902
|
"description": "Added by #4031, read by chant 0.124.28 and newer. The full sha of the first commit on branch whose ci/green and ci/revoked tags must carry a DSSE statement in their annotation, signed by a key .chant/trust.json at base lists under runners for a principal holding the runner role. A tag on a strict first-parent ancestor of this commit counts unsigned, so existing tags keep working; a tag on any other commit, this one included, needs a statement. Position in history decides, never a date, which whoever makes a commit or a tag sets. A reader refuses a sha that names no commit in its clone with declaration-invalid. Absent, no tag needs a statement. Since chant 0.124.32 (#4156) it covers Op run records the same way, when ci.green.source is runs: an unsigned run counts on a strict first-parent ancestor of this commit and needs a statement on any other, ci serve refuses to start without a runner key trusted at the target's head, and ci run and chant run warn about a run they leave unsigned. Absent, an unsigned run counts and nothing asks for a key."
|
|
1903
1903
|
},
|
|
1904
1904
|
"mergeQueue": {
|
|
1905
|
-
"
|
|
1906
|
-
|
|
1905
|
+
"oneOf": [
|
|
1906
|
+
{ "type": "boolean" },
|
|
1907
|
+
{
|
|
1908
|
+
"type": "object",
|
|
1909
|
+
"additionalProperties": false,
|
|
1910
|
+
"required": ["retries"],
|
|
1911
|
+
"properties": {
|
|
1912
|
+
"retries": {
|
|
1913
|
+
"type": "integer",
|
|
1914
|
+
"minimum": 0,
|
|
1915
|
+
"maximum": 5,
|
|
1916
|
+
"description": "Added by #4729. How many times the verdict re-runs a failed workflow run's failed jobs before the group fails. On a failed required job, chant ci verdict --wait waits for the job's workflow run to finish, re-runs its failed jobs while the run's attempt is no more than retries, and fails when it fails on attempt retries + 1. Above 0, chant ci workflow gives the verdict job actions: write and passes --timeout-minutes so the verdict gives up, with a message, before Actions kills the job. 0 is true's behaviour: the first failed job fails the group. Read by chant 0.124.70 and newer, so a declaration that sets the object form sets minReader to 0.124.70 or newer."
|
|
1917
|
+
}
|
|
1918
|
+
}
|
|
1919
|
+
}
|
|
1920
|
+
],
|
|
1921
|
+
"description": "The branch takes changes through GitHub's merge queue (#4644). chant ci workflow adds the merge_group trigger (checks_requested) to each workflow holding a phase that sets build, and writes .github/workflows/chant-merge-queue.yml, whose chant-verdict job runs chant ci verdict --wait on the group's commit: it passes when every required phase passed there, judged as the tick judges, so the queue's ruleset requires that one check. The queue lands the commit its group tested, so the group's runs count for it; a later push run's skipped or cancelled attempt of the same job never replaces the group's decided one. A workflow holding a phase without build gets no trigger: add merge_group to it by hand. true is { \"retries\": 0 }. Written by chant 0.124.61 and newer, so a declaration that sets it sets minReader to 0.124.61 or newer."
|
|
1907
1922
|
},
|
|
1908
1923
|
"notify": {
|
|
1909
1924
|
"type": "object",
|
|
@@ -295,9 +295,10 @@ export interface CiGreen {
|
|
|
295
295
|
* The branch takes changes through GitHub's merge queue (#4644): `ci
|
|
296
296
|
* workflow` adds the `merge_group` trigger to the workflows holding a
|
|
297
297
|
* phase that sets `build`, and writes `chant-merge-queue.yml`, whose
|
|
298
|
-
* `chant-verdict` job the queue's ruleset requires. Absent when unset
|
|
298
|
+
* `chant-verdict` job the queue's ruleset requires. Absent when unset;
|
|
299
|
+
* `true` reads as `{ retries: 0 }`.
|
|
299
300
|
*/
|
|
300
|
-
mergeQueue?:
|
|
301
|
+
mergeQueue?: CiMergeQueue;
|
|
301
302
|
/**
|
|
302
303
|
* Known-flaky tests the phases' runners leave out (#4677), each with the
|
|
303
304
|
* issue tracking its fix and the day it comes back. Absent when the
|
|
@@ -307,6 +308,19 @@ export interface CiGreen {
|
|
|
307
308
|
pointer: string;
|
|
308
309
|
}
|
|
309
310
|
|
|
311
|
+
/** `ci.green.mergeQueue` (#4644), read from `true` or `{ "retries": <n> }` (#4729). */
|
|
312
|
+
export interface CiMergeQueue {
|
|
313
|
+
/**
|
|
314
|
+
* How many times `ci verdict --wait` re-runs a failed workflow run's
|
|
315
|
+
* failed jobs before the group fails (#4729). 0, the default, fails at the
|
|
316
|
+
* first failed job.
|
|
317
|
+
*/
|
|
318
|
+
retries: number;
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
/** The most re-runs `ci.green.mergeQueue.retries` may ask for (#4729). */
|
|
322
|
+
export const MERGE_QUEUE_MAX_RETRIES = 5;
|
|
323
|
+
|
|
310
324
|
/** The default `ci.green.quarantine.maxDays` (#4677): how far out an entry's `until` may be. */
|
|
311
325
|
export const DEFAULT_QUARANTINE_MAX_DAYS = 14;
|
|
312
326
|
|
|
@@ -1455,7 +1469,7 @@ function cloudIdentityOf(raw: CloudIdentityDeclaration, at: (pointer: string, ke
|
|
|
1455
1469
|
|
|
1456
1470
|
function ciOf(raw: unknown, at: (pointer: string, key?: boolean) => ErrorLocation): CiDeclaration | null {
|
|
1457
1471
|
if (raw === undefined) return null;
|
|
1458
|
-
const block = raw as { reportUrl?: string; workflow?: CiWorkflow; hooks?: unknown; identity?: { cloud?: CloudIdentityDeclaration }; branches?: Record<string, string[]>; labels?: Record<string, string[]>; locks?: CiLocks; lockGrain?: CiLockGrain; policy?: unknown; prEnvironment?: { template: string; ttl?: string; teardown?: { name?: string; command: string[]; lexicons?: string[] } }; green?: { branch: string | string[]; window?: string; phases: Record<string, string[] | { runs: string[]; skipped?: CiSkipped; build?: CiBuild }>; require: string[]; preMerge?: string[]; signedSince?: string; source?: CiSource; lanes?: Record<string, { require: string[] }>; notify?: CiNotify; mergeQueue?: boolean; quarantine?: { maxDays?: number; phases: Record<string, { test: string; file?: string; issue: string; until: string; why: string }[]> } } };
|
|
1472
|
+
const block = raw as { reportUrl?: string; workflow?: CiWorkflow; hooks?: unknown; identity?: { cloud?: CloudIdentityDeclaration }; branches?: Record<string, string[]>; labels?: Record<string, string[]>; locks?: CiLocks; lockGrain?: CiLockGrain; policy?: unknown; prEnvironment?: { template: string; ttl?: string; teardown?: { name?: string; command: string[]; lexicons?: string[] } }; green?: { branch: string | string[]; window?: string; phases: Record<string, string[] | { runs: string[]; skipped?: CiSkipped; build?: CiBuild }>; require: string[]; preMerge?: string[]; signedSince?: string; source?: CiSource; lanes?: Record<string, { require: string[] }>; notify?: CiNotify; mergeQueue?: boolean | { retries?: unknown }; quarantine?: { maxDays?: number; phases: Record<string, { test: string; file?: string; issue: string; until: string; why: string }[]> } } };
|
|
1459
1473
|
const hooks = ciHooksOf(block.hooks, "/ci/hooks", at);
|
|
1460
1474
|
const policy = ciPolicyOf(block.policy, at);
|
|
1461
1475
|
const workflow = {
|
|
@@ -1540,8 +1554,18 @@ function ciOf(raw: unknown, at: (pointer: string, key?: boolean) => ErrorLocatio
|
|
|
1540
1554
|
}
|
|
1541
1555
|
}
|
|
1542
1556
|
const quarantine = g.quarantine !== undefined ? quarantineOf(g.quarantine, phases, at) : undefined;
|
|
1557
|
+
// #4729: true, or { retries }, which the schema has checked; false is no merge queue.
|
|
1558
|
+
let mergeQueue: CiMergeQueue | undefined;
|
|
1559
|
+
if (g.mergeQueue === true) mergeQueue = { retries: 0 };
|
|
1560
|
+
else if (typeof g.mergeQueue === "object" && g.mergeQueue !== null) {
|
|
1561
|
+
const retries = g.mergeQueue.retries ?? 0;
|
|
1562
|
+
if (!(typeof retries === "number" && Number.isInteger(retries) && retries >= 0 && retries <= MERGE_QUEUE_MAX_RETRIES)) {
|
|
1563
|
+
throw new WorkspaceReadError("declaration-invalid", `ci.green.mergeQueue.retries: a whole number from 0 to ${MERGE_QUEUE_MAX_RETRIES}`, at("/ci/green/mergeQueue/retries"));
|
|
1564
|
+
}
|
|
1565
|
+
mergeQueue = { retries };
|
|
1566
|
+
}
|
|
1543
1567
|
const judged = Array.isArray(g.branch) ? g.branch : [g.branch];
|
|
1544
|
-
return { green: { branch: judged[0]!, ...(judged.length > 1 ? { branches: [...judged] } : {}), window: g.window ?? DEFAULT_CI_WINDOW, phases, require: [...g.require], ...(g.preMerge !== undefined ? { preMerge: [...g.preMerge] } : {}), signedSince: g.signedSince ?? null, source: g.source ?? "checks", ...(lanes.length > 0 ? { lanes } : {}), ...(g.notify !== undefined ? { notify: { ...g.notify, ...(g.notify.events ? { events: [...g.notify.events] } : {}) } } : {}), ...(
|
|
1568
|
+
return { green: { branch: judged[0]!, ...(judged.length > 1 ? { branches: [...judged] } : {}), window: g.window ?? DEFAULT_CI_WINDOW, phases, require: [...g.require], ...(g.preMerge !== undefined ? { preMerge: [...g.preMerge] } : {}), signedSince: g.signedSince ?? null, source: g.source ?? "checks", ...(lanes.length > 0 ? { lanes } : {}), ...(g.notify !== undefined ? { notify: { ...g.notify, ...(g.notify.events ? { events: [...g.notify.events] } : {}) } } : {}), ...(mergeQueue ? { mergeQueue } : {}), ...(quarantine ? { quarantine } : {}), pointer: "/ci/green" }, ...workflow, reportUrl };
|
|
1545
1569
|
}
|
|
1546
1570
|
|
|
1547
1571
|
/** A YYYY-MM-DD that names a real day, as days since 1970-01-01 UTC, or null. */
|