@intentius/chant 0.124.69 → 0.124.71
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ci-agent.d.ts +12 -1
- package/dist/ci-agent.d.ts.map +1 -1
- package/dist/ci-agent.js +13 -8
- package/dist/ci-commit.d.ts +297 -0
- package/dist/ci-commit.d.ts.map +1 -0
- package/dist/ci-commit.js +696 -0
- package/dist/ci-product-words.d.ts +6 -1
- package/dist/ci-product-words.d.ts.map +1 -1
- package/dist/ci-product-words.js +3 -1
- package/dist/cli/handlers/approve.d.ts.map +1 -1
- package/dist/cli/handlers/approve.js +7 -2
- package/dist/cli/handlers/commit.d.ts +26 -0
- package/dist/cli/handlers/commit.d.ts.map +1 -0
- package/dist/cli/handlers/commit.js +142 -0
- package/dist/cli/handlers/pr.d.ts +6 -0
- package/dist/cli/handlers/pr.d.ts.map +1 -1
- package/dist/cli/handlers/pr.js +16 -7
- package/dist/cli/handlers/release.d.ts +9 -0
- package/dist/cli/handlers/release.d.ts.map +1 -1
- package/dist/cli/handlers/release.js +114 -5
- package/dist/cli/main.d.ts.map +1 -1
- package/dist/cli/main.js +23 -0
- package/dist/cli/parse-args.d.ts.map +1 -1
- package/dist/cli/parse-args.js +15 -0
- package/dist/cli/registry.d.ts +4 -0
- package/dist/cli/registry.d.ts.map +1 -1
- package/dist/components/product-pipeline.d.ts +33 -0
- package/dist/components/product-pipeline.d.ts.map +1 -1
- package/dist/components/product-pipeline.js +34 -1
- package/dist/components/release-compile.d.ts +2 -0
- package/dist/components/release-compile.d.ts.map +1 -1
- package/dist/components/release-compile.js +8 -4
- package/dist/config.d.ts +46 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +9 -0
- package/dist/lexicon.d.ts +37 -0
- package/dist/lexicon.d.ts.map +1 -1
- package/dist/lexicon.js +28 -0
- package/dist/lifecycle/__fixtures__/publish-fakes.d.ts +4 -0
- package/dist/lifecycle/__fixtures__/publish-fakes.d.ts.map +1 -1
- package/dist/lifecycle/__fixtures__/publish-fakes.js +6 -2
- package/dist/lifecycle/gated-request.d.ts +3 -1
- package/dist/lifecycle/gated-request.d.ts.map +1 -1
- package/dist/lifecycle/gated-request.js +5 -2
- package/dist/lifecycle/git.d.ts.map +1 -1
- package/dist/lifecycle/git.js +57 -0
- package/dist/lifecycle/publish-declaration.d.ts +16 -0
- package/dist/lifecycle/publish-declaration.d.ts.map +1 -1
- package/dist/lifecycle/publish-declaration.js +35 -3
- package/dist/lifecycle/publish-forge.d.ts +12 -0
- package/dist/lifecycle/publish-forge.d.ts.map +1 -1
- package/dist/lifecycle/publish-forge.js +103 -19
- package/dist/lifecycle/publish-npm.d.ts +15 -0
- package/dist/lifecycle/publish-npm.d.ts.map +1 -1
- package/dist/lifecycle/publish-npm.js +28 -1
- package/dist/lifecycle/publish-oci.d.ts +2 -0
- package/dist/lifecycle/publish-oci.d.ts.map +1 -1
- package/dist/lifecycle/publish-oci.js +10 -0
- package/dist/lifecycle/release-cut.d.ts +7 -1
- package/dist/lifecycle/release-cut.d.ts.map +1 -1
- package/dist/lifecycle/release-cut.js +8 -1
- package/dist/lifecycle/release-images.d.ts +87 -0
- package/dist/lifecycle/release-images.d.ts.map +1 -0
- package/dist/lifecycle/release-images.js +139 -0
- package/dist/lifecycle/release-publish.d.ts +2 -0
- package/dist/lifecycle/release-publish.d.ts.map +1 -1
- package/dist/op/activities/activity-contracts.d.ts +4 -4
- package/dist/op/activities/review-agent.d.ts.map +1 -1
- package/dist/op/activities/review-agent.js +4 -2
- package/dist/op/gate.d.ts +5 -1
- package/dist/op/gate.d.ts.map +1 -1
- package/dist/op/gate.js +7 -2
- package/dist/op/op-waves-notify.d.ts +1 -1
- package/dist/op/op-waves-notify.d.ts.map +1 -1
- package/dist/op/op-waves-notify.js +2 -1
- package/dist/op/op-waves-run.d.ts.map +1 -1
- package/dist/op/op-waves-run.js +12 -8
- package/dist/op/op-waves.d.ts +9 -0
- package/dist/op/op-waves.d.ts.map +1 -1
- package/dist/op/op-waves.js +11 -0
- package/dist/pr-forge-base.d.ts +18 -0
- package/dist/pr-forge-base.d.ts.map +1 -0
- package/dist/pr-forge-base.js +29 -0
- package/dist/release-floors.d.ts +1 -1
- package/dist/release-floors.js +1 -1
- package/dist/wave-runtime.mjs +305 -305
- package/dist/workspace/ci-green-cli.d.ts +7 -3
- package/dist/workspace/ci-green-cli.d.ts.map +1 -1
- package/dist/workspace/ci-green-cli.js +29 -8
- package/dist/workspace/ci-green-forge.d.ts +29 -0
- package/dist/workspace/ci-green-forge.d.ts.map +1 -1
- package/dist/workspace/ci-green-forge.js +50 -0
- package/dist/workspace/ci-green-workflow.d.ts +9 -1
- package/dist/workspace/ci-green-workflow.d.ts.map +1 -1
- package/dist/workspace/ci-green-workflow.js +15 -3
- package/dist/workspace/ci-green.d.ts +35 -1
- package/dist/workspace/ci-green.d.ts.map +1 -1
- package/dist/workspace/ci-green.js +122 -4
- package/dist/workspace/declaration.d.ts +14 -2
- package/dist/workspace/declaration.d.ts.map +1 -1
- package/dist/workspace/declaration.js +14 -1
- package/dist/workspace/declaration.schema.json +17 -2
- package/dist/workspace/release-workflow.d.ts +23 -0
- package/dist/workspace/release-workflow.d.ts.map +1 -1
- package/dist/workspace/release-workflow.js +203 -12
- package/package.json +1 -1
- package/src/ci-agent.ts +13 -8
- package/src/ci-commit.test.ts +415 -0
- package/src/ci-commit.ts +857 -0
- package/src/ci-product-words.ts +6 -2
- package/src/cli/handlers/approve.ts +6 -2
- package/src/cli/handlers/commit.ts +152 -0
- package/src/cli/handlers/operator.test.ts +2 -0
- package/src/cli/handlers/pr-wave-approve-lines.test.ts +32 -0
- package/src/cli/handlers/pr.ts +17 -7
- package/src/cli/handlers/release.ts +110 -5
- package/src/cli/main.ts +23 -0
- package/src/cli/parse-args.ts +11 -0
- package/src/cli/registry.ts +4 -0
- package/src/components/product-pipeline.test.ts +28 -1
- package/src/components/product-pipeline.ts +54 -1
- package/src/components/release-compile.ts +9 -3
- package/src/config.ts +24 -0
- package/src/lexicon.ts +56 -0
- package/src/lifecycle/__fixtures__/publish-fakes.ts +11 -5
- package/src/lifecycle/gated-request-spend.test.ts +115 -0
- package/src/lifecycle/gated-request.ts +5 -2
- package/src/lifecycle/git.ts +53 -0
- package/src/lifecycle/ledger-two-clones.test.ts +132 -0
- package/src/lifecycle/publish-declaration.ts +35 -3
- package/src/lifecycle/publish-forge-notes.test.ts +193 -0
- package/src/lifecycle/publish-forge.ts +101 -22
- package/src/lifecycle/publish-npm.ts +31 -2
- package/src/lifecycle/publish-oci.ts +10 -0
- package/src/lifecycle/publish-targets.test.ts +2 -1
- package/src/lifecycle/release-cut.ts +17 -1
- package/src/lifecycle/release-images.test.ts +242 -0
- package/src/lifecycle/release-images.ts +175 -0
- package/src/lifecycle/release-publish.ts +2 -0
- package/src/op/activities/review-agent.test.ts +20 -1
- package/src/op/activities/review-agent.ts +5 -2
- package/src/op/gate.ts +7 -2
- package/src/op/op-waves-approve-target.test.ts +108 -0
- package/src/op/op-waves-notify.ts +2 -2
- package/src/op/op-waves-run.ts +12 -7
- package/src/op/op-waves.ts +12 -0
- package/src/pr-forge-base.test.ts +37 -0
- package/src/pr-forge-base.ts +36 -0
- package/src/release-floors.ts +1 -1
- package/src/workspace/__fixtures__/release-workflow/forgejo-plain-member.yml +26 -0
- package/src/workspace/__fixtures__/release-workflow/forgejo-plain.yml +23 -0
- package/src/workspace/__fixtures__/release-workflow/forgejo-secure-member.yml +27 -0
- package/src/workspace/__fixtures__/release-workflow/forgejo-secure.yml +24 -0
- package/src/workspace/__fixtures__/release-workflow/github-plain-member.yml +26 -0
- package/src/workspace/__fixtures__/release-workflow/github-plain.yml +23 -0
- package/src/workspace/__fixtures__/release-workflow/github-secure-member.yml +27 -0
- package/src/workspace/__fixtures__/release-workflow/github-secure.yml +24 -0
- package/src/workspace/__fixtures__/release-workflow/gitlab-plain-member.yml +13 -0
- package/src/workspace/__fixtures__/release-workflow/gitlab-plain.yml +12 -0
- package/src/workspace/__fixtures__/release-workflow/gitlab-secure-member.yml +14 -0
- package/src/workspace/__fixtures__/release-workflow/gitlab-secure.yml +13 -0
- package/src/workspace/ci-green-cli.ts +27 -8
- package/src/workspace/ci-green-forge.ts +70 -0
- package/src/workspace/ci-green-workflow.ts +15 -3
- package/src/workspace/ci-green.ts +130 -5
- package/src/workspace/ci-verdict-retries.test.ts +376 -0
- package/src/workspace/declaration.schema.json +17 -2
- package/src/workspace/declaration.ts +28 -4
- package/src/workspace/release-workflow.test.ts +151 -0
- package/src/workspace/release-workflow.ts +270 -12
|
@@ -31,7 +31,7 @@ import { join, relative, resolve, sep } from "node:path";
|
|
|
31
31
|
import { formatError } from "../cli/format";
|
|
32
32
|
import type { CommandContext } from "../cli/registry";
|
|
33
33
|
import { findWorkspaceRoot } from "../project-root";
|
|
34
|
-
import { ciBase, ciList, ciTick, ciVerdict, CiGreenError, formatList, formatTick, lastGreen, RUNNER_KEY_ENV } from "./ci-green";
|
|
34
|
+
import { ciBase, ciList, ciTick, ciVerdict, CiGreenError, CiVerdictGaveUpError, formatList, formatTick, lastGreen, RUNNER_KEY_ENV } from "./ci-green";
|
|
35
35
|
import { ciForgeFromEnv } from "./ci-green-forge";
|
|
36
36
|
import { ACTIONS_NAME, buildTriggerEdits, CI_GREEN_WORKFLOW_FILE, ciGreenHeader, MERGE_QUEUE_VERDICT_JOB, MERGE_QUEUE_WORKFLOW_FILE, type PushToken, readWorkflowSources, renderCiGreenWorkflow, renderMergeQueueWorkflow, workflowsHolding, writeWorkflowFile } from "./ci-green-workflow";
|
|
37
37
|
import { greenBranches, readDeclaration, readerVersion, WorkspaceReadError, type CiSource } from "./declaration";
|
|
@@ -39,7 +39,7 @@ import { recordGeneratedFiles, resolveMemberContext, shellArg } from "./member-p
|
|
|
39
39
|
import { gitTop, workingTree } from "./tree";
|
|
40
40
|
|
|
41
41
|
const USAGE =
|
|
42
|
-
"chant ci last-green [--json] [--lane <name>] [--from <rev>] [--branch <branch>] | ci base [--json] [--phase <name> | --lane <name>] [--from <rev>] [--branch <branch>] | ci verdict [--commit <sha>] [--wait] | ci list [--json] [--branch <branch>] [--limit <n>] [--since <rev>] [--forge github] [--source checks|runs] | ci tick [--dry-run] [--forge github] [--source checks|runs] [--key <runner key.pem>] | " +
|
|
42
|
+
"chant ci last-green [--json] [--lane <name>] [--from <rev>] [--branch <branch>] | ci base [--json] [--phase <name> | --lane <name>] [--from <rev>] [--branch <branch>] | ci verdict [--commit <sha>] [--wait [--retries <n>] [--timeout-minutes <n>]] | ci list [--json] [--branch <branch>] [--limit <n>] [--since <rev>] [--forge github] [--source checks|runs] | ci tick [--dry-run] [--forge github] [--source checks|runs] [--key <runner key.pem>] | " +
|
|
43
43
|
"ci workflow [--workflow <name>]... [--output <file>] [--chant <command>] [--install <command>] [--token-secret <NAME> | --app-id-var <VAR> --app-key-secret <NAME>] [--runner-key-secret <NAME>]";
|
|
44
44
|
|
|
45
45
|
export { RUNNER_KEY_ENV };
|
|
@@ -107,27 +107,42 @@ export async function runCiBase(ctx: CommandContext): Promise<number> {
|
|
|
107
107
|
}
|
|
108
108
|
|
|
109
109
|
/**
|
|
110
|
-
* `chant ci verdict [--commit <sha>] [--wait]
|
|
111
|
-
* required phase passed on the commit
|
|
112
|
-
* failed or wasn't built, and 3 when one
|
|
110
|
+
* `chant ci verdict [--commit <sha>] [--wait] [--retries <n>] [--timeout-minutes <n>]`
|
|
111
|
+
* (#4644): exits 0 when every required phase passed on the commit
|
|
112
|
+
* (GITHUB_SHA, else HEAD), 1 when one failed or wasn't built, and 3 when one
|
|
113
|
+
* is still pending without --wait. With --wait (#4729) it looks past forge
|
|
114
|
+
* errors that say to look again, re-runs a failed job's workflow run up to
|
|
115
|
+
* --retries times (else ci.green.mergeQueue.retries), and with
|
|
116
|
+
* --timeout-minutes exits 1 with a message once that time is up.
|
|
113
117
|
*/
|
|
114
118
|
export async function runCiVerdict(ctx: CommandContext): Promise<number> {
|
|
115
119
|
try {
|
|
120
|
+
if (!ctx.args.ciWait && (ctx.args.ciRetries !== undefined || ctx.args.ciTimeoutMinutes !== undefined)) {
|
|
121
|
+
throw new Error("--retries and --timeout-minutes go with --wait: without it the verdict reads the check runs once");
|
|
122
|
+
}
|
|
116
123
|
const sha = ctx.args.commit ?? process.env.GITHUB_SHA ?? execFileSync("git", ["rev-parse", "HEAD"], { encoding: "utf-8" }).trim();
|
|
117
124
|
const forge = ciForgeFromEnv("github");
|
|
118
|
-
const { verdict } = await ciVerdict({
|
|
125
|
+
const { verdict, repeated } = await ciVerdict({
|
|
119
126
|
cwd: process.cwd(),
|
|
120
127
|
sha,
|
|
121
128
|
forge,
|
|
122
129
|
...(ctx.args.ciWait ? { wait: true } : {}),
|
|
123
130
|
...(ctx.args.branch !== undefined ? { branch: ctx.args.branch } : {}),
|
|
131
|
+
...(ctx.args.ciRetries !== undefined ? { retries: ctx.args.ciRetries } : {}),
|
|
132
|
+
...(ctx.args.ciTimeoutMinutes !== undefined ? { timeoutMs: ctx.args.ciTimeoutMinutes * 60_000 } : {}),
|
|
124
133
|
onPending: (v) => console.error(`waiting on ${v.phases.filter((p) => p.verdict === "pending").map((p) => p.phase).join(", ")}`),
|
|
134
|
+
onNote: (line) => console.error(line),
|
|
125
135
|
});
|
|
126
136
|
const short = sha.slice(0, 12);
|
|
127
137
|
if (verdict.verdict === "pass") {
|
|
128
138
|
console.log(`${short}: every required phase passed (${verdict.phases.map((p) => p.phase).join(", ")})`);
|
|
129
139
|
return 0;
|
|
130
140
|
}
|
|
141
|
+
if (repeated) {
|
|
142
|
+
const wr = repeated.workflowRun;
|
|
143
|
+
console.error(`${short}: phase ${repeated.phase} failed in ${repeated.run.name} (${repeated.run.conclusion}) again on attempt ${wr.attempt} of workflow run ${wr.name}${wr.url ? ` (${wr.url})` : ""}`);
|
|
144
|
+
return 1;
|
|
145
|
+
}
|
|
131
146
|
if (verdict.verdict === "fail") {
|
|
132
147
|
console.error(`${short}: phase ${verdict.failure!.phase} failed in ${verdict.failure!.run.name} (${verdict.failure!.run.conclusion})`);
|
|
133
148
|
return 1;
|
|
@@ -136,7 +151,9 @@ export async function runCiVerdict(ctx: CommandContext): Promise<number> {
|
|
|
136
151
|
console.error(`${short}: ${verdict.verdict === "pending" ? "still waiting on" : "never built"} ${open.join(", ")}`);
|
|
137
152
|
return verdict.verdict === "pending" ? CHANT_EXIT.waiting : 1;
|
|
138
153
|
} catch (err) {
|
|
139
|
-
|
|
154
|
+
// #4729: out of time or out of forge errors to look past says so, without the usage.
|
|
155
|
+
if (err instanceof CiVerdictGaveUpError) console.error(`chant ci verdict: ${err.message}`);
|
|
156
|
+
else console.error(formatError({ message: `chant ci verdict: ${describe(err)}`, hint: USAGE }));
|
|
140
157
|
return 1;
|
|
141
158
|
}
|
|
142
159
|
}
|
|
@@ -258,7 +275,9 @@ export async function runCiWorkflow(ctx: CommandContext): Promise<number> {
|
|
|
258
275
|
if (green.mergeQueue && !member) {
|
|
259
276
|
const queue = renderMergeQueueWorkflow({ green, chantVersion: readerVersion(), workspaceRoot, chant: ctx.args.ciChant, install: ctx.args.ciInstall, npmLock });
|
|
260
277
|
const at = writeWorkflowFile(join(repoRoot, MERGE_QUEUE_WORKFLOW_FILE), ciGreenHeader(command, "root") + queue);
|
|
261
|
-
|
|
278
|
+
const retries = green.mergeQueue.retries;
|
|
279
|
+
const again = retries > 0 ? `, re-running a failed job's workflow run up to ${retries} time${retries === 1 ? "" : "s"} first` : "";
|
|
280
|
+
console.log(`Wrote ${at}: on each merge group, ${MERGE_QUEUE_VERDICT_JOB} passes when every required phase passed on the group's commit${again}. Require ${MERGE_QUEUE_VERDICT_JOB} in the branch's ruleset.`);
|
|
262
281
|
}
|
|
263
282
|
for (const e of edits) {
|
|
264
283
|
if (!e.changed) continue;
|
|
@@ -69,6 +69,40 @@ export interface CiForge {
|
|
|
69
69
|
* can't ask.
|
|
70
70
|
*/
|
|
71
71
|
dispatch?(workflow: string, ref: string): Promise<void>;
|
|
72
|
+
/**
|
|
73
|
+
* The workflow run a check run's job belongs to, as it is now, at its
|
|
74
|
+
* latest attempt (#4729), for `ci verdict --retries`. Null when the check
|
|
75
|
+
* run is no Actions job. Absent on a forge chant can't ask.
|
|
76
|
+
*/
|
|
77
|
+
workflowRun?(checkRunId: number): Promise<WorkflowRun | null>;
|
|
78
|
+
/**
|
|
79
|
+
* Re-run a workflow run's failed jobs (#4729), each time it is asked,
|
|
80
|
+
* where {@link rerun} asks once per workflow run per client. Needs
|
|
81
|
+
* `actions: write`. Absent on a forge chant can't ask.
|
|
82
|
+
*/
|
|
83
|
+
rerunFailedJobs?(workflowRunId: number): Promise<void>;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/** A GitHub Actions workflow run at its latest attempt (#4729). */
|
|
87
|
+
export interface WorkflowRun {
|
|
88
|
+
id: number;
|
|
89
|
+
name: string;
|
|
90
|
+
/** 1 for the first attempt; a re-run raises it. */
|
|
91
|
+
attempt: number;
|
|
92
|
+
/** `completed` once every job of the attempt has finished. */
|
|
93
|
+
status: string;
|
|
94
|
+
conclusion: string | null;
|
|
95
|
+
url: string | null;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* Whether a forge call's error says to look again rather than that the
|
|
100
|
+
* answer is no (#4729): an answer of 5xx or 429, or a request that never got
|
|
101
|
+
* an answer (fetch throws a TypeError when the network fails).
|
|
102
|
+
*/
|
|
103
|
+
export function isTransientForgeError(err: unknown): boolean {
|
|
104
|
+
if (err instanceof ForgeApiError) return err.status >= 500 || err.status === 429;
|
|
105
|
+
return err instanceof TypeError;
|
|
72
106
|
}
|
|
73
107
|
|
|
74
108
|
export interface GithubCiForgeOptions {
|
|
@@ -89,6 +123,7 @@ export function githubCiForge(opts: GithubCiForgeOptions): CiForge {
|
|
|
89
123
|
const fetchImpl = opts.fetch ?? (globalThis.fetch as unknown as ForgeFetch);
|
|
90
124
|
const headers = { Authorization: `Bearer ${opts.token}`, Accept: "application/vnd.github+json" };
|
|
91
125
|
const rerunRuns = new Set<number>();
|
|
126
|
+
const runOfJob = new Map<number, number>();
|
|
92
127
|
return {
|
|
93
128
|
kind: "github",
|
|
94
129
|
async checkRuns(sha) {
|
|
@@ -140,8 +175,43 @@ export function githubCiForge(opts: GithubCiForgeOptions): CiForge {
|
|
|
140
175
|
async dispatch(workflow, ref) {
|
|
141
176
|
await post(`${root}/repos/${opts.repo}/actions/workflows/${encodeURIComponent(workflow)}/dispatches`, { ref });
|
|
142
177
|
},
|
|
178
|
+
// A job's workflow run never changes, so it is asked once; the run itself
|
|
179
|
+
// is read each time, for its status and attempt. A check run that is no
|
|
180
|
+
// Actions job (another app's) answers 404 here.
|
|
181
|
+
async workflowRun(checkRunId) {
|
|
182
|
+
let runId = runOfJob.get(checkRunId);
|
|
183
|
+
if (runId === undefined) {
|
|
184
|
+
const job = await get(`${root}/repos/${opts.repo}/actions/jobs/${checkRunId}`);
|
|
185
|
+
if (job === null) return null;
|
|
186
|
+
if (!isObject(job) || typeof job.run_id !== "number") return null;
|
|
187
|
+
runId = job.run_id;
|
|
188
|
+
runOfJob.set(checkRunId, runId);
|
|
189
|
+
}
|
|
190
|
+
const run = await get(`${root}/repos/${opts.repo}/actions/runs/${runId}`);
|
|
191
|
+
if (!isObject(run)) return null;
|
|
192
|
+
return {
|
|
193
|
+
id: runId,
|
|
194
|
+
name: str(run.name) ?? String(runId),
|
|
195
|
+
attempt: typeof run.run_attempt === "number" ? run.run_attempt : 1,
|
|
196
|
+
status: str(run.status) ?? "queued",
|
|
197
|
+
conclusion: str(run.conclusion),
|
|
198
|
+
url: str(run.html_url),
|
|
199
|
+
};
|
|
200
|
+
},
|
|
201
|
+
async rerunFailedJobs(workflowRunId) {
|
|
202
|
+
await post(`${root}/repos/${opts.repo}/actions/runs/${workflowRunId}/rerun-failed-jobs`, {});
|
|
203
|
+
},
|
|
143
204
|
};
|
|
144
205
|
|
|
206
|
+
/** The parsed body, or null on a 404. */
|
|
207
|
+
async function get(url: string): Promise<unknown> {
|
|
208
|
+
const res = await fetchImpl(url, { method: "GET", headers });
|
|
209
|
+
const text = await res.text();
|
|
210
|
+
if (res.status === 404) return null;
|
|
211
|
+
if (!res.ok) throw new ForgeApiError("GET", url, res.status, text);
|
|
212
|
+
return text ? JSON.parse(text) : null;
|
|
213
|
+
}
|
|
214
|
+
|
|
145
215
|
async function post(url: string, body: unknown): Promise<void> {
|
|
146
216
|
const res = await fetchImpl(url, { method: "POST", headers: { ...headers, "Content-Type": "application/json" }, body: JSON.stringify(body) });
|
|
147
217
|
if (!res.ok) throw new ForgeApiError("POST", url, res.status, await res.text());
|
|
@@ -62,6 +62,10 @@ export const CI_GREEN_WORKFLOW_NAME = "chant-ci-green";
|
|
|
62
62
|
export const MERGE_QUEUE_WORKFLOW_FILE = ".github/workflows/chant-merge-queue.yml";
|
|
63
63
|
export const MERGE_QUEUE_WORKFLOW_NAME = "chant-merge-queue";
|
|
64
64
|
export const MERGE_QUEUE_VERDICT_JOB = "chant-verdict";
|
|
65
|
+
/** The verdict job's `timeout-minutes` (#4644). */
|
|
66
|
+
export const MERGE_QUEUE_JOB_TIMEOUT_MINUTES = 120;
|
|
67
|
+
/** The `--timeout-minutes` the verdict job passes with retries (#4729): it gives up with a message before Actions kills the job. */
|
|
68
|
+
export const MERGE_QUEUE_VERDICT_TIMEOUT_MINUTES = MERGE_QUEUE_JOB_TIMEOUT_MINUTES - 5;
|
|
65
69
|
|
|
66
70
|
/** One workflow file as the scan reads it. */
|
|
67
71
|
export interface WorkflowSource {
|
|
@@ -306,18 +310,25 @@ export function renderCiGreenWorkflow(input: WorkflowInput): string {
|
|
|
306
310
|
* The merge queue's verdict workflow (#4644): on each group, `chant ci
|
|
307
311
|
* verdict --wait` judges the group's commit by `ci.green`'s required phases
|
|
308
312
|
* as the tick would, and its job, `chant-verdict`, is the one check the
|
|
309
|
-
* queue's ruleset requires. It reads check runs and writes nothing
|
|
313
|
+
* queue's ruleset requires. It reads check runs and writes nothing, unless
|
|
314
|
+
* `ci.green.mergeQueue.retries` is above 0 (#4729): then it re-runs a failed
|
|
315
|
+
* job's workflow run, so the job gets `actions: write`, and passes
|
|
316
|
+
* `--timeout-minutes` to give up before the job's timeout. With retries 0 the
|
|
317
|
+
* file is as #4644 wrote it.
|
|
310
318
|
*/
|
|
311
319
|
export function renderMergeQueueWorkflow(input: Omit<WorkflowInput, "workflows" | "push" | "runnerKeySecret" | "compose">): string {
|
|
312
320
|
const where = input.workspaceRoot === "." ? [] : [` working-directory: ${q(input.workspaceRoot)}`];
|
|
313
321
|
const cache = input.install && input.npmLock ? [" cache: npm", ...(input.npmLock === "package-lock.json" ? [] : [` cache-dependency-path: ${q(input.npmLock)}`])] : [];
|
|
314
|
-
const
|
|
322
|
+
const retries = (input.green.mergeQueue?.retries ?? 0) > 0;
|
|
323
|
+
const args = retries ? ` --timeout-minutes ${MERGE_QUEUE_VERDICT_TIMEOUT_MINUTES}` : "";
|
|
324
|
+
const verdict = input.chant ? q(`${input.chant} ci verdict --wait${args}`) : `npx --yes @intentius/chant@${input.chantVersion} ci verdict --wait${args}`;
|
|
315
325
|
const lines = [
|
|
316
326
|
`name: ${MERGE_QUEUE_WORKFLOW_NAME}`,
|
|
317
327
|
"",
|
|
318
328
|
"# The merge queue's verdict: passes when every phase ci.green requires passed",
|
|
319
329
|
"# on the group's commit, judged as chant ci tick judges. Require its job,",
|
|
320
330
|
`# ${MERGE_QUEUE_VERDICT_JOB}, in the branch's ruleset.`,
|
|
331
|
+
...(retries ? [`# A failed job's workflow run is re-run up to ${input.green.mergeQueue!.retries} time${input.green.mergeQueue!.retries === 1 ? "" : "s"} first (ci.green.mergeQueue.retries).`] : []),
|
|
321
332
|
"on:",
|
|
322
333
|
" merge_group:",
|
|
323
334
|
" types:",
|
|
@@ -326,11 +337,12 @@ export function renderMergeQueueWorkflow(input: Omit<WorkflowInput, "workflows"
|
|
|
326
337
|
"permissions:",
|
|
327
338
|
" contents: read",
|
|
328
339
|
" checks: read",
|
|
340
|
+
...(retries ? [" actions: write"] : []),
|
|
329
341
|
"",
|
|
330
342
|
"jobs:",
|
|
331
343
|
` ${MERGE_QUEUE_VERDICT_JOB}:`,
|
|
332
344
|
" runs-on: ubuntu-latest",
|
|
333
|
-
|
|
345
|
+
` timeout-minutes: ${MERGE_QUEUE_JOB_TIMEOUT_MINUTES}`,
|
|
334
346
|
" steps:",
|
|
335
347
|
` - uses: ${usesRef(ACTION_PINS.checkout6)}`,
|
|
336
348
|
` - uses: ${usesRef(ACTION_PINS.setupNode6)}`,
|
|
@@ -58,7 +58,7 @@ import { execFileSync } from "node:child_process";
|
|
|
58
58
|
import { greenBranches, laneOf, laneTagPrefixes, readDeclaration, readerVersion, WORKSPACE_ERROR_CODES, WorkspaceReadError, type CiGreen, type CiPhase, type CiSource } from "./declaration";
|
|
59
59
|
import { makeNotifyEvent, sendNotify, type NotifyFields } from "../notify";
|
|
60
60
|
import { lifecycleRemoteOverride } from "../lifecycle/ledger-remote";
|
|
61
|
-
import type
|
|
61
|
+
import { isTransientForgeError, type CheckRun, type CiForge, type OpRunRef, type WorkflowRun } from "./ci-green-forge";
|
|
62
62
|
import type { OpRunRecord } from "../op/runtime";
|
|
63
63
|
import { readRunLedger } from "../lifecycle/run-ledger";
|
|
64
64
|
import { fetchLifecycleStatus, listFilesInDir, listLedgerTrees, pushLifecycleStatus, writeLedgerFiles } from "../lifecycle/git";
|
|
@@ -90,6 +90,8 @@ export const CI_STALE_NOTIFY_FLOOR = "0.124.63";
|
|
|
90
90
|
export const CI_STALE_WALK = 200;
|
|
91
91
|
/** The first chant that writes a merge queue's trigger and verdict job, and reads `chant ci verdict` (#4644). */
|
|
92
92
|
export const CI_MERGE_QUEUE_FLOOR = "0.124.61";
|
|
93
|
+
/** The first chant that reads `ci.green.mergeQueue: { retries }` and whose `ci verdict` re-runs a failed job and looks again after a forge 5xx (#4729). */
|
|
94
|
+
export const CI_VERDICT_RETRIES_FLOOR = "0.124.70";
|
|
93
95
|
|
|
94
96
|
export const CI_LAST_GREEN_CONTRACT_VERSION = 1;
|
|
95
97
|
export const CI_LAST_GREEN_SCHEMA_ID = "https://intentius.io/chant/schemas/workspace/ci-last-green/v1/ci-last-green.schema.json";
|
|
@@ -1550,26 +1552,149 @@ export function lastGreen(req: { cwd: string; remote?: string; from?: string; br
|
|
|
1550
1552
|
|
|
1551
1553
|
/** How long `ci verdict --wait` waits between reads of the check runs. */
|
|
1552
1554
|
export const CI_VERDICT_INTERVAL_MS = 30_000;
|
|
1555
|
+
/** How many forge errors in a row (5xx, 429, no answer) `ci verdict --wait` looks past before it fails (#4729). */
|
|
1556
|
+
export const CI_VERDICT_FORGE_ERRORS = 8;
|
|
1557
|
+
/** The wait after the first forge error in a row (#4729); it doubles with each one after, up to {@link CI_VERDICT_BACKOFF_MAX_MS}. */
|
|
1558
|
+
export const CI_VERDICT_BACKOFF_MS = 15_000;
|
|
1559
|
+
export const CI_VERDICT_BACKOFF_MAX_MS = 120_000;
|
|
1560
|
+
|
|
1561
|
+
/** `ci verdict --wait` stopped without a verdict (#4729): out of time, or the forge kept failing. */
|
|
1562
|
+
export class CiVerdictGaveUpError extends Error {
|
|
1563
|
+
constructor(message: string) {
|
|
1564
|
+
super(message);
|
|
1565
|
+
this.name = "CiVerdictGaveUpError";
|
|
1566
|
+
}
|
|
1567
|
+
}
|
|
1568
|
+
|
|
1569
|
+
/** A failed job whose workflow run failed again on its last allowed attempt (#4729). */
|
|
1570
|
+
export interface VerdictRepeat {
|
|
1571
|
+
phase: string;
|
|
1572
|
+
run: RunVerdict;
|
|
1573
|
+
workflowRun: WorkflowRun;
|
|
1574
|
+
}
|
|
1553
1575
|
|
|
1554
1576
|
/**
|
|
1555
1577
|
* `chant ci verdict` (#4644): judge one commit by ci.green's required phases
|
|
1556
1578
|
* from its check runs, as the tick judges a commit, leaving out the
|
|
1557
1579
|
* verdict's own check run (`chant-verdict`). With `wait`, read again until no
|
|
1558
1580
|
* phase is pending. A merge queue's verdict job runs it on the group's commit.
|
|
1581
|
+
*
|
|
1582
|
+
* With `wait`, two more things (#4729). A forge error that says to look again
|
|
1583
|
+
* (5xx, 429, no answer) is looked past, with a backoff, up to
|
|
1584
|
+
* {@link CI_VERDICT_FORGE_ERRORS} in a row. And with `retries` above 0 (from
|
|
1585
|
+
* the call, else `ci.green.mergeQueue.retries`), a failed job doesn't fail
|
|
1586
|
+
* the commit at once: once the job's workflow run has finished (a run can be
|
|
1587
|
+
* re-run only then), its failed jobs are re-run while its attempt is no more
|
|
1588
|
+
* than `retries`, and the commit fails when the run fails on attempt
|
|
1589
|
+
* `retries + 1`, returned as `repeated`. With retries 0 the first failed job
|
|
1590
|
+
* fails it, as before. `timeoutMs` bounds the wait: past it, the verdict
|
|
1591
|
+
* throws {@link CiVerdictGaveUpError} saying what it was waiting for.
|
|
1559
1592
|
*/
|
|
1560
|
-
export async function ciVerdict(req: {
|
|
1593
|
+
export async function ciVerdict(req: {
|
|
1594
|
+
cwd: string;
|
|
1595
|
+
sha: string;
|
|
1596
|
+
forge: CiForge;
|
|
1597
|
+
wait?: boolean;
|
|
1598
|
+
branch?: string;
|
|
1599
|
+
retries?: number;
|
|
1600
|
+
timeoutMs?: number;
|
|
1601
|
+
sleep?: (ms: number) => Promise<void>;
|
|
1602
|
+
now?: () => number;
|
|
1603
|
+
onPending?: (v: CommitVerdict) => void;
|
|
1604
|
+
/** A line about a re-run, a run it waits on, or a forge error it looks past. */
|
|
1605
|
+
onNote?: (line: string) => void;
|
|
1606
|
+
}): Promise<{ sha: string; verdict: CommitVerdict; repeated?: VerdictRepeat }> {
|
|
1561
1607
|
const read = readGreen(req.cwd);
|
|
1562
1608
|
const green = greenOn(read.green, req.branch);
|
|
1563
1609
|
const sleep = req.sleep ?? ((ms: number) => new Promise<void>((r) => setTimeout(r, ms)));
|
|
1610
|
+
const now = req.now ?? Date.now;
|
|
1611
|
+
const retries = req.wait ? (req.retries ?? green.mergeQueue?.retries ?? 0) : 0;
|
|
1612
|
+
const deadline = req.wait && req.timeoutMs !== undefined ? now() + req.timeoutMs : Infinity;
|
|
1613
|
+
const note = req.onNote ?? (() => {});
|
|
1614
|
+
let errors = 0;
|
|
1615
|
+
let waitingOn = "the first read of the check runs";
|
|
1616
|
+
// Each workflow run re-run, at the attempt it was on when asked: until GitHub starts the new attempt, it isn't asked again.
|
|
1617
|
+
const asked = new Map<number, number>();
|
|
1618
|
+
const told = new Set<string>();
|
|
1619
|
+
const once = (line: string) => {
|
|
1620
|
+
if (told.has(line)) return;
|
|
1621
|
+
told.add(line);
|
|
1622
|
+
note(line);
|
|
1623
|
+
};
|
|
1624
|
+
const pause = async (ms: number) => {
|
|
1625
|
+
const left = deadline - now();
|
|
1626
|
+
if (left <= 0) throw new CiVerdictGaveUpError(`no verdict after ${minutes(req.timeoutMs!)}: ${waitingOn}`);
|
|
1627
|
+
await sleep(Math.min(ms, left));
|
|
1628
|
+
};
|
|
1564
1629
|
for (;;) {
|
|
1630
|
+
let step: { done: { verdict: CommitVerdict; repeated?: VerdictRepeat } } | { verdict: CommitVerdict; waiting: boolean };
|
|
1631
|
+
try {
|
|
1632
|
+
step = await look();
|
|
1633
|
+
} catch (err) {
|
|
1634
|
+
if (!req.wait || !isTransientForgeError(err)) throw err;
|
|
1635
|
+
errors++;
|
|
1636
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
1637
|
+
if (errors >= CI_VERDICT_FORGE_ERRORS) throw new CiVerdictGaveUpError(`the forge failed ${errors} times in a row, the last: ${message}`);
|
|
1638
|
+
const backoff = Math.min(CI_VERDICT_BACKOFF_MS * 2 ** (errors - 1), CI_VERDICT_BACKOFF_MAX_MS);
|
|
1639
|
+
note(`the forge failed (${message}); looking again in ${Math.round(backoff / 1000)}s (${errors} of ${CI_VERDICT_FORGE_ERRORS - 1} looked past)`);
|
|
1640
|
+
waitingOn = `the forge failing: ${message}`;
|
|
1641
|
+
await pause(backoff);
|
|
1642
|
+
continue;
|
|
1643
|
+
}
|
|
1644
|
+
errors = 0;
|
|
1645
|
+
if ("done" in step) return { sha: req.sha, ...step.done };
|
|
1646
|
+
if (!step.waiting) req.onPending?.(step.verdict);
|
|
1647
|
+
await pause(CI_VERDICT_INTERVAL_MS);
|
|
1648
|
+
}
|
|
1649
|
+
|
|
1650
|
+
async function look(): Promise<{ done: { verdict: CommitVerdict; repeated?: VerdictRepeat } } | { verdict: CommitVerdict; waiting: boolean }> {
|
|
1565
1651
|
const runs = (await req.forge.checkRuns(req.sha)).filter((r) => r.name !== MERGE_QUEUE_VERDICT_JOB);
|
|
1566
1652
|
const verdict = evaluateCommit(green, runs);
|
|
1567
|
-
if (!req.wait
|
|
1568
|
-
|
|
1569
|
-
|
|
1653
|
+
if (!req.wait) return { done: { verdict } };
|
|
1654
|
+
if (verdict.verdict === "pending") {
|
|
1655
|
+
waitingOn = `still waiting on ${verdict.phases.filter((p) => p.verdict === "pending").map((p) => p.phase).join(", ")}`;
|
|
1656
|
+
return { verdict, waiting: false };
|
|
1657
|
+
}
|
|
1658
|
+
if (verdict.verdict !== "fail" || retries === 0 || !req.forge.workflowRun || !req.forge.rerunFailedJobs) return { done: { verdict } };
|
|
1659
|
+
// #4729: every failed run of a required phase, each job once.
|
|
1660
|
+
const failed: { phase: string; run: RunVerdict }[] = [];
|
|
1661
|
+
for (const p of verdict.phases) for (const r of p.runs) if (r.verdict === "fail" && !failed.some((f) => f.run.id === r.id)) failed.push({ phase: p.phase, run: r });
|
|
1662
|
+
const waits: string[] = [];
|
|
1663
|
+
for (const f of failed) {
|
|
1664
|
+
const wr = await req.forge.workflowRun(f.run.id);
|
|
1665
|
+
// Another app's check run: nothing chant can re-run.
|
|
1666
|
+
if (wr === null) return { done: { verdict } };
|
|
1667
|
+
const what = `${f.run.name} (phase ${f.phase}) in workflow run ${wr.name}${wr.url ? ` ${wr.url}` : ""}`;
|
|
1668
|
+
if (wr.status !== "completed") {
|
|
1669
|
+
once(`${f.run.name} failed (phase ${f.phase}); waiting for workflow run ${wr.name} to finish before re-running it`);
|
|
1670
|
+
waits.push(`workflow run ${wr.name} to finish`);
|
|
1671
|
+
continue;
|
|
1672
|
+
}
|
|
1673
|
+
if (wr.attempt > retries) return { done: { verdict, repeated: { phase: f.phase, run: f.run, workflowRun: wr } } };
|
|
1674
|
+
if (asked.get(wr.id) === wr.attempt) {
|
|
1675
|
+
waits.push(`the re-run of workflow run ${wr.name} to start`);
|
|
1676
|
+
continue;
|
|
1677
|
+
}
|
|
1678
|
+
note(`${what} failed on attempt ${wr.attempt} of ${retries + 1}; re-running its failed jobs`);
|
|
1679
|
+
try {
|
|
1680
|
+
await req.forge.rerunFailedJobs(wr.id);
|
|
1681
|
+
} catch (err) {
|
|
1682
|
+
if (isTransientForgeError(err)) throw err;
|
|
1683
|
+
throw new Error(`could not re-run the failed jobs of workflow run ${wr.name} (${wr.id}): ${err instanceof Error ? err.message : String(err)}. The verdict's job needs actions: write, which chant ci workflow gives it when ci.green.mergeQueue.retries is above 0`);
|
|
1684
|
+
}
|
|
1685
|
+
asked.set(wr.id, wr.attempt);
|
|
1686
|
+
waits.push(`the re-run of workflow run ${wr.name}`);
|
|
1687
|
+
}
|
|
1688
|
+
waitingOn = `waiting for ${[...new Set(waits)].join(", ")}`;
|
|
1689
|
+
return { verdict, waiting: true };
|
|
1570
1690
|
}
|
|
1571
1691
|
}
|
|
1572
1692
|
|
|
1693
|
+
function minutes(ms: number): string {
|
|
1694
|
+
const m = ms / 60_000;
|
|
1695
|
+
return `${Number.isInteger(m) ? m : m.toFixed(1)} minute${m === 1 ? "" : "s"}`;
|
|
1696
|
+
}
|
|
1697
|
+
|
|
1573
1698
|
// ── base ─────────────────────────────────────────────────────────────────────
|
|
1574
1699
|
|
|
1575
1700
|
/**
|