@intentius/chant 0.124.69 → 0.124.71

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (170) hide show
  1. package/dist/ci-agent.d.ts +12 -1
  2. package/dist/ci-agent.d.ts.map +1 -1
  3. package/dist/ci-agent.js +13 -8
  4. package/dist/ci-commit.d.ts +297 -0
  5. package/dist/ci-commit.d.ts.map +1 -0
  6. package/dist/ci-commit.js +696 -0
  7. package/dist/ci-product-words.d.ts +6 -1
  8. package/dist/ci-product-words.d.ts.map +1 -1
  9. package/dist/ci-product-words.js +3 -1
  10. package/dist/cli/handlers/approve.d.ts.map +1 -1
  11. package/dist/cli/handlers/approve.js +7 -2
  12. package/dist/cli/handlers/commit.d.ts +26 -0
  13. package/dist/cli/handlers/commit.d.ts.map +1 -0
  14. package/dist/cli/handlers/commit.js +142 -0
  15. package/dist/cli/handlers/pr.d.ts +6 -0
  16. package/dist/cli/handlers/pr.d.ts.map +1 -1
  17. package/dist/cli/handlers/pr.js +16 -7
  18. package/dist/cli/handlers/release.d.ts +9 -0
  19. package/dist/cli/handlers/release.d.ts.map +1 -1
  20. package/dist/cli/handlers/release.js +114 -5
  21. package/dist/cli/main.d.ts.map +1 -1
  22. package/dist/cli/main.js +23 -0
  23. package/dist/cli/parse-args.d.ts.map +1 -1
  24. package/dist/cli/parse-args.js +15 -0
  25. package/dist/cli/registry.d.ts +4 -0
  26. package/dist/cli/registry.d.ts.map +1 -1
  27. package/dist/components/product-pipeline.d.ts +33 -0
  28. package/dist/components/product-pipeline.d.ts.map +1 -1
  29. package/dist/components/product-pipeline.js +34 -1
  30. package/dist/components/release-compile.d.ts +2 -0
  31. package/dist/components/release-compile.d.ts.map +1 -1
  32. package/dist/components/release-compile.js +8 -4
  33. package/dist/config.d.ts +46 -0
  34. package/dist/config.d.ts.map +1 -1
  35. package/dist/config.js +9 -0
  36. package/dist/lexicon.d.ts +37 -0
  37. package/dist/lexicon.d.ts.map +1 -1
  38. package/dist/lexicon.js +28 -0
  39. package/dist/lifecycle/__fixtures__/publish-fakes.d.ts +4 -0
  40. package/dist/lifecycle/__fixtures__/publish-fakes.d.ts.map +1 -1
  41. package/dist/lifecycle/__fixtures__/publish-fakes.js +6 -2
  42. package/dist/lifecycle/gated-request.d.ts +3 -1
  43. package/dist/lifecycle/gated-request.d.ts.map +1 -1
  44. package/dist/lifecycle/gated-request.js +5 -2
  45. package/dist/lifecycle/git.d.ts.map +1 -1
  46. package/dist/lifecycle/git.js +57 -0
  47. package/dist/lifecycle/publish-declaration.d.ts +16 -0
  48. package/dist/lifecycle/publish-declaration.d.ts.map +1 -1
  49. package/dist/lifecycle/publish-declaration.js +35 -3
  50. package/dist/lifecycle/publish-forge.d.ts +12 -0
  51. package/dist/lifecycle/publish-forge.d.ts.map +1 -1
  52. package/dist/lifecycle/publish-forge.js +103 -19
  53. package/dist/lifecycle/publish-npm.d.ts +15 -0
  54. package/dist/lifecycle/publish-npm.d.ts.map +1 -1
  55. package/dist/lifecycle/publish-npm.js +28 -1
  56. package/dist/lifecycle/publish-oci.d.ts +2 -0
  57. package/dist/lifecycle/publish-oci.d.ts.map +1 -1
  58. package/dist/lifecycle/publish-oci.js +10 -0
  59. package/dist/lifecycle/release-cut.d.ts +7 -1
  60. package/dist/lifecycle/release-cut.d.ts.map +1 -1
  61. package/dist/lifecycle/release-cut.js +8 -1
  62. package/dist/lifecycle/release-images.d.ts +87 -0
  63. package/dist/lifecycle/release-images.d.ts.map +1 -0
  64. package/dist/lifecycle/release-images.js +139 -0
  65. package/dist/lifecycle/release-publish.d.ts +2 -0
  66. package/dist/lifecycle/release-publish.d.ts.map +1 -1
  67. package/dist/op/activities/activity-contracts.d.ts +4 -4
  68. package/dist/op/activities/review-agent.d.ts.map +1 -1
  69. package/dist/op/activities/review-agent.js +4 -2
  70. package/dist/op/gate.d.ts +5 -1
  71. package/dist/op/gate.d.ts.map +1 -1
  72. package/dist/op/gate.js +7 -2
  73. package/dist/op/op-waves-notify.d.ts +1 -1
  74. package/dist/op/op-waves-notify.d.ts.map +1 -1
  75. package/dist/op/op-waves-notify.js +2 -1
  76. package/dist/op/op-waves-run.d.ts.map +1 -1
  77. package/dist/op/op-waves-run.js +12 -8
  78. package/dist/op/op-waves.d.ts +9 -0
  79. package/dist/op/op-waves.d.ts.map +1 -1
  80. package/dist/op/op-waves.js +11 -0
  81. package/dist/pr-forge-base.d.ts +18 -0
  82. package/dist/pr-forge-base.d.ts.map +1 -0
  83. package/dist/pr-forge-base.js +29 -0
  84. package/dist/release-floors.d.ts +1 -1
  85. package/dist/release-floors.js +1 -1
  86. package/dist/wave-runtime.mjs +305 -305
  87. package/dist/workspace/ci-green-cli.d.ts +7 -3
  88. package/dist/workspace/ci-green-cli.d.ts.map +1 -1
  89. package/dist/workspace/ci-green-cli.js +29 -8
  90. package/dist/workspace/ci-green-forge.d.ts +29 -0
  91. package/dist/workspace/ci-green-forge.d.ts.map +1 -1
  92. package/dist/workspace/ci-green-forge.js +50 -0
  93. package/dist/workspace/ci-green-workflow.d.ts +9 -1
  94. package/dist/workspace/ci-green-workflow.d.ts.map +1 -1
  95. package/dist/workspace/ci-green-workflow.js +15 -3
  96. package/dist/workspace/ci-green.d.ts +35 -1
  97. package/dist/workspace/ci-green.d.ts.map +1 -1
  98. package/dist/workspace/ci-green.js +122 -4
  99. package/dist/workspace/declaration.d.ts +14 -2
  100. package/dist/workspace/declaration.d.ts.map +1 -1
  101. package/dist/workspace/declaration.js +14 -1
  102. package/dist/workspace/declaration.schema.json +17 -2
  103. package/dist/workspace/release-workflow.d.ts +23 -0
  104. package/dist/workspace/release-workflow.d.ts.map +1 -1
  105. package/dist/workspace/release-workflow.js +203 -12
  106. package/package.json +1 -1
  107. package/src/ci-agent.ts +13 -8
  108. package/src/ci-commit.test.ts +415 -0
  109. package/src/ci-commit.ts +857 -0
  110. package/src/ci-product-words.ts +6 -2
  111. package/src/cli/handlers/approve.ts +6 -2
  112. package/src/cli/handlers/commit.ts +152 -0
  113. package/src/cli/handlers/operator.test.ts +2 -0
  114. package/src/cli/handlers/pr-wave-approve-lines.test.ts +32 -0
  115. package/src/cli/handlers/pr.ts +17 -7
  116. package/src/cli/handlers/release.ts +110 -5
  117. package/src/cli/main.ts +23 -0
  118. package/src/cli/parse-args.ts +11 -0
  119. package/src/cli/registry.ts +4 -0
  120. package/src/components/product-pipeline.test.ts +28 -1
  121. package/src/components/product-pipeline.ts +54 -1
  122. package/src/components/release-compile.ts +9 -3
  123. package/src/config.ts +24 -0
  124. package/src/lexicon.ts +56 -0
  125. package/src/lifecycle/__fixtures__/publish-fakes.ts +11 -5
  126. package/src/lifecycle/gated-request-spend.test.ts +115 -0
  127. package/src/lifecycle/gated-request.ts +5 -2
  128. package/src/lifecycle/git.ts +53 -0
  129. package/src/lifecycle/ledger-two-clones.test.ts +132 -0
  130. package/src/lifecycle/publish-declaration.ts +35 -3
  131. package/src/lifecycle/publish-forge-notes.test.ts +193 -0
  132. package/src/lifecycle/publish-forge.ts +101 -22
  133. package/src/lifecycle/publish-npm.ts +31 -2
  134. package/src/lifecycle/publish-oci.ts +10 -0
  135. package/src/lifecycle/publish-targets.test.ts +2 -1
  136. package/src/lifecycle/release-cut.ts +17 -1
  137. package/src/lifecycle/release-images.test.ts +242 -0
  138. package/src/lifecycle/release-images.ts +175 -0
  139. package/src/lifecycle/release-publish.ts +2 -0
  140. package/src/op/activities/review-agent.test.ts +20 -1
  141. package/src/op/activities/review-agent.ts +5 -2
  142. package/src/op/gate.ts +7 -2
  143. package/src/op/op-waves-approve-target.test.ts +108 -0
  144. package/src/op/op-waves-notify.ts +2 -2
  145. package/src/op/op-waves-run.ts +12 -7
  146. package/src/op/op-waves.ts +12 -0
  147. package/src/pr-forge-base.test.ts +37 -0
  148. package/src/pr-forge-base.ts +36 -0
  149. package/src/release-floors.ts +1 -1
  150. package/src/workspace/__fixtures__/release-workflow/forgejo-plain-member.yml +26 -0
  151. package/src/workspace/__fixtures__/release-workflow/forgejo-plain.yml +23 -0
  152. package/src/workspace/__fixtures__/release-workflow/forgejo-secure-member.yml +27 -0
  153. package/src/workspace/__fixtures__/release-workflow/forgejo-secure.yml +24 -0
  154. package/src/workspace/__fixtures__/release-workflow/github-plain-member.yml +26 -0
  155. package/src/workspace/__fixtures__/release-workflow/github-plain.yml +23 -0
  156. package/src/workspace/__fixtures__/release-workflow/github-secure-member.yml +27 -0
  157. package/src/workspace/__fixtures__/release-workflow/github-secure.yml +24 -0
  158. package/src/workspace/__fixtures__/release-workflow/gitlab-plain-member.yml +13 -0
  159. package/src/workspace/__fixtures__/release-workflow/gitlab-plain.yml +12 -0
  160. package/src/workspace/__fixtures__/release-workflow/gitlab-secure-member.yml +14 -0
  161. package/src/workspace/__fixtures__/release-workflow/gitlab-secure.yml +13 -0
  162. package/src/workspace/ci-green-cli.ts +27 -8
  163. package/src/workspace/ci-green-forge.ts +70 -0
  164. package/src/workspace/ci-green-workflow.ts +15 -3
  165. package/src/workspace/ci-green.ts +130 -5
  166. package/src/workspace/ci-verdict-retries.test.ts +376 -0
  167. package/src/workspace/declaration.schema.json +17 -2
  168. package/src/workspace/declaration.ts +28 -4
  169. package/src/workspace/release-workflow.test.ts +151 -0
  170. package/src/workspace/release-workflow.ts +270 -12
@@ -31,7 +31,7 @@ import { join, relative, resolve, sep } from "node:path";
31
31
  import { formatError } from "../cli/format";
32
32
  import type { CommandContext } from "../cli/registry";
33
33
  import { findWorkspaceRoot } from "../project-root";
34
- import { ciBase, ciList, ciTick, ciVerdict, CiGreenError, formatList, formatTick, lastGreen, RUNNER_KEY_ENV } from "./ci-green";
34
+ import { ciBase, ciList, ciTick, ciVerdict, CiGreenError, CiVerdictGaveUpError, formatList, formatTick, lastGreen, RUNNER_KEY_ENV } from "./ci-green";
35
35
  import { ciForgeFromEnv } from "./ci-green-forge";
36
36
  import { ACTIONS_NAME, buildTriggerEdits, CI_GREEN_WORKFLOW_FILE, ciGreenHeader, MERGE_QUEUE_VERDICT_JOB, MERGE_QUEUE_WORKFLOW_FILE, type PushToken, readWorkflowSources, renderCiGreenWorkflow, renderMergeQueueWorkflow, workflowsHolding, writeWorkflowFile } from "./ci-green-workflow";
37
37
  import { greenBranches, readDeclaration, readerVersion, WorkspaceReadError, type CiSource } from "./declaration";
@@ -39,7 +39,7 @@ import { recordGeneratedFiles, resolveMemberContext, shellArg } from "./member-p
39
39
  import { gitTop, workingTree } from "./tree";
40
40
 
41
41
  const USAGE =
42
- "chant ci last-green [--json] [--lane <name>] [--from <rev>] [--branch <branch>] | ci base [--json] [--phase <name> | --lane <name>] [--from <rev>] [--branch <branch>] | ci verdict [--commit <sha>] [--wait] | ci list [--json] [--branch <branch>] [--limit <n>] [--since <rev>] [--forge github] [--source checks|runs] | ci tick [--dry-run] [--forge github] [--source checks|runs] [--key <runner key.pem>] | " +
42
+ "chant ci last-green [--json] [--lane <name>] [--from <rev>] [--branch <branch>] | ci base [--json] [--phase <name> | --lane <name>] [--from <rev>] [--branch <branch>] | ci verdict [--commit <sha>] [--wait [--retries <n>] [--timeout-minutes <n>]] | ci list [--json] [--branch <branch>] [--limit <n>] [--since <rev>] [--forge github] [--source checks|runs] | ci tick [--dry-run] [--forge github] [--source checks|runs] [--key <runner key.pem>] | " +
43
43
  "ci workflow [--workflow <name>]... [--output <file>] [--chant <command>] [--install <command>] [--token-secret <NAME> | --app-id-var <VAR> --app-key-secret <NAME>] [--runner-key-secret <NAME>]";
44
44
 
45
45
  export { RUNNER_KEY_ENV };
@@ -107,27 +107,42 @@ export async function runCiBase(ctx: CommandContext): Promise<number> {
107
107
  }
108
108
 
109
109
  /**
110
- * `chant ci verdict [--commit <sha>] [--wait]` (#4644): exits 0 when every
111
- * required phase passed on the commit (GITHUB_SHA, else HEAD), 1 when one
112
- * failed or wasn't built, and 3 when one is still pending without --wait.
110
+ * `chant ci verdict [--commit <sha>] [--wait] [--retries <n>] [--timeout-minutes <n>]`
111
+ * (#4644): exits 0 when every required phase passed on the commit
112
+ * (GITHUB_SHA, else HEAD), 1 when one failed or wasn't built, and 3 when one
113
+ * is still pending without --wait. With --wait (#4729) it looks past forge
114
+ * errors that say to look again, re-runs a failed job's workflow run up to
115
+ * --retries times (else ci.green.mergeQueue.retries), and with
116
+ * --timeout-minutes exits 1 with a message once that time is up.
113
117
  */
114
118
  export async function runCiVerdict(ctx: CommandContext): Promise<number> {
115
119
  try {
120
+ if (!ctx.args.ciWait && (ctx.args.ciRetries !== undefined || ctx.args.ciTimeoutMinutes !== undefined)) {
121
+ throw new Error("--retries and --timeout-minutes go with --wait: without it the verdict reads the check runs once");
122
+ }
116
123
  const sha = ctx.args.commit ?? process.env.GITHUB_SHA ?? execFileSync("git", ["rev-parse", "HEAD"], { encoding: "utf-8" }).trim();
117
124
  const forge = ciForgeFromEnv("github");
118
- const { verdict } = await ciVerdict({
125
+ const { verdict, repeated } = await ciVerdict({
119
126
  cwd: process.cwd(),
120
127
  sha,
121
128
  forge,
122
129
  ...(ctx.args.ciWait ? { wait: true } : {}),
123
130
  ...(ctx.args.branch !== undefined ? { branch: ctx.args.branch } : {}),
131
+ ...(ctx.args.ciRetries !== undefined ? { retries: ctx.args.ciRetries } : {}),
132
+ ...(ctx.args.ciTimeoutMinutes !== undefined ? { timeoutMs: ctx.args.ciTimeoutMinutes * 60_000 } : {}),
124
133
  onPending: (v) => console.error(`waiting on ${v.phases.filter((p) => p.verdict === "pending").map((p) => p.phase).join(", ")}`),
134
+ onNote: (line) => console.error(line),
125
135
  });
126
136
  const short = sha.slice(0, 12);
127
137
  if (verdict.verdict === "pass") {
128
138
  console.log(`${short}: every required phase passed (${verdict.phases.map((p) => p.phase).join(", ")})`);
129
139
  return 0;
130
140
  }
141
+ if (repeated) {
142
+ const wr = repeated.workflowRun;
143
+ console.error(`${short}: phase ${repeated.phase} failed in ${repeated.run.name} (${repeated.run.conclusion}) again on attempt ${wr.attempt} of workflow run ${wr.name}${wr.url ? ` (${wr.url})` : ""}`);
144
+ return 1;
145
+ }
131
146
  if (verdict.verdict === "fail") {
132
147
  console.error(`${short}: phase ${verdict.failure!.phase} failed in ${verdict.failure!.run.name} (${verdict.failure!.run.conclusion})`);
133
148
  return 1;
@@ -136,7 +151,9 @@ export async function runCiVerdict(ctx: CommandContext): Promise<number> {
136
151
  console.error(`${short}: ${verdict.verdict === "pending" ? "still waiting on" : "never built"} ${open.join(", ")}`);
137
152
  return verdict.verdict === "pending" ? CHANT_EXIT.waiting : 1;
138
153
  } catch (err) {
139
- console.error(formatError({ message: `chant ci verdict: ${describe(err)}`, hint: USAGE }));
154
+ // #4729: out of time or out of forge errors to look past says so, without the usage.
155
+ if (err instanceof CiVerdictGaveUpError) console.error(`chant ci verdict: ${err.message}`);
156
+ else console.error(formatError({ message: `chant ci verdict: ${describe(err)}`, hint: USAGE }));
140
157
  return 1;
141
158
  }
142
159
  }
@@ -258,7 +275,9 @@ export async function runCiWorkflow(ctx: CommandContext): Promise<number> {
258
275
  if (green.mergeQueue && !member) {
259
276
  const queue = renderMergeQueueWorkflow({ green, chantVersion: readerVersion(), workspaceRoot, chant: ctx.args.ciChant, install: ctx.args.ciInstall, npmLock });
260
277
  const at = writeWorkflowFile(join(repoRoot, MERGE_QUEUE_WORKFLOW_FILE), ciGreenHeader(command, "root") + queue);
261
- console.log(`Wrote ${at}: on each merge group, ${MERGE_QUEUE_VERDICT_JOB} passes when every required phase passed on the group's commit. Require ${MERGE_QUEUE_VERDICT_JOB} in the branch's ruleset.`);
278
+ const retries = green.mergeQueue.retries;
279
+ const again = retries > 0 ? `, re-running a failed job's workflow run up to ${retries} time${retries === 1 ? "" : "s"} first` : "";
280
+ console.log(`Wrote ${at}: on each merge group, ${MERGE_QUEUE_VERDICT_JOB} passes when every required phase passed on the group's commit${again}. Require ${MERGE_QUEUE_VERDICT_JOB} in the branch's ruleset.`);
262
281
  }
263
282
  for (const e of edits) {
264
283
  if (!e.changed) continue;
@@ -69,6 +69,40 @@ export interface CiForge {
69
69
  * can't ask.
70
70
  */
71
71
  dispatch?(workflow: string, ref: string): Promise<void>;
72
+ /**
73
+ * The workflow run a check run's job belongs to, as it is now, at its
74
+ * latest attempt (#4729), for `ci verdict --retries`. Null when the check
75
+ * run is no Actions job. Absent on a forge chant can't ask.
76
+ */
77
+ workflowRun?(checkRunId: number): Promise<WorkflowRun | null>;
78
+ /**
79
+ * Re-run a workflow run's failed jobs (#4729), each time it is asked,
80
+ * where {@link rerun} asks once per workflow run per client. Needs
81
+ * `actions: write`. Absent on a forge chant can't ask.
82
+ */
83
+ rerunFailedJobs?(workflowRunId: number): Promise<void>;
84
+ }
85
+
86
+ /** A GitHub Actions workflow run at its latest attempt (#4729). */
87
+ export interface WorkflowRun {
88
+ id: number;
89
+ name: string;
90
+ /** 1 for the first attempt; a re-run raises it. */
91
+ attempt: number;
92
+ /** `completed` once every job of the attempt has finished. */
93
+ status: string;
94
+ conclusion: string | null;
95
+ url: string | null;
96
+ }
97
+
98
+ /**
99
+ * Whether a forge call's error says to look again rather than that the
100
+ * answer is no (#4729): an answer of 5xx or 429, or a request that never got
101
+ * an answer (fetch throws a TypeError when the network fails).
102
+ */
103
+ export function isTransientForgeError(err: unknown): boolean {
104
+ if (err instanceof ForgeApiError) return err.status >= 500 || err.status === 429;
105
+ return err instanceof TypeError;
72
106
  }
73
107
 
74
108
  export interface GithubCiForgeOptions {
@@ -89,6 +123,7 @@ export function githubCiForge(opts: GithubCiForgeOptions): CiForge {
89
123
  const fetchImpl = opts.fetch ?? (globalThis.fetch as unknown as ForgeFetch);
90
124
  const headers = { Authorization: `Bearer ${opts.token}`, Accept: "application/vnd.github+json" };
91
125
  const rerunRuns = new Set<number>();
126
+ const runOfJob = new Map<number, number>();
92
127
  return {
93
128
  kind: "github",
94
129
  async checkRuns(sha) {
@@ -140,8 +175,43 @@ export function githubCiForge(opts: GithubCiForgeOptions): CiForge {
140
175
  async dispatch(workflow, ref) {
141
176
  await post(`${root}/repos/${opts.repo}/actions/workflows/${encodeURIComponent(workflow)}/dispatches`, { ref });
142
177
  },
178
+ // A job's workflow run never changes, so it is asked once; the run itself
179
+ // is read each time, for its status and attempt. A check run that is no
180
+ // Actions job (another app's) answers 404 here.
181
+ async workflowRun(checkRunId) {
182
+ let runId = runOfJob.get(checkRunId);
183
+ if (runId === undefined) {
184
+ const job = await get(`${root}/repos/${opts.repo}/actions/jobs/${checkRunId}`);
185
+ if (job === null) return null;
186
+ if (!isObject(job) || typeof job.run_id !== "number") return null;
187
+ runId = job.run_id;
188
+ runOfJob.set(checkRunId, runId);
189
+ }
190
+ const run = await get(`${root}/repos/${opts.repo}/actions/runs/${runId}`);
191
+ if (!isObject(run)) return null;
192
+ return {
193
+ id: runId,
194
+ name: str(run.name) ?? String(runId),
195
+ attempt: typeof run.run_attempt === "number" ? run.run_attempt : 1,
196
+ status: str(run.status) ?? "queued",
197
+ conclusion: str(run.conclusion),
198
+ url: str(run.html_url),
199
+ };
200
+ },
201
+ async rerunFailedJobs(workflowRunId) {
202
+ await post(`${root}/repos/${opts.repo}/actions/runs/${workflowRunId}/rerun-failed-jobs`, {});
203
+ },
143
204
  };
144
205
 
206
+ /** The parsed body, or null on a 404. */
207
+ async function get(url: string): Promise<unknown> {
208
+ const res = await fetchImpl(url, { method: "GET", headers });
209
+ const text = await res.text();
210
+ if (res.status === 404) return null;
211
+ if (!res.ok) throw new ForgeApiError("GET", url, res.status, text);
212
+ return text ? JSON.parse(text) : null;
213
+ }
214
+
145
215
  async function post(url: string, body: unknown): Promise<void> {
146
216
  const res = await fetchImpl(url, { method: "POST", headers: { ...headers, "Content-Type": "application/json" }, body: JSON.stringify(body) });
147
217
  if (!res.ok) throw new ForgeApiError("POST", url, res.status, await res.text());
@@ -62,6 +62,10 @@ export const CI_GREEN_WORKFLOW_NAME = "chant-ci-green";
62
62
  export const MERGE_QUEUE_WORKFLOW_FILE = ".github/workflows/chant-merge-queue.yml";
63
63
  export const MERGE_QUEUE_WORKFLOW_NAME = "chant-merge-queue";
64
64
  export const MERGE_QUEUE_VERDICT_JOB = "chant-verdict";
65
+ /** The verdict job's `timeout-minutes` (#4644). */
66
+ export const MERGE_QUEUE_JOB_TIMEOUT_MINUTES = 120;
67
+ /** The `--timeout-minutes` the verdict job passes with retries (#4729): it gives up with a message before Actions kills the job. */
68
+ export const MERGE_QUEUE_VERDICT_TIMEOUT_MINUTES = MERGE_QUEUE_JOB_TIMEOUT_MINUTES - 5;
65
69
 
66
70
  /** One workflow file as the scan reads it. */
67
71
  export interface WorkflowSource {
@@ -306,18 +310,25 @@ export function renderCiGreenWorkflow(input: WorkflowInput): string {
306
310
  * The merge queue's verdict workflow (#4644): on each group, `chant ci
307
311
  * verdict --wait` judges the group's commit by `ci.green`'s required phases
308
312
  * as the tick would, and its job, `chant-verdict`, is the one check the
309
- * queue's ruleset requires. It reads check runs and writes nothing.
313
+ * queue's ruleset requires. It reads check runs and writes nothing, unless
314
+ * `ci.green.mergeQueue.retries` is above 0 (#4729): then it re-runs a failed
315
+ * job's workflow run, so the job gets `actions: write`, and passes
316
+ * `--timeout-minutes` to give up before the job's timeout. With retries 0 the
317
+ * file is as #4644 wrote it.
310
318
  */
311
319
  export function renderMergeQueueWorkflow(input: Omit<WorkflowInput, "workflows" | "push" | "runnerKeySecret" | "compose">): string {
312
320
  const where = input.workspaceRoot === "." ? [] : [` working-directory: ${q(input.workspaceRoot)}`];
313
321
  const cache = input.install && input.npmLock ? [" cache: npm", ...(input.npmLock === "package-lock.json" ? [] : [` cache-dependency-path: ${q(input.npmLock)}`])] : [];
314
- const verdict = input.chant ? q(`${input.chant} ci verdict --wait`) : `npx --yes @intentius/chant@${input.chantVersion} ci verdict --wait`;
322
+ const retries = (input.green.mergeQueue?.retries ?? 0) > 0;
323
+ const args = retries ? ` --timeout-minutes ${MERGE_QUEUE_VERDICT_TIMEOUT_MINUTES}` : "";
324
+ const verdict = input.chant ? q(`${input.chant} ci verdict --wait${args}`) : `npx --yes @intentius/chant@${input.chantVersion} ci verdict --wait${args}`;
315
325
  const lines = [
316
326
  `name: ${MERGE_QUEUE_WORKFLOW_NAME}`,
317
327
  "",
318
328
  "# The merge queue's verdict: passes when every phase ci.green requires passed",
319
329
  "# on the group's commit, judged as chant ci tick judges. Require its job,",
320
330
  `# ${MERGE_QUEUE_VERDICT_JOB}, in the branch's ruleset.`,
331
+ ...(retries ? [`# A failed job's workflow run is re-run up to ${input.green.mergeQueue!.retries} time${input.green.mergeQueue!.retries === 1 ? "" : "s"} first (ci.green.mergeQueue.retries).`] : []),
321
332
  "on:",
322
333
  " merge_group:",
323
334
  " types:",
@@ -326,11 +337,12 @@ export function renderMergeQueueWorkflow(input: Omit<WorkflowInput, "workflows"
326
337
  "permissions:",
327
338
  " contents: read",
328
339
  " checks: read",
340
+ ...(retries ? [" actions: write"] : []),
329
341
  "",
330
342
  "jobs:",
331
343
  ` ${MERGE_QUEUE_VERDICT_JOB}:`,
332
344
  " runs-on: ubuntu-latest",
333
- " timeout-minutes: 120",
345
+ ` timeout-minutes: ${MERGE_QUEUE_JOB_TIMEOUT_MINUTES}`,
334
346
  " steps:",
335
347
  ` - uses: ${usesRef(ACTION_PINS.checkout6)}`,
336
348
  ` - uses: ${usesRef(ACTION_PINS.setupNode6)}`,
@@ -58,7 +58,7 @@ import { execFileSync } from "node:child_process";
58
58
  import { greenBranches, laneOf, laneTagPrefixes, readDeclaration, readerVersion, WORKSPACE_ERROR_CODES, WorkspaceReadError, type CiGreen, type CiPhase, type CiSource } from "./declaration";
59
59
  import { makeNotifyEvent, sendNotify, type NotifyFields } from "../notify";
60
60
  import { lifecycleRemoteOverride } from "../lifecycle/ledger-remote";
61
- import type { CheckRun, CiForge, OpRunRef } from "./ci-green-forge";
61
+ import { isTransientForgeError, type CheckRun, type CiForge, type OpRunRef, type WorkflowRun } from "./ci-green-forge";
62
62
  import type { OpRunRecord } from "../op/runtime";
63
63
  import { readRunLedger } from "../lifecycle/run-ledger";
64
64
  import { fetchLifecycleStatus, listFilesInDir, listLedgerTrees, pushLifecycleStatus, writeLedgerFiles } from "../lifecycle/git";
@@ -90,6 +90,8 @@ export const CI_STALE_NOTIFY_FLOOR = "0.124.63";
90
90
  export const CI_STALE_WALK = 200;
91
91
  /** The first chant that writes a merge queue's trigger and verdict job, and reads `chant ci verdict` (#4644). */
92
92
  export const CI_MERGE_QUEUE_FLOOR = "0.124.61";
93
+ /** The first chant that reads `ci.green.mergeQueue: { retries }` and whose `ci verdict` re-runs a failed job and looks again after a forge 5xx (#4729). */
94
+ export const CI_VERDICT_RETRIES_FLOOR = "0.124.70";
93
95
 
94
96
  export const CI_LAST_GREEN_CONTRACT_VERSION = 1;
95
97
  export const CI_LAST_GREEN_SCHEMA_ID = "https://intentius.io/chant/schemas/workspace/ci-last-green/v1/ci-last-green.schema.json";
@@ -1550,26 +1552,149 @@ export function lastGreen(req: { cwd: string; remote?: string; from?: string; br
1550
1552
 
1551
1553
  /** How long `ci verdict --wait` waits between reads of the check runs. */
1552
1554
  export const CI_VERDICT_INTERVAL_MS = 30_000;
1555
+ /** How many forge errors in a row (5xx, 429, no answer) `ci verdict --wait` looks past before it fails (#4729). */
1556
+ export const CI_VERDICT_FORGE_ERRORS = 8;
1557
+ /** The wait after the first forge error in a row (#4729); it doubles with each one after, up to {@link CI_VERDICT_BACKOFF_MAX_MS}. */
1558
+ export const CI_VERDICT_BACKOFF_MS = 15_000;
1559
+ export const CI_VERDICT_BACKOFF_MAX_MS = 120_000;
1560
+
1561
+ /** `ci verdict --wait` stopped without a verdict (#4729): out of time, or the forge kept failing. */
1562
+ export class CiVerdictGaveUpError extends Error {
1563
+ constructor(message: string) {
1564
+ super(message);
1565
+ this.name = "CiVerdictGaveUpError";
1566
+ }
1567
+ }
1568
+
1569
+ /** A failed job whose workflow run failed again on its last allowed attempt (#4729). */
1570
+ export interface VerdictRepeat {
1571
+ phase: string;
1572
+ run: RunVerdict;
1573
+ workflowRun: WorkflowRun;
1574
+ }
1553
1575
 
1554
1576
  /**
1555
1577
  * `chant ci verdict` (#4644): judge one commit by ci.green's required phases
1556
1578
  * from its check runs, as the tick judges a commit, leaving out the
1557
1579
  * verdict's own check run (`chant-verdict`). With `wait`, read again until no
1558
1580
  * phase is pending. A merge queue's verdict job runs it on the group's commit.
1581
+ *
1582
+ * With `wait`, two more things (#4729). A forge error that says to look again
1583
+ * (5xx, 429, no answer) is looked past, with a backoff, up to
1584
+ * {@link CI_VERDICT_FORGE_ERRORS} in a row. And with `retries` above 0 (from
1585
+ * the call, else `ci.green.mergeQueue.retries`), a failed job doesn't fail
1586
+ * the commit at once: once the job's workflow run has finished (a run can be
1587
+ * re-run only then), its failed jobs are re-run while its attempt is no more
1588
+ * than `retries`, and the commit fails when the run fails on attempt
1589
+ * `retries + 1`, returned as `repeated`. With retries 0 the first failed job
1590
+ * fails it, as before. `timeoutMs` bounds the wait: past it, the verdict
1591
+ * throws {@link CiVerdictGaveUpError} saying what it was waiting for.
1559
1592
  */
1560
- export async function ciVerdict(req: { cwd: string; sha: string; forge: CiForge; wait?: boolean; branch?: string; sleep?: (ms: number) => Promise<void>; onPending?: (v: CommitVerdict) => void }): Promise<{ sha: string; verdict: CommitVerdict }> {
1593
+ export async function ciVerdict(req: {
1594
+ cwd: string;
1595
+ sha: string;
1596
+ forge: CiForge;
1597
+ wait?: boolean;
1598
+ branch?: string;
1599
+ retries?: number;
1600
+ timeoutMs?: number;
1601
+ sleep?: (ms: number) => Promise<void>;
1602
+ now?: () => number;
1603
+ onPending?: (v: CommitVerdict) => void;
1604
+ /** A line about a re-run, a run it waits on, or a forge error it looks past. */
1605
+ onNote?: (line: string) => void;
1606
+ }): Promise<{ sha: string; verdict: CommitVerdict; repeated?: VerdictRepeat }> {
1561
1607
  const read = readGreen(req.cwd);
1562
1608
  const green = greenOn(read.green, req.branch);
1563
1609
  const sleep = req.sleep ?? ((ms: number) => new Promise<void>((r) => setTimeout(r, ms)));
1610
+ const now = req.now ?? Date.now;
1611
+ const retries = req.wait ? (req.retries ?? green.mergeQueue?.retries ?? 0) : 0;
1612
+ const deadline = req.wait && req.timeoutMs !== undefined ? now() + req.timeoutMs : Infinity;
1613
+ const note = req.onNote ?? (() => {});
1614
+ let errors = 0;
1615
+ let waitingOn = "the first read of the check runs";
1616
+ // Each workflow run re-run, at the attempt it was on when asked: until GitHub starts the new attempt, it isn't asked again.
1617
+ const asked = new Map<number, number>();
1618
+ const told = new Set<string>();
1619
+ const once = (line: string) => {
1620
+ if (told.has(line)) return;
1621
+ told.add(line);
1622
+ note(line);
1623
+ };
1624
+ const pause = async (ms: number) => {
1625
+ const left = deadline - now();
1626
+ if (left <= 0) throw new CiVerdictGaveUpError(`no verdict after ${minutes(req.timeoutMs!)}: ${waitingOn}`);
1627
+ await sleep(Math.min(ms, left));
1628
+ };
1564
1629
  for (;;) {
1630
+ let step: { done: { verdict: CommitVerdict; repeated?: VerdictRepeat } } | { verdict: CommitVerdict; waiting: boolean };
1631
+ try {
1632
+ step = await look();
1633
+ } catch (err) {
1634
+ if (!req.wait || !isTransientForgeError(err)) throw err;
1635
+ errors++;
1636
+ const message = err instanceof Error ? err.message : String(err);
1637
+ if (errors >= CI_VERDICT_FORGE_ERRORS) throw new CiVerdictGaveUpError(`the forge failed ${errors} times in a row, the last: ${message}`);
1638
+ const backoff = Math.min(CI_VERDICT_BACKOFF_MS * 2 ** (errors - 1), CI_VERDICT_BACKOFF_MAX_MS);
1639
+ note(`the forge failed (${message}); looking again in ${Math.round(backoff / 1000)}s (${errors} of ${CI_VERDICT_FORGE_ERRORS - 1} looked past)`);
1640
+ waitingOn = `the forge failing: ${message}`;
1641
+ await pause(backoff);
1642
+ continue;
1643
+ }
1644
+ errors = 0;
1645
+ if ("done" in step) return { sha: req.sha, ...step.done };
1646
+ if (!step.waiting) req.onPending?.(step.verdict);
1647
+ await pause(CI_VERDICT_INTERVAL_MS);
1648
+ }
1649
+
1650
+ async function look(): Promise<{ done: { verdict: CommitVerdict; repeated?: VerdictRepeat } } | { verdict: CommitVerdict; waiting: boolean }> {
1565
1651
  const runs = (await req.forge.checkRuns(req.sha)).filter((r) => r.name !== MERGE_QUEUE_VERDICT_JOB);
1566
1652
  const verdict = evaluateCommit(green, runs);
1567
- if (!req.wait || verdict.verdict !== "pending") return { sha: req.sha, verdict };
1568
- req.onPending?.(verdict);
1569
- await sleep(CI_VERDICT_INTERVAL_MS);
1653
+ if (!req.wait) return { done: { verdict } };
1654
+ if (verdict.verdict === "pending") {
1655
+ waitingOn = `still waiting on ${verdict.phases.filter((p) => p.verdict === "pending").map((p) => p.phase).join(", ")}`;
1656
+ return { verdict, waiting: false };
1657
+ }
1658
+ if (verdict.verdict !== "fail" || retries === 0 || !req.forge.workflowRun || !req.forge.rerunFailedJobs) return { done: { verdict } };
1659
+ // #4729: every failed run of a required phase, each job once.
1660
+ const failed: { phase: string; run: RunVerdict }[] = [];
1661
+ for (const p of verdict.phases) for (const r of p.runs) if (r.verdict === "fail" && !failed.some((f) => f.run.id === r.id)) failed.push({ phase: p.phase, run: r });
1662
+ const waits: string[] = [];
1663
+ for (const f of failed) {
1664
+ const wr = await req.forge.workflowRun(f.run.id);
1665
+ // Another app's check run: nothing chant can re-run.
1666
+ if (wr === null) return { done: { verdict } };
1667
+ const what = `${f.run.name} (phase ${f.phase}) in workflow run ${wr.name}${wr.url ? ` ${wr.url}` : ""}`;
1668
+ if (wr.status !== "completed") {
1669
+ once(`${f.run.name} failed (phase ${f.phase}); waiting for workflow run ${wr.name} to finish before re-running it`);
1670
+ waits.push(`workflow run ${wr.name} to finish`);
1671
+ continue;
1672
+ }
1673
+ if (wr.attempt > retries) return { done: { verdict, repeated: { phase: f.phase, run: f.run, workflowRun: wr } } };
1674
+ if (asked.get(wr.id) === wr.attempt) {
1675
+ waits.push(`the re-run of workflow run ${wr.name} to start`);
1676
+ continue;
1677
+ }
1678
+ note(`${what} failed on attempt ${wr.attempt} of ${retries + 1}; re-running its failed jobs`);
1679
+ try {
1680
+ await req.forge.rerunFailedJobs(wr.id);
1681
+ } catch (err) {
1682
+ if (isTransientForgeError(err)) throw err;
1683
+ throw new Error(`could not re-run the failed jobs of workflow run ${wr.name} (${wr.id}): ${err instanceof Error ? err.message : String(err)}. The verdict's job needs actions: write, which chant ci workflow gives it when ci.green.mergeQueue.retries is above 0`);
1684
+ }
1685
+ asked.set(wr.id, wr.attempt);
1686
+ waits.push(`the re-run of workflow run ${wr.name}`);
1687
+ }
1688
+ waitingOn = `waiting for ${[...new Set(waits)].join(", ")}`;
1689
+ return { verdict, waiting: true };
1570
1690
  }
1571
1691
  }
1572
1692
 
1693
+ function minutes(ms: number): string {
1694
+ const m = ms / 60_000;
1695
+ return `${Number.isInteger(m) ? m : m.toFixed(1)} minute${m === 1 ? "" : "s"}`;
1696
+ }
1697
+
1573
1698
  // ── base ─────────────────────────────────────────────────────────────────────
1574
1699
 
1575
1700
  /**