@intentius/chant 0.124.30 → 0.124.32

package/src/components/fan-out-run.ts

@@ -205,6 +205,25 @@ export interface FanOutRunOptions {
    * the returned result cannot do.
    */
   onComponentSettled?: (result: DriverComponentResult, outputs: Record<string, unknown> | undefined) => void;
+  /**
+   * Asked as each component's wave starts, with the outputs standing then
+   * (#3999). A reason holds the component back: it does not run, and nothing
+   * downstream of it runs either; each is listed in `held` and settles
+   * `waiting`. A pull request's apply holds a member whose plan read an
+   * upstream output that applied as another value, or that was known only
+   * after apply. Not with `waveGate`, which plans each wave after the one
+   * before it applied.
+   */
+  hold?: (component: string, componentOutputs: Record<string, Record<string, unknown>>) => string | undefined;
+}
+
+/** A component {@link FanOutRunOptions.hold} held back, or one downstream of it. */
+export interface FanOutHeld {
+  component: string;
+  /** Why the hook held it, or, downstream, why it held the component named in `heldBy`. */
+  reason: string;
+  /** Downstream of a held component: that component. */
+  heldBy?: string;
 }
 
 export interface FanOutRunResult {
@@ -223,6 +242,8 @@ export interface FanOutRunResult {
   gate?: PendingGateRecord;
   /** With `waveGate`: each wave this attempt planned, in order, the one it stopped at included. */
   waves?: WaveRecord[];
+  /** What {@link FanOutRunOptions.hold} held back, in plan order. Absent when nothing was. */
+  held?: FanOutHeld[];
 }
 
 /**
@@ -296,9 +317,21 @@ export async function runFanOut(
   const completed: string[] = [];
   const failed: string[] = [];
   const blockedBy = new Map<string, string>();
+  const held = new Map<string, FanOutHeld>();
   let gatedFact: PendingGateRecord | undefined;
 
   const inPlan = new Set(active.order);
+  const waveOf = waveNumbers(active.waves);
+  /** Hold `root` and everything downstream of it that has not settled otherwise. */
+  const markHeld = (root: string, reason: string): void => {
+    held.set(root, { component: root, reason });
+    emit?.({ type: "component-settle", wave: waveOf.get(root)!, component: root, status: "waiting" });
+    for (const [component] of downstreamWithin(components, inPlan, [root])) {
+      if (held.has(component) || blockedBy.has(component) || failed.includes(component)) continue;
+      held.set(component, { component, reason, heldBy: root });
+      emit?.({ type: "component-settle", wave: waveOf.get(component)!, component, status: "waiting" });
+    }
+  };
   /** Skip everything downstream of `root`, blaming `root` rather than the nearest edge. */
   const markBlocked = (root: string): void => {
     for (const [component, by] of downstreamWithin(components, inPlan, [root])) {
@@ -310,7 +343,12 @@ export async function runFanOut(
   for (const [waveIndex, wave] of active.waves.entries()) {
     const waveNum = waveIndex + 1;
     // A component whose upstream failed in an earlier wave is not dispatched.
-    const runnable = wave.filter((name) => inPlan.has(name) && !blockedBy.has(name));
+    const candidates = wave.filter((name) => inPlan.has(name) && !blockedBy.has(name) && !held.has(name));
+    for (const name of candidates) {
+      const reason = options.hold?.(name, componentOutputs);
+      if (reason !== undefined) markHeld(name, reason);
+    }
+    const runnable = candidates.filter((name) => !held.has(name));
     if (runnable.length === 0) continue;
 
     emit?.({ type: "wave-start", wave: waveNum, components: runnable });
@@ -384,6 +422,7 @@ export async function runFanOut(
     blocked,
     componentOutputs,
     ...(gatedFact ? { gate: gatedFact } : {}),
+    ...(held.size > 0 ? { held: active.order.filter((name) => held.has(name)).map((name) => held.get(name)!) } : {}),
   };
 }
 

package/src/components/pr-pipeline.test.ts

@@ -0,0 +1,100 @@
+/**
+ * The apply job's base fallback (#4004), run in sh against a real
+ * repository. GitHub and Forgejo set `github.event.before`, and GitLab
+ * `CI_COMMIT_BEFORE_SHA`, to forty zeros on a branch's first push.
+ */
+
+import { execFileSync } from "node:child_process";
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterAll, describe, expect, test } from "vitest";
+import { EMPTY_TREE_SHA } from "./changed-files";
+import { githubBaseFallback, gitlabBaseFallback, prLoopJobs } from "./pr-pipeline";
+
+const ZEROS = "0".repeat(40);
+const dir = mkdtempSync(join(tmpdir(), "chant-pr-base-"));
+afterAll(() => rmSync(dir, { recursive: true, force: true }));
+const ENV = { ...process.env, GIT_AUTHOR_NAME: "t", GIT_AUTHOR_EMAIL: "t@example.com", GIT_COMMITTER_NAME: "t", GIT_COMMITTER_EMAIL: "t@example.com" };
+
+const git = (cwd: string, ...args: string[]): string => execFileSync("git", args, { cwd, encoding: "utf8", env: ENV }).trim();
+
+/** A bare origin and a clone of it, `main` holding `commits` commits. */
+function repo(name: string, commits: number): string {
+  const origin = join(dir, `${name}.git`);
+  const work = join(dir, name);
+  execFileSync("git", ["init", "-q", "--bare", "-b", "main", origin]);
+  execFileSync("git", ["init", "-q", "-b", "main", work]);
+  git(work, "remote", "add", "origin", origin);
+  for (let i = 0; i < commits; i++) {
+    writeFileSync(join(work, "f.txt"), `${i}\n`);
+    git(work, "add", "-A");
+    git(work, "commit", "-q", "-m", `c${i}`);
+  }
+  git(work, "push", "-q", "origin", "main");
+  return work;
+}
+
+/** BASE_SHA after `line` runs in `cwd` with `env`. */
+function settle(cwd: string, line: string, env: Record<string, string>): string {
+  return execFileSync("sh", ["-ec", `${line}\nprintf %s "$BASE_SHA"`], { cwd, encoding: "utf8", env: { ...ENV, ...env } });
+}
+
+describe("githubBaseFallback (#4004)", () => {
+  test("a set base is kept", () => {
+    const work = repo("kept", 2);
+    const parent = git(work, "rev-parse", "HEAD^");
+    expect(settle(work, githubBaseFallback(), { BASE_SHA: parent, GITHUB_SHA: git(work, "rev-parse", "HEAD") })).toBe(parent);
+  });
+
+  test("forty zeros or empty on the target branch's first push: the pushed commit's parent", () => {
+    const work = repo("first", 2);
+    const head = git(work, "rev-parse", "HEAD");
+    const parent = git(work, "rev-parse", "HEAD^");
+    expect(settle(work, githubBaseFallback(), { BASE_SHA: ZEROS, GITHUB_SHA: head })).toBe(parent);
+    expect(settle(work, githubBaseFallback(), { BASE_SHA: "", GITHUB_SHA: head })).toBe(parent);
+  });
+
+  test("a branch's first push: the merge base with the target branch", () => {
+    const work = repo("branch", 1);
+    const fork = git(work, "rev-parse", "HEAD");
+    git(work, "checkout", "-q", "-b", "feature");
+    writeFileSync(join(work, "g.txt"), "g\n");
+    git(work, "add", "-A");
+    git(work, "commit", "-q", "-m", "feature");
+    expect(settle(work, githubBaseFallback(), { BASE_SHA: ZEROS, GITHUB_SHA: git(work, "rev-parse", "HEAD") })).toBe(fork);
+  });
+
+  test("a root commit: the empty tree", () => {
+    const work = repo("root", 1);
+    expect(settle(work, githubBaseFallback(), { BASE_SHA: ZEROS, GITHUB_SHA: git(work, "rev-parse", "HEAD") })).toBe(EMPTY_TREE_SHA);
+  });
+
+  test("a named branch is the one fetched", () => {
+    expect(githubBaseFallback()).toContain('git fetch origin "main"');
+    expect(githubBaseFallback("trunk")).toContain('git fetch origin "trunk"');
+  });
+});
+
+describe("gitlabBaseFallback", () => {
+  test("a root commit's first push: the empty tree, as on GitHub (#4004)", () => {
+    const work = repo("gitlab-root", 1);
+    expect(settle(work, gitlabBaseFallback("main"), { BASE_SHA: ZEROS, CI_COMMIT_SHA: git(work, "rev-parse", "HEAD") })).toBe(EMPTY_TREE_SHA);
+  });
+
+  test("the merge request's diff base comes first", () => {
+    const work = repo("gitlab-mr", 2);
+    const parent = git(work, "rev-parse", "HEAD^");
+    expect(settle(work, gitlabBaseFallback("main"), { BASE_SHA: ZEROS, CI_MERGE_REQUEST_DIFF_BASE_SHA: parent, CI_COMMIT_SHA: git(work, "rev-parse", "HEAD") })).toBe(parent);
+  });
+});
+
+describe("prLoopJobs", () => {
+  test("every forge's apply settles its base first; no plan does", () => {
+    for (const forge of ["github", "forgejo", "gitlab"] as const) {
+      const [plan, apply] = prLoopJobs(forge, "prod", { branch: "trunk" });
+      expect(plan!.setup).toBeUndefined();
+      expect(apply!.setup).toEqual([forge === "gitlab" ? gitlabBaseFallback("trunk") : githubBaseFallback("trunk")]);
+    }
+  });
+});

package/src/components/pr-pipeline.ts

@@ -42,6 +42,25 @@
  * build is `chant build --env <env>`, or `buildCommand` when the templates
  * come from several builds (`npm run build`).
  *
+ * With `waveGate` (#4003) the apply runs a wave at a time: `pr-apply
+ * --wave-gate <gate>`, each wave planned after the one before it applied. A
+ * generator writes one apply job per wave of the component graph
+ * (`prLoopWaveJobs`), job `apply-wave-<k>` passing `--wave <k>` and needing
+ * the job before it, and the attempt record carries the approved set from
+ * one job to the next. A wave whose plan the approved change set covers
+ * applies under the pull request's approval; one whose plan moved stops at
+ * `<gate>-wave-<k>` (exit 3), and re-running its job after the approval
+ * carries on. `chant ci serve` runs the one apply job, which runs the waves
+ * in order until one stops.
+ *
+ * Each generated job runs its command through `chant ci run plan|apply
+ * --in-place` (#4156, `PrLoopJob.run`), so a forge job writes the same run
+ * record the chant runner does: Op `pr-plan` or `pr-apply`, signed with
+ * the runner key a forge secret puts in `CHANT_RUNNER_KEY`, and naming the
+ * forge's run as its `runOrigin`. `ci run` runs `command` in the job's own
+ * checkout with the job's environment, less the key. The plan passes
+ * `--no-push`, so it still writes nothing to the remote `chant/lifecycle`.
+ *
  * In a workspace member (#3465) both commands run in the member's directory
  * and pass `--member <name>`, so the member's gate (`pr-<number>-<member>`),
  * note and statuses stay apart from every other member's on the same pull
@@ -52,6 +71,13 @@
 import type { ForgeKind } from "../pr-forge";
 import { PR_APPLY_GATE } from "../pr-loop";
 import { CI_IMAGE, GITHUB_JOB_TOKEN } from "../ci-jobs";
+import { EMPTY_TREE_SHA } from "./changed-files";
+import { layerWaves } from "../gated-waves";
+import type { DriverComponent } from "./driver";
+import type { GatedWavePipelineOptions } from "./gated-wave-pipeline";
+
+/** The variable `chant ci run` reads its runner key from (`RUNNER_KEY_ENV` in ../workspace/ci-green.ts). */
+const RUNNER_KEY_ENV = "CHANT_RUNNER_KEY";
 
 /** Where both jobs write their reports, kept as a CI artifact. */
 export const PR_LOOP_REPORT_DIR = ".chant/pr";
@@ -125,6 +151,12 @@ export interface PrLoopPipelineOptions {
    * `npm run build`. Setting it turns `build` on.
    */
   buildCommand?: string;
+  /**
+   * A gate per wave on merge (#4003): the apply runs `pr-apply --wave-gate
+   * <waveGate>`, a wave at a time, and a generator writes one apply job per
+   * wave. Set from `--wave-gate` beside `--pr-loop` ({@link prLoopWithWaves}).
+   */
+  waveGate?: string;
   /**
    * Which GitHub-shaped forge the workflow is for. Set by the forgejo
    * generator, which reuses the github one; a caller never needs to.
@@ -148,6 +180,17 @@ export interface PrLoopJob {
   setup?: string[];
   /** With `build` (#4088): the build line and the lint line, run in this order before the command. */
   build?: string[];
+  /** With `waveGate` (#4003): the wave a generated apply job runs, 1-based. */
+  wave?: number;
+  /**
+   * What the generated job runs in place of `command` (#4156): `chant ci run
+   * <job> --in-place` with the same environment, gate, forge, events and
+   * report URL, which runs `command` and records the run. `env` is the
+   * job's `env` and, on GitHub and Forgejo, `CHANT_RUNNER_KEY` from the
+   * repository secret of that name. GitLab hands its CI/CD variables to a
+   * job itself.
+   */
+  run: { command: string; env: Record<string, string> };
 }
 
 /** The lines `build` (#4088) runs before a job's command: the build, then the lint. Empty without the option. */
@@ -176,29 +219,82 @@ export function prLoopJobs(forge: ForgeKind, env: string, options: PrLoopPipelin
     ...(github ? { GITHUB_TOKEN: GITHUB_JOB_TOKEN } : {}),
     ...(options.events ? { CHANT_RUN_EVENTS_DIR: PR_LOOP_EVENTS_DIR } : {}),
   };
+  // #4156: the flags `chant ci run` rebuilds `command` from. It finds the member from the directory it runs in.
+  const ciRun = (job: "plan" | "apply"): string[] => [
+    "chant", "ci", "run", job, "--in-place", ...(job === "plan" ? ["--no-push"] : []),
+    "--base", '"$BASE_SHA"', ...(job === "plan" ? ["--pr", '"$PR_NUMBER"'] : []),
+    "--env", env, "--gate", gate, "--forge", forge,
+    ...(options.waveGate ? ["--wave-gate", options.waveGate] : []),
+    ...(job === "apply" && options.branch ? ["--branch", JSON.stringify(options.branch)] : []),
+    ...(job === "apply" && options.requireReview === false ? ["--no-require-review"] : []),
+    ...(options.events ? ["--events-json"] : []),
+    ...(options.reportUrl ? ["--report-url", JSON.stringify(options.reportUrl)] : []),
+  ];
+  const runEnv = (env: Record<string, string>): Record<string, string> => (github ? { ...env, [RUNNER_KEY_ENV]: `\${{ secrets.${RUNNER_KEY_ENV} }}` } : env);
+  const planEnv = github
+    ? { BASE_SHA: "${{ github.event.pull_request.base.sha }}", PR_NUMBER: "${{ github.event.pull_request.number }}", ...jobEnv }
+    : { BASE_SHA: "$CI_MERGE_REQUEST_DIFF_BASE_SHA", PR_NUMBER: "$CI_MERGE_REQUEST_IID", ...jobEnv };
+  const applyEnv = github ? { BASE_SHA: "${{ github.event.before }}", ...jobEnv } : { BASE_SHA: "$CI_COMMIT_BEFORE_SHA", ...jobEnv };
   return [
     {
       jobName: "plan",
-      command: ["chant", "components", "pr-plan", "--base", '"$BASE_SHA"', "--pr", '"$PR_NUMBER"', ...common].join(" "),
-      env: github
-        ? { BASE_SHA: "${{ github.event.pull_request.base.sha }}", PR_NUMBER: "${{ github.event.pull_request.number }}", ...jobEnv }
-        : { BASE_SHA: "$CI_MERGE_REQUEST_DIFF_BASE_SHA", PR_NUMBER: "$CI_MERGE_REQUEST_IID", ...jobEnv },
+      // With `waveGate` the plan's note says the later waves may stop at their own gates (#4003).
+      command: ["chant", "components", "pr-plan", "--base", '"$BASE_SHA"', "--pr", '"$PR_NUMBER"', ...common, ...(options.waveGate ? ["--wave-gate", options.waveGate] : [])].join(" "),
+      env: planEnv,
       ...build,
+      run: { command: ciRun("plan").join(" "), env: runEnv(planEnv) },
     },
     {
       jobName: "apply",
       command: [
         "chant", "components", "pr-apply", "--base", '"$BASE_SHA"', ...common,
         ...(options.requireReview === false ? [] : ["--require-review"]),
+        ...(options.waveGate ? ["--wave-gate", options.waveGate] : []),
         "--resume", PR_APPLY_RECORD,
       ].join(" "),
-      env: github ? { BASE_SHA: "${{ github.event.before }}", ...jobEnv } : { BASE_SHA: "$CI_COMMIT_BEFORE_SHA", ...jobEnv },
-      ...(github ? {} : { setup: [gitlabBaseFallback(options.branch)] }),
+      env: applyEnv,
+      setup: [github ? githubBaseFallback(options.branch) : gitlabBaseFallback(options.branch)],
       ...build,
+      run: { command: ciRun("apply").join(" "), env: runEnv(applyEnv) },
     },
   ];
 }
 
+/**
+ * The pull-request options with `--wave-gate` beside `--pr-loop` (#4003):
+ * the apply gets a gate per wave. A canary or a fixed base is refused, since
+ * the pull request's waves are its derivation's and each push is measured
+ * from the commit before it.
+ */
+export function prLoopWithWaves(prLoop: PrLoopPipelineOptions, gatedWaves?: GatedWavePipelineOptions): PrLoopPipelineOptions {
+  if (!gatedWaves) return prLoop;
+  if (gatedWaves.canary?.length) throw new Error("a pull-request pipeline's waves are the change's derivation, so it takes no --canary; drop --canary");
+  if (gatedWaves.base !== undefined) throw new Error("a pull-request pipeline measures each push from the commit before it, so it takes no --base; drop --base");
+  return { ...prLoop, waveGate: gatedWaves.gate };
+}
+
+/** The name of the apply job for wave `wave` (#4003). */
+export function prApplyWaveJobName(wave: number): string {
+  return `apply-wave-${wave}`;
+}
+
+/**
+ * One apply job per wave of the component graph (#4003), each `apply`'s
+ * command with `--wave <k>`. A change selects a subset, which never layers
+ * deeper than the graph, so a job past the change's last wave finds nothing
+ * left and exits 0.
+ */
+export function prLoopWaveJobs(components: DriverComponent[], apply: PrLoopJob): PrLoopJob[] {
+  const count = Math.max(1, layerWaves(components.map((c) => ({ name: c.name, dependsOn: c.dependsOn ?? [] }))).length);
+  return Array.from({ length: count }, (_, i) => ({
+    ...apply,
+    command: `${apply.command} --wave ${i + 1}`,
+    wave: i + 1,
+    // #4156: the job runs it through chant ci run, which appends the same --wave to pr-apply.
+    run: { ...apply.run, command: `${apply.run.command} --wave ${i + 1}` },
+  }));
+}
+
 /**
  * The concurrency group (GitHub, Forgejo) or resource group (GitLab) that
  * keeps one apply at a time per environment, and per member in a workspace.
@@ -227,19 +323,38 @@ export function prApplyRecordKey(env: string, commit: string, member?: string):
 }
 
 /**
- * GitLab sets `CI_COMMIT_BEFORE_SHA` to forty zeros on a branch's first push.
- * The line replaces it with the merge request's diff base when the pipeline
- * has one, and otherwise with the merge base of the target branch and the
- * pushed commit, or the pushed commit's parent when that is the commit itself.
+ * The shell that settles an unusable push base. `before` is empty or forty
+ * zeros on a branch's first push; then the base is `diffBase` when the forge
+ * gives one, else the merge base of `target` and `commit`, else `commit`'s
+ * parent when the merge base is `commit` itself, else, for a root commit, the
+ * empty tree.
  */
-export function gitlabBaseFallback(branch?: string): string {
-  const target = branch ? JSON.stringify(branch) : "$CI_DEFAULT_BRANCH";
+function pushBaseFallback(target: string, commit: string, diffBase?: string): string {
+  const fromBranch =
+    `git fetch origin ${target} && BASE_SHA="$(git merge-base FETCH_HEAD "${commit}")"; ` +
+    `if [ "$BASE_SHA" = "${commit}" ]; then BASE_SHA="$(git rev-parse --verify --quiet "${commit}^" || echo ${EMPTY_TREE_SHA})"; fi; `;
   return (
     'if [ -z "$BASE_SHA" ] || [ "$BASE_SHA" = "0000000000000000000000000000000000000000" ]; then ' +
-    'BASE_SHA="${CI_MERGE_REQUEST_DIFF_BASE_SHA:-}"; ' +
-    'if [ -z "$BASE_SHA" ]; then ' +
-    `git fetch origin ${target} && BASE_SHA="$(git merge-base FETCH_HEAD "$CI_COMMIT_SHA")"; ` +
-    'if [ "$BASE_SHA" = "$CI_COMMIT_SHA" ]; then BASE_SHA="$(git rev-parse "$CI_COMMIT_SHA^")"; fi; ' +
-    "fi; fi; export BASE_SHA"
+    (diffBase ? `BASE_SHA="\${${diffBase}:-}"; if [ -z "$BASE_SHA" ]; then ${fromBranch}fi; ` : fromBranch) +
+    "fi; export BASE_SHA"
   );
 }
+
+/**
+ * GitLab sets `CI_COMMIT_BEFORE_SHA` to forty zeros on a branch's first push.
+ * The line replaces it with the merge request's diff base when the pipeline
+ * has one, and otherwise as {@link pushBaseFallback} does.
+ */
+export function gitlabBaseFallback(branch?: string): string {
+  return pushBaseFallback(branch ? JSON.stringify(branch) : "$CI_DEFAULT_BRANCH", "$CI_COMMIT_SHA", "CI_MERGE_REQUEST_DIFF_BASE_SHA");
+}
+
+/**
+ * GitHub and Forgejo set `github.event.before` to forty zeros on a branch's
+ * first push (#4004). The line replaces it as {@link pushBaseFallback} does,
+ * with the target branch `main` unless the pipeline names another, the
+ * generators' default.
+ */
+export function githubBaseFallback(branch?: string): string {
+  return pushBaseFallback(JSON.stringify(branch ?? "main"), "$GITHUB_SHA");
+}