@intentius/chant 0.124.28 → 0.124.30

package/src/cli/handlers/pr.ts

@@ -33,7 +33,15 @@
  * one `run-event.schema.json` line each: the planning as a `plan` phase with
  * a step per component, the gate's decision as a `gate` phase, and on
  * pr-apply the apply's waves and components as `chant run --components`
- * streams them.
+ * streams them. A component the apply does not start settles too, `blocked`
+ * behind a failed dependency or `already-applied` on a resumed apply.
+ * With or without it, pr-apply prints a line on stderr as each wave starts
+ * and as each component settles, so a job log shows the waves in order.
+ *
+ * `--report-url <s3 or file URL>` (#4007) copies `pr-<stage>.json`,
+ * `change-set.json` and, with `--events-json`, the events to
+ * `<url>/<pr>/<commit>/`, the events again as each wave and component
+ * settles (`../../pr-report-url.ts`).
  *
  * `--member <name>` (#3465) runs a workspace member's loop: the member's
  * generated pipeline passes it from the member's directory. The gate is
@@ -59,11 +67,13 @@ import {
   type PrPlanSet,
 } from "../../components/pr-run";
 import { readFanOutAttempt, writeFanOutAttempt, type FanOutAttempt, type PrApplyRecord } from "../../components/fan-out-record";
+import { componentInputDigests, progressThatCarries } from "../../components/fan-out-inputs";
 import { samePlanDigest } from "../../lifecycle/plan-digest";
 import { gitGateLedgerPort, type GateCheck } from "../../op/gate";
 import { emitGateStep, gateEventsOf, GATE_PHASE, type RunEventSink, type RunEventStream } from "../../run-event-stream";
-import type { RunEventSettleStatus } from "../../run-event";
+import type { RunEventBody, RunEventComponentSettle, RunEventSettleStatus } from "../../run-event";
 import { openCliRunEvents, refusesEventsWithJson } from "../run-events";
+import { openPrReportUpload, parseReportUrl, reportKey, type PrReportUpload, type ReportTarget } from "../../pr-report-url";
 import {
   describePlanChanged,
   PR_APPLY_GATE,
@@ -102,6 +112,11 @@ interface Prepared {
   head: string;
   gate: string;
   forge?: PrForge;
+  /**
+   * Every component in the project, not the selection (`deriveFanOut`), so
+   * the plan reads the deployed outputs of a dependency the change did not
+   * select (#4104).
+   */
   components: DriverComponent[];
   plan: FanOutPlan;
   signal: ComponentChangeSignal;
@@ -262,12 +277,18 @@ function settleOf(status: PrReport["status"]): RunEventSettleStatus {
   }
 }
 
-/** Write the report, the change set and the note; print the note's path. */
-function writeOutputs(ctx: CommandContext, report: PrReport, note: string): void {
+/** Write the report, the change set and the note, and hand the first two to `--report-url`'s upload. */
+function writeOutputs(ctx: CommandContext, report: PrReport, note: string, upload?: PrReportUpload): void {
   const dir = resolve(ctx.args.output ?? PR_REPORT_DIR);
   mkdirSync(dir, { recursive: true });
-  writeFileSync(join(dir, `pr-${report.stage}.json`), JSON.stringify(report, null, 2) + "\n");
-  writeFileSync(join(dir, "change-set.json"), JSON.stringify(report.changeSet, null, 2) + "\n");
+  const files: Array<[string, string]> = [
+    [`pr-${report.stage}.json`, JSON.stringify(report, null, 2) + "\n"],
+    ["change-set.json", JSON.stringify(report.changeSet, null, 2) + "\n"],
+  ];
+  for (const [name, body] of files) {
+    writeFileSync(join(dir, name), body);
+    upload?.file(name, body);
+  }
   writeFileSync(join(dir, "pr-note.md"), note);
 }
 
@@ -310,21 +331,32 @@ function printMembers(report: PrReport): void {
   }
 }
 
-async function finish(ctx: CommandContext, p: Prepared, report: PrReport): Promise<void> {
+async function finish(ctx: CommandContext, p: Prepared, report: PrReport, upload?: PrReportUpload): Promise<void> {
   const note = noteFor(report, p.forge);
-  writeOutputs(ctx, report, note);
+  writeOutputs(ctx, report, note, upload);
   await publish(p.forge, report, note);
   if (ctx.args.json) console.log(JSON.stringify(report, null, 2));
 }
 
-/** A stage's stream, held so that a stage that returns or throws before it settles still ends it. */
+/**
+ * A stage's stream and its `--report-url` upload, held so that a stage that
+ * returns or throws before it settles still ends the stream and uploads.
+ */
 interface StageEvents {
   stream?: RunEventStream;
+  /** Where `--report-url` points, read before the stage starts. */
+  target?: ReportTarget;
+  upload?: PrReportUpload;
 }
 
-/** Run a stage, and end its stream as `failed` when the stage did not settle it. */
-async function withStageEvents(stage: string, run: (events: StageEvents) => Promise<number>): Promise<number> {
+/** Run a stage, end its stream as `failed` when the stage did not settle it, and finish its upload. */
+async function withStageEvents(stage: string, ctx: CommandContext, run: (events: StageEvents) => Promise<number>): Promise<number> {
   const events: StageEvents = {};
+  if (ctx.args.reportUrl !== undefined) {
+    const target = parseReportUrl(ctx.args.reportUrl);
+    if ("error" in target) return fail(target.error);
+    events.target = target;
+  }
   try {
     return await run(events);
   } catch (err) {
@@ -332,13 +364,32 @@ async function withStageEvents(stage: string, run: (events: StageEvents) => Prom
     throw err;
   } finally {
     events.stream?.settle("failed", `${stage} stopped before it finished`);
+    if (events.upload) {
+      await events.upload.close();
+      console.error(formatInfo(`report  : ${events.upload.where}/`));
+    }
   }
 }
 
-/** chant components pr-plan --base <ref> --pr <n> [--env <env>] [--gate <name>] [--forge <kind>] [--output <dir>] [--json] [--events-json] */
+/**
+ * Open the stage's `--report-url` upload under its pull request and commit,
+ * when the flag was given. Its events are the lines `openCliRunEvents` writes.
+ */
+function openUpload(events: StageEvents, stage: "plan" | "apply", pr: number | null, p: Prepared): PrReportUpload | undefined {
+  if (!events.target) return undefined;
+  events.upload = openPrReportUpload({
+    target: events.target,
+    key: reportKey(pr, p.head, p.member),
+    stage,
+    warn: (message) => console.error(formatWarning({ message })),
+  });
+  return events.upload;
+}
+
+/** chant components pr-plan --base <ref> --pr <n> [--env <env>] [--gate <name>] [--forge <kind>] [--output <dir>] [--json] [--events-json] [--report-url <url>] */
 export function runComponentsPrPlan(ctx: CommandContext): Promise<number> {
   if (refusesEventsWithJson(ctx.args, "the stage's report")) return Promise.resolve(1);
-  return withStageEvents("pr-plan", (events) => prPlan(ctx, events));
+  return withStageEvents("pr-plan", ctx, (events) => prPlan(ctx, events));
 }
 
 async function prPlan(ctx: CommandContext, stage: StageEvents): Promise<number> {
@@ -348,8 +399,9 @@ async function prPlan(ctx: CommandContext, stage: StageEvents): Promise<number>
   const p = await prepare(ctx, "plan");
   if (typeof p === "number") return p;
   const op = prOp(ctx.args.pr, p.member);
+  const upload = openUpload(stage, "plan", ctx.args.pr, p);
   // `--events-json` (#4041): planning, then where the approval stands.
-  const events = await openCliRunEvents(ctx.args, { kind: "component", name: `pr-plan ${op}`, cwd: resolve(".") });
+  const events = await openCliRunEvents(ctx.args, { kind: "component", name: `pr-plan ${op}`, cwd: resolve(".") }, upload?.line);
   stage.stream = events;
   events?.emit({ type: "run-start", waves: p.plan.waves });
   const set = await planAll(p, events?.emit);
@@ -381,7 +433,7 @@ async function prPlan(ctx: CommandContext, stage: StageEvents): Promise<number>
     approval,
     status,
   });
-  await finish(ctx, p, report);
+  await finish(ctx, p, report, upload);
   // The plan stage settles on its planning: an approval still to come is what it reports, not a stop.
   events?.settle(settleOf(status), status === "plan-failed" ? "a member failed to plan" : undefined);
 
@@ -394,10 +446,10 @@ async function prPlan(ctx: CommandContext, stage: StageEvents): Promise<number>
   return status === "plan-failed" ? 1 : 0;
 }
 
-/** chant components pr-apply --base <ref> [--pr <n>] [--env <env>] [--gate <name>] [--forge <kind>] [--require-review] [--output <dir>] [--json] [--events-json] */
+/** chant components pr-apply --base <ref> [--pr <n>] [--env <env>] [--gate <name>] [--forge <kind>] [--require-review] [--output <dir>] [--json] [--events-json] [--report-url <url>] */
 export function runComponentsPrApply(ctx: CommandContext): Promise<number> {
   if (refusesEventsWithJson(ctx.args, "the stage's report")) return Promise.resolve(1);
-  return withStageEvents("pr-apply", (events) => prApply(ctx, events));
+  return withStageEvents("pr-apply", ctx, (events) => prApply(ctx, events));
 }
 
 async function prApply(ctx: CommandContext, stage: StageEvents): Promise<number> {
@@ -418,7 +470,8 @@ async function prApply(ctx: CommandContext, stage: StageEvents): Promise<number>
   // The run id the release records carry, chosen before the run so that
   // `--events-json` (#4041, #4007) names the run by it too.
   const release = autoReleaseRun(`${op}-${Date.now()}`);
-  const events = await openCliRunEvents(args, { kind: "component", name: `pr-apply ${op}`, runId: release.runId, cwd: resolve(".") });
+  const upload = openUpload(stage, "apply", pr, p);
+  const events = await openCliRunEvents(args, { kind: "component", name: `pr-apply ${op}`, runId: release.runId, cwd: resolve(".") }, upload?.line);
   stage.stream = events;
   events?.emit({ type: "run-start", waves: p.plan.waves });
 
@@ -434,15 +487,28 @@ async function prApply(ctx: CommandContext, stage: StageEvents): Promise<number>
       return fail(`--resume: could not read "${args.resume}": ${err instanceof Error ? err.message : String(err)}`);
     }
   }
-  const resumed = pr !== null && p.plan.order.length > 0 && attempt ? await tryResume(p, attempt, args.resume!, op) : undefined;
+  // What each component would apply now (#4081): progress carries only where it is unchanged.
+  const inputsNow = resumePath ? componentInputDigests(p.components, { projectRoot: resolve("."), env: p.env }) : {};
+  const resumed = pr !== null && p.plan.order.length > 0 && attempt ? await tryResume(p, attempt, args.resume!, op, inputsNow) : undefined;
 
-  if (resumed) emitResumedGate(events, op, p.gate, resumed.check);
+  // The components an earlier attempt applied settle first (#4007), each in
+  // the wave the derivation gave it, which is the run-start's.
+  const log = applyLog();
+  if (resumed) {
+    const waveOf = new Map(p.plan.waves.flatMap((wave, i) => wave.map((name) => [name, i + 1] as const)));
+    for (const component of p.plan.order.filter((name) => resumed.completed.includes(name))) {
+      const body: RunEventComponentSettle = { type: "component-settle", wave: waveOf.get(component)!, component, status: "already-applied" };
+      events?.emit(body);
+      log(body);
+    }
+    emitResumedGate(events, op, p.gate, resumed.check);
+  }
   const set = resumed?.set ?? (await planAll(p, events?.emit));
   const base = { stage: "apply" as const, pr, env: p.env, ...(p.member ? { member: p.member } : {}), base: p.base, head: p.head, op, gate: p.gate, plan: p.plan, signal: p.signal, set };
 
   const stop = async (report: PrReport, code: number, settle?: RunEventSettleStatus): Promise<number> => {
     events?.settle(settle ?? settleOf(report.status), report.status === "plan-failed" || report.status === "failed" ? report.message ?? `pr-apply ${report.status}` : undefined);
-    await finish(ctx, p, report);
+    await finish(ctx, p, report, upload);
     console.error((code === 0 ? formatInfo : (m: string) => formatWarning({ message: m }))(`pr-apply: ${report.status}${report.refusal ? ` (${report.refusal})` : ""}`));
     if (report.message) console.error(report.message);
     printMembers(report);
@@ -538,6 +604,9 @@ async function prApply(ctx: CommandContext, stage: StageEvents): Promise<number>
   // one settles, so a run killed partway still knows what applied.
   const prior = resumed ? { completed: resumed.completed, outputs: resumed.outputs } : { completed: [], outputs: {} };
   const completedSoFar = new Set<string>(prior.completed);
+  const inputsSoFar: Record<string, string> = Object.fromEntries(
+    prior.completed.filter((name) => inputsNow[name] !== undefined).map((name) => [name, inputsNow[name]!]),
+  );
   const failedSoFar: string[] = [];
   const outputsSoFar: Record<string, Record<string, unknown>> = { ...prior.outputs };
   const record: PrApplyRecord = {
@@ -557,6 +626,7 @@ async function prApply(ctx: CommandContext, stage: StageEvents): Promise<number>
       completed: [...completedSoFar].sort(),
       failed: [...failedSoFar].sort(),
       outputs: outputsSoFar,
+      inputs: Object.fromEntries(Object.entries(inputsSoFar).sort(([a], [b]) => a.localeCompare(b))),
       prApply: record,
     });
   };
@@ -569,13 +639,18 @@ async function prApply(ctx: CommandContext, stage: StageEvents): Promise<number>
     env: p.env,
     set,
     ...(resumed ? { resumed: prior } : {}),
-    // The apply is this run's waves, so the fan-out's own run framing is dropped.
-    ...(events ? { onRunEvent: events.nested } : {}),
+    // The apply is this run's waves, so the fan-out's own run framing is
+    // dropped. The log's lines go to stderr with or without --events-json.
+    onRunEvent: (body: RunEventBody) => {
+      events?.nested(body);
+      log(body);
+    },
     ...(resumePath
       ? {
           onComponentSettled: (settled, outputs) => {
             if (settled.status === "ok") {
               completedSoFar.add(settled.component);
+              if (inputsNow[settled.component] !== undefined) inputsSoFar[settled.component] = inputsNow[settled.component]!;
               if (outputs) outputsSoFar[settled.component] = outputs;
             } else if (settled.status === "fail") {
               failedSoFar.push(settled.component);
@@ -600,6 +675,37 @@ async function prApply(ctx: CommandContext, stage: StageEvents): Promise<number>
   return code;
 }
 
+/** What a component's settle says in the job log. */
+function settleWords(body: RunEventComponentSettle): string {
+  switch (body.status) {
+    case "ok":
+      return "applied";
+    case "failed":
+      return "failed";
+    case "gated":
+      return "stopped at its own gate";
+    case "waiting":
+      return "waiting on a decision";
+    case "blocked":
+      return `blocked by ${body.blockedBy ?? "a dependency"}, not run`;
+    case "already-applied":
+      return "already applied in an earlier attempt";
+  }
+}
+
+/**
+ * pr-apply's job log (#4007): a line on stderr as each wave starts and as
+ * each component settles, so the log shows the waves in order while they run.
+ */
+function applyLog(): (body: RunEventBody) => void {
+  let waves = 0;
+  return (body) => {
+    if (body.type === "run-start") waves = body.waves?.length ?? 0;
+    else if (body.type === "wave-start") console.error(formatInfo(`wave ${body.wave}${waves ? `/${waves}` : ""}: ${body.components.join(", ")}`));
+    else if (body.type === "component-settle") console.error(formatInfo(`  ${body.component}: ${settleWords(body)}`));
+  };
+}
+
 /** A resumed apply's gate (#3464), decided against the approved digest, as the run's `gate` phase. */
 function emitResumedGate(events: RunEventStream | undefined, op: string, gate: string, check: GateCheck): void {
   if (!events) return;
@@ -658,8 +764,13 @@ interface Resumed {
  * derivation; when its approval still stands; and when every member left is
  * covered by the approved set. Otherwise it says why on stderr and the apply
  * runs as it would with no record.
+ *
+ * Of the components it applied, only those whose input digest is the same
+ * now carry (#4081), as in `components fan-out --resume`. One that changed
+ * runs again, under the same approval when the approved set covers its new
+ * plan. A record without input digests carries nothing.
  */
-async function tryResume(p: Prepared, attempt: FanOutAttempt, file: string, op: string): Promise<Resumed | undefined> {
+async function tryResume(p: Prepared, attempt: FanOutAttempt, file: string, op: string, inputsNow: Record<string, string>): Promise<Resumed | undefined> {
   const skip = (why: string): undefined => {
     console.error(formatWarning({ message: `--resume: "${file}" does not carry: ${why}`, hint: "Every member is planned again, and the plan needs an approval of its own." }));
     return undefined;
@@ -670,6 +781,11 @@ async function tryResume(p: Prepared, attempt: FanOutAttempt, file: string, op:
   if (record.head !== p.head) return skip(`it records an apply of ${record.head.slice(0, 12)}, not ${p.head.slice(0, 12)}`);
   if (!samePlanDigest(attempt.digest, p.plan.digest)) return skip("the change selects other components than it did");
   if (attempt.completed.length === 0) return undefined;
+  const split = progressThatCarries(attempt, inputsNow);
+  if (split.unrecorded) return skip("it does not record what each component applied, since it was written before chant recorded component inputs");
+  for (const name of split.changed) console.error(formatInfo(`${name}: changed since the earlier attempt, so it runs again`));
+  if (split.carried.length === 0) return undefined;
+  const completed = split.carried;
 
   let check;
   try {
@@ -679,7 +795,7 @@ async function tryResume(p: Prepared, attempt: FanOutAttempt, file: string, op:
   }
   if (!check.satisfied) return skip(`no approval stands for the plan it applied (${record.digest}) any more`);
 
-  const outputs = Object.fromEntries(Object.entries(attempt.outputs).filter(([name]) => attempt.completed.includes(name)));
+  const outputs = Object.fromEntries(Object.entries(attempt.outputs).filter(([name]) => completed.includes(name)));
   const resumed = await resumePrSet({
     components: p.components,
     plan: p.plan,
@@ -687,9 +803,9 @@ async function tryResume(p: Prepared, attempt: FanOutAttempt, file: string, op:
     env: p.env,
     seededOutputs: p.seededOutputs,
     record,
-    completed: attempt.completed,
+    completed,
   });
   if (!resumed.ok) return skip(`what is left is not what was approved. ${resumed.reasons.join("; ")}`);
-  console.error(formatInfo(`resume  : ${attempt.completed.join(", ")} applied in an earlier attempt; applying ${resumed.plan.order.join(", ") || "nothing more"} under the approval of ${record.digest}`));
-  return { plan: resumed.plan, set: resumed.set, check, approval: approvalOf(check), completed: attempt.completed, outputs };
+  console.error(formatInfo(`resume  : ${completed.join(", ")} applied in an earlier attempt; applying ${resumed.plan.order.join(", ") || "nothing more"} under the approval of ${record.digest}`));
+  return { plan: resumed.plan, set: resumed.set, check, approval: approvalOf(check), completed, outputs };
 }