@llm4ts/flow 2.38.0 → 2.40.0

package/src/Stories.ts

@@ -27,7 +27,7 @@ import {
   ensureBaseline as ensureStoredBaseline,
   failingLinesOf,
   storyGateLogDir,
-  triageGates,
+  mergeGateLogDir,
   writeBaseline,
   type GateLogDir
 } from "./Gates.ts"
@@ -74,7 +74,9 @@ import {
   ReviewIssue,
   ReviewResult,
   applyTriage,
+  isBlocking,
   loadRepoReviewRules,
+  nonBlockingIssues,
   type FixPromptOptions,
   type GateTriageOptions
 } from "./Review.ts"
@@ -701,6 +703,13 @@ export interface StoriesOptions {
    * story's plan (coder, review, gates, commit). Default 2: one revision.
    */
   readonly judgeRounds?: number
+  /**
+   * When the epic gates are red after a story merges (the merge is undone),
+   * the story goes back to its worktree this many times: it catches up with
+   * the epic, gets a revision task with the failures, and merges again.
+   * Default 1; 0 fails the story at once, as before.
+   */
+  readonly mergeRevisions?: number
   /**
    * How much of the code a story starts from goes into its coder's system
    * prompt, in characters: the shared read-only files it uses, then its own.
@@ -722,6 +731,14 @@ export interface StoriesOptions {
   readonly failFast?: boolean
   readonly reviewers?: ReadonlyArray<Reviewer>
   readonly maxRounds?: number
+  /**
+   * Defer what does not block (ADR 0031): a task's review settles once its
+   * findings are all non-blocking, the judge clears a story whose findings
+   * are all non-blocking, and what is left is written to
+   * `stories/<id>.deferred.md` (`deferredPath`) when the story finishes,
+   * for a follow-up round. Default false: every finding is fixed or fails.
+   */
+  readonly deferNonBlocking?: boolean
   /**
    * Second opinion on a BLOCKED_ON claim the plan cannot settle (a path no
    * story owns, or a parallel story's). `real: false` sends the coder back
@@ -762,6 +779,24 @@ const combined = (first: ReviewResult, second: ReviewResult): ReviewResult =>
         summary: [first.summary, second.summary].filter((part) => part.length > 0).join("; ")
       })
 
+/**
+ * Where a finished story's deferred findings are kept (ADR 0031): one
+ * Markdown list per story under the executor's state directory, absent when
+ * nothing was deferred.
+ */
+export const deferredPath = (stateDir: string, storyId: string): string =>
+  join(stateDir, `stories/${storyId}.deferred.md`)
+
+/** A deferred finding as one Markdown bullet, prefixed with where it was found. */
+export const deferredLine = (source: string, issue: ReviewIssue): string => {
+  const where =
+    issue.file === undefined
+      ? ""
+      : ` (${issue.file}${issue.line === undefined ? "" : `:${issue.line}`})`
+  const detail = issue.description.trim().length === 0 ? "" : `: ${issue.description.trim()}`
+  return `- [${issue.severity}] ${source} — ${issue.title}${where}${detail}`
+}
+
 const issueLines = (result: ReviewResult): string =>
   result.issues.map((issue) => `- ${issue.title}: ${issue.description}`).join("\n")
 
@@ -843,6 +878,8 @@ export const implementStoriesFlow = Effect.fn("@llm4ts/flow/Stories.implement")(
   const epicBranch = options.epicBranch ?? `epic/${plan.epicId}`
   const concurrency = Math.max(1, options.concurrency ?? 3)
   const judgeRounds = Math.max(1, options.judgeRounds ?? 2)
+  const mergeRevisions = Math.max(0, options.mergeRevisions ?? 1)
+  const deferring = options.deferNonBlocking === true
   const statePath = (story: Story): string => join(options.stateDir, `stories/${story.id}.json`)
   const planPath = (story: Story): string => join(options.stateDir, `stories/${story.id}.plan.md`)
   /** The judge's last verdict beside a fingerprint of what it judged. */
@@ -981,8 +1018,15 @@ export const implementStoriesFlow = Effect.fn("@llm4ts/flow/Stories.implement")(
 
   const mergeLock = yield* Semaphore.make(1)
 
-  /** Merge the story branch into the epic branch and re-gate the epic head, one story at a time. */
-  const integrate = (story: Story, branch: string): Effect.Effect<void, FlowError> =>
+  /**
+   * Merge the story branch into the epic branch and re-gate the epic head,
+   * one story at a time. Red gates undo the merge and are returned (the
+   * failures the merge added); `undefined` means merged.
+   */
+  const integrate = (
+    story: Story,
+    branch: string
+  ): Effect.Effect<ReviewResult | undefined, FlowError> =>
     Effect.flatMap(Clock.currentTimeMillis, (requested) =>
       Effect.andThen(
         laneOf(story).publish(Began.make({ kind: "wait", label: "merge lock" })),
@@ -1003,22 +1047,36 @@ export const implementStoriesFlow = Effect.fn("@llm4ts/flow/Stories.implement")(
                 Timed.make({ kind: "merge", label: "merge", ms, ...(failed ? { failed } : {}) }),
               () => Began.make({ kind: "merge", label: "merge" })
             )
-            const gate = yield* options.gates(context.workDir, lane)
+            // The whole output is kept: the reason below names only the
+            // failing lines, which is not enough to tell why.
+            const gate = yield* options.gates(context.workDir, lane, {
+              files,
+              dir: mergeGateLogDir(options.stateDir, story.id)
+            })
             // Charged only with what the merge added: the head before it had
-            // a baseline (run start or the previous merge).
+            // a baseline (run start or the previous merge). A new line green
+            // on one rerun of the test gate is flaky, as in the worktree.
             const before = yield* ensureBaseline(
               checkpoint,
               Effect.succeed(ReviewResult.make({ issues: [] })),
               context.workDir
             )
-            const verdict = triageGates(gate, before, rootsOf(context.workDir))
-            if (!verdict.blocking.isClean) {
+            const blocking = yield* applyTriage(
+              gate,
+              {
+                baseline: Effect.succeed(before),
+                roots: rootsOf(context.workDir),
+                ...(options.testGate === undefined
+                  ? {}
+                  : { rerunTest: options.testGate(context.workDir, lane) })
+              },
+              lane,
+              yield* Ref.make<ReadonlySet<string>>(new Set())
+            )
+            if (!blocking.isClean) {
               // Never leave a red epic head for the next story to inherit.
               yield* context.git.rollback(checkpoint)
-              return yield* failed(
-                story,
-                `epic gates failed after merging; merge undone:\n${issueLines(verdict.blocking)}`
-              )
+              return blocking
             }
             if (gateCommands !== undefined) {
               const mergedHead = yield* context.git.checkpoint
@@ -1032,6 +1090,7 @@ export const implementStoriesFlow = Effect.fn("@llm4ts/flow/Stories.implement")(
             yield* laneOf(story).publish(
               Info.make({ message: `story ${story.id}: merged into ${epicBranch}` })
             )
+            return undefined
           })
         )
       )
@@ -1144,7 +1203,9 @@ export const implementStoriesFlow = Effect.fn("@llm4ts/flow/Stories.implement")(
   /** Everything that happens inside a story's worktree: plan, implement, judge, perimeter. */
   const implementStory = Effect.fn("@llm4ts/flow/Stories.implementStory")(function* (
     story: Story,
-    state: StoryState
+    state: StoryState,
+    /** The epic gates' failures after the story's last merge: run as a revision task. */
+    mergeFailure?: ReviewResult
   ): Effect.fn.Return<
     {
       readonly judge: string | undefined
@@ -1196,6 +1257,8 @@ export const implementStoriesFlow = Effect.fn("@llm4ts/flow/Stories.implement")(
     )
     /** What the coder claimed to have run and did not, task by task, for the judge. */
     const evidenceNotes: Array<string> = []
+    /** Findings left for a follow-up round (ADR 0031), written when the story finishes. */
+    const deferred: Array<string> = []
     const checkEvidence = (
       task: Task,
       trailer: Trailer,
@@ -1466,6 +1529,34 @@ export const implementStoriesFlow = Effect.fn("@llm4ts/flow/Stories.implement")(
       yield* git.commitAll(`${story.id}: put paths outside the perimeter back`)
     })
 
+    /** A revision task on the story's own plan, run by the task loop like any other. */
+    const appendRevision = (subject: string, lead: string, verdict: ReviewResult) =>
+      Effect.gen(function* () {
+        const store = makePlanStore(files)
+        const current = yield* store.load(planPath(story))
+        if (current === undefined) {
+          return
+        }
+        const number = current.tasks.filter((task) => revisionTitle.test(task.title)).length + 1
+        yield* store.save(
+          planPath(story),
+          Plan.make({
+            ...current,
+            tasks: [
+              ...current.tasks,
+              Task.make({
+                title: `Revision ${number}: ${subject}`,
+                description: [
+                  lead,
+                  "Close these gaps without weakening any test and without leaving your owned paths:",
+                  issueLines(verdict)
+                ].join("\n")
+              })
+            ]
+          })
+        )
+      })
+
     const planTasks = options.planTasks ?? defaultPlanTasks
     // Taken after catch-up and setup: the code as the first task finds it.
     const startingCode = yield* startingCodeOf(
@@ -1513,13 +1604,21 @@ export const implementStoriesFlow = Effect.fn("@llm4ts/flow/Stories.implement")(
       // the story's findings log for people to read.
       reviewCache: { files, dir: reviewCacheDir(story) },
       onReview: (task, round, result, settled) =>
-        appendFindings(
-          story,
-          `review "${task.title}" round ${round} — ${
-            result.isClean ? "clean" : `${result.issues.length} issue(s)`
-          }${settled ? "" : ", fixing"}`,
-          result
-        ),
+        Effect.gen(function* () {
+          // A settled round's leftovers are deferred, not dropped: when the
+          // rounds ran out, blocking ones too, since the task commits anyway.
+          const left = settled && deferring ? result.issues : []
+          for (const issue of left) {
+            deferred.push(deferredLine(`review "${task.title}"`, issue))
+          }
+          yield* appendFindings(
+            story,
+            `review "${task.title}" round ${round} — ${
+              result.isClean ? "clean" : `${result.issues.length} issue(s)`
+            }${settled ? (left.length === 0 ? "" : ", deferred") : ", fixing"}`,
+            result
+          )
+        }),
       onTaskReply: (task, _reply, trailer, startedAt) => checkEvidence(task, trailer, startedAt),
       // The repository's own review rules ride along as one extra lens
       // (ADR 0027 decision 9), read from the epic checkout.
@@ -1528,8 +1627,22 @@ export const implementStoriesFlow = Effect.fn("@llm4ts/flow/Stories.implement")(
       ...(options.fixer === undefined ? {} : { fixer: options.fixer }),
       ...(options.stall === undefined ? {} : { stall: options.stall }),
       ...(options.reviewers === undefined ? {} : { reviewers: options.reviewers }),
-      ...(options.maxRounds === undefined ? {} : { maxRounds: options.maxRounds })
+      ...(options.maxRounds === undefined ? {} : { maxRounds: options.maxRounds }),
+      ...(deferring ? { settle: "blocking" as const } : {})
     })
+    if (mergeFailure !== undefined) {
+      // The branch already caught up with the epic above: the gates in the
+      // worktree now see the combination that was red after the merge.
+      yield* appendRevision(
+        "make the epic gates green after merging",
+        [
+          `The story "${story.title}" was merged into ${epicBranch} and the epic's gates went red, so the merge was undone.`,
+          `Your branch now includes everything merged into ${epicBranch} since you started: the failures`,
+          "come from your change combined with that work. Fix them on your side."
+        ].join("\n"),
+        mergeFailure
+      )
+    }
     yield* guarded(implementTasks, implementTasks)
 
     // Other stories may have merged while this one ran: judge the branch
@@ -1540,33 +1653,13 @@ export const implementStoriesFlow = Effect.fn("@llm4ts/flow/Stories.implement")(
     // A judge finding becomes a revision task on the story's own plan, run by
     // the task loop like any other: coder, review, gates, commit.
     const addRevision = (verdict: ReviewResult, empty: boolean): Effect.Effect<void, FlowError> =>
-      Effect.gen(function* () {
-        const store = makePlanStore(files)
-        const current = yield* store.load(planPath(story))
-        if (current === undefined) {
-          return
-        }
-        const number = current.tasks.filter((task) => revisionTitle.test(task.title)).length + 1
-        yield* store.save(
-          planPath(story),
-          Plan.make({
-            ...current,
-            tasks: [
-              ...current.tasks,
-              Task.make({
-                title: `Revision ${number}: close the judge's findings`,
-                description: [
-                  empty
-                    ? `The story "${story.title}" has no changes yet, and what it must provide is not all in place on the epic branch.`
-                    : `The story "${story.title}" was judged short of done.`,
-                  "Close these gaps without weakening any test and without leaving your owned paths:",
-                  issueLines(verdict)
-                ].join("\n")
-              })
-            ]
-          })
-        )
-      })
+      appendRevision(
+        "close the judge's findings",
+        empty
+          ? `The story "${story.title}" has no changes yet, and what it must provide is not all in place on the epic branch.`
+          : `The story "${story.title}" was judged short of done.`,
+        verdict
+      )
 
     let judgeNote: string | undefined
     let inPlace = false
@@ -1684,11 +1777,17 @@ export const implementStoriesFlow = Effect.fn("@llm4ts/flow/Stories.implement")(
             dimensions: verdict instanceof StoryVerdict ? verdict.dimensions : []
           })
         )
-        if (verdict.isClean) {
+        // Deferring, a verdict with nothing blocking clears: what it found
+        // waits for the follow-up round instead of another revision.
+        if (verdict.isClean || (deferring && !verdict.issues.some(isBlocking))) {
+          const waiting = nonBlockingIssues(verdict)
+          for (const issue of waiting) {
+            deferred.push(deferredLine(`judge round ${round}`, issue))
+          }
           inPlace = empty
-          judgeNote = empty
-            ? `verified already in place (round ${round})`
-            : `judge cleared (round ${round})`
+          judgeNote = `${empty ? "verified already in place" : "judge cleared"} (round ${round}${
+            waiting.length === 0 ? "" : `, ${waiting.length} finding(s) deferred`
+          })`
           break
         }
         if (round >= judgeRounds) {
@@ -1709,6 +1808,19 @@ export const implementStoriesFlow = Effect.fn("@llm4ts/flow/Stories.implement")(
 
     const changed = yield* git.changedFilesVsBase(epicBranch)
     yield* enforcePerimeter(changed, story)
+    if (deferring) {
+      // A merge revision adds to what the story's first run deferred.
+      if (mergeFailure !== undefined) {
+        const earlier = yield* files.read(deferredPath(options.stateDir, story.id))
+        deferred.unshift(...(earlier ?? "").split("\n").filter((line) => line.startsWith("- [")))
+      }
+      yield* deferred.length === 0
+        ? files.remove(deferredPath(options.stateDir, story.id))
+        : files.writeAtomic(
+            deferredPath(options.stateDir, story.id),
+            [`# Deferred from ${story.id}: ${story.title}`, "", ...new Set(deferred), ""].join("\n")
+          )
+    }
     const totals = seats.totals === undefined ? undefined : yield* seats.totals
     const history = roster === undefined ? [] : yield* roster.history
     return {
@@ -1736,14 +1848,38 @@ export const implementStoriesFlow = Effect.fn("@llm4ts/flow/Stories.implement")(
         judge: "merged on a previous run"
       })
     }
-    const result = yield* Effect.scoped(implementStory(story, state))
-    if (result.inPlace) {
-      // Verified already on the epic: there is nothing to merge or re-gate.
+    let result = yield* Effect.scoped(implementStory(story, state))
+    for (let revision = 0; ; revision += 1) {
+      if (result.inPlace) {
+        // Verified already on the epic: there is nothing to merge or re-gate.
+        yield* laneOf(story).publish(
+          Info.make({ message: `story ${story.id}: verified already in place; nothing to merge` })
+        )
+        break
+      }
+      const red = yield* integrate(story, state.branch)
+      if (red === undefined) {
+        break
+      }
+      const logs = red.issues.flatMap((issue) =>
+        issue.logPath === undefined ? [] : [issue.logPath]
+      )
+      const why = `${issueLines(red)}${logs.length === 0 ? "" : `\nfull output: ${logs.join(", ")}`}`
+      if (revision >= mergeRevisions) {
+        return yield* failed(
+          story,
+          `epic gates failed after merging${
+            revision === 0 ? "" : ` (after ${revision} merge revision${revision === 1 ? "" : "s"})`
+          }; merge undone:\n${why}`
+        )
+      }
+      yield* appendFindings(story, `epic gates after merging — red, merge undone; revising`, red)
       yield* laneOf(story).publish(
-        Info.make({ message: `story ${story.id}: verified already in place; nothing to merge` })
+        Info.make({
+          message: `story ${story.id}: epic gates red after merging; merge undone, sending it back to catch up and fix (merge revision ${revision + 1} of ${mergeRevisions})`
+        })
       )
-    } else {
-      yield* integrate(story, state.branch)
+      result = yield* Effect.scoped(implementStory(story, state, red))
     }
     const last = result.executor?.split(" → ").at(-1)
     yield* saveVersioned(