@llm4ts/flow 2.45.0 → 2.47.0

package/src/Stories.ts

@@ -148,6 +148,50 @@ export class StoryState extends Schema.Class<StoryState>("StoryState")({
   executor: Schema.optionalKey(Schema.String)
 }) {}
 
+/**
+ * What a story's red phase left (ADR 0042): the test paths it froze, the
+ * commit that holds them, and what it parked on the way. Written once; its
+ * presence is what tells a resumed story the red phase is done.
+ */
+export const RedTestsVersion = 1
+
+export class RedTests extends Schema.Class<RedTests>("RedTests")({
+  frozen: Schema.Array(Schema.String),
+  commit: Schema.String,
+  /** Criteria the red phase could not prove red, each with why: parked for the operator. */
+  parked: Schema.Array(Schema.String)
+}) {}
+
+/** What a red phase answers: the test paths to freeze and the criteria it parked. */
+export interface RedPhase {
+  readonly frozen: ReadonlyArray<string>
+  readonly parked: ReadonlyArray<string>
+}
+
+/** Where a story's red phase is recorded, beside its task plan. */
+export const redTestsPath = (stateDir: string, storyId: string): string =>
+  join(stateDir, `stories/${storyId}.red.json`)
+
+/** A change to a test the red phase froze: a gate failure, whatever the change. */
+export const frozenTestsGate = (
+  changed: ReadonlyArray<string>,
+  frozen: ReadonlyArray<string>
+): ReviewResult => {
+  const touched = changed.filter((path) => frozen.includes(path))
+  return ReviewResult.make({
+    issues: touched.map((path) =>
+      ReviewIssue.make({
+        severity: "Critical",
+        title: `frozen red test changed: ${path}`,
+        description:
+          "The red phase froze this test: make it pass by changing production code, and put the test back as it was.",
+        file: path
+      })
+    ),
+    summary: touched.length === 0 ? "" : "frozen red tests"
+  })
+}
+
 export const EpicReportVersion = 1
 
 /** `waiting`: on hold behind a failed predecessor, runs once that is fixed. */
@@ -724,6 +768,19 @@ export interface StoriesOptions {
   readonly contextChars?: number
   /** Extra system context per story (house rules, shared read-only excerpts). */
   readonly system?: (story: Story) => Effect.Effect<string, FlowError>
+  /**
+   * Test-first (ADR 0042): runs once per story, in its worktree, before its
+   * tasks are planned — writes the red tests and proves them red. The
+   * executor commits what it wrote and freezes the paths it names: a later
+   * change to any of them is a gate failure. Omit for no red phase.
+   */
+  readonly redPhase?: (
+    story: Story,
+    seats: StorySeats,
+    workDir: string,
+    system: string,
+    events: FlowEventsShape
+  ) => Effect.Effect<RedPhase, FlowError>
   /** How a story's task plan is produced. Default: the story's own coder plans it. */
   readonly planTasks?: (
     seats: StorySeats,
@@ -1145,6 +1202,7 @@ export const implementStoriesFlow = Effect.fn("@llm4ts/flow/Stories.implement")(
       // The notes its tasks carried describe the old definition: gone too.
       yield* files.remove(planPath(story))
       yield* files.remove(notesPath(story))
+      yield* files.remove(redTestsPath(options.stateDir, story.id))
       stored = undefined
     }
     if (stored !== undefined && stored.status === "merged") {
@@ -1394,6 +1452,13 @@ export const implementStoriesFlow = Effect.fn("@llm4ts/flow/Stories.implement")(
             )
             return ReviewResult.make({ issues, summary: issues.length === 0 ? "" : "oracle guard" })
           })
+    // The red phase's tests, once written (ADR 0042): frozen for every later change.
+    let redTests = yield* loadVersioned(
+      files,
+      redTestsPath(options.stateDir, story.id),
+      RedTestsVersion,
+      RedTests
+    )
     /**
      * The gates in a worktree of the story: the target's, the story's
      * perimeter, the oracle guard and — for a task run aside — the paths the
@@ -1420,12 +1485,28 @@ export const implementStoriesFlow = Effect.fn("@llm4ts/flow/Stories.implement")(
                 ],
                 task.owns
               )
+        const frozen =
+          redTests === undefined
+            ? ReviewResult.make({ issues: [] })
+            : frozenTestsGate(
+                [
+                  ...(yield* where.changedFilesVsBase(redTests.commit)),
+                  ...(yield* where.uncommittedFiles)
+                ],
+                redTests.frozen
+              )
         return combined(
           combined(
-            combined(target, perimeterGate([...changed.sharedReadOnly, ...changed.outside], story)),
-            yield* oracleGate(target, where)
+            combined(
+              combined(
+                target,
+                perimeterGate([...changed.sharedReadOnly, ...changed.outside], story)
+              ),
+              yield* oracleGate(target, where)
+            ),
+            owned
           ),
-          owned
+          frozen
         )
       })
     const gates = gatesAt(state.worktree, git, storyGateLogDir(options.stateDir, story.id))
@@ -1599,6 +1680,35 @@ export const implementStoriesFlow = Effect.fn("@llm4ts/flow/Stories.implement")(
         )
       })
 
+    // Test-first (ADR 0042): the red tests before any task is planned, once.
+    // A story whose task plan exists already ran it, or predates it.
+    if (
+      options.redPhase !== undefined &&
+      redTests === undefined &&
+      (yield* files.read(planPath(story))) === undefined
+    ) {
+      const red = yield* stage(
+        laneOf(story),
+        `story ${story.id}: red tests`,
+        options.redPhase(story, watchedSeats, state.worktree, system, laneOf(story))
+      )
+      if ((yield* git.uncommittedFiles).length > 0) {
+        yield* git.commitAll(`${story.id}: red tests`)
+      }
+      redTests = RedTests.make({
+        frozen: [...red.frozen],
+        commit: yield* git.checkpoint,
+        parked: [...red.parked]
+      })
+      yield* saveVersioned(
+        files,
+        redTestsPath(options.stateDir, story.id),
+        RedTestsVersion,
+        RedTests,
+        redTests
+      )
+    }
+
     const planTasks = options.planTasks ?? defaultPlanTasks
     // Taken after catch-up and setup: the code as the first task finds it.
     const startingCode = yield* startingCodeOf(

package/src/StoryPlan.ts

@@ -45,6 +45,12 @@ export class Story extends Schema.Class<Story>("Story")({
     Schema.withConstructorDefault(Effect.succeed([])),
     Schema.withDecodingDefaultKey(Effect.succeed([]))
   ),
+  /**
+   * Outcomes cut from this iteration (ADR 0042, epic-mvp): planned, shown,
+   * never built now. `acceptance` is the MVP; a story with only `later`
+   * outcomes is parked whole. Absent outside epic-mvp.
+   */
+  later: Schema.optionalKey(Schema.Array(Schema.String)),
   /**
    * The story may delete, skip or rewrite tests (ADR 0027): the oracle guard
    * stays silent for it. Absent or false, a deleted test file, an added skip
@@ -376,6 +382,7 @@ export const storyHash = (story: Story): string =>
         ? {}
         : { readFirst: [...story.readFirst].map(normalizePath) }),
       ...(story.acceptance.length === 0 ? {} : { acceptance: [...story.acceptance] }),
+      ...(story.later === undefined || story.later.length === 0 ? {} : { later: [...story.later] }),
       ...(story.testsChange ? { testsChange: true } : {})
     })
   )

package/src/TestEvidence.ts

@@ -0,0 +1,137 @@
+// Per-criterion test evidence (ADR 0042): every MVP acceptance criterion is
+// encoded by tests whose names carry its tag, `[<story-id>#<n>]`, and the
+// criterion is green only when the test run says so — from a JUnit XML
+// report, or from one filtered test command per criterion. Pass/fail counts
+// alone (GateTriage, OracleGuard) cannot say WHICH criterion holds.
+import * as Schema from "effect/Schema"
+import type { OracleRules } from "./OracleGuard.ts"
+import { defaultOracleRules } from "./OracleGuard.ts"
+
+/** The tag tests of a criterion carry in their name: `[story-id#2]` for its second criterion. */
+export const criterionTag = (storyId: string, index: number): string => `[${storyId}#${index}]`
+
+/** The tags of a story's MVP criteria, in order. */
+export const criterionTags = (
+  storyId: string,
+  acceptance: ReadonlyArray<string>
+): ReadonlyArray<string> => acceptance.map((_, index) => criterionTag(storyId, index + 1))
+
+export const TestCaseStatus = Schema.Literals(["passed", "failed", "skipped"])
+export type TestCaseStatus = typeof TestCaseStatus.Type
+
+export interface TestCase {
+  readonly name: string
+  readonly classname: string
+  readonly file: string | undefined
+  readonly status: TestCaseStatus
+}
+
+const attribute = (attributes: string, name: string): string | undefined => {
+  const match = new RegExp(`(?:^|\\s)${name}\\s*=\\s*"([^"]*)"`, "u").exec(attributes)
+  return match?.[1] === undefined ? undefined : unescapeXml(match[1])
+}
+
+const unescapeXml = (text: string): string =>
+  text
+    .replace(/&lt;/g, "<")
+    .replace(/&gt;/g, ">")
+    .replace(/&quot;/g, '"')
+    .replace(/&apos;/g, "'")
+    .replace(/&#(\d+);/g, (_, code: string) => String.fromCodePoint(Number(code)))
+    .replace(/&amp;/g, "&")
+
+/**
+ * The test cases of a JUnit XML report — the format vitest, jest, pytest,
+ * Surefire/Gradle and cargo-nextest all write. A case with a `<failure>` or
+ * `<error>` failed, one with `<skipped>` was skipped, any other passed.
+ */
+export const parseJUnit = (xml: string): ReadonlyArray<TestCase> => {
+  const cases: Array<TestCase> = []
+  const pattern = /<testcase\b([^>]*?)(?:\/>|>([\s\S]*?)<\/testcase>)/gu
+  for (const match of xml.matchAll(pattern)) {
+    const attributes = match[1] ?? ""
+    const body = match[2] ?? ""
+    const status: TestCaseStatus = /<(?:failure|error)\b/u.test(body)
+      ? "failed"
+      : /<skipped\b/u.test(body)
+        ? "skipped"
+        : "passed"
+    cases.push({
+      name: attribute(attributes, "name") ?? "",
+      classname: attribute(attributes, "classname") ?? "",
+      file: attribute(attributes, "file"),
+      status
+    })
+  }
+  return cases
+}
+
+/**
+ * `missing`: no test names the criterion. `red`: one does and not all of
+ * them pass. `green`: some do and every one passes.
+ */
+export const CriterionStatus = Schema.Literals(["missing", "red", "green"])
+export type CriterionStatus = typeof CriterionStatus.Type
+
+const named = (testCase: TestCase, tag: string): boolean =>
+  testCase.name.includes(tag) || testCase.classname.includes(tag)
+
+/**
+ * A criterion's status from a report. A tag the report does not show but
+ * the test sources carry (`written`) is red: a suite that cannot even load
+ * — the module under test does not exist yet — reports no case.
+ */
+export const statusFromReport = (
+  tag: string,
+  cases: ReadonlyArray<TestCase>,
+  written: boolean
+): CriterionStatus => {
+  const mine = cases.filter((testCase) => named(testCase, tag))
+  if (mine.length === 0) {
+    return written ? "red" : "missing"
+  }
+  return mine.every((testCase) => testCase.status === "passed") ? "green" : "red"
+}
+
+/**
+ * Where the evidence comes from: the JUnit report the test gate writes
+ * (`LLM4TS_TEST_REPORT`, relative to the app directory), or a command run
+ * once per criterion with `{tag}` replaced (`LLM4TS_TEST_ONE`), green when
+ * it exits 0.
+ */
+export type EvidenceSource =
+  | { readonly _tag: "Report"; readonly path: string }
+  | { readonly _tag: "Command"; readonly argv: ReadonlyArray<string> }
+
+export const evidenceSourceFromEnvironment = (
+  environment: Readonly<Record<string, string | undefined>>
+): EvidenceSource | undefined => {
+  const report = environment.LLM4TS_TEST_REPORT?.trim()
+  if (report !== undefined && report.length > 0) {
+    return { _tag: "Report", path: report }
+  }
+  const command = environment.LLM4TS_TEST_ONE?.trim()
+  if (command !== undefined && (command.includes("{tag}") || command.includes("{tagRegex}"))) {
+    return { _tag: "Command", argv: command.split(/\s+/u) }
+  }
+  return undefined
+}
+
+const escapeRegex = (text: string): string => text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")
+
+/**
+ * The per-criterion command with its tag in place: `{tag}` verbatim,
+ * `{tagRegex}` escaped for runners whose name filter is a regex (vitest `-t`,
+ * jest `-t`, pytest `-k` takes neither — use `{tag}` there).
+ */
+export const commandFor = (argv: ReadonlyArray<string>, tag: string): ReadonlyArray<string> =>
+  argv.map((part) => part.split("{tagRegex}").join(escapeRegex(tag)).split("{tag}").join(tag))
+
+/** The changed paths that are test files by the oracle's rules: what a red phase freezes. */
+export const testPaths = (
+  paths: ReadonlyArray<string>,
+  rules: OracleRules = defaultOracleRules
+): ReadonlyArray<string> => {
+  const pattern = new RegExp(rules.testFiles, "u")
+  return paths.filter((path) => pattern.test(path))
+}