@llm4ts/flow 2.30.1 → 2.31.0

package/src/Flow.ts

@@ -1,10 +1,15 @@
+import { join } from "node:path"
+import * as Clock from "effect/Clock"
 import * as Effect from "effect/Effect"
+import * as Ref from "effect/Ref"
 import type { LlmServiceShape } from "@llm4ts/core/LlmService"
 import { collect } from "@llm4ts/core/Streaming"
 import { withToolActivity } from "./Activity.ts"
 import {
   appendNote,
   findingsIn,
+  trailerIn,
+  type Trailer,
   findingsRequest,
   withNotes,
   type CarriedNotes
@@ -21,16 +26,23 @@ import {
   type FlowEventsShape
 } from "./FlowEvents.ts"
 import type { Plan, Task } from "./Plan.ts"
-import type { PlanStoreShape } from "./Persistence.ts"
+import type { PlainFileStoreShape, PlanStoreShape } from "./Persistence.ts"
+import { ensureBaseline } from "./GateTriage.ts"
 import { implementTaskLoop, stage } from "./PlanExecution.ts"
 import { truth } from "@llm4ts/core/judgment/Schemas"
 import { certaintyOf, decide, judgmentOf, type JudgmentMode } from "./Judgment.ts"
 import {
+  applyTriage,
   minimalReviewers,
   reviewAndFixLoop,
+  withOracle,
+  type FixPromptOptions,
+  type GateTriageOptions,
+  type OracleGateOptions,
   type ReviewResult,
   type ReviewCacheLocation
 } from "./Review.ts"
+import type { OracleRules } from "./OracleGuard.ts"
 import type { Reviewer } from "./Reviewer.ts"
 
 export { publishUsage, structuredAndPublish } from "./Usage.ts"
@@ -77,6 +89,28 @@ export interface ImplementPlanOptions {
   readonly maxRounds?: number
   readonly lint?: Effect.Effect<ReviewResult, FlowError>
   readonly format?: Effect.Effect<void, FlowError>
+  /** Charge the lint gate only with failures the task caused (ADR 0027). */
+  readonly triage?: GateTriageOptions
+  /**
+   * Without `triage`: record the gates' result on each task's starting
+   * commit under this store and triage against it (ADR 0027). The gates run
+   * once per new commit, before the task; `lint` is what they are.
+   */
+  readonly baseline?: {
+    readonly files: PlainFileStoreShape
+    readonly dir: string
+    readonly commands: ReadonlyArray<ReadonlyArray<string>>
+    readonly appDir?: string
+  }
+  /**
+   * The oracle guard (ADR 0027 decision 4) over each task's diff: deleted
+   * test files, skip or focus markers and a passed-count drop (against the
+   * baseline, when both counts are known) fail the round unless
+   * `testsChange` is true.
+   */
+  readonly oracle?: { readonly rules?: OracleRules; readonly testsChange?: boolean }
+  /** How gate output reaches the coder in the fix prompt. */
+  readonly fix?: FixPromptOptions
   /** Where each review lens's answer is kept, so a rerun over the same diff asks nothing. */
   readonly reviewCache?: ReviewCacheLocation
   /** Told after every review round of every task, settled or not. */
@@ -111,6 +145,17 @@ export interface ImplementPlanOptions {
    * appended. Omit to carry nothing.
    */
   readonly carry?: CarriedNotes
+  /**
+   * Called with every task's final reply and its parsed trailer, after the
+   * findings are kept (ADR 0027 decision 6): the caller checks the
+   * `verified:` claims against what actually ran.
+   */
+  readonly onTaskReply?: (
+    task: Task,
+    reply: string,
+    trailer: Trailer,
+    startedAt: number
+  ) => Effect.Effect<void, FlowError>
 }
 
 /** The judgment form of the empty-diff probe: one Truth question over the coder's reply. */
@@ -166,6 +211,40 @@ const defaultCommitMessage = (plan: Plan, task: Task): string => `${plan.epicId}
 const composeSystem = (base: string | undefined, note: string): string =>
   [base, note].filter((part): part is string => part !== undefined && part.length > 0).join("\n\n")
 
+/**
+ * The task's triage: the explicit one, or one built from `baseline` over the
+ * commit the task starts from (HEAD now), or none.
+ */
+const triageFor = (
+  context: FlowContextShape,
+  options: ImplementPlanOptions
+): Effect.Effect<GateTriageOptions | undefined, FlowError> =>
+  Effect.gen(function* () {
+    if (options.triage !== undefined) {
+      return options.triage
+    }
+    if (options.baseline === undefined || options.lint === undefined) {
+      return undefined
+    }
+    const base = options.baseline
+    const appDir = base.appDir ?? "."
+    const roots = [join(context.workDir, appDir), context.workDir]
+    const commit = yield* context.git.checkpoint
+    return {
+      baseline: ensureBaseline({
+        files: base.files,
+        stateDir: base.dir,
+        commit,
+        appDir,
+        commands: base.commands,
+        run: options.lint,
+        roots,
+        now: Clock.currentTimeMillis
+      }),
+      roots
+    }
+  })
+
 export const implementPlanFlow = Effect.fn("@llm4ts/flow/Flow.implementPlan")(function* (
   context: FlowContextShape,
   options: ImplementPlanOptions
@@ -208,13 +287,17 @@ export const implementPlanFlow = Effect.fn("@llm4ts/flow/Flow.implementPlan")(fu
         // details. `planSoFar`, threaded through by implementTaskLoop, is the
         // single source of truth for completion progress instead.
         const notes = options.carry === undefined ? undefined : yield* options.carry.read
+        const taskStartedAt = yield* Clock.currentTimeMillis
         /** What the task's reply learned goes to the carried notes, whichever turn did the work. */
         const keepFindings = (reply: string): Effect.Effect<void, FlowError> =>
           Effect.gen(function* () {
-            const found = options.carry === undefined ? undefined : findingsIn(reply)
+            const found = findingsIn(reply)
             if (options.carry !== undefined && found !== undefined) {
               yield* options.carry.write(appendNote(notes, task.title, found))
             }
+            if (options.onTaskReply !== undefined) {
+              yield* options.onTaskReply(task, reply, trailerIn(found), taskStartedAt)
+            }
           })
         yield* keepFindings(
           yield* coder.ask(
@@ -270,6 +353,20 @@ export const implementPlanFlow = Effect.fn("@llm4ts/flow/Flow.implementPlan")(fu
             })
           }
         }
+        const triage = yield* triageFor(context, options)
+        const oracle: OracleGateOptions | undefined =
+          options.oracle === undefined
+            ? undefined
+            : {
+                diff: context.git.diffAll,
+                ...(options.oracle.rules === undefined ? {} : { rules: options.oracle.rules }),
+                ...(options.oracle.testsChange === undefined
+                  ? {}
+                  : { declared: options.oracle.testsChange }),
+                ...(triage === undefined
+                  ? {}
+                  : { baseCount: Effect.map(triage.baseline, (base) => base?.passedCount) })
+              }
         yield* reviewAndFixLoop({
           reviewers: options.reviewers ?? minimalReviewers,
           reviewerService: flowReviewer(context),
@@ -280,6 +377,9 @@ export const implementPlanFlow = Effect.fn("@llm4ts/flow/Flow.implementPlan")(fu
           ...(options.maxRounds === undefined ? {} : { maxRounds: options.maxRounds }),
           ...(options.lint === undefined ? {} : { lint: options.lint }),
           ...(options.format === undefined ? {} : { format: options.format }),
+          ...(triage === undefined ? {} : { triage }),
+          ...(oracle === undefined ? {} : { oracle }),
+          ...(options.fix === undefined ? {} : { fix: options.fix }),
           ...(options.reviewCache === undefined ? {} : { cache: options.reviewCache }),
           ...(options.onReview === undefined
             ? {}
@@ -291,7 +391,13 @@ export const implementPlanFlow = Effect.fn("@llm4ts/flow/Flow.implementPlan")(fu
               })
         })
         if (options.lint !== undefined) {
-          const gate = yield* options.lint
+          const reported = yield* Ref.make<ReadonlySet<string>>(new Set())
+          const gate = yield* withOracle(
+            yield* applyTriage(yield* options.lint, triage, context.events, reported),
+            oracle,
+            context.events,
+            yield* Ref.make(true)
+          )
           if (!gate.isClean) {
             return yield* FlowAborted.make({
               message: [

package/src/FlowEvents.ts

@@ -171,6 +171,18 @@ export class CapabilityUnenforceable extends Schema.TaggedClass<CapabilityUnenfo
   }
 ) {}
 
+/** A task's `verified:` claims against the tool calls its transcript shows (ADR 0027). */
+export class EvidenceChecked extends Schema.TaggedClass<EvidenceChecked>()("EvidenceChecked", {
+  task: Schema.String,
+  claimed: Schema.Int,
+  /** Claims no tool call carried: fabricated status. */
+  unverified: Schema.Int,
+  confidence: Schema.optionalKey(Schema.Literals(["high", "medium", "low"])),
+  /** The run kept no tool transcript for this coder (an API seat), so nothing was checked. */
+  unchecked: Schema.optionalKey(Schema.Boolean),
+  lane: Schema.optionalKey(Schema.String)
+}) {}
+
 export class Declassified extends Schema.TaggedClass<Declassified>()("Declassified", {
   label: Schema.String
 }) {}
@@ -334,6 +346,7 @@ export const FlowEvent = Schema.Union([
   CapabilityUsedEvent,
   CapabilityDeniedEvent,
   CapabilityUnenforceable,
+  EvidenceChecked,
   Declassified,
   ExecutorLeased,
   ExecutorReleased,

package/src/GateTriage.ts

@@ -0,0 +1,250 @@
+// Gates with a memory (ADR 0027): what the target's gates said on the code a
+// change started from, and which failing lines the change can be charged
+// with. Pure helpers and a small store; the seams are lintCommand and
+// reviewAndFixLoop (Review.ts), implementPlanFlow (Flow.ts) and the story
+// executor (Stories.ts). Gates.ts re-exports this module beside gatesIn.
+import * as Effect from "effect/Effect"
+import * as Schema from "effect/Schema"
+import type { PersistenceError } from "./FlowError.ts"
+import type { PlainFileStoreShape } from "./Persistence.ts"
+import { ReviewIssue, ReviewResult } from "./Review.ts"
+import { fingerprintOf } from "./ReviewCache.ts"
+
+export const GateClass = Schema.Literals(["red", "hang", "crash"])
+export type GateClass = typeof GateClass.Type
+
+export const FailureOrigin = Schema.Literals(["new", "base", "flaky"])
+export type FailureOrigin = typeof FailureOrigin.Type
+
+export class GateBaseline extends Schema.Class<GateBaseline>("GateBaseline")({
+  baseCommit: Schema.String,
+  appDir: Schema.String,
+  /** Each gate command as configured, joined with one space. */
+  commands: Schema.Array(Schema.String),
+  failingLines: Schema.Array(Schema.String),
+  /** Tests passing on the base, when the test gate's output said (the oracle guard compares). */
+  passedCount: Schema.optionalKey(Schema.Int),
+  recordedAt: Schema.Number
+}) {}
+
+/** Every baseline a run recorded, by `baselineKey`; one file, deleted on land. */
+const Baselines = Schema.Record(Schema.String, GateBaseline)
+const BaselinesJson = Schema.fromJsonString(Baselines)
+const decodeBaselines = Schema.decodeUnknownEffect(BaselinesJson)
+const encodeBaselines = Schema.encodeSync(BaselinesJson)
+
+// Built from the escape code point so the pattern holds no control character literal.
+const ansi = new RegExp(`${String.fromCharCode(27)}\\[[0-9;]*m`, "gu")
+const duration = /\b\d+(?:\.\d+)?\s?(?:ms|s|m)\b/gu
+const isoTimestamp = /\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})?/gu
+const stackFrame = /^\s*at\s/u
+const summaryLine =
+  /^\s*(?:Test Files|Tests|Duration|Start at|Snapshots|Found \d+ errors?|\d+ (?:passing|failing|pending))\b/u
+const failureMarker =
+  /FAIL|✗|×|✘|✖|❯|error|Error|AssertionError|Expected|Received|expected|TS\d{4}/u
+
+/**
+ * The lines of a gate's output a change can be blamed for, made comparable
+ * across runs and worktrees: colours, timings, timestamps and the given root
+ * prefixes removed; stack frames, summaries and blank lines dropped; only
+ * lines carrying a failure marker kept, each once. A heuristic by design:
+ * per-tool parsers are kit material (a pack `diagnostics:` command, later).
+ */
+export const normalizeGateOutput = (
+  text: string,
+  roots: ReadonlyArray<string>
+): ReadonlyArray<string> => {
+  const prefixes = roots.filter((root) => root.length > 0).map((root) => root.replace(/\/+$/u, ""))
+  const seen = new Set<string>()
+  const out: Array<string> = []
+  for (const raw of text.split(/\r?\n/u)) {
+    let line = raw.replace(ansi, "")
+    for (const prefix of prefixes) {
+      line = line.replaceAll(`${prefix}/`, "")
+    }
+    line = line.replace(isoTimestamp, "<ts>").replace(duration, "<t>").trim()
+    if (line.length === 0 || stackFrame.test(line) || summaryLine.test(line)) {
+      continue
+    }
+    if (!failureMarker.test(line)) {
+      continue
+    }
+    line = line.replace(/\s+/gu, " ")
+    if (!seen.has(line)) {
+      seen.add(line)
+      out.push(line)
+    }
+  }
+  return out
+}
+
+/** The title prefix `lintCommand` gives every gate issue; nothing else is a gate. */
+export const gateIssuePrefix = "lint failed: "
+
+export const isGateIssue = (issue: ReviewIssue): boolean => issue.title.startsWith(gateIssuePrefix)
+
+/** Every gate issue's normalized failing lines, in order, each once. */
+export const failingLinesOf = (
+  result: ReviewResult,
+  roots: ReadonlyArray<string>
+): ReadonlyArray<string> =>
+  Array.from(
+    new Set(
+      result.issues
+        .filter(isGateIssue)
+        .flatMap((issue) => normalizeGateOutput(issue.description, roots))
+    )
+  )
+
+export interface GateTriage {
+  /** The lint result with only the failures the change caused; clean when it caused none. */
+  readonly blocking: ReviewResult
+  readonly newLines: ReadonlyArray<string>
+  /** Failing lines already red on the base: listed, never charged. */
+  readonly inherited: ReadonlyArray<string>
+}
+
+/**
+ * Splits a lint result against the baseline. A gate issue whose every
+ * failing line is on the baseline is dropped (inherited); one with new lines
+ * is kept with `origin: "new"` and its description reduced to those lines,
+ * so the fix prompt names only what the change broke. A gate issue with no
+ * recognizable failing line at all (a hang, a crash, an unparsed output) is
+ * kept whole. Non-gate issues (the perimeter, the oracle guard) pass through.
+ * Without a baseline the result is returned as is.
+ */
+export const triageGates = (
+  result: ReviewResult,
+  baseline: GateBaseline | undefined,
+  roots: ReadonlyArray<string>
+): GateTriage => {
+  if (baseline === undefined || result.isClean) {
+    return { blocking: result, newLines: [], inherited: [] }
+  }
+  const known = new Set(baseline.failingLines)
+  const newLines: Array<string> = []
+  const inherited: Array<string> = []
+  const issues: Array<ReviewIssue> = []
+  for (const issue of result.issues) {
+    if (!isGateIssue(issue)) {
+      issues.push(issue)
+      continue
+    }
+    const lines = normalizeGateOutput(issue.description, roots)
+    const fresh = lines.filter((line) => !known.has(line))
+    inherited.push(...lines.filter((line) => known.has(line)))
+    if (lines.length > 0 && fresh.length === 0) {
+      continue
+    }
+    newLines.push(...fresh)
+    issues.push(
+      ReviewIssue.make({
+        ...issue,
+        origin: "new",
+        description: fresh.length > 0 ? fresh.join("\n") : issue.description
+      })
+    )
+  }
+  return {
+    blocking: ReviewResult.make({
+      issues,
+      summary: issues.length === 0 ? "lint passed (inherited failures only)" : result.summary
+    }),
+    newLines: Array.from(new Set(newLines)),
+    inherited: Array.from(new Set(inherited))
+  }
+}
+
+export const baselineKey = (parts: {
+  readonly baseCommit: string
+  readonly appDir: string
+  readonly commands: ReadonlyArray<ReadonlyArray<string>>
+}): string =>
+  fingerprintOf([
+    parts.baseCommit,
+    parts.appDir,
+    ...parts.commands.map((command) => command.join(" "))
+  ]).slice(0, 16)
+
+/** The one file a run keeps its baselines in. */
+export const baselinesPath = (stateDir: string): string =>
+  `${stateDir.replace(/\/+$/u, "")}/gates/baselines.json`
+
+const readAll = (
+  files: PlainFileStoreShape,
+  path: string
+): Effect.Effect<Readonly<Record<string, GateBaseline>>, PersistenceError> =>
+  files.read(path).pipe(
+    Effect.flatMap((text) =>
+      text === undefined
+        ? Effect.succeed({})
+        : decodeBaselines(text).pipe(
+            // An unreadable file is no baseline: the caller records a fresh one.
+            Effect.catch(() => Effect.succeed({}))
+          )
+    )
+  )
+
+export const readBaseline = (
+  files: PlainFileStoreShape,
+  stateDir: string,
+  key: string
+): Effect.Effect<GateBaseline | undefined, PersistenceError> =>
+  Effect.map(readAll(files, baselinesPath(stateDir)), (all) => all[key])
+
+export interface EnsureBaselineArgs<E> {
+  readonly files: PlainFileStoreShape
+  readonly stateDir: string
+  /** The commit the change starts from. */
+  readonly commit: string
+  readonly appDir: string
+  readonly commands: ReadonlyArray<ReadonlyArray<string>>
+  /** Runs the gates on that commit; called only when no baseline is stored. */
+  readonly run: Effect.Effect<ReviewResult, E>
+  /** Root prefixes stripped from gate output (the work dir, the app dir). */
+  readonly roots: ReadonlyArray<string>
+  readonly now: Effect.Effect<number>
+}
+
+/**
+ * The baseline for a commit: the stored one, or a fresh one recorded from
+ * `run`. Lazy on purpose, so a second change on an unchanged commit costs
+ * no gate run.
+ */
+export const ensureBaseline = <E>(
+  args: EnsureBaselineArgs<E>
+): Effect.Effect<GateBaseline, E | PersistenceError> =>
+  Effect.gen(function* () {
+    const key = baselineKey({
+      baseCommit: args.commit,
+      appDir: args.appDir,
+      commands: args.commands
+    })
+    const stored = yield* readBaseline(args.files, args.stateDir, key)
+    if (stored !== undefined) {
+      return stored
+    }
+    const result = yield* args.run
+    const baseline = GateBaseline.make({
+      baseCommit: args.commit,
+      appDir: args.appDir,
+      commands: args.commands.map((command) => command.join(" ")),
+      failingLines: failingLinesOf(result, args.roots),
+      ...(result.passed === undefined ? {} : { passedCount: result.passed }),
+      recordedAt: yield* args.now
+    })
+    yield* writeBaseline(args.files, args.stateDir, key, baseline)
+    return baseline
+  })
+
+export const writeBaseline = (
+  files: PlainFileStoreShape,
+  stateDir: string,
+  key: string,
+  baseline: GateBaseline
+): Effect.Effect<void, PersistenceError> => {
+  const path = baselinesPath(stateDir)
+  return Effect.flatMap(readAll(files, path), (all) =>
+    files.writeAtomic(path, encodeBaselines({ ...all, [key]: baseline }))
+  )
+}

package/src/Gates.ts

@@ -0,0 +1,104 @@
+// The target's gates as a module (ADR 0027): run them in a directory with a
+// timeout and a log per gate, remember what they said on a base, and keep
+// only what matters when an epic lands. The pure half lives in
+// GateTriage.ts and is re-exported here.
+import { join } from "node:path"
+import type * as Duration from "effect/Duration"
+import * as Effect from "effect/Effect"
+import type { ProcessExecutorShape } from "@llm4ts/core/ProcessExecutor"
+import type { FlowError } from "./FlowError.ts"
+import type { FlowEventsShape } from "./FlowEvents.ts"
+import { baselinesPath, normalizeGateOutput } from "./GateTriage.ts"
+import type { PlainFileStoreShape } from "./Persistence.ts"
+import { lintCommand, mergeReviewResults, type ReviewResult } from "./Review.ts"
+
+export * from "./GateTriage.ts"
+
+export interface GateLogDir {
+  readonly files: PlainFileStoreShape
+  readonly dir: string
+}
+
+export interface GateRunOptions {
+  readonly timeout?: Duration.Duration
+  /** Where this runner's logs go unless a call names its own; `undefined` writes none. */
+  readonly logDir?: GateLogDir
+}
+
+/** `<index>-<command slug>.log`: one file per gate command, overwritten each run. */
+export const gateLogName = (index: number, command: ReadonlyArray<string>): string =>
+  `${index}-${command
+    .join(" ")
+    .replace(/[^a-z0-9]+/giu, "-")
+    .replace(/^-|-$/gu, "")
+    .toLowerCase()}.log`
+
+/** Runs the gates in a directory, stopping at the first red one (later output would be noise). */
+export const gatesIn =
+  (
+    process: ProcessExecutorShape,
+    events: FlowEventsShape,
+    commands: ReadonlyArray<ReadonlyArray<string>>,
+    options: GateRunOptions = {}
+  ) =>
+  (
+    workDir: string,
+    laneEvents?: FlowEventsShape,
+    log?: GateLogDir
+  ): Effect.Effect<ReviewResult, FlowError> =>
+    Effect.gen(function* () {
+      const sink = log ?? options.logDir
+      const results: Array<ReviewResult> = []
+      for (const [index, command] of commands.entries()) {
+        const result = yield* lintCommand(process, laneEvents ?? events, command, workDir, {
+          ...(options.timeout === undefined ? {} : { timeout: options.timeout }),
+          ...(sink === undefined
+            ? {}
+            : { log: { files: sink.files, path: join(sink.dir, gateLogName(index, command)) } })
+        })
+        results.push(result)
+        if (!result.isClean) {
+          break
+        }
+      }
+      return mergeReviewResults(results)
+    })
+
+/** A landed epic keeps why a gate was red, not the target's whole test output. */
+export const compactGateLog = (text: string, roots: ReadonlyArray<string>): string =>
+  normalizeGateOutput(text, roots).join("\n")
+
+/** The gate log directory of one story under a run's state folder. */
+export const storyGateLogDir = (stateDir: string, storyId: string): string =>
+  join(stateDir, "stories", storyId, "gates")
+
+/**
+ * What `--land` keeps of the gates: no baselines (reproducible), and each
+ * story's gate logs reduced to their failing lines. Logs are enumerated from
+ * the story ids and the gate commands, so no listing is needed.
+ */
+export const compactGateArtifacts = Effect.fn("@llm4ts/flow/Gates.compactArtifacts")(function* (
+  files: PlainFileStoreShape,
+  stateDir: string,
+  storyIds: ReadonlyArray<string>,
+  commands: ReadonlyArray<ReadonlyArray<string>>
+): Effect.fn.Return<{ readonly baselines: number; readonly logs: number }, FlowError> {
+  const path = baselinesPath(stateDir)
+  const hadBaselines = (yield* files.read(path)) !== undefined
+  if (hadBaselines) {
+    yield* files.remove(path)
+  }
+  let logs = 0
+  for (const storyId of storyIds) {
+    for (const [index, command] of commands.entries()) {
+      const logPath = join(storyGateLogDir(stateDir, storyId), gateLogName(index, command))
+      const text = yield* files.read(logPath)
+      if (text === undefined) {
+        continue
+      }
+      yield* files.writeAtomic(logPath, compactGateLog(text, []))
+      logs += 1
+    }
+  }
+  return { baselines: hadBaselines ? 1 : 0, logs }
+})