@llm4ts/flow 0.10.0 → 0.11.0

package/src/ProgramJudge.ts

@@ -0,0 +1,186 @@
+import * as Effect from "effect/Effect"
+import { Sample, type Dimension, type EvalResult } from "@llm4ts/core/eval/Eval"
+import type { Evaluator } from "@llm4ts/core/eval/Evaluator"
+import { capped, withShrink } from "./Context.ts"
+import { FlowLlmError, type FlowError } from "./FlowError.ts"
+import { FlowEvents, Info } from "./FlowEvents.ts"
+import type { GitToolShape } from "./GitTool.ts"
+import type { Pack } from "./Pack.ts"
+import type { PlainFileStoreShape } from "./Persistence.ts"
+import { ReviewIssue, ReviewResult, mergeReviewResults } from "./Review.ts"
+import { cachedReview } from "./ReviewCache.ts"
+
+// Per-program spec-compliance judging, shared by the implement and review
+// phases. A whole-branch judge call carries every spec and every diff hunk in
+// the estate, which is what blows a provider's input window. Judging one
+// program at a time against only that program's slice of the diff keeps each
+// call small, and — wrapped in the review cache — makes the gate resumable: an
+// unchanged program reuses its stored verdict with no LLM call.
+
+export interface GroupedFiles {
+  readonly byProgram: Readonly<Record<string, ReadonlyArray<string>>>
+  readonly unassigned: ReadonlyArray<string>
+}
+
+/**
+ * Partition `changed` by which program's file regex matches. A file matching
+ * several programs is judged with each of them (a shared bridge class is
+ * genuinely part of both). The remainder — build files, shared utilities — is
+ * returned separately for the unassigned pass.
+ */
+export const groupFiles = (
+  pack: Pack,
+  programs: ReadonlyArray<string>,
+  changed: ReadonlyArray<string>
+): GroupedFiles => {
+  const byProgram = Object.fromEntries(
+    programs.map((program) => {
+      const matches = pack.filesFor(program)
+      return [program, changed.filter((file) => matches.test(file))] as const
+    })
+  )
+  const assigned = new Set(Object.values(byProgram).flat())
+  return {
+    byProgram,
+    unassigned: changed.filter((file) => !assigned.has(file))
+  }
+}
+
+export interface ProgramJudgeOptions {
+  readonly pack: Pack
+  readonly judge: Evaluator<Sample>
+  readonly dimensions: ReadonlyArray<Dimension>
+  readonly git: GitToolShape
+  readonly files: PlainFileStoreShape
+  /** Directory holding the per-program verdict cache (`<gateDir>/<NAME>.json`). */
+  readonly gateDir: string
+  readonly base: string
+  readonly programs: ReadonlyArray<string>
+  /** A program's spec text; the caller owns where specs live. */
+  readonly specFor: (program: string) => Effect.Effect<string, FlowError>
+  readonly query: string
+  /** Content fingerprint for the verdict cache (hashing lives outside flow). */
+  readonly fingerprint: (...parts: ReadonlyArray<string>) => string
+}
+
+const join = (root: string, path: string): string =>
+  `${root.replace(/[\\/]+$/, "")}/${path.replace(/^[\\/]+/, "")}`
+
+const rubricText = (dimensions: ReadonlyArray<Dimension>): string =>
+  dimensions
+    .map((dimension) => `${dimension.name} (0..${dimension.maxScore}): ${dimension.rubric}`)
+    .join("\n")
+
+/**
+ * Sub-bar dimensions as Critical review issues, titled with the program they
+ * belong to — the same shape the extract gate produces, so fix loops and
+ * `ReviewResult.isClean` work unchanged.
+ */
+const issues = (
+  scored: EvalResult,
+  dimensions: ReadonlyArray<Dimension>,
+  program: string
+): ReviewResult => {
+  const subBar = scored.scores.filter(
+    (score) =>
+      score.score < (dimensions.find((dimension) => dimension.name === score.name)?.maxScore ?? 2)
+  )
+  return ReviewResult.make({
+    issues: subBar.map((score) =>
+      ReviewIssue.make({
+        severity: "Critical",
+        title: `judge[${program}]: ${score.name} scored ${score.score}`,
+        description: score.reasoning
+      })
+    ),
+    summary: `judge:${program}`
+  })
+}
+
+/**
+ * A spec'd program with NO matching changed file is a deterministic gate
+ * failure, not a silent pass. Skipping it would let the branch clear a bar the
+ * old whole-branch judge would have failed. It also surfaces a mis-set
+ * `programFiles:` immediately — the top documented risk of the per-program
+ * design — instead of quietly degrading coverage.
+ */
+export const unimplemented = (programs: ReadonlyArray<string>): ReviewResult =>
+  ReviewResult.make({
+    issues: programs.map((program) =>
+      ReviewIssue.make({
+        severity: "Critical",
+        title: `judge[${program}]: spec'd but no implementation files changed`,
+        description:
+          `${program} has a committed spec but no file on this branch matches the pack's ` +
+          "programFiles regex for it. Either the program is unimplemented, or the pack's " +
+          "`programFiles:` template does not match this repo's layout — check that before " +
+          "assuming the former."
+      })
+    ),
+    summary: "judge:unimplemented"
+  })
+
+const judgeSlice = Effect.fn("@llm4ts/flow/ProgramJudge.judgeSlice")(function* (
+  options: ProgramJudgeOptions,
+  label: string,
+  spec: string,
+  diff: string
+): Effect.fn.Return<ReviewResult, FlowError, FlowEvents> {
+  const events = yield* FlowEvents
+  const rubric = rubricText(options.dimensions)
+  return yield* cachedReview(
+    options.files,
+    join(options.gateDir, `${label}.json`),
+    options.fingerprint(spec, diff, rubric),
+    events.publish(Info.make({ message: `judging ${label}` })).pipe(
+      Effect.andThen(
+        withShrink(`judge[${label}]`, (cap) =>
+          Effect.gen(function* () {
+            const cappedSpec = yield* capped(`spec[${label}]`, spec, cap)
+            const cappedDiff = yield* capped(`diff[${label}]`, diff, cap)
+            return yield* options.judge
+              .evaluate(
+                Sample.make({ response: cappedDiff, context: cappedSpec, query: options.query })
+              )
+              .pipe(Effect.mapError(FlowLlmError.from))
+          })
+        )
+      ),
+      Effect.map((scored) => issues(scored, options.dimensions, label))
+    )
+  )
+})
+
+/**
+ * Judge every program whose files changed, plus one pass over the unassigned
+ * remainder. Each verdict is cached at `gateDir/<NAME>.json`, fingerprinted
+ * over the spec, the diff slice, and the rubric it judged — so re-running
+ * after a crash re-judges only what changed.
+ */
+export const judgeAllPrograms = Effect.fn("@llm4ts/flow/ProgramJudge.judgeAll")(function* (
+  options: ProgramJudgeOptions
+): Effect.fn.Return<ReviewResult, FlowError, FlowEvents> {
+  const changed = yield* options.git.changedFilesVsBase(options.base)
+  const { byProgram, unassigned } = groupFiles(options.pack, options.programs, changed)
+  const active = options.programs.filter((program) => (byProgram[program] ?? []).length > 0)
+  const untouched = options.programs.filter((program) => (byProgram[program] ?? []).length === 0)
+
+  const perProgram: Array<ReviewResult> = []
+  for (const program of active) {
+    const spec = yield* options.specFor(program)
+    // The file slice comes from groupFiles — re-deriving it per program would
+    // be N+1 git invocations for the same answer.
+    const diff = yield* options.git.diffVsBaseScoped(options.base, byProgram[program] ?? [])
+    perProgram.push(yield* judgeSlice(options, program, spec, diff))
+  }
+
+  if (unassigned.length > 0) {
+    const diff = yield* options.git.diffVsBaseScoped(options.base, unassigned)
+    const specs = yield* Effect.forEach(options.programs, options.specFor).pipe(
+      Effect.map((all) => all.join("\n\n"))
+    )
+    perProgram.push(yield* judgeSlice(options, "unassigned", specs, diff))
+  }
+
+  return mergeReviewResults([...perProgram, unimplemented(untouched)])
+})

package/src/Provenance.ts

@@ -13,7 +13,16 @@ export class Provenance extends Schema.Class<Provenance>("Provenance")({
   specs: Schema.Record(Schema.String, Schema.String),
   gateVerdicts: Schema.Record(Schema.String, Schema.String),
   equivalenceReport: Schema.optionalKey(Schema.String),
-  fixSpecs: Schema.Array(Schema.String)
+  fixSpecs: Schema.Array(Schema.String),
+  // Context truncations recorded while producing this evidence
+  // (Context.renderTruncation strings). A gate verdict rendered on a
+  // partially-read spec pack says so HERE — that visibility is the whole
+  // reason truncation is allowed at all. Defaulted so manifests written
+  // before this field still load.
+  contextTruncations: Schema.Array(Schema.String).pipe(
+    Schema.withDecodingDefaultKey(Effect.succeed([])),
+    Schema.withConstructorDefault(Effect.succeed([]))
+  )
 }) {}
 
 const join = (root: string, path: string): string =>

package/src/Survey.ts

@@ -29,6 +29,43 @@ export class SurveyGraph extends Schema.Class<SurveyGraph>("SurveyGraph")({
   }
 }
 
+/**
+ * The transitive dependency closure of `program` as repo-relative paths,
+ * breadth-first, excluding the program itself. The `seen` set is required, not
+ * optional bookkeeping: COBOL copybook graphs genuinely contain cycles (a
+ * copybook that COPYs something which eventually COPYs back), and without it
+ * this walk would never terminate on a real estate. Truncated to `maxFiles` so
+ * a program pulling hundreds of copybooks still gets a bounded, visible subset
+ * instead of an unbounded read.
+ */
+export const closureFor = (
+  graph: SurveyGraph,
+  program: string,
+  maxFiles: number
+): ReadonlyArray<string> => {
+  const pathOf = new Map(graph.nodes.map((node) => [node.name, node.path]))
+  const walk = (
+    frontier: ReadonlyArray<string>,
+    seen: ReadonlySet<string>,
+    acc: ReadonlyArray<string>
+  ): ReadonlyArray<string> => {
+    if (frontier.length === 0 || acc.length >= maxFiles) {
+      return acc.slice(0, maxFiles)
+    }
+    const next = [
+      ...new Set(frontier.flatMap((from) => graph.outgoing(from).map((edge) => edge.to)))
+    ].filter((name) => !seen.has(name))
+    return walk(next, new Set([...seen, ...next]), [
+      ...acc,
+      ...next.flatMap((name) => {
+        const path = pathOf.get(name)
+        return path === undefined ? [] : [path]
+      })
+    ])
+  }
+  return walk([program], new Set([program]), [])
+}
+
 const unitName = (path: string): string => {
   const base = path.split("/").at(-1) ?? path
   const dot = base.lastIndexOf(".")

package/src/TransientRetry.ts

@@ -27,12 +27,40 @@ const transientSignals: ReadonlyArray<string> = [
   " 504"
 ]
 
+// An "empty response" is ambiguous: gemini returns one both for a random mid-stream
+// flake and for a prompt too large to start on. This classifier sees only the message,
+// so it deliberately treats both as flaky — a fresh process fixes the common (flake)
+// case, and the oversized-prompt case is resolved a layer up by Context.withShrink.
+// Do not route empty responses into isContextOverflow.
 const flakyStreamSignals: ReadonlyArray<string> = [
   "empty response",
   "malformed tool call",
   "invalid stream"
 ]
 
+// Deterministic client errors a retry can never fix. Gemini wraps every error in
+// "[API Error: {...}]", including 400s, so the "api error" transient signal would
+// otherwise swallow them.
+const deterministic4xxSignals: ReadonlyArray<string> = [
+  "invalid_argument",
+  '"code": 400',
+  '"code":400',
+  "code=400",
+  "exceeds the maximum number of tokens"
+]
+
+// The message signatures of a prompt-too-large failure, in one place: isContextOverflow
+// matches them on a typed LlmError, and Context.withShrink matches them on a bare
+// message string when no typed cause survived. Two copies of this list would drift.
+const contextOverflowSignals: ReadonlyArray<string> = [
+  "exceeds the maximum number of tokens",
+  "input token count exceeds",
+  "context length exceeded",
+  "maximum context length",
+  "prompt is too long",
+  "request too large"
+]
+
 const includesSignal = (message: string, signals: ReadonlyArray<string>): boolean => {
   const normalized = message.toLowerCase()
   return signals.some((signal) => normalized.includes(signal))
@@ -41,13 +69,29 @@ const includesSignal = (message: string, signals: ReadonlyArray<string>): boolea
 export const isFlakyStream = (error: LlmError): boolean =>
   error._tag === "ProviderError" && includesSignal(error.message, flakyStreamSignals)
 
+const isDeterministic4xx = (message: string): boolean =>
+  includesSignal(message, deterministic4xxSignals)
+
+export const isContextOverflowMessage = (message: string): boolean =>
+  includesSignal(message, contextOverflowSignals)
+
+// The prompt was larger than the model's input window. Deterministic: the same prompt
+// always fails, so it is not transient — it routes to Context.withShrink, which retries
+// at a smaller budget.
+export const isContextOverflow = (error: LlmError): boolean =>
+  error._tag === "ProviderError" && isContextOverflowMessage(error.message)
+
 export const isTransient = (error: LlmError): boolean => {
   switch (error._tag) {
     case "TimeoutError":
     case "RateLimitError":
       return true
     case "ProviderError":
-      return !isFlakyStream(error) && includesSignal(error.message, transientSignals)
+      return (
+        !isFlakyStream(error) &&
+        !isDeterministic4xx(error.message) &&
+        includesSignal(error.message, transientSignals)
+      )
     default:
       return false
   }