@llm4ts/shell 0.2.2 → 0.3.1

package/flows/modernize-bench.ts

@@ -0,0 +1,362 @@
+// Legacy modernization benchmark: measure an extraction run and report the cost of a wave.
+//
+// Two modes:
+//
+//   report  — `modernize-bench --repo <anything> report` (or LLM4TS_BENCH_MODE=report)
+//             loads bench-results.jsonl next to the launch directory and prints
+//             the comparison report. LLM4TS_BENCH_PROJECT=<n> adds the per-wave
+//             projection modernize-survey embeds in its wave plan.
+//
+//   measure — the default. Runs the extraction pipeline over the estate at
+//             `--repo`, taps the flow's own events for per-stage tokens, cost,
+//             and self-healing counters, then appends one schema-versioned
+//             BenchRecord to bench-results.jsonl. Extraction is the dominant
+//             cost of a modernization wave, so its measurement is what a wave
+//             projection needs.
+//
+// The measured run writes into the estate exactly as modernize-extract would;
+// point it at a disposable fixture copy, not a live estate.
+//
+// Run: modernize-bench --repo ~/estates/fixture-copy
+import { arch, cpus, hostname, totalmem, type as osType } from "node:os"
+import { join } from "node:path"
+import * as Effect from "effect/Effect"
+import { Sample } from "@llm4ts/core/eval/Eval"
+import type { TokenUsage } from "@llm4ts/core/Models"
+import { judge } from "@llm4ts/core/eval/Judge"
+import type { LlmServiceShape } from "@llm4ts/core/LlmService"
+import {
+  BenchJudge,
+  BenchMachine,
+  BenchQuality,
+  BenchRecord,
+  BenchScore,
+  CurrentBenchSchema,
+  makeBenchPhase,
+  makeBenchTap,
+  type BenchPhase
+} from "@llm4ts/flow/Bench"
+import { loadBenchRecords, appendBenchRecord, renderBenchReport } from "@llm4ts/flow/BenchReport"
+import { FlowAborted, FlowLlmError } from "@llm4ts/flow/FlowError"
+import { Info, TokensUsed } from "@llm4ts/flow/FlowEvents"
+import { packageVersion } from "@llm4ts/flow/Package"
+import { loadPack } from "@llm4ts/flow/Pack"
+import { stage } from "@llm4ts/flow/PlanExecution"
+import { mergeReviewResults } from "@llm4ts/flow/Review"
+import { coverage, coverageUnits, features, matchingFiles } from "@llm4ts/flow/SpecChecks"
+import {
+  ProgramArtifacts,
+  ProgramUnit,
+  extractProgramsResumably
+} from "@llm4ts/modernize/Artifacts"
+import { asReadOnly, coderFromEnv } from "@llm4ts/runner/Connectors"
+import { resolveFlowInput } from "@llm4ts/runner/FlowArgs"
+import { runFlowMain, runNode } from "@llm4ts/runner/FlowRunner"
+import { nodePlainFileStore } from "@llm4ts/runner/NodePlainFileStore"
+import { makeNodeWorkspace } from "@llm4ts/runner/NodeWorkspace"
+import { reviewFingerprint } from "@llm4ts/runner/ReviewFingerprint"
+
+const ModDir = "docs/modernization"
+const BenchFile = "bench-results.jsonl"
+
+const programName = (relativePath: string): string => {
+  const base = relativePath.slice(relativePath.lastIndexOf("/") + 1)
+  const dot = base.lastIndexOf(".")
+  return dot > 0 ? base.slice(0, dot) : base
+}
+
+const machine = (): BenchMachine =>
+  BenchMachine.make({
+    hostname: hostname(),
+    os: osType(),
+    arch: arch(),
+    cores: cpus().length,
+    memoryGb: Math.round((totalmem() / 1024 ** 3) * 10) / 10,
+    runtime: `node ${process.versions.node}`
+  })
+
+const program = Effect.gen(function* () {
+  const input = yield* resolveFlowInput("Benchmark a modernization extraction run")
+  const packDir = process.env.LLM4TS_PACK ?? "packs/cobol-springboot"
+  const mode =
+    process.env.LLM4TS_BENCH_MODE?.trim() ??
+    (input.prompt.trim().toLowerCase() === "report" ? "report" : "measure")
+  const files = nodePlainFileStore
+  const benchPath = join(input.workspace, BenchFile)
+  const coder = coderFromEnv(process.env)
+  const startedAt = new Date()
+
+  if (mode === "report") {
+    const records = yield* loadBenchRecords(files, benchPath)
+    if (records.length === 0) {
+      return yield* FlowAborted.make({ message: `no benchmark records at ${benchPath}` })
+    }
+    const projected = Number.parseInt(process.env.LLM4TS_BENCH_PROJECT ?? "", 10)
+    process.stdout.write(
+      `${renderBenchReport(records, Number.isFinite(projected) && projected > 0 ? projected : undefined)}\n`
+    )
+    return
+  }
+
+  yield* runNode(
+    {
+      workDir: input.workDir,
+      workspace: input.workspace,
+      userPrompt: input.prompt,
+      coder,
+      reasoning: asReadOnly(coder),
+      environment: process.env
+    },
+    (context) =>
+      Effect.scoped(
+        Effect.gen(function* () {
+          const launchWorkspace = yield* makeNodeWorkspace(input.workspace)
+          const estate = yield* makeNodeWorkspace(input.workDir)
+          const pack = yield* stage(context.events, "pack", loadPack(launchWorkspace, packDir))
+          const modDirAbs = join(input.workDir, ModDir)
+          // The tap observes the same events the terminal renders, so the
+          // measurement never needs its own instrumentation inside the flow.
+          const tap = yield* makeBenchTap(context.events)
+          const phases: Array<BenchPhase> = []
+
+          const measured = <A, E>(name: string, body: Effect.Effect<A, E>) =>
+            Effect.gen(function* () {
+              yield* tap.reset
+              const began = Date.now()
+              const value = yield* stage(context.events, name, body)
+              const observation = yield* tap.get
+              phases.push(makeBenchPhase(observation, name, Date.now() - began))
+              return value
+            })
+
+          const programs = yield* measured(
+            "inventory",
+            matchingFiles(estate, pack.programs ?? pack.sources ?? ".*")
+          )
+          if (programs.length === 0) {
+            return yield* FlowAborted.make({
+              message: `no source units matched the pack's programs/sources regex under ${input.workDir}`
+            })
+          }
+          const units = programs.map((relativePath) =>
+            ProgramUnit.make({ name: programName(relativePath), sourcePath: relativePath })
+          )
+          const system = [
+            pack.prompt("analysis"),
+            pack.lessons === undefined
+              ? undefined
+              : `Lessons from previous modernization runs — apply them:\n${pack.lessons}`
+          ]
+            .filter((part) => part !== undefined)
+            .join("\n\n")
+
+          /**
+           * Structured calls report usage only to their caller; the Chat seam
+           * is what normally publishes `TokensUsed` (ADR 0005). A benchmark
+           * that skipped this would measure wall-clock and nothing else, so
+           * every measured call republishes its own usage for the tap.
+           */
+          const publishUsage = (agent: string, usage: TokenUsage, model: string | undefined) =>
+            context.events.publish(
+              TokensUsed.make({
+                agent,
+                usage,
+                ...(model === undefined ? {} : { model })
+              })
+            )
+
+          yield* measured(
+            "extract",
+            extractProgramsResumably(
+              files,
+              units,
+              (unit) =>
+                context.coder
+                  .executeStructuredWithUsage(
+                    [
+                      system,
+                      "",
+                      `Extract the behavioural spec for ONE source unit: ${unit.sourcePath}`,
+                      'Respond only with JSON: {"spec":"…","feature":"…","traceability":"…","mapping":"…"}.',
+                      pack.prompt("spec") ?? "",
+                      pack.prompt("bdd") ?? ""
+                    ].join("\n"),
+                    ProgramArtifacts,
+                    { type: "object" }
+                  )
+                  .pipe(
+                    Effect.mapError(FlowLlmError.from),
+                    Effect.tap(([, usage, model]) =>
+                      usage === undefined ? Effect.void : publishUsage("coder", usage, model)
+                    ),
+                    Effect.map(([artifacts]) => artifacts)
+                  ),
+              modDirAbs
+            )
+          )
+
+          // The deterministic half of the extraction gate, measured on its own:
+          // it is the part a slower model cannot make cheaper.
+          const deterministic = yield* measured(
+            "gate",
+            Effect.gen(function* () {
+              const fragments: Array<string> = []
+              for (const unit of units) {
+                const text = yield* files.read(join(modDirAbs, "traceability", `${unit.name}.md`))
+                if (text !== undefined) {
+                  fragments.push(text)
+                }
+              }
+              const trace = fragments.join("\n\n")
+              const covered = yield* coverage(estate, pack.coverage, trace)
+              const wellFormed = yield* features(estate, `${ModDir}/features`)
+              return mergeReviewResults([covered, wellFormed])
+            })
+          )
+
+          /**
+           * The evaluator has no usage-reporting variant, so the seat it runs
+           * on is wrapped instead: every structured call it makes reports its
+           * usage to the tap before returning the value.
+           */
+          const metered: LlmServiceShape = {
+            ...context.reasoning,
+            executeStructured: (prompt, schema, jsonSchema) =>
+              context.reasoning.executeStructuredWithUsage(prompt, schema, jsonSchema).pipe(
+                Effect.tap(([, usage, model]) =>
+                  usage === undefined ? Effect.void : publishUsage("reasoning", usage, model)
+                ),
+                Effect.map(([value]) => value)
+              )
+          }
+
+          const judged = yield* measured(
+            "judge",
+            Effect.gen(function* () {
+              const packJudge = judge(metered, pack.judgeDimensions)
+              const scores: Array<BenchScore> = []
+              let findings = 0
+              for (const unit of units) {
+                const spec = (yield* files.read(join(modDirAbs, "specs", `${unit.name}.md`))) ?? ""
+                const source = (yield* files.read(join(input.workDir, unit.sourcePath))) ?? ""
+                const scored = yield* packJudge
+                  .evaluate(Sample.make({ response: spec, context: source, query: input.prompt }))
+                  .pipe(Effect.mapError(FlowLlmError.from))
+                for (const score of scored.scores) {
+                  const max = pack.judgeDimensions.find((d) => d.name === score.name)?.maxScore ?? 2
+                  scores.push(
+                    BenchScore.make({
+                      name: `${unit.name}:${score.name}`,
+                      score: score.score,
+                      max,
+                      reasoning: score.reasoning
+                    })
+                  )
+                  if (score.score < max) {
+                    findings += 1
+                  }
+                }
+              }
+              return { scores, findings }
+            })
+          )
+
+          const quality = yield* Effect.gen(function* () {
+            const specFiles = yield* matchingFiles(estate, `^${ModDir}/specs/.*\\.md$`).pipe(
+              Effect.orElseSucceed(() => [])
+            )
+            const featureFiles = yield* matchingFiles(
+              estate,
+              `^${ModDir}/features/.*\\.feature$`
+            ).pipe(Effect.orElseSucceed(() => []))
+            let scenarios = 0
+            for (const path of featureFiles) {
+              const text = yield* estate.read(path).pipe(Effect.orElseSucceed(() => ""))
+              scenarios += text
+                .split(/\r?\n/)
+                .filter((line) => line.trim().startsWith("Scenario")).length
+            }
+            let sourceLines = 0
+            for (const path of programs) {
+              const text = yield* estate.read(path).pipe(Effect.orElseSucceed(() => ""))
+              sourceLines += text.split(/\r?\n/).length
+            }
+            const units_ = yield* coverageUnits(estate, pack.coverage)
+            const malformed = (yield* features(estate, `${ModDir}/features`)).issues.length
+            return BenchQuality.make({
+              featureFiles: featureFiles.length,
+              scenarios,
+              malformedFeatures: malformed,
+              specFiles: specFiles.length,
+              programs: programs.length,
+              sourceFiles: programs.length,
+              sourceLines,
+              testFiles: featureFiles.length,
+              testLines: scenarios,
+              deterministicFindings: deterministic.issues.length,
+              judgeFindings: judged.findings,
+              gateCleared: deterministic.issues.length === 0 && judged.findings === 0,
+              gateOpenIssues: deterministic.issues.length + judged.findings,
+              judgeScores: judged.scores,
+              // The rule universe is the estate's own coverage units.
+              coveragePct:
+                Object.values(units_).flat().length === 0
+                  ? 0
+                  : Math.round(
+                      (Object.values(units_)
+                        .flat()
+                        .filter((unit) => unit.length > 0).length /
+                        Object.values(units_).flat().length) *
+                        100
+                    )
+            })
+          })
+
+          // The fingerprint groups records of the SAME estate so the report can
+          // compare providers rather than mixing unrelated fixtures.
+          const fingerprint = reviewFingerprint(
+            pack.name,
+            ...[...programs].sort(),
+            String(quality.sourceLines)
+          ).slice(0, 16)
+
+          const finishedAt = new Date()
+          const record = BenchRecord.make({
+            schemaVersion: CurrentBenchSchema,
+            runId: `${startedAt.toISOString()}-${process.pid}`,
+            startedAt: startedAt.toISOString(),
+            finishedAt: finishedAt.toISOString(),
+            provider: coder.connectorId.value,
+            modelRequested: coder.model ?? "(harness default)",
+            modelsServed: [],
+            judge: BenchJudge.make({
+              provider: coder.connectorId.value,
+              model: coder.model ?? "(harness default)",
+              fixed: false
+            }),
+            llm4tsVersion: packageVersion,
+            pack: pack.name,
+            fixtureFingerprint: fingerprint,
+            machine: machine(),
+            outcome: quality.gateCleared === true ? "completed" : "gate-open",
+            totalMs: finishedAt.getTime() - startedAt.getTime(),
+            phases,
+            quality
+          })
+
+          yield* appendBenchRecord(files, benchPath, record)
+          yield* context.events.publish(
+            Info.make({
+              message:
+                `benchmark appended to ${benchPath} — ${record.totalTokens} token(s) over ` +
+                `${phases.length} phase(s); report with LLM4TS_BENCH_MODE=report`
+            })
+          )
+          const all = yield* loadBenchRecords(files, benchPath)
+          process.stdout.write(`${renderBenchReport(all)}\n`)
+        })
+      )
+  )
+})
+
+runFlowMain(program)

package/flows/modernize-extract.ts

@@ -23,16 +23,18 @@
 // plan. Judge context is bounded by LLM4TS_JUDGE_SOURCES_LIMIT (chars).
 import { join } from "node:path"
 import * as Effect from "effect/Effect"
-import { Sample } from "@llm4ts/core/eval/Eval"
+import { Sample, type EvalResult } from "@llm4ts/core/eval/Eval"
 import { judge } from "@llm4ts/core/eval/Judge"
 import type { JsonSchema } from "@llm4ts/core/Models"
 import { FlowAborted, FlowLlmError } from "@llm4ts/flow/FlowError"
 import { Info } from "@llm4ts/flow/FlowEvents"
 import { makeChat } from "@llm4ts/flow/Chat"
 import { loadPack, type Pack } from "@llm4ts/flow/Pack"
+import { loadPatternCards, matchingPatternCards } from "@llm4ts/flow/Patterns"
 import { stage } from "@llm4ts/flow/PlanExecution"
 import { defaultPlanInstructions, planFrom } from "@llm4ts/flow/Planner"
 import { ReviewIssue, ReviewResult, mergeReviewResults } from "@llm4ts/flow/Review"
+import { cachedReview } from "@llm4ts/flow/ReviewCache"
 import { coverage, coverageUnits, features, matchingFiles } from "@llm4ts/flow/SpecChecks"
 import { withDraftApproval, requireApproval } from "@llm4ts/modernize/Approval"
 import {
@@ -43,6 +45,7 @@ import {
 import { asReadOnly, coderFromEnv } from "@llm4ts/runner/Connectors"
 import { resolveFlowInput } from "@llm4ts/runner/FlowArgs"
 import { runFlowMain, runNode } from "@llm4ts/runner/FlowRunner"
+import { reviewFingerprint } from "@llm4ts/runner/ReviewFingerprint"
 import { nodePlainFileStore } from "@llm4ts/runner/NodePlainFileStore"
 import { makeNodeWorkspace } from "@llm4ts/runner/NodeWorkspace"
 
@@ -267,6 +270,13 @@ const program = Effect.gen(function* () {
           ProgramUnit.make({ name: programName(rel), sourcePath: rel })
         )
 
+        // Pattern cards are selected deterministically from the SOURCE, never by
+        // the model: implementation later injects exactly the cards cited here.
+        const cards = [
+          ...(yield* loadPatternCards(launchWorkspace, `${packDir}/patterns`)),
+          ...(yield* loadPatternCards(launchWorkspace, "patterns"))
+        ]
+
         // One structured analyst call per program, resumable per program: a rerun
         // skips every program whose spec exists, and each program gets its own commit.
         yield* stage(
@@ -290,11 +300,51 @@ const program = Effect.gen(function* () {
                         ProgramArtifacts,
                         programArtifactsJsonSchema
                       )
-                      .pipe(Effect.mapError(FlowLlmError.from))
+                      .pipe(
+                        Effect.mapError(FlowLlmError.from),
+                        // A turn-limit trip is the wedged-agent tail, not a
+                        // failure: keep whatever the analyst already produced.
+                        Effect.catchIf(
+                          (error) => error.cause?._tag === "TurnLimitError",
+                          (error) =>
+                            Effect.gen(function* () {
+                              const existing = yield* files.read(
+                                join(modDirAbs, "specs", `${target.name}.md`)
+                              )
+                              if (existing === undefined) {
+                                return yield* Effect.fail(error)
+                              }
+                              yield* context.events.publish(
+                                Info.make({
+                                  message: `turn limit hit on ${target.sourcePath} after its spec was written — keeping the work`
+                                })
+                              )
+                              return ProgramArtifacts.make({
+                                spec: existing,
+                                feature: "",
+                                traceability: "",
+                                mapping: ""
+                              })
+                            })
+                        )
+                      )
                   }),
                 modDirAbs
               )
               if (summary.created.length > 0) {
+                // Tag the traceability fragment with the cards this program's
+                // source matches — regex-decided, so implementation's playbook
+                // is reproducible.
+                const source = yield* files.read(join(input.workDir, unit.sourcePath))
+                const matched = source === undefined ? [] : matchingPatternCards(source, cards)
+                if (matched.length > 0) {
+                  const fragmentPath = join(modDirAbs, "traceability", `${unit.name}.md`)
+                  const fragment = (yield* files.read(fragmentPath)) ?? ""
+                  yield* files.writeAtomic(
+                    fragmentPath,
+                    `${fragment.trimEnd()}\n\nPatterns: ${matched.map((card) => card.id).join(", ")}\n`
+                  )
+                }
                 yield* context.git
                   .commitAll(`modernize(${pack.name}): spec ${unit.name}`)
                   .pipe(Effect.asVoid)
@@ -355,6 +405,50 @@ const program = Effect.gen(function* () {
         const packJudge = judge(context.reasoning, pack.judgeDimensions)
         const limit = judgeSourcesLimit()
 
+        /**
+         * An empty structured response that survives the in-run retries is
+         * usually DETERMINISTIC (context overflow), so repeating the same
+         * prompt cannot succeed — retry at half, then quarter context instead.
+         */
+        const judgeWithShrink = (spec: string, feature: string, source: string) => {
+          const attempt = (
+            cap: number,
+            rest: ReadonlyArray<number>
+          ): Effect.Effect<EvalResult, FlowLlmError> =>
+            packJudge
+              .evaluate(
+                Sample.make({
+                  response: capText(`${spec}\n\n${feature}`, cap),
+                  context: capText(source, cap),
+                  query: input.prompt
+                })
+              )
+              .pipe(
+                Effect.mapError(FlowLlmError.from),
+                Effect.catchIf(
+                  (error) => rest.length > 0 && error.message.includes("empty response"),
+                  (error) => {
+                    const [next, ...remaining] = rest
+                    return context.events
+                      .publish(
+                        Info.make({
+                          message: `judge returned empty at cap ${cap} chars — shrinking to ${next ?? cap}: ${error.message}`
+                        })
+                      )
+                      .pipe(Effect.andThen(attempt(next ?? cap, remaining)))
+                  }
+                )
+              )
+          return attempt(limit, [Math.floor(limit / 2), Math.floor(limit / 4)])
+        }
+
+        /**
+         * Judging is resumable per program: the verdict persists under
+         * `gate/<NAME>.json`, fingerprinted over the source, spec, feature, and
+         * rubric it judged. Unchanged content reuses the stored verdict with NO
+         * model call, so a crash or quota death re-judges only what changed.
+         * Delete `gate/` to force a full re-judge.
+         */
         const judgeProgram = (unit: ProgramUnit) =>
           Effect.gen(function* () {
             const spec = (yield* files.read(join(modDirAbs, "specs", `${unit.name}.md`))) ?? ""
@@ -363,17 +457,20 @@ const program = Effect.gen(function* () {
                 join(modDirAbs, "features", `${unit.name.toLowerCase()}.feature`)
               )) ?? ""
             const source = (yield* files.read(join(input.workDir, unit.sourcePath))) ?? ""
-            yield* context.events.publish(Info.make({ message: `judging ${unit.name}` }))
-            const scored = yield* packJudge
-              .evaluate(
-                Sample.make({
-                  response: capText(`${spec}\n\n${feature}`, limit),
-                  context: capText(source, limit),
-                  query: input.prompt
-                })
+            const rubric = pack.judgeDimensions
+              .map(
+                (dimension) => `${dimension.name} (0..${dimension.maxScore}): ${dimension.rubric}`
+              )
+              .join("\n")
+            return yield* cachedReview(
+              files,
+              join(modDirAbs, "gate", `${unit.name}.json`),
+              reviewFingerprint(source, spec, feature, rubric),
+              context.events.publish(Info.make({ message: `judging ${unit.name}` })).pipe(
+                Effect.andThen(judgeWithShrink(spec, feature, source)),
+                Effect.map((scored) => judgeIssues(pack, scored, unit.name))
               )
-              .pipe(Effect.mapError(FlowLlmError.from))
-            return judgeIssues(pack, scored, unit.name)
+            )
           })
 
         const gateEvaluate = Effect.gen(function* () {
@@ -432,7 +529,21 @@ const program = Effect.gen(function* () {
             const turn = (ask: string, commitMessage: string) =>
               Effect.gen(function* () {
                 const chat = yield* makeChat(context.coder, { system })
-                yield* chat.ask(ask)
+                yield* chat.ask(ask).pipe(
+                  Effect.asVoid,
+                  // A wedged agent that trips its turn limit mid-fix still wrote
+                  // something; re-evaluate what landed instead of failing.
+                  Effect.catchIf(
+                    (error) => error._tag === "Llm" && error.cause?._tag === "TurnLimitError",
+                    () =>
+                      context.events.publish(
+                        Info.make({
+                          message:
+                            "turn limit hit during a fix turn — re-evaluating what was written"
+                        })
+                      )
+                  )
+                )
                 yield* context.git.commitAll(commitMessage).pipe(Effect.asVoid)
               })
             for (const [name, issues] of [...scoped.entries()].sort()) {