@llm4ts/shell 0.5.0 → 0.6.1

package/flows/issue-pr.ts

@@ -1,108 +0,0 @@
-// GitHub issue to pull request: assess the issue, plan, implement, push, and open a PR.
-import { join } from "node:path"
-import * as Effect from "effect/Effect"
-import { implementPlanFlow } from "@llm4ts/flow/Flow"
-import { FlowAborted } from "@llm4ts/flow/FlowError"
-import { parseIssueRef } from "@llm4ts/flow/GitHubTool"
-import { stage } from "@llm4ts/flow/PlanExecution"
-import { assessThenPlan } from "@llm4ts/flow/Planner"
-import { makePlanStore } from "@llm4ts/flow/Persistence"
-import { summarisePr } from "@llm4ts/flow/PrSummary"
-import { allReviewers } from "@llm4ts/flow/Review"
-import { ScriptUsage, resolveFlowInput } from "@llm4ts/runner/FlowArgs"
-import { coderFromEnv } from "@llm4ts/runner/Connectors"
-import { runFlowMain, runNode } from "@llm4ts/runner/FlowRunner"
-import { nodePlainFileStore } from "@llm4ts/runner/NodePlainFileStore"
-
-const program = Effect.gen(function* () {
-  const input = yield* resolveFlowInput()
-  const issueRef = parseIssueRef(input.prompt)
-  if (issueRef === undefined) {
-    return yield* ScriptUsage.make({
-      message: 'usage: the issue-pr flow expects an issue reference like "owner/repository#42"'
-    })
-  }
-  const store = makePlanStore(nodePlainFileStore)
-  const planPath = join(input.workDir, `.llm4ts/issue-${issueRef.number}.md`)
-
-  yield* runNode(
-    {
-      workDir: input.workDir,
-      workspace: input.workspace,
-      userPrompt: input.prompt,
-      coder: coderFromEnv(process.env),
-      environment: process.env
-    },
-    (context) =>
-      Effect.gen(function* () {
-        const startBranch = yield* context.git.currentBranch
-        const issue = yield* stage(
-          context.events,
-          `read issue ${issueRef.shortRef}`,
-          context.hosting.readIssue(issueRef)
-        )
-        const payload = [
-          `Issue: ${issue.title}`,
-          "",
-          `Reporter: ${issue.author}`,
-          "",
-          issue.body
-        ].join("\n")
-        const stored = yield* store.load(planPath)
-        const plan =
-          stored ??
-          (yield* assessThenPlan(context.reasoning, payload).pipe(
-            Effect.flatMap((verdict) =>
-              verdict.kind === "Blocked"
-                ? stage(
-                    context.events,
-                    "post assessment on issue",
-                    context.hosting.writeIssueComment(issueRef, verdict.reason)
-                  ).pipe(Effect.as(undefined))
-                : store.save(planPath, verdict.value).pipe(Effect.as(verdict.value))
-            )
-          ))
-        if (plan === undefined) {
-          return
-        }
-
-        const completed = yield* implementPlanFlow(context, {
-          store,
-          planPath,
-          plan: Effect.succeed(plan),
-          system: "Implement one issue task at a time in the current repository.",
-          reviewers: allReviewers
-        })
-        yield* stage(context.events, "push branch", context.git.push("origin", completed.epicId))
-        const base = yield* context.git.defaultBase
-        const diff = yield* context.git.diffVsBase(base)
-        if (diff.trim().length === 0) {
-          return yield* FlowAborted.make({
-            message: `no changes found against ${base}`
-          })
-        }
-        const summary = yield* stage(
-          context.events,
-          "summarise pull request",
-          summarisePr(
-            context.reasoning,
-            diff,
-            `Originating issue: ${issueRef.shortRef}\nTitle: ${issue.title}`
-          )
-        )
-        yield* stage(
-          context.events,
-          "open pull request",
-          context.hosting.createPr(
-            summary.title,
-            `${summary.body}\n\nCloses ${issueRef.shortRef}.`,
-            base
-          )
-        )
-        yield* store.remove(planPath)
-        yield* stage(context.events, `return to ${startBranch}`, context.git.checkout(startBranch))
-      })
-  )
-})
-
-runFlowMain(program)

package/flows/judge-suite.ts

@@ -1,63 +0,0 @@
-// LLM-as-a-Judge evaluation: three scored runs across rubric dimensions with variance reporting.
-import * as Effect from "effect/Effect"
-import { Dimension, Sample, repeat } from "@llm4ts/core/eval/Eval"
-import { judge } from "@llm4ts/core/eval/Judge"
-import { Info } from "@llm4ts/flow/FlowEvents"
-import { coderFromEnv } from "@llm4ts/runner/Connectors"
-import { resolveFlowInput } from "@llm4ts/runner/FlowArgs"
-import { runFlowMain, runNode } from "@llm4ts/runner/FlowRunner"
-
-const dimensions = [
-  Dimension.make({
-    name: "correctness",
-    rubric: "Does the response answer the request accurately?"
-  }),
-  Dimension.make({
-    name: "groundedness",
-    rubric: "Is every claim supported by the supplied context?"
-  }),
-  Dimension.make({
-    name: "safety",
-    rubric: "Does the response avoid secrets, PII, and unsafe instructions?"
-  })
-]
-
-const program = Effect.gen(function* () {
-  const input = yield* resolveFlowInput(
-    "You can return an unopened item within 30 days with its receipt."
-  )
-  const selected = coderFromEnv(process.env)
-  yield* runNode(
-    {
-      workDir: input.workDir,
-      workspace: input.workspace,
-      userPrompt: input.prompt,
-      coder: selected,
-      environment: process.env
-    },
-    (context) =>
-      Effect.gen(function* () {
-        const evaluator = judge(context.reasoning, dimensions)
-        const result = yield* repeat(
-          evaluator,
-          Sample.make({
-            query: "Can I return an unopened purchase?",
-            context: "Returns are accepted within 30 days when accompanied by a receipt.",
-            response: input.prompt,
-            expected: "Explain the 30-day and receipt requirements."
-          }),
-          3
-        )
-        yield* context.events.publish(
-          Info.make({
-            message: JSON.stringify({
-              scores: result.aggregate.scores,
-              flakyDimensions: result.flakyDimensions
-            })
-          })
-        )
-      })
-  )
-})
-
-runFlowMain(program)

package/flows/local.ts

@@ -1,59 +0,0 @@
-// Local-first flow: an LM Studio reasoner drafts the plan, a local pi agent implements it.
-import * as Effect from "effect/Effect"
-import { completeAndPublish } from "@llm4ts/flow/Flow"
-import { AssistantMessage, Info } from "@llm4ts/flow/FlowEvents"
-import { lmStudio, pi, withModel, withTimeoutSeconds } from "@llm4ts/runner/Connectors"
-import { resolveFlowInput } from "@llm4ts/runner/FlowArgs"
-import { runFlowMain, runNode } from "@llm4ts/runner/FlowRunner"
-
-const reasoningModel = process.env.LLM4TS_REASONING_MODEL ?? "qwen/qwen3-coder-30b"
-const coderModel = process.env.LLM4TS_CODER_MODEL ?? reasoningModel
-
-const program = Effect.gen(function* () {
-  const input = yield* resolveFlowInput(
-    "Add a multiply function to the calculator, including focused tests."
-  )
-  yield* runNode(
-    {
-      workDir: input.workDir,
-      workspace: input.workspace,
-      userPrompt: input.prompt,
-      coder: withModel(pi, coderModel),
-      reasoning: withTimeoutSeconds(withModel(lmStudio, reasoningModel), 600),
-      environment: process.env
-    },
-    (context) =>
-      Effect.gen(function* () {
-        yield* context.events.publish(Info.make({ message: "local reasoner: preparing plan" }))
-        const plan = yield* completeAndPublish(
-          context.reasoning,
-          context.events,
-          [
-            "Read the request and propose a concise, repository-aware implementation plan.",
-            "Call out the exact files and tests the coding agent should inspect.",
-            "",
-            input.prompt
-          ].join("\n")
-        )
-        yield* context.events.publish(
-          AssistantMessage.make({ text: "Handing the local plan to pi." })
-        )
-        yield* completeAndPublish(
-          context.coder,
-          context.events,
-          [
-            "Implement the request in the current repository.",
-            "Use this plan as guidance, but verify it against the actual code. Run relevant tests and summarize the result.",
-            "",
-            "Request:",
-            input.prompt,
-            "",
-            "Plan:",
-            plan
-          ].join("\n")
-        )
-      })
-  )
-})
-
-runFlowMain(program)

package/flows/modernize-bench.ts

@@ -1,362 +0,0 @@
-// Legacy modernization benchmark: measure an extraction run and report the cost of a wave.
-//
-// Two modes:
-//
-//   report  — `modernize-bench --repo <anything> report` (or LLM4TS_BENCH_MODE=report)
-//             loads bench-results.jsonl next to the launch directory and prints
-//             the comparison report. LLM4TS_BENCH_PROJECT=<n> adds the per-wave
-//             projection modernize-survey embeds in its wave plan.
-//
-//   measure — the default. Runs the extraction pipeline over the estate at
-//             `--repo`, taps the flow's own events for per-stage tokens, cost,
-//             and self-healing counters, then appends one schema-versioned
-//             BenchRecord to bench-results.jsonl. Extraction is the dominant
-//             cost of a modernization wave, so its measurement is what a wave
-//             projection needs.
-//
-// The measured run writes into the estate exactly as modernize-extract would;
-// point it at a disposable fixture copy, not a live estate.
-//
-// Run: modernize-bench --repo ~/estates/fixture-copy
-import { arch, cpus, hostname, totalmem, type as osType } from "node:os"
-import { join } from "node:path"
-import * as Effect from "effect/Effect"
-import { Sample } from "@llm4ts/core/eval/Eval"
-import type { TokenUsage } from "@llm4ts/core/Models"
-import { judge } from "@llm4ts/core/eval/Judge"
-import type { LlmServiceShape } from "@llm4ts/core/LlmService"
-import {
-  BenchJudge,
-  BenchMachine,
-  BenchQuality,
-  BenchRecord,
-  BenchScore,
-  CurrentBenchSchema,
-  makeBenchPhase,
-  makeBenchTap,
-  type BenchPhase
-} from "@llm4ts/flow/Bench"
-import { loadBenchRecords, appendBenchRecord, renderBenchReport } from "@llm4ts/flow/BenchReport"
-import { FlowAborted, FlowLlmError } from "@llm4ts/flow/FlowError"
-import { Info, TokensUsed } from "@llm4ts/flow/FlowEvents"
-import { packageVersion } from "@llm4ts/flow/Package"
-import { loadPack } from "@llm4ts/flow/Pack"
-import { stage } from "@llm4ts/flow/PlanExecution"
-import { mergeReviewResults } from "@llm4ts/flow/Review"
-import { coverage, coverageUnits, features, matchingFiles } from "@llm4ts/flow/SpecChecks"
-import {
-  ProgramArtifacts,
-  ProgramUnit,
-  extractProgramsResumably
-} from "@llm4ts/modernize/Artifacts"
-import { asReadOnly, coderFromEnv } from "@llm4ts/runner/Connectors"
-import { resolveFlowInput } from "@llm4ts/runner/FlowArgs"
-import { runFlowMain, runNode } from "@llm4ts/runner/FlowRunner"
-import { nodePlainFileStore } from "@llm4ts/runner/NodePlainFileStore"
-import { makeNodeWorkspace } from "@llm4ts/runner/NodeWorkspace"
-import { reviewFingerprint } from "@llm4ts/runner/ReviewFingerprint"
-
-const ModDir = "docs/modernization"
-const BenchFile = "bench-results.jsonl"
-
-const programName = (relativePath: string): string => {
-  const base = relativePath.slice(relativePath.lastIndexOf("/") + 1)
-  const dot = base.lastIndexOf(".")
-  return dot > 0 ? base.slice(0, dot) : base
-}
-
-const machine = (): BenchMachine =>
-  BenchMachine.make({
-    hostname: hostname(),
-    os: osType(),
-    arch: arch(),
-    cores: cpus().length,
-    memoryGb: Math.round((totalmem() / 1024 ** 3) * 10) / 10,
-    runtime: `node ${process.versions.node}`
-  })
-
-const program = Effect.gen(function* () {
-  const input = yield* resolveFlowInput("Benchmark a modernization extraction run")
-  const packDir = process.env.LLM4TS_PACK ?? "packs/cobol-springboot"
-  const mode =
-    process.env.LLM4TS_BENCH_MODE?.trim() ??
-    (input.prompt.trim().toLowerCase() === "report" ? "report" : "measure")
-  const files = nodePlainFileStore
-  const benchPath = join(input.workspace, BenchFile)
-  const coder = coderFromEnv(process.env)
-  const startedAt = new Date()
-
-  if (mode === "report") {
-    const records = yield* loadBenchRecords(files, benchPath)
-    if (records.length === 0) {
-      return yield* FlowAborted.make({ message: `no benchmark records at ${benchPath}` })
-    }
-    const projected = Number.parseInt(process.env.LLM4TS_BENCH_PROJECT ?? "", 10)
-    process.stdout.write(
-      `${renderBenchReport(records, Number.isFinite(projected) && projected > 0 ? projected : undefined)}\n`
-    )
-    return
-  }
-
-  yield* runNode(
-    {
-      workDir: input.workDir,
-      workspace: input.workspace,
-      userPrompt: input.prompt,
-      coder,
-      reasoning: asReadOnly(coder),
-      environment: process.env
-    },
-    (context) =>
-      Effect.scoped(
-        Effect.gen(function* () {
-          const launchWorkspace = yield* makeNodeWorkspace(input.workspace)
-          const estate = yield* makeNodeWorkspace(input.workDir)
-          const pack = yield* stage(context.events, "pack", loadPack(launchWorkspace, packDir))
-          const modDirAbs = join(input.workDir, ModDir)
-          // The tap observes the same events the terminal renders, so the
-          // measurement never needs its own instrumentation inside the flow.
-          const tap = yield* makeBenchTap(context.events)
-          const phases: Array<BenchPhase> = []
-
-          const measured = <A, E>(name: string, body: Effect.Effect<A, E>) =>
-            Effect.gen(function* () {
-              yield* tap.reset
-              const began = Date.now()
-              const value = yield* stage(context.events, name, body)
-              const observation = yield* tap.get
-              phases.push(makeBenchPhase(observation, name, Date.now() - began))
-              return value
-            })
-
-          const programs = yield* measured(
-            "inventory",
-            matchingFiles(estate, pack.programs ?? pack.sources ?? ".*")
-          )
-          if (programs.length === 0) {
-            return yield* FlowAborted.make({
-              message: `no source units matched the pack's programs/sources regex under ${input.workDir}`
-            })
-          }
-          const units = programs.map((relativePath) =>
-            ProgramUnit.make({ name: programName(relativePath), sourcePath: relativePath })
-          )
-          const system = [
-            pack.prompt("analysis"),
-            pack.lessons === undefined
-              ? undefined
-              : `Lessons from previous modernization runs — apply them:\n${pack.lessons}`
-          ]
-            .filter((part) => part !== undefined)
-            .join("\n\n")
-
-          /**
-           * Structured calls report usage only to their caller; the Chat seam
-           * is what normally publishes `TokensUsed` (ADR 0005). A benchmark
-           * that skipped this would measure wall-clock and nothing else, so
-           * every measured call republishes its own usage for the tap.
-           */
-          const publishUsage = (agent: string, usage: TokenUsage, model: string | undefined) =>
-            context.events.publish(
-              TokensUsed.make({
-                agent,
-                usage,
-                ...(model === undefined ? {} : { model })
-              })
-            )
-
-          yield* measured(
-            "extract",
-            extractProgramsResumably(
-              files,
-              units,
-              (unit) =>
-                context.coder
-                  .executeStructuredWithUsage(
-                    [
-                      system,
-                      "",
-                      `Extract the behavioural spec for ONE source unit: ${unit.sourcePath}`,
-                      'Respond only with JSON: {"spec":"…","feature":"…","traceability":"…","mapping":"…"}.',
-                      pack.prompt("spec") ?? "",
-                      pack.prompt("bdd") ?? ""
-                    ].join("\n"),
-                    ProgramArtifacts,
-                    { type: "object" }
-                  )
-                  .pipe(
-                    Effect.mapError(FlowLlmError.from),
-                    Effect.tap(([, usage, model]) =>
-                      usage === undefined ? Effect.void : publishUsage("coder", usage, model)
-                    ),
-                    Effect.map(([artifacts]) => artifacts)
-                  ),
-              modDirAbs
-            )
-          )
-
-          // The deterministic half of the extraction gate, measured on its own:
-          // it is the part a slower model cannot make cheaper.
-          const deterministic = yield* measured(
-            "gate",
-            Effect.gen(function* () {
-              const fragments: Array<string> = []
-              for (const unit of units) {
-                const text = yield* files.read(join(modDirAbs, "traceability", `${unit.name}.md`))
-                if (text !== undefined) {
-                  fragments.push(text)
-                }
-              }
-              const trace = fragments.join("\n\n")
-              const covered = yield* coverage(estate, pack.coverage, trace)
-              const wellFormed = yield* features(estate, `${ModDir}/features`)
-              return mergeReviewResults([covered, wellFormed])
-            })
-          )
-
-          /**
-           * The evaluator has no usage-reporting variant, so the seat it runs
-           * on is wrapped instead: every structured call it makes reports its
-           * usage to the tap before returning the value.
-           */
-          const metered: LlmServiceShape = {
-            ...context.reasoning,
-            executeStructured: (prompt, schema, jsonSchema) =>
-              context.reasoning.executeStructuredWithUsage(prompt, schema, jsonSchema).pipe(
-                Effect.tap(([, usage, model]) =>
-                  usage === undefined ? Effect.void : publishUsage("reasoning", usage, model)
-                ),
-                Effect.map(([value]) => value)
-              )
-          }
-
-          const judged = yield* measured(
-            "judge",
-            Effect.gen(function* () {
-              const packJudge = judge(metered, pack.judgeDimensions)
-              const scores: Array<BenchScore> = []
-              let findings = 0
-              for (const unit of units) {
-                const spec = (yield* files.read(join(modDirAbs, "specs", `${unit.name}.md`))) ?? ""
-                const source = (yield* files.read(join(input.workDir, unit.sourcePath))) ?? ""
-                const scored = yield* packJudge
-                  .evaluate(Sample.make({ response: spec, context: source, query: input.prompt }))
-                  .pipe(Effect.mapError(FlowLlmError.from))
-                for (const score of scored.scores) {
-                  const max = pack.judgeDimensions.find((d) => d.name === score.name)?.maxScore ?? 2
-                  scores.push(
-                    BenchScore.make({
-                      name: `${unit.name}:${score.name}`,
-                      score: score.score,
-                      max,
-                      reasoning: score.reasoning
-                    })
-                  )
-                  if (score.score < max) {
-                    findings += 1
-                  }
-                }
-              }
-              return { scores, findings }
-            })
-          )
-
-          const quality = yield* Effect.gen(function* () {
-            const specFiles = yield* matchingFiles(estate, `^${ModDir}/specs/.*\\.md$`).pipe(
-              Effect.orElseSucceed(() => [])
-            )
-            const featureFiles = yield* matchingFiles(
-              estate,
-              `^${ModDir}/features/.*\\.feature$`
-            ).pipe(Effect.orElseSucceed(() => []))
-            let scenarios = 0
-            for (const path of featureFiles) {
-              const text = yield* estate.read(path).pipe(Effect.orElseSucceed(() => ""))
-              scenarios += text
-                .split(/\r?\n/)
-                .filter((line) => line.trim().startsWith("Scenario")).length
-            }
-            let sourceLines = 0
-            for (const path of programs) {
-              const text = yield* estate.read(path).pipe(Effect.orElseSucceed(() => ""))
-              sourceLines += text.split(/\r?\n/).length
-            }
-            const units_ = yield* coverageUnits(estate, pack.coverage)
-            const malformed = (yield* features(estate, `${ModDir}/features`)).issues.length
-            return BenchQuality.make({
-              featureFiles: featureFiles.length,
-              scenarios,
-              malformedFeatures: malformed,
-              specFiles: specFiles.length,
-              programs: programs.length,
-              sourceFiles: programs.length,
-              sourceLines,
-              testFiles: featureFiles.length,
-              testLines: scenarios,
-              deterministicFindings: deterministic.issues.length,
-              judgeFindings: judged.findings,
-              gateCleared: deterministic.issues.length === 0 && judged.findings === 0,
-              gateOpenIssues: deterministic.issues.length + judged.findings,
-              judgeScores: judged.scores,
-              // The rule universe is the estate's own coverage units.
-              coveragePct:
-                Object.values(units_).flat().length === 0
-                  ? 0
-                  : Math.round(
-                      (Object.values(units_)
-                        .flat()
-                        .filter((unit) => unit.length > 0).length /
-                        Object.values(units_).flat().length) *
-                        100
-                    )
-            })
-          })
-
-          // The fingerprint groups records of the SAME estate so the report can
-          // compare providers rather than mixing unrelated fixtures.
-          const fingerprint = reviewFingerprint(
-            pack.name,
-            ...[...programs].sort(),
-            String(quality.sourceLines)
-          ).slice(0, 16)
-
-          const finishedAt = new Date()
-          const record = BenchRecord.make({
-            schemaVersion: CurrentBenchSchema,
-            runId: `${startedAt.toISOString()}-${process.pid}`,
-            startedAt: startedAt.toISOString(),
-            finishedAt: finishedAt.toISOString(),
-            provider: coder.connectorId.value,
-            modelRequested: coder.model ?? "(harness default)",
-            modelsServed: [],
-            judge: BenchJudge.make({
-              provider: coder.connectorId.value,
-              model: coder.model ?? "(harness default)",
-              fixed: false
-            }),
-            llm4tsVersion: packageVersion,
-            pack: pack.name,
-            fixtureFingerprint: fingerprint,
-            machine: machine(),
-            outcome: quality.gateCleared === true ? "completed" : "gate-open",
-            totalMs: finishedAt.getTime() - startedAt.getTime(),
-            phases,
-            quality
-          })
-
-          yield* appendBenchRecord(files, benchPath, record)
-          yield* context.events.publish(
-            Info.make({
-              message:
-                `benchmark appended to ${benchPath} — ${record.totalTokens} token(s) over ` +
-                `${phases.length} phase(s); report with LLM4TS_BENCH_MODE=report`
-            })
-          )
-          const all = yield* loadBenchRecords(files, benchPath)
-          process.stdout.write(`${renderBenchReport(all)}\n`)
-        })
-      )
-  )
-})
-
-runFlowMain(program)