@llm4ts/runner 2.5.0 → 2.7.0

package/src/FlowRunner.ts

@@ -1,3 +1,4 @@
+import { join } from "node:path"
 import * as Clock from "effect/Clock"
 import * as Effect from "effect/Effect"
 import type * as Scope from "effect/Scope"
@@ -5,8 +6,13 @@ import type { ConnectorConfig } from "@llm4ts/core/ConnectorConfig"
 import { CliConnectorConfig, defaultReasoningConfig } from "@llm4ts/core/ConnectorConfig"
 import type { ConnectorRegistryShape } from "@llm4ts/core/ConnectorRegistry"
 import type { HttpClientShape } from "@llm4ts/core/HttpClient"
+import type { JudgmentBackend } from "@llm4ts/core/judgment/Schemas"
+import type { JudgmentShape } from "@llm4ts/core/judgment/Judgment"
+import { LlmJudgmentConfig, makeLlmJudgment } from "@llm4ts/core/judgment/LlmJudgment"
+import { makeTypeSafeJudgment } from "@llm4ts/core/judgment/TypeSafeJudgment"
+import * as Redacted from "effect/Redacted"
 import type { LlmServiceShape } from "@llm4ts/core/LlmService"
-import type { ConnectorCapabilities } from "@llm4ts/core/Models"
+import type { ConnectorCapabilities, TokenUsage } from "@llm4ts/core/Models"
 import type { ProcessExecutorShape } from "@llm4ts/core/ProcessExecutor"
 import type { TemporaryFilesShape } from "@llm4ts/core/TemporaryFiles"
 import type { GeminiCliExecutorShape } from "@llm4ts/core/providers/GeminiCliProvider"
@@ -21,16 +27,26 @@ import {
   geminiBridgePort,
   resolveBridgeModel
 } from "./PiModels.ts"
-import { checkCostBudget, makeCostRecord, type CostBudget } from "@llm4ts/flow/CostLedger"
+import {
+  appendCostRecord,
+  checkCostBudget,
+  CostLedgerFileName,
+  makeCostRecord,
+  type CostBudget
+} from "@llm4ts/flow/CostLedger"
+import { estimatedUsageOptionsFromEnv, makeEstimatedUsageMeter } from "@llm4ts/flow/EstimatedUsage"
 import { FlowLlmError, describeFlowError, type FlowError } from "@llm4ts/flow/FlowError"
 import {
   FlowEvents,
   FlowEventsValues,
+  Info,
+  TokensUsed,
   makeFlowEventHub,
   type FlowEventHub
 } from "@llm4ts/flow/FlowEvents"
 import { FlowContext, type FlowContextShape } from "@llm4ts/flow/FlowContext"
 import { makeFlowRecorder } from "@llm4ts/flow/FlowRecorder"
+import { makeJudgmentLog, type JudgmentLogShape } from "@llm4ts/flow/JudgmentLog"
 import { makeGitHubTool } from "@llm4ts/flow/GitHubTool"
 import { makeGitTool } from "@llm4ts/flow/GitTool"
 import type { PlainFileStoreShape } from "@llm4ts/flow/Persistence"
@@ -61,6 +77,8 @@ export interface FlowRunnerDependencies {
   readonly files: PlainFileStoreShape
   /** pi's own model config (ADR 0016), injectable so a test never reads $HOME. */
   readonly readPiModelsJson?: () => string | undefined
+  /** For the hosted TypeSafe judgment backend; defaults to the Node client. */
+  readonly http?: HttpClientShape
 }
 
 export interface NodeConnectorDependencies {
@@ -73,6 +91,7 @@ export interface NodeConnectorDependencies {
 export const nodeFlowRunnerDependencies = (): FlowRunnerDependencies => ({
   process: nodeProcessExecutor,
   files: nodePlainFileStore,
+  http: nodeHttpClient,
   registry: createConnectorRegistry({
     http: nodeHttpClient,
     process: nodeProcessExecutor,
@@ -88,7 +107,44 @@ export interface FlowRunnerOptions {
   readonly coder: ConnectorConfig
   readonly reasoning?: ConnectorConfig
   readonly reviewers?: ReadonlyArray<ConnectorConfig>
+  /**
+   * The seat typed judgments run on (ADR 0017). Defaults to the reasoning
+   * seat; point it at a small non-thinking model for one-token label scoring.
+   */
+  readonly judgment?: ConnectorConfig
+  /**
+   * `llm` (default) answers on the judgment seat; `typesafe` uses the hosted
+   * Jev model with `TYPESAFE_API_KEY` from the environment. Unset, the
+   * environment's `LLM4TS_JUDGMENT_BACKEND` decides, then `llm`.
+   */
+  readonly judgmentBackend?: JudgmentBackend
+  /** Persist judgment observations under workDir/.llm4ts/judgments; off by default. */
+  readonly judgmentLog?: boolean
+  /**
+   * Meter every seat with `EstimatedUsage`, so a backend that reports no
+   * token counts still yields usage — counted from characters and
+   * labelled `estimated:<model>`, never mixed with measured usage. Default
+   * `true`; `false`, or `LLM4TS_ESTIMATE_USAGE=0` in the environment, leaves
+   * the seats raw and those runs accrue nothing.
+   */
+  readonly estimateUsage?: boolean
+  /**
+   * Where the run's trace goes. Default `workDir/.llm4ts/trace-<start ms>.jsonl`
+   * while `persistRun` holds.
+   */
   readonly tracePath?: string
+  /**
+   * Append-only ledger (`CostLedger`) that receives one `CostRecord` per run
+   * that accrued usage, so `llm4ts costs` and budgets can look across runs.
+   * Default `workDir/.llm4ts/costs.jsonl` while `persistRun` holds.
+   */
+  readonly costLedgerPath?: string
+  /**
+   * `false` writes neither the trace nor the ledger unless their paths are
+   * given explicitly. Default `true`: every run leaves its record under
+   * `workDir/.llm4ts/`, which `llm4ts costs` and replay read back.
+   */
+  readonly persistRun?: boolean
   readonly runId?: string
   readonly verbosity?: Verbosity
   readonly surface?: TerminalSurface
@@ -108,11 +164,16 @@ export interface FlowRunnerBundle {
   readonly events: FlowEventHub
   /** The run's cost tracker, already subscribed to `events`. */
   readonly tracker: CostTracker
+  readonly runId: string
+  readonly judgmentLog?: JudgmentLogShape
 }
 
 const truthy = (value: string | undefined): boolean =>
   value !== undefined && ["1", "true", "yes"].includes(value.trim().toLowerCase())
 
+const switchedOff = (value: string | undefined): boolean =>
+  value !== undefined && ["0", "false", "no", "off"].includes(value.trim().toLowerCase())
+
 const isPiSeat = (config: ConnectorConfig | undefined): config is CliConnectorConfig =>
   config instanceof CliConnectorConfig && config.connectorId.value === ConnectorIds.Pi.value
 
@@ -177,6 +238,12 @@ export const makeFlowRunnerContext = Effect.fn("@llm4ts/runner/FlowRunner.makeCo
   dependencies: FlowRunnerDependencies
 ): Effect.fn.Return<FlowRunnerBundle, FlowError, Scope.Scope> {
   const events = yield* makeFlowEventHub()
+  const runId = options.runId ?? `run-${(yield* Clock.currentTimeMillis).toString()}`
+  const judgmentLog =
+    options.judgmentLog === true
+      ? yield* makeJudgmentLog({ files: dependencies.files, root: options.workDir, runId })
+      : undefined
+  if (judgmentLog !== undefined) yield* judgmentLog.consume(events)
   const tracker = yield* makeCostTracker()
   yield* tracker.consume(events)
   const reasoning = defaultReasoningConfig(options.coder, options.reasoning)
@@ -212,19 +279,96 @@ export const makeFlowRunnerContext = Effect.fn("@llm4ts/runner/FlowRunner.makeCo
           )
         )
       : Effect.void
+  // Every seat is metered, so no run is silently free: a backend that reports
+  // no token counts (`capabilities.usageReporting` false) is estimated from
+  // character counts and labelled `estimated:<model>`, which is what makes
+  // `TokensUsed` — and therefore cost summaries, budgets, the ledger, and
+  // `llm4ts costs` — answer for those seats at all. Measured usage always
+  // wins; the estimate only fills a gap the backend left. Metering here rather
+  // than in each flow also covers the seats a flow never touches directly,
+  // notably the judgment seat and the worktree seats `contextFor` rebinds.
+  const runEnvironment = options.environment ?? process.env
+  const estimateOptions = estimatedUsageOptionsFromEnv(runEnvironment)
+  const meterUsage = options.estimateUsage ?? !switchedOff(runEnvironment.LLM4TS_ESTIMATE_USAGE)
+  const metered = (service: LlmServiceShape): Effect.Effect<LlmServiceShape> =>
+    meterUsage
+      ? Effect.map(makeEstimatedUsageMeter(service, estimateOptions), (meter) => meter.service)
+      : Effect.succeed(service)
   // The spread keeps everything the connector carries beyond the service
-  // methods — notably `capabilities`, which the flow context exposes.
+  // methods — notably `capabilities`, which the flow context exposes. The
+  // meter decorates the retrying seat, so an estimate reflects the attempt
+  // that actually succeeded, and it sees `scoreLabelSequence` when the
+  // connector has one (the retry wrapper does not carry it).
   const resolveSeat = (configuration: ConnectorConfig) =>
     dependencies.registry.resolve(configuration).pipe(
       Effect.mapError(FlowLlmError.from),
       Effect.flatMap((connector) =>
         announceReadOnlyGrade(configuration, connector).pipe(
           Effect.andThen(
-            Effect.map(resilient(connector), (retrying) => ({ ...connector, ...retrying }))
+            Effect.flatMap(resilient(connector), (retrying) => {
+              const seat = { ...connector, ...retrying }
+              return Effect.map(metered(seat), (measured) => ({ ...seat, ...measured }))
+            })
           )
         )
       )
     )
+  // Judgment usage is metered like any chat (ADR 0005 amendment) and every
+  // fallback from label scoring to the verbalized path is announced.
+  const judgmentHooks = {
+    onUsage: (usage: TokenUsage, model: string | undefined) =>
+      events.publish(
+        new TokensUsed({ agent: "judgment", usage, ...(model === undefined ? {} : { model }) })
+      ),
+    onFallback: (key: string, reason: string) =>
+      events.publish(
+        new Info({ message: `judgment '${key}' fell back to verbalized scoring: ${reason}` })
+      )
+  }
+  const judgmentFor = (
+    seat: LlmServiceShape,
+    seatConfig: ConnectorConfig,
+    environment: Readonly<Record<string, string | undefined>>
+  ): Effect.Effect<JudgmentShape, FlowError> => {
+    const backend =
+      options.judgmentBackend ??
+      (environment.LLM4TS_JUDGMENT_BACKEND?.trim().toLowerCase() === "typesafe"
+        ? "typesafe"
+        : "llm")
+    if (backend !== "typesafe") {
+      return Effect.succeed(
+        makeLlmJudgment(
+          seat,
+          LlmJudgmentConfig.make({
+            connector: seatConfig.connectorId.value,
+            ...(seatConfig.model === undefined ? {} : { model: seatConfig.model }),
+            // LLM4TS_JUDGMENT_BATCHING=shared-prefix sends each request's state
+            // once with every question; anything else keeps them independent.
+            ...(environment.LLM4TS_JUDGMENT_BATCHING?.trim().toLowerCase() === "shared-prefix"
+              ? { batching: "shared-prefix" as const }
+              : {})
+          }),
+          judgmentHooks
+        )
+      )
+    }
+    const key = environment.TYPESAFE_API_KEY?.trim()
+    if (key === undefined || key.length === 0) {
+      return Effect.fail(
+        FlowLlmError.from(
+          ProviderError.make({
+            message: "judgment backend 'typesafe' needs TYPESAFE_API_KEY in the environment"
+          })
+        )
+      )
+    }
+    return Effect.succeed(
+      makeTypeSafeJudgment(
+        { apiKey: Redacted.make(key), onUsage: judgmentHooks.onUsage },
+        dependencies.http ?? nodeHttpClient
+      )
+    )
+  }
   const seatsFor = Effect.fn("@llm4ts/runner/FlowRunner.seatsFor")(function* (
     workDir: string
   ): Effect.fn.Return<
@@ -232,6 +376,7 @@ export const makeFlowRunnerContext = Effect.fn("@llm4ts/runner/FlowRunner.makeCo
       readonly coder: LlmServiceShape & { readonly capabilities: ConnectorCapabilities }
       readonly reasoning: LlmServiceShape
       readonly reviewers: ReadonlyArray<LlmServiceShape>
+      readonly judgment: JudgmentShape
     },
     FlowError,
     Scope.Scope
@@ -244,7 +389,14 @@ export const makeFlowRunnerContext = Effect.fn("@llm4ts/runner/FlowRunner.makeCo
     const reviewers = yield* Effect.forEach(options.reviewers ?? [], (configuration) =>
       resolveSeat(prepareConnector(bridged(configuration), workDir, environment))
     )
-    return { coder, reasoning: reasoningService, reviewers }
+    // The judgment seat defaults to the reasoning seat's service itself, so
+    // an unconfigured run resolves nothing extra.
+    const judgmentSeat =
+      options.judgment === undefined
+        ? reasoningService
+        : yield* resolveSeat(prepareConnector(bridged(options.judgment), workDir, environment))
+    const judgment = yield* judgmentFor(judgmentSeat, options.judgment ?? reasoning, environment)
+    return { coder, reasoning: reasoningService, reviewers, judgment }
   })
   const contextAt = (
     workDir: string,
@@ -252,12 +404,14 @@ export const makeFlowRunnerContext = Effect.fn("@llm4ts/runner/FlowRunner.makeCo
       readonly coder: LlmServiceShape & { readonly capabilities: ConnectorCapabilities }
       readonly reasoning: LlmServiceShape
       readonly reviewers: ReadonlyArray<LlmServiceShape>
+      readonly judgment: JudgmentShape
     },
     rebind: boolean
   ): FlowContextShape =>
     FlowContext.of({
       reasoning: seats.reasoning,
       coder: seats.coder,
+      judgment: seats.judgment,
       git: makeGitTool(dependencies.process, workDir, events),
       hosting: makeGitHubTool(dependencies.process, workDir, events),
       events,
@@ -279,6 +433,8 @@ export const makeFlowRunnerContext = Effect.fn("@llm4ts/runner/FlowRunner.makeCo
   return {
     events,
     tracker,
+    runId,
+    ...(judgmentLog === undefined ? {} : { judgmentLog }),
     context: contextAt(options.workDir, seats, true)
   }
 })
@@ -296,6 +452,13 @@ export const runWithBundle = Effect.fn("@llm4ts/runner/FlowRunner.runWithBundle"
   const startedAt = yield* Clock.currentTimeMillis
   const environment = options.environment ?? process.env
   const verbosity = options.verbosity ?? "Normal"
+  const persistRun = options.persistRun ?? true
+  const tracePath =
+    options.tracePath ??
+    (persistRun ? join(options.workDir, ".llm4ts", `trace-${startedAt}.jsonl`) : undefined)
+  const costLedgerPath =
+    options.costLedgerPath ??
+    (persistRun ? join(options.workDir, ".llm4ts", CostLedgerFileName) : undefined)
   const tracker = bundle.tracker
   const surface = options.surface ?? (yield* makeTerminalSurface(environment))
   const palette = surface.palette
@@ -310,22 +473,18 @@ export const runWithBundle = Effect.fn("@llm4ts/runner/FlowRunner.runWithBundle"
       ...(reviewers.length === 0 ? [] : [`reviewers ${reviewers.map(describeSeat).join(", ")}`])
     ].join(" · ")
     yield* surface.log(palette.info(`${seats} · ${options.workDir}`))
-    if (options.tracePath !== undefined) {
+    if (tracePath !== undefined) {
       yield* surface.log(
         palette.info(
-          `trace ${options.tracePath}${options.runId === undefined ? "" : ` · run ${options.runId}`}`
+          `trace ${tracePath}${options.runId === undefined ? "" : ` · run ${options.runId}`}`
         )
       )
     }
   }
   const recorder =
-    options.tracePath === undefined
+    tracePath === undefined
       ? undefined
-      : yield* makeFlowRecorder(
-          dependencies.files,
-          options.tracePath,
-          options.runId ?? `run-${(yield* Clock.currentTimeMillis).toString()}`
-        )
+      : yield* makeFlowRecorder(dependencies.files, tracePath, bundle.runId)
   if (recorder !== undefined) {
     yield* recorder.consume(bundle.events)
   }
@@ -349,6 +508,32 @@ export const runWithBundle = Effect.fn("@llm4ts/runner/FlowRunner.runWithBundle"
             budget
           )
         })
+  // Runs without usage leave no record: a ledger line of zeros would only
+  // inflate run counts for backends that report nothing.
+  const appendLedger =
+    costLedgerPath === undefined
+      ? Effect.void
+      : Effect.gen(function* () {
+          const cells = yield* tracker.cells
+          if (cells.length === 0) {
+            return
+          }
+          yield* appendCostRecord(
+            dependencies.files,
+            costLedgerPath,
+            makeCostRecord({
+              runId: bundle.runId,
+              at: new Date(startedAt).toISOString(),
+              repo: options.workDir,
+              prompt: options.userPrompt,
+              cells
+            })
+          )
+        }).pipe(
+          Effect.catch((error) =>
+            surface.log(palette.fail(`cost ledger not written: ${describeFlowError(error)}`))
+          )
+        )
   return yield* body(bundle.context).pipe(
     Effect.provideService(FlowContext, bundle.context),
     Effect.andThen((value) => Effect.as(enforceBudget, value)),
@@ -357,10 +542,14 @@ export const runWithBundle = Effect.fn("@llm4ts/runner/FlowRunner.runWithBundle"
         [
           terminal.awaitDrained(),
           tracker.awaitDrained(bundle.events),
-          recorder === undefined ? Effect.void : recorder.awaitDrained(bundle.events)
+          recorder === undefined ? Effect.void : recorder.awaitDrained(bundle.events),
+          bundle.judgmentLog === undefined
+            ? Effect.void
+            : bundle.judgmentLog.awaitDrained(bundle.events)
         ],
         { concurrency: "unbounded" }
       ).pipe(
+        Effect.andThen(appendLedger),
         Effect.andThen(surface.setStatus(undefined)),
         Effect.andThen(tracker.summary),
         Effect.flatMap((summary) => surface.log(`\n${summary}`))
@@ -375,8 +564,8 @@ export const runWithBundle = Effect.fn("@llm4ts/runner/FlowRunner.runWithBundle"
             `flow failed after ${formatDurationMs(finishedAt - startedAt)}: ${describeFlowError(error)}`
           )}`
         )
-        if (options.tracePath !== undefined) {
-          yield* surface.log(palette.info(`trace ${options.tracePath}`))
+        if (tracePath !== undefined) {
+          yield* surface.log(palette.info(`trace ${tracePath}`))
         }
       })
     ),

package/src/Terminal.ts

@@ -26,6 +26,9 @@ export const parseVerbosity = (value: string | undefined): Verbosity => {
 
 export const rendersEvent = (verbosity: Verbosity, event: FlowEvent): boolean => {
   switch (event._tag) {
+    case "JudgmentObserved":
+      // Advise mode publishes a separate Info event for the operator.
+      return false
     case "StageStarted":
     case "StageCompleted":
     case "StageFailed":
@@ -141,6 +144,8 @@ export const terminalLine = (
 ): string => {
   const safe = terminalSafe
   switch (event._tag) {
+    case "JudgmentObserved":
+      return ""
     case "StageStarted":
       return palette.stageStart(safe(event.stage))
     case "StageCompleted":