@llm4ts/flow 2.5.0 → 2.6.0

package/src/JudgmentEval.ts

@@ -0,0 +1,306 @@
+import * as Schema from "effect/Schema"
+import { expectedScore, truth, type Answer } from "@llm4ts/core/judgment/Schemas"
+import { decide, type Decision } from "./Judgment.ts"
+import { candidateId, DatasetDecision, LabelledItem } from "./JudgmentDataset.ts"
+import type { ReviewPrescreenResult, ReviewResult, Severity } from "./Review.ts"
+import type { Reviewer } from "./Reviewer.ts"
+
+export interface EvalItem {
+  readonly item: LabelledItem
+  readonly answer: Answer | undefined
+  readonly latencyMs: number
+  readonly failure?: string
+  readonly decision?: Decision
+  readonly severities?: ReadonlyArray<Severity>
+}
+
+/** One completed full-path result per lens; missing screens remain failed eval items. */
+export const replayEvalItems = (
+  commit: { readonly sha: string; readonly diff: string },
+  runs: ReadonlyArray<{ readonly lens: Reviewer; readonly result: ReviewResult }>,
+  observations: ReviewPrescreenResult["observations"],
+  screeningMs: number,
+  labelledAt: string
+): ReadonlyArray<EvalItem> =>
+  runs.map(({ lens, result }) => {
+    const observation = observations.find(({ key }) => key === lens.name)
+    const source = `commit:${commit.sha}:${lens.name}`
+    return {
+      item: LabelledItem.make({
+        id: candidateId("review-prescreen", source),
+        decision: "review-prescreen",
+        source,
+        state: observation?.state ?? { diff: commit.diff },
+        question: observation?.question ?? truth(lens.screeningStatement),
+        label: result.issues.length > 0,
+        labelledBy: "outcome",
+        labelledAt
+      }),
+      answer: observation?.answer,
+      ...(observation === undefined
+        ? { failure: "Pre-screen returned no matching answer." }
+        : { decision: observation.decision }),
+      severities: result.issues.map(({ severity }) => severity),
+      latencyMs: screeningMs / runs.length
+    }
+  })
+
+const SeverityCounts = Schema.Struct({
+  Critical: Schema.Int,
+  Warning: Schema.Int,
+  Info: Schema.Int
+})
+
+/** Lost issues and missed positive lenses have different units; verify both independently. */
+export const replayMissesAgree = (
+  report: EvalReport,
+  lost: typeof SeverityCounts.Type,
+  missedPositiveLenses: number
+): boolean =>
+  report.missedIssues?.count === missedPositiveLenses &&
+  report.missedIssues.severities !== undefined &&
+  (["Critical", "Warning", "Info"] as const).every(
+    (severity) => report.missedIssues?.severities?.[severity] === lost[severity]
+  )
+
+const Metric = Schema.NullOr(Schema.Number)
+export const EvalMemory = Schema.Struct({ restMb: Schema.Number, peakMb: Schema.Number })
+export const CalibrationBin = Schema.Struct({
+  lower: Schema.Number,
+  upper: Schema.Number,
+  count: Schema.Int,
+  meanLabelProbability: Metric
+})
+export const EvalMeasures = Schema.Struct({
+  items: Schema.Int,
+  answered: Schema.Int,
+  failed: Schema.Int,
+  accuracy: Metric,
+  ece: Metric,
+  brier: Metric,
+  bins: Schema.Array(CalibrationBin),
+  decisions: Schema.Struct({ act: Schema.Int, caution: Schema.Int, hold: Schema.Int }),
+  latencyMs: Schema.Struct({ p50: Metric, p95: Metric })
+})
+
+export class EvalReport extends Schema.Class<EvalReport>("EvalReport")({
+  overall: EvalMeasures,
+  decisions: Schema.Array(Schema.Struct({ decision: DatasetDecision, measures: EvalMeasures })),
+  missedIssues: Schema.optionalKey(
+    Schema.Struct({
+      count: Schema.Int,
+      positiveAnswered: Schema.Int,
+      rate: Metric,
+      severities: Schema.optionalKey(SeverityCounts)
+    })
+  ),
+  memory: Schema.optionalKey(EvalMemory)
+}) {}
+
+const measured = ({ item, answer, failure, latencyMs, decision }: EvalItem) => {
+  if (failure !== undefined || answer === undefined) return undefined
+  if (
+    item.question.type === "truth" &&
+    answer.type === "truth" &&
+    typeof item.label === "boolean"
+  ) {
+    return {
+      answer,
+      decision: decision ?? decide(answer),
+      latencyMs,
+      correct: answer.truth >= 0.5 === item.label,
+      probability: item.label ? answer.truth : 1 - answer.truth
+    }
+  }
+  if (item.question.type === "score" && answer.type === "score" && typeof item.label === "number") {
+    return {
+      answer,
+      decision: decision ?? decide(answer),
+      latencyMs,
+      correct: Math.round(expectedScore(answer.probabilities)) === item.label,
+      probability: answer.probabilities[String(item.label)] ?? 0
+    }
+  }
+  return undefined
+}
+
+/** Nearest rank, with no interpolation; no observations means no metric. */
+const percentile = (sorted: ReadonlyArray<number>, fraction: number): number | null =>
+  sorted[Math.ceil(sorted.length * fraction) - 1] ?? null
+
+const measures = (items: ReadonlyArray<EvalItem>): typeof EvalMeasures.Type => {
+  const answered = items.flatMap((item) => {
+    const value = measured(item)
+    return value === undefined ? [] : [value]
+  })
+  const count = answered.length
+  const average = (sum: number): number | null => (count === 0 ? null : sum / count)
+  const bins = Array.from({ length: 10 }, (_, index) => {
+    const members = answered.filter(
+      ({ probability }) => Math.min(9, Math.floor(probability * 10)) === index
+    )
+    return {
+      lower: index / 10,
+      upper: (index + 1) / 10,
+      count: members.length,
+      meanLabelProbability:
+        members.length === 0
+          ? null
+          : members.reduce((sum, member) => sum + member.probability, 0) / members.length
+    }
+  })
+  const decisions = answered.map(({ decision }) => decision)
+  const latencies = answered.map(({ latencyMs }) => latencyMs).sort((a, b) => a - b)
+  return {
+    items: items.length,
+    answered: count,
+    failed: items.length - count,
+    accuracy: average(answered.filter(({ correct }) => correct).length),
+    // The labelled outcome occurred (target 1), regardless of the predicted class.
+    ece: average(
+      bins.reduce((sum, bin) => sum + bin.count * Math.abs(1 - (bin.meanLabelProbability ?? 1)), 0)
+    ),
+    brier: average(answered.reduce((sum, value) => sum + (1 - value.probability) ** 2, 0)),
+    bins,
+    decisions: {
+      act: decisions.filter((decision) => decision === "act").length,
+      caution: decisions.filter((decision) => decision === "caution").length,
+      hold: decisions.filter((decision) => decision === "hold").length
+    },
+    latencyMs: { p50: percentile(latencies, 0.5), p95: percentile(latencies, 0.95) }
+  }
+}
+
+/** Failures and mismatched answer kinds count as failed and are excluded from all metrics. */
+export const evaluateJudgments = (
+  items: ReadonlyArray<EvalItem>,
+  memory?: typeof EvalMemory.Type
+): EvalReport => {
+  const decisions = [...new Set(items.map(({ item }) => item.decision))].sort()
+  const positives = items
+    .filter(({ item }) => item.decision === "review-prescreen" && item.label === true)
+    .flatMap((item) => {
+      const value = measured(item)
+      return value === undefined ? [] : [{ ...value, severities: item.severities }]
+    })
+  const missed = positives.filter(
+    ({ answer, decision }) => answer.type === "truth" && answer.truth < 0.5 && decision === "act"
+  )
+  const severities = missed.flatMap((item) => item.severities ?? [])
+  return EvalReport.make({
+    overall: measures(items),
+    decisions: decisions.map((decision) => ({
+      decision,
+      measures: measures(items.filter(({ item }) => item.decision === decision))
+    })),
+    ...(decisions.includes("review-prescreen")
+      ? {
+          missedIssues: {
+            count: missed.length,
+            positiveAnswered: positives.length,
+            rate: positives.length === 0 ? null : missed.length / positives.length,
+            ...(items.every((item) => item.severities !== undefined)
+              ? {
+                  severities: {
+                    Critical: severities.filter((severity) => severity === "Critical").length,
+                    Warning: severities.filter((severity) => severity === "Warning").length,
+                    Info: severities.filter((severity) => severity === "Info").length
+                  }
+                }
+              : {})
+          }
+        }
+      : {}),
+    ...(memory === undefined ? {} : { memory })
+  })
+}
+
+export interface EvalReportMeta {
+  readonly date: string
+  readonly decision: string
+  readonly backend: string
+  readonly model: string
+  readonly memoryNote?: string
+  readonly title?: string
+  readonly latencyNote?: string
+}
+
+const cell = (value: string): string => value.replace(/\|/g, "&#124;").replace(/[\r\n]/g, " ")
+const number = (value: number | null): string => (value === null ? "n/a" : value.toFixed(4))
+
+export const renderEvalReport = (report: EvalReport, meta: EvalReportMeta): string => {
+  const rows = [
+    ...report.decisions.map(({ decision, measures }) => ({ name: decision, measures })),
+    { name: "Overall", measures: report.overall }
+  ]
+  return [
+    `# ${cell(meta.title ?? "Judgment evaluation")}`,
+    "",
+    `- Date: ${cell(meta.date)}`,
+    `- Decision: ${cell(meta.decision)}`,
+    `- Backend identity: ${cell(meta.backend)}`,
+    `- Model: ${cell(meta.model)}`,
+    "",
+    "## Measures",
+    "",
+    "| Decision | Items | Answered | Failed | Accuracy | ECE (10 bins) | Brier | p50 ms | p95 ms |",
+    "| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |",
+    ...rows.map(
+      ({ name, measures: m }) =>
+        `| ${name} | ${m.items} | ${m.answered} | ${m.failed} | ${number(m.accuracy)} | ${number(m.ece)} | ${number(m.brier)} | ${number(m.latencyMs.p50)} | ${number(m.latencyMs.p95)} |`
+    ),
+    "",
+    "Metrics use answered items only; n/a means no observations. Overall is item-weighted.",
+    "Accuracy: Truth >= 0.5; Score rounds the expected level index.",
+    "ECE bins the labelled-outcome probability p into [0, 0.1), ..., [0.9, 1]; target = 1.",
+    "ECE = sum(bin count / answered * |1 - mean p|); Brier = mean((1 - p)^2).",
+    "These are labelled-outcome metrics, not predicted-confidence ECE or multiclass Brier.",
+    meta.latencyNote ??
+      "Latency uses nearest-rank percentiles per independent question, excluding failures.",
+    "",
+    "## Default policy decisions",
+    "",
+    "| Decision | Act | Caution | Hold |",
+    "| --- | ---: | ---: | ---: |",
+    ...rows.map(
+      ({ name, measures: m }) =>
+        `| ${name} | ${m.decisions.act} | ${m.decisions.caution} | ${m.decisions.hold} |`
+    ),
+    ...(report.missedIssues === undefined
+      ? []
+      : [
+          "",
+          "## Pre-screen missed issues",
+          "",
+          "| Missed positive items | Answered positive items | Missed rate |",
+          "| ---: | ---: | ---: |",
+          `| ${report.missedIssues.count} | ${report.missedIssues.positiveAnswered} | ${number(report.missedIssues.rate)} |`,
+          "",
+          "A miss requires label=true, truth<0.5 and decide=act (the lens would be skipped).",
+          ...(report.missedIssues.severities === undefined
+            ? [
+                "Severity breakdown unavailable: LabelledItem records presence, not issue counts or severity."
+              ]
+            : [
+                "",
+                "| Lost Critical issues | Lost Warning issues | Lost Info issues |",
+                "| ---: | ---: | ---: |",
+                `| ${report.missedIssues.severities.Critical} | ${report.missedIssues.severities.Warning} | ${report.missedIssues.severities.Info} |`,
+                "",
+                "Severity counts are issues on missed positive lenses; one lens can report multiple issues."
+              ])
+        ]),
+    "",
+    "## Judgment seat resident memory",
+    "",
+    "| Rest MiB | Sampled peak MiB |",
+    "| ---: | ---: |",
+    `| ${number(report.memory?.restMb ?? null)} | ${number(report.memory?.peakMb ?? null)} |`,
+    "",
+    meta.memoryNote ??
+      (report.memory === undefined
+        ? "Memory not supplied."
+        : "Memory supplied by the caller; peak is the maximum sampled RSS."),
+    ""
+  ].join("\n")
+}

package/src/JudgmentLog.ts

@@ -0,0 +1,96 @@
+import * as Clock from "effect/Clock"
+import * as Effect from "effect/Effect"
+import * as Ref from "effect/Ref"
+import * as Schema from "effect/Schema"
+import * as Semaphore from "effect/Semaphore"
+import type * as Scope from "effect/Scope"
+import * as Stream from "effect/Stream"
+import { JudgmentObserved, type FlowEvent, type FlowEventHub } from "./FlowEvents.ts"
+import type { PlainFileStoreShape } from "./Persistence.ts"
+
+export class JudgmentObservation extends Schema.Class<JudgmentObservation>("JudgmentObservation")({
+  runId: Schema.String,
+  /** Epoch milliseconds, matching FlowRecorder's timestamp. */
+  at: Schema.Number,
+  consumer: JudgmentObserved.fields.consumer,
+  key: JudgmentObserved.fields.key,
+  state: JudgmentObserved.fields.state,
+  question: JudgmentObserved.fields.question,
+  answer: JudgmentObserved.fields.answer,
+  outcome: JudgmentObserved.fields.outcome,
+  decision: JudgmentObserved.fields.decision,
+  mode: JudgmentObserved.fields.mode,
+  judgmentIdentity: JudgmentObserved.fields.judgmentIdentity
+}) {}
+
+export const judgmentLogPath = (root: string, consumer: JudgmentObservation["consumer"]): string =>
+  `${root.replace(/[\\/]+$/, "")}/.llm4ts/judgments/${consumer}.jsonl`
+
+export interface JudgmentLogShape {
+  readonly record: (event: FlowEvent) => Effect.Effect<void>
+  readonly consume: (hub: FlowEventHub) => Effect.Effect<void, never, Scope.Scope>
+  readonly awaitDrained: (hub: FlowEventHub) => Effect.Effect<void>
+}
+
+const codec = Schema.fromJsonString(JudgmentObservation)
+
+/** A best-effort subscriber, with the same permanent degradation policy as FlowRecorder. */
+export const makeJudgmentLog = Effect.fn("@llm4ts/flow/JudgmentLog.make")(function* (options: {
+  readonly files: PlainFileStoreShape
+  readonly root: string
+  readonly runId: string
+}): Effect.fn.Return<JudgmentLogShape> {
+  const consumed = yield* Ref.make(0)
+  const degraded = yield* Ref.make(false)
+  const lock = yield* Semaphore.make(1)
+  const record = (event: FlowEvent): Effect.Effect<void> =>
+    event._tag !== "JudgmentObserved"
+      ? Effect.void
+      : lock.withPermit(
+          Effect.gen(function* () {
+            if (yield* Ref.get(degraded)) return
+            const at = yield* Clock.currentTimeMillis
+            // Classified remains sealed until an explicit, audited declassify.
+            // Text uses its toString; JSON uses its toJSON. Serialize before
+            // schema traversal so even nested sealed values use that redaction,
+            // never their private payload. Already declassified plain text has
+            // no taint metadata; callers must retain the wrapper for secrets.
+            const observation = yield* Schema.decodeUnknownEffect(codec)(
+              JSON.stringify({ ...event, runId: options.runId, at })
+            )
+            const line = yield* Schema.encodeEffect(codec)(observation)
+            yield* options.files.append(judgmentLogPath(options.root, event.consumer), `${line}\n`)
+          }).pipe(Effect.catch(() => Ref.set(degraded, true)))
+        )
+
+  const awaitDrained = (hub: FlowEventHub): Effect.Effect<void> =>
+    Effect.gen(function* () {
+      const target = yield* hub.publishedCount
+      const drain: Effect.Effect<void> = Effect.suspend(() =>
+        Ref.get(consumed).pipe(
+          Effect.flatMap((count) =>
+            count >= target ? Effect.void : Effect.yieldNow.pipe(Effect.andThen(drain))
+          )
+        )
+      )
+      yield* drain
+    })
+
+  return {
+    record,
+    awaitDrained,
+    consume: (hub) =>
+      Effect.gen(function* () {
+        yield* Ref.set(consumed, yield* hub.publishedCount)
+        const subscription = yield* hub.subscribe
+        yield* Stream.fromSubscription(subscription).pipe(
+          Stream.runForEach((event) =>
+            record(event).pipe(Effect.andThen(Ref.update(consumed, (count) => count + 1)))
+          ),
+          Effect.forkScoped
+        )
+        // Registered after the consumer fiber: drain before scope interrupts it.
+        yield* Effect.addFinalizer(() => awaitDrained(hub))
+      })
+  }
+})

package/src/JudgmentTypes.ts

@@ -0,0 +1,15 @@
+import * as Schema from "effect/Schema"
+
+/**
+ * The two literals shared by the judgment policy and the flow events. They
+ * live apart from `Judgment.ts` so `FlowEvents.ts` can import them without
+ * a module cycle (the policy module publishes events).
+ */
+
+/** Consumers observe by default; automation requires an explicit act mode. */
+export const JudgmentMode = Schema.Literals(["observe", "advise", "act"])
+export type JudgmentMode = typeof JudgmentMode.Type
+
+/** The three bands Jev documents: act, proceed with caution, or hold. */
+export const Decision = Schema.Literals(["act", "caution", "hold"])
+export type Decision = typeof Decision.Type

package/src/ProgramJudge.ts

@@ -1,9 +1,18 @@
 import * as Effect from "effect/Effect"
 import { Sample, type Dimension, type EvalResult } from "@llm4ts/core/eval/Eval"
-import type { Evaluator } from "@llm4ts/core/eval/Evaluator"
+import { makeEvaluator, type Evaluator } from "@llm4ts/core/eval/Evaluator"
+import { judgeWithJudgment } from "@llm4ts/core/eval/Judge"
+import type { JudgmentShape } from "@llm4ts/core/judgment/Judgment"
 import { capped, withShrink } from "./Context.ts"
 import { FlowLlmError, type FlowError } from "./FlowError.ts"
-import { FlowEvents, Info } from "./FlowEvents.ts"
+import {
+  FlowEvents,
+  Info,
+  JudgmentObserved,
+  publishJudgmentObserved,
+  type FlowEventsShape
+} from "./FlowEvents.ts"
+import { certaintyOf, decide, type JudgmentMode } from "./Judgment.ts"
 import type { GitToolShape } from "./GitTool.ts"
 import type { Pack } from "./Pack.ts"
 import type { PlainFileStoreShape } from "./Persistence.ts"
@@ -49,6 +58,8 @@ export const groupFiles = (
 export interface ProgramJudgeOptions {
   readonly pack: Pack
   readonly judge: Evaluator<Sample>
+  /** Observe alongside the generative judge by default; only act replaces its scores. */
+  readonly judgment?: { readonly judgment: JudgmentShape; readonly mode?: JudgmentMode }
   readonly dimensions: ReadonlyArray<Dimension>
   readonly git: GitToolShape
   readonly files: PlainFileStoreShape
@@ -63,6 +74,70 @@ export interface ProgramJudgeOptions {
   readonly fingerprint: (...parts: ReadonlyArray<string>) => string
 }
 
+/**
+ * Compare typed answers with generative scores without changing the result in
+ * observe/advise. Failed questions have no answer to observe; backend failure
+ * keeps the generative result. Act uses core's judgment evaluator unchanged.
+ */
+export const withJudgment = (
+  generative: Evaluator<Sample>,
+  dimensions: ReadonlyArray<Dimension>,
+  options: NonNullable<ProgramJudgeOptions["judgment"]>,
+  events: FlowEventsShape
+): Evaluator<Sample> => {
+  const mode = options.mode ?? "observe"
+  if (mode === "act") return judgeWithJudgment(options.judgment, dimensions)
+  return makeEvaluator(
+    Effect.fn("@llm4ts/flow/ProgramJudge.withJudgment")(function* (sample: Sample) {
+      const full = yield* generative.evaluate(sample)
+      const observing: JudgmentShape = {
+        ...options.judgment,
+        judge: (request) =>
+          options.judgment.judge(request).pipe(
+            Effect.tap(
+              Effect.fnUntraced(function* (result) {
+                for (const dimension of dimensions) {
+                  const answer = result.answers[dimension.name]
+                  const score = full.scores.find((entry) => entry.name === dimension.name)
+                  if (answer?.type !== "score" || score === undefined) continue
+                  yield* publishJudgmentObserved(
+                    events,
+                    JudgmentObserved.make({
+                      consumer: "program-judge",
+                      state: request.state,
+                      question: request.questions[dimension.name],
+                      answer,
+                      judgmentIdentity: options.judgment.identity,
+                      key: dimension.name,
+                      decision: decide(answer),
+                      certainty: certaintyOf(answer),
+                      support: answer.support,
+                      origin: answer.origin,
+                      outcome: { _tag: "ProgramJudge", score: score.score },
+                      mode
+                    })
+                  )
+                }
+              })
+            )
+          )
+      }
+      yield* judgeWithJudgment(observing, dimensions)
+        .evaluate(sample)
+        .pipe(
+          Effect.catch(() =>
+            events.publish(
+              Info.make({
+                message: "program judgment unavailable; keeping the generative result"
+              })
+            )
+          )
+        )
+      return full
+    })
+  )
+}
+
 const join = (root: string, path: string): string =>
   `${root.replace(/[\\/]+$/, "")}/${path.replace(/^[\\/]+/, "")}`
 
@@ -128,17 +203,27 @@ const judgeSlice = Effect.fn("@llm4ts/flow/ProgramJudge.judgeSlice")(function* (
 ): Effect.fn.Return<ReviewResult, FlowError, FlowEvents> {
   const events = yield* FlowEvents
   const rubric = rubricText(options.dimensions)
+  const judge =
+    options.judgment === undefined
+      ? options.judge
+      : withJudgment(options.judge, options.dimensions, options.judgment, events)
+  // A mode switch (especially act back to observe) must not reuse the other
+  // path's verdict. Checkpoint changes must run the comparison again too.
+  const judgmentKey =
+    options.judgment === undefined
+      ? []
+      : [options.judgment.mode ?? "observe", options.judgment.judgment.identity]
   return yield* cachedReview(
     options.files,
     join(options.gateDir, `${label}.json`),
-    options.fingerprint(spec, diff, rubric),
+    options.fingerprint(spec, diff, rubric, ...judgmentKey),
     events.publish(Info.make({ message: `judging ${label}` })).pipe(
       Effect.andThen(
         withShrink(`judge[${label}]`, (cap) =>
           Effect.gen(function* () {
             const cappedSpec = yield* capped(`spec[${label}]`, spec, cap)
             const cappedDiff = yield* capped(`diff[${label}]`, diff, cap)
-            return yield* options.judge
+            return yield* judge
               .evaluate(
                 Sample.make({ response: cappedDiff, context: cappedSpec, query: options.query })
               )

package/src/Replay.ts

@@ -3,6 +3,7 @@ import * as Ref from "effect/Ref"
 import * as Schema from "effect/Schema"
 import * as Stream from "effect/Stream"
 import { InvalidRequestError, ParseError, ProviderError } from "@llm4ts/core/Errors"
+import { verbalizedScoreLabels } from "@llm4ts/core/LabelScoring"
 import type { LlmServiceShape } from "@llm4ts/core/LlmService"
 import { LlmChunk, TokenUsage } from "@llm4ts/core/Models"
 import { collect } from "@llm4ts/core/Streaming"
@@ -211,6 +212,24 @@ export const makeReplayConnector = Effect.fn("@llm4ts/flow/Replay.makeConnector"
     )
   )
 
+  const executeStructuredWithUsage = <A, E, RD, RE>(
+    _prompt: string,
+    schema: Schema.ConstraintCodec<A, E, RD, RE>
+  ) =>
+    collect(next).pipe(
+      Effect.flatMap((response) =>
+        Schema.decodeUnknownEffect(Schema.fromJsonString(schema))(response.content).pipe(
+          Effect.map((value) => [value, response.usage, response.metadata.model] as const)
+        )
+      ),
+      Effect.mapError((error) =>
+        ParseError.make({
+          message: `replay structured parse error: ${String(error)}`,
+          raw: ""
+        })
+      )
+    )
+
   return {
     executeStream: (_prompt) => next,
     executeStreamWithHistory: (_messages) => next,
@@ -235,23 +254,10 @@ export const makeReplayConnector = Effect.fn("@llm4ts/flow/Replay.makeConnector"
           })
         )
       ),
-    executeStructuredWithUsage: <A, E, RD, RE>(
-      _prompt: string,
-      schema: Schema.ConstraintCodec<A, E, RD, RE>
-    ) =>
-      collect(next).pipe(
-        Effect.flatMap((response) =>
-          Schema.decodeUnknownEffect(Schema.fromJsonString(schema))(response.content).pipe(
-            Effect.map((value) => [value, response.usage, response.metadata.model] as const)
-          )
-        ),
-        Effect.mapError((error) =>
-          ParseError.make({
-            message: `replay structured parse error: ${String(error)}`,
-            raw: ""
-          })
-        )
-      ),
+    executeStructuredWithUsage,
+    // A replayed label question decodes the recorded JSON reply like any
+    // other structured call: the trace holds the verbalized answer.
+    scoreLabels: verbalizedScoreLabels(executeStructuredWithUsage),
     isAvailable: Effect.succeed(true)
   }
 })