@llm4ts/core 2.4.2 → 2.6.0

package/src/LabelScoring.ts

@@ -0,0 +1,211 @@
+import * as Effect from "effect/Effect"
+import * as Schema from "effect/Schema"
+import { InvalidRequestError, ParseError, type LlmError } from "./Errors.ts"
+import * as Result from "effect/Result"
+import type { LabelSequence, LlmServiceShape } from "./LlmService.ts"
+import { LabelDistribution, type JsonSchema, type LabelMethod } from "./Models.ts"
+
+/**
+ * Label scoring: the one classification primitive every connector offers.
+ *
+ * A caller hands over a prompt and a closed set of labels and gets a
+ * probability per label back. Connectors that expose token log-probabilities
+ * implement it natively in one forward pass; everything else derives it here
+ * from `executeStructured`, asking the model to write the numbers down
+ * (method `verbalized`, which callers should hold to a higher bar).
+ */
+
+const raw = (text: string): string => (text.length > 200 ? `${text.slice(0, 200)}…` : text)
+
+/**
+ * Keep only the offered labels, clamp negatives to zero and renormalize.
+ * `support` records how much mass the offered labels held before
+ * renormalization: for log-probabilities that is their absolute share of
+ * the vocabulary distribution (so one label found with 5% of the mass
+ * yields probability 1 but support 0.05); a backend whose numbers were
+ * declared over the labels alone passes `support: 1`. Fails typed when
+ * none of the offered labels carries any mass, so a caller never branches
+ * on a silent uniform distribution.
+ */
+export const normalizeLabelProbabilities = (
+  labels: ReadonlyArray<string>,
+  observed: Readonly<Record<string, number>>,
+  method: LabelMethod,
+  extra: {
+    readonly usage?: LabelDistribution["usage"]
+    readonly model?: string
+    readonly support?: number
+  } = {}
+): Effect.Effect<LabelDistribution, ParseError> => {
+  const kept = labels.map((label) => [label, Math.max(0, observed[label] ?? 0)] as const)
+  const total = kept.reduce((sum, [, value]) => sum + value, 0)
+  if (!(total > 0)) {
+    return Effect.fail(
+      ParseError.make({
+        message: `none of the offered labels was observed: ${labels.join(", ")}`,
+        raw: raw(JSON.stringify(observed))
+      })
+    )
+  }
+  return Effect.succeed(
+    LabelDistribution.make({
+      probabilities: Object.fromEntries(kept.map(([label, value]) => [label, value / total])),
+      method,
+      support: Math.max(0, Math.min(1, extra.support ?? total)),
+      ...(extra.usage === undefined ? {} : { usage: extra.usage }),
+      ...(extra.model === undefined ? {} : { model: extra.model })
+    })
+  )
+}
+
+export class VerbalizedLabels extends Schema.Class<VerbalizedLabels>("VerbalizedLabels")({
+  label: Schema.String,
+  probabilities: Schema.Record(Schema.String, Schema.Number)
+}) {}
+
+export const verbalizedLabelsJsonSchema = (labels: ReadonlyArray<string>): JsonSchema => ({
+  type: "object",
+  properties: {
+    label: { type: "string", enum: [...labels] },
+    probabilities: {
+      type: "object",
+      properties: Object.fromEntries(labels.map((label) => [label, { type: "number" }])),
+      required: [...labels],
+      additionalProperties: false
+    }
+  },
+  required: ["label", "probabilities"],
+  additionalProperties: false
+})
+
+export const verbalizedLabelsPrompt = (prompt: string, labels: ReadonlyArray<string>): string =>
+  `${prompt}\n\nReply with JSON only: {"label": <the one label you choose>, "probabilities": {<a probability for every label, summing to 1>}}. Labels: ${labels.join(", ")}.`
+
+/**
+ * The default `scoreLabels` for a connector without log-probabilities: one
+ * schema-constrained JSON reply, renormalized over the offered labels.
+ */
+export const verbalizedScoreLabels =
+  (executeStructuredWithUsage: LlmServiceShape["executeStructuredWithUsage"]) =>
+  (prompt: string, labels: ReadonlyArray<string>): Effect.Effect<LabelDistribution, LlmError> =>
+    executeStructuredWithUsage(
+      verbalizedLabelsPrompt(prompt, labels),
+      VerbalizedLabels,
+      verbalizedLabelsJsonSchema(labels)
+    ).pipe(
+      Effect.flatMap(([reply, usage, model]) =>
+        // A reply that names a label but gives it no mass contradicts itself;
+        // failing here is what lets the judgment layer retry or hold, instead
+        // of a manufactured 1.0 that could wave a review through.
+        labels.includes(reply.label) && !((reply.probabilities[reply.label] ?? 0) > 0)
+          ? Effect.fail(
+              ParseError.make({
+                message: `the model chose "${reply.label}" but gave it no probability`,
+                raw: raw(JSON.stringify(reply.probabilities))
+              })
+            )
+          : normalizeLabelProbabilities(labels, reply.probabilities, "verbalized", {
+              // Declared over the labels alone: support is by construction.
+              support: 1,
+              ...(usage === undefined ? {} : { usage }),
+              ...(model === undefined ? {} : { model })
+            })
+      )
+    )
+
+/** For fakes and adapters that cannot classify: fails typed instead of guessing. */
+export const unsupportedScoreLabels: LlmServiceShape["scoreLabels"] = (_prompt, _labels) =>
+  Effect.fail(InvalidRequestError.make({ message: "label scoring is not supported here" }))
+
+export class VerbalizedLabelSequence extends Schema.Class<VerbalizedLabelSequence>(
+  "VerbalizedLabelSequence"
+)({
+  answers: Schema.Array(VerbalizedLabels)
+}) {}
+
+export const verbalizedLabelSequenceJsonSchema = (
+  labelSets: ReadonlyArray<ReadonlyArray<string>>
+): JsonSchema => ({
+  type: "object",
+  properties: {
+    answers: {
+      type: "array",
+      minItems: labelSets.length,
+      maxItems: labelSets.length,
+      items: {
+        type: "object",
+        properties: {
+          label: { type: "string" },
+          probabilities: { type: "object", additionalProperties: { type: "number" } }
+        },
+        required: ["label", "probabilities"]
+      }
+    }
+  },
+  required: ["answers"],
+  additionalProperties: false
+})
+
+export const verbalizedLabelSequencePrompt = (
+  prompt: string,
+  labelSets: ReadonlyArray<ReadonlyArray<string>>
+): string =>
+  `${prompt}\n\nReply with JSON only: {"answers": [<one entry per question, in order>]}, each entry {"label": <the one label you choose>, "probabilities": {<a probability for every label of that question, summing to 1>}}. Labels per question: ${labelSets
+    .map((labels, index) => `${index + 1}: ${labels.join(", ")}`)
+    .join("; ")}.`
+
+/**
+ * The derived `scoreLabelSequence`: one schema-constrained JSON reply with
+ * an answer per question. A missing position, or a chosen label with no
+ * mass, fails that position only.
+ */
+export const verbalizedScoreLabelSequence =
+  (executeStructuredWithUsage: LlmServiceShape["executeStructuredWithUsage"]) =>
+  (
+    prompt: string,
+    labelSets: ReadonlyArray<ReadonlyArray<string>>
+  ): Effect.Effect<LabelSequence, LlmError> =>
+    executeStructuredWithUsage(
+      verbalizedLabelSequencePrompt(prompt, labelSets),
+      VerbalizedLabelSequence,
+      verbalizedLabelSequenceJsonSchema(labelSets)
+    ).pipe(
+      Effect.flatMap(([reply, usage, model]) =>
+        Effect.forEach(labelSets, (labels, index) => {
+          const answer = reply.answers[index]
+          const entry: Effect.Effect<LabelDistribution, ParseError> =
+            answer === undefined
+              ? Effect.fail(
+                  ParseError.make({
+                    message: `no answer for question ${index + 1} of ${labelSets.length}`,
+                    raw: raw(JSON.stringify(reply.answers))
+                  })
+                )
+              : labels.includes(answer.label) && !((answer.probabilities[answer.label] ?? 0) > 0)
+                ? Effect.fail(
+                    ParseError.make({
+                      message: `question ${index + 1}: the model chose "${answer.label}" but gave it no probability`,
+                      raw: raw(JSON.stringify(answer.probabilities))
+                    })
+                  )
+                : normalizeLabelProbabilities(labels, answer.probabilities, "verbalized", {
+                    support: 1
+                  })
+          return Effect.result(entry)
+        }).pipe(
+          Effect.map(
+            (entries): LabelSequence => ({
+              entries,
+              ...(usage === undefined ? {} : { usage }),
+              ...(model === undefined ? {} : { model })
+            })
+          )
+        )
+      )
+    )
+
+/** Convenience for callers that want the sequence's successes only. */
+export const sequenceSuccesses = (
+  sequence: LabelSequence
+): ReadonlyArray<LabelDistribution | undefined> =>
+  sequence.entries.map((entry) => (Result.isSuccess(entry) ? entry.success : undefined))

package/src/LlmService.ts

@@ -2,9 +2,11 @@ import * as Context from "effect/Context"
 import type * as Effect from "effect/Effect"
 import type * as Stream from "effect/Stream"
 import type * as Schema from "effect/Schema"
-import type { LlmError } from "./Errors.ts"
+import type * as Result from "effect/Result"
+import type { LlmError, ParseError } from "./Errors.ts"
 import type {
   JsonSchema,
+  LabelDistribution,
   LlmChunk,
   Message,
   TokenUsage,
@@ -37,9 +39,43 @@ export interface LlmServiceShape {
     schema: Schema.ConstraintCodec<A, E, RD, RE>,
     jsonSchema: JsonSchema
   ) => Effect.Effect<StructuredResult<A>, LlmError, RD>
+  /**
+   * One atomic classification: the probability of each offered label given
+   * the prompt. Connectors with token log-probabilities answer in one forward
+   * pass; the rest derive it from `executeStructured` (see `LabelScoring`).
+   * Fails with `ParseError` when no offered label could be observed.
+   */
+  readonly scoreLabels: (
+    prompt: string,
+    labels: ReadonlyArray<string>
+  ) => Effect.Effect<LabelDistribution, LlmError>
+  /**
+   * Several label questions answered in one call over a shared prompt
+   * prefix (the judgment layer's `shared-prefix` batching): the reply is one
+   * label per question in order, and each position is read as its own
+   * distribution. Optional, unlike `scoreLabels`: it is a cost optimization
+   * that only backends with token log-probabilities implement natively, and
+   * the judgment layer derives a verbalized version from structured output
+   * when it is absent, so no fake or decorator has to carry it.
+   */
+  readonly scoreLabelSequence?: (
+    prompt: string,
+    labelSets: ReadonlyArray<ReadonlyArray<string>>
+  ) => Effect.Effect<LabelSequence, LlmError>
   readonly isAvailable: Effect.Effect<boolean>
 }
 
+/**
+ * The reply to `scoreLabelSequence`: one outcome per question in order (a
+ * position that could not be read fails on its own, so the caller can fall
+ * back for that question only), plus the call's usage and model once.
+ */
+export interface LabelSequence {
+  readonly entries: ReadonlyArray<Result.Result<LabelDistribution, ParseError>>
+  readonly usage?: TokenUsage
+  readonly model?: string
+}
+
 export class LlmService extends Context.Service<LlmService, LlmServiceShape>()(
   "@llm4ts/core/LlmService"
 ) {}

package/src/Models.ts

@@ -9,6 +9,7 @@ export const LlmProvider = Schema.Literals([
   "Anthropic",
   "LmStudio",
   "Ollama",
+  "MlxLm",
   "OpenCode",
   "Mock"
 ])
@@ -24,6 +25,7 @@ export const ConnectorIds = Object.freeze({
   GeminiApi: new ConnectorId({ value: "gemini-api" }),
   LmStudio: new ConnectorId({ value: "lm-studio" }),
   Ollama: new ConnectorId({ value: "ollama" }),
+  MlxLm: new ConnectorId({ value: "mlx-lm" }),
   ClaudeCli: new ConnectorId({ value: "claude-cli" }),
   GeminiCli: new ConnectorId({ value: "gemini-cli" }),
   OpenCode: new ConnectorId({ value: "opencode" }),
@@ -41,7 +43,8 @@ export const apiConnectorIds: ReadonlyArray<ConnectorId> = Object.freeze([
   ConnectorIds.Anthropic,
   ConnectorIds.GeminiApi,
   ConnectorIds.LmStudio,
-  ConnectorIds.Ollama
+  ConnectorIds.Ollama,
+  ConnectorIds.MlxLm
 ])
 
 export const cliConnectorIds: ReadonlyArray<ConnectorId> = Object.freeze([
@@ -77,6 +80,8 @@ export const defaultBaseUrl = (provider: LlmProvider): string | undefined => {
       return "http://localhost:1234/v1"
     case "Ollama":
       return "http://localhost:11434"
+    case "MlxLm":
+      return "http://localhost:8080"
     case "OpenCode":
       return "http://localhost:4096"
   }
@@ -88,6 +93,7 @@ const connectorProviderTable: Readonly<Record<string, LlmProvider>> = {
   "gemini-api": "GeminiApi",
   "lm-studio": "LmStudio",
   ollama: "Ollama",
+  "mlx-lm": "MlxLm",
   opencode: "OpenCode",
   "gemini-cli": "GeminiCli",
   mock: "Mock"
@@ -115,6 +121,8 @@ export const providerConnectorId = (provider: LlmProvider): ConnectorId => {
       return ConnectorIds.LmStudio
     case "Ollama":
       return ConnectorIds.Ollama
+    case "MlxLm":
+      return ConnectorIds.MlxLm
     case "OpenCode":
       return ConnectorIds.OpenCode
     case "Mock":
@@ -255,6 +263,37 @@ export type InteractionSupport = typeof InteractionSupport.Type
 export const ReadOnlyEnforcement = Schema.Literals(["enforced", "advisory", "ignored"])
 export type ReadOnlyEnforcement = typeof ReadOnlyEnforcement.Type
 
+/**
+ * How a connector produces the per-label probabilities behind `scoreLabels`:
+ * - "logprobs": read off the backend's token log-probabilities in one
+ *   forward pass (a real distribution, still uncalibrated).
+ * - "verbalized": the model writes the numbers itself in a JSON reply
+ *   (typically overconfident; hold to a higher bar).
+ * - "none": the connector cannot answer label questions at all.
+ */
+export const LabelProbabilities = Schema.Literals(["logprobs", "verbalized", "none"])
+export type LabelProbabilities = typeof LabelProbabilities.Type
+
+/** How a label distribution's numbers were extracted (see `LabelProbabilities`). */
+export const LabelMethod = Schema.Literals(["logprobs", "verbalized", "sampled"])
+export type LabelMethod = typeof LabelMethod.Type
+
+/**
+ * A probability per label, normalized over the labels the caller offered.
+ * `support` is the probability mass the backend actually placed on those
+ * labels before renormalization (1 when the numbers were declared over the
+ * labels alone): a distribution renormalized from a sliver of mass is not
+ * a confident one, and callers must not treat it as one.
+ * `usage` is whatever the backend reported for the call, if anything.
+ */
+export class LabelDistribution extends Schema.Class<LabelDistribution>("LabelDistribution")({
+  probabilities: Schema.Record(Schema.String, Schema.Number),
+  method: LabelMethod,
+  support: Schema.Number,
+  usage: Schema.optionalKey(TokenUsage),
+  model: Schema.optionalKey(Schema.String)
+}) {}
+
 export class ConnectorCapabilities extends Schema.Class<ConnectorCapabilities>(
   "ConnectorCapabilities"
 )({
@@ -265,6 +304,9 @@ export class ConnectorCapabilities extends Schema.Class<ConnectorCapabilities>(
   approval: Schema.Boolean.pipe(Schema.withConstructorDefault(Effect.succeed(false))),
   structuredOutput: Schema.Boolean.pipe(Schema.withConstructorDefault(Effect.succeed(true))),
   usageReporting: Schema.Boolean.pipe(Schema.withConstructorDefault(Effect.succeed(true))),
+  labelProbabilities: LabelProbabilities.pipe(
+    Schema.withConstructorDefault(Effect.succeed<LabelProbabilities>("verbalized"))
+  ),
   readOnlyEnforcement: ReadOnlyEnforcement.pipe(
     Schema.withConstructorDefault(Effect.succeed<ReadOnlyEnforcement>("advisory"))
   )

package/src/eval/Judge.ts

@@ -1,5 +1,8 @@
 import * as Effect from "effect/Effect"
 import * as Schema from "effect/Schema"
+import { ProviderError } from "../Errors.ts"
+import type { JudgmentShape } from "../judgment/Judgment.ts"
+import { score, type ScoreQuestion, type State } from "../judgment/Schemas.ts"
 import type { LlmServiceShape } from "../LlmService.ts"
 import type { JsonSchema } from "../Models.ts"
 import { DimensionScore, EvalResult } from "./Eval.ts"
@@ -90,3 +93,74 @@ export const judge = (
       .executeStructured(buildPrompt(system, dimensions, sample), JudgeResponse, judgeJsonSchema)
       .pipe(Effect.map((response) => toResult(dimensions, response)))
   )
+
+/**
+ * A dimension as a Score question: one level per rubric point, so the
+ * judgment returns a distribution over the scale instead of one integer.
+ */
+export const dimensionQuestion = (dimension: Dimension): ScoreQuestion =>
+  score(
+    `${dimension.name}: ${dimension.rubric}`,
+    Array.from({ length: dimension.maxScore + 1 }, (_, level) => ({
+      level,
+      of: dimension.maxScore,
+      meaning:
+        level === 0
+          ? "does not meet the rubric at all"
+          : level === dimension.maxScore
+            ? "fully meets the rubric"
+            : `partially meets the rubric (${level} of ${dimension.maxScore})`
+    }))
+  )
+
+export const sampleState = (sample: Sample): State => ({
+  ...(sample.query === undefined ? {} : { query: sample.query }),
+  ...(sample.context === undefined ? {} : { context: sample.context }),
+  response: sample.response,
+  ...(sample.expected === undefined ? {} : { expected: sample.expected })
+})
+
+/**
+ * The rubric judge over the Judgment service (ADR 0017, consumer 1): every
+ * dimension is an independent Score question over the same sample. A
+ * dimension the backend could not answer scores 0 with the reason recorded,
+ * matching the generative judge's treatment of a missing score.
+ */
+export const judgeWithJudgment = (
+  judgment: JudgmentShape,
+  dimensions: ReadonlyArray<Dimension>
+): Evaluator<Sample> =>
+  makeEvaluator((sample) =>
+    judgment
+      .judge({
+        state: sampleState(sample),
+        questions: Object.fromEntries(
+          dimensions.map((dimension) => [dimension.name, dimensionQuestion(dimension)])
+        )
+      })
+      .pipe(
+        Effect.mapError((error) =>
+          ProviderError.make({ message: `judgment backend failed: ${error.message}`, cause: error })
+        ),
+        Effect.map((result) =>
+          EvalResult.make({
+            scores: dimensions.map((dimension) => {
+              const answer = result.answers[dimension.name]
+              if (answer === undefined || answer.type !== "score") {
+                const failure = result.failures.find((entry) => entry.key === dimension.name)
+                return DimensionScore.make({
+                  name: dimension.name,
+                  score: 0,
+                  reasoning: failure === undefined ? "missing" : `failed: ${failure.reason}`
+                })
+              }
+              return DimensionScore.make({
+                name: dimension.name,
+                score: Math.max(0, Math.min(dimension.maxScore, Math.round(answer.score))),
+                reasoning: `${answer.origin.method} judgment (${answer.origin.backend}), confidence ${answer.confidence.toFixed(2)}, support ${answer.support.toFixed(2)}, expected level ${answer.score.toFixed(2)}`
+              })
+            })
+          })
+        )
+      )
+  )

package/src/judgment/FakeJudgment.ts

@@ -0,0 +1,90 @@
+import * as Effect from "effect/Effect"
+import * as Layer from "effect/Layer"
+import * as Ref from "effect/Ref"
+import { Judgment, type JudgmentInput, type JudgmentShape } from "./Judgment.ts"
+import {
+  choiceAnswer,
+  JudgmentRequest,
+  JudgmentResult,
+  origins,
+  QuestionFailure,
+  scoreAnswer,
+  truthAnswer,
+  type Answer,
+  type Question
+} from "./Schemas.ts"
+
+/**
+ * `FakeJudgment`: deterministic answers for tests. A plan maps question keys
+ * to canned answers (or a failure reason); unplanned questions get a
+ * predictable default so a consumer test can run without listing every key.
+ */
+
+export interface FakeJudgmentPlan {
+  readonly answers?: Readonly<Record<string, Answer>>
+  readonly failures?: Readonly<Record<string, string>>
+  /** Truth probability for unplanned truth questions (default 1). */
+  readonly defaultTruth?: number
+}
+
+export interface FakeJudgment {
+  readonly judgment: JudgmentShape
+  readonly recorded: Effect.Effect<ReadonlyArray<JudgmentRequest>>
+}
+
+const defaultAnswer = (question: Question, defaultTruth: number): Answer => {
+  switch (question.type) {
+    case "choice": {
+      const keys = Object.keys(question.criteria)
+      return choiceAnswer(
+        Object.fromEntries(keys.map((key, index) => [key, index === 0 ? 1 : 0])),
+        origins.fake()
+      )
+    }
+    case "score":
+      return scoreAnswer(
+        question,
+        Object.fromEntries(
+          question.criteria.map((_, index) => [String(index), index === 0 ? 1 : 0])
+        ),
+        origins.fake()
+      )
+    case "truth":
+      return truthAnswer(defaultTruth, origins.fake())
+  }
+}
+
+export const makeFakeJudgment = Effect.fn("@llm4ts/core/judgment/FakeJudgment.make")(function* (
+  plan: FakeJudgmentPlan = {}
+): Effect.fn.Return<FakeJudgment> {
+  const requests = yield* Ref.make<ReadonlyArray<JudgmentRequest>>([])
+  const judge = (input: JudgmentInput): Effect.Effect<JudgmentResult> =>
+    Ref.update(requests, (all) => [
+      ...all,
+      JudgmentRequest.make({ state: input.state, questions: input.questions })
+    ]).pipe(
+      Effect.map(() => {
+        const answers: Record<string, Answer> = {}
+        const failures: Array<QuestionFailure> = []
+        for (const [key, question] of Object.entries(input.questions)) {
+          const failure = plan.failures?.[key]
+          if (failure !== undefined) {
+            failures.push(QuestionFailure.make({ key, reason: failure }))
+            continue
+          }
+          answers[key] = plan.answers?.[key] ?? defaultAnswer(question, plan.defaultTruth ?? 1)
+        }
+        return JudgmentResult.make({ answers, failures, backend: "fake" })
+      })
+    )
+  return {
+    judgment: { backend: "fake", identity: "fake", judge },
+    recorded: Ref.get(requests)
+  }
+})
+
+export const FakeJudgmentLive = (plan?: FakeJudgmentPlan): Layer.Layer<Judgment> =>
+  Layer.effect(
+    Judgment,
+    Effect.map(makeFakeJudgment(plan), (fake) => fake.judgment)
+  )

package/src/judgment/Judgment.ts

@@ -0,0 +1,123 @@
+import * as Context from "effect/Context"
+import * as Effect from "effect/Effect"
+import * as Schema from "effect/Schema"
+import {
+  ChoiceAnswer,
+  ScoreAnswer,
+  TruthAnswer,
+  type Answer,
+  type JudgmentBackend,
+  type JudgmentResult,
+  type Question,
+  type State
+} from "./Schemas.ts"
+
+/**
+ * The Judgment service (ADR 0017): evaluate typed questions against one
+ * State and return typed answers with probabilities and their origin. It
+ * answers; it never decides. Policy (thresholds, escalation, caching) lives
+ * in the flow package.
+ */
+
+export class JudgmentBackendError extends Schema.TaggedError<JudgmentBackendError>()(
+  "JudgmentBackendError",
+  {
+    backend: Schema.String,
+    message: Schema.String,
+    cause: Schema.optionalKey(Schema.Defect())
+  }
+) {}
+
+/** Raised by the typed accessors when a key is missing or of another kind. */
+export class AnswerMismatch extends Schema.TaggedError<AnswerMismatch>()("AnswerMismatch", {
+  key: Schema.String,
+  expected: Schema.String,
+  actual: Schema.optionalKey(Schema.String),
+  reason: Schema.optionalKey(Schema.String)
+}) {
+  get message(): string {
+    return this.actual === undefined
+      ? `no ${this.expected} answer for "${this.key}"${this.reason === undefined ? "" : `: ${this.reason}`}`
+      : `answer "${this.key}" is a ${this.actual}, not a ${this.expected}`
+  }
+}
+
+export const JudgmentError = Schema.Union([JudgmentBackendError, AnswerMismatch])
+export type JudgmentError = typeof JudgmentError.Type
+
+export interface JudgmentInput {
+  readonly state: State
+  readonly questions: Readonly<Record<string, Question>>
+}
+
+export interface JudgmentShape {
+  readonly backend: JudgmentBackend
+  /**
+   * Backend plus checkpoint, e.g. `llm:mlx-lm:/models/qwen3-4b` or
+   * `typesafe:jev-1.13.0`: what a cached answer is keyed on, so swapping a
+   * local model never reuses its predecessor's answers.
+   */
+  readonly identity: string
+  /**
+   * Answer every question independently over the same state. A question the
+   * backend could not answer lands in `failures`; the request as a whole
+   * fails only when the backend itself is unreachable or rejects it.
+   */
+  readonly judge: (input: JudgmentInput) => Effect.Effect<JudgmentResult, JudgmentBackendError>
+}
+
+export class Judgment extends Context.Service<Judgment, JudgmentShape>()(
+  "@llm4ts/core/judgment/Judgment"
+) {}
+
+const answerOf = (
+  result: JudgmentResult,
+  key: string,
+  expected: Answer["type"]
+): Effect.Effect<Answer, AnswerMismatch> => {
+  const answer = result.answers[key]
+  if (answer === undefined) {
+    const failure = result.failures.find((entry) => entry.key === key)
+    return Effect.fail(
+      AnswerMismatch.make({
+        key,
+        expected,
+        ...(failure === undefined ? {} : { reason: failure.reason })
+      })
+    )
+  }
+  return answer.type === expected
+    ? Effect.succeed(answer)
+    : Effect.fail(AnswerMismatch.make({ key, expected, actual: answer.type }))
+}
+
+/** Typed accessors: the honest way to read an answer by key. */
+export const choiceOf = (
+  result: JudgmentResult,
+  key: string
+): Effect.Effect<ChoiceAnswer, AnswerMismatch> =>
+  Effect.flatMap(answerOf(result, key, "choice"), (answer) =>
+    answer instanceof ChoiceAnswer
+      ? Effect.succeed(answer)
+      : Effect.fail(AnswerMismatch.make({ key, expected: "choice", actual: answer.type }))
+  )
+
+export const scoreOf = (
+  result: JudgmentResult,
+  key: string
+): Effect.Effect<ScoreAnswer, AnswerMismatch> =>
+  Effect.flatMap(answerOf(result, key, "score"), (answer) =>
+    answer instanceof ScoreAnswer
+      ? Effect.succeed(answer)
+      : Effect.fail(AnswerMismatch.make({ key, expected: "score", actual: answer.type }))
+  )
+
+export const truthOf = (
+  result: JudgmentResult,
+  key: string
+): Effect.Effect<TruthAnswer, AnswerMismatch> =>
+  Effect.flatMap(answerOf(result, key, "truth"), (answer) =>
+    answer instanceof TruthAnswer
+      ? Effect.succeed(answer)
+      : Effect.fail(AnswerMismatch.make({ key, expected: "truth", actual: answer.type }))
+  )