@llm4ts/flow 2.5.0 → 2.6.0

package/src/GitTool.ts

@@ -90,6 +90,17 @@ const problem = (result: ProcessResult): string => {
   return detail.length === 0 ? `process exited with code ${result.exitCode}` : detail
 }
 
+/**
+ * The runner's own bookkeeping under `workDir/.llm4ts/`: the run trace and
+ * the cost ledger. `commitAll` never stages them — they grow while the run
+ * commits and belong to the machine, not the repository — while the rest of
+ * `.llm4ts/` (plans, forked packs) is committed as before.
+ */
+export const runnerBookkeeping: ReadonlyArray<string> = [
+  ".llm4ts/trace-*.jsonl",
+  ".llm4ts/costs.jsonl"
+]
+
 export const makeGitTool = (
   process: ProcessExecutorShape,
   workDir: string,
@@ -200,7 +211,14 @@ export const makeGitTool = (
   const checkout = (name: string): Effect.Effect<void, FlowError> =>
     write("git checkout", runOrFail(["checkout", name]).pipe(Effect.asVoid))
 
-  const addAll = runOrFail(["add", "-A"]).pipe(Effect.asVoid)
+  // Stage everything, then take the runner's bookkeeping back out of the
+  // index: `reset -- <glob>` leaves an ignored or unmatched glob alone and
+  // keeps a previously committed trace at its HEAD version, whereas an
+  // `:(exclude)` pathspec makes `add` refuse the ignored files outright.
+  const addAll = runOrFail(["add", "-A"]).pipe(
+    Effect.andThen(runOrFail(["reset", "-q", "--", ...runnerBookkeeping])),
+    Effect.asVoid
+  )
 
   const commitStaged = (message: string): Effect.Effect<CommitResult, FlowError> =>
     Effect.gen(function* () {

package/src/Judgment.ts

@@ -0,0 +1,418 @@
+import * as Effect from "effect/Effect"
+import * as Schema from "effect/Schema"
+import { ProviderError } from "@llm4ts/core/Errors"
+import type { LlmServiceShape } from "@llm4ts/core/LlmService"
+import type { JsonSchema } from "@llm4ts/core/Models"
+import type {
+  JudgmentBackendError,
+  JudgmentInput,
+  JudgmentShape
+} from "@llm4ts/core/judgment/Judgment"
+import { makeLlmJudgment } from "@llm4ts/core/judgment/LlmJudgment"
+import {
+  ChoiceAnswer,
+  JudgmentResult,
+  ScoreAnswer,
+  TruthAnswer,
+  confidenceOf,
+  expectedScore,
+  origins,
+  renderState,
+  type Answer,
+  type AnswerOrigin,
+  type JudgmentBackend,
+  type Question
+} from "@llm4ts/core/judgment/Schemas"
+import { ParseError } from "@llm4ts/core/Errors"
+import type { FlowContextShape } from "./FlowContext.ts"
+import { FlowLlmError, PersistenceError, type FlowError } from "./FlowError.ts"
+import { Info, type FlowEventsShape } from "./FlowEvents.ts"
+import type { PlainFileStoreShape } from "./Persistence.ts"
+import { type Decision } from "./JudgmentTypes.ts"
+
+export { Decision, JudgmentMode } from "./JudgmentTypes.ts"
+
+/**
+ * Judgment policy (ADR 0017): core answers questions and reports their
+ * origin; this module decides what a flow does with the number. Thresholds
+ * are keyed first by calibration evidence (`measured`, then `claimed`) and
+ * otherwise by extraction method, because a model that writes its own
+ * probabilities (`verbalized`) is held to a higher bar than one whose
+ * log-probabilities were read (`logprobs`), and a reasoning model's typed
+ * reply (`reasoning`) sits between. An answer whose `support` (mass the
+ * backend actually placed on the offered options) is below `minSupport` is
+ * held whatever its confidence. Jev's advice, kept: questions and
+ * thresholds in one place.
+ */
+
+export class CertaintyBands extends Schema.Class<CertaintyBands>("CertaintyBands")({
+  /** At or above: act automatically. */
+  act: Schema.Number,
+  /** Below: do not act; escalate or hold. Between the two: proceed with caution. */
+  hold: Schema.Number
+}) {}
+
+const thresholds = (act: number, hold: number) =>
+  CertaintyBands.pipe(
+    Schema.withConstructorDefault(Effect.succeed(CertaintyBands.make({ act, hold })))
+  )
+
+export class JudgmentPolicy extends Schema.Class<JudgmentPolicy>("JudgmentPolicy")({
+  /** Calibration evidence produced in this project: trusted most. */
+  measured: thresholds(0.8, 0.5),
+  /** The provider claims calibration (TypeSafe). */
+  claimed: thresholds(0.85, 0.5),
+  /** No evidence; keyed by how the numbers were extracted. */
+  logprobs: thresholds(0.9, 0.6),
+  sampled: thresholds(0.9, 0.6),
+  reasoning: thresholds(0.9, 0.6),
+  verbalized: thresholds(0.95, 0.7),
+  /** Below this share of mass on the offered options, hold regardless. */
+  minSupport: Schema.Number.pipe(Schema.withConstructorDefault(Effect.succeed(0.5)))
+}) {
+  thresholds(origin: AnswerOrigin): CertaintyBands {
+    switch (origin.calibration) {
+      case "measured":
+        return this.measured
+      case "claimed":
+        return this.claimed
+      case "none":
+        return origin.method === "hosted" ? this.claimed : this[origin.method]
+    }
+  }
+}
+
+export const defaultJudgmentPolicy = JudgmentPolicy.make({})
+
+/**
+ * How sure an answer is, whatever its kind. A Choice or Score carries its
+ * confidence (the peak probability). A Truth's certainty is its distance
+ * from even, scaled so 0.5 is 0 and either extreme is 1: `|2·truth − 1|`.
+ */
+export const certaintyOf = (answer: Answer): number =>
+  answer.type === "truth" ? Math.abs(2 * answer.truth - 1) : answer.confidence
+
+/**
+ * The three bands Jev documents: act, proceed with caution, or hold. An
+ * answer rebuilt from too little mass on the offered options is always held.
+ */
+export const decide = (
+  answer: Answer,
+  policy: JudgmentPolicy = defaultJudgmentPolicy
+): Decision => {
+  if (answer.support < policy.minSupport) {
+    return "hold"
+  }
+  const certainty = certaintyOf(answer)
+  const bands = policy.thresholds(answer.origin)
+  return certainty >= bands.act ? "act" : certainty < bands.hold ? "hold" : "caution"
+}
+
+/** The context's judgment service, or one derived from its reasoning seat. */
+export const judgmentOf = (context: FlowContextShape): JudgmentShape =>
+  context.judgment ?? makeLlmJudgment(context.reasoning)
+
+const backendFailure = (error: JudgmentBackendError): FlowError =>
+  FlowLlmError.from(ProviderError.make({ message: error.message, cause: error }))
+
+// Escalation asks a reasoning model the same question in its own words and
+// decodes a typed reply, so the schema per kind mirrors the answer shape.
+class EscalatedChoice extends Schema.Class<EscalatedChoice>("EscalatedChoice")({
+  choice: Schema.String,
+  probabilities: Schema.Record(Schema.String, Schema.Number)
+}) {}
+class EscalatedScore extends Schema.Class<EscalatedScore>("EscalatedScore")({
+  probabilities: Schema.Record(Schema.String, Schema.Number)
+}) {}
+class EscalatedTruth extends Schema.Class<EscalatedTruth>("EscalatedTruth")({
+  truth: Schema.Number
+}) {}
+
+const numberMap = (keys: ReadonlyArray<string>): JsonSchema => ({
+  type: "object",
+  properties: Object.fromEntries(keys.map((key) => [key, { type: "number" }])),
+  required: [...keys],
+  additionalProperties: false
+})
+
+const escalationPrompt = (state: JudgmentInput["state"], question: Question): string => {
+  const header = `State:\n${renderState(state)}\n\nThink carefully, then answer with JSON only.\n`
+  switch (question.type) {
+    case "choice":
+      return (
+        `${header}Question: ${question.instructions}\nOptions:\n` +
+        Object.entries(question.criteria)
+          .map(([key, description]) => `- ${key}: ${JSON.stringify(description)}`)
+          .join("\n") +
+        '\nReply: {"choice": <option key>, "probabilities": {<a probability per option, summing to 1>}}'
+      )
+    case "score":
+      return (
+        `${header}Rate: ${question.instructions}\nLevels:\n` +
+        question.criteria.map((level, index) => `- ${index}: ${JSON.stringify(level)}`).join("\n") +
+        '\nReply: {"probabilities": {<a probability per level index, summing to 1>}}'
+      )
+    case "truth":
+      return (
+        `${header}Statement: ${question.instructions}\n` +
+        (question.criteria === undefined
+          ? ""
+          : `True means: ${JSON.stringify(question.criteria.true)}\nFalse means: ${JSON.stringify(question.criteria.false)}\n`) +
+        'Reply: {"truth": <probability between 0 and 1 that the statement holds>}'
+      )
+  }
+}
+
+/** Renormalize over the offered keys; fails typed when nothing was placed on them. */
+const normalized = (
+  keys: ReadonlyArray<string>,
+  raw: Readonly<Record<string, number>>
+): Effect.Effect<Record<string, number>, ParseError> => {
+  const kept = keys.map((key) => [key, Math.max(0, raw[key] ?? 0)] as const)
+  const total = kept.reduce((sum, [, value]) => sum + value, 0)
+  return total > 0
+    ? Effect.succeed(Object.fromEntries(kept.map(([key, value]) => [key, value / total])))
+    : Effect.fail(
+        ParseError.make({
+          message: "the reasoning seat placed no probability on any offered option",
+          raw: JSON.stringify(raw)
+        })
+      )
+}
+
+const escalate = (
+  reasoning: LlmServiceShape,
+  backend: JudgmentBackend,
+  state: JudgmentInput["state"],
+  question: Question
+): Effect.Effect<Answer, FlowError> => {
+  const prompt = escalationPrompt(state, question)
+  const origin = origins.escalated(backend)
+  switch (question.type) {
+    case "choice": {
+      const keys = Object.keys(question.criteria)
+      return reasoning
+        .executeStructured(prompt, EscalatedChoice, {
+          type: "object",
+          properties: {
+            choice: { type: "string", enum: [...keys] },
+            probabilities: numberMap(keys)
+          },
+          required: ["choice", "probabilities"]
+        })
+        .pipe(
+          Effect.flatMap((reply) =>
+            // A choice the model gave no mass contradicts itself: fail rather
+            // than invent certainty, and the caller keeps its earlier answer.
+            keys.includes(reply.choice) && !((reply.probabilities[reply.choice] ?? 0) > 0)
+              ? Effect.fail(
+                  ParseError.make({
+                    message: `the reasoning seat chose "${reply.choice}" but gave it no probability`,
+                    raw: JSON.stringify(reply.probabilities)
+                  })
+                )
+              : Effect.map(normalized(keys, reply.probabilities), (probabilities) =>
+                  ChoiceAnswer.make({
+                    type: "choice",
+                    choice: keys.includes(reply.choice) ? reply.choice : (keys[0] ?? ""),
+                    probabilities,
+                    confidence: confidenceOf(probabilities),
+                    origin
+                  })
+                )
+          ),
+          Effect.mapError(FlowLlmError.from)
+        )
+    }
+    case "score": {
+      const keys = question.criteria.map((_, index) => String(index))
+      return reasoning
+        .executeStructured(prompt, EscalatedScore, {
+          type: "object",
+          properties: { probabilities: numberMap(keys) },
+          required: ["probabilities"]
+        })
+        .pipe(
+          Effect.flatMap((reply) =>
+            Effect.map(normalized(keys, reply.probabilities), (probabilities) =>
+              ScoreAnswer.make({
+                type: "score",
+                score: expectedScore(probabilities),
+                legend: Object.fromEntries(
+                  question.criteria.map((level, index) => [String(index), level])
+                ),
+                probabilities,
+                confidence: confidenceOf(probabilities),
+                origin
+              })
+            )
+          ),
+          Effect.mapError(FlowLlmError.from)
+        )
+    }
+    case "truth":
+      return reasoning
+        .executeStructured(prompt, EscalatedTruth, {
+          type: "object",
+          properties: { truth: { type: "number" } },
+          required: ["truth"]
+        })
+        .pipe(
+          Effect.map((reply) =>
+            TruthAnswer.make({
+              type: "truth",
+              truth: Math.max(0, Math.min(1, reply.truth)),
+              origin
+            })
+          ),
+          Effect.mapError(FlowLlmError.from)
+        )
+  }
+}
+
+export interface JudgeOrEscalateOptions {
+  readonly judgment: JudgmentShape
+  readonly reasoning: LlmServiceShape
+  readonly events: FlowEventsShape
+  readonly request: JudgmentInput
+  readonly policy?: JudgmentPolicy
+}
+
+/**
+ * Run the judgment; every answer the policy would `hold`, and every question
+ * that failed, is asked again of the reasoning seat and replaced with an
+ * `escalated` answer. An escalation that fails keeps what it had, so the
+ * caller always sees the most informed answer available.
+ */
+export const judgeOrEscalate = Effect.fn("@llm4ts/flow/Judgment.judgeOrEscalate")(function* (
+  options: JudgeOrEscalateOptions
+): Effect.fn.Return<JudgmentResult, FlowError> {
+  const policy = options.policy ?? defaultJudgmentPolicy
+  const result = yield* options.judgment
+    .judge(options.request)
+    .pipe(Effect.mapError(backendFailure))
+  const answers: Record<string, Answer> = { ...result.answers }
+  const failures = [...result.failures]
+  for (const [key, question] of Object.entries(options.request.questions)) {
+    const current = answers[key]
+    const reason =
+      current === undefined
+        ? (failures.find((failure) => failure.key === key)?.reason ?? "no answer")
+        : decide(current, policy) === "hold"
+          ? current.support < policy.minSupport
+            ? `${current.origin.method} support ${current.support.toFixed(2)} below ${policy.minSupport}`
+            : `${current.origin.method} certainty ${certaintyOf(current).toFixed(2)} below hold`
+          : undefined
+    if (reason === undefined) {
+      continue
+    }
+    yield* options.events.publish(
+      Info.make({ message: `judgment '${key}' escalated to the reasoning seat: ${reason}` })
+    )
+    const escalated = yield* escalate(
+      options.reasoning,
+      result.backend,
+      options.request.state,
+      question
+    ).pipe(
+      Effect.map((answer): Answer | undefined => answer),
+      Effect.catch((error) =>
+        options.events
+          .publish(
+            Info.make({
+              message: `judgment '${key}' escalation failed (${error._tag}); keeping the original answer`
+            })
+          )
+          .pipe(Effect.as<Answer | undefined>(undefined))
+      )
+    )
+    if (escalated !== undefined) {
+      answers[key] = escalated
+      const index = failures.findIndex((failure) => failure.key === key)
+      if (index >= 0) {
+        failures.splice(index, 1)
+      }
+    }
+  }
+  return JudgmentResult.make({
+    answers,
+    failures,
+    backend: result.backend,
+    ...(result.usage === undefined ? {} : { usage: result.usage }),
+    ...(result.model === undefined ? {} : { model: result.model })
+  })
+})
+
+class JudgmentCacheEntry extends Schema.Class<JudgmentCacheEntry>("JudgmentCacheEntry")({
+  fingerprint: Schema.String,
+  result: JudgmentResult
+}) {}
+
+const hexDigest = (buffer: ArrayBuffer): string =>
+  Array.from(new Uint8Array(buffer))
+    .map((byte) => byte.toString(16).padStart(2, "0"))
+    .join("")
+
+/**
+ * A stable digest of state, questions and the judgment's identity (backend
+ * plus checkpoint): the cache key for one request. A retrained or swapped
+ * local model therefore never reuses its predecessor's answers.
+ */
+export const judgmentFingerprint = (
+  request: JudgmentInput,
+  identity: string
+): Effect.Effect<string, PersistenceError> =>
+  Effect.tryPromise({
+    try: async () =>
+      hexDigest(
+        await globalThis.crypto.subtle.digest(
+          "SHA-256",
+          new TextEncoder().encode(
+            JSON.stringify({ identity, state: request.state, questions: request.questions })
+          )
+        )
+      ),
+    catch: (error) =>
+      PersistenceError.make({
+        message: `failed to fingerprint judgment: ${error instanceof Error ? error.message : String(error)}`,
+        cause: error
+      })
+  })
+
+/**
+ * Answers persisted like `cachedReview`: a re-run re-judges only what
+ * changed (state, questions, or backend). A cached result carries the
+ * origin it was answered with, so policy still applies as before.
+ */
+export const cachedJudgment = Effect.fn("@llm4ts/flow/Judgment.cached")(function* (
+  files: PlainFileStoreShape,
+  path: string,
+  judgment: JudgmentShape,
+  request: JudgmentInput
+): Effect.fn.Return<JudgmentResult, FlowError> {
+  const fingerprint = yield* judgmentFingerprint(request, judgment.identity)
+  const contents = yield* files.read(path).pipe(Effect.catch(() => Effect.succeed(undefined)))
+  const entry =
+    contents === undefined
+      ? undefined
+      : yield* Schema.decodeUnknownEffect(Schema.fromJsonString(JudgmentCacheEntry))(contents).pipe(
+          Effect.option,
+          Effect.map((option) => (option._tag === "Some" ? option.value : undefined))
+        )
+  if (entry?.fingerprint === fingerprint) {
+    return entry.result
+  }
+  const result = yield* judgment.judge(request).pipe(Effect.mapError(backendFailure))
+  const encoded = yield* Schema.encodeEffect(Schema.fromJsonString(JudgmentCacheEntry))(
+    JudgmentCacheEntry.make({ fingerprint, result })
+  ).pipe(
+    Effect.mapError((error) =>
+      PersistenceError.make({
+        message: `failed to encode judgment cache: ${String(error)}`,
+        cause: error
+      })
+    )
+  )
+  yield* files.writeAtomic(path, encoded)
+  return result
+})

package/src/JudgmentDataset.ts

@@ -0,0 +1,254 @@
+import * as Effect from "effect/Effect"
+import * as Schema from "effect/Schema"
+import { State, ScoreQuestion, TruthQuestion, truth } from "@llm4ts/core/judgment/Schemas"
+import type { JudgmentObservation } from "./JudgmentLog.ts"
+import type { PlainFileStoreShape } from "./Persistence.ts"
+import type { Reviewer } from "./Reviewer.ts"
+
+export const DatasetDecision = Schema.Literals([
+  "review-prescreen",
+  "satisfied-probe",
+  "program-judge"
+])
+export type DatasetDecision = typeof DatasetDecision.Type
+const Text = Schema.String.check(Schema.makeFilter((value) => value.trim().length > 0))
+const DatasetQuestion = Schema.Union([TruthQuestion, ScoreQuestion])
+const baseFields = {
+  id: Text,
+  decision: DatasetDecision,
+  state: State,
+  question: DatasetQuestion,
+  source: Text
+}
+const correctQuestion = (item: {
+  readonly decision: DatasetDecision
+  readonly question: typeof DatasetQuestion.Type
+}) => item.question.type === (item.decision === "program-judge" ? "score" : "truth")
+
+export const validLabel = (question: typeof DatasetQuestion.Type, label: unknown): boolean =>
+  question.type === "truth"
+    ? typeof label === "boolean"
+    : typeof label === "number" &&
+      Number.isInteger(label) &&
+      label >= 0 &&
+      label < question.criteria.length
+
+const LabelledRecord = Schema.Struct({
+  ...baseFields,
+  label: Schema.Union([Schema.Boolean, Schema.Int]),
+  labelledBy: Text,
+  /** Human-supplied ISO 8601 timestamp. */
+  labelledAt: Schema.String.check(
+    Schema.makeFilter(
+      (value) => /^\d{4}-\d{2}-\d{2}T/.test(value) && Number.isFinite(Date.parse(value))
+    )
+  )
+}).check(
+  Schema.makeFilter(correctQuestion),
+  Schema.makeFilter(
+    (item) =>
+      validLabel(item.question, item.label) ||
+      `Invalid label for ${item.id} (${item.question.type})`
+  )
+)
+
+export class LabelledItem extends Schema.Class<LabelledItem>("LabelledItem")(LabelledRecord) {}
+
+/** Seeds omit the label; edits remain readable so promotion can report all refused ids. */
+export class PendingItem extends Schema.Class<PendingItem>("PendingItem")(
+  Schema.Struct({
+    ...baseFields,
+    label: Schema.optionalKey(Schema.Json),
+    labelledBy: Schema.optionalKey(Schema.NullOr(Schema.String)),
+    labelledAt: Schema.optionalKey(Schema.NullOr(Schema.String))
+  }).check(Schema.makeFilter(correctQuestion))
+) {}
+
+export class DatasetParseError extends Schema.TaggedError<DatasetParseError>()(
+  "DatasetParseError",
+  {
+    path: Schema.String,
+    line: Schema.Int,
+    message: Schema.String
+  }
+) {}
+
+export class PromotionRefused extends Schema.TaggedError<PromotionRefused>()("PromotionRefused", {
+  ids: Schema.Array(Schema.String),
+  message: Schema.String
+}) {}
+
+/** A missing file is an empty set. Blank interior lines are malformed; a final newline is optional. */
+export const readJsonLines = Effect.fn("JudgmentDataset.readJsonLines")(function* <A, I>(
+  files: PlainFileStoreShape,
+  path: string,
+  schema: Schema.Codec<A, I>
+) {
+  const contents = yield* files.read(path)
+  if (contents === undefined || contents === "") return []
+  const lines = contents.split("\n")
+  if (lines.at(-1) === "") lines.pop()
+  return yield* Effect.forEach(lines, (line, index) =>
+    Schema.decodeUnknownEffect(Schema.fromJsonString(schema))(line).pipe(
+      Effect.mapError(() =>
+        DatasetParseError.make({
+          path,
+          line: index + 1,
+          message: `Malformed JSONL record at ${path}:${index + 1}`
+        })
+      )
+    )
+  )
+})
+
+export const readDataset = (files: PlainFileStoreShape, path: string) =>
+  readJsonLines(files, path, LabelledItem)
+export const readPending = (files: PlainFileStoreShape, path: string) =>
+  readJsonLines(files, path, PendingItem)
+
+const encodeLines = Effect.fn("JudgmentDataset.encodeLines")(function* <A, I>(
+  path: string,
+  schema: Schema.Codec<A, I>,
+  items: ReadonlyArray<A>
+) {
+  const lines = yield* Effect.forEach(items, (item, index) =>
+    Schema.encodeEffect(Schema.fromJsonString(schema))(item).pipe(
+      Effect.mapError(() =>
+        DatasetParseError.make({
+          path,
+          line: index + 1,
+          message: `Invalid record for ${path}:${index + 1}`
+        })
+      )
+    )
+  )
+  return lines.length === 0 ? "" : `${lines.join("\n")}\n`
+})
+
+export const appendDataset = Effect.fn("JudgmentDataset.appendDataset")(function* (
+  files: PlainFileStoreShape,
+  path: string,
+  items: ReadonlyArray<LabelledItem>
+) {
+  yield* readDataset(files, path)
+  const encoded = yield* encodeLines(path, LabelledItem, items)
+  if (encoded === "") return
+  const previous = yield* files.read(path)
+  yield* files.append(path, `${previous && !previous.endsWith("\n") ? "\n" : ""}${encoded}`)
+})
+
+export const writePending = Effect.fn("JudgmentDataset.writePending")(function* (
+  files: PlainFileStoreShape,
+  path: string,
+  items: ReadonlyArray<PendingItem>
+) {
+  const encoded = yield* encodeLines(path, PendingItem, items)
+  yield* files.writeAtomic(path, encoded)
+})
+
+/** Escaping keeps source components unambiguous without a runtime-specific hash dependency. */
+export const candidateId = (decision: DatasetDecision, source: string): string =>
+  `${decision}:${encodeURIComponent(source)}`
+
+export const reviewCandidates = (
+  commits: ReadonlyArray<{ readonly sha: string; readonly diff: string }>,
+  lenses: ReadonlyArray<Reviewer>
+): ReadonlyArray<PendingItem> =>
+  commits.flatMap((commit) =>
+    lenses.map((lens) => {
+      const source = `commit:${commit.sha}:${lens.name}`
+      return PendingItem.make({
+        id: candidateId("review-prescreen", source),
+        decision: "review-prescreen",
+        state: { diff: commit.diff },
+        question: truth(lens.screeningStatement),
+        source
+      })
+    })
+  )
+
+/** Line position distinguishes repeated observations within a run, even at the same timestamp. */
+export const observationCandidates = Effect.fn("JudgmentDataset.observationCandidates")(function* (
+  decision: DatasetDecision,
+  observations: ReadonlyArray<JudgmentObservation>
+) {
+  return yield* Effect.forEach(
+    observations
+      .map((observation, index) => ({ observation, index }))
+      .filter(({ observation }) => observation.consumer === decision),
+    ({ observation, index }) => {
+      const source = `observation:${encodeURIComponent(observation.runId)}:${observation.consumer}:${encodeURIComponent(observation.key)}:${observation.at}:${index + 1}`
+      return Schema.decodeUnknownEffect(PendingItem)({
+        id: candidateId(decision, source),
+        decision,
+        state: observation.state,
+        question: observation.question,
+        source
+      }).pipe(
+        Effect.mapError(() =>
+          DatasetParseError.make({
+            path: source,
+            line: index + 1,
+            message: `Question kind does not match decision ${decision} at ${source}`
+          })
+        )
+      )
+    }
+  )
+})
+
+/** Existing human edits win; duplicates in the incoming batch are skipped too. */
+export const mergeCandidates = (
+  pending: ReadonlyArray<PendingItem>,
+  dataset: ReadonlyArray<LabelledItem>,
+  candidates: ReadonlyArray<PendingItem>
+): ReadonlyArray<PendingItem> => {
+  const seen = new Set([...pending, ...dataset].map((item) => item.id))
+  return [
+    ...pending,
+    ...candidates.filter((item) => {
+      if (seen.has(item.id)) return false
+      seen.add(item.id)
+      return true
+    })
+  ]
+}
+
+export const isLabelledPending = (item: PendingItem): boolean => Schema.is(LabelledRecord)(item)
+
+/**
+ * Validate the whole batch before any writes. An item nobody has labelled yet
+ * (label absent or null) simply stays pending, so labelling can proceed in
+ * batches; a label that is present but of the wrong kind or out of range
+ * refuses the whole promotion, naming the ids. Complete labels with
+ * incomplete attribution stay pending too.
+ */
+export const preparePromotion = Effect.fn("JudgmentDataset.preparePromotion")(function* (
+  pending: ReadonlyArray<PendingItem>,
+  dataset: ReadonlyArray<LabelledItem>
+) {
+  const ids = pending
+    .filter(
+      (item) =>
+        item.label !== undefined && item.label !== null && !validLabel(item.question, item.label)
+    )
+    .map((item) => item.id)
+  if (ids.length > 0)
+    return yield* PromotionRefused.make({
+      ids,
+      message: `Refusing promotion: missing or invalid labels for ids: ${ids.join(", ")}`
+    })
+  const ready = pending.filter(isLabelledPending)
+  const labelled = yield* Effect.forEach(ready, (item) =>
+    Schema.decodeUnknownEffect(LabelledItem)(item)
+  )
+  const existing = new Set(dataset.map((item) => item.id))
+  return {
+    items: labelled.filter((item) => {
+      if (existing.has(item.id)) return false
+      existing.add(item.id)
+      return true
+    }),
+    remaining: pending.filter((item) => !isLabelledPending(item))
+  }
+})