@llm4ts/flow 2.5.0 → 2.7.0

package/src/JudgmentDataset.ts

@@ -0,0 +1,254 @@
+import * as Effect from "effect/Effect"
+import * as Schema from "effect/Schema"
+import { State, ScoreQuestion, TruthQuestion, truth } from "@llm4ts/core/judgment/Schemas"
+import type { JudgmentObservation } from "./JudgmentLog.ts"
+import type { PlainFileStoreShape } from "./Persistence.ts"
+import type { Reviewer } from "./Reviewer.ts"
+
+export const DatasetDecision = Schema.Literals([
+  "review-prescreen",
+  "satisfied-probe",
+  "program-judge"
+])
+export type DatasetDecision = typeof DatasetDecision.Type
+const Text = Schema.String.check(Schema.makeFilter((value) => value.trim().length > 0))
+const DatasetQuestion = Schema.Union([TruthQuestion, ScoreQuestion])
+const baseFields = {
+  id: Text,
+  decision: DatasetDecision,
+  state: State,
+  question: DatasetQuestion,
+  source: Text
+}
+const correctQuestion = (item: {
+  readonly decision: DatasetDecision
+  readonly question: typeof DatasetQuestion.Type
+}) => item.question.type === (item.decision === "program-judge" ? "score" : "truth")
+
+export const validLabel = (question: typeof DatasetQuestion.Type, label: unknown): boolean =>
+  question.type === "truth"
+    ? typeof label === "boolean"
+    : typeof label === "number" &&
+      Number.isInteger(label) &&
+      label >= 0 &&
+      label < question.criteria.length
+
+const LabelledRecord = Schema.Struct({
+  ...baseFields,
+  label: Schema.Union([Schema.Boolean, Schema.Int]),
+  labelledBy: Text,
+  /** Human-supplied ISO 8601 timestamp. */
+  labelledAt: Schema.String.check(
+    Schema.makeFilter(
+      (value) => /^\d{4}-\d{2}-\d{2}T/.test(value) && Number.isFinite(Date.parse(value))
+    )
+  )
+}).check(
+  Schema.makeFilter(correctQuestion),
+  Schema.makeFilter(
+    (item) =>
+      validLabel(item.question, item.label) ||
+      `Invalid label for ${item.id} (${item.question.type})`
+  )
+)
+
+export class LabelledItem extends Schema.Class<LabelledItem>("LabelledItem")(LabelledRecord) {}
+
+/** Seeds omit the label; edits remain readable so promotion can report all refused ids. */
+export class PendingItem extends Schema.Class<PendingItem>("PendingItem")(
+  Schema.Struct({
+    ...baseFields,
+    label: Schema.optionalKey(Schema.Json),
+    labelledBy: Schema.optionalKey(Schema.NullOr(Schema.String)),
+    labelledAt: Schema.optionalKey(Schema.NullOr(Schema.String))
+  }).check(Schema.makeFilter(correctQuestion))
+) {}
+
+export class DatasetParseError extends Schema.TaggedError<DatasetParseError>()(
+  "DatasetParseError",
+  {
+    path: Schema.String,
+    line: Schema.Int,
+    message: Schema.String
+  }
+) {}
+
+export class PromotionRefused extends Schema.TaggedError<PromotionRefused>()("PromotionRefused", {
+  ids: Schema.Array(Schema.String),
+  message: Schema.String
+}) {}
+
+/** A missing file is an empty set. Blank interior lines are malformed; a final newline is optional. */
+export const readJsonLines = Effect.fn("JudgmentDataset.readJsonLines")(function* <A, I>(
+  files: PlainFileStoreShape,
+  path: string,
+  schema: Schema.Codec<A, I>
+) {
+  const contents = yield* files.read(path)
+  if (contents === undefined || contents === "") return []
+  const lines = contents.split("\n")
+  if (lines.at(-1) === "") lines.pop()
+  return yield* Effect.forEach(lines, (line, index) =>
+    Schema.decodeUnknownEffect(Schema.fromJsonString(schema))(line).pipe(
+      Effect.mapError(() =>
+        DatasetParseError.make({
+          path,
+          line: index + 1,
+          message: `Malformed JSONL record at ${path}:${index + 1}`
+        })
+      )
+    )
+  )
+})
+
+export const readDataset = (files: PlainFileStoreShape, path: string) =>
+  readJsonLines(files, path, LabelledItem)
+export const readPending = (files: PlainFileStoreShape, path: string) =>
+  readJsonLines(files, path, PendingItem)
+
+const encodeLines = Effect.fn("JudgmentDataset.encodeLines")(function* <A, I>(
+  path: string,
+  schema: Schema.Codec<A, I>,
+  items: ReadonlyArray<A>
+) {
+  const lines = yield* Effect.forEach(items, (item, index) =>
+    Schema.encodeEffect(Schema.fromJsonString(schema))(item).pipe(
+      Effect.mapError(() =>
+        DatasetParseError.make({
+          path,
+          line: index + 1,
+          message: `Invalid record for ${path}:${index + 1}`
+        })
+      )
+    )
+  )
+  return lines.length === 0 ? "" : `${lines.join("\n")}\n`
+})
+
+export const appendDataset = Effect.fn("JudgmentDataset.appendDataset")(function* (
+  files: PlainFileStoreShape,
+  path: string,
+  items: ReadonlyArray<LabelledItem>
+) {
+  yield* readDataset(files, path)
+  const encoded = yield* encodeLines(path, LabelledItem, items)
+  if (encoded === "") return
+  const previous = yield* files.read(path)
+  yield* files.append(path, `${previous && !previous.endsWith("\n") ? "\n" : ""}${encoded}`)
+})
+
+export const writePending = Effect.fn("JudgmentDataset.writePending")(function* (
+  files: PlainFileStoreShape,
+  path: string,
+  items: ReadonlyArray<PendingItem>
+) {
+  const encoded = yield* encodeLines(path, PendingItem, items)
+  yield* files.writeAtomic(path, encoded)
+})
+
+/** Escaping keeps source components unambiguous without a runtime-specific hash dependency. */
+export const candidateId = (decision: DatasetDecision, source: string): string =>
+  `${decision}:${encodeURIComponent(source)}`
+
+export const reviewCandidates = (
+  commits: ReadonlyArray<{ readonly sha: string; readonly diff: string }>,
+  lenses: ReadonlyArray<Reviewer>
+): ReadonlyArray<PendingItem> =>
+  commits.flatMap((commit) =>
+    lenses.map((lens) => {
+      const source = `commit:${commit.sha}:${lens.name}`
+      return PendingItem.make({
+        id: candidateId("review-prescreen", source),
+        decision: "review-prescreen",
+        state: { diff: commit.diff },
+        question: truth(lens.screeningStatement),
+        source
+      })
+    })
+  )
+
+/** Line position distinguishes repeated observations within a run, even at the same timestamp. */
+export const observationCandidates = Effect.fn("JudgmentDataset.observationCandidates")(function* (
+  decision: DatasetDecision,
+  observations: ReadonlyArray<JudgmentObservation>
+) {
+  return yield* Effect.forEach(
+    observations
+      .map((observation, index) => ({ observation, index }))
+      .filter(({ observation }) => observation.consumer === decision),
+    ({ observation, index }) => {
+      const source = `observation:${encodeURIComponent(observation.runId)}:${observation.consumer}:${encodeURIComponent(observation.key)}:${observation.at}:${index + 1}`
+      return Schema.decodeUnknownEffect(PendingItem)({
+        id: candidateId(decision, source),
+        decision,
+        state: observation.state,
+        question: observation.question,
+        source
+      }).pipe(
+        Effect.mapError(() =>
+          DatasetParseError.make({
+            path: source,
+            line: index + 1,
+            message: `Question kind does not match decision ${decision} at ${source}`
+          })
+        )
+      )
+    }
+  )
+})
+
+/** Existing human edits win; duplicates in the incoming batch are skipped too. */
+export const mergeCandidates = (
+  pending: ReadonlyArray<PendingItem>,
+  dataset: ReadonlyArray<LabelledItem>,
+  candidates: ReadonlyArray<PendingItem>
+): ReadonlyArray<PendingItem> => {
+  const seen = new Set([...pending, ...dataset].map((item) => item.id))
+  return [
+    ...pending,
+    ...candidates.filter((item) => {
+      if (seen.has(item.id)) return false
+      seen.add(item.id)
+      return true
+    })
+  ]
+}
+
+export const isLabelledPending = (item: PendingItem): boolean => Schema.is(LabelledRecord)(item)
+
+/**
+ * Validate the whole batch before any writes. An item nobody has labelled yet
+ * (label absent or null) simply stays pending, so labelling can proceed in
+ * batches; a label that is present but of the wrong kind or out of range
+ * refuses the whole promotion, naming the ids. Complete labels with
+ * incomplete attribution stay pending too.
+ */
+export const preparePromotion = Effect.fn("JudgmentDataset.preparePromotion")(function* (
+  pending: ReadonlyArray<PendingItem>,
+  dataset: ReadonlyArray<LabelledItem>
+) {
+  const ids = pending
+    .filter(
+      (item) =>
+        item.label !== undefined && item.label !== null && !validLabel(item.question, item.label)
+    )
+    .map((item) => item.id)
+  if (ids.length > 0)
+    return yield* PromotionRefused.make({
+      ids,
+      message: `Refusing promotion: missing or invalid labels for ids: ${ids.join(", ")}`
+    })
+  const ready = pending.filter(isLabelledPending)
+  const labelled = yield* Effect.forEach(ready, (item) =>
+    Schema.decodeUnknownEffect(LabelledItem)(item)
+  )
+  const existing = new Set(dataset.map((item) => item.id))
+  return {
+    items: labelled.filter((item) => {
+      if (existing.has(item.id)) return false
+      existing.add(item.id)
+      return true
+    }),
+    remaining: pending.filter((item) => !isLabelledPending(item))
+  }
+})

package/src/JudgmentEval.ts

@@ -0,0 +1,306 @@
+import * as Schema from "effect/Schema"
+import { expectedScore, truth, type Answer } from "@llm4ts/core/judgment/Schemas"
+import { decide, type Decision } from "./Judgment.ts"
+import { candidateId, DatasetDecision, LabelledItem } from "./JudgmentDataset.ts"
+import type { ReviewPrescreenResult, ReviewResult, Severity } from "./Review.ts"
+import type { Reviewer } from "./Reviewer.ts"
+
+export interface EvalItem {
+  readonly item: LabelledItem
+  readonly answer: Answer | undefined
+  readonly latencyMs: number
+  readonly failure?: string
+  readonly decision?: Decision
+  readonly severities?: ReadonlyArray<Severity>
+}
+
+/** One completed full-path result per lens; missing screens remain failed eval items. */
+export const replayEvalItems = (
+  commit: { readonly sha: string; readonly diff: string },
+  runs: ReadonlyArray<{ readonly lens: Reviewer; readonly result: ReviewResult }>,
+  observations: ReviewPrescreenResult["observations"],
+  screeningMs: number,
+  labelledAt: string
+): ReadonlyArray<EvalItem> =>
+  runs.map(({ lens, result }) => {
+    const observation = observations.find(({ key }) => key === lens.name)
+    const source = `commit:${commit.sha}:${lens.name}`
+    return {
+      item: LabelledItem.make({
+        id: candidateId("review-prescreen", source),
+        decision: "review-prescreen",
+        source,
+        state: observation?.state ?? { diff: commit.diff },
+        question: observation?.question ?? truth(lens.screeningStatement),
+        label: result.issues.length > 0,
+        labelledBy: "outcome",
+        labelledAt
+      }),
+      answer: observation?.answer,
+      ...(observation === undefined
+        ? { failure: "Pre-screen returned no matching answer." }
+        : { decision: observation.decision }),
+      severities: result.issues.map(({ severity }) => severity),
+      latencyMs: screeningMs / runs.length
+    }
+  })
+
+const SeverityCounts = Schema.Struct({
+  Critical: Schema.Int,
+  Warning: Schema.Int,
+  Info: Schema.Int
+})
+
+/** Lost issues and missed positive lenses have different units; verify both independently. */
+export const replayMissesAgree = (
+  report: EvalReport,
+  lost: typeof SeverityCounts.Type,
+  missedPositiveLenses: number
+): boolean =>
+  report.missedIssues?.count === missedPositiveLenses &&
+  report.missedIssues.severities !== undefined &&
+  (["Critical", "Warning", "Info"] as const).every(
+    (severity) => report.missedIssues?.severities?.[severity] === lost[severity]
+  )
+
+const Metric = Schema.NullOr(Schema.Number)
+export const EvalMemory = Schema.Struct({ restMb: Schema.Number, peakMb: Schema.Number })
+export const CalibrationBin = Schema.Struct({
+  lower: Schema.Number,
+  upper: Schema.Number,
+  count: Schema.Int,
+  meanLabelProbability: Metric
+})
+export const EvalMeasures = Schema.Struct({
+  items: Schema.Int,
+  answered: Schema.Int,
+  failed: Schema.Int,
+  accuracy: Metric,
+  ece: Metric,
+  brier: Metric,
+  bins: Schema.Array(CalibrationBin),
+  decisions: Schema.Struct({ act: Schema.Int, caution: Schema.Int, hold: Schema.Int }),
+  latencyMs: Schema.Struct({ p50: Metric, p95: Metric })
+})
+
+export class EvalReport extends Schema.Class<EvalReport>("EvalReport")({
+  overall: EvalMeasures,
+  decisions: Schema.Array(Schema.Struct({ decision: DatasetDecision, measures: EvalMeasures })),
+  missedIssues: Schema.optionalKey(
+    Schema.Struct({
+      count: Schema.Int,
+      positiveAnswered: Schema.Int,
+      rate: Metric,
+      severities: Schema.optionalKey(SeverityCounts)
+    })
+  ),
+  memory: Schema.optionalKey(EvalMemory)
+}) {}
+
+const measured = ({ item, answer, failure, latencyMs, decision }: EvalItem) => {
+  if (failure !== undefined || answer === undefined) return undefined
+  if (
+    item.question.type === "truth" &&
+    answer.type === "truth" &&
+    typeof item.label === "boolean"
+  ) {
+    return {
+      answer,
+      decision: decision ?? decide(answer),
+      latencyMs,
+      correct: answer.truth >= 0.5 === item.label,
+      probability: item.label ? answer.truth : 1 - answer.truth
+    }
+  }
+  if (item.question.type === "score" && answer.type === "score" && typeof item.label === "number") {
+    return {
+      answer,
+      decision: decision ?? decide(answer),
+      latencyMs,
+      correct: Math.round(expectedScore(answer.probabilities)) === item.label,
+      probability: answer.probabilities[String(item.label)] ?? 0
+    }
+  }
+  return undefined
+}
+
+/** Nearest rank, with no interpolation; no observations means no metric. */
+const percentile = (sorted: ReadonlyArray<number>, fraction: number): number | null =>
+  sorted[Math.ceil(sorted.length * fraction) - 1] ?? null
+
+const measures = (items: ReadonlyArray<EvalItem>): typeof EvalMeasures.Type => {
+  const answered = items.flatMap((item) => {
+    const value = measured(item)
+    return value === undefined ? [] : [value]
+  })
+  const count = answered.length
+  const average = (sum: number): number | null => (count === 0 ? null : sum / count)
+  const bins = Array.from({ length: 10 }, (_, index) => {
+    const members = answered.filter(
+      ({ probability }) => Math.min(9, Math.floor(probability * 10)) === index
+    )
+    return {
+      lower: index / 10,
+      upper: (index + 1) / 10,
+      count: members.length,
+      meanLabelProbability:
+        members.length === 0
+          ? null
+          : members.reduce((sum, member) => sum + member.probability, 0) / members.length
+    }
+  })
+  const decisions = answered.map(({ decision }) => decision)
+  const latencies = answered.map(({ latencyMs }) => latencyMs).sort((a, b) => a - b)
+  return {
+    items: items.length,
+    answered: count,
+    failed: items.length - count,
+    accuracy: average(answered.filter(({ correct }) => correct).length),
+    // The labelled outcome occurred (target 1), regardless of the predicted class.
+    ece: average(
+      bins.reduce((sum, bin) => sum + bin.count * Math.abs(1 - (bin.meanLabelProbability ?? 1)), 0)
+    ),
+    brier: average(answered.reduce((sum, value) => sum + (1 - value.probability) ** 2, 0)),
+    bins,
+    decisions: {
+      act: decisions.filter((decision) => decision === "act").length,
+      caution: decisions.filter((decision) => decision === "caution").length,
+      hold: decisions.filter((decision) => decision === "hold").length
+    },
+    latencyMs: { p50: percentile(latencies, 0.5), p95: percentile(latencies, 0.95) }
+  }
+}
+
+/** Failures and mismatched answer kinds count as failed and are excluded from all metrics. */
+export const evaluateJudgments = (
+  items: ReadonlyArray<EvalItem>,
+  memory?: typeof EvalMemory.Type
+): EvalReport => {
+  const decisions = [...new Set(items.map(({ item }) => item.decision))].sort()
+  const positives = items
+    .filter(({ item }) => item.decision === "review-prescreen" && item.label === true)
+    .flatMap((item) => {
+      const value = measured(item)
+      return value === undefined ? [] : [{ ...value, severities: item.severities }]
+    })
+  const missed = positives.filter(
+    ({ answer, decision }) => answer.type === "truth" && answer.truth < 0.5 && decision === "act"
+  )
+  const severities = missed.flatMap((item) => item.severities ?? [])
+  return EvalReport.make({
+    overall: measures(items),
+    decisions: decisions.map((decision) => ({
+      decision,
+      measures: measures(items.filter(({ item }) => item.decision === decision))
+    })),
+    ...(decisions.includes("review-prescreen")
+      ? {
+          missedIssues: {
+            count: missed.length,
+            positiveAnswered: positives.length,
+            rate: positives.length === 0 ? null : missed.length / positives.length,
+            ...(items.every((item) => item.severities !== undefined)
+              ? {
+                  severities: {
+                    Critical: severities.filter((severity) => severity === "Critical").length,
+                    Warning: severities.filter((severity) => severity === "Warning").length,
+                    Info: severities.filter((severity) => severity === "Info").length
+                  }
+                }
+              : {})
+          }
+        }
+      : {}),
+    ...(memory === undefined ? {} : { memory })
+  })
+}
+
+export interface EvalReportMeta {
+  readonly date: string
+  readonly decision: string
+  readonly backend: string
+  readonly model: string
+  readonly memoryNote?: string
+  readonly title?: string
+  readonly latencyNote?: string
+}
+
+const cell = (value: string): string => value.replace(/\|/g, "&#124;").replace(/[\r\n]/g, " ")
+const number = (value: number | null): string => (value === null ? "n/a" : value.toFixed(4))
+
+export const renderEvalReport = (report: EvalReport, meta: EvalReportMeta): string => {
+  const rows = [
+    ...report.decisions.map(({ decision, measures }) => ({ name: decision, measures })),
+    { name: "Overall", measures: report.overall }
+  ]
+  return [
+    `# ${cell(meta.title ?? "Judgment evaluation")}`,
+    "",
+    `- Date: ${cell(meta.date)}`,
+    `- Decision: ${cell(meta.decision)}`,
+    `- Backend identity: ${cell(meta.backend)}`,
+    `- Model: ${cell(meta.model)}`,
+    "",
+    "## Measures",
+    "",
+    "| Decision | Items | Answered | Failed | Accuracy | ECE (10 bins) | Brier | p50 ms | p95 ms |",
+    "| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |",
+    ...rows.map(
+      ({ name, measures: m }) =>
+        `| ${name} | ${m.items} | ${m.answered} | ${m.failed} | ${number(m.accuracy)} | ${number(m.ece)} | ${number(m.brier)} | ${number(m.latencyMs.p50)} | ${number(m.latencyMs.p95)} |`
+    ),
+    "",
+    "Metrics use answered items only; n/a means no observations. Overall is item-weighted.",
+    "Accuracy: Truth >= 0.5; Score rounds the expected level index.",
+    "ECE bins the labelled-outcome probability p into [0, 0.1), ..., [0.9, 1]; target = 1.",
+    "ECE = sum(bin count / answered * |1 - mean p|); Brier = mean((1 - p)^2).",
+    "These are labelled-outcome metrics, not predicted-confidence ECE or multiclass Brier.",
+    meta.latencyNote ??
+      "Latency uses nearest-rank percentiles per independent question, excluding failures.",
+    "",
+    "## Default policy decisions",
+    "",
+    "| Decision | Act | Caution | Hold |",
+    "| --- | ---: | ---: | ---: |",
+    ...rows.map(
+      ({ name, measures: m }) =>
+        `| ${name} | ${m.decisions.act} | ${m.decisions.caution} | ${m.decisions.hold} |`
+    ),
+    ...(report.missedIssues === undefined
+      ? []
+      : [
+          "",
+          "## Pre-screen missed issues",
+          "",
+          "| Missed positive items | Answered positive items | Missed rate |",
+          "| ---: | ---: | ---: |",
+          `| ${report.missedIssues.count} | ${report.missedIssues.positiveAnswered} | ${number(report.missedIssues.rate)} |`,
+          "",
+          "A miss requires label=true, truth<0.5 and decide=act (the lens would be skipped).",
+          ...(report.missedIssues.severities === undefined
+            ? [
+                "Severity breakdown unavailable: LabelledItem records presence, not issue counts or severity."
+              ]
+            : [
+                "",
+                "| Lost Critical issues | Lost Warning issues | Lost Info issues |",
+                "| ---: | ---: | ---: |",
+                `| ${report.missedIssues.severities.Critical} | ${report.missedIssues.severities.Warning} | ${report.missedIssues.severities.Info} |`,
+                "",
+                "Severity counts are issues on missed positive lenses; one lens can report multiple issues."
+              ])
+        ]),
+    "",
+    "## Judgment seat resident memory",
+    "",
+    "| Rest MiB | Sampled peak MiB |",
+    "| ---: | ---: |",
+    `| ${number(report.memory?.restMb ?? null)} | ${number(report.memory?.peakMb ?? null)} |`,
+    "",
+    meta.memoryNote ??
+      (report.memory === undefined
+        ? "Memory not supplied."
+        : "Memory supplied by the caller; peak is the maximum sampled RSS."),
+    ""
+  ].join("\n")
+}

package/src/JudgmentLog.ts

@@ -0,0 +1,96 @@
+import * as Clock from "effect/Clock"
+import * as Effect from "effect/Effect"
+import * as Ref from "effect/Ref"
+import * as Schema from "effect/Schema"
+import * as Semaphore from "effect/Semaphore"
+import type * as Scope from "effect/Scope"
+import * as Stream from "effect/Stream"
+import { JudgmentObserved, type FlowEvent, type FlowEventHub } from "./FlowEvents.ts"
+import type { PlainFileStoreShape } from "./Persistence.ts"
+
+export class JudgmentObservation extends Schema.Class<JudgmentObservation>("JudgmentObservation")({
+  runId: Schema.String,
+  /** Epoch milliseconds, matching FlowRecorder's timestamp. */
+  at: Schema.Number,
+  consumer: JudgmentObserved.fields.consumer,
+  key: JudgmentObserved.fields.key,
+  state: JudgmentObserved.fields.state,
+  question: JudgmentObserved.fields.question,
+  answer: JudgmentObserved.fields.answer,
+  outcome: JudgmentObserved.fields.outcome,
+  decision: JudgmentObserved.fields.decision,
+  mode: JudgmentObserved.fields.mode,
+  judgmentIdentity: JudgmentObserved.fields.judgmentIdentity
+}) {}
+
+export const judgmentLogPath = (root: string, consumer: JudgmentObservation["consumer"]): string =>
+  `${root.replace(/[\\/]+$/, "")}/.llm4ts/judgments/${consumer}.jsonl`
+
+export interface JudgmentLogShape {
+  readonly record: (event: FlowEvent) => Effect.Effect<void>
+  readonly consume: (hub: FlowEventHub) => Effect.Effect<void, never, Scope.Scope>
+  readonly awaitDrained: (hub: FlowEventHub) => Effect.Effect<void>
+}
+
+const codec = Schema.fromJsonString(JudgmentObservation)
+
+/** A best-effort subscriber, with the same permanent degradation policy as FlowRecorder. */
+export const makeJudgmentLog = Effect.fn("@llm4ts/flow/JudgmentLog.make")(function* (options: {
+  readonly files: PlainFileStoreShape
+  readonly root: string
+  readonly runId: string
+}): Effect.fn.Return<JudgmentLogShape> {
+  const consumed = yield* Ref.make(0)
+  const degraded = yield* Ref.make(false)
+  const lock = yield* Semaphore.make(1)
+  const record = (event: FlowEvent): Effect.Effect<void> =>
+    event._tag !== "JudgmentObserved"
+      ? Effect.void
+      : lock.withPermit(
+          Effect.gen(function* () {
+            if (yield* Ref.get(degraded)) return
+            const at = yield* Clock.currentTimeMillis
+            // Classified remains sealed until an explicit, audited declassify.
+            // Text uses its toString; JSON uses its toJSON. Serialize before
+            // schema traversal so even nested sealed values use that redaction,
+            // never their private payload. Already declassified plain text has
+            // no taint metadata; callers must retain the wrapper for secrets.
+            const observation = yield* Schema.decodeUnknownEffect(codec)(
+              JSON.stringify({ ...event, runId: options.runId, at })
+            )
+            const line = yield* Schema.encodeEffect(codec)(observation)
+            yield* options.files.append(judgmentLogPath(options.root, event.consumer), `${line}\n`)
+          }).pipe(Effect.catch(() => Ref.set(degraded, true)))
+        )
+
+  const awaitDrained = (hub: FlowEventHub): Effect.Effect<void> =>
+    Effect.gen(function* () {
+      const target = yield* hub.publishedCount
+      const drain: Effect.Effect<void> = Effect.suspend(() =>
+        Ref.get(consumed).pipe(
+          Effect.flatMap((count) =>
+            count >= target ? Effect.void : Effect.yieldNow.pipe(Effect.andThen(drain))
+          )
+        )
+      )
+      yield* drain
+    })
+
+  return {
+    record,
+    awaitDrained,
+    consume: (hub) =>
+      Effect.gen(function* () {
+        yield* Ref.set(consumed, yield* hub.publishedCount)
+        const subscription = yield* hub.subscribe
+        yield* Stream.fromSubscription(subscription).pipe(
+          Stream.runForEach((event) =>
+            record(event).pipe(Effect.andThen(Ref.update(consumed, (count) => count + 1)))
+          ),
+          Effect.forkScoped
+        )
+        // Registered after the consumer fiber: drain before scope interrupts it.
+        yield* Effect.addFinalizer(() => awaitDrained(hub))
+      })
+  }
+})

package/src/JudgmentTypes.ts

@@ -0,0 +1,15 @@
+import * as Schema from "effect/Schema"
+
+/**
+ * The two literals shared by the judgment policy and the flow events. They
+ * live apart from `Judgment.ts` so `FlowEvents.ts` can import them without
+ * a module cycle (the policy module publishes events).
+ */
+
+/** Consumers observe by default; automation requires an explicit act mode. */
+export const JudgmentMode = Schema.Literals(["observe", "advise", "act"])
+export type JudgmentMode = typeof JudgmentMode.Type
+
+/** The three bands Jev documents: act, proceed with caution, or hold. */
+export const Decision = Schema.Literals(["act", "caution", "hold"])
+export type Decision = typeof Decision.Type