@agentsfleet/orly 0.12.1 → 0.14.0

package/src/judgment_comparison_jev_consumer.test.ts

@@ -0,0 +1,144 @@
+import { expect, test } from "bun:test";
+import { mkdirSync, mkdtempSync, realpathSync, rmSync, symlinkSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { dirname, join, resolve } from "node:path";
+
+import { CONSUMER_SCENARIOS, CONSUMER_SOURCE, CONSUMER_TEST, OWNER_FILES, consumerProbe, mutateConsumer } from "../evals/judgments/comparison/consumer-checks";
+import { consumerJevExamples, consumerReceiptSchema, CONSUMER_JEV_ERROR } from "../evals/judgments/comparison/jev-consumer";
+import { MAX_JEV_REPORT_BYTES } from "../evals/judgments/comparison/jev-report";
+import { CANDIDATE, CLASSIFICATION } from "../evals/judgments/comparison/types";
+import { validateCatalog } from "../evals/judgments/comparison/validate";
+import { MAX_SOURCE_BYTES } from "./judgments/constants";
+import { digest, readBounded } from "./judgments/files";
+
+const ROOT = resolve(import.meta.dir, "..");
+const FIXTURE_ROOT = join(ROOT, "evals/judgments/comparison/fixtures");
+const RECEIPT = join(ROOT, "evals/judgments/comparison/receipts/consumer-final.json");
+const CLI = join(ROOT, "evals/judgments/comparison/jev-run.ts");
+const PRIVATE_OWNER = "Owner bytes preserved in this isolated test.";
+const TIMEOUT = 30_000;
+
+class ConsumerFixture {
+  readonly parent = realpathSync(mkdtempSync(join(tmpdir(), "orly-jev-consumer-")));
+  readonly root = join(this.parent, "consumer");
+  readonly receipt = join(this.parent, "consumer.json");
+
+  async setup() {
+    const source = await readBounded(join(FIXTURE_ROOT, "consumer-source.txt"), MAX_SOURCE_BYTES);
+    const tests = await readBounded(join(FIXTURE_ROOT, "consumer-tests.txt"), MAX_SOURCE_BYTES);
+    for (const file of [CONSUMER_SOURCE, ...OWNER_FILES]) {
+      const path = join(this.root, file);
+      mkdirSync(dirname(path), { recursive: true });
+      await Bun.write(path, file === CONSUMER_SOURCE ? source : file === CONSUMER_TEST ? tests : PRIVATE_OWNER);
+    }
+    const report = consumerReceiptSchema.parse(JSON.parse(await readBounded(RECEIPT, MAX_SOURCE_BYTES)));
+    report.metadata.owner_files = Object.fromEntries(OWNER_FILES.map((file) => [file, digest(file === CONSUMER_TEST ? tests : PRIVATE_OWNER)]));
+    for (const scenario of CONSUMER_SCENARIOS) for (const suffix of ["submitted", "hidden", "repaired"]) {
+      const text = suffix === "repaired" ? source : mutateConsumer(source, scenario);
+      const identity = digest(JSON.stringify([{ path: "consumer-source.txt", digest: digest(text) },
+        { path: "probe.ts", digest: digest(consumerProbe(this.root, suffix !== "submitted")) }]));
+      const entry = report.attempts.find((item) => item.label === `${scenario}:${suffix}`);
+      if (!entry) throw new Error("Missing recorded example.");
+      entry.receipt.source_before = identity;
+      entry.receipt.source_after = identity;
+    }
+    await Bun.write(this.receipt, JSON.stringify(report));
+    return report;
+  }
+
+  async examples() {
+    const catalog = validateCatalog(await Bun.file(join(ROOT, "evals/judgments/comparison/catalog.json")).json());
+    return consumerJevExamples(this.root, this.receipt, catalog);
+  }
+
+  [Symbol.dispose]() { rmSync(this.parent, { recursive: true, force: true }); }
+}
+
+test("jev_inputs_exclude_expected_labels_and_validate_consumer_receipts", async () => {
+  using fixture = new ConsumerFixture();
+  await fixture.setup();
+  const examples = await fixture.examples();
+  expect(examples).toHaveLength(12);
+  expect(examples.find((entry) => entry.id === "consumer-context-known-gap")).toMatchObject({ question: CANDIDATE.evidence, expected: CLASSIFICATION.defective });
+  expect(examples.find((entry) => entry.id === "consumer-context-unknown-dependencies")).toMatchObject({ question: CANDIDATE.evidence, expected: CLASSIFICATION.insufficient });
+  for (const entry of examples) {
+    const state = JSON.parse(entry.input.request).state;
+    expect(Object.keys(state).sort()).toEqual(["evidence", "requirement"]);
+    expect(entry.input.request).not.toContain(entry.id);
+  }
+});
+
+test("jev_consumer_refuses_each_tampered_probe_missing_outcome_and_duplicate", async () => {
+  using fixture = new ConsumerFixture();
+  const original = await fixture.setup();
+  for (const scenario of CONSUMER_SCENARIOS) for (const suffix of ["submitted", "hidden", "repaired"]) {
+    const changed = structuredClone(original);
+    const entry = changed.attempts.find((item: { label: string }) => item.label === `${scenario}:${suffix}`);
+    if (!entry) throw new Error("Missing attempt.");
+    entry.receipt.source_after = digest("unrelated source");
+    await Bun.write(fixture.receipt, JSON.stringify(changed));
+    await expect(fixture.examples()).rejects.toThrow(CONSUMER_JEV_ERROR);
+  }
+  for (const mutate of [
+    (report: typeof original) => { const entry = report.attempts.find((item) => item.label.endsWith(":submitted")); if (!entry) throw new Error("Missing attempt."); entry.label = "unrelated-attempt"; },
+    (report: typeof original) => { const entry = report.attempts.find((item) => item.label.endsWith(":hidden")); if (!entry) throw new Error("Missing attempt."); entry.receipt.result.exit_code = 0; },
+    (report: typeof original) => { const entry = report.attempts.find((item) => item.label.endsWith(":repaired")); if (!entry) throw new Error("Missing attempt."); entry.receipt.result.failure = "injected failure"; },
+    (report: typeof original) => { const entry = report.attempts[1]; if (!entry) throw new Error("Missing attempt."); report.attempts[0] = entry; },
+    (report: typeof original) => { const entry = report.attempts.find((item) => item.label.endsWith(":hidden")); if (!entry) throw new Error("Missing attempt."); entry.receipt.stdout = "[]"; },
+    (report: typeof original) => { report.metadata.restored_source = digest("not restored"); },
+    (report: typeof original) => { const owner = OWNER_FILES[0]; if (!owner) throw new Error("Missing owner."); report.metadata.owner_files[owner] = digest("changed owner"); },
+  ]) {
+    const changed = structuredClone(original);
+    mutate(changed);
+    await Bun.write(fixture.receipt, JSON.stringify(changed));
+    await expect(fixture.examples()).rejects.toThrow(CONSUMER_JEV_ERROR);
+  }
+});
+
+test("jev_cli_without_credentials_retains_every_scheduled_attempt_and_preserves_output", async () => {
+  using fixture = new ConsumerFixture();
+  await fixture.setup();
+  const output = join(fixture.parent, "results");
+  const invoke = (destination: string) => Bun.spawnSync([process.execPath, CLI, "--live", fixture.root, fixture.receipt, destination], {
+    env: { PATH: Bun.env.PATH ?? "", TYPESAFE_API_KEY: "" }, stdout: "pipe", stderr: "pipe", timeout: TIMEOUT,
+  });
+  const first = invoke(output);
+  if (first.exitCode !== 0) throw new Error(first.stderr.toString());
+  expect(first.exitCode).toBe(0);
+  const file = Bun.file(join(output, "report.json"));
+  const bytes = await file.text();
+  const report = JSON.parse(bytes);
+  expect(report.completed).toBe(true);
+  expect(report.requests).toBe(0);
+  expect(report.attempts).toHaveLength(188);
+  expect(report.results.every((entry: { status: string }) => entry.status === "unavailable")).toBe(true);
+  expect(invoke(output).exitCode).not.toBe(0);
+  expect(await file.text()).toBe(bytes);
+  const inside = invoke(join(fixture.root, "results"));
+  expect(inside.exitCode).toBe(2);
+  expect(inside.stderr.toString()).toContain("outside the authoring and consumer worktrees");
+  expect(await Bun.file(join(fixture.root, "results/report.json")).exists()).toBe(false);
+}, TIMEOUT);
+
+test("jev_cli_refuses_symlink_outputs_authoring_outputs_and_oversized_replay", async () => {
+  using fixture = new ConsumerFixture();
+  await fixture.setup();
+  const alias = join(fixture.parent, "consumer-alias");
+  symlinkSync(fixture.root, alias);
+  const oversized = join(fixture.parent, "oversized.json");
+  await Bun.write(oversized, " ".repeat(MAX_JEV_REPORT_BYTES + 1));
+  for (const [destination, mode, saved] of [
+    [join(alias, "results"), "--live", []],
+    [join(ROOT, "forbidden-jev-test-output"), "--live", []],
+    [alias, "--live", []],
+    [join(fixture.parent, "replay"), "--replay", [oversized]],
+  ] as const) {
+    const result = Bun.spawnSync([process.execPath, CLI, mode, fixture.root, fixture.receipt, destination, ...saved], {
+      env: { PATH: Bun.env.PATH ?? "", TYPESAFE_API_KEY: "" }, stdout: "pipe", stderr: "pipe", timeout: TIMEOUT,
+    });
+    expect(result.exitCode).toBe(2);
+    expect(result.stdout.toString()).toBe("");
+  }
+  expect(await Bun.file(join(fixture.root, "results/report.json")).exists()).toBe(false);
+  expect(await Bun.file(join(ROOT, "forbidden-jev-test-output/report.json")).exists()).toBe(false);
+}, TIMEOUT);

package/src/judgment_comparison_jev_report.test.ts

@@ -0,0 +1,167 @@
+import { expect, test } from "bun:test";
+import { resolve } from "node:path";
+
+import { loadCases } from "../evals/judgments/comparison/corpus";
+import { corpusJevExamples, gradeJev, jevInput, JEV_PREFLIGHT_ERROR, type JevAttempt, type JevExample } from "../evals/judgments/comparison/jev";
+import { JEV_MODE, jevReport, renderJevReport, replayJev } from "../evals/judgments/comparison/jev-report";
+import { CANDIDATE, CASE_CLASS, CLASSIFICATION, labelsSchema, MEASUREMENT, NATIVE, OBSERVATION } from "../evals/judgments/comparison/types";
+import { validateCatalog } from "../evals/judgments/comparison/validate";
+import { ANSWER_KIND, MODEL, REPLY_KEY } from "./judgments/constants";
+import { digest } from "./judgments/files";
+import type { Reply } from "./judgments/wire";
+
+const ROOT = resolve(import.meta.dir, "..");
+const MISSING = "Request failed before a usable reply.";
+
+async function inputs() {
+  const catalog = validateCatalog(await Bun.file(resolve(ROOT, "evals/judgments/comparison/catalog.json")).json());
+  const cases = await loadCases(ROOT);
+  const labels = labelsSchema.parse(await Bun.file(resolve(ROOT, "evals/judgments/comparison/labels.json")).json());
+  return { catalog, cases, labels, examples: corpusJevExamples(catalog, cases, labels) };
+}
+
+function attempt(entry: JevExample, reply: Reply | null = null): JevAttempt {
+  return { id: entry.id, question: entry.question, identity: entry.identity, request_digest: digest(entry.input.request),
+    requests: reply === null ? 0 : 1, elapsed_ms: 0, reply, failure: reply === null ? MISSING : null };
+}
+
+function reply(entry: JevExample, decision: string, strength = 1): Reply {
+  const question = entry.input.definition.question;
+  const keys = Object.keys(question.criteria);
+  return { model: MODEL, answers: { [REPLY_KEY]: question.type === ANSWER_KIND.noul ? { type: question.type, noul: strength } : {
+    type: question.type, choice: decision, confidence: strength,
+    probabilities: Object.fromEntries(keys.map((key) => [key, key === decision ? strength : (1 - strength) / (keys.length - 1)])),
+  } }, usage: { input_tokens: 100, output_tokens: 20 } };
+}
+
+test("jev_inputs_exclude_expected_labels_and_freeze_native_questions", async () => {
+  const { catalog, cases, labels, examples } = await inputs();
+  for (const entry of examples) {
+    const request = JSON.parse(entry.input.request);
+    expect(request.model).toBe(MODEL);
+    expect(Object.keys(request.questions)).toEqual([REPLY_KEY]);
+    expect(request.questions[REPLY_KEY]).toEqual(entry.input.definition.question);
+    expect(Object.keys(request.state).sort()).toEqual(["evidence", "requirement"]);
+    expect(request.state.evidence.every((item: Record<string, unknown>) => JSON.stringify(Object.keys(item).sort()) === JSON.stringify(["role", "text"]))).toBe(true);
+  }
+  const first = cases[0];
+  const original = examples[0];
+  if (!first || !original) throw new Error("Missing cases.");
+  const changed = corpusJevExamples(catalog, [{ ...first, classification: CASE_CLASS.defective, author: "different-author", id: first.id }], labels)[0];
+  expect(changed?.input.request).toBe(original.input.request);
+  expect(changed?.identity).not.toBe(original.identity);
+  expect(() => jevInput(original.input.definition.question, { ...JSON.parse(original.input.request).state, expected: NATIVE.yes })).toThrow();
+});
+
+test("jev_report_grades_actual_answers_without_model_accuracy_substitution", async () => {
+  const { cases, labels, examples } = await inputs();
+  const attempts = examples.map((entry) => attempt(entry));
+  const yesIndex = examples.findIndex((entry) => entry.expected === NATIVE.yes);
+  const noIndex = examples.findIndex((entry) => entry.expected === NATIVE.no);
+  const candidateIndex = examples.findIndex((entry) => entry.question === CANDIDATE.wiring && entry.expected === CLASSIFICATION.defective);
+  const uncertainIndex = examples.findIndex((entry) => entry.question === CANDIDATE.wiring && entry.expected === CLASSIFICATION.supported);
+  for (const [index, choice, strength] of [[yesIndex, NATIVE.yes, 1], [noIndex, NATIVE.yes, 1],
+    [candidateIndex, CLASSIFICATION.supported, 1], [uncertainIndex, CLASSIFICATION.supported, 0.4]] as const) {
+    const entry = examples[index];
+    if (!entry) throw new Error("Missing selected case.");
+    attempts[index] = attempt(entry, reply(entry, choice, strength));
+  }
+  const report = jevReport(examples, attempts, cases, labels, [], JEV_MODE.live, true);
+  expect(report.results[yesIndex]?.status).toBe("pass");
+  expect(report.results[noIndex]?.status).toBe("fail");
+  expect(report.results[candidateIndex]?.actual).toBe(CLASSIFICATION.supported);
+  expect(report.results[candidateIndex]?.status).toBe("fail");
+  expect(report.results[uncertainIndex]).toMatchObject({ actual: CLASSIFICATION.supported, graded: NATIVE.abstain, uncertain: true, status: "fail" });
+  expect(report.observations[uncertainIndex]).toMatchObject({ status: OBSERVATION.valid, answer: { decision: CLASSIFICATION.supported, strength: 0.4 } });
+  expect(report.observations.every((entry) => entry.measurement === MEASUREMENT.live)).toBe(true);
+  expect(report.adoption.every((entry) => !entry.eligible && entry.decision === "retain")).toBe(true);
+  expect(report.requests).toBe(4);
+  expect(report.usage).toEqual({ input_tokens: 400, output_tokens: 80 });
+  expect(report.scores).toHaveLength(22);
+  expect(report.results.filter((entry) => entry.status === "unavailable")).toHaveLength(examples.length - 4);
+  const rendered = renderJevReport(report);
+  for (const glyph of ["✅", "❌", "⚪"]) expect(rendered).toContain(glyph);
+  expect(rendered).toContain("(uncertain)");
+  expect(rendered).toContain("not measured autonomous agent improvement");
+});
+
+test("jev_replay_preserves_original_answers_without_new_request_credit", async () => {
+  const { cases, labels, examples } = await inputs();
+  const attempts = examples.map((entry) => attempt(entry));
+  const first = examples[0];
+  if (!first) throw new Error("Missing case.");
+  attempts[0] = attempt(first, reply(first, NATIVE.yes));
+  const live = jevReport(examples, attempts, cases, labels, [], JEV_MODE.live, true);
+  const replay = jevReport(examples, replayJev(examples, live, []), cases, labels, [], JEV_MODE.replay, true);
+  expect(replay.results).toEqual(live.results);
+  expect(replay.attempts).toEqual(live.attempts);
+  expect(replay.requests).toBe(0);
+  expect(replay.original_requests).toBe(1);
+  expect(replay.observations.every((entry) => entry.measurement === MEASUREMENT.editable)).toBe(true);
+  expect(replay.adoption.every((entry) => !entry.eligible)).toBe(true);
+});
+
+test("jev_replay_rejects_changed_input_order_identity_reply_and_request_count", async () => {
+  const { cases, labels, examples } = await inputs();
+  const attempts = examples.map((entry) => attempt(entry));
+  const first = examples[0];
+  if (!first) throw new Error("Missing case.");
+  attempts[0] = attempt(first, reply(first, NATIVE.yes));
+  const report = jevReport(examples, attempts, cases, labels, [], JEV_MODE.live, true);
+  for (const field of ["id", "question", "identity", "request_digest"] as const) {
+    const changed = structuredClone(report);
+    const entry = changed.attempts[0];
+    if (!entry) throw new Error("Missing attempt.");
+    entry[field] = "altered";
+    expect(() => replayJev(examples, changed, [])).toThrow(JEV_PREFLIGHT_ERROR);
+  }
+  expect(() => replayJev([...examples].reverse(), report, [])).toThrow(JEV_PREFLIGHT_ERROR);
+  expect(() => replayJev(examples, { ...report, attempts: attempts.slice(1) }, [])).toThrow(JEV_PREFLIGHT_ERROR);
+  const firstAttempt = attempts[0];
+  if (!firstAttempt) throw new Error("Missing attempt.");
+  expect(() => gradeJev(first, { ...firstAttempt, requests: 0 })).toThrow(JEV_PREFLIGHT_ERROR);
+});
+
+test("jev_replay_rejects_wrong_native_types_and_invalid_probability_distributions", async () => {
+  const { cases, labels, examples } = await inputs();
+  const index = examples.findIndex((entry) => entry.question === CANDIDATE.wiring);
+  const entry = examples[index];
+  if (!entry) throw new Error("Missing Choice example.");
+  const attempts = examples.map((value) => attempt(value));
+  for (const answer of [
+    { type: ANSWER_KIND.noul, noul: 1 },
+    { type: ANSWER_KIND.choice, choice: CLASSIFICATION.supported, confidence: 1, probabilities: {} },
+    { type: ANSWER_KIND.choice, choice: CLASSIFICATION.supported, confidence: 2, probabilities: {} },
+  ]) {
+    const report = jevReport(examples, attempts, cases, labels, [], JEV_MODE.live, true);
+    const changed = { ...attempt(entry), requests: 1, failure: null,
+      reply: { model: MODEL, answers: { [REPLY_KEY]: answer }, usage: { input_tokens: 1, output_tokens: 1 } } };
+    expect(() => replayJev(examples, { ...report, attempts: report.attempts.map((value, position) => position === index ? changed : value) }, [])).toThrow();
+  }
+});
+
+test("jev_noul_uncertainty_preserves_native_probability_and_abstains", async () => {
+  const { examples } = await inputs();
+  const entry = examples.find((value) => value.input.definition.question.type === ANSWER_KIND.noul);
+  if (!entry) throw new Error("Missing Noul example.");
+  for (const probability of [0.49, 0.5, 0.51]) {
+    const native = reply(entry, NATIVE.yes, probability);
+    const result = gradeJev(entry, attempt(entry, native));
+    expect(result.actual).toBe(probability < 0.5 ? NATIVE.no : NATIVE.yes);
+    expect(result.graded).toBe(NATIVE.abstain);
+    expect(result.uncertain).toBe(true);
+    expect(native.answers[REPLY_KEY]).toEqual({ type: ANSWER_KIND.noul, noul: probability });
+  }
+});
+
+// Saved observations must retain the evaluator that produced them.
+test("jev_replay_rejects_missing_changed_or_reordered_source_inventory", async () => {
+  const { cases, labels, examples } = await inputs();
+  const sources = [{ path: "score.ts", digest: digest("scorer") }, { path: "jev.ts", digest: digest("transport") }];
+  const live = jevReport(examples, examples.map((entry) => attempt(entry)), cases, labels, sources, JEV_MODE.live, true);
+  expect(replayJev(examples, live, sources)).toEqual(live.attempts);
+  for (const changed of [undefined, [], sources.slice(1), [...sources].reverse(), [...sources, sources[0]],
+    sources.map((entry) => ({ ...entry, digest: "0".repeat(64) })), sources.map((entry) => ({ ...entry, path: "other.ts" }))]) {
+    expect(() => replayJev(examples, { ...live, sources: changed }, sources)).toThrow();
+  }
+});

package/src/judgment_labels.test.ts

@@ -0,0 +1,119 @@
+import { afterEach, expect, test } from "bun:test";
+import { join } from "node:path";
+
+import catalogInput from "../evals/judgments/comparison/catalog.json";
+import { validateLabels } from "../evals/judgments/comparison/labels";
+import { validateCatalog } from "../evals/judgments/comparison/validate";
+import { ADJUDICATION, CANDIDATE, CASE_CLASS, CHECK_ERROR, CLASSIFICATION, LABEL_AUTHORITY, MIN_RESOLVED_PER_GROUP, NATIVE, SPLIT,
+  type ComparisonCase, type IndependentLabel, type LabelOracle } from "../evals/judgments/comparison/types";
+import { cleanupTemporaryDirectories, temporaryDirectory } from "./gates_test_support";
+import { CHOICE, QUESTIONS } from "./judgments/constants";
+import { digest } from "./judgments/files";
+
+const FIXTURE_AUTHOR = "label-fixture-author";
+const ORACLE_NAME = "independent-test-boundary";
+const SOURCE_PATH = "reference.md";
+const SOURCE_CONTENT = "An independently sourced expected value is compared with the selected result.\n";
+const REVISION_LENGTH = 40;
+const REVISION = "1".repeat(REVISION_LENGTH);
+const catalog = validateCatalog(catalogInput);
+
+afterEach(cleanupTemporaryDirectories);
+
+function nativeAnswer(question: string, classification: ComparisonCase["classification"]): IndependentLabel["expected_native"] {
+  if (question === QUESTIONS[2]) return classification === CASE_CLASS.healthy ? CHOICE.exact : classification === CASE_CLASS.defective ? CHOICE.weak
+    : classification === CASE_CLASS.ambiguous ? NATIVE.abstain : CHOICE.insufficient;
+  if (QUESTIONS.some((item) => item === question)) return classification === CASE_CLASS.healthy ? NATIVE.yes
+    : classification === CASE_CLASS.defective ? NATIVE.no : NATIVE.abstain;
+  return classification === CASE_CLASS.healthy ? CLASSIFICATION.supported : classification === CASE_CLASS.defective ? CLASSIFICATION.defective
+    : classification === CASE_CLASS.ambiguous ? CLASSIFICATION.ambiguous : CLASSIFICATION.insufficient;
+}
+
+async function fixture() {
+  const root = temporaryDirectory();
+  await Bun.write(join(root, SOURCE_PATH), SOURCE_CONTENT);
+  const cases: ComparisonCase[] = [];
+  const labels: IndependentLabel[] = [];
+  for (const question of [...QUESTIONS, ...Object.values(CANDIDATE)]) for (const classification of Object.values(CASE_CLASS)) {
+    for (const split of Object.values(SPLIT)) for (let index = 0; index < MIN_RESOLVED_PER_GROUP; index++) {
+      const id = `${question}-${classification}-${split}-${index}`;
+      cases.push({ id, question, classification, split, author: FIXTURE_AUTHOR, origin: `${question}-${split}`,
+        requirement: `Independent unit boundary ${id}`, evidence: [] });
+      labels.push({ case_id: id, classification, expected_native: nativeAnswer(question, classification), rationale: SOURCE_CONTENT.trim(),
+        author: FIXTURE_AUTHOR, adjudication: ADJUDICATION.resolved, authority: { kind: LABEL_AUTHORITY.oracle, name: ORACLE_NAME },
+        source: { path: SOURCE_PATH, digest: digest(SOURCE_CONTENT), revision: REVISION, selected: SOURCE_CONTENT.trim() } });
+    }
+  }
+  const oracle: LabelOracle = async (entry) => ({ classification: entry.classification, expected_native: nativeAnswer(entry.question, entry.classification),
+    source_digest: digest(SOURCE_CONTENT), revision: REVISION });
+  return { root, cases, labels, oracle };
+}
+
+test("labels_require_independent_evidence", async () => {
+  const { root, cases, labels, oracle } = await fixture();
+  expect(await validateLabels(root, catalog, cases, labels, oracle)).toEqual(labels);
+  const selfReviewed = labels.map((label) => ({ ...label, authority: { kind: LABEL_AUTHORITY.reviewer, name: FIXTURE_AUTHOR } }));
+  await expect(validateLabels(root, catalog, cases, selfReviewed, oracle)).rejects.toThrow(CHECK_ERROR.independence);
+});
+
+test("all_pending_and_all_disputed_labels_cannot_satisfy_minimums", async () => {
+  const { root, cases, labels, oracle } = await fixture();
+  for (const adjudication of [ADJUDICATION.pending, ADJUDICATION.disputed]) {
+    await expect(validateLabels(root, catalog, cases, labels.map((label) => ({ ...label, adjudication })), oracle)).rejects.toThrow(CHECK_ERROR.minimum);
+  }
+});
+
+test("a_missing_independent_source_and_invented_selected_text_refuse_labels", async () => {
+  const { root, cases, labels, oracle } = await fixture();
+  const absent = labels.map((label) => ({ ...label, source: { ...label.source, path: "absent-reference.md" } }));
+  await expect(validateLabels(root, catalog, cases, absent, oracle)).rejects.toThrow();
+  const fabricated = labels.map((label) => ({ ...label, source: { ...label.source, selected: "Invented label approval." } }));
+  await expect(validateLabels(root, catalog, cases, fabricated, oracle)).rejects.toThrow(CHECK_ERROR.independence);
+});
+
+test("an_independence_flag_cannot_replace_external_oracle_proof", async () => {
+  const { root, cases, labels, oracle } = await fixture();
+  await expect(validateLabels(root, catalog, cases, labels.map((label) => ({ ...label, independent: true })), oracle)).rejects.toThrow(CHECK_ERROR.labels);
+  const wrongOracle: LabelOracle = async (entry, label) => ({ ...(await oracle(entry, label)), expected_native: CLASSIFICATION.defective });
+  await expect(validateLabels(root, catalog, cases, labels, wrongOracle)).rejects.toThrow(CHECK_ERROR.independence);
+});
+
+test("stale_oracle_revision_and_changed_label_evidence_refuse_labels", async () => {
+  const { root, cases, labels, oracle } = await fixture();
+  const stale: LabelOracle = async (entry, label) => ({ ...(await oracle(entry, label)), revision: "2".repeat(REVISION_LENGTH) });
+  await expect(validateLabels(root, catalog, cases, labels, stale)).rejects.toThrow(CHECK_ERROR.independence);
+  await Bun.write(join(root, SOURCE_PATH), "Changed independent evidence.\n");
+  await expect(validateLabels(root, catalog, cases, labels, oracle)).rejects.toThrow(CHECK_ERROR.independence);
+});
+
+test("missing_duplicate_and_mismatched_labels_refuse_scoring", async () => {
+  const { root, cases, labels, oracle } = await fixture();
+  await expect(validateLabels(root, catalog, cases, labels.slice(1), oracle)).rejects.toThrow(CHECK_ERROR.labelIdentity);
+  await expect(validateLabels(root, catalog, cases, [...labels, labels[0]], oracle)).rejects.toThrow(CHECK_ERROR.labelIdentity);
+  await expect(validateLabels(root, catalog, cases, labels.map((label) => ({ ...label, author: "invented-author" })), oracle)).rejects.toThrow(CHECK_ERROR.labelIdentity);
+});
+
+test("labels_cannot_change_answer_types_or_relabel_case_health", async () => {
+  const { root, cases, labels, oracle } = await fixture();
+  await expect(validateLabels(root, catalog, cases, labels.map((label) => ({ ...label, expected_native: CHOICE.exact })), oracle)).rejects.toThrow(CHECK_ERROR.native);
+  await expect(validateLabels(root, catalog, cases, labels.map((label) => ({ ...label, classification: CASE_CLASS.defective })), oracle)).rejects.toThrow(CHECK_ERROR.labelIdentity);
+});
+
+test("frozen_reference_bytes_validate_without_git_history_and_reject_modified_sources", async () => {
+  const { resolve } = await import("node:path");
+  const { loadCases } = await import("../evals/judgments/comparison/corpus");
+  const { verifyCaseLabel } = await import("../evals/judgments/comparison/hidden/label-oracle");
+  const { labelsSchema } = await import("../evals/judgments/comparison/types");
+  const { default: input } = await import("../evals/judgments/comparison/labels.json");
+  const sourceRoot = resolve(import.meta.dir, "..");
+  const label = labelsSchema.parse(input)[0];
+  if (!label) throw new Error("Missing independent label.");
+  const entry = (await loadCases(sourceRoot)).find((entry) => entry.id === label.case_id);
+  if (!entry) throw new Error("Missing sourced case.");
+  const root = temporaryDirectory();
+  const source = join(root, label.source.path);
+  await Bun.write(source, Bun.file(join(sourceRoot, label.source.path)));
+  expect((await verifyCaseLabel(root, entry, label)).source_digest).toBe(label.source.digest);
+  await Bun.write(source, "Changed reference bytes.");
+  await expect(verifyCaseLabel(root, entry, label)).rejects.toThrow(CHECK_ERROR.independence);
+});

package/src/judgment_score.test.ts

@@ -0,0 +1,166 @@
+import { expect, test } from "bun:test";
+
+import { adoptionDecision, missingObservations, scoreQuestion, validateObservations } from "../evals/judgments/comparison/score";
+import { ADJUDICATION, CANDIDATE, CASE_CLASS, CHECK_ERROR, CLASSIFICATION, LABEL_AUTHORITY, MEASUREMENT, NATIVE, OBSERVATION, SPLIT,
+  type ComparisonCase, type IndependentLabel, type Observation } from "../evals/judgments/comparison/types";
+import { CHOICE, MODEL, QUESTIONS } from "./judgments/constants";
+import { OrlyError } from "./model";
+
+const CERTAIN = 1;
+const UNCERTAIN = 0.5;
+const TEST_AUTHOR = "score-fixture-author";
+const TEST_ORACLE = "unit-boundary-oracle";
+const DIGEST_LENGTH = 64;
+const REVISION_LENGTH = 40;
+const REFERENCE_SOURCE = "reference.md";
+
+function fixture(question: ComparisonCase["question"], classifications: ComparisonCase["classification"][], expected: IndependentLabel["expected_native"][]) {
+  const cases: ComparisonCase[] = classifications.map((classification, index) => ({ id: `case-${index}`, question, classification,
+    origin: `family-${index}`, split: SPLIT.heldout, author: TEST_AUTHOR, requirement: "Assert the sourced result.", evidence: [] }));
+  const labels: IndependentLabel[] = cases.map((entry, index) => {
+    const answer = expected[index];
+    if (!answer) throw new Error("Missing unit expectation.");
+    return { case_id: entry.id, classification: entry.classification, expected_native: answer, rationale: "Exact unit boundary expectation.",
+      author: TEST_AUTHOR, adjudication: ADJUDICATION.resolved, authority: { kind: LABEL_AUTHORITY.oracle, name: TEST_ORACLE },
+      source: { path: REFERENCE_SOURCE, digest: "0".repeat(DIGEST_LENGTH), revision: "0".repeat(REVISION_LENGTH), selected: "Expected result." } };
+  });
+  const observations: Observation[] = missingObservations(cases).map((entry, index) => {
+    const decision = expected[index];
+    if (!decision) throw new Error("Missing unit answer.");
+    return { ...entry, status: OBSERVATION.valid, answer: { decision, strength: CERTAIN } };
+  });
+  return { cases, labels, observations };
+}
+
+test("metrics_preserve_missing_outcomes", () => {
+  const { cases, labels, observations } = fixture(QUESTIONS[0], [CASE_CLASS.healthy, CASE_CLASS.defective, CASE_CLASS.ambiguous, CASE_CLASS.insufficient],
+    [NATIVE.yes, NATIVE.no, NATIVE.abstain, NATIVE.abstain]);
+  const mixed = observations.map((item, index) => index === 0 ? { ...item, status: OBSERVATION.unavailable, answer: null } : item);
+  const result = scoreQuestion(QUESTIONS[0], cases, labels, validateObservations(cases, mixed));
+  expect(result.native_correctness).toEqual({ numerator: 3, denominator: 4, value: 0.75 });
+  expect(result.defect_recall).toEqual({ numerator: 1, denominator: 1, value: 1 });
+  expect(result.false_concerns_bound).toEqual({ numerator: 1, denominator: 1, value: 1 });
+  expect(result.insufficient_detection).toBeNull();
+  expect(result.shared_abstention).toEqual({ numerator: 2, denominator: 2, value: 1 });
+  expect(() => validateObservations(cases, mixed.slice(1))).toThrow(new OrlyError(CHECK_ERROR.observations));
+});
+
+test("healthy_applicable_yes_is_native_correctness_without_defect_metrics", () => {
+  const { cases, labels, observations } = fixture(QUESTIONS[4], [CASE_CLASS.healthy], [NATIVE.yes]);
+  const result = scoreQuestion(QUESTIONS[4], cases, labels, observations);
+  expect(result.native_correctness?.value).toBe(1);
+  expect(result.false_concerns).toBeNull();
+  expect(result.defect_recall).toBeNull();
+});
+
+test("noul_half_probability_abstains_without_inventing_insufficiency", () => {
+  const { cases, labels, observations } = fixture(QUESTIONS[0], [CASE_CLASS.insufficient], [NATIVE.abstain]);
+  const answer = observations.map((item) => ({ ...item, answer: { decision: NATIVE.yes, strength: UNCERTAIN } }));
+  const result = scoreQuestion(QUESTIONS[0], cases, labels, answer);
+  expect(result.shared_abstention?.value).toBe(1);
+  expect(result.insufficient_detection).toBeNull();
+});
+
+test("zero_responses_and_empty_denominators_are_unavailable", () => {
+  const { cases, labels } = fixture(CANDIDATE.obligation, [CASE_CLASS.healthy], [CLASSIFICATION.supported]);
+  const result = scoreQuestion(CANDIDATE.obligation, cases, labels, missingObservations(cases));
+  expect(result.attempted).toBe(1);
+  expect(result.unavailable).toBe(1);
+  expect(result.native_correctness).toBeNull();
+  expect(result.defect_recall).toBeNull();
+  expect(result.false_concerns_bound).toBeNull();
+  expect(scoreQuestion(CANDIDATE.wiring, [], [], []).native_correctness).toBeNull();
+});
+
+test("invalid_defective_answers_are_misses_and_invalid_healthy_answers_raise_the_bound", () => {
+  const { cases, labels, observations } = fixture(CANDIDATE.failure, [CASE_CLASS.healthy, CASE_CLASS.defective], [CLASSIFICATION.supported, CLASSIFICATION.defective]);
+  const answer = observations.map((item, index) => index === 0 ? item : { ...item, status: OBSERVATION.invalid, answer: null });
+  const result = scoreQuestion(CANDIDATE.failure, cases, labels, answer);
+  expect(result.native_correctness?.value).toBe(0.5);
+  expect(result.defect_recall?.value).toBe(0);
+  expect(result.invalid).toBe(1);
+});
+
+test("weak_low_confidence_assertion_is_an_abstention", () => {
+  const { cases, labels, observations } = fixture(QUESTIONS[2], [CASE_CLASS.defective], [CHOICE.weak]);
+  const answer = observations.map((item) => ({ ...item, answer: { decision: CHOICE.weak, strength: UNCERTAIN } }));
+  const result = scoreQuestion(QUESTIONS[2], cases, labels, answer);
+  expect(result.native_correctness?.value).toBe(0);
+  expect(result.defect_recall?.value).toBe(0);
+});
+
+test("explicit_choice_insufficiency_is_scored_separately", () => {
+  const { cases, labels, observations } = fixture(QUESTIONS[2], [CASE_CLASS.insufficient], [CHOICE.insufficient]);
+  expect(scoreQuestion(QUESTIONS[2], cases, labels, observations).insufficient_detection?.value).toBe(1);
+});
+
+test("observations_reject_stale_duplicate_unknown_and_executable_answers", () => {
+  const { cases, observations } = fixture(CANDIDATE.wiring, [CASE_CLASS.healthy, CASE_CLASS.defective], [CLASSIFICATION.supported, CLASSIFICATION.defective]);
+  expect(() => validateObservations(cases, observations.map((item) => ({ ...item, identity: "0".repeat(DIGEST_LENGTH) })))).toThrow(CHECK_ERROR.observations);
+  expect(() => validateObservations(cases, [observations[0], observations[0]])).toThrow(CHECK_ERROR.observations);
+  expect(() => validateObservations(cases, observations.map((item) => ({ ...item, execute: "true" })))).toThrow(CHECK_ERROR.observations);
+  expect(() => validateObservations(cases.map((item) => ({ ...item, requirement: "Edited required behavior." })), observations)).toThrow(CHECK_ERROR.observations);
+});
+
+test("adoption_requires_independent_measured_benefit", () => {
+  const { cases, labels, observations } = fixture(CANDIDATE.resolution, Object.values(CASE_CLASS), Object.values(CLASSIFICATION));
+  const decision = adoptionDecision(CANDIDATE.resolution, cases, labels, missingObservations(cases), null);
+  expect(decision.eligible).toBe(false);
+  expect(decision.reasons.length).toBe(4);
+  expect(decision.decision).toBe("retain");
+  expect(adoptionDecision(CANDIDATE.resolution, cases, labels, observations, null).eligible).toBe(false);
+});
+
+test("unmeasured_candidates_remain_evaluation_only", () => {
+  for (const question of Object.values(CANDIDATE)) {
+    const { cases, labels } = fixture(question, [CASE_CLASS.healthy], [CLASSIFICATION.supported]);
+    const result = adoptionDecision(question, cases, labels, missingObservations(cases), null);
+    expect(result.eligible).toBe(false);
+    expect(result.metrics.native_correctness).toBeNull();
+  }
+  expect(QUESTIONS.length).toBe(6);
+});
+
+test("development_cases_cannot_fill_the_heldout_adoption_minimum", () => {
+  const { cases, labels, observations } = fixture(CANDIDATE.wiring, [CASE_CLASS.healthy], [CLASSIFICATION.supported]);
+  const result = adoptionDecision(CANDIDATE.wiring, cases.map((entry) => ({ ...entry, split: SPLIT.development })), labels,
+    observations.map((entry) => ({ ...entry, measurement: MEASUREMENT.independent })), { baseline_resolved: 0, candidate_resolved: 1, unsupported_completions: 0, authority_violations: 0 });
+  expect(result.eligible).toBe(false);
+  expect(result.metrics.attempted).toBe(0);
+});
+
+test("answers_from_another_question_format_count_as_invalid_attempts", () => {
+  const { cases, labels, observations } = fixture(CANDIDATE.wiring, [CASE_CLASS.healthy], [CLASSIFICATION.supported]);
+  const malformed = observations.map((entry) => ({ ...entry, answer: { decision: NATIVE.yes, strength: CERTAIN } }));
+  const validated = validateObservations(cases, malformed);
+  expect(validated.map((entry) => entry.status)).toEqual([OBSERVATION.invalid]);
+  const result = scoreQuestion(CANDIDATE.wiring, cases, labels, malformed);
+  expect(result.invalid).toBe(1);
+  expect(result.valid).toBe(0);
+  expect(result.false_concerns_bound?.value).toBe(1);
+});
+
+test("pending_extra_cases_are_unscored_and_do_not_change_denominators", () => {
+  const { cases, labels, observations } = fixture(CANDIDATE.evidence, [CASE_CLASS.healthy, CASE_CLASS.healthy], [CLASSIFICATION.supported, CLASSIFICATION.supported]);
+  const pending = labels.map((entry, index) => index === 0 ? entry : { ...entry, adjudication: ADJUDICATION.pending });
+  const result = scoreQuestion(CANDIDATE.evidence, cases, pending, observations);
+  expect(result.attempted).toBe(1);
+  expect(result.unscored).toBe(1);
+  expect(result.native_correctness).toEqual({ numerator: 1, denominator: 1, value: 1 });
+});
+
+test("fabricated_measurement_enums_and_paired_counts_never_qualify_for_adoption", () => {
+  const minimum = 20;
+  const classes = Object.values(CASE_CLASS).flatMap((classification) => Array.from({ length: minimum }, () => classification));
+  const answers = Object.values(CLASSIFICATION).flatMap((answer) => Array.from({ length: minimum }, () => answer));
+  const { cases, labels, observations } = fixture(CANDIDATE.resolution, classes, answers);
+  const fabricated = observations.map((entry) => ({ ...entry, measurement: MEASUREMENT.independent }));
+  const invalidPair = { baseline_resolved: -1, candidate_resolved: 0, unsupported_completions: 0, authority_violations: 0 };
+  const ordinaryPair = { ...invalidPair, baseline_resolved: 1, candidate_resolved: 2 };
+  for (const pair of [invalidPair, ordinaryPair]) {
+    const result = adoptionDecision(CANDIDATE.resolution, cases, labels, fabricated, pair);
+    expect(result.eligible).toBe(false);
+    expect(result.decision).toBe("retain");
+    expect(result.reasons.some((reason) => reason.includes("provenance"))).toBe(true);
+  }
+});