@agentsfleet/orly 0.12.0 → 0.13.0

package/src/judgment_comparison_jev.integration.test.ts

@@ -0,0 +1,134 @@
+import { expect, test } from "bun:test";
+import { resolve } from "node:path";
+
+import { loadCases } from "../evals/judgments/comparison/corpus";
+import { corpusJevExamples, evaluateJev, JEV_PREFLIGHT_ERROR, MAX_JEV_REQUESTS, type JevAttempt, type JevExample } from "../evals/judgments/comparison/jev";
+import { CANDIDATE, labelsSchema } from "../evals/judgments/comparison/types";
+import { validateCatalog } from "../evals/judgments/comparison/validate";
+import { ANSWER_KIND, MAX_REQUEST_BYTES, MAX_STATE_BYTES, MODEL, REPLY_KEY } from "./judgments/constants";
+import type { Reply } from "./judgments/wire";
+import { OrlyError } from "./model";
+
+const ROOT = resolve(import.meta.dir, "..");
+const CREDENTIAL = "test-credential";
+const PRIVATE_DETAIL = "private-provider-detail";
+const FAILURE = "retention write failed";
+const SCAN = "scan:";
+const REQUEST = "request:";
+
+async function examples(): Promise<JevExample[]> {
+  const catalog = validateCatalog(await Bun.file(resolve(ROOT, "evals/judgments/comparison/catalog.json")).json());
+  const cases = await loadCases(ROOT);
+  const labels = labelsSchema.parse(await Bun.file(resolve(ROOT, "evals/judgments/comparison/labels.json")).json());
+  const all = corpusJevExamples(catalog, cases, labels);
+  return [all.find((entry) => entry.question === "document.claim"), all.find((entry) => entry.question === "verify.assertion"),
+    all.find((entry) => entry.question === CANDIDATE.wiring)].filter((entry) => entry !== undefined);
+}
+
+function response(entry: JevExample): Reply {
+  const question = entry.input.definition.question;
+  const answer = question.type === ANSWER_KIND.noul ? { type: question.type, noul: 1 } : {
+    type: question.type, choice: entry.expected, confidence: 1,
+    probabilities: Object.fromEntries(Object.keys(question.criteria).map((key) => [key, key === entry.expected ? 1 : 0])),
+  };
+  return { model: MODEL, answers: { [REPLY_KEY]: answer }, usage: { input_tokens: 100, output_tokens: 20 } };
+}
+
+test("jev_requests_preserve_actual_native_answers_and_failures", async () => {
+  const selected = await examples();
+  const order: string[] = [];
+  const retained: JevAttempt[] = [];
+  let ordinal = 0;
+  using server = Bun.serve({ port: 0, async fetch(request) {
+    const entry = selected[ordinal++];
+    if (!entry) throw new Error("Unexpected request.");
+    order.push(REQUEST + entry.id);
+    expect(await request.text()).toBe(entry.input.request);
+    expect(request.headers.get("authorization")).toBe(`Bearer ${CREDENTIAL}`);
+    expect(retained.at(-1)).toMatchObject({ id: entry.id, requests: 1, reply: null });
+    return ordinal === 2 ? new Response(PRIVATE_DETAIL, { status: 429 }) : Response.json(response(entry));
+  } });
+  const actual = await evaluateJev(selected, CREDENTIAL, {
+    scan: async (input) => { order.push(SCAN + selected.find((entry) => entry.input.request === input)?.id); },
+    fetch: (request) => fetch(new Request(server.url, request)),
+  }, async (attempt) => { retained.push(attempt); });
+  expect(ordinal).toBe(3);
+  expect(order.slice(0, 3)).toEqual(selected.map((entry) => SCAN + entry.id));
+  expect(actual.map((entry) => entry.requests)).toEqual([1, 1, 1]);
+  expect(actual.map((entry) => entry.reply !== null)).toEqual([true, false, true]);
+  expect(actual[1]?.failure).toContain("HTTP 429; no automatic retry");
+  expect(JSON.stringify(actual)).not.toContain(PRIVATE_DETAIL);
+  expect(JSON.stringify(actual)).not.toContain(CREDENTIAL);
+  expect(retained).toHaveLength(6);
+  const first = selected[0];
+  if (!first) throw new Error("Missing input.");
+  expect(actual[0]?.reply).toEqual(response(first));
+});
+
+test("jev_missing_credentials_and_each_scanner_failure_prevent_every_upload", async () => {
+  const selected = await examples();
+  for (const failAt of [0, 1, 2, 3]) {
+    let scans = 0;
+    let uploads = 0;
+    const attempts = await evaluateJev(selected, failAt === 0 ? undefined : CREDENTIAL, {
+      scan: async () => { if (++scans === failAt) throw new OrlyError("Secret scanner failed; nothing was uploaded."); },
+      fetch: async () => { uploads++; throw new Error(PRIVATE_DETAIL); },
+    });
+    expect(scans).toBe(failAt);
+    expect(uploads).toBe(0);
+    expect(attempts).toHaveLength(3);
+    expect(attempts.every((entry) => entry.requests === 0 && entry.reply === null && entry.failure !== null)).toBe(true);
+  }
+});
+
+test("jev_invalid_schedule_and_forged_upload_refuse_before_scanning", async () => {
+  const selected = await examples();
+  const entry = selected[0];
+  if (!entry) throw new Error("Missing input.");
+  let touched = 0;
+  const dependencies = { scan: async () => { touched++; }, fetch: async () => { touched++; return Response.json({}); } };
+  const badInputs = ["{", "x".repeat(MAX_REQUEST_BYTES + 1), JSON.stringify({ ...JSON.parse(entry.input.request), model: "unpinned" }),
+    JSON.stringify({ ...JSON.parse(entry.input.request), state: { requirement: "bounded", evidence: [{ role: "spec", text: "x".repeat(MAX_STATE_BYTES + 1) }] } })];
+  const invalid = [[], [entry, entry], Array.from({ length: MAX_JEV_REQUESTS + 1 }, (_, id) => ({ ...entry, id: String(id) })),
+    ...badInputs.map((request) => [{ ...entry, input: { ...entry.input, request } }])];
+  for (const schedule of invalid) await expect(evaluateJev(schedule, CREDENTIAL, dependencies)).rejects.toThrow(JEV_PREFLIGHT_ERROR);
+  expect(touched).toBe(0);
+});
+
+test("jev_malformed_native_reply_and_transport_failure_remain_missing_and_continue", async () => {
+  const selected = await examples();
+  let ordinal = 0;
+  const attempts = await evaluateJev(selected, CREDENTIAL, { scan: async () => {}, fetch: async () => {
+    ordinal++;
+    if (ordinal === 1) return Response.json({ model: "wrong-model", answers: {}, usage: {} });
+    if (ordinal === 2) throw new Error(CREDENTIAL + PRIVATE_DETAIL);
+    return new Response("{");
+  } });
+  expect(ordinal).toBe(3);
+  expect(attempts.every((entry) => entry.reply === null && entry.requests === 1 && entry.failure !== null)).toBe(true);
+  expect(attempts[0]?.failure).toContain("pinned model and answer schema");
+  expect(attempts[1]?.failure).toContain("Provider transport failed");
+  expect(attempts[2]?.failure).toContain("not valid JSON");
+  expect(JSON.stringify(attempts)).not.toContain(CREDENTIAL);
+  expect(JSON.stringify(attempts)).not.toContain(PRIVATE_DETAIL);
+});
+
+test("jev_retention_failure_at_each_ordered_write_prevents_the_next_upload", async () => {
+  const selected = await examples();
+  for (const failAt of [1, 2, 3, 4, 5, 6]) {
+    let writes = 0;
+    let uploads = 0;
+    const durable: JevAttempt[] = [];
+    await expect(evaluateJev(selected, CREDENTIAL, { scan: async () => {}, fetch: async () => {
+      const entry = selected[uploads++];
+      if (!entry) throw new Error("Unexpected upload.");
+      return Response.json(response(entry));
+    } }, async (attempt) => {
+      if (++writes === failAt) throw new Error(FAILURE);
+      durable.push(attempt);
+    })).rejects.toThrow(FAILURE);
+    expect(uploads).toBe(Math.floor(failAt / 2));
+    expect(durable).toHaveLength(failAt - 1);
+    if (failAt % 2 === 0) expect(durable.at(-1)).toMatchObject({ requests: 1, reply: null });
+  }
+});

package/src/judgment_comparison_jev_consumer.test.ts

@@ -0,0 +1,144 @@
+import { expect, test } from "bun:test";
+import { mkdirSync, mkdtempSync, realpathSync, rmSync, symlinkSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { dirname, join, resolve } from "node:path";
+
+import { CONSUMER_SCENARIOS, CONSUMER_SOURCE, CONSUMER_TEST, OWNER_FILES, consumerProbe, mutateConsumer } from "../evals/judgments/comparison/consumer-checks";
+import { consumerJevExamples, consumerReceiptSchema, CONSUMER_JEV_ERROR } from "../evals/judgments/comparison/jev-consumer";
+import { MAX_JEV_REPORT_BYTES } from "../evals/judgments/comparison/jev-report";
+import { CANDIDATE, CLASSIFICATION } from "../evals/judgments/comparison/types";
+import { validateCatalog } from "../evals/judgments/comparison/validate";
+import { MAX_SOURCE_BYTES } from "./judgments/constants";
+import { digest, readBounded } from "./judgments/files";
+
+const ROOT = resolve(import.meta.dir, "..");
+const FIXTURE_ROOT = join(ROOT, "evals/judgments/comparison/fixtures");
+const RECEIPT = join(ROOT, "evals/judgments/comparison/receipts/consumer-final.json");
+const CLI = join(ROOT, "evals/judgments/comparison/jev-run.ts");
+const PRIVATE_OWNER = "Owner bytes preserved in this isolated test.";
+const TIMEOUT = 30_000;
+
+class ConsumerFixture {
+  readonly parent = realpathSync(mkdtempSync(join(tmpdir(), "orly-jev-consumer-")));
+  readonly root = join(this.parent, "consumer");
+  readonly receipt = join(this.parent, "consumer.json");
+
+  async setup() {
+    const source = await readBounded(join(FIXTURE_ROOT, "consumer-source.txt"), MAX_SOURCE_BYTES);
+    const tests = await readBounded(join(FIXTURE_ROOT, "consumer-tests.txt"), MAX_SOURCE_BYTES);
+    for (const file of [CONSUMER_SOURCE, ...OWNER_FILES]) {
+      const path = join(this.root, file);
+      mkdirSync(dirname(path), { recursive: true });
+      await Bun.write(path, file === CONSUMER_SOURCE ? source : file === CONSUMER_TEST ? tests : PRIVATE_OWNER);
+    }
+    const report = consumerReceiptSchema.parse(JSON.parse(await readBounded(RECEIPT, MAX_SOURCE_BYTES)));
+    report.metadata.owner_files = Object.fromEntries(OWNER_FILES.map((file) => [file, digest(file === CONSUMER_TEST ? tests : PRIVATE_OWNER)]));
+    for (const scenario of CONSUMER_SCENARIOS) for (const suffix of ["submitted", "hidden", "repaired"]) {
+      const text = suffix === "repaired" ? source : mutateConsumer(source, scenario);
+      const identity = digest(JSON.stringify([{ path: "consumer-source.txt", digest: digest(text) },
+        { path: "probe.ts", digest: digest(consumerProbe(this.root, suffix !== "submitted")) }]));
+      const entry = report.attempts.find((item) => item.label === `${scenario}:${suffix}`);
+      if (!entry) throw new Error("Missing recorded example.");
+      entry.receipt.source_before = identity;
+      entry.receipt.source_after = identity;
+    }
+    await Bun.write(this.receipt, JSON.stringify(report));
+    return report;
+  }
+
+  async examples() {
+    const catalog = validateCatalog(await Bun.file(join(ROOT, "evals/judgments/comparison/catalog.json")).json());
+    return consumerJevExamples(this.root, this.receipt, catalog);
+  }
+
+  [Symbol.dispose]() { rmSync(this.parent, { recursive: true, force: true }); }
+}
+
+test("jev_inputs_exclude_expected_labels_and_validate_consumer_receipts", async () => {
+  using fixture = new ConsumerFixture();
+  await fixture.setup();
+  const examples = await fixture.examples();
+  expect(examples).toHaveLength(12);
+  expect(examples.find((entry) => entry.id === "consumer-context-known-gap")).toMatchObject({ question: CANDIDATE.evidence, expected: CLASSIFICATION.defective });
+  expect(examples.find((entry) => entry.id === "consumer-context-unknown-dependencies")).toMatchObject({ question: CANDIDATE.evidence, expected: CLASSIFICATION.insufficient });
+  for (const entry of examples) {
+    const state = JSON.parse(entry.input.request).state;
+    expect(Object.keys(state).sort()).toEqual(["evidence", "requirement"]);
+    expect(entry.input.request).not.toContain(entry.id);
+  }
+});
+
+test("jev_consumer_refuses_each_tampered_probe_missing_outcome_and_duplicate", async () => {
+  using fixture = new ConsumerFixture();
+  const original = await fixture.setup();
+  for (const scenario of CONSUMER_SCENARIOS) for (const suffix of ["submitted", "hidden", "repaired"]) {
+    const changed = structuredClone(original);
+    const entry = changed.attempts.find((item: { label: string }) => item.label === `${scenario}:${suffix}`);
+    if (!entry) throw new Error("Missing attempt.");
+    entry.receipt.source_after = digest("unrelated source");
+    await Bun.write(fixture.receipt, JSON.stringify(changed));
+    await expect(fixture.examples()).rejects.toThrow(CONSUMER_JEV_ERROR);
+  }
+  for (const mutate of [
+    (report: typeof original) => { const entry = report.attempts.find((item) => item.label.endsWith(":submitted")); if (!entry) throw new Error("Missing attempt."); entry.label = "unrelated-attempt"; },
+    (report: typeof original) => { const entry = report.attempts.find((item) => item.label.endsWith(":hidden")); if (!entry) throw new Error("Missing attempt."); entry.receipt.result.exit_code = 0; },
+    (report: typeof original) => { const entry = report.attempts.find((item) => item.label.endsWith(":repaired")); if (!entry) throw new Error("Missing attempt."); entry.receipt.result.failure = "injected failure"; },
+    (report: typeof original) => { const entry = report.attempts[1]; if (!entry) throw new Error("Missing attempt."); report.attempts[0] = entry; },
+    (report: typeof original) => { const entry = report.attempts.find((item) => item.label.endsWith(":hidden")); if (!entry) throw new Error("Missing attempt."); entry.receipt.stdout = "[]"; },
+    (report: typeof original) => { report.metadata.restored_source = digest("not restored"); },
+    (report: typeof original) => { const owner = OWNER_FILES[0]; if (!owner) throw new Error("Missing owner."); report.metadata.owner_files[owner] = digest("changed owner"); },
+  ]) {
+    const changed = structuredClone(original);
+    mutate(changed);
+    await Bun.write(fixture.receipt, JSON.stringify(changed));
+    await expect(fixture.examples()).rejects.toThrow(CONSUMER_JEV_ERROR);
+  }
+});
+
+test("jev_cli_without_credentials_retains_every_scheduled_attempt_and_preserves_output", async () => {
+  using fixture = new ConsumerFixture();
+  await fixture.setup();
+  const output = join(fixture.parent, "results");
+  const invoke = (destination: string) => Bun.spawnSync([process.execPath, CLI, "--live", fixture.root, fixture.receipt, destination], {
+    env: { PATH: Bun.env.PATH ?? "", TYPESAFE_API_KEY: "" }, stdout: "pipe", stderr: "pipe", timeout: TIMEOUT,
+  });
+  const first = invoke(output);
+  if (first.exitCode !== 0) throw new Error(first.stderr.toString());
+  expect(first.exitCode).toBe(0);
+  const file = Bun.file(join(output, "report.json"));
+  const bytes = await file.text();
+  const report = JSON.parse(bytes);
+  expect(report.completed).toBe(true);
+  expect(report.requests).toBe(0);
+  expect(report.attempts).toHaveLength(188);
+  expect(report.results.every((entry: { status: string }) => entry.status === "unavailable")).toBe(true);
+  expect(invoke(output).exitCode).not.toBe(0);
+  expect(await file.text()).toBe(bytes);
+  const inside = invoke(join(fixture.root, "results"));
+  expect(inside.exitCode).toBe(2);
+  expect(inside.stderr.toString()).toContain("outside the authoring and consumer worktrees");
+  expect(await Bun.file(join(fixture.root, "results/report.json")).exists()).toBe(false);
+}, TIMEOUT);
+
+test("jev_cli_refuses_symlink_outputs_authoring_outputs_and_oversized_replay", async () => {
+  using fixture = new ConsumerFixture();
+  await fixture.setup();
+  const alias = join(fixture.parent, "consumer-alias");
+  symlinkSync(fixture.root, alias);
+  const oversized = join(fixture.parent, "oversized.json");
+  await Bun.write(oversized, " ".repeat(MAX_JEV_REPORT_BYTES + 1));
+  for (const [destination, mode, saved] of [
+    [join(alias, "results"), "--live", []],
+    [join(ROOT, "forbidden-jev-test-output"), "--live", []],
+    [alias, "--live", []],
+    [join(fixture.parent, "replay"), "--replay", [oversized]],
+  ] as const) {
+    const result = Bun.spawnSync([process.execPath, CLI, mode, fixture.root, fixture.receipt, destination, ...saved], {
+      env: { PATH: Bun.env.PATH ?? "", TYPESAFE_API_KEY: "" }, stdout: "pipe", stderr: "pipe", timeout: TIMEOUT,
+    });
+    expect(result.exitCode).toBe(2);
+    expect(result.stdout.toString()).toBe("");
+  }
+  expect(await Bun.file(join(fixture.root, "results/report.json")).exists()).toBe(false);
+  expect(await Bun.file(join(ROOT, "forbidden-jev-test-output/report.json")).exists()).toBe(false);
+}, TIMEOUT);

package/src/judgment_comparison_jev_report.test.ts

@@ -0,0 +1,167 @@
+import { expect, test } from "bun:test";
+import { resolve } from "node:path";
+
+import { loadCases } from "../evals/judgments/comparison/corpus";
+import { corpusJevExamples, gradeJev, jevInput, JEV_PREFLIGHT_ERROR, type JevAttempt, type JevExample } from "../evals/judgments/comparison/jev";
+import { JEV_MODE, jevReport, renderJevReport, replayJev } from "../evals/judgments/comparison/jev-report";
+import { CANDIDATE, CASE_CLASS, CLASSIFICATION, labelsSchema, MEASUREMENT, NATIVE, OBSERVATION } from "../evals/judgments/comparison/types";
+import { validateCatalog } from "../evals/judgments/comparison/validate";
+import { ANSWER_KIND, MODEL, REPLY_KEY } from "./judgments/constants";
+import { digest } from "./judgments/files";
+import type { Reply } from "./judgments/wire";
+
+const ROOT = resolve(import.meta.dir, "..");
+const MISSING = "Request failed before a usable reply.";
+
+async function inputs() {
+  const catalog = validateCatalog(await Bun.file(resolve(ROOT, "evals/judgments/comparison/catalog.json")).json());
+  const cases = await loadCases(ROOT);
+  const labels = labelsSchema.parse(await Bun.file(resolve(ROOT, "evals/judgments/comparison/labels.json")).json());
+  return { catalog, cases, labels, examples: corpusJevExamples(catalog, cases, labels) };
+}
+
+function attempt(entry: JevExample, reply: Reply | null = null): JevAttempt {
+  return { id: entry.id, question: entry.question, identity: entry.identity, request_digest: digest(entry.input.request),
+    requests: reply === null ? 0 : 1, elapsed_ms: 0, reply, failure: reply === null ? MISSING : null };
+}
+
+function reply(entry: JevExample, decision: string, strength = 1): Reply {
+  const question = entry.input.definition.question;
+  const keys = Object.keys(question.criteria);
+  return { model: MODEL, answers: { [REPLY_KEY]: question.type === ANSWER_KIND.noul ? { type: question.type, noul: strength } : {
+    type: question.type, choice: decision, confidence: strength,
+    probabilities: Object.fromEntries(keys.map((key) => [key, key === decision ? strength : (1 - strength) / (keys.length - 1)])),
+  } }, usage: { input_tokens: 100, output_tokens: 20 } };
+}
+
+test("jev_inputs_exclude_expected_labels_and_freeze_native_questions", async () => {
+  const { catalog, cases, labels, examples } = await inputs();
+  for (const entry of examples) {
+    const request = JSON.parse(entry.input.request);
+    expect(request.model).toBe(MODEL);
+    expect(Object.keys(request.questions)).toEqual([REPLY_KEY]);
+    expect(request.questions[REPLY_KEY]).toEqual(entry.input.definition.question);
+    expect(Object.keys(request.state).sort()).toEqual(["evidence", "requirement"]);
+    expect(request.state.evidence.every((item: Record<string, unknown>) => JSON.stringify(Object.keys(item).sort()) === JSON.stringify(["role", "text"]))).toBe(true);
+  }
+  const first = cases[0];
+  const original = examples[0];
+  if (!first || !original) throw new Error("Missing cases.");
+  const changed = corpusJevExamples(catalog, [{ ...first, classification: CASE_CLASS.defective, author: "different-author", id: first.id }], labels)[0];
+  expect(changed?.input.request).toBe(original.input.request);
+  expect(changed?.identity).not.toBe(original.identity);
+  expect(() => jevInput(original.input.definition.question, { ...JSON.parse(original.input.request).state, expected: NATIVE.yes })).toThrow();
+});
+
+test("jev_report_grades_actual_answers_without_model_accuracy_substitution", async () => {
+  const { cases, labels, examples } = await inputs();
+  const attempts = examples.map((entry) => attempt(entry));
+  const yesIndex = examples.findIndex((entry) => entry.expected === NATIVE.yes);
+  const noIndex = examples.findIndex((entry) => entry.expected === NATIVE.no);
+  const candidateIndex = examples.findIndex((entry) => entry.question === CANDIDATE.wiring && entry.expected === CLASSIFICATION.defective);
+  const uncertainIndex = examples.findIndex((entry) => entry.question === CANDIDATE.wiring && entry.expected === CLASSIFICATION.supported);
+  for (const [index, choice, strength] of [[yesIndex, NATIVE.yes, 1], [noIndex, NATIVE.yes, 1],
+    [candidateIndex, CLASSIFICATION.supported, 1], [uncertainIndex, CLASSIFICATION.supported, 0.4]] as const) {
+    const entry = examples[index];
+    if (!entry) throw new Error("Missing selected case.");
+    attempts[index] = attempt(entry, reply(entry, choice, strength));
+  }
+  const report = jevReport(examples, attempts, cases, labels, [], JEV_MODE.live, true);
+  expect(report.results[yesIndex]?.status).toBe("pass");
+  expect(report.results[noIndex]?.status).toBe("fail");
+  expect(report.results[candidateIndex]?.actual).toBe(CLASSIFICATION.supported);
+  expect(report.results[candidateIndex]?.status).toBe("fail");
+  expect(report.results[uncertainIndex]).toMatchObject({ actual: CLASSIFICATION.supported, graded: NATIVE.abstain, uncertain: true, status: "fail" });
+  expect(report.observations[uncertainIndex]).toMatchObject({ status: OBSERVATION.valid, answer: { decision: CLASSIFICATION.supported, strength: 0.4 } });
+  expect(report.observations.every((entry) => entry.measurement === MEASUREMENT.live)).toBe(true);
+  expect(report.adoption.every((entry) => !entry.eligible && entry.decision === "retain")).toBe(true);
+  expect(report.requests).toBe(4);
+  expect(report.usage).toEqual({ input_tokens: 400, output_tokens: 80 });
+  expect(report.scores).toHaveLength(22);
+  expect(report.results.filter((entry) => entry.status === "unavailable")).toHaveLength(examples.length - 4);
+  const rendered = renderJevReport(report);
+  for (const glyph of ["✅", "❌", "⚪"]) expect(rendered).toContain(glyph);
+  expect(rendered).toContain("(uncertain)");
+  expect(rendered).toContain("not measured autonomous agent improvement");
+});
+
+test("jev_replay_preserves_original_answers_without_new_request_credit", async () => {
+  const { cases, labels, examples } = await inputs();
+  const attempts = examples.map((entry) => attempt(entry));
+  const first = examples[0];
+  if (!first) throw new Error("Missing case.");
+  attempts[0] = attempt(first, reply(first, NATIVE.yes));
+  const live = jevReport(examples, attempts, cases, labels, [], JEV_MODE.live, true);
+  const replay = jevReport(examples, replayJev(examples, live, []), cases, labels, [], JEV_MODE.replay, true);
+  expect(replay.results).toEqual(live.results);
+  expect(replay.attempts).toEqual(live.attempts);
+  expect(replay.requests).toBe(0);
+  expect(replay.original_requests).toBe(1);
+  expect(replay.observations.every((entry) => entry.measurement === MEASUREMENT.editable)).toBe(true);
+  expect(replay.adoption.every((entry) => !entry.eligible)).toBe(true);
+});
+
+test("jev_replay_rejects_changed_input_order_identity_reply_and_request_count", async () => {
+  const { cases, labels, examples } = await inputs();
+  const attempts = examples.map((entry) => attempt(entry));
+  const first = examples[0];
+  if (!first) throw new Error("Missing case.");
+  attempts[0] = attempt(first, reply(first, NATIVE.yes));
+  const report = jevReport(examples, attempts, cases, labels, [], JEV_MODE.live, true);
+  for (const field of ["id", "question", "identity", "request_digest"] as const) {
+    const changed = structuredClone(report);
+    const entry = changed.attempts[0];
+    if (!entry) throw new Error("Missing attempt.");
+    entry[field] = "altered";
+    expect(() => replayJev(examples, changed, [])).toThrow(JEV_PREFLIGHT_ERROR);
+  }
+  expect(() => replayJev([...examples].reverse(), report, [])).toThrow(JEV_PREFLIGHT_ERROR);
+  expect(() => replayJev(examples, { ...report, attempts: attempts.slice(1) }, [])).toThrow(JEV_PREFLIGHT_ERROR);
+  const firstAttempt = attempts[0];
+  if (!firstAttempt) throw new Error("Missing attempt.");
+  expect(() => gradeJev(first, { ...firstAttempt, requests: 0 })).toThrow(JEV_PREFLIGHT_ERROR);
+});
+
+test("jev_replay_rejects_wrong_native_types_and_invalid_probability_distributions", async () => {
+  const { cases, labels, examples } = await inputs();
+  const index = examples.findIndex((entry) => entry.question === CANDIDATE.wiring);
+  const entry = examples[index];
+  if (!entry) throw new Error("Missing Choice example.");
+  const attempts = examples.map((value) => attempt(value));
+  for (const answer of [
+    { type: ANSWER_KIND.noul, noul: 1 },
+    { type: ANSWER_KIND.choice, choice: CLASSIFICATION.supported, confidence: 1, probabilities: {} },
+    { type: ANSWER_KIND.choice, choice: CLASSIFICATION.supported, confidence: 2, probabilities: {} },
+  ]) {
+    const report = jevReport(examples, attempts, cases, labels, [], JEV_MODE.live, true);
+    const changed = { ...attempt(entry), requests: 1, failure: null,
+      reply: { model: MODEL, answers: { [REPLY_KEY]: answer }, usage: { input_tokens: 1, output_tokens: 1 } } };
+    expect(() => replayJev(examples, { ...report, attempts: report.attempts.map((value, position) => position === index ? changed : value) }, [])).toThrow();
+  }
+});
+
+test("jev_noul_uncertainty_preserves_native_probability_and_abstains", async () => {
+  const { examples } = await inputs();
+  const entry = examples.find((value) => value.input.definition.question.type === ANSWER_KIND.noul);
+  if (!entry) throw new Error("Missing Noul example.");
+  for (const probability of [0.49, 0.5, 0.51]) {
+    const native = reply(entry, NATIVE.yes, probability);
+    const result = gradeJev(entry, attempt(entry, native));
+    expect(result.actual).toBe(probability < 0.5 ? NATIVE.no : NATIVE.yes);
+    expect(result.graded).toBe(NATIVE.abstain);
+    expect(result.uncertain).toBe(true);
+    expect(native.answers[REPLY_KEY]).toEqual({ type: ANSWER_KIND.noul, noul: probability });
+  }
+});
+
+// Saved observations must retain the evaluator that produced them.
+test("jev_replay_rejects_missing_changed_or_reordered_source_inventory", async () => {
+  const { cases, labels, examples } = await inputs();
+  const sources = [{ path: "score.ts", digest: digest("scorer") }, { path: "jev.ts", digest: digest("transport") }];
+  const live = jevReport(examples, examples.map((entry) => attempt(entry)), cases, labels, sources, JEV_MODE.live, true);
+  expect(replayJev(examples, live, sources)).toEqual(live.attempts);
+  for (const changed of [undefined, [], sources.slice(1), [...sources].reverse(), [...sources, sources[0]],
+    sources.map((entry) => ({ ...entry, digest: "0".repeat(64) })), sources.map((entry) => ({ ...entry, path: "other.ts" }))]) {
+    expect(() => replayJev(examples, { ...live, sources: changed }, sources)).toThrow();
+  }
+});

package/src/judgment_labels.test.ts

@@ -0,0 +1,119 @@
+import { afterEach, expect, test } from "bun:test";
+import { join } from "node:path";
+
+import catalogInput from "../evals/judgments/comparison/catalog.json";
+import { validateLabels } from "../evals/judgments/comparison/labels";
+import { validateCatalog } from "../evals/judgments/comparison/validate";
+import { ADJUDICATION, CANDIDATE, CASE_CLASS, CHECK_ERROR, CLASSIFICATION, LABEL_AUTHORITY, MIN_RESOLVED_PER_GROUP, NATIVE, SPLIT,
+  type ComparisonCase, type IndependentLabel, type LabelOracle } from "../evals/judgments/comparison/types";
+import { cleanupTemporaryDirectories, temporaryDirectory } from "./gates_test_support";
+import { CHOICE, QUESTIONS } from "./judgments/constants";
+import { digest } from "./judgments/files";
+
+const FIXTURE_AUTHOR = "label-fixture-author";
+const ORACLE_NAME = "independent-test-boundary";
+const SOURCE_PATH = "reference.md";
+const SOURCE_CONTENT = "An independently sourced expected value is compared with the selected result.\n";
+const REVISION_LENGTH = 40;
+const REVISION = "1".repeat(REVISION_LENGTH);
+const catalog = validateCatalog(catalogInput);
+
+afterEach(cleanupTemporaryDirectories);
+
+function nativeAnswer(question: string, classification: ComparisonCase["classification"]): IndependentLabel["expected_native"] {
+  if (question === QUESTIONS[2]) return classification === CASE_CLASS.healthy ? CHOICE.exact : classification === CASE_CLASS.defective ? CHOICE.weak
+    : classification === CASE_CLASS.ambiguous ? NATIVE.abstain : CHOICE.insufficient;
+  if (QUESTIONS.some((item) => item === question)) return classification === CASE_CLASS.healthy ? NATIVE.yes
+    : classification === CASE_CLASS.defective ? NATIVE.no : NATIVE.abstain;
+  return classification === CASE_CLASS.healthy ? CLASSIFICATION.supported : classification === CASE_CLASS.defective ? CLASSIFICATION.defective
+    : classification === CASE_CLASS.ambiguous ? CLASSIFICATION.ambiguous : CLASSIFICATION.insufficient;
+}
+
+async function fixture() {
+  const root = temporaryDirectory();
+  await Bun.write(join(root, SOURCE_PATH), SOURCE_CONTENT);
+  const cases: ComparisonCase[] = [];
+  const labels: IndependentLabel[] = [];
+  for (const question of [...QUESTIONS, ...Object.values(CANDIDATE)]) for (const classification of Object.values(CASE_CLASS)) {
+    for (const split of Object.values(SPLIT)) for (let index = 0; index < MIN_RESOLVED_PER_GROUP; index++) {
+      const id = `${question}-${classification}-${split}-${index}`;
+      cases.push({ id, question, classification, split, author: FIXTURE_AUTHOR, origin: `${question}-${split}`,
+        requirement: `Independent unit boundary ${id}`, evidence: [] });
+      labels.push({ case_id: id, classification, expected_native: nativeAnswer(question, classification), rationale: SOURCE_CONTENT.trim(),
+        author: FIXTURE_AUTHOR, adjudication: ADJUDICATION.resolved, authority: { kind: LABEL_AUTHORITY.oracle, name: ORACLE_NAME },
+        source: { path: SOURCE_PATH, digest: digest(SOURCE_CONTENT), revision: REVISION, selected: SOURCE_CONTENT.trim() } });
+    }
+  }
+  const oracle: LabelOracle = async (entry) => ({ classification: entry.classification, expected_native: nativeAnswer(entry.question, entry.classification),
+    source_digest: digest(SOURCE_CONTENT), revision: REVISION });
+  return { root, cases, labels, oracle };
+}
+
+test("labels_require_independent_evidence", async () => {
+  const { root, cases, labels, oracle } = await fixture();
+  expect(await validateLabels(root, catalog, cases, labels, oracle)).toEqual(labels);
+  const selfReviewed = labels.map((label) => ({ ...label, authority: { kind: LABEL_AUTHORITY.reviewer, name: FIXTURE_AUTHOR } }));
+  await expect(validateLabels(root, catalog, cases, selfReviewed, oracle)).rejects.toThrow(CHECK_ERROR.independence);
+});
+
+test("all_pending_and_all_disputed_labels_cannot_satisfy_minimums", async () => {
+  const { root, cases, labels, oracle } = await fixture();
+  for (const adjudication of [ADJUDICATION.pending, ADJUDICATION.disputed]) {
+    await expect(validateLabels(root, catalog, cases, labels.map((label) => ({ ...label, adjudication })), oracle)).rejects.toThrow(CHECK_ERROR.minimum);
+  }
+});
+
+test("a_missing_independent_source_and_invented_selected_text_refuse_labels", async () => {
+  const { root, cases, labels, oracle } = await fixture();
+  const absent = labels.map((label) => ({ ...label, source: { ...label.source, path: "absent-reference.md" } }));
+  await expect(validateLabels(root, catalog, cases, absent, oracle)).rejects.toThrow();
+  const fabricated = labels.map((label) => ({ ...label, source: { ...label.source, selected: "Invented label approval." } }));
+  await expect(validateLabels(root, catalog, cases, fabricated, oracle)).rejects.toThrow(CHECK_ERROR.independence);
+});
+
+test("an_independence_flag_cannot_replace_external_oracle_proof", async () => {
+  const { root, cases, labels, oracle } = await fixture();
+  await expect(validateLabels(root, catalog, cases, labels.map((label) => ({ ...label, independent: true })), oracle)).rejects.toThrow(CHECK_ERROR.labels);
+  const wrongOracle: LabelOracle = async (entry, label) => ({ ...(await oracle(entry, label)), expected_native: CLASSIFICATION.defective });
+  await expect(validateLabels(root, catalog, cases, labels, wrongOracle)).rejects.toThrow(CHECK_ERROR.independence);
+});
+
+test("stale_oracle_revision_and_changed_label_evidence_refuse_labels", async () => {
+  const { root, cases, labels, oracle } = await fixture();
+  const stale: LabelOracle = async (entry, label) => ({ ...(await oracle(entry, label)), revision: "2".repeat(REVISION_LENGTH) });
+  await expect(validateLabels(root, catalog, cases, labels, stale)).rejects.toThrow(CHECK_ERROR.independence);
+  await Bun.write(join(root, SOURCE_PATH), "Changed independent evidence.\n");
+  await expect(validateLabels(root, catalog, cases, labels, oracle)).rejects.toThrow(CHECK_ERROR.independence);
+});
+
+test("missing_duplicate_and_mismatched_labels_refuse_scoring", async () => {
+  const { root, cases, labels, oracle } = await fixture();
+  await expect(validateLabels(root, catalog, cases, labels.slice(1), oracle)).rejects.toThrow(CHECK_ERROR.labelIdentity);
+  await expect(validateLabels(root, catalog, cases, [...labels, labels[0]], oracle)).rejects.toThrow(CHECK_ERROR.labelIdentity);
+  await expect(validateLabels(root, catalog, cases, labels.map((label) => ({ ...label, author: "invented-author" })), oracle)).rejects.toThrow(CHECK_ERROR.labelIdentity);
+});
+
+test("labels_cannot_change_answer_types_or_relabel_case_health", async () => {
+  const { root, cases, labels, oracle } = await fixture();
+  await expect(validateLabels(root, catalog, cases, labels.map((label) => ({ ...label, expected_native: CHOICE.exact })), oracle)).rejects.toThrow(CHECK_ERROR.native);
+  await expect(validateLabels(root, catalog, cases, labels.map((label) => ({ ...label, classification: CASE_CLASS.defective })), oracle)).rejects.toThrow(CHECK_ERROR.labelIdentity);
+});
+
+test("frozen_reference_bytes_validate_without_git_history_and_reject_modified_sources", async () => {
+  const { resolve } = await import("node:path");
+  const { loadCases } = await import("../evals/judgments/comparison/corpus");
+  const { verifyCaseLabel } = await import("../evals/judgments/comparison/hidden/label-oracle");
+  const { labelsSchema } = await import("../evals/judgments/comparison/types");
+  const { default: input } = await import("../evals/judgments/comparison/labels.json");
+  const sourceRoot = resolve(import.meta.dir, "..");
+  const label = labelsSchema.parse(input)[0];
+  if (!label) throw new Error("Missing independent label.");
+  const entry = (await loadCases(sourceRoot)).find((entry) => entry.id === label.case_id);
+  if (!entry) throw new Error("Missing sourced case.");
+  const root = temporaryDirectory();
+  const source = join(root, label.source.path);
+  await Bun.write(source, Bun.file(join(sourceRoot, label.source.path)));
+  expect((await verifyCaseLabel(root, entry, label)).source_digest).toBe(label.source.digest);
+  await Bun.write(source, "Changed reference bytes.");
+  await expect(verifyCaseLabel(root, entry, label)).rejects.toThrow(CHECK_ERROR.independence);
+});