@agentsfleet/orly 0.12.0 → 0.13.0

package/src/judgment_score.test.ts

@@ -0,0 +1,166 @@
+import { expect, test } from "bun:test";
+
+import { adoptionDecision, missingObservations, scoreQuestion, validateObservations } from "../evals/judgments/comparison/score";
+import { ADJUDICATION, CANDIDATE, CASE_CLASS, CHECK_ERROR, CLASSIFICATION, LABEL_AUTHORITY, MEASUREMENT, NATIVE, OBSERVATION, SPLIT,
+  type ComparisonCase, type IndependentLabel, type Observation } from "../evals/judgments/comparison/types";
+import { CHOICE, MODEL, QUESTIONS } from "./judgments/constants";
+import { OrlyError } from "./model";
+
+const CERTAIN = 1;
+const UNCERTAIN = 0.5;
+const TEST_AUTHOR = "score-fixture-author";
+const TEST_ORACLE = "unit-boundary-oracle";
+const DIGEST_LENGTH = 64;
+const REVISION_LENGTH = 40;
+const REFERENCE_SOURCE = "reference.md";
+
+function fixture(question: ComparisonCase["question"], classifications: ComparisonCase["classification"][], expected: IndependentLabel["expected_native"][]) {
+  const cases: ComparisonCase[] = classifications.map((classification, index) => ({ id: `case-${index}`, question, classification,
+    origin: `family-${index}`, split: SPLIT.heldout, author: TEST_AUTHOR, requirement: "Assert the sourced result.", evidence: [] }));
+  const labels: IndependentLabel[] = cases.map((entry, index) => {
+    const answer = expected[index];
+    if (!answer) throw new Error("Missing unit expectation.");
+    return { case_id: entry.id, classification: entry.classification, expected_native: answer, rationale: "Exact unit boundary expectation.",
+      author: TEST_AUTHOR, adjudication: ADJUDICATION.resolved, authority: { kind: LABEL_AUTHORITY.oracle, name: TEST_ORACLE },
+      source: { path: REFERENCE_SOURCE, digest: "0".repeat(DIGEST_LENGTH), revision: "0".repeat(REVISION_LENGTH), selected: "Expected result." } };
+  });
+  const observations: Observation[] = missingObservations(cases).map((entry, index) => {
+    const decision = expected[index];
+    if (!decision) throw new Error("Missing unit answer.");
+    return { ...entry, status: OBSERVATION.valid, answer: { decision, strength: CERTAIN } };
+  });
+  return { cases, labels, observations };
+}
+
+test("metrics_preserve_missing_outcomes", () => {
+  const { cases, labels, observations } = fixture(QUESTIONS[0], [CASE_CLASS.healthy, CASE_CLASS.defective, CASE_CLASS.ambiguous, CASE_CLASS.insufficient],
+    [NATIVE.yes, NATIVE.no, NATIVE.abstain, NATIVE.abstain]);
+  const mixed = observations.map((item, index) => index === 0 ? { ...item, status: OBSERVATION.unavailable, answer: null } : item);
+  const result = scoreQuestion(QUESTIONS[0], cases, labels, validateObservations(cases, mixed));
+  expect(result.native_correctness).toEqual({ numerator: 3, denominator: 4, value: 0.75 });
+  expect(result.defect_recall).toEqual({ numerator: 1, denominator: 1, value: 1 });
+  expect(result.false_concerns_bound).toEqual({ numerator: 1, denominator: 1, value: 1 });
+  expect(result.insufficient_detection).toBeNull();
+  expect(result.shared_abstention).toEqual({ numerator: 2, denominator: 2, value: 1 });
+  expect(() => validateObservations(cases, mixed.slice(1))).toThrow(new OrlyError(CHECK_ERROR.observations));
+});
+
+test("healthy_applicable_yes_is_native_correctness_without_defect_metrics", () => {
+  const { cases, labels, observations } = fixture(QUESTIONS[4], [CASE_CLASS.healthy], [NATIVE.yes]);
+  const result = scoreQuestion(QUESTIONS[4], cases, labels, observations);
+  expect(result.native_correctness?.value).toBe(1);
+  expect(result.false_concerns).toBeNull();
+  expect(result.defect_recall).toBeNull();
+});
+
+test("noul_half_probability_abstains_without_inventing_insufficiency", () => {
+  const { cases, labels, observations } = fixture(QUESTIONS[0], [CASE_CLASS.insufficient], [NATIVE.abstain]);
+  const answer = observations.map((item) => ({ ...item, answer: { decision: NATIVE.yes, strength: UNCERTAIN } }));
+  const result = scoreQuestion(QUESTIONS[0], cases, labels, answer);
+  expect(result.shared_abstention?.value).toBe(1);
+  expect(result.insufficient_detection).toBeNull();
+});
+
+test("zero_responses_and_empty_denominators_are_unavailable", () => {
+  const { cases, labels } = fixture(CANDIDATE.obligation, [CASE_CLASS.healthy], [CLASSIFICATION.supported]);
+  const result = scoreQuestion(CANDIDATE.obligation, cases, labels, missingObservations(cases));
+  expect(result.attempted).toBe(1);
+  expect(result.unavailable).toBe(1);
+  expect(result.native_correctness).toBeNull();
+  expect(result.defect_recall).toBeNull();
+  expect(result.false_concerns_bound).toBeNull();
+  expect(scoreQuestion(CANDIDATE.wiring, [], [], []).native_correctness).toBeNull();
+});
+
+test("invalid_defective_answers_are_misses_and_invalid_healthy_answers_raise_the_bound", () => {
+  const { cases, labels, observations } = fixture(CANDIDATE.failure, [CASE_CLASS.healthy, CASE_CLASS.defective], [CLASSIFICATION.supported, CLASSIFICATION.defective]);
+  const answer = observations.map((item, index) => index === 0 ? item : { ...item, status: OBSERVATION.invalid, answer: null });
+  const result = scoreQuestion(CANDIDATE.failure, cases, labels, answer);
+  expect(result.native_correctness?.value).toBe(0.5);
+  expect(result.defect_recall?.value).toBe(0);
+  expect(result.invalid).toBe(1);
+});
+
+test("weak_low_confidence_assertion_is_an_abstention", () => {
+  const { cases, labels, observations } = fixture(QUESTIONS[2], [CASE_CLASS.defective], [CHOICE.weak]);
+  const answer = observations.map((item) => ({ ...item, answer: { decision: CHOICE.weak, strength: UNCERTAIN } }));
+  const result = scoreQuestion(QUESTIONS[2], cases, labels, answer);
+  expect(result.native_correctness?.value).toBe(0);
+  expect(result.defect_recall?.value).toBe(0);
+});
+
+test("explicit_choice_insufficiency_is_scored_separately", () => {
+  const { cases, labels, observations } = fixture(QUESTIONS[2], [CASE_CLASS.insufficient], [CHOICE.insufficient]);
+  expect(scoreQuestion(QUESTIONS[2], cases, labels, observations).insufficient_detection?.value).toBe(1);
+});
+
+test("observations_reject_stale_duplicate_unknown_and_executable_answers", () => {
+  const { cases, observations } = fixture(CANDIDATE.wiring, [CASE_CLASS.healthy, CASE_CLASS.defective], [CLASSIFICATION.supported, CLASSIFICATION.defective]);
+  expect(() => validateObservations(cases, observations.map((item) => ({ ...item, identity: "0".repeat(DIGEST_LENGTH) })))).toThrow(CHECK_ERROR.observations);
+  expect(() => validateObservations(cases, [observations[0], observations[0]])).toThrow(CHECK_ERROR.observations);
+  expect(() => validateObservations(cases, observations.map((item) => ({ ...item, execute: "true" })))).toThrow(CHECK_ERROR.observations);
+  expect(() => validateObservations(cases.map((item) => ({ ...item, requirement: "Edited required behavior." })), observations)).toThrow(CHECK_ERROR.observations);
+});
+
+test("adoption_requires_independent_measured_benefit", () => {
+  const { cases, labels, observations } = fixture(CANDIDATE.resolution, Object.values(CASE_CLASS), Object.values(CLASSIFICATION));
+  const decision = adoptionDecision(CANDIDATE.resolution, cases, labels, missingObservations(cases), null);
+  expect(decision.eligible).toBe(false);
+  expect(decision.reasons.length).toBe(4);
+  expect(decision.decision).toBe("retain");
+  expect(adoptionDecision(CANDIDATE.resolution, cases, labels, observations, null).eligible).toBe(false);
+});
+
+test("unmeasured_candidates_remain_evaluation_only", () => {
+  for (const question of Object.values(CANDIDATE)) {
+    const { cases, labels } = fixture(question, [CASE_CLASS.healthy], [CLASSIFICATION.supported]);
+    const result = adoptionDecision(question, cases, labels, missingObservations(cases), null);
+    expect(result.eligible).toBe(false);
+    expect(result.metrics.native_correctness).toBeNull();
+  }
+  expect(QUESTIONS.length).toBe(6);
+});
+
+test("development_cases_cannot_fill_the_heldout_adoption_minimum", () => {
+  const { cases, labels, observations } = fixture(CANDIDATE.wiring, [CASE_CLASS.healthy], [CLASSIFICATION.supported]);
+  const result = adoptionDecision(CANDIDATE.wiring, cases.map((entry) => ({ ...entry, split: SPLIT.development })), labels,
+    observations.map((entry) => ({ ...entry, measurement: MEASUREMENT.independent })), { baseline_resolved: 0, candidate_resolved: 1, unsupported_completions: 0, authority_violations: 0 });
+  expect(result.eligible).toBe(false);
+  expect(result.metrics.attempted).toBe(0);
+});
+
+test("answers_from_another_question_format_count_as_invalid_attempts", () => {
+  const { cases, labels, observations } = fixture(CANDIDATE.wiring, [CASE_CLASS.healthy], [CLASSIFICATION.supported]);
+  const malformed = observations.map((entry) => ({ ...entry, answer: { decision: NATIVE.yes, strength: CERTAIN } }));
+  const validated = validateObservations(cases, malformed);
+  expect(validated.map((entry) => entry.status)).toEqual([OBSERVATION.invalid]);
+  const result = scoreQuestion(CANDIDATE.wiring, cases, labels, malformed);
+  expect(result.invalid).toBe(1);
+  expect(result.valid).toBe(0);
+  expect(result.false_concerns_bound?.value).toBe(1);
+});
+
+test("pending_extra_cases_are_unscored_and_do_not_change_denominators", () => {
+  const { cases, labels, observations } = fixture(CANDIDATE.evidence, [CASE_CLASS.healthy, CASE_CLASS.healthy], [CLASSIFICATION.supported, CLASSIFICATION.supported]);
+  const pending = labels.map((entry, index) => index === 0 ? entry : { ...entry, adjudication: ADJUDICATION.pending });
+  const result = scoreQuestion(CANDIDATE.evidence, cases, pending, observations);
+  expect(result.attempted).toBe(1);
+  expect(result.unscored).toBe(1);
+  expect(result.native_correctness).toEqual({ numerator: 1, denominator: 1, value: 1 });
+});
+
+test("fabricated_measurement_enums_and_paired_counts_never_qualify_for_adoption", () => {
+  const minimum = 20;
+  const classes = Object.values(CASE_CLASS).flatMap((classification) => Array.from({ length: minimum }, () => classification));
+  const answers = Object.values(CLASSIFICATION).flatMap((answer) => Array.from({ length: minimum }, () => answer));
+  const { cases, labels, observations } = fixture(CANDIDATE.resolution, classes, answers);
+  const fabricated = observations.map((entry) => ({ ...entry, measurement: MEASUREMENT.independent }));
+  const invalidPair = { baseline_resolved: -1, candidate_resolved: 0, unsupported_completions: 0, authority_violations: 0 };
+  const ordinaryPair = { ...invalidPair, baseline_resolved: 1, candidate_resolved: 2 };
+  for (const pair of [invalidPair, ordinaryPair]) {
+    const result = adoptionDecision(CANDIDATE.resolution, cases, labels, fabricated, pair);
+    expect(result.eligible).toBe(false);
+    expect(result.decision).toBe("retain");
+    expect(result.reasons.some((reason) => reason.includes("provenance"))).toBe(true);
+  }
+});

package/src/judgment_tasks.test.ts

@@ -0,0 +1,164 @@
+import { afterEach, expect, mock, spyOn, test } from "bun:test";
+import * as fileSystem from "node:fs";
+import { join } from "node:path";
+
+import { EvaluationRun, EXECUTION_ERROR, sourceIdentity } from "../evals/judgments/comparison/execution";
+import { cleanupTemporaryDirectories, temporaryDirectory } from "./gates_test_support";
+import { createOwnership, OWNER_ERROR, recoverStaleOwnership } from "../evals/judgments/comparison/recovery";
+
+const INPUT_FILE = "input.txt";
+const INITIAL_CONTENT = "2\n3\n";
+const BUN_EVAL = "-e";
+const COMMAND_SUCCESS = 'process.stdout.write("measured\\n")';
+const COMMAND_FAILURE = 'process.stderr.write("required refusal\\n");process.exit(2)';
+const COMMAND_WAIT = 'process.stdout.write("ready\\n");setInterval(()=>{},1000)';
+const COMMAND_OUTPUT = 'process.stdout.write("x".repeat(1000000));setInterval(()=>{},1000)';
+const SUCCESS_LIMITS = { timeout_ms: 2_000, output_bytes: 4_096 };
+const DEADLINE_LIMITS = { timeout_ms: 150, output_bytes: SUCCESS_LIMITS.output_bytes };
+const OUTPUT_LIMITS = { timeout_ms: SUCCESS_LIMITS.timeout_ms, output_bytes: 128 };
+const TEST_TIMEOUT_MS = 10_000;
+const READY_POLL_MS = 10;
+const { existsSync } = fileSystem;
+const START_SENTINEL = "child-started.txt";
+const SENTINEL_COMMAND = `await Bun.write('${START_SENTINEL}','started')`;
+const ALLOCATION_FAILURE = "second temporary-directory allocation failed";
+type SyncOptions = NonNullable<Parameters<typeof Bun.spawnSync>[1]>;
+
+afterEach(() => { mock.restore(); cleanupTemporaryDirectories(); });
+
+test("task_receipts_bind_actual_execution", async () => {
+  const run = await EvaluationRun.create();
+  try {
+    await Bun.write(join(run.workspace, INPUT_FILE), INITIAL_CONTENT);
+    const actual = await run.execute([process.execPath, BUN_EVAL, COMMAND_SUCCESS], [INPUT_FILE], SUCCESS_LIMITS);
+    expect(actual.result.exit_code).toBe(0);
+    expect(actual.stdout).toBe("measured\n");
+    expect(actual.result.output_bytes).toBe(new TextEncoder().encode(actual.stdout).byteLength);
+    expect(actual.source_before).toBe(await sourceIdentity(run.workspace, [INPUT_FILE]));
+    expect(() => run.verify(actual)).not.toThrow();
+    expect(() => run.verify({ ...actual, result: { ...actual.result, exit_code: 2 } })).toThrow(EXECUTION_ERROR);
+    expect(() => run.verify({ ...actual, command: ["true"] })).toThrow(EXECUTION_ERROR);
+    await Bun.write(join(run.workspace, INPUT_FILE), "9\n");
+    expect(await sourceIdentity(run.workspace, [INPUT_FILE])).not.toBe(actual.source_after);
+  } finally { await run.close(); }
+});
+
+test("offline_tasks_bound_and_clean_owned_work", async () => {
+  const sentinel = join(temporaryDirectory(), "unrelated.txt");
+  await Bun.write(sentinel, INITIAL_CONTENT);
+  const controls = [
+    { command: COMMAND_SUCCESS, limits: SUCCESS_LIMITS, exit: 0, failure: undefined },
+    { command: COMMAND_FAILURE, limits: SUCCESS_LIMITS, exit: 2, failure: undefined },
+    { command: COMMAND_WAIT, limits: DEADLINE_LIMITS, exit: null, failure: "deadline exceeded" },
+    { command: COMMAND_OUTPUT, limits: OUTPUT_LIMITS, exit: null, failure: "output limit exceeded" },
+  ];
+  for (const control of controls) {
+    const run = await EvaluationRun.create();
+    const owned = run.ownership;
+    try {
+      const receipt = await run.execute([process.execPath, BUN_EVAL, control.command], [], control.limits);
+      run.verify(receipt);
+      if (control.failure === undefined) expect(receipt.result.exit_code).toBe(control.exit);
+      else expect(receipt.result.failure).toBe(control.failure);
+      expect(existsSync(receipt.receipt_directory)).toBe(false);
+      if (receipt.child_pid !== null) expect(() => process.kill(receipt.child_pid ?? 0, 0)).toThrow();
+    } finally { await run.close(); }
+    expect(existsSync(owned.workspace)).toBe(false);
+    expect(existsSync(owned.receipts)).toBe(false);
+    expect(await Bun.file(sentinel).text()).toBe(INITIAL_CONTENT);
+  }
+}, TEST_TIMEOUT_MS);
+
+test("handled_interruption_captures_failure_and_removes_owned_directories", async () => {
+  const run = await EvaluationRun.create();
+  const owned = run.ownership;
+  const controller = new AbortController();
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  try {
+    timer = setTimeout(() => controller.abort(), DEADLINE_LIMITS.timeout_ms);
+    const receipt = await run.execute([process.execPath, BUN_EVAL, COMMAND_WAIT], [], SUCCESS_LIMITS, controller.signal);
+    expect(receipt.result.failure).toBe("supervisor received SIGTERM");
+    expect(receipt.stdout).toBe("ready\n");
+    if (receipt.child_pid !== null) expect(() => process.kill(receipt.child_pid ?? 0, 0)).toThrow();
+  } finally {
+    if (timer !== undefined) clearTimeout(timer);
+    await run.close();
+  }
+  expect(existsSync(owned.workspace)).toBe(false);
+  expect(existsSync(owned.receipts)).toBe(false);
+}, TEST_TIMEOUT_MS);
+
+test("recovery_refuses_live_owner_or_altered_ownership_without_deleting_resources", async () => {
+  const run = await EvaluationRun.create();
+  const owned = run.ownership;
+  try {
+    await expect(recoverStaleOwnership(owned)).rejects.toThrow(OWNER_ERROR);
+    await expect(recoverStaleOwnership({ ...owned, token: crypto.randomUUID() })).rejects.toThrow(OWNER_ERROR);
+    expect(existsSync(owned.workspace)).toBe(true);
+    expect(existsSync(owned.receipts)).toBe(true);
+  } finally { await run.close(); }
+});
+
+test("closed_evaluation_refuses_new_commands_and_double_close_is_safe", async () => {
+  const run = await EvaluationRun.create();
+  await run.close();
+  await run.close();
+  await expect(run.execute([process.execPath, BUN_EVAL, COMMAND_SUCCESS], [], SUCCESS_LIMITS)).rejects.toThrow(EXECUTION_ERROR);
+});
+
+test("cancellation_during_request_preparation_prevents_any_child_start", async () => {
+  const run = await EvaluationRun.create();
+  const controller = new AbortController();
+  const write = Bun.write.bind(Bun);
+  let injections = 0;
+  try {
+    spyOn(Bun, "write").mockImplementation(async (destination, input, settings) => {
+      if (String(destination).endsWith("/request.json")) { injections++; controller.abort(); }
+      return Reflect.apply(write, Bun, [destination, input, settings]);
+    });
+    await expect(run.execute([process.execPath, BUN_EVAL, SENTINEL_COMMAND], [], SUCCESS_LIMITS, controller.signal)).rejects.toThrow(EXECUTION_ERROR);
+    expect(injections).toBe(1);
+    expect(existsSync(join(run.workspace, START_SENTINEL))).toBe(false);
+  } finally { mock.restore(); await run.close(); }
+});
+
+test("failed_process_inspection_never_authorizes_live_owner_cleanup", async () => {
+  const run = await EvaluationRun.create();
+  const owned = run.ownership;
+  const spawn = Bun.spawnSync.bind(Bun);
+  let inspections = 0;
+  try {
+    spyOn(Bun, "spawnSync").mockImplementation((argument: string[] | (SyncOptions & { cmd: string[] }), options?: SyncOptions) => {
+      const actual = Reflect.apply(spawn, Bun, Array.isArray(argument) ? [argument, options] : [argument]);
+      const argv = Array.isArray(argument) ? argument : argument.cmd;
+      if (Array.isArray(argv) && argv[0] === "ps") { inspections++; return { ...actual, exitCode: 1, stdout: Buffer.alloc(0) }; }
+      return actual;
+    });
+    await expect(recoverStaleOwnership(owned)).rejects.toThrow(OWNER_ERROR);
+    expect(inspections).toBe(1);
+    expect(existsSync(owned.workspace)).toBe(true);
+    expect(existsSync(owned.receipts)).toBe(true);
+  } finally { mock.restore(); await run.close(); }
+});
+
+test("failed_second_directory_allocation_cleans_the_real_first_directory", async () => {
+  const allocate = fileSystem.mkdtempSync;
+  const allocated: string[] = [];
+  let attempts = 0;
+  try {
+    spyOn(fileSystem, "mkdtempSync").mockImplementation((prefix, options) => {
+      attempts++;
+      if (attempts === 2) throw new Error(ALLOCATION_FAILURE);
+      const path = Reflect.apply(allocate, fileSystem, [prefix, options]);
+      if (typeof path === "string") allocated.push(path);
+      return path;
+    });
+    await expect(createOwnership()).rejects.toThrow(ALLOCATION_FAILURE);
+    expect(attempts).toBe(2);
+    expect(allocated.length).toBe(1);
+    for (const path of allocated) expect(existsSync(path)).toBe(false);
+  } finally {
+    mock.restore();
+    for (const path of allocated) if (existsSync(path)) fileSystem.rmSync(path, { recursive: true, force: true });
+  }
+});

package/src/judgment_tasks_findings.test.ts

@@ -0,0 +1,76 @@
+import { expect, test } from "bun:test";
+import { join, resolve } from "node:path";
+
+import { EvaluationRun } from "../evals/judgments/comparison/execution";
+import { checkWholeTask } from "../evals/judgments/comparison/hidden/assertions";
+import { CORRECT_COMMAND, CORRECT_HELPER, EXACT_TEST } from "../evals/judgments/comparison/task-fixtures";
+import { loadTasks, OWNER_CONTENT } from "../evals/judgments/comparison/tasks";
+import { TASK_FAMILY, TASK_FILES } from "../evals/judgments/comparison/types";
+
+const ROOT = resolve(import.meta.dir, "..");
+const TEST_TIMEOUT = 30_000;
+
+for (const family of [TASK_FAMILY.wiring, TASK_FAMILY.failure]) {
+  test(`hidden_checks_reject_inline_completion_without_required_${family}`, async () => {
+    const task = (await loadTasks(ROOT)).find((entry) => entry.id === family);
+    if (!task) throw new Error("Missing task family.");
+    const run = await EvaluationRun.create();
+    try {
+      await Bun.write(join(run.workspace, TASK_FILES.entry), CORRECT_COMMAND);
+      await Bun.write(join(run.workspace, TASK_FILES.helper), CORRECT_HELPER);
+      await Bun.write(join(run.workspace, TASK_FILES.linked), EXACT_TEST);
+      await Bun.write(join(run.workspace, TASK_FILES.owner), OWNER_CONTENT);
+      const result = await checkWholeTask(run, task, OWNER_CONTENT);
+      expect(result.passed).toBe(false);
+      expect(result.checks.some((check) => !check.passed)).toBe(true);
+    } finally { await run.close(); }
+  }, TEST_TIMEOUT);
+}
+
+test("consumer_hidden_evidence_refuses_missing_malformed_and_unrelated_checks", async () => {
+  const { validConsumerChecks } = await import("../evals/judgments/comparison/consumer-evidence");
+  for (const stdout of ["", "SyntaxError: unexpected token", "[]", JSON.stringify([{ name: "unrelated", passed: false }])]) {
+    expect(validConsumerChecks(stdout, true)).toBe(false);
+  }
+  expect(validConsumerChecks(JSON.stringify([{ name: "output-exists", passed: true }]), false)).toBe(true);
+  expect(validConsumerChecks(JSON.stringify([{ name: "output-exists", passed: false }]), false)).toBe(false);
+});
+
+test("consumer_package_identity_changes_when_same_version_source_changes", async () => {
+  const { packageIdentity } = await import("../evals/judgments/comparison/consumer-evidence");
+  const run = await EvaluationRun.create();
+  try {
+    const source = join(run.workspace, "entry.ts");
+    await Bun.write(source, "export const value = true;");
+    const before = await packageIdentity(run.workspace);
+    await Bun.write(source, "export const value = false;");
+    const after = await packageIdentity(run.workspace);
+    expect(after.digest).not.toBe(before.digest);
+    expect(after.files).toBe(before.files);
+  } finally { await run.close(); }
+});
+
+test("consumer_update_requires_guide_and_pin_without_unrelated_changes", async () => {
+  const { validConsumerDiff } = await import("../evals/judgments/comparison/consumer-evidence");
+  const guide = ".orly/docs/REST_API_DESIGN_GUIDELINES.md";
+  const config = ".orly/orly.json";
+  const expected = `${guide}\n${config}\n`;
+  expect(validConsumerDiff(expected)).toBe(true);
+  for (const changed of ["", guide, config, `${expected}AGENTS.md\n`, `${expected}${config}\n`,
+    `${config}\n${guide}\n`, `${guide}\n../.orly/orly.json\n`]) expect(validConsumerDiff(changed)).toBe(false);
+});
+
+test("consumer_report_refuses_to_overwrite_prior_attempts", async () => {
+  const run = await EvaluationRun.create();
+  const report = join(run.workspace, "retained.json");
+  const retained = "previous failed attempt";
+  try {
+    await Bun.write(report, retained);
+    const processResult = Bun.spawnSync([process.execPath, join(ROOT, "evals/judgments/comparison/consumer.ts"), run.workspace, run.workspace, report], {
+      stdout: "pipe", stderr: "pipe", timeout: TEST_TIMEOUT,
+    });
+    expect(processResult.exitCode).not.toBe(0);
+    expect(processResult.stderr.toString()).toContain("Report must be a new file");
+    expect(await Bun.file(report).text()).toBe(retained);
+  } finally { await run.close(); }
+});

package/src/judgment_tasks_recovery.test.ts

@@ -0,0 +1,119 @@
+import { afterEach, expect, test } from "bun:test";
+import { existsSync, symlinkSync, unlinkSync } from "node:fs";
+import { join, resolve } from "node:path";
+
+import { EvaluationRun, EXECUTION_ERROR, TASK_LIMITS, CONSUMER_SUITE_LIMITS } from "../evals/judgments/comparison/execution";
+import { recoverStaleOwnership, OWNER_ERROR } from "../evals/judgments/comparison/recovery";
+import { cleanupTemporaryDirectories, temporaryDirectory } from "./gates_test_support";
+
+const RECOVERY = resolve(import.meta.dir, "../evals/judgments/comparison/recovery.ts");
+const OWNER_RECEIPT = "owner.json";
+const SENTINEL_CONTENT = "unrelated owner bytes\n";
+const BUN_EVAL = "-e";
+const TIMEOUT = 10_000;
+const EXCESSIVE_TIMEOUT = 100_000;
+const OUTPUT_LIMIT = 1024;
+const RECOVERY_POLL_MS = 10;
+const SUCCESS_COMMAND = [process.execPath, BUN_EVAL, "process.stdout.write('done')"];
+
+afterEach(cleanupTemporaryDirectories);
+
+test("restart_recovers_verified_stale_resources_without_completion_credit", async () => {
+  const root = temporaryDirectory();
+  const marker = join(root, OWNER_RECEIPT);
+  const sentinel = join(root, "unrelated.txt");
+  await Bun.write(sentinel, SENTINEL_CONTENT);
+  const script = `import {createOwnership} from ${JSON.stringify(RECOVERY)};await Bun.write(${JSON.stringify(marker)},JSON.stringify(await createOwnership()));`;
+  const owner = Bun.spawn([process.execPath, BUN_EVAL, script], { stdout: "ignore", stderr: "pipe" });
+  const timeout = setTimeout(() => owner.kill(), TIMEOUT);
+  try {
+    expect(await owner.exited).toBe(0);
+    const ownership: unknown = await Bun.file(marker).json();
+    expect(await recoverStaleOwnership(ownership)).toEqual({ recovered: true, completion: false });
+    expect(await Bun.file(sentinel).text()).toBe(SENTINEL_CONTENT);
+    await expect(recoverStaleOwnership(ownership)).rejects.toThrow();
+  } finally { clearTimeout(timeout); owner.kill(); await owner.exited; }
+}, TIMEOUT);
+
+test("foreign_paths_and_symlinked_markers_cannot_authorize_cleanup", async () => {
+  const run = await EvaluationRun.create();
+  const foreign = temporaryDirectory();
+  const sentinel = join(foreign, "unrelated.txt");
+  await Bun.write(sentinel, SENTINEL_CONTENT);
+  try {
+    await expect(recoverStaleOwnership({ ...run.ownership, workspace: foreign })).rejects.toThrow(OWNER_ERROR);
+    symlinkSync(foreign, join(run.ownership.receipts, "foreign"));
+    expect(await Bun.file(sentinel).text()).toBe(SENTINEL_CONTENT);
+    expect(existsSync(run.workspace)).toBe(true);
+  } finally { unlinkSync(join(run.ownership.receipts, "foreign")); await run.close(); }
+});
+
+test("invalid_limits_refuse_before_starting_any_process", async () => {
+  const run = await EvaluationRun.create();
+  try {
+    for (const timeout_ms of [0, -1, Number.NaN, CONSUMER_SUITE_LIMITS.timeout_ms, EXCESSIVE_TIMEOUT]) {
+      await expect(run.execute(SUCCESS_COMMAND, [], { timeout_ms, output_bytes: OUTPUT_LIMIT })).rejects.toThrow(EXECUTION_ERROR);
+    }
+  } finally { await run.close(); }
+});
+
+test("only_the_fixed_consumer_suite_command_receives_the_longer_budget", async () => {
+  const run = await EvaluationRun.create();
+  try {
+    await Bun.write(join(run.workspace, "Makefile"), "test-unit-design-system:\n\t@printf verified\n");
+    const suite = await run.executeConsumerSuite(run.workspace, []);
+    run.verify(suite);
+    expect(suite.command).toEqual(["make", "-C", run.workspace, "test-unit-design-system"]);
+    expect(suite.limits).toEqual(CONSUMER_SUITE_LIMITS);
+    expect(suite.result.exit_code).toBe(0);
+    expect(suite.stdout).toContain("verified");
+    expect((await run.execute(SUCCESS_COMMAND, [])).limits).toEqual(TASK_LIMITS);
+  } finally { await run.close(); }
+});
+
+test("one_owner_refuses_concurrent_execution_and_close_until_work_finishes", async () => {
+  const run = await EvaluationRun.create();
+  try {
+    const pending = run.execute(SUCCESS_COMMAND, []);
+    await expect(run.execute(SUCCESS_COMMAND, [])).rejects.toThrow(EXECUTION_ERROR);
+    await expect(run.close()).rejects.toThrow(EXECUTION_ERROR);
+    expect((await pending).result.exit_code).toBe(0);
+  } finally { await run.close(); }
+});
+
+test("recovery_refuses_live_descendants_after_the_group_leader_exits", async () => {
+  const root = temporaryDirectory();
+  const marker = join(root, OWNER_RECEIPT);
+  const groupMarker = join(root, "group.txt");
+  const script = `import {createOwnership} from ${JSON.stringify(RECOVERY)};
+const owner=await createOwnership();
+const leader=Bun.spawn(['sh','-c','sleep 30 >/dev/null 2>&1 &'],{detached:true,stdout:'ignore',stderr:'ignore'});
+await Bun.write(owner.receipts+'/command/child.pid',String(leader.pid));
+await Bun.write(${JSON.stringify(groupMarker)},String(leader.pid));
+await leader.exited;
+await Bun.write(${JSON.stringify(marker)},JSON.stringify(owner));`;
+  const owner = Bun.spawn([process.execPath, BUN_EVAL, script], { stdout: "ignore", stderr: "pipe" });
+  const timeout = setTimeout(() => owner.kill(), TIMEOUT);
+  try {
+    expect(await owner.exited).toBe(0);
+    const ownership: unknown = await Bun.file(marker).json();
+    const group = Number(await Bun.file(groupMarker).text());
+    process.kill(-group, 0);
+    await expect(recoverStaleOwnership(ownership)).rejects.toThrow(OWNER_ERROR);
+  } finally {
+    clearTimeout(timeout);
+    owner.kill();
+    await owner.exited;
+    const group = Number(await Bun.file(groupMarker).text());
+    try { process.kill(-group, "SIGKILL"); } catch { /* Already exited. */ }
+    const ownership: unknown = await Bun.file(marker).json();
+    const deadline = Date.now() + TIMEOUT;
+    while (true) {
+      try { await recoverStaleOwnership(ownership); break; }
+      catch (error) {
+        if (Date.now() >= deadline) throw error;
+        await Bun.sleep(RECOVERY_POLL_MS);
+      }
+    }
+  }
+}, TIMEOUT * 2);

package/src/judgments/transport.ts

@@ -1,4 +1,4 @@
-import type { Prepared } from "./types";
+import type { ProviderInput } from "./types";
 
 import { OrlyError } from "../model";
 import { ENDPOINT, MAX_RESPONSE_BYTES, REQUEST_TIMEOUT_MS } from "./constants";
@@ -7,7 +7,7 @@ import { parseReply, type Reply } from "./wire";
 
 export type Fetch = (request: Request) => Promise<Response>;
 
-export async function requestAdvice(prepared: Prepared, credential: string, fetcher: Fetch = fetch, timeoutMs = REQUEST_TIMEOUT_MS): Promise<Reply> {
+export async function requestAdvice(prepared: ProviderInput, credential: string, fetcher: Fetch = fetch, timeoutMs = REQUEST_TIMEOUT_MS): Promise<Reply> {
   const controller = new AbortController();
   const timer = setTimeout(() => controller.abort(), timeoutMs);
   let response: Response | undefined;

package/src/judgments/types.ts

@@ -34,6 +34,7 @@ export type QuestionId = Item["question"];
 export type ProviderQuestion =
   | { type: "noul"; instructions: string; criteria: { true: string; false: string } }
   | { type: "choice"; instructions: string; criteria: Record<string, string> };
+export type ProviderInput = { request: string; definition: { question: ProviderQuestion } };
 export type Definition = {
   stage: Stage;
   requiredRoles: readonly Role[];

package/src/judgments/wire.ts

@@ -1,4 +1,4 @@
-import type { Prepared, ProviderQuestion } from "./types";
+import type { ProviderInput, ProviderQuestion } from "./types";
 
 import { z } from "zod";
 
@@ -15,7 +15,7 @@ export const replySchema = z.object({ model: z.literal(MODEL), answers: z.record
 export type Reply = z.infer<typeof replySchema>;
 export type Answer = z.infer<typeof answerSchema>;
 
-export function parseReply(value: unknown, prepared: Prepared): Reply {
+export function parseReply(value: unknown, prepared: ProviderInput): Reply {
   const parsed = replySchema.safeParse(value);
   if (!parsed.success) throw new OrlyError("Provider reply does not match the pinned model and answer schema.");
   const reply = parsed.data;

package/src/review_poll.test.ts

@@ -30,7 +30,11 @@ else if(args[0]==="pr" && args[1]==="view") {
   else { const prior=(await Bun.file(process.env.POLL_CALLS).text()).split("\\n").filter(x=>x.includes("headRefOid")).length; console.log((process.env.POLL_MODE==="changed" && prior>1) || (process.env.POLL_MODE==="changed-check" && prior>2) ? "changed-revision" : process.env.POLL_HEAD); }
 } else if(args[0]==="pr" && args[1]==="checks") {
   if(process.env.POLL_STATE==="fetch-failed"){console.error("temporary fetch refusal");process.exit(2);}
-  console.log(JSON.stringify(process.env.POLL_STATE==="empty" ? [] : [{name:"tests",state:"SUCCESS",bucket:process.env.POLL_STATE==="failed" ? "fail" : "pass",link:"https://example.invalid/check"}]));
+  const state=process.env.POLL_STATE;
+  const bucket=state==="failed" ? "fail" : state==="pending" ? "pending" : "pass";
+  console.log(JSON.stringify(state==="empty" ? [] : state==="malformed-check" ? [{name:"tests"}] : [{name:"tests",state:bucket==="pass" ? "SUCCESS" : bucket==="fail" ? "FAILURE" : "IN_PROGRESS",bucket,link:"https://example.invalid/check"}]));
+  if(state==="failed") process.exit(1);
+  if(state==="pending") process.exit(8);
 } else if(args[0]==="api") {
   if(process.env.POLL_MODE==="fetch-failed" && args[1].includes("/issues/")){console.error("temporary summary refusal");process.exit(2);}
   const revision=process.env.POLL_MODE==="stale" ? "old-revision" : process.env.POLL_HEAD;
@@ -75,7 +79,7 @@ for (const mode of ["fetch-failed", "malformed", "changed", "changed-check", "st
   });
 }
 
-for (const state of ["empty", "fetch-failed", "failed"]) {
+for (const state of ["empty", "fetch-failed", "failed", "pending", "malformed-check"]) {
   test(`${state} check discovery stays incomplete`, () => {
     const result = poll("complete", state);
     expect(result.code).not.toBe(0);
@@ -84,6 +88,17 @@ for (const state of ["empty", "fetch-failed", "failed"]) {
   });
 }
 
+for (const state of ["failed", "pending"]) {
+  test(`${state} GitHub checks retain job details despite nonzero status`, () => {
+    const result = poll("complete", state);
+    expect(result.code).not.toBe(0);
+    expect(result.output).toContain("tests |");
+    expect(result.output).toContain("https://example.invalid/check");
+    expect(result.output).toContain(state === "failed" ? "FAILURE | fail" : "IN_PROGRESS | pending");
+    expect(result.output).not.toContain("check fetch incomplete");
+  });
+}
+
 function gitlabPoll(mode = "complete", state = "green") {
   const root = newRepository();
   const tools = join(root, "tools");
@@ -105,7 +120,7 @@ else if(args[0]==="mr") {
   if(process.env.POLL_MODE==="fetch-failed" && endpoint.endsWith("/notes")){console.error("temporary summary refusal");process.exit(2);}
   if(endpoint.endsWith("/pipelines")) {
     if(process.env.POLL_STATE==="fetch-failed"){console.error("temporary pipeline refusal");process.exit(2);}
-    console.log(JSON.stringify(process.env.POLL_STATE==="empty" ? [] : [{id:3,sha:process.env.POLL_HEAD,status:process.env.POLL_STATE==="failed" ? "failed" : "success"}]));
+    console.log(JSON.stringify(process.env.POLL_STATE==="empty" ? [] : [{id:3,sha:process.env.POLL_HEAD,status:process.env.POLL_STATE==="failed" ? "failed" : process.env.POLL_STATE==="pending" ? "running" : "success",web_url:"https://example.invalid/pipeline"}]));
   } else if(process.env.POLL_MODE==="malformed") console.log(JSON.stringify({message:"not review evidence"}));
   else {
     const note=id=>({id,author:{username:"greptile"},body:id===1 ? "first-page finding" : "second-page finding",type:null});
@@ -148,7 +163,7 @@ for (const mode of ["fetch-failed", "malformed", "changed", "changed-check"]) {
   });
 }
 
-for (const state of ["empty", "fetch-failed", "failed"]) {
+for (const state of ["empty", "fetch-failed", "failed", "pending"]) {
   test(`GitLab ${state} pipeline discovery stays incomplete`, () => {
     const result = gitlabPoll("complete", state);
     expect(result.code).not.toBe(0);
@@ -156,3 +171,12 @@ for (const state of ["empty", "fetch-failed", "failed"]) {
     expect(result.output).not.toContain("ci=green");
   });
 }
+
+for (const state of ["failed", "pending"]) {
+  test(`GitLab ${state} pipeline retains its state and link`, () => {
+    const result = gitlabPoll("complete", state);
+    expect(result.code).not.toBe(0);
+    expect(result.output).toContain(`pipeline 3 | ${state === "failed" ? "failed" : "running"}`);
+    expect(result.output).toContain("https://example.invalid/pipeline");
+  });
+}