@agentsfleet/orly 0.12.0 → 0.13.0

package/README.md

@@ -42,22 +42,30 @@ Works with Claude Code, Codex, OpenCode, and Amp.
 
 ---
 
-## 0.12.0: migration verification
-
-0.12.0 is a local release candidate. Published 0.11.0 already provides the
-`.orly/` layout and the six existing `orly judge` questions.
-This release focuses on package verification, migration evidence and clearer setup guidance; it adds no new runtime judgment question.
-
-The `agentsfleet` trial upgraded a dedicated worktree from 0.10.14 using the packed candidate.
-Owner files, hooks and verification commands survived; a second update wrote nothing.
-See the [release report](evals/release/release-report.md) for exact proof and remaining release checks.
-
-Jev, TypeSafe's bounded judgment model, supplies advice only when requested.
-Existing deterministic checks retain control of gate results.
-The comparison evaluator is parked for proposed 0.13 work; model accuracy and candidate benefits remain unmeasured.
-See the [judgment reference](docs/JUDGMENTS.md) for evidence selection, offline replay and explicit source upload.
-
-The install commands below fetch the published package. They do not select this unpublished candidate.
+## 0.13: measured judgment comparison
+
+The 0.13.0 source candidate adds a deterministic examiner for unsupported completion claims.
+It freezes the six runtime questions and evaluates five proposed checks separately.
+Those checks cover obligations, evidence, production wiring, failure tests and finding resolution.
+
+From a source checkout, run `bun evals/judgments/comparison/run.ts --check` to validate the frozen inputs.
+Expected output is JSON with six baseline questions, five candidates and `passed: true`.
+Run `bun evals/judgments/comparison/run.ts --offline` for whole-task exercises with hidden behavioral checks.
+A successful run reports `passed: true`; seeded false completions remain failed task outcomes.
+
+A separate explicit `jev-run.ts --live` command measures TypeSafe Jev answers against the frozen expectations.
+Its retained report lists every measured answer, grading match, disagreement and unavailable response.
+Replay reproduces every result with zero new requests. Autonomous agent improvement remains unmeasured.
+All five candidates remain evaluation-only; uncertain and failed replies stay in the report.
+An additional rehearsal uses the packed candidate and actual `agentsfleet` source in an isolated worktree.
+See the [evaluation design](docs/architecture/judgment-evaluation.md) for commands, evidence and limits.
+
+The `.orly/` layout, six runtime questions and 0.12.1 audit and check-diagnostic fixes remain unchanged.
+The [0.12 report](evals/release/release-report.md) retains the earlier package and consumer migration evidence.
+TypeSafe's Jev model supplies runtime advice through explicit `orly judge` requests; contributor measurement uses the separate live evaluator.
+
+See [releases](https://github.com/agentsfleet/orly/releases) for published versions.
+The install commands below fetch the published package; 0.13.0 publication is a separate owner action.
 
 ---
 

package/audits/deinit-pairs.sh

@@ -48,7 +48,7 @@ set -euo pipefail
 
 source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/scope.sh"
 audit_scope_init --all "$@"
-audit_index_snapshot "$@"
+audit_index_snapshot '*.zig'
 
 MODE="${1:-${SCOPE:-all}}"
 ROOT="$(git rev-parse --show-toplevel)"

package/audits/design-tokens.sh

@@ -39,7 +39,7 @@ set -euo pipefail
 
 source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/scope.sh"
 audit_scope_init --all "$@"
-audit_index_snapshot "$@"
+audit_index_snapshot '*.tsx' '*.jsx' 'ui/packages/app/*' 'ui/packages/website/*' 'ui/packages/design-system/*'
 
 MODE="$AUDIT_MODE"
 ROOT="$(git rev-parse --show-toplevel)"

package/audits/error-codes.sh

@@ -51,7 +51,7 @@ set -euo pipefail
 
 source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/scope.sh"
 audit_scope_init --all "$@"
-audit_index_snapshot "$@"
+audit_index_snapshot
 
 MODE="$AUDIT_MODE"
 ROOT="$(git rev-parse --show-toplevel)"

package/audits/logging.sh

@@ -43,7 +43,7 @@ set -euo pipefail
 
 source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/scope.sh"
 audit_scope_init --all "$@"
-audit_index_snapshot "$@"
+audit_index_snapshot '*.zig' '*.rs' 'agentsfleet/src/*.js' 'agentsfleet/src/*.jsx' 'agentsfleet/src/*.ts' 'agentsfleet/src/*.tsx'
 
 MODE="--all"
 STRICT=0

package/audits/rust-error.sh

@@ -44,7 +44,7 @@ set -euo pipefail
 
 source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/scope.sh"
 audit_scope_init --all "$@"
-audit_index_snapshot "$@"
+audit_index_snapshot '*.rs'
 
 MODE_STAGED="--staged"
 MODE_ALL="--all"

package/audits/scope.sh

@@ -37,7 +37,7 @@ audit_scope_init() {
 # the index copy and checked-out files belong to this invocation alone.
 audit_index_snapshot() {
   [ "$AUDIT_MODE" = --staged ] || return 0
-  local gitdir index
+  local gitdir index changed=0
   AUDIT_SNAPSHOT_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/orly-index.XXXXXX")" || return 1
   trap 'rm -rf -- "$AUDIT_SNAPSHOT_ROOT"' EXIT
   trap 'exit 130' INT
@@ -46,6 +46,14 @@ audit_index_snapshot() {
   index="$(git rev-parse --git-path index)" || exit 1
   cp "$index" "$AUDIT_SNAPSHOT_ROOT/index" || exit 1
   export GIT_DIR="$gitdir" GIT_INDEX_FILE="$AUDIT_SNAPSHOT_ROOT/index" GIT_OPTIONAL_LOCKS=0
+  if [ "$#" -gt 0 ]; then
+    git diff --cached --quiet --diff-filter=ACMRT -- "$@" || changed=$?
+    case "$changed" in
+      0) return 0 ;;
+      1) ;;
+      *) printf 'cannot select staged audit files (git exit %s)\n' "$changed" >&2; exit 2 ;;
+    esac
+  fi
   export GIT_WORK_TREE="$AUDIT_SNAPSHOT_ROOT/tree"
   mkdir "$GIT_WORK_TREE" || exit 1
   git checkout-index --all --prefix="$GIT_WORK_TREE/" || exit 1

package/audits/sql-mod.sh

@@ -25,7 +25,7 @@ set -euo pipefail
 
 source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/scope.sh"
 audit_scope_init --all "$@"
-audit_index_snapshot "$@"
+audit_index_snapshot '*.zig'
 
 MODE="${1:-${SCOPE:---all}}"
 ROOT="$(git rev-parse --show-toplevel)"

package/audits/ufs.sh

@@ -55,7 +55,7 @@ set -euo pipefail
 
 source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/scope.sh"
 audit_scope_init --all "$@"
-audit_index_snapshot "$@"
+audit_index_snapshot '*.zig' '*.ts' '*.tsx' '*.js' '*.jsx' '*.rs' '*.go'
 
 # Default: full-codebase scan (`--all` is an explicit alias). `--staged` is the
 # pre-commit lens — it narrows the per-file checks to `git diff --cached`. The

package/dispatch/lib.sh

@@ -125,7 +125,7 @@ dispatch_managed_paths() {
 dispatch_resolve_files() {
   DISPATCH_FILES=()
   audit_scope_init --all "$@"
-  audit_index_snapshot "$@"
+  audit_index_snapshot
   DISPATCH_STAGED=0
   if [ "${1:-}" = "--staged" ]; then
     DISPATCH_STAGED=1

package/docs/CHANGELOG.md

@@ -1,6 +1,26 @@
 # Changelog
 
-## 0.12.0 — unreleased
+## 0.13.0 — 2026-10-07
+
+A source-checkout examiner checks unsupported completion claims and grades actual TypeSafe Jev answers against frozen expectations.
+
+- **Comparison** — freezes six runtime questions and evaluates five proposed checks with separately validated expected answers.
+- **Task evidence** — hidden behavioral checks reject seeded false completions; reports retain failed and missing attempts.
+- **Measurement** — retains typed Jev replies, uncertainty, failures and per-example grades; replay sends no new requests.
+- **OpenAPI guidance** — names the two surviving parity tests and makes build regeneration plus a clean diff an explicit verify step.
+- **Adoption** — keeps candidates evaluation-only because independent provenance and paired-task improvement remain unproved.
+
+The installed `.orly/` layout and 0.12.1 fixes remain unchanged.
+
+## 0.12.1
+
+Staged audits avoid copying repository files when their filename scope is empty.
+
+- **Audit preflight** — selects filenames from a copied index before materialising repository files; relevant audits retain coherent staged reads.
+- **Check diagnostics** — prints hosted job or pipeline states and links before reporting pending or failed checks.
+- **Release status** — corrects documentation that still described published 0.12.0 as an unpublished candidate.
+
+## 0.12.0 — 2026-10-07
 
 Setup guidance now distinguishes the published package from the local release candidate.
 

package/docs/JUDGMENTS.md

@@ -2,7 +2,7 @@
 type: reference
 audience: contributor
 verified: 2026-10-07
-product_version: 0.12.0
+product_version: 0.13.0
 executable: false
 ---
 
@@ -10,8 +10,9 @@ executable: false
 
 ## Synopsis
 
-The 0.12.0 candidate retains the six questions shipped in 0.11.0.
-The comparison evaluator is parked for proposed 0.13 work; this release makes no new model-quality claim.
+The 0.13.0 candidate retains the six questions shipped in 0.11.0.
+The contributor [comparison examiner](architecture/judgment-evaluation.md) tests five evaluation-only additions with offline controls and explicitly requested TypeSafe Jev measurements.
+Its results establish examiner behavior; model quality and autonomous agent improvement remain unmeasured.
 
 `orly judge` asks one bounded question about evidence you select.
 It accompanies planning, verification, review and documentation.

package/docs/ORLY_ARCHITECTURE.md

@@ -84,6 +84,13 @@ Every criterion is mechanical — it reads an exit code or a file. Claims that
 cannot be proven that way stay prose and are graded by the spec's rubric; they
 never become fake criteria.
 
+Filename-scoped staged audits copy Git's index before selecting relevant paths.
+An empty selection skips the repository checkout; matching paths still trigger
+a private checkout, dependency reads and indexed-link checks. Each audit owns
+its temporary files and cleanup. Audits with unconditional registry checks
+retain a complete checkout. Separate relevant audits retain separate snapshots;
+this optimization does not introduce a shared snapshot cache.
+
 Four behaviours worth knowing:
 
 - **No spec, no problem — but closing is not escaping.** Spec criteria skip

package/docs/REST_API_DESIGN_GUIDELINES.md

@@ -452,7 +452,7 @@ table decides the paths and methods, `#[utoipa::path]` beside each handler
 carries the prose and the status codes, and the `afd_wire` types carry the
 schemas. The `public/openapi/` YAML tree this section used to describe was
 retired with the Zig daemon; a hand-edit to the JSON is reverted by the next
-regeneration and fails a test before it gets that far.
+regeneration.
 
 ### Adding, renaming, or removing an endpoint
 
@@ -468,15 +468,24 @@ regeneration and fails a test before it gets that far.
      --bin agentsfleetd -- --no-banner openapi > ../public/openapi.json
    ```
 
-**Parity is mechanical, not reviewer-enforced.** Three tests grade it, and the
+**Parity is mechanical, not reviewer-enforced.** Two tests grade it, and the
 first is the one that used to be a review obligation:
 
 | Test | What it refuses |
 |---|---|
 | `test_coverage_gate_rust_source` | a served route with no annotation, or an annotation for a route nobody mounts — named with its method and direction |
-| `test_openapi_build_is_the_source` | a committed artifact that is not what the build emits |
 | `test_documented_codes_match_refusals` | an operation that omits a refusal its guard or scope rung guarantees |
 
+No test checks that `public/openapi.json` matches the build. Regeneration is a
+verify step: from the repository root, run the generator, then check for drift.
+The subshell keeps the diff command at the repository root.
+
+```bash
+(cd rustd && cargo run -q -p agentsfleetd --features openapi \
+  --bin agentsfleetd -- --no-banner openapi > ../public/openapi.json)
+git diff --exit-code public/openapi.json
+```
+
 The prose is graded too: `scripts/check_documentation_rules.py` reads the
 generated document, so a description that breaks the wording rules fails
 `make lint-all`. Fix it at the annotation — or, for a schema description, at the

package/llms.txt

@@ -12,11 +12,19 @@ Package: `@agentsfleet/orly` (npm)
 Source: https://github.com/agentsfleet/orly
 License: Massachusetts Institute of Technology (MIT)
 
-This checkout prepares the unpublished 0.12.0 candidate. Published 0.11.0 already
-contains the same runtime commands and `.orly/` layout. The `bunx` commands below
-fetch the published package. Use the installed local tarball for candidate checks.
-The comparison evaluator is parked for proposed 0.13 work; do not claim measured
-judgment improvement or enable new questions from its unfinished results.
+0.12.0 shipped migration verification; the runtime commands and `.orly/` layout
+originally shipped in 0.11.0. The 0.12.1 patch avoids irrelevant staged audit
+snapshots and preserves hosted-check diagnostics. The `bunx` commands below
+fetch the published package; available versions are listed at
+https://github.com/agentsfleet/orly/releases. Use an installed local tarball for candidate checks.
+The 0.13.0 source candidate adds contributor-only offline controls and a separate explicit TypeSafe Jev measurement command.
+Run `bun evals/judgments/comparison/run.ts --check` or `--offline` from source.
+The separate `jev-run.ts --live` collects actual model replies; `--replay` checks saved answers with no new requests.
+See `docs/architecture/judgment-evaluation.md` for required inputs and measured limitations.
+Expected output is JSON with `passed: true`; deliberate false completions stay failed.
+All five proposed checks remain evaluation-only. Do not infer model accuracy or
+agent improvement from scripted controls. Live model runs, publication, merge
+and live consumer upgrades need subsequent owner instruction.
 
 ---
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@agentsfleet/orly",
-  "version": "0.12.0",
+  "version": "0.13.0",
   "description": "Repository-scoped engineering harness: renders agent rules, materialises the gates that enforce them, and proves the PR boundary \u2014 for any coding agent.",
   "license": "MIT",
   "type": "module",

package/skills/orly-babysit-prs/SKILL.md

@@ -93,8 +93,17 @@ Each cycle, after the review-thread walk, poll CI and act by cause:
 
 ```bash
 set -euo pipefail
-CHECKS=$(gh pr checks "$PR_NUMBER" --json name,state,bucket,link) \
-  || { echo "BABYSIT: check fetch incomplete; reset quiet polls" >&2; exit 1; }
+CHECK_STATUS=0
+CHECKS=$(gh pr checks "$PR_NUMBER" --json name,state,bucket,link) || CHECK_STATUS=$?
+printf '%s' "$CHECKS" | jq -e 'type == "array" and length > 0 and all(.[];
+  (.name | type == "string") and (.state | type == "string") and
+  (.link | type == "string") and (.bucket | IN("pass", "fail", "pending", "skipping", "cancel")))' >/dev/null \
+  || { printf 'BABYSIT: invalid check response (exit %s); reset quiet polls\n%s\n' "$CHECK_STATUS" "$CHECKS" >&2; exit 1; }
+printf '%s' "$CHECKS" | jq -r '.[] | "\(.name) | \(.state) | \(.bucket) | \(.link)"'
+case "$CHECK_STATUS" in
+  0|1|8) ;; # GitHub reports failed and pending checks with nonzero statuses.
+  *) printf 'BABYSIT: check fetch incomplete (exit %s); reset quiet polls\n' "$CHECK_STATUS" >&2; exit 1 ;;
+esac
 printf '%s' "$CHECKS" | jq -e 'type == "array" and length > 0 and all(.[]; .bucket == "pass" or .bucket == "skipping")' >/dev/null \
   || { echo "BABYSIT: checks empty or not green; reset quiet polls" >&2; exit 1; }
 CHECK_HEAD=$(gh pr view "$PR_NUMBER" --json headRefOid --jq '.headRefOid')
@@ -107,8 +116,14 @@ echo "BABYSIT: ci=green revision=$CHECK_HEAD"
 
 ```bash
 set -euo pipefail
-PIPELINES=$(glab api "projects/$PROJ_ENC/merge_requests/$MR_IID/pipelines" --paginate | jq -sc 'add') \
-  || { echo "BABYSIT: pipeline fetch incomplete; reset quiet polls" >&2; exit 1; }
+PIPELINE_STATUS=0
+PIPELINES=$(glab api "projects/$PROJ_ENC/merge_requests/$MR_IID/pipelines" --paginate | jq -sc 'add') || PIPELINE_STATUS=$?
+printf '%s' "$PIPELINES" | jq -e 'type == "array" and length > 0 and all(.[];
+  (.id | type == "number") and (.sha | type == "string") and (.status | type == "string"))' >/dev/null \
+  || { printf 'BABYSIT: invalid pipeline response (exit %s); reset quiet polls\n%s\n' "$PIPELINE_STATUS" "$PIPELINES" >&2; exit 1; }
+printf '%s' "$PIPELINES" | jq -r '.[] | "pipeline \(.id) | \(.status) | \(.web_url // "URL not reported")"'
+[ "$PIPELINE_STATUS" -eq 0 ] \
+  || { printf 'BABYSIT: pipeline fetch incomplete (exit %s); reset quiet polls\n' "$PIPELINE_STATUS" >&2; exit 1; }
 printf '%s' "$PIPELINES" | jq -e --arg revision "$POLL_HEAD" \
   '[.[] | select(.sha == $revision)] | sort_by(.id) | last | .status == "success"' >/dev/null \
   || { echo "BABYSIT: current revision has no successful pipeline; reset quiet polls" >&2; exit 1; }

package/src/judgment_comparison.test.ts

@@ -0,0 +1,138 @@
+import { afterEach, expect, test } from "bun:test";
+import { join } from "node:path";
+
+import catalog from "../evals/judgments/comparison/catalog.json";
+import { CASE_CLASS, CHECK_ERROR, SPLIT } from "../evals/judgments/comparison/types";
+import { validateCases, validateCatalog } from "../evals/judgments/comparison/validate";
+import { OrlyError } from "./model";
+import { MODEL, QUESTIONS, ROLE } from "./judgments/constants";
+import { digest } from "./judgments/files";
+import { cleanupTemporaryDirectories, temporaryDirectory } from "./gates_test_support";
+
+const OTHER_MODEL = "different-model";
+const DIGEST_HEX_LENGTH = 64;
+const WRONG_DIGEST = "0".repeat(DIGEST_HEX_LENGTH);
+const SOURCE_PATH = "requirement.md";
+const SOURCE_CONTENT = "Input must include a locale. The caller supplies the locale before formatting.\n";
+
+afterEach(cleanupTemporaryDirectories);
+
+async function caseFixture() {
+  const root = temporaryDirectory();
+  await Bun.write(join(root, SOURCE_PATH), SOURCE_CONTENT);
+  const entry = {
+    id: "locale-prerequisite", question: QUESTIONS[0], origin: "locale-source", split: SPLIT.development,
+    classification: CASE_CLASS.healthy, author: "fixture-author", requirement: "Name the required locale before formatting.",
+    evidence: [{ role: ROLE.spec, content: SOURCE_CONTENT.trim(), source: { path: SOURCE_PATH, digest: digest(SOURCE_CONTENT) } }],
+  };
+  return { root, entry };
+}
+
+test("comparison_preserves_runtime_baseline", () => {
+  const parsed = validateCatalog(catalog);
+  expect(parsed.model).toBe(MODEL);
+  expect(parsed.baseline.map((entry) => entry.question)).toEqual([...QUESTIONS]);
+  expect(parsed.candidates.map((entry) => entry.question)).toEqual([
+    "plan.obligation_coverage", "evidence.sufficiency", "verify.production_wiring",
+    "review.failure_coverage", "review.finding_resolution",
+  ]);
+});
+
+test("comparison_rejects_changed_model", () => {
+  expect(() => validateCatalog({ ...catalog, model: OTHER_MODEL })).toThrow(new OrlyError(CHECK_ERROR.schema));
+});
+
+test("comparison_rejects_changed_baseline_definition", () => {
+  const changed = structuredClone(catalog);
+  for (const entry of changed.baseline) entry.definition_digest = WRONG_DIGEST;
+  expect(() => validateCatalog(changed)).toThrow(new OrlyError(CHECK_ERROR.baseline));
+});
+
+test("comparison_rejects_missing_baseline_question", () => {
+  expect(() => validateCatalog({ ...catalog, baseline: catalog.baseline.slice(1) })).toThrow(new OrlyError(CHECK_ERROR.schema));
+});
+
+test("comparison_rejects_duplicate_baseline_question", () => {
+  const first = catalog.baseline.at(0);
+  expect(first).toBeDefined();
+  expect(() => validateCatalog({ ...catalog, baseline: catalog.baseline.map(() => first) })).toThrow(new OrlyError(CHECK_ERROR.baseline));
+});
+
+test("candidate_inputs_are_complete", () => {
+  const parsed = validateCatalog(catalog);
+  for (const candidate of parsed.candidates) {
+    expect(candidate.required_roles.length).toBeGreaterThan(0);
+    expect(Object.keys(candidate.criteria).sort()).toEqual(["ambiguous", "defective", "insufficient", "supported"]);
+  }
+});
+
+test("comparison_rejects_missing_candidate", () => {
+  expect(() => validateCatalog({ ...catalog, candidates: catalog.candidates.slice(1) })).toThrow(new OrlyError(CHECK_ERROR.schema));
+});
+
+test("comparison_rejects_duplicate_candidates", () => {
+  const first = catalog.candidates.at(0);
+  expect(first).toBeDefined();
+  expect(() => validateCatalog({ ...catalog, candidates: catalog.candidates.map(() => first) })).toThrow(new OrlyError(CHECK_ERROR.candidates));
+});
+
+test("comparison_rejects_missing_evidence_roles", () => {
+  const candidates = catalog.candidates.map((entry) => ({ ...entry, required_roles: entry.required_roles.slice(1) }));
+  expect(() => validateCatalog({ ...catalog, candidates })).toThrow(new OrlyError(CHECK_ERROR.roles));
+});
+
+test("comparison_rejects_empty_question_instructions", () => {
+  const candidates = catalog.candidates.map((entry) => ({ ...entry, instructions: "" }));
+  expect(() => validateCatalog({ ...catalog, candidates })).toThrow(new OrlyError(CHECK_ERROR.schema));
+});
+
+test("comparison_rejects_unbounded_answer_fields", () => {
+  const candidates = catalog.candidates.map((entry) => ({ ...entry, criteria: { ...entry.criteria, run_command: "true" } }));
+  expect(() => validateCatalog({ ...catalog, candidates })).toThrow(new OrlyError(CHECK_ERROR.schema));
+});
+
+test("comparison_rejects_unknown_catalog_fields", () => {
+  expect(() => validateCatalog({ ...catalog, enable_remote: true })).toThrow(new OrlyError(CHECK_ERROR.schema));
+});
+
+test("comparison_rejects_invalid_provenance", async () => {
+  const { root, entry } = await caseFixture();
+  const checked = validateCatalog(catalog);
+  expect(await validateCases(root, checked, [entry])).toEqual([entry]);
+  const changed = { ...entry, evidence: entry.evidence.map((item) => ({ ...item, source: { ...item.source, digest: WRONG_DIGEST } })) };
+  await expect(validateCases(root, checked, [changed])).rejects.toThrow(new OrlyError(CHECK_ERROR.source));
+});
+
+test("comparison_rejects_duplicate_cases", async () => {
+  const { root, entry } = await caseFixture();
+  await expect(validateCases(root, validateCatalog(catalog), [entry, entry])).rejects.toThrow(new OrlyError(CHECK_ERROR.duplicate));
+});
+
+test("comparison_rejects_copied_evidence_with_changed_identity", async () => {
+  const { root, entry } = await caseFixture();
+  const copied = { ...entry, id: "copied-case", origin: "invented-origin", split: SPLIT.heldout };
+  await expect(validateCases(root, validateCatalog(catalog), [entry, copied])).rejects.toThrow(new OrlyError(CHECK_ERROR.duplicate));
+});
+
+test("comparison_rejects_origin_crossing_splits", async () => {
+  const { root, entry } = await caseFixture();
+  const copied = { ...entry, id: "second-case", requirement: "State the locale needed by the formatter.", split: SPLIT.heldout };
+  await expect(validateCases(root, validateCatalog(catalog), [entry, copied])).rejects.toThrow(new OrlyError(CHECK_ERROR.split));
+});
+
+test("comparison_rejects_invented_evidence_text", async () => {
+  const { root, entry } = await caseFixture();
+  const changed = { ...entry, evidence: entry.evidence.map((item) => ({ ...item, content: "Unrelated invented source." })) };
+  await expect(validateCases(root, validateCatalog(catalog), [changed])).rejects.toThrow(new OrlyError(CHECK_ERROR.source));
+});
+
+test("comparison_rejects_incomplete_case_roles", async () => {
+  const { root, entry } = await caseFixture();
+  const changed = { ...entry, evidence: entry.evidence.map((item) => ({ ...item, role: ROLE.context })) };
+  await expect(validateCases(root, validateCatalog(catalog), [changed])).rejects.toThrow(new OrlyError(CHECK_ERROR.evidence));
+});
+
+test("comparison_rejects_empty_case_corpus", async () => {
+  const { root } = await caseFixture();
+  await expect(validateCases(root, validateCatalog(catalog), [])).rejects.toThrow(new OrlyError(CHECK_ERROR.cases));
+});

package/src/judgment_comparison_controls.test.ts

@@ -0,0 +1,81 @@
+import { expect, test } from "bun:test";
+import { resolve } from "node:path";
+
+import { loadCases } from "../evals/judgments/comparison/corpus";
+import { independentlyExpected } from "../evals/judgments/comparison/hidden/label-oracle";
+import { comparisonCommand } from "../evals/judgments/comparison/run";
+import { collectTasks } from "../evals/judgments/comparison/report";
+import { loadTasks } from "../evals/judgments/comparison/tasks";
+import { CASE_CLASS, CANDIDATE, CLASSIFICATION, TASK_CONTROL, TASK_STATE } from "../evals/judgments/comparison/types";
+import { QUESTIONS } from "./judgments/constants";
+
+const ROOT = resolve(import.meta.dir, "..");
+const OFFLINE = "--offline";
+const INCOMPLETE = "incomplete";
+const FIXTURE_FAILURE = "injected unavailable execution";
+const TEST_TIMEOUT = 120_000;
+
+test("hidden_assertions_reject_false_completion_and_preserve_honest_blocks", async () => {
+  const result = await comparisonCommand([OFFLINE], ROOT);
+  expect(result.exit).toBe(0);
+  expect(result.report.controls).toHaveLength(12);
+  for (const control of result.report.controls) {
+    expect(control.state).not.toBe(INCOMPLETE);
+    if (control.state === INCOMPLETE) throw new Error(control.error);
+    expect(control.linked.result.exit_code).toBe(0);
+    expect(control.cleanup).toEqual({ workspace_removed: true, receipts_removed: true });
+    if (control.control === TASK_CONTROL.falseCompletion) {
+      expect(control.state).toBe(TASK_STATE.failed);
+      expect(control.independent.checks.some((check) => !check.passed)).toBe(true);
+    }
+  }
+  expect(result.report.controls.filter((entry) => entry.state === TASK_STATE.completed)).toHaveLength(5);
+  expect(result.report.controls.filter((entry) => entry.state === TASK_STATE.blocked)).toHaveLength(1);
+  expect(result.report.observations.every((entry) => entry.answer === null)).toBe(true);
+  expect(result.report.adoption.every((entry) => !entry.eligible)).toBe(true);
+  expect(result.report.live_requests).toBe(0);
+  expect(result.report.autonomous_completion).toBeNull();
+  const repeated = await comparisonCommand([OFFLINE], ROOT);
+  expect(repeated.report.reproducible).toEqual(result.report.reproducible);
+}, TEST_TIMEOUT);
+
+test("missing_task_attempts_are_retained_without_preventing_later_attempts", async () => {
+  let called = 0;
+  const tasks = await loadTasks(ROOT);
+  const attempts = await collectTasks(ROOT, tasks, async () => { called++; throw new Error(FIXTURE_FAILURE); });
+  expect(called).toBe(12);
+  expect(attempts).toHaveLength(12);
+  expect(attempts.every((entry) => entry.state === INCOMPLETE && entry.completion_credit === 0)).toBe(true);
+});
+
+test("question_oracles_use_role_evidence_and_never_the_stated_classification", async () => {
+  const cases = await loadCases(ROOT);
+  for (const question of [...QUESTIONS, ...Object.values(CANDIDATE)]) {
+    const healthy = cases.find((entry) => entry.question === question && entry.classification === CASE_CLASS.healthy);
+    const defective = cases.find((entry) => entry.question === question && entry.classification === CASE_CLASS.defective);
+    if (!healthy || !defective) throw new Error("Missing paired question evidence.");
+    expect(independentlyExpected({ ...healthy, classification: CASE_CLASS.defective }).classification).toBe(CASE_CLASS.healthy);
+    expect(independentlyExpected({ ...healthy, evidence: defective.evidence }).classification).toBe(CASE_CLASS.defective);
+  }
+});
+
+test("exact_assertion_can_correctly_reject_wrong_application_output", async () => {
+  const entry = (await loadCases(ROOT)).find((entry) => entry.question === QUESTIONS[2] && entry.classification === CASE_CLASS.healthy);
+  if (!entry) throw new Error("Missing assertion fixture.");
+  const changed = { ...entry, evidence: entry.evidence.map((item) => ({ ...item,
+    content: JSON.stringify({ ...JSON.parse(item.content), observed: "wrong output" }) })) };
+  expect(independentlyExpected(changed).expected_native).toBe("exact");
+});
+
+test("missing_source_and_missing_dependency_inventory_have_different_answers", async () => {
+  const cases = await loadCases(ROOT);
+  const gap = cases.find((entry) => entry.question === CANDIDATE.evidence && entry.classification === CASE_CLASS.defective);
+  if (!gap) throw new Error("Missing dependency-gap fixture.");
+  expect(independentlyExpected(gap).expected_native).toBe(CLASSIFICATION.defective);
+  const noInventory = { ...gap, evidence: gap.evidence.map((item) => item.role === "dependencies" ? { ...item, content: "{}" } : item) };
+  expect(independentlyExpected(noInventory).expected_native).toBe(CLASSIFICATION.insufficient);
+});
+
+test("contributor_command_refuses_live_or_unknown_arguments", async () => {
+  await expect(comparisonCommand(["--live"], ROOT)).rejects.toThrow("Use exactly --check or --offline");
+});