devlyn-cli 3.2.0 → 3.3.0

package/benchmark/auto-resolve/scripts/iter-0033c-l1-summary.py

@@ -1,97 +0,0 @@
-#!/usr/bin/env python3
-"""Build iter-0033c L1 rerun summary from per-fixture judge/result artifacts."""
-from __future__ import annotations
-
-import argparse
-import json
-import sys
-from pathlib import Path
-from typing import Any
-
-SCRIPT_DIR = Path(__file__).resolve().parent
-if str(SCRIPT_DIR) not in sys.path:
-    sys.path.insert(0, str(SCRIPT_DIR))
-
-from pair_evidence_contract import is_score, is_strict_number, loads_strict_json_object
-
-
-SCORE_ARMS = ("solo_claude", "l2_gated", "l2_forced", "bare")
-
-
-def load_json(path: Path) -> dict[str, Any]:
-    try:
-        data = loads_strict_json_object(path.read_text(encoding="utf8"))
-    except (ValueError, json.JSONDecodeError):
-        return {}
-    return data
-
-
-def score_for(judge: dict[str, Any], arm: str, mapping: dict[str, Any]) -> int | None:
-    letter = next(
-        (slot for slot, mapped in mapping.items() if slot in {"A", "B", "C"} and mapped == arm),
-        None,
-    )
-    if letter is None:
-        return None
-    raw_scores = judge.get("scores_by_arm")
-    scores = raw_scores if isinstance(raw_scores, dict) else {}
-    score = scores.get(arm)
-    if is_score(score):
-        return score
-    legacy = judge.get(f"{letter.lower()}_score")
-    return legacy if is_score(legacy) else None
-
-
-def strict_number(value: object) -> object:
-    return value if is_strict_number(value) else None
-
-
-def build_summary(results_dir: Path, run_id: str, git_sha: str) -> dict[str, Any]:
-    rows = []
-    for fx_dir in sorted(p for p in results_dir.iterdir() if p.is_dir()):
-        judge_path = fx_dir / "judge.json"
-        if not judge_path.is_file():
-            continue
-        judge = load_json(judge_path)
-        raw_mapping = judge.get("_blind_mapping")
-        mapping = raw_mapping if isinstance(raw_mapping, dict) else {}
-        arms = {}
-        for arm_name in SCORE_ARMS:
-            score = score_for(judge, arm_name, mapping)
-            if score is None and arm_name not in set(mapping.values()):
-                continue
-            arm_dir = fx_dir / arm_name
-            result = load_json(arm_dir / "result.json") if (arm_dir / "result.json").is_file() else {}
-            arms[arm_name] = {
-                "score": score,
-                "wall_s": strict_number(result.get("elapsed_seconds")),
-                "verify_score": strict_number(result.get("verify_score")),
-                "files_changed": result.get("files_changed"),
-                "timed_out": result.get("timed_out"),
-                "disqualifier": result.get("disqualifier"),
-            }
-        rows.append({"fixture": fx_dir.name, "arms": arms})
-    return {
-        "run_id": run_id,
-        "git_sha": git_sha,
-        "fixtures_total": len(rows),
-        "rows": rows,
-    }
-
-
-def main() -> int:
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--results-dir", required=True, type=Path)
-    parser.add_argument("--out", required=True, type=Path)
-    parser.add_argument("--run-id", required=True)
-    parser.add_argument("--git-sha", required=True)
-    args = parser.parse_args()
-
-    summary = build_summary(args.results_dir, args.run_id, args.git_sha)
-    args.out.write_text(json.dumps(summary, indent=2) + "\n", encoding="utf8")
-    print(f"[l1-rerun-summary] wrote {args.out} (fixtures={summary['fixtures_total']})")
-    return 0
-
-
-if __name__ == "__main__":
-    raise SystemExit(main())

package/benchmark/auto-resolve/scripts/judge-opus-pass.sh

@@ -1,542 +0,0 @@
-#!/usr/bin/env bash
-# judge-opus-pass.sh — POST-RUN Opus dual-judge sidecar.
-#
-# Runs Opus (the Claude CLI's default model) over the SAME sanitized blind prompts that judge.sh wrote
-# (judge-prompt.txt), producing a parallel judge-opus.json alongside the
-# canonical judge.json. Does NOT modify judge.sh and does NOT change the
-# measurement contract for the in-flight run — this is a sidecar that adds
-# cross-judge data after the suite finishes.
-#
-# Why: judge.sh runs GPT-5.5 today; variant arms run Codex BUILD/FIX on
-# GPT-5.5 too — self-judgment bias. Opus pass tests inter-judge agreement.
-#
-# iter-0025 (B-1) rewrite — 3-arm aware. Prior version computed only legacy
-# `variant_score / bare_score / margin` from A/B slots (Codex R0 caught: F1
-# has variant in C; F6/F9 have bare in C — legacy parser silently produced
-# wrong margins on iter-0020 mappings). This version mirrors judge.sh post
-# iter-0023:
-#   - Reads `_blind_mapping` from gpt judge.json (A/B/C → arm name).
-#   - Requires C-slot score when C is present in mapping.
-#   - Computes `scores_by_arm`, `margins.{variant_over_bare, solo_over_bare,
-#     variant_over_solo}`, and per-arm 4-axis breakdown.
-#   - Validates each axis cell ∈ [0, 25] (clamp + record under
-#     `_axis_validation`, same shape as judge.sh).
-#   - Always re-judges (no skip-on-exists) so cross-judge results never go
-#     stale.
-#   - Aggregator computes per-axis solo_claude-bare (L1-L0) disagreement vs
-#     GPT (the decisive metric per Codex R0 Q1 — falsification rule: any axis
-#     disagreement >2 means iter-0021/0023 L1 readout is single-judge artifact).
-#
-# Usage:
-#   judge-opus-pass.sh --run-id <ID>
-#
-# Reads:   results/<run-id>/<fixture>/judge-prompt.txt  (blind A/B/C prompt)
-#          results/<run-id>/<fixture>/judge.json        (blind mapping)
-# Writes:  results/<run-id>/<fixture>/judge-opus.json   (Opus's parsed result)
-#          results/<run-id>/cross-judge-summary.json    (agreement metrics)
-
-set -euo pipefail
-
-usage() { echo "usage: $0 --run-id <ID>"; exit 1; }
-require_value() {
-  local flag="$1"
-  local value="${2:-}"
-  if [ -z "$value" ] || [[ "$value" == --* ]]; then
-    echo "$flag requires a value" >&2
-    exit 1
-  fi
-}
-
-RUN_ID=""
-while [ $# -gt 0 ]; do
-  case "$1" in
-    --run-id) require_value "$1" "${2:-}"; RUN_ID="$2"; shift 2;;
-    *) usage;;
-  esac
-done
-[ -n "$RUN_ID" ] || usage
-
-BENCH_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
-RES_ROOT="$BENCH_ROOT/results/$RUN_ID"
-[ -d "$RES_ROOT" ] || { echo "no results dir: $RES_ROOT"; exit 1; }
-
-python3 - "$RES_ROOT" "$BENCH_ROOT/scripts" <<'PY'
-import json
-import pathlib
-import sys
-sys.path.insert(0, sys.argv[2])
-from pair_evidence_contract import loads_strict_json_object
-
-def is_score(value):
-    return isinstance(value, int) and not isinstance(value, bool) and 0 <= value <= 100
-
-res_root = pathlib.Path(sys.argv[1])
-errors = []
-for fixture_dir in sorted(p for p in res_root.glob("F*/") if p.is_dir()):
-    judge_path = fixture_dir / "judge.json"
-    prompt_path = fixture_dir / "judge-prompt.txt"
-    if not judge_path.exists() or not prompt_path.exists():
-        continue
-    judge = loads_strict_json_object(judge_path.read_text())
-    mapping = judge.get("_blind_mapping")
-    if not isinstance(mapping, dict):
-        errors.append(f"{fixture_dir.name}: judge blind mapping missing")
-        continue
-    mapped_arms = {arm for slot, arm in mapping.items() if slot in {"A", "B", "C"}}
-    required = {"bare", "solo_claude"}
-    raw_scores = judge.get("scores_by_arm")
-    scores = raw_scores if isinstance(raw_scores, dict) else {}
-    if "variant" in scores:
-        required.add("variant")
-    malformed_scores = sorted(arm for arm, score in scores.items() if not is_score(score))
-    if malformed_scores:
-        errors.append(f"{fixture_dir.name}: scores_by_arm malformed score(s): {', '.join(malformed_scores)}")
-    missing = sorted(required - mapped_arms)
-    if missing:
-        errors.append(f"{fixture_dir.name}: judge blind mapping missing arm(s): {', '.join(missing)}")
-    unmapped_scores = sorted(set(scores) - mapped_arms)
-    if unmapped_scores:
-        errors.append(f"{fixture_dir.name}: scores_by_arm without blind mapping: {', '.join(unmapped_scores)}")
-
-if errors:
-    for error in errors:
-        print(f"[opus-judge] ✗ {error}", file=sys.stderr)
-    raise SystemExit(2)
-PY
-
-command -v claude >/dev/null 2>&1 || { echo "claude CLI not on PATH"; exit 1; }
-
-CLAUDE_CLI_VER=$(claude --version 2>/dev/null || echo "claude-cli unknown")
-JUDGE_MODEL_ALIAS="opus-default"
-
-echo "[opus-judge] run-id=$RUN_ID  cli=$CLAUDE_CLI_VER  model=$JUDGE_MODEL_ALIAS"
-
-processed=0
-skipped=0
-failed=0
-
-for fixture_dir in "$RES_ROOT"/F*/; do
-  [ -d "$fixture_dir" ] || continue
-  fid=$(basename "$fixture_dir")
-  prompt_f="$fixture_dir/judge-prompt.txt"
-  gpt_judge_f="$fixture_dir/judge.json"
-  opus_out_raw="$fixture_dir/judge-opus-output.txt"
-  opus_judge_f="$fixture_dir/judge-opus.json"
-
-  if [ ! -f "$prompt_f" ] || [ ! -f "$gpt_judge_f" ]; then
-    echo "[opus-judge] skip $fid (missing prompt or gpt judge)"
-    skipped=$((skipped + 1))
-    continue
-  fi
-
-  echo "[opus-judge] judging $fid ..."
-
-  # Codex R1 #2: drop any stale judge-opus.json before re-judging. Otherwise
-  # a parse failure here keeps the previous run's file and the aggregator
-  # silently reads it.
-  rm -f "$opus_judge_f"
-
-  # Strict MCP isolation matches the variant arm contract — no user MCP
-  # leakage into the judge call. Run from a clean tmp CWD so it can't peek
-  # at project files that might leak arm identity.
-  JUDGE_CWD="/tmp/opus-judge-$RUN_ID-$fid"
-  rm -rf "$JUDGE_CWD"
-  mkdir -p "$JUDGE_CWD"
-
-  set +e
-  ( cd "$JUDGE_CWD" && claude -p "$(cat "$prompt_f")" \
-      --dangerously-skip-permissions \
-      --strict-mcp-config --mcp-config '{"mcpServers":{}}' \
-  ) > "$opus_out_raw" 2>&1
-  rc=$?
-  set -e
-  rm -rf "$JUDGE_CWD"
-
-  if [ $rc -ne 0 ]; then
-    echo "[opus-judge] ✗ $fid claude -p exit=$rc (output preserved at $opus_out_raw)"
-    failed=$((failed + 1))
-    continue
-  fi
-
-  python3 - "$opus_out_raw" "$gpt_judge_f" "$opus_judge_f" "$CLAUDE_CLI_VER" "$JUDGE_MODEL_ALIAS" "$BENCH_ROOT/scripts" <<'PY' || { echo "[opus-judge] ✗ $fid parse failed"; failed=$((failed + 1)); continue; }
-import sys, json, pathlib
-sys.path.insert(0, sys.argv[6])
-from pair_evidence_contract import loads_strict_json_object, reject_json_constant
-
-def is_score(value):
-    return isinstance(value, int) and not isinstance(value, bool) and 0 <= value <= 100
-
-def is_bool(value):
-    return isinstance(value, bool)
-
-raw = pathlib.Path(sys.argv[1]).read_text()
-gpt = loads_strict_json_object(pathlib.Path(sys.argv[2]).read_text())
-target = pathlib.Path(sys.argv[3])
-cli_ver = sys.argv[4].strip()
-model_alias = sys.argv[5].strip()
-
-# Robust JSON extraction — last valid {} block with required score keys.
-mapping = gpt.get("_blind_mapping")
-if not isinstance(mapping, dict):
-    raise SystemExit("gpt judge.json _blind_mapping must be an object")
-required_score_keys = ["a_score", "b_score"]
-if "C" in mapping:
-    required_score_keys.append("c_score")
-
-decoder = json.JSONDecoder(parse_constant=reject_json_constant)
-brace_positions = [i for i, c in enumerate(raw) if c == '{']
-chosen = None
-for pos in reversed(brace_positions):
-    try:
-        obj, _ = decoder.raw_decode(raw[pos:])
-    except json.JSONDecodeError:
-        continue
-    if isinstance(obj, dict) and all(k in obj for k in required_score_keys):
-        chosen = obj
-        break
-if chosen is None:
-    raise SystemExit(
-        f"no valid JSON with keys {required_score_keys} in opus output: {sys.argv[1]}"
-    )
-invalid_scores = [key for key in required_score_keys if not is_score(chosen.get(key))]
-if invalid_scores:
-    raise SystemExit(f"invalid opus score value(s): {', '.join(invalid_scores)}")
-
-# Axis validation — mirror judge.sh post iter-0023.
-AXIS_KEYS = ("spec", "constraint", "scope", "quality")
-BREAKDOWN_KEYS = ("a_breakdown", "b_breakdown", "c_breakdown")
-axis_invalid_cells = []
-for bk in BREAKDOWN_KEYS:
-    if bk not in chosen or not isinstance(chosen[bk], dict):
-        continue
-    for axis in AXIS_KEYS:
-        if axis not in chosen[bk]:
-            continue
-        v = chosen[bk][axis]
-        if isinstance(v, bool) or not isinstance(v, (int, float)) or v < 0 or v > 25:
-            axis_invalid_cells.append({"breakdown": bk, "axis": axis, "value": v})
-            chosen[bk][axis] = max(0, min(25, int(v) if not isinstance(v, bool) and isinstance(v, (int, float)) else 0))
-chosen["_axis_validation"] = {
-    "out_of_range_count": len(axis_invalid_cells),
-    "out_of_range_cells": axis_invalid_cells,
-    "axis_range": [0, 25],
-}
-if axis_invalid_cells:
-    sys.stderr.write(
-        f"[opus-judge] WARNING: {len(axis_invalid_cells)} axis cell(s) out of [0,25] "
-        f"clamped: {axis_invalid_cells}\n"
-    )
-
-# Reuse GPT's blind mapping verbatim (same blind A/B/C decoded the same way).
-chosen["_blind_mapping"] = mapping
-chosen["_judge_cli"] = cli_ver
-chosen["_judge_model"] = model_alias
-
-# scores_by_arm + margins — mirror judge.sh post iter-0023.
-slot_keys = ["a_score", "b_score", "c_score"]
-slot_letters = ["A", "B", "C"]
-scores_by_arm = {}
-for letter, key in zip(slot_letters, slot_keys):
-    arm = mapping.get(letter)
-    if arm is not None and key in chosen and is_score(chosen[key]):
-        scores_by_arm[arm] = chosen[key]
-chosen["scores_by_arm"] = scores_by_arm
-
-def margin(left, right):
-    if left in scores_by_arm and right in scores_by_arm:
-        return scores_by_arm[left] - scores_by_arm[right]
-    return None
-
-chosen["margins"] = {
-    "variant_over_bare":   margin("variant", "bare"),
-    "solo_over_bare":      margin("solo_claude", "bare"),
-    "variant_over_solo":   margin("variant", "solo_claude"),
-}
-
-# Per-arm axis breakdown.
-breakdowns_by_arm = {}
-for letter, bk in zip(slot_letters, BREAKDOWN_KEYS):
-    arm = mapping.get(letter)
-    if arm is not None and bk in chosen:
-        breakdowns_by_arm[arm] = chosen[bk]
-chosen["breakdowns_by_arm"] = breakdowns_by_arm
-
-# Per-arm critical_findings + disqualifiers (same shape judge.sh emits).
-raw_findings_letters = chosen.get("critical_findings")
-findings_letters = raw_findings_letters if isinstance(raw_findings_letters, dict) else {}
-chosen["findings_by_arm"] = {
-    mapping[l]: findings_letters.get(l, []) for l in slot_letters if l in mapping
-}
-raw_dq_letters = chosen.get("disqualifiers")
-dq_letters = raw_dq_letters if isinstance(raw_dq_letters, dict) else {}
-invalid_dq_letters = [
-    letter for letter in slot_letters
-    if letter in dq_letters and not is_bool(dq_letters.get(letter))
-]
-if invalid_dq_letters:
-    raise SystemExit(f"invalid opus disqualifier value(s): {', '.join(invalid_dq_letters)}")
-dq_by_arm = {}
-for l in slot_letters:
-    if l not in mapping:
-        continue
-    arm = mapping[l]
-    dq_by_arm[arm] = {
-        "disqualifier": dq_letters.get(l, False) is True,
-        "reason": str(dq_letters.get(f"{l}_reason", "") or ""),
-    }
-chosen["disqualifiers_by_arm"] = dq_by_arm
-
-# Winner letter → arm.
-w = chosen.get("winner")
-if w in slot_letters and w in mapping:
-    chosen["winner_arm"] = mapping[w]
-elif w == "tie":
-    chosen["winner_arm"] = "tie"
-else:
-    chosen["winner_arm"] = None
-
-# Legacy fields kept for back-compat.
-a_arm = mapping.get("A")
-b_arm = mapping.get("B")
-chosen["variant_score"] = scores_by_arm.get("variant")
-chosen["bare_score"]    = scores_by_arm.get("bare")
-chosen["solo_score"]    = scores_by_arm.get("solo_claude")
-if chosen["variant_score"] is not None and chosen["bare_score"] is not None:
-    chosen["margin"] = chosen["variant_score"] - chosen["bare_score"]
-else:
-    chosen["margin"] = None
-
-target.write_text(json.dumps(chosen, indent=2))
-print(
-    f"[opus-judge] {target.parent.name} "
-    f"v={chosen.get('variant_score')} l1={chosen.get('solo_score')} l0={chosen.get('bare_score')} "
-    f"solo_claude-bare={chosen['margins']['solo_over_bare']} "
-    f"variant-solo_claude={chosen['margins']['variant_over_solo']}"
-)
-PY
-  processed=$((processed + 1))
-done
-
-echo "[opus-judge] judge passes: processed=$processed skipped=$skipped failed=$failed"
-
-# Aggregate cross-judge agreement, including per-axis solo_claude-bare (L1-L0) disagreement.
-python3 - "$RES_ROOT" "$BENCH_ROOT/scripts" <<'PY'
-import json, pathlib, sys, math
-sys.path.insert(0, sys.argv[2])
-from pair_evidence_contract import loads_strict_json_object
-
-def is_score(value):
-    return isinstance(value, int) and not isinstance(value, bool) and 0 <= value <= 100
-
-res_root = pathlib.Path(sys.argv[1])
-rows = []
-axis_keys = ("spec", "constraint", "scope", "quality")
-
-for fdir in sorted(res_root.glob("F*/")):
-    g_f = fdir / "judge.json"
-    o_f = fdir / "judge-opus.json"
-    if not g_f.exists() or not o_f.exists():
-        continue
-    g = loads_strict_json_object(g_f.read_text())
-    o = loads_strict_json_object(o_f.read_text())
-
-    # Per-axis solo_claude-bare (L1-L0) for both judges.
-    # Codex R1 #1: judge.sh historically writes `a/b/c_breakdown` plus
-    # `_blind_mapping`, NOT `breakdowns_by_arm`. iter-0020 judge.json files
-    # are in that historical shape. Derive per-arm breakdowns from letter
-    # fields when `breakdowns_by_arm` is absent; fail loudly when neither
-    # source is available so axis disagreement never silently falls to zero.
-    def blind_mapping(j):
-        raw_mapping = j.get("_blind_mapping")
-        return raw_mapping if isinstance(raw_mapping, dict) else {}
-
-    def mapped_arm_set(j):
-        mapping = blind_mapping(j)
-        return {arm for slot, arm in mapping.items() if slot in {"A", "B", "C"}}
-
-    def axis_l1_l0(j, label):
-        mapped_arms = mapped_arm_set(j)
-        bka = j.get("breakdowns_by_arm") or {}
-        if {"solo_claude", "bare"}.issubset(mapped_arms) and "solo_claude" in bka and "bare" in bka:
-            l1 = bka["solo_claude"]; l0 = bka["bare"]
-        else:
-            mapping = blind_mapping(j)
-            slot_letters = ["A", "B", "C"]
-            slot_breakdowns = ["a_breakdown", "b_breakdown", "c_breakdown"]
-            derived = {}
-            for letter, bk in zip(slot_letters, slot_breakdowns):
-                arm = mapping.get(letter)
-                if arm is not None and bk in j:
-                    derived[arm] = j[bk]
-            if "solo_claude" not in derived or "bare" not in derived:
-                raise SystemExit(
-                    f"[cross-judge] {label} judge.json missing breakdowns for solo_claude/bare; "
-                    "expected either `breakdowns_by_arm` or `a/b/c_breakdown` + `_blind_mapping`"
-                )
-            l1 = derived["solo_claude"]; l0 = derived["bare"]
-        return {a: (l1.get(a, 0) - l0.get(a, 0)) for a in axis_keys}
-
-    def mapped_scores(j):
-        mapped_arms = mapped_arm_set(j)
-        raw_scores = j.get("scores_by_arm")
-        scores = raw_scores if isinstance(raw_scores, dict) else {}
-        return {arm: score for arm, score in scores.items() if arm in mapped_arms and is_score(score)}
-
-    def margin_from_scores(scores, left, right):
-        if left in scores and right in scores:
-            return scores[left] - scores[right]
-        return None
-
-    def mapped_winner(j, scores):
-        winner = j.get("winner_arm")
-        if winner == "tie" or winner in scores:
-            return winner
-        return None
-
-    g_axes = axis_l1_l0(g, f"gpt {fdir.name}")
-    o_axes = axis_l1_l0(o, f"opus {fdir.name}")
-    axis_disagreement = {a: o_axes[a] - g_axes[a] for a in axis_keys}
-
-    g_scores = mapped_scores(g)
-    o_scores = mapped_scores(o)
-    g_l1_l0 = margin_from_scores(g_scores, "solo_claude", "bare")
-    o_l1_l0 = margin_from_scores(o_scores, "solo_claude", "bare")
-    g_v_l0 = margin_from_scores(g_scores, "variant", "bare")
-    o_v_l0 = margin_from_scores(o_scores, "variant", "bare")
-    g_winner = mapped_winner(g, g_scores)
-    o_winner = mapped_winner(o, o_scores)
-    margin_l1_l0_diff = (
-        abs(g_l1_l0 - o_l1_l0) if g_l1_l0 is not None and o_l1_l0 is not None else None
-    )
-    margin_v_l0_diff = (
-        abs(g_v_l0 - o_v_l0) if g_v_l0 is not None and o_v_l0 is not None else None
-    )
-
-    rows.append({
-        "fixture": fdir.name,
-        "gpt_scores":   g_scores,
-        "opus_scores":  o_scores,
-        "gpt_margin_l1_l0":  g_l1_l0,
-        "opus_margin_l1_l0": o_l1_l0,
-        "margin_l1_l0_diff": margin_l1_l0_diff,
-        "gpt_margin_v_l0":   g_v_l0,
-        "opus_margin_v_l0":  o_v_l0,
-        "margin_v_l0_diff":  margin_v_l0_diff,
-        "gpt_axis_l1_l0":    g_axes,
-        "opus_axis_l1_l0":   o_axes,
-        "axis_disagreement": axis_disagreement,
-        "winner_agree":   g_winner is not None and o_winner is not None and g_winner == o_winner,
-        "gpt_winner":     g_winner,
-        "opus_winner":    o_winner,
-    })
-
-if not rows:
-    print("[cross-judge] no paired judgements found")
-    sys.exit(0)
-
-n = len(rows)
-
-# Suite-level per-axis solo_claude-bare (L1-L0) sum (both judges) and disagreement.
-g_axis_sum = {a: sum(r["gpt_axis_l1_l0"][a] for r in rows) for a in axis_keys}
-o_axis_sum = {a: sum(r["opus_axis_l1_l0"][a] for r in rows) for a in axis_keys}
-axis_sum_disagreement = {a: o_axis_sum[a] - g_axis_sum[a] for a in axis_keys}
-max_abs_axis_disagreement = max(abs(v) for v in axis_sum_disagreement.values())
-
-# Per-axis falsification rule (Codex R0 Q1): any axis-sum disagreement >2 → single-judge artifact.
-THRESHOLD = 2
-falsified_by_axis = max_abs_axis_disagreement > THRESHOLD
-flipped_axes = [a for a, v in axis_sum_disagreement.items() if abs(v) > THRESHOLD]
-
-# Suite avg solo_claude-bare (L1-L0, both judges) — Codex R1 #3: divide by valid-count, report denom.
-gpt_l1_l0_valid = [r["gpt_margin_l1_l0"] for r in rows if r["gpt_margin_l1_l0"] is not None]
-opus_l1_l0_valid = [r["opus_margin_l1_l0"] for r in rows if r["opus_margin_l1_l0"] is not None]
-gpt_l1_l0_avg = (sum(gpt_l1_l0_valid) / len(gpt_l1_l0_valid)) if gpt_l1_l0_valid else None
-opus_l1_l0_avg = (sum(opus_l1_l0_valid) / len(opus_l1_l0_valid)) if opus_l1_l0_valid else None
-suite_avg_diff = (
-    abs(gpt_l1_l0_avg - opus_l1_l0_avg)
-    if gpt_l1_l0_avg is not None and opus_l1_l0_avg is not None else None
-)
-
-# Sign agreement — Codex R1 #4: 3-way sign (-, 0, +) so tie vs positive don't agree.
-def sign3(v):
-    if v is None:
-        return None
-    return 1 if v > 0 else (-1 if v < 0 else 0)
-
-sign_agree_v_l0 = sum(
-    1 for r in rows
-    if r["gpt_margin_v_l0"] is not None and r["opus_margin_v_l0"] is not None
-    and sign3(r["gpt_margin_v_l0"]) == sign3(r["opus_margin_v_l0"])
-)
-sign_agree_l1_l0 = sum(
-    1 for r in rows
-    if r["gpt_margin_l1_l0"] is not None and r["opus_margin_l1_l0"] is not None
-    and sign3(r["gpt_margin_l1_l0"]) == sign3(r["opus_margin_l1_l0"])
-)
-sign_valid_l1_l0 = sum(
-    1 for r in rows
-    if r["gpt_margin_l1_l0"] is not None and r["opus_margin_l1_l0"] is not None
-)
-sign_valid_v_l0 = sum(
-    1 for r in rows
-    if r["gpt_margin_v_l0"] is not None and r["opus_margin_v_l0"] is not None
-)
-winner_agree = sum(1 for r in rows if r["winner_agree"])
-
-l1_l0_diffs = [r["margin_l1_l0_diff"] for r in rows if r["margin_l1_l0_diff"] is not None]
-mean_abs_l1_l0_diff = (sum(l1_l0_diffs) / len(l1_l0_diffs)) if l1_l0_diffs else None
-
-summary = {
-    "n_fixtures": n,
-    "threshold": THRESHOLD,
-    "falsified_by_axis_disagreement": falsified_by_axis,
-    "flipped_axes": flipped_axes,
-    "axis_sum_l1_l0": {
-        "gpt": g_axis_sum,
-        "opus": o_axis_sum,
-        "disagreement": axis_sum_disagreement,
-        "max_abs_disagreement": max_abs_axis_disagreement,
-    },
-    "suite_avg_l1_l0": {
-        "gpt": gpt_l1_l0_avg,
-        "opus": opus_l1_l0_avg,
-        "abs_diff": suite_avg_diff,
-        "gpt_valid_count":  len(gpt_l1_l0_valid),
-        "opus_valid_count": len(opus_l1_l0_valid),
-    },
-    "winner_agree_count": winner_agree,
-    "sign_agree_l1_l0":               sign_agree_l1_l0,
-    "sign_valid_count_l1_l0":         sign_valid_l1_l0,
-    "sign_agree_variant_over_bare":   sign_agree_v_l0,
-    "sign_valid_count_variant_over_bare": sign_valid_v_l0,
-    "mean_abs_l1_l0_margin_diff": mean_abs_l1_l0_diff,
-    "mean_abs_l1_l0_valid_count": len(l1_l0_diffs),
-    "rows": rows,
-}
-out = res_root / "cross-judge-summary.json"
-out.write_text(json.dumps(summary, indent=2))
-
-def fmt_metric(value):
-    return f"{value:.2f}" if value is not None else "na"
-
-print(
-    f"[cross-judge] n={n} "
-    f"falsified={falsified_by_axis} flipped_axes={flipped_axes} "
-    f"max_axis_disagreement={max_abs_axis_disagreement} "
-    f"gpt_l1_l0_avg={fmt_metric(gpt_l1_l0_avg)} "
-    f"opus_l1_l0_avg={fmt_metric(opus_l1_l0_avg)} "
-    f"suite_avg_diff={fmt_metric(suite_avg_diff)}"
-)
-print(f"[cross-judge] axis_sum_l1_l0: gpt={g_axis_sum} opus={o_axis_sum} disagree={axis_sum_disagreement}")
-print(f"[cross-judge] wrote {out}")
-PY
-
-# Hard-fail summary if any fixture in the selected result set lacks a paired judgement.
-EXPECTED_FIXTURES=$(ls -d "$RES_ROOT"/F*/ 2>/dev/null | wc -l | awk '{print $1}')
-PAIRED=$(find "$RES_ROOT" -maxdepth 2 -name 'judge-opus.json' | wc -l | awk '{print $1}')
-echo "[opus-judge] expected_fixtures=$EXPECTED_FIXTURES paired=$PAIRED"
-if [ "$PAIRED" -ne "$EXPECTED_FIXTURES" ]; then
-  echo "[opus-judge] ✗ paired ($PAIRED) != expected ($EXPECTED_FIXTURES) — sidecar incomplete; do not interpret cross-judge until all fixtures judged."
-  exit 2
-fi
-echo "[opus-judge] done: processed=$processed skipped=$skipped failed=$failed"