devlyn-cli 3.2.0 → 3.3.0

package/benchmark/auto-resolve/scripts/run-headroom-candidate.sh

@@ -1,336 +0,0 @@
-#!/usr/bin/env bash
-# run-headroom-candidate.sh — calibrate candidate fixtures for L2/pair headroom.
-#
-# Runs only the arms needed by headroom-gate.py: bare and solo_claude.
-# Then blind-judges those two arms and applies the mechanical gate.
-
-set -euo pipefail
-
-usage() {
-  local code="${1:-1}"
-  cat >&2 <<'EOF'
-usage: run-headroom-candidate.sh [options] <fixture> [<fixture> ...]
-
-Options:
-  --run-id ID
-  --bare-max N       (default: 60)
-  --solo-max N       (default: 80)
-  --min-bare-headroom N  (default: 5)
-  --min-solo-headroom N  (default: 5)
-  --min-fixtures N   (default: 2)
-  --allow-rejected-fixtures
-                    allow rejected/ceiling fixtures for diagnostics only
-  --dry-run          validate args/fixtures and print replay command only
-EOF
-  exit "$code"
-}
-
-require_value() {
-  local flag="$1"
-  local value="${2:-}"
-  if [ -z "$value" ] || [[ "$value" == --* ]]; then
-    echo "$flag requires a value" >&2
-    exit 1
-  fi
-}
-
-RUN_ID=""
-BARE_MAX=60
-SOLO_MAX=80
-MIN_BARE_HEADROOM=5
-MIN_SOLO_HEADROOM=5
-MIN_FIXTURES=2
-ALLOW_REJECTED_FIXTURES=0
-DRY_RUN=0
-FIXTURES=()
-while [ $# -gt 0 ]; do
-  case "$1" in
-    --run-id) require_value "$1" "${2:-}"; RUN_ID="$2"; shift 2;;
-    --bare-max) require_value "$1" "${2:-}"; BARE_MAX="$2"; shift 2;;
-    --solo-max) require_value "$1" "${2:-}"; SOLO_MAX="$2"; shift 2;;
-    --min-bare-headroom) require_value "$1" "${2:-}"; MIN_BARE_HEADROOM="$2"; shift 2;;
-    --min-solo-headroom) require_value "$1" "${2:-}"; MIN_SOLO_HEADROOM="$2"; shift 2;;
-    --min-fixtures) require_value "$1" "${2:-}"; MIN_FIXTURES="$2"; shift 2;;
-    --allow-rejected-fixtures) ALLOW_REJECTED_FIXTURES=1; shift;;
-    --dry-run) DRY_RUN=1; shift;;
-    -h|--help) usage 0;;
-    [FS][0-9]*) FIXTURES+=("$1"); shift;;
-    *) echo "unknown arg: $1" >&2; usage;;
-  esac
-done
-
-for threshold in BARE_MAX SOLO_MAX MIN_BARE_HEADROOM MIN_SOLO_HEADROOM MIN_FIXTURES; do
-  value="${!threshold}"
-  case "$threshold" in
-    BARE_MAX) flag="bare-max" ;;
-    SOLO_MAX) flag="solo-max" ;;
-    MIN_BARE_HEADROOM) flag="min-bare-headroom" ;;
-    MIN_SOLO_HEADROOM) flag="min-solo-headroom" ;;
-    MIN_FIXTURES) flag="min-fixtures" ;;
-  esac
-  if [[ ! "$value" =~ ^[0-9]+$ ]]; then
-    echo "--$flag must be an integer: $value" >&2
-    exit 1
-  fi
-done
-if [ "$MIN_FIXTURES" -lt 1 ]; then
-  echo "--min-fixtures must be >= 1" >&2
-  exit 1
-fi
-if [ "$MIN_BARE_HEADROOM" -lt 0 ]; then
-  echo "--min-bare-headroom must be >= 0" >&2
-  exit 1
-fi
-if [ "$MIN_SOLO_HEADROOM" -lt 0 ]; then
-  echo "--min-solo-headroom must be >= 0" >&2
-  exit 1
-fi
-
-[ ${#FIXTURES[@]} -gt 0 ] || usage
-
-BENCH_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
-REPO_ROOT="$(cd "$BENCH_ROOT/../.." && pwd)"
-source "$BENCH_ROOT/scripts/pair-rejected-fixtures.sh"
-if ! declare -F rejected_pair_fixture_reason >/dev/null; then
-  echo "rejected fixture registry must define rejected_pair_fixture_reason" >&2
-  exit 1
-fi
-
-if [ -z "$RUN_ID" ]; then
-  TS=$(date -u +%Y%m%dT%H%M%SZ)
-  SHA=$(git -C "$REPO_ROOT" rev-parse --short HEAD 2>/dev/null || echo nogit)
-  RUN_ID="${TS}-${SHA}-headroom"
-fi
-
-print_command() {
-  local cmd
-  if [ "${DEVLYN_BENCHMARK_CLI_SUBCOMMAND:-}" = "headroom" ]; then
-    cmd=(npx devlyn-cli benchmark headroom --run-id "$RUN_ID")
-  else
-    cmd=(bash "$0" --run-id "$RUN_ID")
-  fi
-  cmd+=(--bare-max "$BARE_MAX")
-  cmd+=(--solo-max "$SOLO_MAX")
-  cmd+=(--min-bare-headroom "$MIN_BARE_HEADROOM")
-  cmd+=(--min-solo-headroom "$MIN_SOLO_HEADROOM")
-  cmd+=(--min-fixtures "$MIN_FIXTURES")
-  [ "$ALLOW_REJECTED_FIXTURES" -eq 0 ] || cmd+=(--allow-rejected-fixtures)
-  [ "$DRY_RUN" -eq 0 ] || cmd+=(--dry-run)
-  cmd+=("${FIXTURES[@]}")
-  printf 'Command: '
-  printf '%q ' "${cmd[@]}"
-  printf '\n'
-}
-
-fixture_exists() {
-  local fid="$1"
-  [ -d "$BENCH_ROOT/fixtures/$fid" ] || [ -d "$BENCH_ROOT/shadow-fixtures/$fid" ]
-}
-
-fixture_dir() {
-  local fid="$1"
-  if [ -d "$BENCH_ROOT/fixtures/$fid" ]; then
-    printf '%s\n' "$BENCH_ROOT/fixtures/$fid"
-  else
-    printf '%s\n' "$BENCH_ROOT/shadow-fixtures/$fid"
-  fi
-}
-
-is_shadow_fixture() {
-  local fid="$1"
-  [ -d "$BENCH_ROOT/shadow-fixtures/$fid" ]
-}
-
-retired_fixture_exists() {
-  local fid="$1"
-  [ -d "$BENCH_ROOT/fixtures/retired/$fid" ]
-}
-
-fixture_smoke_only() {
-  local fid="$1"
-  [[ "$fid" == S1 || "$fid" == S1-* ]]
-}
-
-fixture_category() {
-  local dir="$1"
-  python3 - "$dir/metadata.json" <<'PY'
-import json
-import sys
-
-try:
-    with open(sys.argv[1], encoding="utf-8") as handle:
-        print(json.load(handle).get("category", ""))
-except FileNotFoundError:
-    print("")
-PY
-}
-
-fixture_has_solo_headroom_hypothesis() {
-  local dir="$1"
-  python3 "$BENCH_ROOT/scripts/solo-headroom-hypothesis.py" --expected-json "$dir/expected.json" "$dir/spec.md"
-}
-
-fixture_has_solo_ceiling_avoidance_note() {
-  local dir="$1"
-  python3 "$BENCH_ROOT/scripts/solo-ceiling-avoidance.py" "$dir/NOTES.md"
-}
-
-fixture_has_pair_evidence() {
-  local fid="$1"
-  python3 - "$BENCH_ROOT/results" "$fid" <<'PY'
-import json
-import pathlib
-import sys
-
-results = pathlib.Path(sys.argv[1])
-fixture = sys.argv[2]
-if not results.is_dir():
-    sys.exit(1)
-for path in results.glob("*/full-pipeline-pair-gate.json"):
-    try:
-        data = json.loads(path.read_text(encoding="utf-8"))
-    except (OSError, json.JSONDecodeError):
-        continue
-    if data.get("verdict") != "PASS":
-        continue
-    rows = data.get("rows")
-    if not isinstance(rows, list):
-        continue
-    for row in rows:
-        if isinstance(row, dict) and row.get("fixture") == fixture and row.get("status") == "PASS":
-            sys.exit(0)
-sys.exit(1)
-PY
-}
-
-validate_fixtures() {
-  local missing=0
-  local fid reason dir category
-  for fid in "${FIXTURES[@]}"; do
-    if ! fixture_exists "$fid"; then
-      if retired_fixture_exists "$fid"; then
-        echo "fixture is retired and is not rerun by pair-candidate runners: $fid. Use preserved results/docs for historical replay." >&2
-        missing=1
-        continue
-      fi
-      echo "fixture not found in fixtures/ or shadow-fixtures/: $fid" >&2
-      missing=1
-      continue
-    fi
-    if [ "$DRY_RUN" -eq 0 ] && fixture_smoke_only "$fid"; then
-      echo "fixture is smoke-only and cannot run providers: $fid. Use --dry-run for runner/package validation." >&2
-      missing=1
-      continue
-    fi
-    reason="$(rejected_pair_fixture_reason "$fid" || true)"
-    if [ "$ALLOW_REJECTED_FIXTURES" -eq 0 ]; then
-      if [ -n "$reason" ]; then
-        echo "fixture rejected for pair-candidate runs: $fid ($reason). Use --allow-rejected-fixtures for diagnostics only." >&2
-        missing=1
-        continue
-      fi
-    fi
-    if [ -z "$reason" ]; then
-      dir="$(fixture_dir "$fid")"
-      category="$(fixture_category "$dir")"
-      if [ "$category" = "high-risk" ] && ! fixture_has_pair_evidence "$fid"; then
-        if ! fixture_has_solo_headroom_hypothesis "$dir"; then
-          echo "fixture spec.md needs a solo-headroom hypothesis with solo_claude miss and observable command from expected.json before provider spend: $fid" >&2
-          missing=1
-        fi
-        if is_shadow_fixture "$fid" && ! fixture_has_solo_ceiling_avoidance_note "$dir"; then
-          echo "shadow fixture NOTES.md needs ## Solo ceiling avoidance with solo_claude, a rejected/solo-saturated control comparison, and headroom reasoning before provider spend: $fid" >&2
-          missing=1
-        fi
-      fi
-    fi
-  done
-  [ "$missing" -eq 0 ] || exit 1
-}
-
-echo ""
-echo "═══ Headroom Candidate Run ═══"
-echo "Run-id:   $RUN_ID"
-echo "Fixtures: ${FIXTURES[*]}"
-echo "Arms:     bare solo_claude"
-echo "Gate:     bare <= $BARE_MAX (headroom >= $MIN_BARE_HEADROOM), solo_claude <= $SOLO_MAX (headroom >= $MIN_SOLO_HEADROOM), baseline evidence-complete, min fixtures $MIN_FIXTURES"
-[ "$DRY_RUN" -eq 0 ] || echo "Mode:     DRY RUN (no model/provider invocations)"
-print_command
-if [ ${#FIXTURES[@]} -lt "$MIN_FIXTURES" ]; then
-  echo "Gate:     will FAIL set gate unless at least $MIN_FIXTURES fixtures are supplied"
-fi
-echo ""
-
-validate_fixtures
-
-if [ "$DRY_RUN" -eq 1 ] && [ "${#FIXTURES[@]}" -lt "$MIN_FIXTURES" ]; then
-  echo "[headroom] DRY RUN failed — ${#FIXTURES[@]} fixture(s) supplied, --min-fixtures requires $MIN_FIXTURES." >&2
-  exit 1
-fi
-
-if [ "$DRY_RUN" -eq 1 ]; then
-  echo "[headroom] DRY RUN complete — fixtures resolved, no arms or judges executed."
-  exit 0
-fi
-
-SRC_SKILLS="$REPO_ROOT/config/skills"
-DST_SKILLS="$REPO_ROOT/.claude/skills"
-mkdir -p "$DST_SKILLS"
-mirrored=0
-for src_dir in "$SRC_SKILLS"/*/; do
-  [ -d "$src_dir" ] || continue
-  name=$(basename "$src_dir")
-  case "$name" in
-    devlyn:auto-resolve-workspace|devlyn:ideate-workspace|preflight-workspace|roadmap-archival-workspace)
-      continue ;;
-  esac
-  staging="$DST_SKILLS/.${name}.staging"
-  rm -rf "$staging"
-  cp -R "$src_dir" "$staging"
-  rm -rf "$DST_SKILLS/$name"
-  mv "$staging" "$DST_SKILLS/$name"
-  mirrored=$((mirrored + 1))
-done
-echo "[headroom] mirrored $mirrored committed skill(s): config/skills/ -> .claude/skills/"
-
-for fid in "${FIXTURES[@]}"; do
-  echo "[headroom] ► $fid / bare"
-  bash "$BENCH_ROOT/scripts/run-fixture.sh" \
-    --fixture "$fid" --arm bare --run-id "$RUN_ID" \
-    || echo "[headroom] ✗ $fid / bare (arm failure tolerated; artifacts may still exist)"
-
-  echo "[headroom] ► $fid / solo_claude"
-  bash "$BENCH_ROOT/scripts/run-fixture.sh" \
-    --fixture "$fid" --arm solo_claude --run-id "$RUN_ID" \
-    || echo "[headroom] ✗ $fid / solo_claude (arm failure tolerated; artifacts may still exist)"
-
-  echo "[headroom] ► judge $fid"
-  bash "$BENCH_ROOT/scripts/judge.sh" --fixture "$fid" --run-id "$RUN_ID" \
-    || echo "[headroom] ✗ judge failed for $fid"
-done
-
-echo ""
-set +e
-python3 "$BENCH_ROOT/scripts/headroom-gate.py" \
-  --run-id "$RUN_ID" \
-  --bare-max "$BARE_MAX" \
-  --solo-max "$SOLO_MAX" \
-  --min-bare-headroom "$MIN_BARE_HEADROOM" \
-  --min-solo-headroom "$MIN_SOLO_HEADROOM" \
-  --min-fixtures "$MIN_FIXTURES" \
-  --out-json "$BENCH_ROOT/results/$RUN_ID/headroom-gate.json" \
-  --out-md "$BENCH_ROOT/results/$RUN_ID/headroom-gate.md"
-GATE_EXIT=$?
-set -e
-
-if [ -f "$BENCH_ROOT/results/$RUN_ID/headroom-gate.md" ]; then
-  cat "$BENCH_ROOT/results/$RUN_ID/headroom-gate.md"
-else
-  echo "[headroom] headroom gate report missing: $BENCH_ROOT/results/$RUN_ID/headroom-gate.md" >&2
-fi
-if [ "$GATE_EXIT" -eq 0 ]; then
-  echo "[headroom] headroom gate passed — candidate set accepted."
-else
-  echo "[headroom] headroom gate failed — candidate set rejected."
-fi
-exit "$GATE_EXIT"

package/benchmark/auto-resolve/scripts/run-iter-0033c.sh

@@ -1,208 +0,0 @@
-#!/usr/bin/env bash
-# run-iter-0033c.sh — orchestrate the iter-0033c suite (NEW L2 vs NEW L1).
-#
-# Codex R0.5-infra design: bypass run-suite.sh + ship-gate.py + compile-report.py
-# (those enforce variant/bare semantics that don't apply here). Call run-fixture.sh
-# directly per fixture per arm; per-fixture interleaving for fail-early on hard-floor
-# violations (Codex R0.5-infra Q4).
-#
-# Per Mission 1: serial only, no parallel-fleet.
-#
-# Usage:
-#   run-iter-0033c.sh --label <label> [--fixtures F1,F2,...] [--c1-summary <path>] [--f9-judge <path>]
-#                     [--manifest-out <path>] [--results-out-dir <path>] [--skip-judge]
-#
-# Pre-flight: smoke 1b (codex availability) — fail-fast.
-# Arms per fixture:
-#   - All fixtures: solo_claude (L1 rerun) + l2_gated (L2 natural triggers)
-#   - Pair-eligible (per manifest): also l2_forced (L2 diagnostic)
-# After arms: judge.sh per fixture; manifest build; iter-0033c-compare.py.
-set -euo pipefail
-
-usage() {
-  cat >&2 <<EOF
-usage: $0 --label <label>
-          [--fixtures F1,F2,F3,F4,F5,F6,F7,F8,F9]
-          [--c1-summary <path>]   # default: benchmark/auto-resolve/results/3bc86dd-iter0033c1-new-20260501T004229Z/summary.json
-          [--f9-judge <path>]     # default: benchmark/auto-resolve/results/4e3d89a-iter-0033a-f9-smoke3-20260430T232747Z/F9-e2e-ideate-to-resolve/judge.json
-          [--results-root <path>] # default: benchmark/auto-resolve/results
-          [--skip-judge]          # skip judge.sh (re-runnable post-hoc)
-EOF
-  exit 1
-}
-
-LABEL=""
-FIXTURES_CSV="F1,F2,F3,F4,F5,F6,F7,F8,F9"
-C1_SUMMARY="benchmark/auto-resolve/results/3bc86dd-iter0033c1-new-20260501T004229Z/summary.json"
-F9_JUDGE="benchmark/auto-resolve/results/4e3d89a-iter-0033a-f9-smoke3-20260430T232747Z/F9-e2e-ideate-to-resolve/judge.json"
-RESULTS_ROOT="benchmark/auto-resolve/results"
-SKIP_JUDGE=0
-while [ $# -gt 0 ]; do
-  case "$1" in
-    --label)         LABEL="$2"; shift 2;;
-    --fixtures)      FIXTURES_CSV="$2"; shift 2;;
-    --c1-summary)    C1_SUMMARY="$2"; shift 2;;
-    --f9-judge)      F9_JUDGE="$2"; shift 2;;
-    --results-root)  RESULTS_ROOT="$2"; shift 2;;
-    --skip-judge)    SKIP_JUDGE=1; shift;;
-    *) usage;;
-  esac
-done
-[ -n "$LABEL" ] || usage
-
-REPO_ROOT="$(cd "$(dirname "$0")/../../.." && pwd)"
-cd "$REPO_ROOT"
-
-# --- Smoke 1b: codex availability fail-fast ---
-echo "=== Smoke 1b: Codex availability ==="
-if ! command -v codex >/dev/null 2>&1; then
-  echo "FAIL: codex not on PATH — iter-0033c L2 arms cannot run" >&2
-  exit 1
-fi
-echo "PASS: $(command -v codex) ($(codex --version 2>&1 | head -1))"
-
-# --- Mirror committed skills to .claude/skills (parity with run-suite.sh:111-141) ---
-# Iteration commits land in config/skills/; the variant-arm runtime resolves
-# from .claude/skills/. Without this step, edits to SKILL.md / phase prompts /
-# _shared scripts (e.g. archive_run.py iter-0033c fix) silently run against the
-# stale mirror. UNSHIPPED list mirrors bin/devlyn.js:299-304.
-SRC_SKILLS="$REPO_ROOT/config/skills"
-DST_SKILLS="$REPO_ROOT/.claude/skills"
-mkdir -p "$DST_SKILLS"
-mirrored=0
-for src_dir in "$SRC_SKILLS"/*/; do
-  [ -d "$src_dir" ] || continue
-  name=$(basename "$src_dir")
-  case "$name" in
-    devlyn:auto-resolve-workspace|devlyn:ideate-workspace|preflight-workspace|roadmap-archival-workspace)
-      continue ;;
-  esac
-  staging="$DST_SKILLS/.${name}.staging"
-  rm -rf "$staging"
-  cp -R "$src_dir" "$staging"
-  rm -rf "$DST_SKILLS/$name"
-  mv "$staging" "$DST_SKILLS/$name"
-  mirrored=$((mirrored + 1))
-done
-echo "[run-iter-0033c] mirrored $mirrored committed skill(s): config/skills/ -> .claude/skills/"
-
-# --- Setup ---
-HEAD_SHA=$(git rev-parse --short HEAD)
-TS=$(date -u +%Y%m%dT%H%M%SZ)
-RUN_ID="${HEAD_SHA}-iter0033c-${LABEL}-${TS}"
-RESULTS_DIR="$RESULTS_ROOT/$RUN_ID"
-mkdir -p "$RESULTS_DIR"
-echo "[run-iter-0033c] RUN_ID=$RUN_ID"
-echo "[run-iter-0033c] RESULTS_DIR=$RESULTS_DIR"
-
-# --- Determine pair-eligible set from manifest input bundle ---
-# Pair eligibility is pre-registered from C1/F9 before any iter-0033c arms run.
-# The later L1 rerun summary is archived into the final manifest for provenance;
-# it must not change the arm-selection set after execution has begun.
-DRAFT_MANIFEST="$RESULTS_DIR/manifest-draft.json"
-python3 benchmark/auto-resolve/scripts/build-pair-eligible-manifest.py \
-  --c1-summary "$C1_SUMMARY" \
-  --f9-judge "$F9_JUDGE" \
-  --l1-rerun-summary "$C1_SUMMARY" \
-  --output "$DRAFT_MANIFEST"
-PAIR_ELIGIBLE=$(python3 - "$DRAFT_MANIFEST" "$REPO_ROOT/benchmark/auto-resolve/scripts" <<'PY'
-import pathlib
-import sys
-
-sys.path.insert(0, sys.argv[2])
-from pair_evidence_contract import loads_strict_json_object
-
-manifest = loads_strict_json_object(pathlib.Path(sys.argv[1]).read_text())
-fixtures = manifest.get("fixtures_pair_eligible")
-if not isinstance(fixtures, list) or not all(isinstance(item, str) for item in fixtures):
-    raise SystemExit("manifest fixtures_pair_eligible must be a string array")
-print(" ".join(fixtures))
-PY
-)
-echo "[run-iter-0033c] pair-eligible: $PAIR_ELIGIBLE"
-
-# --- Per-fixture interleaved arm loop ---
-IFS=',' read -ra FIXTURES <<< "$FIXTURES_CSV"
-declare -a TIMINGS=()
-for short in "${FIXTURES[@]}"; do
-  # Resolve short ID to canonical fixture dir name.
-  case "$short" in
-    F1) fx="F1-cli-trivial-flag";;
-    F2) fx="F2-cli-medium-subcommand";;
-    F3) fx="F3-backend-contract-risk";;
-    F4) fx="F4-web-browser-design";;
-    F5) fx="F5-fix-loop-red-green";;
-    F6) fx="F6-dep-audit-native-module";;
-    F7) fx="F7-out-of-scope-trap";;
-    F8) fx="F8-known-limit-ambiguous";;
-    F9) fx="F9-e2e-ideate-to-resolve";;
-    *) echo "[run-iter-0033c] unknown fixture short id: $short" >&2; exit 1;;
-  esac
-  echo ""
-  echo "=== Fixture $fx ==="
-  ARMS=("solo_claude" "l2_gated")
-  if [[ " $PAIR_ELIGIBLE " =~ " $short " ]]; then
-    ARMS+=("l2_forced")
-  fi
-  for arm in "${ARMS[@]}"; do
-    echo "[run-iter-0033c] $fx :: $arm START $(date -u +%FT%TZ)"
-    arm_t0=$(date +%s)
-    if ! bash benchmark/auto-resolve/scripts/run-fixture.sh \
-         --fixture "$fx" --arm "$arm" \
-         --run-id "$RUN_ID" --resolve-skill new \
-         > "$RESULTS_DIR/${fx}-${arm}.log" 2>&1; then
-      echo "[run-iter-0033c] $fx :: $arm FAILED — see $RESULTS_DIR/${fx}-${arm}.log"
-      # Continue to next arm; full failure surface goes through compare.py gates.
-    fi
-    arm_t1=$(date +%s)
-    elapsed=$((arm_t1 - arm_t0))
-    TIMINGS+=("$fx:$arm:${elapsed}s")
-    echo "[run-iter-0033c] $fx :: $arm END elapsed=${elapsed}s"
-  done
-
-  # Per-fixture judge (graded across ARMS_PRESENT)
-  if [ "$SKIP_JUDGE" -eq 0 ]; then
-    echo "[run-iter-0033c] $fx :: judge START"
-    if ! bash benchmark/auto-resolve/scripts/judge.sh \
-         --fixture "$fx" --run-id "$RUN_ID" \
-         > "$RESULTS_DIR/${fx}-judge.log" 2>&1; then
-      echo "[run-iter-0033c] $fx :: judge FAILED — see $RESULTS_DIR/${fx}-judge.log"
-    fi
-    echo "[run-iter-0033c] $fx :: judge END"
-  fi
-done
-
-# --- Build L1 rerun summary from solo_claude arm result.json + judge.json ---
-L1_RERUN_SUMMARY="$RESULTS_DIR/l1-rerun-summary.json"
-python3 benchmark/auto-resolve/scripts/iter-0033c-l1-summary.py \
-  --results-dir "$RESULTS_DIR" \
-  --out "$L1_RERUN_SUMMARY" \
-  --run-id "$RUN_ID" \
-  --git-sha "$HEAD_SHA"
-
-# --- Build final manifest with real L1 rerun summary ---
-FINAL_MANIFEST="$RESULTS_DIR/iter-0033c-pair-eligible.json"
-python3 benchmark/auto-resolve/scripts/build-pair-eligible-manifest.py \
-  --c1-summary "$C1_SUMMARY" \
-  --f9-judge "$F9_JUDGE" \
-  --l1-rerun-summary "$L1_RERUN_SUMMARY" \
-  --output "$FINAL_MANIFEST"
-
-# --- Run iter-0033c gate compare ---
-GATES_JSON="$RESULTS_DIR/gates.json"
-GATES_MD="$RESULTS_DIR/gates.md"
-python3 benchmark/auto-resolve/scripts/iter-0033c-compare.py \
-  --manifest "$FINAL_MANIFEST" \
-  --results-dir "$RESULTS_DIR" \
-  --work-dir-root /tmp \
-  --run-id "$RUN_ID" \
-  --out-json "$GATES_JSON" \
-  --out-md "$GATES_MD" \
-  || true  # gates may FAIL — exit non-zero handled by inspecting gates.json
-
-echo ""
-echo "=== iter-0033c done ==="
-echo "RESULTS_DIR=$RESULTS_DIR"
-echo "MANIFEST=$FINAL_MANIFEST"
-echo "GATES=$GATES_MD"
-printf '\n--- per-arm wall ---\n%s\n' "$(printf '%s\n' "${TIMINGS[@]}")"