devlyn-cli 3.2.1 → 3.3.1

package/benchmark/auto-resolve/scripts/run-suite.sh

@@ -1,251 +0,0 @@
-#!/usr/bin/env bash
-# run-suite.sh — the single-command benchmark entry.
-#
-# Orchestrates: fixture setup + arm invocations + blind judge + report + ship
-# gate. Called by `npx devlyn-cli benchmark` as well as directly.
-#
-# Usage:
-#   run-suite.sh                            # all fixtures, n=1 smoke
-#   run-suite.sh F2 F5                      # specific fixtures only
-#   run-suite.sh --dry-run                  # skip model invocations, validate setup
-#   run-suite.sh --judge-only --run-id X    # re-judge an existing run
-#   run-suite.sh --label v3.6               # tag this run
-#   run-suite.sh --bless                    # if ship-gate PASS, promote to baselines/shipped.json
-#   run-suite.sh --resolve-skill new        # invoke /devlyn:resolve --spec (the only supported value post iter-0034 cutover; flag kept as accepted no-op for historical runners)
-#   run-suite.sh --suite shadow --dry-run   # list shadow tasks; shadow suite refuses provider/judge runs
-#
-# Exits 0 on PASS, 1 on FAIL.
-
-set -euo pipefail
-
-BENCH_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
-REPO_ROOT="$(cd "$BENCH_ROOT/../.." && pwd)"
-
-N=1
-LABEL=""
-DRY_RUN=0
-JUDGE_ONLY=0
-RUN_ID_ARG=""
-BLESS=0
-ACCEPT_MISSING=0
-SUITE="golden"
-RESOLVE_SKILL="new"
-FIXTURES=()
-
-require_value() {
-  local flag="$1"
-  local value="${2:-}"
-  if [ -z "$value" ] || [[ "$value" == --* ]]; then
-    echo "$flag requires a value" >&2
-    exit 1
-  fi
-}
-
-while [ $# -gt 0 ]; do
-  case "$1" in
-    --n)              require_value "$1" "${2:-}"; N="$2"; shift 2;;
-    --label)          require_value "$1" "${2:-}"; LABEL="$2"; shift 2;;
-    --dry-run)        DRY_RUN=1; shift;;
-    --judge-only)     JUDGE_ONLY=1; shift;;
-    --run-id)         require_value "$1" "${2:-}"; RUN_ID_ARG="$2"; shift 2;;
-    --bless)          BLESS=1; shift;;
-    --accept-missing) ACCEPT_MISSING=1; shift;;
-    --suite)          require_value "$1" "${2:-}"; SUITE="$2"; shift 2;;
-    --resolve-skill)  require_value "$1" "${2:-}"; RESOLVE_SKILL="$2"; shift 2;;
-    -h|--help)
-      head -22 "$0" | sed -n '3,22p'; exit 0;;
-    [FS][0-9]*)       FIXTURES+=("$1"); shift;;
-    *)
-      echo "unknown arg: $1" >&2; exit 1;;
-  esac
-done
-
-# iter-0034 Phase 4 cutover (2026-05-03): OLD `/devlyn:auto-resolve` deleted.
-# Only `new` (= /devlyn:resolve --spec) is supported. The flag is retained as
-# an accepted no-op so historical runners (e.g. run-iter-0033c.sh) keep working
-# without edit. `old` is hard-errored with a pointer at the cutover commit.
-if [ "$RESOLVE_SKILL" = "old" ]; then
-  echo "--resolve-skill old is no longer supported: /devlyn:auto-resolve was deleted in the iter-0034 Phase 4 cutover. Use --resolve-skill new (default) or omit the flag." >&2
-  exit 1
-fi
-[ "$RESOLVE_SKILL" = "new" ] || \
-  { echo "--resolve-skill must be 'new' (got '$RESOLVE_SKILL')" >&2; exit 1; }
-
-# Suite → fixtures directory + discovery prefix.
-case "$SUITE" in
-  golden)  FIXTURES_DIR="$BENCH_ROOT/fixtures";        FIXTURES_GLOB="F*";;
-  shadow)  FIXTURES_DIR="$BENCH_ROOT/shadow-fixtures"; FIXTURES_GLOB="S*";;
-  *)       echo "error: --suite must be 'golden' or 'shadow' (got '$SUITE')" >&2; exit 1;;
-esac
-
-if [ "$SUITE" = "shadow" ] && [ "$DRY_RUN" -eq 0 ]; then
-  echo "shadow suite run-suite is dry-run only. Use benchmark headroom/pair with explicit S* candidates for real provider measurement." >&2
-  exit 1
-fi
-
-# n must be 1 while iteration semantics aren't wired through judge/report.
-# Remove this block when compile-report.py gains multi-iter aggregation.
-case "$N" in ''|*[!0-9]*) echo "error: --n must be an integer" >&2; exit 1;; esac
-[ "$N" -gt 0 ] || { echo "error: --n must be > 0" >&2; exit 1; }
-if [ "$N" -ne 1 ]; then
-  echo "error: --n $N not yet supported — judge/report currently expect a single iteration per fixture." >&2
-  echo "       Track progress in benchmark/auto-resolve/BENCHMARK-DESIGN.md (#multi-iter-roadmap)." >&2
-  exit 2
-fi
-
-# Auto-discover fixtures if none specified
-if [ ${#FIXTURES[@]} -eq 0 ]; then
-  for d in "$FIXTURES_DIR"/$FIXTURES_GLOB/; do
-    [ -d "$d" ] && FIXTURES+=("$(basename "$d")")
-  done
-fi
-
-if [ ${#FIXTURES[@]} -eq 0 ]; then
-  echo "no fixtures found in $FIXTURES_DIR/ — build the suite first" >&2
-  exit 1
-fi
-
-# RUN_ID
-if [ -n "$RUN_ID_ARG" ]; then
-  RUN_ID="$RUN_ID_ARG"
-else
-  TS=$(date -u +%Y%m%dT%H%M%SZ)
-  SHA=$(git -C "$REPO_ROOT" rev-parse --short HEAD 2>/dev/null || echo nogit)
-  RUN_ID="${TS}-${SHA}${LABEL:+-$LABEL}"
-fi
-
-RES_DIR="$BENCH_ROOT/results/$RUN_ID"
-mkdir -p "$RES_DIR"
-
-print_command() {
-  local cmd=(bash "$0" --n "$N" --suite "$SUITE" --resolve-skill "$RESOLVE_SKILL")
-  [ -z "$LABEL" ] || cmd+=(--label "$LABEL")
-  cmd+=(--run-id "$RUN_ID")
-  [ $DRY_RUN -eq 0 ] || cmd+=(--dry-run)
-  [ $JUDGE_ONLY -eq 0 ] || cmd+=(--judge-only)
-  [ $BLESS -eq 0 ] || cmd+=(--bless)
-  [ $ACCEPT_MISSING -eq 0 ] || cmd+=(--accept-missing)
-  if [ ${#FIXTURES[@]} -gt 0 ]; then
-    cmd+=("${FIXTURES[@]}")
-  fi
-  printf 'Command: '
-  printf '%q ' "${cmd[@]}"
-  printf '\n'
-}
-
-echo ""
-echo "═══ Benchmark Suite Run ═══"
-echo "Run-id:        $RUN_ID"
-echo "Label:         ${LABEL:-(unlabeled)}"
-echo "Suite:         $SUITE ($FIXTURES_DIR)"
-echo "Fixtures:      ${FIXTURES[*]}"
-echo "n:             $N"
-echo "Resolve skill: $RESOLVE_SKILL"
-[ $DRY_RUN -eq 1 ] && echo "Mode:          DRY RUN (no model invocations)"
-[ $JUDGE_ONLY -eq 1 ] && echo "Mode:          JUDGE ONLY (re-judging existing artifacts)"
-print_command
-echo ""
-
-# ---- Mirror committed skills into .claude/skills (iter-0017) --------------
-# The variant arm reads $REPO_ROOT/.claude/skills/, but iteration commits land
-# in config/skills/. Without this step every checkout/revert that touches
-# SKILL.md or phase prompts requires a manual `node bin/devlyn.js -y` or
-# surgical cp; forgetting it silently runs the suite against stale skills.
-# Replicates the clean-then-copy semantics of bin/devlyn.js
-# (cleanManagedSkillDirs ~L313 + copyRecursive ~L274). Per-skill staging dir
-# + atomic mv keeps a Ctrl-C window from leaving a managed skill missing.
-# UNSHIPPED list mirrors bin/devlyn.js:299-304 — keep them in sync.
-# Skipped only in --judge-only (no model invocations); runs in --dry-run.
-if [ $JUDGE_ONLY -eq 0 ]; then
-  SRC_SKILLS="$REPO_ROOT/config/skills"
-  DST_SKILLS="$REPO_ROOT/.claude/skills"
-  mkdir -p "$DST_SKILLS"
-  mirrored=0
-  for src_dir in "$SRC_SKILLS"/*/; do
-    [ -d "$src_dir" ] || continue
-    name=$(basename "$src_dir")
-    case "$name" in
-      devlyn:auto-resolve-workspace|devlyn:ideate-workspace|preflight-workspace|roadmap-archival-workspace)
-        continue ;;
-    esac
-    staging="$DST_SKILLS/.${name}.staging"
-    rm -rf "$staging"
-    cp -R "$src_dir" "$staging"
-    rm -rf "$DST_SKILLS/$name"
-    mv "$staging" "$DST_SKILLS/$name"
-    mirrored=$((mirrored + 1))
-  done
-  echo "[suite] mirrored $mirrored committed skill(s): config/skills/ -> .claude/skills/"
-fi
-
-# Prereq checks
-if [ $DRY_RUN -eq 0 ] && [ $JUDGE_ONLY -eq 0 ]; then
-  command -v claude >/dev/null 2>&1 || { echo "claude CLI missing; install Claude Code first"; exit 1; }
-fi
-if [ $JUDGE_ONLY -eq 0 ]; then
-  command -v codex  >/dev/null 2>&1 || echo "warning: codex CLI missing — judge will fail"
-fi
-command -v python3 >/dev/null 2>&1 || { echo "python3 missing"; exit 1; }
-
-# Install test-repo deps once per suite run (shared cache)
-if [ $DRY_RUN -eq 0 ] && [ $JUDGE_ONLY -eq 0 ]; then
-  TEST_REPO="$BENCH_ROOT/fixtures/test-repo"
-  if [ ! -d "$TEST_REPO/node_modules" ]; then
-    echo "[suite] installing test-repo deps (one-time)"
-    if ! (cd "$TEST_REPO" && npm install --no-audit --no-fund --loglevel=error); then
-      echo "[suite] ✗ npm install in test-repo failed — check network/npm auth. Aborting." >&2
-      exit 1
-    fi
-  fi
-fi
-
-# ---- Run arms ---------------------------------------------------------------
-if [ $JUDGE_ONLY -eq 0 ]; then
-  for fid in "${FIXTURES[@]}"; do
-    [ -d "$FIXTURES_DIR/$fid" ] || { echo "[suite] skip $fid (missing)"; continue; }
-    for arm in variant solo_claude bare; do
-      echo "[suite] ► $fid / $arm (resolve-skill=$RESOLVE_SKILL)"
-      extra=""
-      [ $DRY_RUN -eq 1 ] && extra="--dry-run"
-      bash "$BENCH_ROOT/scripts/run-fixture.sh" \
-        --fixture "$fid" --arm "$arm" --run-id "$RUN_ID" \
-        --resolve-skill "$RESOLVE_SKILL" $extra \
-        || echo "[suite] ✗ $fid / $arm (arm failure tolerated; artifacts still captured)"
-    done
-  done
-fi
-
-# ---- Judge ------------------------------------------------------------------
-for fid in "${FIXTURES[@]}"; do
-  if [ ! -d "$BENCH_ROOT/results/$RUN_ID/$fid" ]; then
-    echo "[suite] skip judge for $fid (no results)"
-    continue
-  fi
-  if [ $DRY_RUN -eq 1 ]; then
-    echo "[suite] DRY RUN — skipping judge for $fid"
-    continue
-  fi
-  echo "[suite] ► judge $fid"
-  bash "$BENCH_ROOT/scripts/judge.sh" --fixture "$fid" --run-id "$RUN_ID" \
-    || echo "[suite] ✗ judge failed for $fid (will appear as NO_JUDGE in report)"
-done
-
-# ---- Compile report + ship gate --------------------------------------------
-if [ $DRY_RUN -eq 1 ]; then
-  echo ""
-  echo "[suite] DRY RUN complete — results in $RES_DIR"
-  if [ "$SUITE" = "shadow" ]; then
-    echo "Use benchmark headroom/pair with explicit S* candidates for real provider measurement."
-  else
-    echo "Run without --dry-run to invoke models."
-  fi
-  exit 0
-fi
-
-echo ""
-python3 "$BENCH_ROOT/scripts/compile-report.py" --run-id "$RUN_ID" ${LABEL:+--label "$LABEL"}
-
-extra_flag=""
-[ $BLESS -eq 1 ] && extra_flag="$extra_flag --bless"
-[ $ACCEPT_MISSING -eq 1 ] && extra_flag="$extra_flag --accept-missing"
-python3 "$BENCH_ROOT/scripts/ship-gate.py" --run-id "$RUN_ID" $extra_flag

package/benchmark/auto-resolve/scripts/run-swebench-frozen-corpus.sh

@@ -1,310 +0,0 @@
-#!/usr/bin/env bash
-# Run a prepared SWE-bench frozen VERIFY corpus and gate the result set.
-set -euo pipefail
-
-usage() {
-  cat >&2 <<EOF
-usage: $0 --manifest <path> [--run-prefix ID] [--pair-mode forced|gated]
-          [--min-runs N] [--out-json <path>] [--out-md <path>]
-          [--max-pair-solo-wall-ratio N] [--timeout-seconds N]
-          [--run-ids-out <path>] [--resume-completed-arms]
-          [--prepare-only] [--gate-only-run-ids <path>]
-
-Reads the manifest from prepare-swebench-frozen-corpus.py, runs each prepared
-case through run-frozen-verify-pair.sh, then applies frozen-verify-gate.py to
-the resulting run ids. --prepare-only validates patch application without
-provider calls and skips the gate. --gate-only-run-ids reruns the gate over an
-existing newline-delimited run-id file without invoking providers.
-EOF
-  exit "${1:-1}"
-}
-
-require_value() {
-  local flag="$1"
-  local value="${2:-}"
-  if [ -z "$value" ] || [[ "$value" == --* ]]; then
-    echo "$flag requires a value" >&2
-    exit 1
-  fi
-}
-
-MANIFEST=""
-RUN_PREFIX=""
-PAIR_MODE="gated"
-MIN_RUNS=2
-OUT_JSON=""
-OUT_MD=""
-MAX_PAIR_SOLO_WALL_RATIO=""
-PREPARE_ONLY=0
-GATE_ONLY_RUN_IDS=""
-TIMEOUT_SECONDS=""
-RUN_IDS_OUT=""
-RESUME_COMPLETED_ARMS=0
-while [ $# -gt 0 ]; do
-  case "$1" in
-    --manifest) require_value "$1" "${2:-}"; MANIFEST="$2"; shift 2;;
-    --run-prefix) require_value "$1" "${2:-}"; RUN_PREFIX="$2"; shift 2;;
-    --pair-mode) require_value "$1" "${2:-}"; PAIR_MODE="$2"; shift 2;;
-    --min-runs) require_value "$1" "${2:-}"; MIN_RUNS="$2"; shift 2;;
-    --out-json) require_value "$1" "${2:-}"; OUT_JSON="$2"; shift 2;;
-    --out-md) require_value "$1" "${2:-}"; OUT_MD="$2"; shift 2;;
-    --max-pair-solo-wall-ratio) require_value "$1" "${2:-}"; MAX_PAIR_SOLO_WALL_RATIO="$2"; shift 2;;
-    --timeout-seconds) require_value "$1" "${2:-}"; TIMEOUT_SECONDS="$2"; shift 2;;
-    --run-ids-out) require_value "$1" "${2:-}"; RUN_IDS_OUT="$2"; shift 2;;
-    --resume-completed-arms) RESUME_COMPLETED_ARMS=1; shift;;
-    --prepare-only) PREPARE_ONLY=1; shift;;
-    --gate-only-run-ids) require_value "$1" "${2:-}"; GATE_ONLY_RUN_IDS="$2"; shift 2;;
-    -h|--help) usage 0;;
-    *) echo "unknown arg: $1" >&2; usage 1;;
-  esac
-done
-
-[ -n "$MANIFEST" ] || usage 1
-[ -f "$MANIFEST" ] || { echo "manifest not found: $MANIFEST" >&2; exit 1; }
-[ "$PAIR_MODE" = "forced" ] || [ "$PAIR_MODE" = "gated" ] || { echo "--pair-mode must be forced|gated" >&2; exit 1; }
-case "$MIN_RUNS" in ''|*[!0-9]*) echo "--min-runs must be an integer" >&2; exit 1;; esac
-[ "$MIN_RUNS" -gt 0 ] || { echo "--min-runs must be > 0" >&2; exit 1; }
-if [ -n "$TIMEOUT_SECONDS" ]; then
-  case "$TIMEOUT_SECONDS" in ''|*[!0-9]*) echo "--timeout-seconds must be an integer" >&2; exit 1;; esac
-  [ "$TIMEOUT_SECONDS" -gt 0 ] || { echo "--timeout-seconds must be > 0" >&2; exit 1; }
-fi
-if [ -n "$MAX_PAIR_SOLO_WALL_RATIO" ]; then
-  python3 - "$MAX_PAIR_SOLO_WALL_RATIO" <<'PY' || { echo "--max-pair-solo-wall-ratio must be a positive number" >&2; exit 1; }
-import sys
-try:
-    value = float(sys.argv[1])
-except ValueError:
-    raise SystemExit(1)
-if value <= 0:
-    raise SystemExit(1)
-PY
-fi
-[ -z "$GATE_ONLY_RUN_IDS" ] || [ -f "$GATE_ONLY_RUN_IDS" ] || { echo "run ids file not found: $GATE_ONLY_RUN_IDS" >&2; exit 1; }
-[ "$PREPARE_ONLY" -eq 0 ] || [ -z "$GATE_ONLY_RUN_IDS" ] || { echo "--prepare-only and --gate-only-run-ids are mutually exclusive" >&2; exit 1; }
-
-SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
-BENCH_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
-if [ -z "$RUN_PREFIX" ]; then
-  RUN_PREFIX="$(date -u +%Y%m%dT%H%M%SZ)-swebench-frozen"
-fi
-
-print_command() {
-  local cmd=(bash "$0" --manifest "$MANIFEST" --run-prefix "$RUN_PREFIX")
-  cmd+=(--pair-mode "$PAIR_MODE")
-  cmd+=(--min-runs "$MIN_RUNS")
-  [ -z "$OUT_JSON" ] || cmd+=(--out-json "$OUT_JSON")
-  [ -z "$OUT_MD" ] || cmd+=(--out-md "$OUT_MD")
-  [ -z "$MAX_PAIR_SOLO_WALL_RATIO" ] || cmd+=(--max-pair-solo-wall-ratio "$MAX_PAIR_SOLO_WALL_RATIO")
-  [ -z "$TIMEOUT_SECONDS" ] || cmd+=(--timeout-seconds "$TIMEOUT_SECONDS")
-  [ -z "$RUN_IDS_OUT" ] || cmd+=(--run-ids-out "$RUN_IDS_OUT")
-  [ "$RESUME_COMPLETED_ARMS" -eq 0 ] || cmd+=(--resume-completed-arms)
-  [ "$PREPARE_ONLY" -eq 0 ] || cmd+=(--prepare-only)
-  [ -z "$GATE_ONLY_RUN_IDS" ] || cmd+=(--gate-only-run-ids "$GATE_ONLY_RUN_IDS")
-  printf 'Command: '
-  printf '%q ' "${cmd[@]}"
-  printf '\n'
-}
-
-echo ""
-echo "═══ SWE-bench Frozen VERIFY Corpus Run ═══"
-echo "Run-prefix: $RUN_PREFIX"
-echo "Pair mode:  $PAIR_MODE"
-echo "Min runs:   $MIN_RUNS"
-[ -z "$MAX_PAIR_SOLO_WALL_RATIO" ] || echo "Wall cap:   pair/solo <= ${MAX_PAIR_SOLO_WALL_RATIO}x"
-print_command
-echo ""
-
-TMP_RUN_IDS="$(mktemp)"
-trap 'rm -f "$TMP_RUN_IDS"' EXIT
-ROW_FAILURES=0
-
-python3 - "$MANIFEST" "$GATE_ONLY_RUN_IDS" "$SCRIPT_DIR" <<'PY'
-import pathlib
-import sys
-
-sys.path.insert(0, sys.argv[3])
-from pair_evidence_contract import loads_strict_json_object
-
-manifest_path = pathlib.Path(sys.argv[1])
-gate_only_run_ids = sys.argv[2]
-try:
-    manifest = loads_strict_json_object(manifest_path.read_text())
-except ValueError as exc:
-    if str(exc) == "top-level JSON value must be an object":
-        raise SystemExit("manifest malformed: expected JSON object") from exc
-    raise
-if not isinstance(manifest, dict):
-    raise SystemExit("manifest malformed: expected JSON object")
-cases_root = manifest.get("cases_root")
-if not isinstance(cases_root, str) or not cases_root.strip():
-    raise SystemExit("manifest malformed: missing non-empty cases_root")
-if gate_only_run_ids:
-    raise SystemExit(0)
-prepared = manifest.get("prepared")
-if not isinstance(prepared, list) or not prepared:
-    raise SystemExit("manifest malformed: prepared must be a non-empty array")
-for index, row in enumerate(prepared, start=1):
-    if not isinstance(row, dict):
-        raise SystemExit(f"manifest malformed: prepared[{index}] expected JSON object")
-    for key in ("instance_id", "case_dir", "repo_dir"):
-        value = row.get(key)
-        if not isinstance(value, str) or not value.strip():
-            raise SystemExit(f"manifest malformed: prepared[{index}] missing non-empty {key}")
-PY
-
-if [ -n "$GATE_ONLY_RUN_IDS" ]; then
-  python3 - "$GATE_ONLY_RUN_IDS" "$TMP_RUN_IDS" <<'PY'
-import pathlib
-import re
-import sys
-
-source = pathlib.Path(sys.argv[1])
-dest = pathlib.Path(sys.argv[2])
-safe = re.compile(r"^[A-Za-z0-9_.-]+$")
-run_ids: list[str] = []
-for line_no, line in enumerate(source.read_text(encoding="utf8").splitlines(), start=1):
-    run_id = line.strip()
-    if not run_id:
-        raise SystemExit(f"run ids malformed: line {line_no} is empty")
-    if not safe.match(run_id):
-        raise SystemExit(f"run ids malformed: line {line_no} has unsafe run id")
-    run_ids.append(run_id)
-if not run_ids:
-    raise SystemExit("run ids malformed: no run ids")
-dest.write_text("\n".join(run_ids) + "\n", encoding="utf8")
-PY
-else
-  while IFS=$'\t' read -r index instance_id cases_root repo_dir diff_path; do
-    [ -n "$instance_id" ] || continue
-    run_id="${RUN_PREFIX}-${index}-${instance_id}"
-    safe_run_id="$(printf '%s' "$run_id" | tr -c 'A-Za-z0-9_.-' '-')"
-    echo "[swebench-frozen-corpus] ${index}: ${instance_id} -> ${safe_run_id}"
-    cmd=(
-      bash "$SCRIPT_DIR/run-frozen-verify-pair.sh"
-      --fixture "$instance_id"
-      --fixtures-root "$cases_root"
-      --base-repo "$repo_dir"
-      --diff "$diff_path"
-      --run-id "$safe_run_id"
-      --pair-mode "$PAIR_MODE"
-    )
-    if [ -n "$TIMEOUT_SECONDS" ]; then
-      cmd+=(--timeout-seconds "$TIMEOUT_SECONDS")
-    fi
-    if [ "$PREPARE_ONLY" -eq 1 ]; then
-      cmd+=(--prepare-only)
-    fi
-    if [ "$RESUME_COMPLETED_ARMS" -eq 1 ]; then
-      cmd+=(--resume-completed-arms)
-    fi
-    set +e
-    "${cmd[@]}" </dev/null
-    row_exit=$?
-    set -e
-    if [ "$row_exit" -ne 0 ]; then
-      echo "[swebench-frozen-corpus] row failed: ${safe_run_id} exit=${row_exit}" >&2
-      ROW_FAILURES=$((ROW_FAILURES + 1))
-      python3 - "$BENCH_ROOT/results/$safe_run_id" "$instance_id" "$row_exit" <<'PY'
-import json
-import pathlib
-import sys
-
-run_root = pathlib.Path(sys.argv[1])
-instance_id = sys.argv[2]
-row_exit = int(sys.argv[3])
-run_root.mkdir(parents=True, exist_ok=True)
-for arm in ("solo", "pair"):
-    arm_root = run_root / arm
-    arm_root.mkdir(parents=True, exist_ok=True)
-    input_path = arm_root / "input.md"
-    if not input_path.exists():
-        input_path.write_text(
-            f"Use /devlyn:resolve --verify-only --spec docs/roadmap/phase-1/{instance_id}.md.\n",
-            encoding="utf8",
-        )
-compare_path = run_root / "compare.json"
-if not compare_path.exists():
-    compare_path.write_text(
-        json.dumps(
-            {
-                "solo": {"invoke_exit": row_exit, "timed_out": False},
-                "pair": {"invoke_exit": row_exit, "timed_out": False, "pair_mode": False},
-                "comparison": {
-                    "pair_trigger_missed": False,
-                    "pair_verdict_lift": False,
-                    "pair_internal_verdict_lift": False,
-                    "row_failed_before_compare": True,
-                    "row_exit": row_exit,
-                },
-            },
-            indent=2,
-        )
-        + "\n",
-        encoding="utf8",
-    )
-PY
-    fi
-    printf '%s\n' "$safe_run_id" >> "$TMP_RUN_IDS"
-  done < <(python3 - "$MANIFEST" "$SCRIPT_DIR" <<'PY'
-import pathlib, sys
-
-sys.path.insert(0, sys.argv[2])
-from pair_evidence_contract import loads_strict_json_object
-
-manifest = loads_strict_json_object(pathlib.Path(sys.argv[1]).read_text())
-for index, row in enumerate(manifest["prepared"], start=1):
-    instance_id = row["instance_id"]
-    case_dir = pathlib.Path(row["case_dir"])
-    repo_dir = pathlib.Path(row["repo_dir"])
-    print("\t".join([
-        str(index),
-        instance_id,
-        str(case_dir.parent),
-        str(repo_dir),
-        str(case_dir / "model.patch"),
-    ]))
-PY
-)
-fi
-
-if [ -n "$RUN_IDS_OUT" ]; then
-  mkdir -p "$(dirname "$RUN_IDS_OUT")"
-  cp "$TMP_RUN_IDS" "$RUN_IDS_OUT"
-fi
-
-if [ "$PREPARE_ONLY" -eq 1 ]; then
-  echo "[swebench-frozen-corpus] prepare-only complete; gate skipped"
-  if [ "$ROW_FAILURES" -gt 0 ]; then
-    echo "[swebench-frozen-corpus] row failures: $ROW_FAILURES" >&2
-    exit 1
-  fi
-  exit 0
-fi
-
-run_count="$(wc -l < "$TMP_RUN_IDS" | tr -d ' ')"
-[ "$run_count" -gt 0 ] || { echo "manifest prepared no runs" >&2; exit 1; }
-
-fixtures_root="$(python3 - "$MANIFEST" "$SCRIPT_DIR" <<'PY'
-import pathlib, sys
-
-sys.path.insert(0, sys.argv[2])
-from pair_evidence_contract import loads_strict_json_object
-
-manifest = loads_strict_json_object(pathlib.Path(sys.argv[1]).read_text())
-print(manifest["cases_root"])
-PY
-)"
-gate_args=(
-  python3 "$SCRIPT_DIR/frozen-verify-gate.py"
-  --fixtures-root "$fixtures_root"
-  --min-runs "$MIN_RUNS"
-  --require-hypothesis-trigger
-)
-[ -z "$OUT_JSON" ] || gate_args+=(--out-json "$OUT_JSON")
-[ -z "$OUT_MD" ] || gate_args+=(--out-md "$OUT_MD")
-[ -z "$MAX_PAIR_SOLO_WALL_RATIO" ] || gate_args+=(--max-pair-solo-wall-ratio "$MAX_PAIR_SOLO_WALL_RATIO")
-while IFS= read -r run_id; do
-  gate_args+=(--run-id "$run_id")
-done < "$TMP_RUN_IDS"
-
-"${gate_args[@]}"