devlyn-cli 1.15.0 → 2.1.0

package/benchmark/auto-resolve/scripts/run-iter-0033c.sh

@@ -0,0 +1,234 @@
+#!/usr/bin/env bash
+# run-iter-0033c.sh — orchestrate the iter-0033c suite (NEW L2 vs NEW L1).
+#
+# Codex R0.5-infra design: bypass run-suite.sh + ship-gate.py + compile-report.py
+# (those enforce variant/bare semantics that don't apply here). Call run-fixture.sh
+# directly per fixture per arm; per-fixture interleaving for fail-early on hard-floor
+# violations (Codex R0.5-infra Q4).
+#
+# Per Mission 1: serial only, no parallel-fleet.
+#
+# Usage:
+#   run-iter-0033c.sh --label <label> [--fixtures F1,F2,...] [--c1-summary <path>] [--f9-judge <path>]
+#                     [--manifest-out <path>] [--results-out-dir <path>] [--skip-judge]
+#
+# Pre-flight: smoke 1b (codex availability) — fail-fast.
+# Arms per fixture:
+#   - All fixtures: solo_claude (L1 rerun) + l2_gated (L2 natural triggers)
+#   - Pair-eligible (per manifest): also l2_forced (L2 diagnostic)
+# After arms: judge.sh per fixture; manifest build; iter-0033c-compare.py.
+set -euo pipefail
+
+usage() {
+  cat >&2 <<EOF
+usage: $0 --label <label>
+          [--fixtures F1,F2,F3,F4,F5,F6,F7,F8,F9]
+          [--c1-summary <path>]   # default: benchmark/auto-resolve/results/3bc86dd-iter0033c1-new-20260501T004229Z/summary.json
+          [--f9-judge <path>]     # default: benchmark/auto-resolve/results/4e3d89a-iter-0033a-f9-smoke3-20260430T232747Z/F9-e2e-ideate-to-resolve/judge.json
+          [--results-root <path>] # default: benchmark/auto-resolve/results
+          [--skip-judge]          # skip judge.sh (re-runnable post-hoc)
+EOF
+  exit 1
+}
+
+LABEL=""
+FIXTURES_CSV="F1,F2,F3,F4,F5,F6,F7,F8,F9"
+C1_SUMMARY="benchmark/auto-resolve/results/3bc86dd-iter0033c1-new-20260501T004229Z/summary.json"
+F9_JUDGE="benchmark/auto-resolve/results/4e3d89a-iter-0033a-f9-smoke3-20260430T232747Z/F9-e2e-ideate-to-resolve/judge.json"
+RESULTS_ROOT="benchmark/auto-resolve/results"
+SKIP_JUDGE=0
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --label)         LABEL="$2"; shift 2;;
+    --fixtures)      FIXTURES_CSV="$2"; shift 2;;
+    --c1-summary)    C1_SUMMARY="$2"; shift 2;;
+    --f9-judge)      F9_JUDGE="$2"; shift 2;;
+    --results-root)  RESULTS_ROOT="$2"; shift 2;;
+    --skip-judge)    SKIP_JUDGE=1; shift;;
+    *) usage;;
+  esac
+done
+[ -n "$LABEL" ] || usage
+
+REPO_ROOT="$(cd "$(dirname "$0")/../../.." && pwd)"
+cd "$REPO_ROOT"
+
+# --- Smoke 1b: codex availability fail-fast ---
+echo "=== Smoke 1b: Codex availability ==="
+if ! command -v codex >/dev/null 2>&1; then
+  echo "FAIL: codex not on PATH — iter-0033c L2 arms cannot run" >&2
+  exit 1
+fi
+echo "PASS: $(command -v codex) ($(codex --version 2>&1 | head -1))"
+
+# --- Mirror committed skills to .claude/skills (parity with run-suite.sh:111-141) ---
+# Iteration commits land in config/skills/; the variant-arm runtime resolves
+# from .claude/skills/. Without this step, edits to SKILL.md / phase prompts /
+# _shared scripts (e.g. archive_run.py iter-0033c fix) silently run against the
+# stale mirror. UNSHIPPED list mirrors bin/devlyn.js:299-304.
+SRC_SKILLS="$REPO_ROOT/config/skills"
+DST_SKILLS="$REPO_ROOT/.claude/skills"
+mkdir -p "$DST_SKILLS"
+mirrored=0
+for src_dir in "$SRC_SKILLS"/*/; do
+  [ -d "$src_dir" ] || continue
+  name=$(basename "$src_dir")
+  case "$name" in
+    devlyn:auto-resolve-workspace|devlyn:ideate-workspace|preflight-workspace|roadmap-archival-workspace)
+      continue ;;
+  esac
+  staging="$DST_SKILLS/.${name}.staging"
+  rm -rf "$staging"
+  cp -R "$src_dir" "$staging"
+  rm -rf "$DST_SKILLS/$name"
+  mv "$staging" "$DST_SKILLS/$name"
+  mirrored=$((mirrored + 1))
+done
+echo "[run-iter-0033c] mirrored $mirrored committed skill(s): config/skills/ -> .claude/skills/"
+
+# --- Setup ---
+HEAD_SHA=$(git rev-parse --short HEAD)
+TS=$(date -u +%Y%m%dT%H%M%SZ)
+RUN_ID="${HEAD_SHA}-iter0033c-${LABEL}-${TS}"
+RESULTS_DIR="$RESULTS_ROOT/$RUN_ID"
+mkdir -p "$RESULTS_DIR"
+echo "[run-iter-0033c] RUN_ID=$RUN_ID"
+echo "[run-iter-0033c] RESULTS_DIR=$RESULTS_DIR"
+
+# --- Determine pair-eligible set from manifest input bundle ---
+# Build a draft manifest using the C1 summary as the L1 placeholder; we'll
+# rebuild with the real L1 rerun summary at the end. For now we just need
+# the pair-eligible set for arm-selection per fixture.
+DRAFT_MANIFEST="$RESULTS_DIR/manifest-draft.json"
+python3 benchmark/auto-resolve/scripts/build-pair-eligible-manifest.py \
+  --c1-summary "$C1_SUMMARY" \
+  --f9-judge "$F9_JUDGE" \
+  --l1-rerun-summary "$C1_SUMMARY" \
+  --output "$DRAFT_MANIFEST"
+PAIR_ELIGIBLE=$(python3 -c "import json;print(' '.join(json.load(open('$DRAFT_MANIFEST'))['fixtures_pair_eligible']))")
+echo "[run-iter-0033c] pair-eligible: $PAIR_ELIGIBLE"
+
+# --- Per-fixture interleaved arm loop ---
+IFS=',' read -ra FIXTURES <<< "$FIXTURES_CSV"
+declare -a TIMINGS=()
+for short in "${FIXTURES[@]}"; do
+  # Resolve short ID to canonical fixture dir name.
+  case "$short" in
+    F1) fx="F1-cli-trivial-flag";;
+    F2) fx="F2-cli-medium-subcommand";;
+    F3) fx="F3-backend-contract-risk";;
+    F4) fx="F4-web-browser-design";;
+    F5) fx="F5-fix-loop-red-green";;
+    F6) fx="F6-dep-audit-native-module";;
+    F7) fx="F7-out-of-scope-trap";;
+    F8) fx="F8-known-limit-ambiguous";;
+    F9) fx="F9-e2e-ideate-to-resolve";;
+    *) echo "[run-iter-0033c] unknown fixture short id: $short" >&2; exit 1;;
+  esac
+  echo ""
+  echo "=== Fixture $fx ==="
+  ARMS=("solo_claude" "l2_gated")
+  if [[ " $PAIR_ELIGIBLE " =~ " $short " ]]; then
+    ARMS+=("l2_forced")
+  fi
+  for arm in "${ARMS[@]}"; do
+    echo "[run-iter-0033c] $fx :: $arm START $(date -u +%FT%TZ)"
+    arm_t0=$(date +%s)
+    if ! bash benchmark/auto-resolve/scripts/run-fixture.sh \
+         --fixture "$fx" --arm "$arm" \
+         --run-id "$RUN_ID" --resolve-skill new \
+         > "$RESULTS_DIR/${fx}-${arm}.log" 2>&1; then
+      echo "[run-iter-0033c] $fx :: $arm FAILED — see $RESULTS_DIR/${fx}-${arm}.log"
+      # Continue to next arm; full failure surface goes through compare.py gates.
+    fi
+    arm_t1=$(date +%s)
+    elapsed=$((arm_t1 - arm_t0))
+    TIMINGS+=("$fx:$arm:${elapsed}s")
+    echo "[run-iter-0033c] $fx :: $arm END elapsed=${elapsed}s"
+  done
+
+  # Per-fixture judge (graded across ARMS_PRESENT)
+  if [ "$SKIP_JUDGE" -eq 0 ]; then
+    echo "[run-iter-0033c] $fx :: judge START"
+    if ! bash benchmark/auto-resolve/scripts/judge.sh \
+         --fixture "$fx" --run-id "$RUN_ID" \
+         > "$RESULTS_DIR/${fx}-judge.log" 2>&1; then
+      echo "[run-iter-0033c] $fx :: judge FAILED — see $RESULTS_DIR/${fx}-judge.log"
+    fi
+    echo "[run-iter-0033c] $fx :: judge END"
+  fi
+done
+
+# --- Build L1 rerun summary from solo_claude arm result.json + judge.json ---
+L1_RERUN_SUMMARY="$RESULTS_DIR/l1-rerun-summary.json"
+python3 - "$RESULTS_DIR" "$L1_RERUN_SUMMARY" "$RUN_ID" "$HEAD_SHA" <<'PY'
+import json, sys
+from pathlib import Path
+results_dir = Path(sys.argv[1])
+out_path = Path(sys.argv[2])
+run_id = sys.argv[3]
+head_sha = sys.argv[4]
+rows = []
+for fx_dir in sorted(results_dir.iterdir()):
+    if not fx_dir.is_dir():
+        continue
+    judge_p = fx_dir / "judge.json"
+    if not judge_p.is_file():
+        continue
+    judge = json.loads(judge_p.read_text())
+    mapping = judge.get("_blind_mapping") or {}
+    inv = {v: k for k, v in mapping.items()}
+    arms = {}
+    for arm_name in ("solo_claude", "l2_gated", "l2_forced", "bare"):
+        letter = inv.get(arm_name)
+        if not letter:
+            continue
+        arm_dir = fx_dir / arm_name
+        result = {}
+        if (arm_dir / "result.json").is_file():
+            result = json.loads((arm_dir / "result.json").read_text())
+        arms[arm_name] = {
+            "score": judge.get(f"{letter}_score"),
+            "wall_s": result.get("elapsed_seconds"),
+            "verify_score": result.get("verify_score"),
+            "files_changed": result.get("files_changed"),
+            "timed_out": result.get("timed_out"),
+            "disqualifier": result.get("disqualifier"),
+        }
+    rows.append({"fixture": fx_dir.name, "arms": arms})
+out = {
+    "run_id": run_id,
+    "git_sha": head_sha,
+    "fixtures_total": len(rows),
+    "rows": rows,
+}
+out_path.write_text(json.dumps(out, indent=2) + "\n")
+print(f"[l1-rerun-summary] wrote {out_path} (fixtures={len(rows)})")
+PY
+
+# --- Build final manifest with real L1 rerun summary ---
+FINAL_MANIFEST="$RESULTS_DIR/iter-0033c-pair-eligible.json"
+python3 benchmark/auto-resolve/scripts/build-pair-eligible-manifest.py \
+  --c1-summary "$C1_SUMMARY" \
+  --f9-judge "$F9_JUDGE" \
+  --l1-rerun-summary "$L1_RERUN_SUMMARY" \
+  --output "$FINAL_MANIFEST"
+
+# --- Run iter-0033c gate compare ---
+GATES_JSON="$RESULTS_DIR/gates.json"
+GATES_MD="$RESULTS_DIR/gates.md"
+python3 benchmark/auto-resolve/scripts/iter-0033c-compare.py \
+  --manifest "$FINAL_MANIFEST" \
+  --results-dir "$RESULTS_DIR" \
+  --work-dir-root /tmp \
+  --run-id "$RUN_ID" \
+  --out-json "$GATES_JSON" \
+  --out-md "$GATES_MD" \
+  || true  # gates may FAIL — exit non-zero handled by inspecting gates.json
+
+echo ""
+echo "=== iter-0033c done ==="
+echo "RESULTS_DIR=$RESULTS_DIR"
+echo "MANIFEST=$FINAL_MANIFEST"
+echo "GATES=$GATES_MD"
+printf '\n--- per-arm wall ---\n%s\n' "$(printf '%s\n' "${TIMINGS[@]}")"

package/benchmark/auto-resolve/scripts/run-suite.sh

@@ -0,0 +1,214 @@
+#!/usr/bin/env bash
+# run-suite.sh — the single-command benchmark entry.
+#
+# Orchestrates: fixture setup + arm invocations + blind judge + report + ship
+# gate. Called by `npx devlyn-cli benchmark` as well as directly.
+#
+# Usage:
+#   run-suite.sh                            # all fixtures, n=1 smoke
+#   run-suite.sh --n 3                      # 3 runs per fixture for ship decisions
+#   run-suite.sh F2 F5                      # specific fixtures only
+#   run-suite.sh --dry-run                  # skip model invocations, validate setup
+#   run-suite.sh --judge-only --run-id X    # re-judge an existing run
+#   run-suite.sh --label v3.6               # tag this run
+#   run-suite.sh --bless                    # if ship-gate PASS, promote to baselines/shipped.json
+#   run-suite.sh --resolve-skill new        # invoke /devlyn:resolve --spec (the only supported value post iter-0034 cutover; flag kept as accepted no-op for historical runners)
+#
+# Exits 0 on PASS, 1 on FAIL.
+
+set -euo pipefail
+
+BENCH_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
+REPO_ROOT="$(cd "$BENCH_ROOT/../.." && pwd)"
+
+N=1
+LABEL=""
+DRY_RUN=0
+JUDGE_ONLY=0
+RUN_ID_ARG=""
+BLESS=0
+ACCEPT_MISSING=0
+SUITE="golden"
+RESOLVE_SKILL="new"
+FIXTURES=()
+
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --n)              N="$2"; shift 2;;
+    --label)          LABEL="$2"; shift 2;;
+    --dry-run)        DRY_RUN=1; shift;;
+    --judge-only)     JUDGE_ONLY=1; shift;;
+    --run-id)         RUN_ID_ARG="$2"; shift 2;;
+    --bless)          BLESS=1; shift;;
+    --accept-missing) ACCEPT_MISSING=1; shift;;
+    --suite)          SUITE="$2"; shift 2;;
+    --resolve-skill)  RESOLVE_SKILL="$2"; shift 2;;
+    -h|--help)
+      head -22 "$0" | sed -n '3,22p'; exit 0;;
+    [FS][0-9]*)       FIXTURES+=("$1"); shift;;
+    *)
+      echo "unknown arg: $1" >&2; exit 1;;
+  esac
+done
+
+# iter-0034 Phase 4 cutover (2026-05-03): OLD `/devlyn:auto-resolve` deleted.
+# Only `new` (= /devlyn:resolve --spec) is supported. The flag is retained as
+# an accepted no-op so historical runners (e.g. run-iter-0033c.sh) keep working
+# without edit. `old` is hard-errored with a pointer at the cutover commit.
+if [ "$RESOLVE_SKILL" = "old" ]; then
+  echo "--resolve-skill old is no longer supported: /devlyn:auto-resolve was deleted in the iter-0034 Phase 4 cutover. Use --resolve-skill new (default) or omit the flag." >&2
+  exit 1
+fi
+[ "$RESOLVE_SKILL" = "new" ] || \
+  { echo "--resolve-skill must be 'new' (got '$RESOLVE_SKILL')" >&2; exit 1; }
+
+# Suite → fixtures directory + discovery prefix.
+case "$SUITE" in
+  golden)  FIXTURES_DIR="$BENCH_ROOT/fixtures";        FIXTURES_GLOB="F*";;
+  shadow)  FIXTURES_DIR="$BENCH_ROOT/shadow-fixtures"; FIXTURES_GLOB="S*";;
+  *)       echo "error: --suite must be 'golden' or 'shadow' (got '$SUITE')" >&2; exit 1;;
+esac
+
+# n must be 1 while iteration semantics aren't wired through judge/report.
+# Remove this block when compile-report.py gains multi-iter aggregation.
+if [ "$N" -ne 1 ]; then
+  echo "error: --n $N not yet supported — judge/report currently expect a single iteration per fixture." >&2
+  echo "       Track progress in benchmark/auto-resolve/BENCHMARK-DESIGN.md (#multi-iter-roadmap)." >&2
+  exit 2
+fi
+
+# Auto-discover fixtures if none specified
+if [ ${#FIXTURES[@]} -eq 0 ]; then
+  for d in "$FIXTURES_DIR"/$FIXTURES_GLOB/; do
+    [ -d "$d" ] && FIXTURES+=("$(basename "$d")")
+  done
+fi
+
+if [ ${#FIXTURES[@]} -eq 0 ]; then
+  echo "no fixtures found in $FIXTURES_DIR/ — build the suite first" >&2
+  exit 1
+fi
+
+# RUN_ID
+if [ -n "$RUN_ID_ARG" ]; then
+  RUN_ID="$RUN_ID_ARG"
+else
+  TS=$(date -u +%Y%m%dT%H%M%SZ)
+  SHA=$(git -C "$REPO_ROOT" rev-parse --short HEAD 2>/dev/null || echo nogit)
+  RUN_ID="${TS}-${SHA}${LABEL:+-$LABEL}"
+fi
+
+RES_DIR="$BENCH_ROOT/results/$RUN_ID"
+mkdir -p "$RES_DIR"
+
+echo ""
+echo "═══ Benchmark Suite Run ═══"
+echo "Run-id:        $RUN_ID"
+echo "Label:         ${LABEL:-(unlabeled)}"
+echo "Suite:         $SUITE ($FIXTURES_DIR)"
+echo "Fixtures:      ${FIXTURES[*]}"
+echo "n:             $N"
+echo "Resolve skill: $RESOLVE_SKILL"
+[ $DRY_RUN -eq 1 ] && echo "Mode:          DRY RUN (no model invocations)"
+[ $JUDGE_ONLY -eq 1 ] && echo "Mode:          JUDGE ONLY (re-judging existing artifacts)"
+echo ""
+
+# ---- Mirror committed skills into .claude/skills (iter-0017) --------------
+# The variant arm reads $REPO_ROOT/.claude/skills/, but iteration commits land
+# in config/skills/. Without this step every checkout/revert that touches
+# SKILL.md or phase prompts requires a manual `node bin/devlyn.js -y` or
+# surgical cp; forgetting it silently runs the suite against stale skills.
+# Replicates the clean-then-copy semantics of bin/devlyn.js
+# (cleanManagedSkillDirs ~L313 + copyRecursive ~L274). Per-skill staging dir
+# + atomic mv keeps a Ctrl-C window from leaving a managed skill missing.
+# UNSHIPPED list mirrors bin/devlyn.js:299-304 — keep them in sync.
+# Skipped only in --judge-only (no model invocations); runs in --dry-run.
+if [ $JUDGE_ONLY -eq 0 ]; then
+  SRC_SKILLS="$REPO_ROOT/config/skills"
+  DST_SKILLS="$REPO_ROOT/.claude/skills"
+  mkdir -p "$DST_SKILLS"
+  mirrored=0
+  for src_dir in "$SRC_SKILLS"/*/; do
+    [ -d "$src_dir" ] || continue
+    name=$(basename "$src_dir")
+    case "$name" in
+      devlyn:auto-resolve-workspace|devlyn:ideate-workspace|preflight-workspace|roadmap-archival-workspace)
+        continue ;;
+    esac
+    staging="$DST_SKILLS/.${name}.staging"
+    rm -rf "$staging"
+    cp -R "$src_dir" "$staging"
+    rm -rf "$DST_SKILLS/$name"
+    mv "$staging" "$DST_SKILLS/$name"
+    mirrored=$((mirrored + 1))
+  done
+  echo "[suite] mirrored $mirrored committed skill(s): config/skills/ -> .claude/skills/"
+fi
+
+# Prereq checks
+if [ $DRY_RUN -eq 0 ] && [ $JUDGE_ONLY -eq 0 ]; then
+  command -v claude >/dev/null 2>&1 || { echo "claude CLI missing; install Claude Code first"; exit 1; }
+fi
+if [ $JUDGE_ONLY -eq 0 ]; then
+  command -v codex  >/dev/null 2>&1 || echo "warning: codex CLI missing — judge will fail"
+fi
+command -v python3 >/dev/null 2>&1 || { echo "python3 missing"; exit 1; }
+
+# Install test-repo deps once per suite run (shared cache)
+if [ $DRY_RUN -eq 0 ] && [ $JUDGE_ONLY -eq 0 ]; then
+  TEST_REPO="$BENCH_ROOT/fixtures/test-repo"
+  if [ ! -d "$TEST_REPO/node_modules" ]; then
+    echo "[suite] installing test-repo deps (one-time)"
+    if ! (cd "$TEST_REPO" && npm install --no-audit --no-fund --loglevel=error); then
+      echo "[suite] ✗ npm install in test-repo failed — check network/npm auth. Aborting." >&2
+      exit 1
+    fi
+  fi
+fi
+
+# ---- Run arms ---------------------------------------------------------------
+if [ $JUDGE_ONLY -eq 0 ]; then
+  for fid in "${FIXTURES[@]}"; do
+    [ -d "$FIXTURES_DIR/$fid" ] || { echo "[suite] skip $fid (missing)"; continue; }
+    for arm in variant solo_claude bare; do
+      echo "[suite] ► $fid / $arm (resolve-skill=$RESOLVE_SKILL)"
+      extra=""
+      [ $DRY_RUN -eq 1 ] && extra="--dry-run"
+      bash "$BENCH_ROOT/scripts/run-fixture.sh" \
+        --fixture "$fid" --arm "$arm" --run-id "$RUN_ID" \
+        --resolve-skill "$RESOLVE_SKILL" $extra \
+        || echo "[suite] ✗ $fid / $arm (arm failure tolerated; artifacts still captured)"
+    done
+  done
+fi
+
+# ---- Judge ------------------------------------------------------------------
+for fid in "${FIXTURES[@]}"; do
+  if [ ! -d "$BENCH_ROOT/results/$RUN_ID/$fid" ]; then
+    echo "[suite] skip judge for $fid (no results)"
+    continue
+  fi
+  if [ $DRY_RUN -eq 1 ]; then
+    echo "[suite] DRY RUN — skipping judge for $fid"
+    continue
+  fi
+  echo "[suite] ► judge $fid"
+  bash "$BENCH_ROOT/scripts/judge.sh" --fixture "$fid" --run-id "$RUN_ID" \
+    || echo "[suite] ✗ judge failed for $fid (will appear as NO_JUDGE in report)"
+done
+
+# ---- Compile report + ship gate --------------------------------------------
+if [ $DRY_RUN -eq 1 ]; then
+  echo ""
+  echo "[suite] DRY RUN complete — results in $RES_DIR"
+  echo "Run without --dry-run to invoke models."
+  exit 0
+fi
+
+echo ""
+python3 "$BENCH_ROOT/scripts/compile-report.py" --run-id "$RUN_ID" ${LABEL:+--label "$LABEL"}
+
+extra_flag=""
+[ $BLESS -eq 1 ] && extra_flag="$extra_flag --bless"
+[ $ACCEPT_MISSING -eq 1 ] && extra_flag="$extra_flag --accept-missing"
+python3 "$BENCH_ROOT/scripts/ship-gate.py" --run-id "$RUN_ID" $extra_flag

package/benchmark/auto-resolve/scripts/ship-gate.py

@@ -0,0 +1,222 @@
+#!/usr/bin/env python3
+"""
+ship-gate.py — apply RUBRIC.md ship thresholds to a suite run's summary.json.
+
+Usage:
+    ship-gate.py --run-id <ID>                   # check gates, return 0/1 via exit code
+    ship-gate.py --run-id <ID> --bless           # if PASS, promote summary to baselines/shipped.json
+
+Exits 0 on PASS, 1 on FAIL.
+"""
+from __future__ import annotations
+import argparse, json, pathlib, sys, shutil, datetime
+
+
+def main() -> int:
+    p = argparse.ArgumentParser()
+    p.add_argument("--run-id", required=True)
+    p.add_argument("--bless", action="store_true")
+    p.add_argument("--accept-missing", action="store_true",
+                   help="skip hard-floor gates that require fixtures not yet implemented "
+                        "(F9 and the 7-of-9 count) — only for suites in bootstrap")
+    args = p.parse_args()
+
+    root = pathlib.Path(__file__).resolve().parent.parent
+    summary_p = root / "results" / args.run_id / "summary.json"
+    if not summary_p.exists():
+        print(f"no summary at {summary_p}", file=sys.stderr); return 1
+    summary = json.loads(summary_p.read_text())
+
+    baseline_p = root / "history" / "baselines" / "shipped.json"
+    baseline = None
+    if baseline_p.exists():
+        try:
+            baseline = json.loads(baseline_p.read_text())
+        except Exception:
+            baseline = None
+
+    failures: list[str] = []
+    warnings: list[str] = []
+
+    # Hard floor 1: no disqualifier in variant
+    if summary["hard_floor_violations"] > 0:
+        failures.append(f"{summary['hard_floor_violations']} variant disqualifier(s) — see report")
+
+    # Hard floor 2: F9 must pass (skipped during bootstrap via --accept-missing)
+    # Variant arm legacy gate kept for L2 baseline comparability.
+    # iter-0033a (2026-04-30): renamed F9 dir from -to-preflight to -to-resolve to
+    # match the shipped 2-skill contract (no preflight). The OLD pre-rename id
+    # is preserved in fixtures/retired/ for replay.
+    f9_row = next((r for r in summary["rows"] if r.get("fixture") == "F9-e2e-ideate-to-resolve"), None)
+    if f9_row is None:
+        if not args.accept_missing:
+            failures.append("F9 (E2E novice flow) missing — add fixture or run with --accept-missing")
+    else:
+        if (f9_row.get("margin") or -999) < 5:
+            failures.append("F9 (E2E novice flow) must have variant margin ≥ +5")
+
+    # Hard floor 3: ≥ 7 of 9 gated fixtures with margin ≥ +5
+    # (skipped during bootstrap via --accept-missing)
+    if summary["gated_fixtures"] > 0 and summary["margin_ge_5_count"] < 7:
+        if not args.accept_missing:
+            failures.append(
+                f"only {summary['margin_ge_5_count']} of {summary['gated_fixtures']} "
+                f"gated fixtures have variant margin ≥ +5 (need ≥ 7)"
+            )
+
+    # iter-0023 — L1 (solo_claude) gates per NORTH-STAR.md ops test #1.
+    # Codex R1 (this iter) caught that ship-gate enforced only legacy L2
+    # `variant` margin and never read `solo_over_bare`. Now NORTH-STAR's
+    # documented L1 floor (≥ +5, ≥ 7/9 fixtures, F9 ≥ +5, no L1
+    # disqualifier) is mechanically enforced.
+    arms_present = summary.get("arms_present", {})
+    margins_avg = summary.get("margins_avg", {})
+    if arms_present.get("solo_claude"):
+        l1_avg = margins_avg.get("solo_over_bare")
+        if l1_avg is not None and l1_avg < 5:
+            warnings.append(
+                f"L1 (solo_over_bare) suite avg {l1_avg:+.1f} below NORTH-STAR floor +5 "
+                "(reporting only — per-fixture L1 gates below are decisive)"
+            )
+        # F9 L1 floor
+        if f9_row is not None:
+            f9_l1 = (f9_row.get("margins") or {}).get("solo_over_bare")
+            if f9_l1 is None:
+                if not args.accept_missing:
+                    failures.append("F9 L1 (solo_over_bare) margin missing — measurement invalid")
+            elif f9_l1 < 5:
+                failures.append(f"F9 L1 (solo_over_bare) margin {f9_l1:+d} < +5 floor")
+        # 7-of-9 L1 floor — headroom-aware (added 2026-05-02 per iter-0033 R4
+        # Codex collab + NORTH-STAR amendment + RUBRIC hard-floor 3 update).
+        # A fixture is excluded from the denominator when 100 - L0_score < 5
+        # AND L1_score >= 95 AND the L1 arm has no disqualifier / CRITICAL-HIGH
+        # finding / watchdog timeout / regression worse than gate #4. Excluded
+        # fixtures become fixture-rotation candidates if RUBRIC's
+        # two-shipped-version saturation rule fires.
+        l1_ge_5 = 0
+        l1_gated = 0
+        l1_excluded_headroom = []
+        for r in summary.get("rows", []):
+            if (r.get("category") or "").lower() == "known-limit":
+                continue
+            arms = r.get("arms") or {}
+            l0 = arms.get("bare") or {}
+            l1 = arms.get("solo_claude") or {}
+            l0_score = l0.get("score")
+            l1_score = l1.get("score")
+            m = (r.get("margins") or {}).get("solo_over_bare")
+            if m is None:
+                continue
+            # Headroom carve-out — must satisfy ALL conditions:
+            # (a) bare ceiling-near (100 - L0 < 5)
+            # (b) L1 also ceiling-near (>=95)
+            # (c) L1 arm clean (no disqualifier, no axis-invalid, fix-loop didn't fail)
+            l1_dq_here = bool(l1.get("disqualifier"))
+            l1_axis_inv = (l1.get("_axis_validation_out_of_range_count") or 0) > 0
+            if (
+                isinstance(l0_score, (int, float)) and isinstance(l1_score, (int, float))
+                and (100 - l0_score) < 5 and l1_score >= 95
+                and not l1_dq_here and not l1_axis_inv
+            ):
+                l1_excluded_headroom.append({
+                    "fixture": r.get("fixture"),
+                    "l0_score": l0_score,
+                    "l1_score": l1_score,
+                    "margin": m,
+                })
+                continue
+            l1_gated += 1
+            if m >= 5:
+                l1_ge_5 += 1
+        if l1_gated > 0 and l1_ge_5 < 7 and not args.accept_missing:
+            failures.append(
+                f"L1: only {l1_ge_5} of {l1_gated} headroom-available fixtures have solo_over_bare ≥ +5 (need ≥ 7)"
+            )
+        if l1_excluded_headroom:
+            warnings.append(
+                "L1 headroom-excluded (saturation candidates per RUBRIC two-shipped-version rule): "
+                + ", ".join(
+                    f"{x['fixture']} (L0={x['l0_score']} L1={x['l1_score']} margin={x['margin']:+d})"
+                    for x in l1_excluded_headroom
+                )
+            )
+        # L1 disqualifier floor
+        l1_dq = sum(
+            1 for r in summary.get("rows", [])
+            if ((r.get("arms") or {}).get("solo_claude") or {}).get("disqualifier")
+        )
+        if l1_dq > 0:
+            failures.append(f"L1 disqualifier(s): {l1_dq} solo_claude arm(s) hit a disqualifier")
+        # L1 axis-validity gate (judge.sh records out-of-range axis cells under
+        # `_axis_validation` per fixture). If any L1 row has invalid axis data,
+        # the L1 score for that row is not trustworthy.
+        l1_axis_invalid = 0
+        for r in summary.get("rows", []):
+            av = (r.get("arms") or {}).get("solo_claude") or {}
+            inv = av.get("_axis_validation_out_of_range_count")
+            if inv is not None and inv > 0:
+                l1_axis_invalid += 1
+        if l1_axis_invalid > 0:
+            failures.append(
+                f"L1 axis-invalid: {l1_axis_invalid} fixture(s) have out-of-range axis cells — "
+                "re-judge before trusting L1 margins"
+            )
+
+    # Hard floor 4: no per-fixture regression worse than −5 vs shipped baseline
+    if baseline:
+        prev_rows = {r["fixture"]: r for r in baseline.get("rows", [])}
+        for r in summary["rows"]:
+            fid = r.get("fixture")
+            prev = prev_rows.get(fid)
+            if prev and r.get("variant_score") is not None and prev.get("variant_score") is not None:
+                delta = r["variant_score"] - prev["variant_score"]
+                if delta < -5:
+                    failures.append(f"{fid} regressed {delta:+d} vs shipped (floor: −5)")
+
+    # Soft gate: suite average margin drop > 3
+    if baseline:
+        margin_delta = summary["margin_avg"] - baseline.get("margin_avg", 0)
+        if margin_delta < -3:
+            warnings.append(f"suite margin dropped {margin_delta:+.1f} vs shipped (soft gate: > −3)")
+
+    # Soft gate: any fixture that was > +5 before is now ≤ 0
+    if baseline:
+        prev_rows = {r["fixture"]: r for r in baseline.get("rows", [])}
+        for r in summary["rows"]:
+            fid = r.get("fixture")
+            prev = prev_rows.get(fid)
+            if prev and (prev.get("margin") or 0) > 5 and (r.get("margin") or 0) <= 0:
+                warnings.append(
+                    f"{fid} lost its margin: was {prev['margin']:+d}, now {r['margin']:+d}"
+                )
+
+    verdict = "PASS" if not failures else "FAIL"
+    print(f"\n═══ SHIP-GATE VERDICT: {verdict} ═══\n")
+    if failures:
+        print("Hard-floor failures:")
+        for f in failures:
+            print(f"  ✗ {f}")
+        print()
+    if warnings:
+        print("Soft-gate warnings:")
+        for w in warnings:
+            print(f"  ⚠ {w}")
+        print()
+    if not failures and not warnings:
+        print("No gate violations. Suite is ship-ready.")
+
+    # Bless if PASS + --bless — opt-in promotion to shipped baseline.
+    # Per BENCHMARK-DESIGN.md Karpathy Check, automatic history mutation is
+    # deferred until after the suite format stabilizes; `--bless` stays as
+    # the explicit promotion path, and `summary.json` inside the run dir
+    # is the durable record for ad-hoc inspection.
+    if verdict == "PASS" and args.bless:
+        baseline_p.parent.mkdir(parents=True, exist_ok=True)
+        shutil.copyfile(summary_p, baseline_p)
+        print(f"\nBlessed: {baseline_p}")
+
+    return 0 if verdict == "PASS" else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())