devlyn-cli 2.2.2 → 2.3.1

package/benchmark/auto-resolve/scripts/test-run-headroom-candidate.sh

@@ -0,0 +1,401 @@
+#!/usr/bin/env bash
+# Regression tests for run-headroom-candidate.sh argument and output guards.
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+RUNNER="$SCRIPT_DIR/run-headroom-candidate.sh"
+REJECTED="$SCRIPT_DIR/pair-rejected-fixtures.sh"
+TMP_DIR="$(mktemp -d /tmp/run-headroom-candidate-test.XXXXXX)"
+BENCH_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+TEST_RUN="headroom-cli-replay-$(basename "$TMP_DIR")"
+TEST_SHADOW="$BENCH_ROOT/shadow-fixtures/S98-runner-hypothesis"
+trap 'rm -rf "$TMP_DIR" "$BENCH_ROOT/results/$TEST_RUN"* "$TEST_SHADOW"' EXIT
+
+expect_fail_contains() {
+  local label="$1"
+  local needle="$2"
+  shift 2
+  local out="$TMP_DIR/$label.out"
+  if "$@" > "$out" 2>&1; then
+    echo "expected failure for $label" >&2
+    cat "$out" >&2
+    exit 1
+  fi
+  if ! grep -Fq -- "$needle" "$out"; then
+    echo "missing expected text for $label: $needle" >&2
+    cat "$out" >&2
+    exit 1
+  fi
+}
+
+bash "$RUNNER" --help > "$TMP_DIR/help.out" 2>&1
+grep -Fq 'usage:' "$TMP_DIR/help.out"
+grep -Fq -- '--bare-max N' "$TMP_DIR/help.out"
+grep -Fq -- '--solo-max N' "$TMP_DIR/help.out"
+grep -Fq -- '--min-bare-headroom N' "$TMP_DIR/help.out"
+grep -Fq -- '--min-solo-headroom N' "$TMP_DIR/help.out"
+grep -Fq -- '--min-fixtures N' "$TMP_DIR/help.out"
+grep -Fq -- '--allow-rejected-fixtures' "$TMP_DIR/help.out"
+grep -Fq -- '--dry-run' "$TMP_DIR/help.out"
+grep -Fq 'print_command' "$RUNNER"
+grep -Fq 'Command: ' "$RUNNER"
+grep -Fq 'DEVLYN_BENCHMARK_CLI_SUBCOMMAND' "$RUNNER"
+grep -Fq 'cmd=(npx devlyn-cli benchmark headroom --run-id "$RUN_ID")' "$RUNNER"
+grep -Fq 'cmd=(bash "$0" --run-id "$RUN_ID")' "$RUNNER"
+grep -Fq 'cmd+=(--bare-max "$BARE_MAX")' "$RUNNER"
+grep -Fq 'cmd+=(--solo-max "$SOLO_MAX")' "$RUNNER"
+grep -Fq 'cmd+=(--min-bare-headroom "$MIN_BARE_HEADROOM")' "$RUNNER"
+grep -Fq 'cmd+=(--min-solo-headroom "$MIN_SOLO_HEADROOM")' "$RUNNER"
+grep -Fq 'cmd+=(--min-fixtures "$MIN_FIXTURES")' "$RUNNER"
+grep -Fq 'cmd+=(--allow-rejected-fixtures)' "$RUNNER"
+grep -Fq 'cmd+=(--dry-run)' "$RUNNER"
+grep -Fq 'baseline evidence-complete' "$RUNNER"
+grep -Fq 'headroom gate passed — candidate set accepted' "$RUNNER"
+grep -Fq 'headroom gate failed — candidate set rejected' "$RUNNER"
+grep -Fq -- '--bare-max "$BARE_MAX"' "$RUNNER"
+grep -Fq -- '--solo-max "$SOLO_MAX"' "$RUNNER"
+grep -Fq -- '--min-bare-headroom "$MIN_BARE_HEADROOM"' "$RUNNER"
+grep -Fq -- '--min-solo-headroom "$MIN_SOLO_HEADROOM"' "$RUNNER"
+grep -Fq -- '--min-fixtures "$MIN_FIXTURES"' "$RUNNER"
+grep -Fq 'cat "$BENCH_ROOT/results/$RUN_ID/headroom-gate.md"' "$RUNNER"
+grep -Fq 'headroom gate report missing' "$RUNNER"
+grep -Fq 'validate_fixtures' "$RUNNER"
+grep -Fq 'fixture_has_solo_ceiling_avoidance_note' "$RUNNER"
+grep -Fq 'shadow fixture NOTES.md needs ## Solo ceiling avoidance' "$RUNNER"
+grep -Fq 'fixture not found in fixtures/ or shadow-fixtures/' "$RUNNER"
+grep -Fq '[FS][0-9]*) FIXTURES+=("$1")' "$RUNNER"
+grep -Fq 'retired_fixture_exists' "$RUNNER"
+grep -Fq 'fixture is retired and is not rerun by pair-candidate runners' "$RUNNER"
+grep -Fq 'fixture_smoke_only' "$RUNNER"
+grep -Fq 'fixture is smoke-only and cannot run providers' "$RUNNER"
+grep -Fq 'rejected_pair_fixture_reason' "$RUNNER"
+grep -Fq 'source "$BENCH_ROOT/scripts/pair-rejected-fixtures.sh"' "$RUNNER"
+grep -Fq 'declare -F rejected_pair_fixture_reason' "$RUNNER"
+grep -Fq '20260511-f3-http-error-headroom' "$REJECTED"
+grep -Fq '20260507-f10-f11-tier1-full-pipeline' "$REJECTED"
+grep -Fq '20260511-f12-webhook-headroom' "$REJECTED"
+grep -Fq '20260511-f15-concurrency-headroom' "$REJECTED"
+grep -Fq '20260511-f28-policy-oraclefix-reverified-pair' "$REJECTED"
+grep -Fq '20260511-f30-headroom-v1' "$REJECTED"
+grep -Fq '20260513-s2-inventory-headroom' "$REJECTED"
+grep -Fq '20260513-s3-ticket-headroom' "$REJECTED"
+grep -Fq '20260513-s4-return-headroom' "$REJECTED"
+grep -Fq '20260513-s5-credit-headroom' "$REJECTED"
+grep -Fq 'Use --allow-rejected-fixtures for diagnostics only' "$RUNNER"
+
+expect_fail_contains missing-fixture 'usage:' \
+  bash "$RUNNER" --run-id headroom-arg-test
+
+expect_fail_contains unknown-arg 'unknown arg: --bad-flag' \
+  bash "$RUNNER" --bad-flag F21-cli-scheduler-priority
+
+expect_fail_contains missing-bare-max-value '--bare-max requires a value' \
+  bash "$RUNNER" --bare-max
+
+expect_fail_contains invalid-bare-max '--bare-max must be an integer: nope' \
+  bash "$RUNNER" --bare-max nope F21-cli-scheduler-priority
+
+expect_fail_contains invalid-min-fixtures '--min-fixtures must be >= 1' \
+  bash "$RUNNER" --min-fixtures 0 F21-cli-scheduler-priority
+
+expect_fail_contains invalid-min-bare-headroom '--min-bare-headroom must be an integer: nope' \
+  bash "$RUNNER" --min-bare-headroom nope F21-cli-scheduler-priority
+
+expect_fail_contains negative-min-bare-headroom '--min-bare-headroom must be an integer: -1' \
+  bash "$RUNNER" --min-bare-headroom -1 F21-cli-scheduler-priority
+
+expect_fail_contains negative-min-solo-headroom '--min-solo-headroom must be an integer: -1' \
+  bash "$RUNNER" --min-solo-headroom -1 F21-cli-scheduler-priority
+
+expect_fail_contains missing-fixture-fast \
+  'fixture not found in fixtures/ or shadow-fixtures/: F999-not-a-fixture' \
+  bash "$RUNNER" --run-id "$TEST_RUN-missing" F999-not-a-fixture
+
+expect_fail_contains rejected-f1-fixture \
+  'fixture rejected for pair-candidate runs: F1-cli-trivial-flag' \
+  bash "$RUNNER" --run-id "$TEST_RUN-rejected-f1" --dry-run --min-fixtures 1 F1-cli-trivial-flag
+
+expect_fail_contains rejected-f2-fixture \
+  'fixture rejected for pair-candidate runs: F2-cli-medium-subcommand' \
+  bash "$RUNNER" --run-id "$TEST_RUN-rejected-f2" --dry-run --min-fixtures 1 F2-cli-medium-subcommand
+
+expect_fail_contains rejected-fixture \
+  'fixture rejected for pair-candidate runs: F26-cli-payout-ledger-rules' \
+  bash "$RUNNER" --run-id "$TEST_RUN-rejected" --dry-run --min-fixtures 1 F26-cli-payout-ledger-rules
+
+expect_fail_contains rejected-f3-fixture \
+  'fixture rejected for pair-candidate runs: F3-backend-contract-risk' \
+  bash "$RUNNER" --run-id "$TEST_RUN-rejected-f3" --dry-run --min-fixtures 1 F3-backend-contract-risk
+
+expect_fail_contains rejected-f4-fixture \
+  'fixture rejected for pair-candidate runs: F4-web-browser-design' \
+  bash "$RUNNER" --run-id "$TEST_RUN-rejected-f4" --dry-run --min-fixtures 1 F4-web-browser-design
+
+expect_fail_contains rejected-f5-fixture \
+  'fixture rejected for pair-candidate runs: F5-fix-loop-red-green' \
+  bash "$RUNNER" --run-id "$TEST_RUN-rejected-f5" --dry-run --min-fixtures 1 F5-fix-loop-red-green
+
+expect_fail_contains rejected-f6-fixture \
+  'fixture rejected for pair-candidate runs: F6-dep-audit-native-module' \
+  bash "$RUNNER" --run-id "$TEST_RUN-rejected-f6" --dry-run --min-fixtures 1 F6-dep-audit-native-module
+
+expect_fail_contains rejected-f7-fixture \
+  'fixture rejected for pair-candidate runs: F7-out-of-scope-trap' \
+  bash "$RUNNER" --run-id "$TEST_RUN-rejected-f7" --dry-run --min-fixtures 1 F7-out-of-scope-trap
+
+expect_fail_contains rejected-f8-fixture \
+  'fixture rejected for pair-candidate runs: F8-known-limit-ambiguous' \
+  bash "$RUNNER" --run-id "$TEST_RUN-rejected-f8" --dry-run --min-fixtures 1 F8-known-limit-ambiguous
+
+expect_fail_contains rejected-f9-fixture \
+  'fixture rejected for pair-candidate runs: F9-e2e-ideate-to-resolve' \
+  bash "$RUNNER" --run-id "$TEST_RUN-rejected-f9" --dry-run --min-fixtures 1 F9-e2e-ideate-to-resolve
+
+expect_fail_contains rejected-f10-fixture \
+  'fixture rejected for pair-candidate runs: F10-persist-write-collision' \
+  bash "$RUNNER" --run-id "$TEST_RUN-rejected-f10" --dry-run --min-fixtures 1 F10-persist-write-collision
+
+expect_fail_contains rejected-f11-fixture \
+  'fixture rejected for pair-candidate runs: F11-batch-import-all-or-nothing' \
+  bash "$RUNNER" --run-id "$TEST_RUN-rejected-f11" --dry-run --min-fixtures 1 F11-batch-import-all-or-nothing
+
+expect_fail_contains rejected-f12-fixture \
+  'fixture rejected for pair-candidate runs: F12-webhook-raw-body-signature' \
+  bash "$RUNNER" --run-id "$TEST_RUN-rejected-f12" --dry-run --min-fixtures 1 F12-webhook-raw-body-signature
+
+expect_fail_contains rejected-f15-fixture \
+  'fixture rejected for pair-candidate runs: F15-frozen-diff-race-review' \
+  bash "$RUNNER" --run-id "$TEST_RUN-rejected-f15" --dry-run --min-fixtures 1 F15-frozen-diff-race-review
+
+expect_fail_contains rejected-f31-fixture \
+  'fixture rejected for pair-candidate runs: F31-cli-seat-rebalance' \
+  bash "$RUNNER" --run-id "$TEST_RUN-rejected-f31" --dry-run --min-fixtures 1 F31-cli-seat-rebalance
+
+expect_fail_contains rejected-f32-fixture \
+  'fixture rejected for pair-candidate runs: F32-cli-subscription-renewal' \
+  bash "$RUNNER" --run-id "$TEST_RUN-rejected-f32" --dry-run --min-fixtures 1 F32-cli-subscription-renewal
+
+expect_fail_contains rejected-s2-shadow-fixture \
+  'fixture rejected for pair-candidate runs: S2-cli-inventory-reservation' \
+  bash "$RUNNER" --run-id "$TEST_RUN-rejected-s2" --dry-run --min-fixtures 1 S2-cli-inventory-reservation
+
+expect_fail_contains rejected-s3-shadow-fixture \
+  'fixture rejected for pair-candidate runs: S3-cli-ticket-assignment' \
+  bash "$RUNNER" --run-id "$TEST_RUN-rejected-s3" --dry-run --min-fixtures 1 S3-cli-ticket-assignment
+
+expect_fail_contains rejected-s4-shadow-fixture \
+  'fixture rejected for pair-candidate runs: S4-cli-return-routing' \
+  bash "$RUNNER" --run-id "$TEST_RUN-rejected-s4" --dry-run --min-fixtures 1 S4-cli-return-routing
+
+expect_fail_contains rejected-s5-shadow-fixture \
+  'fixture rejected for pair-candidate runs: S5-cli-credit-grant-ledger' \
+  bash "$RUNNER" --run-id "$TEST_RUN-rejected-s5" --dry-run --min-fixtures 1 S5-cli-credit-grant-ledger
+
+expect_fail_contains rejected-s6-shadow-fixture \
+  'fixture rejected for pair-candidate runs: S6-cli-refund-window-ledger' \
+  bash "$RUNNER" --run-id "$TEST_RUN-rejected-s6" --dry-run --min-fixtures 1 S6-cli-refund-window-ledger
+
+expect_fail_contains retired-fixture \
+  'fixture is retired and is not rerun by pair-candidate runners: F28-cli-return-authorization' \
+  bash "$RUNNER" --run-id "$TEST_RUN-retired" --dry-run --min-fixtures 1 F28-cli-return-authorization
+
+expect_fail_contains smoke-only-s1-provider-run \
+  'fixture is smoke-only and cannot run providers: S1-cli-lang-flag' \
+  bash "$RUNNER" --run-id "$TEST_RUN-smoke-only" --min-fixtures 1 S1-cli-lang-flag
+
+expect_fail_contains cli-replay-command \
+  "Command: npx devlyn-cli benchmark headroom --run-id $TEST_RUN" \
+  env DEVLYN_BENCHMARK_CLI_SUBCOMMAND=headroom \
+    bash "$RUNNER" --run-id "$TEST_RUN" --min-fixtures 2 F999-not-a-fixture
+
+expect_fail_contains dry-run-min-fixtures \
+  '[headroom] DRY RUN failed' \
+  bash "$RUNNER" --run-id "$TEST_RUN-dry-run-fail" --dry-run F21-cli-scheduler-priority
+
+bash "$RUNNER" --run-id "$TEST_RUN-dry-run" --dry-run --min-fixtures 1 F21-cli-scheduler-priority \
+  > "$TMP_DIR/dry-run.out" 2>&1
+grep -Fq 'Mode:     DRY RUN (no model/provider invocations)' "$TMP_DIR/dry-run.out"
+grep -Fq 'Command: ' "$TMP_DIR/dry-run.out"
+grep -Fq -- '--dry-run' "$TMP_DIR/dry-run.out"
+grep -Fq -- '--min-bare-headroom 5' "$TMP_DIR/dry-run.out"
+grep -Fq -- '--min-solo-headroom 5' "$TMP_DIR/dry-run.out"
+grep -Fq -- '--min-fixtures 1' "$TMP_DIR/dry-run.out"
+grep -Fq '[headroom] DRY RUN complete' "$TMP_DIR/dry-run.out"
+
+bash "$RUNNER" --run-id "$TEST_RUN-shadow-dry-run" --dry-run --min-fixtures 1 S1-cli-lang-flag \
+  > "$TMP_DIR/shadow-dry-run.out" 2>&1
+grep -Fq 'Fixtures: S1-cli-lang-flag' "$TMP_DIR/shadow-dry-run.out"
+grep -Fq '[headroom] DRY RUN complete' "$TMP_DIR/shadow-dry-run.out"
+
+mkdir -p "$TEST_SHADOW"
+cat > "$TEST_SHADOW/metadata.json" <<'EOF'
+{
+  "id": "S98-runner-hypothesis",
+  "category": "high-risk"
+}
+EOF
+cat > "$TEST_SHADOW/spec.md" <<'EOF'
+# Runner hypothesis fixture
+
+Add idempotency handling for duplicate requests.
+EOF
+cat > "$TEST_SHADOW/expected.json" <<'EOF'
+{
+  "verification_commands": [
+    {
+      "cmd": "node -e \"process.exit(0)\"",
+      "exit_code": 0
+    }
+  ]
+}
+EOF
+cat > "$TEST_SHADOW/NOTES.md" <<'EOF'
+# Notes
+
+Synthetic runner guard fixture.
+EOF
+expect_fail_contains missing-solo-headroom-hypothesis \
+  'fixture spec.md needs a solo-headroom hypothesis with solo_claude miss and observable command from expected.json before provider spend: S98-runner-hypothesis' \
+  bash "$RUNNER" --run-id "$TEST_RUN-missing-hypothesis" --dry-run --min-fixtures 1 S98-runner-hypothesis
+cat >> "$TEST_SHADOW/spec.md" <<'EOF'
+
+## Solo-headroom hypothesis
+
+A capable solo_claude baseline is expected to miss duplicate idempotency ordering.
+EOF
+expect_fail_contains weak-solo-headroom-hypothesis \
+  'fixture spec.md needs a solo-headroom hypothesis with solo_claude miss and observable command from expected.json before provider spend: S98-runner-hypothesis' \
+  bash "$RUNNER" --run-id "$TEST_RUN-weak-hypothesis" --dry-run --min-fixtures 1 S98-runner-hypothesis
+cat >> "$TEST_SHADOW/spec.md" <<'EOF'
+
+Implementation marker: `duplicate-idempotency`.
+EOF
+expect_fail_contains unrelated-backtick-solo-headroom-hypothesis \
+  'fixture spec.md needs a solo-headroom hypothesis with solo_claude miss and observable command from expected.json before provider spend: S98-runner-hypothesis' \
+  bash "$RUNNER" --run-id "$TEST_RUN-unrelated-backtick-hypothesis" --dry-run --min-fixtures 1 S98-runner-hypothesis
+cat >> "$TEST_SHADOW/spec.md" <<'EOF'
+
+Observable command: `node -e "process.exit(0)"` exposes behavior.
+EOF
+expect_fail_contains observable-without-miss-solo-headroom-hypothesis \
+  'fixture spec.md needs a solo-headroom hypothesis with solo_claude miss and observable command from expected.json before provider spend: S98-runner-hypothesis' \
+  bash "$RUNNER" --run-id "$TEST_RUN-observable-without-miss-hypothesis" --dry-run --min-fixtures 1 S98-runner-hypothesis
+cat >> "$TEST_SHADOW/spec.md" <<'EOF'
+
+Observable command: `node -e "process.exit(0)"` exposes the miss.
+EOF
+expect_fail_contains missing-solo-ceiling-avoidance \
+  'shadow fixture NOTES.md needs ## Solo ceiling avoidance with solo_claude, a rejected/solo-saturated control comparison, and headroom reasoning before provider spend: S98-runner-hypothesis' \
+  bash "$RUNNER" --run-id "$TEST_RUN-missing-ceiling" --dry-run --min-fixtures 1 S98-runner-hypothesis
+cat >> "$TEST_SHADOW/NOTES.md" <<'EOF'
+
+## Solo ceiling avoidance
+
+This candidate mentions solo_claude but gives no control comparison.
+EOF
+expect_fail_contains weak-solo-ceiling-avoidance \
+  'shadow fixture NOTES.md needs ## Solo ceiling avoidance with solo_claude, a rejected/solo-saturated control comparison, and headroom reasoning before provider spend: S98-runner-hypothesis' \
+  bash "$RUNNER" --run-id "$TEST_RUN-weak-ceiling" --dry-run --min-fixtures 1 S98-runner-hypothesis
+cat >> "$TEST_SHADOW/NOTES.md" <<'EOF'
+
+Unlike solo-saturated S2-S6 controls, this fixture should preserve
+solo_claude headroom because it targets a multi-run state dependency.
+EOF
+bash "$RUNNER" --run-id "$TEST_RUN-hypothesis" --dry-run --min-fixtures 1 S98-runner-hypothesis \
+  > "$TMP_DIR/hypothesis.out" 2>&1
+grep -Fq '[headroom] DRY RUN complete' "$TMP_DIR/hypothesis.out"
+
+bash "$RUNNER" --run-id "$TEST_RUN-rejected-override" --dry-run --min-fixtures 1 \
+  --allow-rejected-fixtures F26-cli-payout-ledger-rules \
+  > "$TMP_DIR/rejected-override.out" 2>&1
+grep -Fq -- '--allow-rejected-fixtures' "$TMP_DIR/rejected-override.out"
+grep -Fq '[headroom] DRY RUN complete' "$TMP_DIR/rejected-override.out"
+
+bash "$RUNNER" --run-id "$TEST_RUN-shadow-rejected-override" --dry-run --min-fixtures 1 \
+  --allow-rejected-fixtures S3-cli-ticket-assignment \
+  > "$TMP_DIR/shadow-rejected-override.out" 2>&1
+grep -Fq -- '--allow-rejected-fixtures' "$TMP_DIR/shadow-rejected-override.out"
+grep -Fq '[headroom] DRY RUN complete' "$TMP_DIR/shadow-rejected-override.out"
+
+STUB_REPO="$TMP_DIR/stub-repo"
+STUB_BENCH="$STUB_REPO/benchmark/auto-resolve"
+mkdir -p \
+  "$STUB_BENCH/scripts" \
+  "$STUB_BENCH/fixtures/F21-cli-scheduler-priority" \
+  "$STUB_REPO/config/skills/devlyn:resolve"
+cp "$RUNNER" "$STUB_BENCH/scripts/run-headroom-candidate.sh"
+cp "$REJECTED" "$STUB_BENCH/scripts/pair-rejected-fixtures.sh"
+chmod +x "$STUB_BENCH/scripts/run-headroom-candidate.sh"
+chmod +x "$STUB_BENCH/scripts/pair-rejected-fixtures.sh"
+printf -- '---\nname: devlyn:resolve\n---\n' > "$STUB_REPO/config/skills/devlyn:resolve/SKILL.md"
+cat > "$STUB_BENCH/scripts/run-fixture.sh" <<'EOF'
+#!/usr/bin/env bash
+set -euo pipefail
+echo "[stub-run-fixture] $*"
+EOF
+chmod +x "$STUB_BENCH/scripts/run-fixture.sh"
+cat > "$STUB_BENCH/scripts/judge.sh" <<'EOF'
+#!/usr/bin/env bash
+set -euo pipefail
+echo "[stub-judge] $*"
+EOF
+chmod +x "$STUB_BENCH/scripts/judge.sh"
+cat > "$STUB_BENCH/scripts/headroom-gate.py" <<'PY'
+#!/usr/bin/env python3
+import json
+import os
+import pathlib
+import sys
+
+out_json = None
+out_md = None
+args = sys.argv[1:]
+for index, arg in enumerate(args):
+    if arg == "--out-json":
+        out_json = pathlib.Path(args[index + 1])
+    if arg == "--out-md":
+        out_md = pathlib.Path(args[index + 1])
+payload = {"verdict": "PASS" if os.environ.get("STUB_HEADROOM_EXIT", "0") == "0" else "FAIL"}
+if out_json:
+    out_json.parent.mkdir(parents=True, exist_ok=True)
+    out_json.write_text(json.dumps(payload) + "\n", encoding="utf8")
+if out_md:
+    out_md.parent.mkdir(parents=True, exist_ok=True)
+    out_md.write_text(
+        "# stub headroom\n\n"
+        "Verdict: **%s**\n\n"
+        "| fixture | bare | solo_claude | solo_claude-bare |\n"
+        "| --- | ---: | ---: | ---: |\n"
+        "| F21-cli-scheduler-priority | 50 | 75 | 25 |\n"
+        % payload["verdict"],
+        encoding="utf8",
+    )
+sys.exit(int(os.environ.get("STUB_HEADROOM_EXIT", "0")))
+PY
+
+STUB_RUNNER="$STUB_BENCH/scripts/run-headroom-candidate.sh"
+STUB_HEADROOM_EXIT=0 \
+  bash "$STUB_RUNNER" --run-id "$TEST_RUN-stub-success" --min-fixtures 1 F21-cli-scheduler-priority \
+  > "$TMP_DIR/stub-success.out" 2>&1
+grep -Fq '[headroom] headroom gate passed — candidate set accepted.' "$TMP_DIR/stub-success.out"
+grep -Fq '| F21-cli-scheduler-priority | 50 | 75 | 25 |' "$TMP_DIR/stub-success.out"
+grep -Fq '[stub-run-fixture] --fixture F21-cli-scheduler-priority --arm bare' "$TMP_DIR/stub-success.out"
+grep -Fq '[stub-run-fixture] --fixture F21-cli-scheduler-priority --arm solo_claude' "$TMP_DIR/stub-success.out"
+
+if STUB_HEADROOM_EXIT=1 \
+  bash "$STUB_RUNNER" --run-id "$TEST_RUN-stub-fail" --min-fixtures 1 F21-cli-scheduler-priority \
+  > "$TMP_DIR/stub-fail.out" 2>&1; then
+  echo "expected stub headroom gate failure" >&2
+  cat "$TMP_DIR/stub-fail.out" >&2
+  exit 1
+fi
+grep -Fq '[headroom] headroom gate failed — candidate set rejected.' "$TMP_DIR/stub-fail.out"
+grep -Fq '| F21-cli-scheduler-priority | 50 | 75 | 25 |' "$TMP_DIR/stub-fail.out"
+if grep -Fq '[headroom] headroom gate passed — candidate set accepted.' "$TMP_DIR/stub-fail.out"; then
+  echo "accepted message must not print after headroom gate failure" >&2
+  cat "$TMP_DIR/stub-fail.out" >&2
+  exit 1
+fi
+
+echo "PASS test-run-headroom-candidate"

package/benchmark/auto-resolve/scripts/test-run-swebench-solver-batch.sh

@@ -0,0 +1,111 @@
+#!/usr/bin/env bash
+# Regression tests for run-swebench-solver-batch.sh argument guards.
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+RUNNER="$SCRIPT_DIR/run-swebench-solver-batch.sh"
+TMP_DIR="$(mktemp -d /tmp/run-swebench-solver-batch-test.XXXXXX)"
+trap 'rm -rf "$TMP_DIR"' EXIT
+FAKEBIN="$TMP_DIR/fakebin"
+mkdir -p "$FAKEBIN"
+cat > "$FAKEBIN/claude" <<'EOF'
+#!/usr/bin/env bash
+echo "fake claude should not be reached" >&2
+exit 1
+EOF
+chmod +x "$FAKEBIN/claude"
+
+expect_fail_contains() {
+  local label="$1"
+  local needle="$2"
+  shift 2
+  local out="$TMP_DIR/$label.out"
+  if "$@" > "$out" 2>&1; then
+    echo "expected failure for $label" >&2
+    cat "$out" >&2
+    exit 1
+  fi
+  if ! grep -Fq -- "$needle" "$out"; then
+    echo "missing expected text for $label: $needle" >&2
+    cat "$out" >&2
+    exit 1
+  fi
+}
+
+bash "$RUNNER" --help > "$TMP_DIR/help.out" 2>&1
+grep -Fq 'usage:' "$TMP_DIR/help.out"
+grep -Fq -- '--instances-jsonl <path>' "$TMP_DIR/help.out"
+grep -Fq -- '--predictions-out <path>' "$TMP_DIR/help.out"
+grep -Fq -- '--timeout-seconds N' "$TMP_DIR/help.out"
+grep -Fq 'require_value()' "$RUNNER"
+
+expect_fail_contains missing-instances-jsonl-value \
+  '--instances-jsonl requires a value' \
+  bash "$RUNNER" --instances-jsonl
+
+expect_fail_contains missing-predictions-out-value \
+  '--predictions-out requires a value' \
+  bash "$RUNNER" --instances-jsonl "$TMP_DIR/instances.jsonl" --predictions-out
+
+expect_fail_contains missing-model-name-value \
+  '--model-name requires a value' \
+  bash "$RUNNER" --instances-jsonl "$TMP_DIR/instances.jsonl" --predictions-out "$TMP_DIR/predictions.jsonl" --model-name
+
+expect_fail_contains missing-repos-root-value \
+  '--repos-root requires a value' \
+  bash "$RUNNER" --instances-jsonl "$TMP_DIR/instances.jsonl" --predictions-out "$TMP_DIR/predictions.jsonl" --repos-root
+
+expect_fail_contains missing-worktrees-root-value \
+  '--worktrees-root requires a value' \
+  bash "$RUNNER" --instances-jsonl "$TMP_DIR/instances.jsonl" --predictions-out "$TMP_DIR/predictions.jsonl" --worktrees-root
+
+expect_fail_contains missing-timeout-value \
+  '--timeout-seconds requires a value' \
+  bash "$RUNNER" --instances-jsonl "$TMP_DIR/instances.jsonl" --predictions-out "$TMP_DIR/predictions.jsonl" --timeout-seconds
+
+expect_fail_contains missing-limit-value \
+  '--limit requires a value' \
+  bash "$RUNNER" --instances-jsonl "$TMP_DIR/instances.jsonl" --predictions-out "$TMP_DIR/predictions.jsonl" --limit
+
+expect_fail_contains missing-instance-id-value \
+  '--instance-id requires a value' \
+  bash "$RUNNER" --instances-jsonl "$TMP_DIR/instances.jsonl" --predictions-out "$TMP_DIR/predictions.jsonl" --instance-id
+
+touch "$TMP_DIR/instances.jsonl"
+expect_fail_contains invalid-timeout \
+  '--timeout-seconds must be an integer' \
+  bash "$RUNNER" --instances-jsonl "$TMP_DIR/instances.jsonl" --predictions-out "$TMP_DIR/predictions.jsonl" --timeout-seconds nope
+
+expect_fail_contains zero-timeout \
+  '--timeout-seconds must be > 0' \
+  bash "$RUNNER" --instances-jsonl "$TMP_DIR/instances.jsonl" --predictions-out "$TMP_DIR/predictions.jsonl" --timeout-seconds 0
+
+expect_fail_contains invalid-limit \
+  '--limit must be an integer' \
+  bash "$RUNNER" --instances-jsonl "$TMP_DIR/instances.jsonl" --predictions-out "$TMP_DIR/predictions.jsonl" --limit nope
+
+expect_fail_contains zero-limit \
+  '--limit must be > 0' \
+  bash "$RUNNER" --instances-jsonl "$TMP_DIR/instances.jsonl" --predictions-out "$TMP_DIR/predictions.jsonl" --limit 0
+
+expect_fail_contains missing-claude \
+  'claude command not found' \
+  env PATH="/usr/bin:/bin" bash "$RUNNER" --instances-jsonl "$TMP_DIR/instances.jsonl" --predictions-out "$TMP_DIR/predictions.jsonl"
+
+printf '[]\n' > "$TMP_DIR/non-object-instances.jsonl"
+expect_fail_contains non-object-instance-row \
+  'expected JSON object' \
+  env PATH="$FAKEBIN:/usr/bin:/bin" bash "$RUNNER" --instances-jsonl "$TMP_DIR/non-object-instances.jsonl" --predictions-out "$TMP_DIR/predictions.jsonl"
+
+printf '{"instance_id": NaN}\n' > "$TMP_DIR/nan-instances.jsonl"
+expect_fail_contains nan-instance-row \
+  'invalid JSON numeric constant: NaN' \
+  env PATH="$FAKEBIN:/usr/bin:/bin" bash "$RUNNER" --instances-jsonl "$TMP_DIR/nan-instances.jsonl" --predictions-out "$TMP_DIR/predictions.jsonl"
+
+printf '{"repo":"local/repo"}\n' > "$TMP_DIR/missing-id-instances.jsonl"
+expect_fail_contains missing-instance-id-row \
+  'missing instance_id' \
+  env PATH="$FAKEBIN:/usr/bin:/bin" bash "$RUNNER" --instances-jsonl "$TMP_DIR/missing-id-instances.jsonl" --predictions-out "$TMP_DIR/predictions.jsonl"
+
+echo "PASS test-run-swebench-solver-batch"