devlyn-cli 2.2.2 → 2.3.1

package/benchmark/auto-resolve/scripts/pair-rejected-fixtures.sh

@@ -0,0 +1,91 @@
+#!/usr/bin/env bash
+# Shared rejected/ceiling pair-candidate fixture registry.
+# Includes active fixtures and calibrated shadow controls that should not spend
+# pair-candidate runs unless explicitly requested for diagnostics.
+
+rejected_pair_fixture_reason() {
+  local fid="$1"
+  case "$fid" in
+    F1-*|F1)
+      echo "trivial calibration fixture; every arm is expected to one-shot it"
+      ;;
+    F2-*|F2)
+      echo "bare 83 / solo_claude 95 in 20260512-f2-medium-headroom"
+      ;;
+    F3-*|F3)
+      echo "bare 97 / solo_claude 99 in 20260511-f3-http-error-headroom"
+      ;;
+    F4-*|F4)
+      echo "bare 70 / solo_claude 92 with bare disqualifier in 20260512-f4-web-headroom"
+      ;;
+    F5-*|F5)
+      echo "bare 99 / solo_claude 99 in 20260512-f5-fixloop-headroom"
+      ;;
+    F6-*|F6)
+      echo "bare 97 / solo_claude 96 in 20260512-f6-checksum-headroom"
+      ;;
+    F7-*|F7)
+      echo "bare 99 / solo_claude 100 in 20260512-f7-scope-headroom"
+      ;;
+    F8-*|F8)
+      echo "known-limit ambiguity fixture; expected margin is [-3,+3], not pair-lift evidence"
+      ;;
+    F9-*|F9)
+      echo "bare 60 / solo_claude 90 with bare headroom 0 and bare judge disqualifier in 20260512-f9-e2e-headroom"
+      ;;
+    F10-*|F10)
+      echo "bare 75 / solo_claude 94 in 20260507-f10-f11-tier1-full-pipeline"
+      ;;
+    F11-*|F11)
+      echo "bare 98 / solo_claude 97 in 20260507-f10-f11-tier1-full-pipeline"
+      ;;
+    F12-*|F12)
+      echo "bare 85 / solo_claude 99 in 20260511-f12-webhook-headroom"
+      ;;
+    F15-*|F15)
+      echo "bare 99 / solo_claude 94 in 20260511-f15-concurrency-headroom"
+      ;;
+    F22-*|F22)
+      echo "bare 94 / solo_claude 98 in 20260508-f22-exact-error-headroom"
+      ;;
+    F26-*|F26)
+      echo "solo_claude scored 98 in 20260508-f26-headroom"
+      ;;
+    F27-*|F27)
+      echo "solo_claude scored 94 in 20260511-f27-headroom-smoke-061401"
+      ;;
+    F28-*|F28)
+      echo "corrected-oracle reverify scored solo_claude 98 in 20260511-f28-policy-oraclefix-reverified-pair"
+      ;;
+    F29-*|F29)
+      echo "corrected headroom scored solo_claude 92 in 20260510-f29-headroom-v2"
+      ;;
+    F30-*|F30)
+      echo "solo_claude scored 98 in 20260511-f30-headroom-v1"
+      ;;
+    F31-*|F31)
+      echo "solo_claude scored 98 with bare disqualifiers in 20260512-f31-seat-rebalance-headroom"
+      ;;
+    F32-*|F32)
+      echo "bare 33 / solo_claude 98 in 20260512-f32-subscription-renewal-headroom"
+      ;;
+    S2-*|S2)
+      echo "bare 33 / solo_claude 99 with solo timeout in 20260513-s2-inventory-headroom"
+      ;;
+    S3-*|S3)
+      echo "bare 33 / solo_claude 99 with solo timeout in 20260513-s3-ticket-headroom"
+      ;;
+    S4-*|S4)
+      echo "bare 33 / solo_claude 98 with solo timeout in 20260513-s4-return-headroom"
+      ;;
+    S5-*|S5)
+      echo "bare 33 / solo_claude 98 with solo timeout in 20260513-s5-credit-headroom"
+      ;;
+    S6-*|S6)
+      echo "bare 33 / solo_claude 98 with solo timeout in 20260514-s6-refund-headroom-v1"
+      ;;
+    *)
+      return 1
+      ;;
+  esac
+}

package/benchmark/auto-resolve/scripts/pair_evidence_contract.py

@@ -0,0 +1,269 @@
+"""Shared pair-evidence contract for benchmark audits."""
+from __future__ import annotations
+
+import json
+import math
+import re
+from pathlib import Path
+from typing import Any
+
+
+ALLOWED_PAIR_ARMS = {"l2_risk_probes", "l2_gated"}
+CANONICAL_PAIR_TRIGGER_REASONS = {
+    "mode.verify-only",
+    "mode.pair-verify",
+    "complexity.high",
+    "complexity.large",
+    "spec.complexity.high",
+    "spec.complexity.large",
+    "spec.solo_headroom_hypothesis",
+    "risk.high",
+    "risk_probes.enabled",
+    "risk_probes.present",
+    "coverage.failed",
+    "mechanical.warning",
+    "judge.warning",
+}
+HISTORICAL_PAIR_TRIGGER_REASON_ALIASES = {
+    "risk_profile.high_risk",
+    "risk_probes_enabled",
+}
+HISTORICAL_NORMALIZED_PAIR_TRIGGER_REASON_ALIASES = {
+    "complexity.high.spec.frontmatter",
+    "frontmatter.complexity.high",
+    "high.complexity.spec",
+    "high.risk.profile",
+    "spec.frontmatter.complexity.high",
+    "state.complexity.high",
+}
+# Benchmark readers accept historical aliases only for archived artifacts.
+# Runtime /devlyn:resolve state must continue to emit canonical reasons.
+KNOWN_PAIR_TRIGGER_REASONS = (
+    CANONICAL_PAIR_TRIGGER_REASONS | HISTORICAL_PAIR_TRIGGER_REASON_ALIASES
+)
+OBSERVABLE_COMMAND_MARKERS = ("command", "observable", "expose")
+BACKTICKED_TEXT_RE = re.compile(r"`[^`\n]+`")
+RESERVED_BACKTICK_TERMS = {"solo-headroom hypothesis", "solo_claude", "miss"}
+COMMAND_PREFIXES = {
+    "bash",
+    "bun",
+    "cargo",
+    "git",
+    "go",
+    "jest",
+    "make",
+    "node",
+    "npm",
+    "pnpm",
+    "printf",
+    "pytest",
+    "python",
+    "python3",
+    "ruff",
+    "sh",
+    "uv",
+    "vitest",
+    "yarn",
+}
+
+
+def reject_json_constant(token: str) -> None:
+    raise ValueError(f"invalid JSON numeric constant: {token}")
+
+
+def loads_strict_json_object(text: str) -> dict[str, Any]:
+    data = json.loads(text, parse_constant=reject_json_constant)
+    if not isinstance(data, dict):
+        raise ValueError("top-level JSON value must be an object")
+    return data
+
+
+def normalized_pair_trigger_reason(reason: str) -> str:
+    return re.sub(r"[^a-z0-9]+", ".", reason.lower()).strip(".")
+
+
+def is_known_pair_trigger_reason(reason: str) -> bool:
+    normalized = normalized_pair_trigger_reason(reason)
+    return (
+        reason in CANONICAL_PAIR_TRIGGER_REASONS
+        or reason in HISTORICAL_PAIR_TRIGGER_REASON_ALIASES
+        or normalized in HISTORICAL_NORMALIZED_PAIR_TRIGGER_REASON_ALIASES
+    )
+
+
+def is_canonical_pair_trigger_reason(reason: str) -> bool:
+    return reason in CANONICAL_PAIR_TRIGGER_REASONS
+
+
+def is_historical_pair_trigger_reason(reason: str) -> bool:
+    normalized = normalized_pair_trigger_reason(reason)
+    return (
+        reason in HISTORICAL_PAIR_TRIGGER_REASON_ALIASES
+        or normalized in HISTORICAL_NORMALIZED_PAIR_TRIGGER_REASON_ALIASES
+    )
+
+
+def has_known_pair_trigger_reason(reasons: list[str]) -> bool:
+    return any(is_known_pair_trigger_reason(reason) for reason in reasons)
+
+
+def all_known_pair_trigger_reasons(reasons: list[str]) -> bool:
+    return all(is_known_pair_trigger_reason(reason) for reason in reasons)
+
+
+def has_canonical_pair_trigger_reason(reasons: list[str]) -> bool:
+    return any(is_canonical_pair_trigger_reason(reason) for reason in reasons)
+
+
+def has_historical_pair_trigger_reason(reasons: list[str]) -> bool:
+    return any(is_historical_pair_trigger_reason(reason) for reason in reasons)
+
+
+def is_command_like_backtick(value: str) -> bool:
+    stripped = value.strip()
+    lower = stripped.lower()
+    if not stripped or lower in RESERVED_BACKTICK_TERMS:
+        return False
+    first = lower.split(maxsplit=1)[0]
+    return (
+        first in COMMAND_PREFIXES
+        or any(marker in stripped for marker in ("/", "$", "=", "|", "&&", ";"))
+        or stripped.endswith((".js", ".py", ".sh"))
+    )
+
+
+def actionable_observable_commands(text: str) -> list[str]:
+    commands: list[str] = []
+    for line in text.splitlines():
+        lower = line.lower()
+        if "miss" not in lower or not any(marker in lower for marker in OBSERVABLE_COMMAND_MARKERS):
+            continue
+        for match in BACKTICKED_TEXT_RE.finditer(line):
+            value = match.group(0).strip("`")
+            if is_command_like_backtick(value):
+                commands.append(value)
+    return commands
+
+
+def has_actionable_solo_headroom_hypothesis_text(text: str) -> bool:
+    lower = text.lower()
+    return (
+        "solo-headroom hypothesis" in lower
+        and "solo_claude" in lower
+        and "miss" in lower
+        and bool(actionable_observable_commands(text))
+    )
+
+
+def path_has_actionable_solo_headroom_hypothesis(path: Path) -> bool:
+    try:
+        text = path.read_text(encoding="utf-8")
+    except OSError:
+        return False
+    return has_actionable_solo_headroom_hypothesis_text(text)
+
+
+def normalize_pair_evidence_row(
+    *,
+    fixture: str,
+    run_id: str,
+    pair_arm: object,
+    row: dict[str, Any],
+) -> dict[str, Any] | None:
+    bare_score = row.get("bare_score")
+    solo_score = row.get("solo_score")
+    pair_score = row.get("pair_score")
+    pair_margin = row.get("pair_margin")
+    pair_mode = row.get("pair_mode")
+    pair_trigger_eligible = row.get("pair_trigger_eligible")
+    pair_trigger_reasons = row.get("pair_trigger_reasons")
+    wall_ratio = row.get("pair_solo_wall_ratio")
+    if not fixture or not run_id:
+        return None
+    if not isinstance(pair_arm, str) or pair_arm not in ALLOWED_PAIR_ARMS:
+        return None
+    if not all(is_score(value) for value in [bare_score, solo_score, pair_score]):
+        return None
+    if not is_strict_int(pair_margin):
+        return None
+    if pair_margin != pair_score - solo_score:
+        return None
+    if pair_mode is not True:
+        return None
+    if pair_trigger_eligible is not True:
+        return None
+    if not (
+        isinstance(pair_trigger_reasons, list)
+        and pair_trigger_reasons
+        and all(isinstance(reason, str) for reason in pair_trigger_reasons)
+        and all_known_pair_trigger_reasons(pair_trigger_reasons)
+        and has_canonical_pair_trigger_reason(pair_trigger_reasons)
+    ):
+        return None
+    if not is_strict_number(wall_ratio):
+        return None
+    normalized = {
+        "run_id": run_id,
+        "pair_arm": pair_arm,
+        "bare_score": bare_score,
+        "solo_score": solo_score,
+        "pair_score": pair_score,
+        "pair_margin": pair_margin,
+        "pair_mode": pair_mode,
+        "pair_trigger_eligible": pair_trigger_eligible,
+        "pair_trigger_reasons": pair_trigger_reasons,
+        "pair_trigger_has_canonical_reason": True,
+        "pair_trigger_has_hypothesis_reason": (
+            "spec.solo_headroom_hypothesis" in pair_trigger_reasons
+        ),
+        "pair_solo_wall_ratio": wall_ratio,
+    }
+    return normalized
+
+
+def best_pair_evidence(evidence: list[object]) -> dict[str, Any] | None:
+    candidates = [
+        normalized
+        for item in evidence
+        if isinstance(item, dict)
+        if isinstance(item.get("run_id"), str)
+        for normalized in [
+            normalize_pair_evidence_row(
+                fixture="_",
+                run_id=item["run_id"],
+                pair_arm=item.get("pair_arm"),
+                row=item,
+            )
+        ]
+        if normalized is not None
+    ]
+    if not candidates:
+        return None
+
+    def key(item: dict[str, Any]) -> tuple[int, int, str]:
+        margin = item.get("pair_margin")
+        pair_score = item.get("pair_score")
+        return (
+            margin if isinstance(margin, int) else -10_000,
+            pair_score if isinstance(pair_score, int) else -10_000,
+            str(item.get("run_id") or ""),
+        )
+
+    return max(candidates, key=key)
+
+
+def is_strict_int(value: object) -> bool:
+    return isinstance(value, int) and not isinstance(value, bool)
+
+
+def is_score(value: object) -> bool:
+    return is_strict_int(value) and 0 <= value <= 100
+
+
+def is_strict_number(value: object) -> bool:
+    return (
+        isinstance(value, (int, float))
+        and not isinstance(value, bool)
+        and math.isfinite(value)
+        and value > 0
+    )

package/benchmark/auto-resolve/scripts/prepare-swebench-frozen-case.py

@@ -17,8 +17,12 @@ import subprocess
 from pathlib import Path
 from typing import Any
 
+from pair_evidence_contract import loads_strict_json_object
+
 
 SAFE_ID = re.compile(r"^[A-Za-z0-9_.-]+$")
+SAFE_REPO = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$")
+SAFE_COMMIT = re.compile(r"^[0-9a-fA-F]{7,40}$")
 
 
 def run(cmd: list[str], cwd: Path | None = None) -> None:
@@ -26,11 +30,12 @@ def run(cmd: list[str], cwd: Path | None = None) -> None:
 
 
 def read_json(path: Path) -> dict[str, Any]:
-    with path.open(encoding="utf8") as f:
-        data = json.load(f)
-    if not isinstance(data, dict):
-        raise ValueError(f"expected JSON object: {path}")
-    return data
+    try:
+        return loads_strict_json_object(path.read_text(encoding="utf8"))
+    except ValueError as exc:
+        if str(exc) == "top-level JSON value must be an object":
+            raise ValueError(f"expected JSON object: {path}") from exc
+        raise
 
 
 def require_text(instance: dict[str, Any], key: str) -> str:
@@ -40,14 +45,38 @@ def require_text(instance: dict[str, Any], key: str) -> str:
     return value.strip()
 
 
+def positive_int(value: str) -> int:
+    try:
+        parsed = int(value)
+    except ValueError as exc:
+        raise argparse.ArgumentTypeError("must be an integer") from exc
+    if parsed <= 0:
+        raise argparse.ArgumentTypeError("must be > 0")
+    return parsed
+
+
+def require_safe_repo(instance: dict[str, Any]) -> str:
+    repo = require_text(instance, "repo")
+    if not SAFE_REPO.match(repo):
+        raise ValueError(f"unsafe SWE-bench repo: {repo!r}")
+    return repo
+
+
+def require_safe_base_commit(instance: dict[str, Any]) -> str:
+    base_commit = require_text(instance, "base_commit")
+    if not SAFE_COMMIT.match(base_commit):
+        raise ValueError(f"unsafe SWE-bench base_commit: {base_commit!r}")
+    return base_commit
+
+
 def repo_cache_name(repo: str, base_commit: str) -> str:
     safe_repo = repo.replace("/", "__")
     return f"{safe_repo}-{base_commit[:12]}"
 
 
 def prepare_repo(instance: dict[str, Any], repo_dir: Path | None, repos_root: Path) -> Path:
-    repo = require_text(instance, "repo")
-    base_commit = require_text(instance, "base_commit")
+    repo = require_safe_repo(instance)
+    base_commit = require_safe_base_commit(instance)
     repos_root.mkdir(parents=True, exist_ok=True)
     dest = repos_root / repo_cache_name(repo, base_commit)
 
@@ -72,8 +101,8 @@ def write_case_files(
     timeout_seconds: int,
 ) -> None:
     instance_id = require_text(instance, "instance_id")
-    repo = require_text(instance, "repo")
-    base_commit = require_text(instance, "base_commit")
+    repo = require_safe_repo(instance)
+    base_commit = require_safe_base_commit(instance)
     problem = require_text(instance, "problem_statement")
     case_dir.mkdir(parents=True, exist_ok=True)
 
@@ -196,7 +225,7 @@ def main() -> int:
         type=Path,
         help="Local clone/source repo to copy instead of cloning GitHub; useful for tests and cached runs.",
     )
-    parser.add_argument("--timeout-seconds", type=int, default=2400)
+    parser.add_argument("--timeout-seconds", type=positive_int, default=2400)
     args = parser.parse_args()
 
     instance = read_json(args.instance_json)

package/benchmark/auto-resolve/scripts/prepare-swebench-frozen-corpus.py

@@ -10,6 +10,8 @@ import tempfile
 from pathlib import Path
 from typing import Any
 
+from pair_evidence_contract import loads_strict_json_object, reject_json_constant
+
 
 def read_jsonl(path: Path) -> list[dict[str, Any]]:
     rows: list[dict[str, Any]] = []
@@ -17,7 +19,7 @@ def read_jsonl(path: Path) -> list[dict[str, Any]]:
         for line_no, line in enumerate(f, start=1):
             if not line.strip():
                 continue
-            value = json.loads(line)
+            value = json.loads(line, parse_constant=reject_json_constant)
             if not isinstance(value, dict):
                 raise ValueError(f"{path}:{line_no}: expected JSON object")
             rows.append(value)
@@ -31,6 +33,32 @@ def require_text(row: dict[str, Any], key: str, source: str) -> str:
     return value.strip()
 
 
+def positive_int(value: str) -> int:
+    try:
+        parsed = int(value)
+    except ValueError as exc:
+        raise argparse.ArgumentTypeError("must be an integer") from exc
+    if parsed <= 0:
+        raise argparse.ArgumentTypeError("must be > 0")
+    return parsed
+
+
+def parse_prepared_case(stdout: str, source: str) -> dict[str, Any]:
+    try:
+        value = loads_strict_json_object(stdout)
+    except ValueError as exc:
+        if str(exc) == "top-level JSON value must be an object":
+            raise ValueError(f"{source}: expected JSON object") from exc
+        raise
+    for key in ("instance_id", "case_dir", "repo_dir", "run_command"):
+        if key == "run_command":
+            if not isinstance(value.get(key), list) or not value[key]:
+                raise ValueError(f"{source}: missing non-empty {key!r}")
+        elif not isinstance(value.get(key), str) or not value[key].strip():
+            raise ValueError(f"{source}: missing non-empty {key!r}")
+    return value
+
+
 def main() -> int:
     parser = argparse.ArgumentParser()
     parser.add_argument("--instances-jsonl", required=True, type=Path)
@@ -47,8 +75,8 @@ def main() -> int:
     )
     parser.add_argument("--repo-dir", type=Path, help="Use one local repo clone for every selected instance.")
     parser.add_argument("--instance-id", action="append", help="Prepare only these instance ids.")
-    parser.add_argument("--limit", type=int, help="Prepare at most N matched instances after filtering.")
-    parser.add_argument("--timeout-seconds", type=int, default=2400)
+    parser.add_argument("--limit", type=positive_int, help="Prepare at most N matched instances after filtering.")
+    parser.add_argument("--timeout-seconds", type=positive_int, default=2400)
     parser.add_argument("--out-manifest", type=Path)
     args = parser.parse_args()
 
@@ -61,6 +89,8 @@ def main() -> int:
         predictions[instance_id] = row
 
     selected_ids = args.instance_id or list(predictions)
+    if not selected_ids:
+        raise ValueError("no prediction instances selected")
     script = Path(__file__).with_name("prepare-swebench-frozen-case.py")
     prepared: list[dict[str, Any]] = []
     with tempfile.TemporaryDirectory() as tmp:
@@ -98,7 +128,7 @@ def main() -> int:
             if args.repo_dir is not None:
                 cmd.extend(["--repo-dir", str(args.repo_dir)])
             completed = subprocess.run(cmd, check=True, text=True, capture_output=True)
-            prepared.append(json.loads(completed.stdout))
+            prepared.append(parse_prepared_case(completed.stdout, f"prepared case {instance_id}"))
 
     manifest = {
         "instances_jsonl": str(args.instances_jsonl),

package/benchmark/auto-resolve/scripts/prepare-swebench-solver-worktree.py

@@ -11,8 +11,12 @@ import subprocess
 from pathlib import Path
 from typing import Any
 
+from pair_evidence_contract import reject_json_constant
+
 
 SAFE_ID = re.compile(r"^[A-Za-z0-9_.-]+$")
+SAFE_REPO = re.compile(r"^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$")
+SAFE_COMMIT = re.compile(r"^[0-9a-fA-F]{7,40}$")
 
 
 def run(cmd: list[str], cwd: Path | None = None) -> None:
@@ -25,7 +29,7 @@ def read_instances(path: Path) -> list[dict[str, Any]]:
         for line_no, line in enumerate(f, start=1):
             if not line.strip():
                 continue
-            value = json.loads(line)
+            value = json.loads(line, parse_constant=reject_json_constant)
             if not isinstance(value, dict):
                 raise ValueError(f"{path}:{line_no}: expected JSON object")
             rows.append(value)
@@ -39,6 +43,20 @@ def require_text(instance: dict[str, Any], key: str) -> str:
     return value.strip()
 
 
+def require_safe_repo(instance: dict[str, Any]) -> str:
+    repo = require_text(instance, "repo")
+    if not SAFE_REPO.match(repo):
+        raise ValueError(f"unsafe SWE-bench repo: {repo!r}")
+    return repo
+
+
+def require_safe_base_commit(instance: dict[str, Any]) -> str:
+    base_commit = require_text(instance, "base_commit")
+    if not SAFE_COMMIT.match(base_commit):
+        raise ValueError(f"unsafe SWE-bench base_commit: {base_commit!r}")
+    return base_commit
+
+
 def pick_instance(path: Path, instance_id: str) -> dict[str, Any]:
     matches = [row for row in read_instances(path) if row.get("instance_id") == instance_id]
     if len(matches) != 1:
@@ -51,8 +69,8 @@ def repo_cache_name(repo: str, base_commit: str) -> str:
 
 
 def prepare_repo(instance: dict[str, Any], repos_root: Path) -> Path:
-    repo = require_text(instance, "repo")
-    base_commit = require_text(instance, "base_commit")
+    repo = require_safe_repo(instance)
+    base_commit = require_safe_base_commit(instance)
     repos_root.mkdir(parents=True, exist_ok=True)
     dest = repos_root / repo_cache_name(repo, base_commit)
 
@@ -77,8 +95,8 @@ def copy_worktree(repo_path: Path, worktree: Path) -> None:
 
 def write_spec(instance: dict[str, Any], worktree: Path) -> Path:
     instance_id = require_text(instance, "instance_id")
-    repo = require_text(instance, "repo")
-    base_commit = require_text(instance, "base_commit")
+    repo = require_safe_repo(instance)
+    base_commit = require_safe_base_commit(instance)
     problem = require_text(instance, "problem_statement")
     spec_path = worktree / "docs" / "roadmap" / "phase-1" / f"{instance_id}.md"
     spec_path.parent.mkdir(parents=True, exist_ok=True)