master-skill 0.10.1 → 0.12.0

package/scripts/regrade-report.py

@@ -0,0 +1,157 @@
+#!/usr/bin/env python3
+"""Re-grade a committed run against the current judge and fixtures. Offline, free.
+
+`check_response` is deterministic, and every answer has been stored since PR
+#142. So any change to the judge or to the fixtures can be measured against an
+already-paid-for run for nothing, instead of claimed.
+
+That matters most for changes that *relax* something. Moving a requirement into
+`must_convey` — "the substring matcher cannot decide this" — is honest when the
+adjudication says so and is laundering when it does not. Re-grading the ¥3.89
+sweep shows exactly which cases move and whether anything moved the wrong way.
+
+    python3 scripts/regrade-report.py eval/reports/0.11.0-06b8142-deepseek.json
+
+Results are joined to fixtures **by question text**, never by position: a
+fixture added or removed shifts every index after it, and grading an answer
+against someone else's question would produce a confident, meaningless number.
+"""
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+
+import importlib.util
+
+ROOT = Path(__file__).resolve().parent.parent
+_spec = importlib.util.spec_from_file_location(
+    "_fidelity", ROOT / "scripts" / "test-fidelity.py"
+)
+_fidelity = importlib.util.module_from_spec(_spec)
+sys.modules["_fidelity"] = _fidelity
+_spec.loader.exec_module(_fidelity)
+
+from verify_citations import load_declared_ids, load_member_aliases  # noqa: E402
+
+
+def load_fixtures() -> dict[str, list[dict]]:
+    fixtures: dict[str, list[dict]] = {}
+    for path in sorted((ROOT / "prebuilt").glob("*/tests/fidelity.jsonl")):
+        fixtures[path.parent.parent.name] = [
+            json.loads(line) for line in path.read_text().splitlines() if line.strip()
+        ]
+    return fixtures
+
+
+def regrade(report: dict, fixtures: dict[str, list[dict]]) -> dict:
+    """Re-run the judge over every stored answer. Raises if the run stored none."""
+    if not any(
+        "response" in result
+        for suite in report["suites"]
+        for result in suite["results"]
+    ):
+        raise ValueError("no stored answers in this report — nothing to re-grade")
+
+    cases: list[dict] = []
+    graded_results: list[dict] = []
+    by_test_type: dict[str, dict[str, int]] = {}
+
+    for suite in report["suites"]:
+        master = suite["master"]
+        cases_for_master = fixtures.get(master, [])
+        try:
+            declared = load_declared_ids(master) or None
+            aliases = load_member_aliases(master) or None
+        except (FileNotFoundError, ValueError):
+            declared = None
+            aliases = None
+
+        for result in suite["results"]:
+            if result.get("status") in ("truncated", "api_error"):
+                continue
+            index = result["index"]
+            fixture = cases_for_master[index] if index < len(cases_for_master) else None
+            if fixture is None or fixture.get("q") != result["question"]:
+                raise ValueError(
+                    f"{master} #{index}: the fixture at this position does not match "
+                    f"the question that was graded. Fixtures moved; re-grading by "
+                    f"position would compare an answer with someone else's question."
+                )
+            check = _fidelity.check_response(
+                result.get("response") or "",
+                fixture,
+                declared_ids=declared,
+                member_aliases=aliases,
+            )
+            entry = _fidelity.result_entry(index, fixture, check, result.get("response") or "")
+            graded_results.append(entry)
+            was = result["status"]
+            now = "PASS" if check["passed"] else "FAIL"
+            bucket = by_test_type.setdefault(
+                result["test_type"], {"graded": 0, "was": 0, "now": 0}
+            )
+            bucket["graded"] += 1
+            bucket["was"] += was == "PASS"
+            bucket["now"] += now == "PASS"
+            cases.append(
+                {
+                    "master": master,
+                    "index": index,
+                    "test_type": result["test_type"],
+                    "was": was,
+                    "now": now,
+                    "needs_review": check["needs_review"],
+                }
+            )
+
+    return {
+        "meta": report.get("meta", {}),
+        "cases": cases,
+        "by_test_type": by_test_type,
+        "mentions": _fidelity.summarize_mentions(graded_results),
+        "needs_review": sum(1 for c in cases if c["needs_review"]),
+    }
+
+
+def main(argv: list[str]) -> int:
+    if len(argv) != 2:
+        print(f"usage: {Path(argv[0]).name} <eval/reports/*.json>")
+        return 2
+    path = Path(argv[1])
+    out = regrade(json.loads(path.read_text()), load_fixtures())
+    meta = out["meta"]
+    print(f"{path.name} — commit {meta.get('commit_short')}, model {meta.get('model')}")
+    print(f"\n{'test_type':12} {'graded':>7} {'as graded':>11} {'re-graded':>11}")
+    total = {"graded": 0, "was": 0, "now": 0}
+    for test_type, b in sorted(out["by_test_type"].items()):
+        for key in total:
+            total[key] += b[key]
+        print(f"{test_type:12} {b['graded']:>7} "
+              f"{b['was']:>6} {b['was']/b['graded']:>4.0%} "
+              f"{b['now']:>6} {b['now']/b['graded']:>4.0%}")
+    print(f"{'TOTAL':12} {total['graded']:>7} "
+          f"{total['was']:>6} {total['was']/total['graded']:>4.0%} "
+          f"{total['now']:>6} {total['now']/total['graded']:>4.0%}")
+
+    m = out["mentions"]
+    print(
+        f"\nmention coverage {m['mentions_decided']}/{m['mention_requirements']} "
+        f"= {m['mention_coverage']}  "
+        f"({m['mentions_unverified']} undecidable, "
+        f"{m['script_mismatches']} answers in the wrong script)"
+    )
+    print(f"cases needing adjudication: {out['needs_review']}")
+
+    moved_wrong = [c for c in out["cases"] if c["was"] == "PASS" and c["now"] == "FAIL"]
+    if moved_wrong:
+        print(f"\nWARNING: {len(moved_wrong)} case(s) that passed now fail:")
+        for c in moved_wrong:
+            print(f"  {c['master']} #{c['index']} ({c['test_type']})")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv))

package/scripts/smoke-eval-sdk.py

@@ -0,0 +1,174 @@
+#!/usr/bin/env python3
+"""Keyless end-to-end smoke for the pinned eval SDKs.
+
+`check-eval-sdk-surface.py` reads signatures. That catches a renamed keyword,
+not a changed wire format, a response model that parses differently, or a
+client that stops honouring its base URL. And the real path —
+`scripts/test-fidelity.py` — never runs in CI, because grading needs an API
+key and there is none. So every Dependabot bump of `anthropic` or `openai`
+arrived green having exercised nothing, which `requirements-eval.txt` says in
+as many words.
+
+This runs that real path without a key or a network. A local HTTP server
+answers in each provider's own wire format; `test-fidelity.py` builds its
+request, the pinned SDK sends and parses it, and the answer goes through
+grading and the citation audit exactly as in a sweep. Then it checks what
+arrived at the server and what came back.
+
+    python3 scripts/smoke-eval-sdk.py           # must exit 0
+    python3 scripts/smoke-eval-sdk.py --break   # must exit 1
+
+`--break` makes the server return a reply with no answer text while every
+assertion stays the same. CI runs both: a smoke that cannot fail is a green
+tick, not a check.
+
+First written 2026-09-14 to verify anthropic 1.4.0 → 1.5.0 and openai
+3.10.0 → 3.13.0 (#174, #175).
+"""
+
+from __future__ import annotations
+
+import contextlib
+import importlib.util
+import io
+import json
+import os
+import sys
+import threading
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from pathlib import Path
+
+REPO = Path(__file__).resolve().parent.parent
+ANSWER = "菩提自性，本来清净，但用此心，直了成佛。【《六祖大师法宝坛经》行由品，T48n2008】"
+CACHE_CREATED = 6700
+
+
+def _server(broken: bool, received: list) -> ThreadingHTTPServer:
+    class Handler(BaseHTTPRequestHandler):
+        def log_message(self, *args):  # keep CI logs quiet
+            pass
+
+        def do_POST(self):
+            length = int(self.headers.get("content-length", 0))
+            body = json.loads(self.rfile.read(length) or b"{}")
+            headers = {k.lower(): v for k, v in self.headers.items()}
+            received.append((self.path, body, headers))
+            if self.path.endswith("/messages"):
+                reply = {
+                    "id": "msg_smoke", "type": "message", "role": "assistant",
+                    "model": body.get("model"),
+                    "content": [] if broken else [{"type": "text", "text": ANSWER}],
+                    "stop_reason": "end_turn", "stop_sequence": None,
+                    "usage": {
+                        "input_tokens": 21, "output_tokens": 40,
+                        "cache_creation_input_tokens": CACHE_CREATED,
+                        "cache_read_input_tokens": 0,
+                    },
+                }
+            else:
+                reply = {
+                    "id": "chatcmpl-smoke", "object": "chat.completion", "created": 1,
+                    "model": body.get("model"),
+                    "choices": [{
+                        "index": 0, "finish_reason": "stop",
+                        "message": {"role": "assistant", "content": None if broken else ANSWER},
+                    }],
+                    "usage": {"prompt_tokens": 21, "completion_tokens": 40, "total_tokens": 61},
+                }
+            data = json.dumps(reply).encode()
+            self.send_response(200)
+            self.send_header("content-type", "application/json")
+            self.send_header("content-length", str(len(data)))
+            self.end_headers()
+            self.wfile.write(data)
+
+    return ThreadingHTTPServer(("127.0.0.1", 0), Handler)
+
+
+def main(argv: list[str]) -> int:
+    broken = "--break" in argv
+    try:
+        import anthropic
+        import openai
+    except ImportError as error:
+        # Missing is a failure, not a skip: this only runs where
+        # requirements-eval.txt is installed, and silence here is how a bump
+        # used to pass.
+        print(f"✗ eval SDK not importable: {error} (pip install -r requirements-eval.txt)")
+        return 1
+
+    received: list = []
+    server = _server(broken, received)
+    port = server.server_address[1]
+    threading.Thread(target=server.serve_forever, daemon=True).start()
+
+    # A proxy variable would send 127.0.0.1 somewhere else; a real key must
+    # never be read by a smoke that is supposed to cost nothing.
+    for key in ("HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "http_proxy", "https_proxy", "all_proxy"):
+        os.environ.pop(key, None)
+    os.environ.update({
+        "ANTHROPIC_API_KEY": "sk-ant-smoke",
+        "DEEPSEEK_API_KEY": "sk-smoke",
+        "ANTHROPIC_BASE_URL": f"http://127.0.0.1:{port}",
+        "NO_PROXY": "127.0.0.1,localhost",
+    })
+    os.chdir(REPO)
+    sys.path.insert(0, str(REPO / "scripts"))
+    spec = importlib.util.spec_from_file_location("test_fidelity", REPO / "scripts" / "test-fidelity.py")
+    fidelity = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(fidelity)
+    fidelity.PROVIDERS["deepseek"]["base_url"] = f"http://127.0.0.1:{port}/v1"
+
+    print(f"eval SDK smoke — anthropic {anthropic.__version__}, openai {openai.__version__}"
+          f"{' (--break: server returns no answer text)' if broken else ''}")
+    failures = 0
+
+    def check(label: str, ok: bool) -> None:
+        nonlocal failures
+        print(f"  {'✓' if ok else '✗'} {label}")
+        failures += 0 if ok else 1
+
+    for provider, extra in (("anthropic", []), ("deepseek", ["--model", "deepseek-v4-flash"])):
+        received.clear()
+        sys.argv = ["test-fidelity.py", "--master", "master-huineng", "--provider", provider,
+                    "--max-tests", "1", "--concurrency", "1", "--json", *extra]
+        out = io.StringIO()
+        with contextlib.redirect_stdout(out), contextlib.redirect_stderr(io.StringIO()):
+            fidelity.main()
+        suite = json.loads(out.getvalue())[0]
+        result = suite["results"][0]
+        print(f"── {provider}: {result.get('status')}")
+
+        check("the request reached the local server, not the internet", bool(received))
+        if not received:
+            continue
+        path, body, headers = received[0]
+        check("the reply was graded, not recorded as api_error",
+              result.get("status") in ("PASS", "FAIL"))
+        check("the answer text survived SDK parsing intact", result.get("response") == ANSWER)
+        check("the citation audit ran on it",
+              (suite.get("audit") or {}).get("citations_checked", 0) >= 1)
+        check("model, max_tokens and messages were sent",
+              {"model", "max_tokens", "messages"} <= set(body))
+        if provider == "anthropic":
+            check(f"sent to /v1/messages (got {path})", path == "/v1/messages")
+            check("the persona prompt carries cache_control",
+                  (body.get("system") or [{}])[0].get("cache_control") == {"type": "ephemeral"})
+            check("cache token usage was read back",
+                  (suite.get("cache") or {}).get("created") == CACHE_CREATED)
+            check("the pinned SDK is the one that sent it",
+                  anthropic.__version__ in headers.get("user-agent", ""))
+        else:
+            check(f"sent to /v1/chat/completions (got {path})", path == "/v1/chat/completions")
+            check("the key went in the Authorization header",
+                  headers.get("authorization") == "Bearer sk-smoke")
+            check("the pinned SDK is the one that sent it",
+                  openai.__version__ in headers.get("user-agent", ""))
+
+    server.shutdown()
+    print(f"{'✓ eval SDK path intact' if not failures else f'✗ {failures} check(s) failed'}")
+    return 1 if failures else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv[1:]))