master-skill 0.11.0 → 0.12.0

package/scripts/check-gate-liveness.py

@@ -25,12 +25,15 @@ Usage:
 from __future__ import annotations
 
 import argparse
+import functools
 import json
 import re
 import subprocess
 import sys
 from pathlib import Path
 
+import yaml
+
 # A verdict — as opposed to a skip, an error, or a dry run.
 GRADED_STATUSES = {"PASS", "FAIL"}
 
@@ -133,6 +136,116 @@ def check_every_skill_has_fixtures(prebuilt_dir: Path) -> list[str]:
     return problems
 
 
+# A gate is "advisory" when it can exit 0 without doing the work its name
+# promises — the fidelity smoke passing in 10s because no API key is set. That
+# is a legitimate project decision (CONTRIBUTING.md §2: grading is a local /
+# pre-release step, not a CI expense). What is NOT legitimate is it being
+# invisible: a required check's green tick looks identical either way.
+#
+# So each advisory gate must be declared here, saying what it does not check.
+# An undeclared one fails this script, and the declared roster is printed on
+# every run — `npm test` always answers "what did the green tick examine?".
+#
+# Detection is automatic only for the "missing secret -> exit 0" shape, which
+# is the one this repo shipped. A job that goes quiet for some other reason —
+# `Dependency review` skipping because the dependency graph is off — has to be
+# added by hand. `check_declared_gates_still_exist` then keeps the entry from
+# outliving the job, but nothing can force a new *shape* to be noticed. If you
+# add a gate that can pass without working, put it here yourself.
+ADVISORY_GATES = {
+    "Fidelity smoke (1 master × 1 fixture)": (
+        "grades nothing when ANTHROPIC_API_KEY is unset (it always has been) — "
+        "the green tick means structure validation passed, not that a model "
+        "response was graded. Set repo variable FIDELITY_GRADING_REQUIRED=true "
+        "once the secret exists to make the skip a hard failure."
+    ),
+    "Fidelity tests — full suite (weekly + manual)": (
+        "same skip as the smoke, on the weekly cron"
+    ),
+    "Persona-fidelity schema + advisory eval": (
+        "llm-rubric eval is `|| true` and is skipped entirely without a key; "
+        "only the promptfoo schema + repo-convention validation is real"
+    ),
+}
+
+# The shape of a silent skip: a step that exits 0 because a secret is missing.
+_SKIP_ON_MISSING_SECRET = re.compile(r'\[\s+-z\s+"\$\{[A-Z_]+:-\}"\s+\]')
+
+
+def _job_display_name(job_id: str, job: dict) -> str:
+    """The name this file can see for a job.
+
+    NOT, in general, "the string branch protection matches on" — an earlier
+    version of this docstring said that and it is false for any matrix job.
+    GitHub expands `name: CodeQL (${{ matrix.language }})` into one check run
+    per leg (`CodeQL (python)`, …); statically all that is visible here is the
+    unexpanded template. So a matrix job can never be matched by an
+    ADVISORY_GATES key, and this module cannot police one. The roster covers
+    the non-matrix jobs, which are the ones this repo has actually shipped a
+    silent skip in; a matrix job that grows one has to be caught by review.
+    """
+    name = job.get("name") if isinstance(job, dict) else None
+    return str(name) if name else job_id
+
+
+def _job_skips_on_missing_secret(job: dict) -> bool:
+    steps = job.get("steps") or [] if isinstance(job, dict) else []
+    for step in steps:
+        if not isinstance(step, dict):
+            continue
+        run = step.get("run")
+        if isinstance(run, str) and _SKIP_ON_MISSING_SECRET.search(run):
+            return True
+    return False
+
+
+def _iter_jobs(workflow_docs: dict[str, dict]):
+    """Yield (path, display_name, job) for every job in every workflow.
+
+    Per **job**, not per file: `validate-and-test.yml` holds six jobs, and a
+    file-level check would let a newly-silent seventh hide behind its declared
+    siblings.
+    """
+    for path, doc in sorted(workflow_docs.items()):
+        jobs = (doc or {}).get("jobs") or {}
+        if not isinstance(jobs, dict):
+            continue
+        for job_id, job in jobs.items():
+            if isinstance(job, dict):
+                yield path, _job_display_name(job_id, job), job
+
+
+def check_advisory_gates_declared(workflow_docs: dict[str, dict]) -> list[str]:
+    """Every job that can exit 0 on a missing secret must be declared above.
+
+    Catches the case this repo actually shipped: a *required* branch-protection
+    check that has never once graded a response, with nothing in the repo
+    saying so.
+    """
+    return [
+        f"{path}: job {name!r} exits 0 when a secret is missing but is not in "
+        "ADVISORY_GATES — a gate that can pass without checking anything must "
+        "say so, or stop doing it"
+        for path, name, job in _iter_jobs(workflow_docs)
+        if _job_skips_on_missing_secret(job) and name not in ADVISORY_GATES
+    ]
+
+
+def check_declared_gates_still_exist(workflow_docs: dict[str, dict]) -> list[str]:
+    """The reverse drift: a declaration outliving the job it describes.
+
+    A stale entry is worse than none — it asserts a caveat about a gate that no
+    longer exists, and hides the day a real gate quietly becomes advisory.
+    """
+    live = {name for _, name, _ in _iter_jobs(workflow_docs)}
+    return [
+        f"ADVISORY_GATES declares {name!r}, but no workflow job has that name "
+        "— stale declaration, or the job was renamed"
+        for name in sorted(ADVISORY_GATES)
+        if name not in live
+    ]
+
+
 # ---------------------------------------------------------------------------
 # Repo-level wiring
 # ---------------------------------------------------------------------------
@@ -164,8 +277,17 @@ def read_testpaths(root: Path) -> list[str]:
     return []
 
 
-def collect_counts(root: Path) -> dict[str, int]:
-    """Ask pytest what it actually collects, per file."""
+@functools.lru_cache(maxsize=None)
+def _collect_counts_cached(root: str) -> tuple[tuple[str, int], ...]:
+    """One `pytest --collect-only` per process, not per caller.
+
+    The CLI calls `run_all` once and never noticed. The test suite calls it six
+    times, and each call forked a full collection of the whole suite — measured
+    1.11s each, ~7s of a 20s `npm test`. Worse, it grew with the suite: every
+    test added made those six slower, so the cost compounded exactly as the
+    project got more tests. Cached on the resolved root; the collection cannot
+    change within a process.
+    """
     proc = subprocess.run(
         [sys.executable, "-m", "pytest", "--collect-only", "-q"],
         cwd=root, capture_output=True, text=True,
@@ -175,10 +297,46 @@ def collect_counts(root: Path) -> dict[str, int]:
         match = re.match(r"^([\w./-]+\.py)::", line.strip())
         if match:
             counts[match.group(1)] = counts.get(match.group(1), 0) + 1
-    return counts
+    return tuple(sorted(counts.items()))
+
+
+def collect_counts(root: Path) -> dict[str, int]:
+    """Ask pytest what it actually collects, per file."""
+    return dict(_collect_counts_cached(str(Path(root).resolve())))
+
+
+def read_workflows(root: Path) -> dict[str, dict]:
+    wf_dir = root / ".github" / "workflows"
+    if not wf_dir.is_dir():
+        return {}
+    return {
+        str(p.relative_to(root)): yaml.safe_load(p.read_text(encoding="utf-8")) or {}
+        for p in sorted(wf_dir.glob("*.yml"))
+    }
+
 
+def _declares_a_skip(report: Path) -> bool:
+    """Whether the report says outright that it graded nothing on purpose."""
+    data = json.loads(report.read_text(encoding="utf-8"))
+    return isinstance(data, dict) and bool(data.get("skipped"))
 
-def run_all(root: Path) -> list[str]:
+
+def load_fidelity_suites(report: Path) -> list[dict]:
+    """Read a `test-fidelity.py --json` report into a suite list.
+
+    A declared skip (`{"skipped": true, "reason": "no_api_key"}`) is not a
+    suite — it is the advisory path, already accounted for by ADVISORY_GATES.
+    Anything else claiming to be a run gets checked for actual verdicts.
+    """
+    data = json.loads(report.read_text(encoding="utf-8"))
+    if isinstance(data, dict):
+        if data.get("skipped"):
+            return []
+        data = [data]
+    return [s for s in data if isinstance(s, dict) and not s.get("skipped")]
+
+
+def run_all(root: Path, fidelity_report: Path | None = None) -> list[str]:
     problems: list[str] = []
 
     test_files = discover_test_files(root)
@@ -194,6 +352,32 @@ def run_all(root: Path) -> list[str]:
         problems += check_catalog_matches_filesystem(catalog, root / "prebuilt", root)
 
     problems += check_every_skill_has_fixtures(root / "prebuilt")
+
+    workflows = read_workflows(root)
+    problems += check_advisory_gates_declared(workflows)
+    problems += check_declared_gates_still_exist(workflows)
+
+    # check_graded_suites_graded_something shipped fully written and unit-tested
+    # but unreferenced by run_all — the anti-fake-green script had a check that
+    # itself never ran. This is where it runs.
+    if fidelity_report is not None:
+        # A missing file is a problem, not a pass. `… and fidelity_report.exists()`
+        # meant `--fidelity-report /nonexistent.json` printed "every gate examined
+        # a non-empty set" about a report it never opened — the exact statement
+        # this script exists to make impossible.
+        if not fidelity_report.exists():
+            problems.append(
+                f"{fidelity_report} was named as the fidelity report but does "
+                "not exist — nothing was examined"
+            )
+        else:
+            suites = load_fidelity_suites(fidelity_report)
+            if not suites and not _declares_a_skip(fidelity_report):
+                problems.append(
+                    f"{fidelity_report} contains no suites and does not declare "
+                    "a skip — it grades nothing but reads as clean"
+                )
+            problems += check_graded_suites_graded_something(suites)
     return problems
 
 
@@ -201,12 +385,21 @@ def main() -> int:
     parser = argparse.ArgumentParser(description=__doc__)
     parser.add_argument("--root", type=Path, default=Path(__file__).resolve().parent.parent)
     parser.add_argument("--json", action="store_true", help="machine-readable output")
+    parser.add_argument(
+        "--fidelity-report",
+        type=Path,
+        default=None,
+        help="a test-fidelity.py --json report; assert it produced real verdicts",
+    )
     args = parser.parse_args()
 
-    problems = run_all(args.root)
+    problems = run_all(args.root, args.fidelity_report)
 
     if args.json:
-        print(json.dumps({"problems": problems, "ok": not problems}, ensure_ascii=False, indent=2))
+        print(json.dumps(
+            {"problems": problems, "ok": not problems, "advisory_gates": ADVISORY_GATES},
+            ensure_ascii=False, indent=2,
+        ))
     elif problems:
         print(f"✗ {len(problems)} gate-liveness problem(s):\n")
         for p in problems:
@@ -214,6 +407,12 @@ def main() -> int:
         print("\nA gate that examines nothing reports the same green as one that passes.")
     else:
         print("✓ gate liveness ok — every gate examined a non-empty set")
+        # Printed on success, not just failure: the roster is the answer to
+        # "what did that green tick actually examine?", and it is only useful
+        # if you see it without going looking.
+        print(f"\n⚠ {len(ADVISORY_GATES)} advisory gate(s) — green does NOT mean these ran:")
+        for name, caveat in sorted(ADVISORY_GATES.items()):
+            print(f"  - {name}\n      {caveat}")
 
     return 1 if problems else 0
 

package/scripts/reaudit-report.py

@@ -0,0 +1,163 @@
+#!/usr/bin/env python3
+"""Re-run the citation audit over a committed run's stored answers. Offline, free.
+
+The fabrication audit is a mechanical string-resolution test — does a citation
+resolve to a declared source — so every change to `verify_citations.py` changes
+what an already-paid-for run would have seen. Coverage is otherwise frozen at run
+time: `eval/reports/0.11.0-06b8142-deepseek.json` records `master-ajahn-chah` at
+0 of 48 citations readable, and that number describes the auditor of 2026-08-31,
+not the persona.
+
+Since PR #142 every answer is stored, so the audit can simply be run again.
+The ¥3.89 sweep is re-measurable for nothing, and a family added to the auditor
+has to show what it bought instead of asserting it.
+
+    python3 scripts/reaudit-report.py eval/reports/0.11.0-06b8142-deepseek.json
+
+The report file itself is never rewritten: it is the record of what that run
+measured with that instrument, and editing it would be rewriting the experiment.
+"""
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from verify_citations import (  # noqa: E402
+    audit_answer,
+    load_declared_ids,
+    load_member_aliases,
+    load_title_aliases,
+)
+
+
+def reaudit(report: dict) -> dict:
+    """Re-audit every stored answer. Raises if the run stored none."""
+    if not any(
+        "response" in result
+        for suite in report["suites"]
+        for result in suite["results"]
+    ):
+        raise ValueError(
+            "no stored answers in this report — nothing to re-audit. Runs before "
+            "PR #142 kept only response_length."
+        )
+
+    suites = []
+    totals = {
+        "recorded": {"checked": 0, "unparsed": 0},
+        "recomputed": {"checked": 0, "unparsed": 0},
+    }
+    for suite in report["suites"]:
+        recorded_block = suite.get("audit") or {}
+        recorded = {
+            "checked": recorded_block.get("citations_checked", 0),
+            "unparsed": recorded_block.get("citations_unparsed", 0),
+        }
+        totals["recorded"]["checked"] += recorded["checked"]
+        totals["recorded"]["unparsed"] += recorded["unparsed"]
+
+        try:
+            declared = load_declared_ids(suite["master"])
+            aliases = load_member_aliases(suite["master"])
+            titles = load_title_aliases(suite["master"])
+        except (FileNotFoundError, ValueError):
+            # No meta.json, so no declared set to audit against. Recorded as
+            # unavailable rather than as a clean zero — the distinction this
+            # repo lost once already.
+            suites.append(
+                {
+                    "master": suite["master"],
+                    "status": "unavailable",
+                    "recorded": recorded,
+                    "recomputed": None,
+                    "fabricated": [],
+                }
+            )
+            totals["recomputed"]["checked"] += recorded["checked"]
+            totals["recomputed"]["unparsed"] += recorded["unparsed"]
+            continue
+
+        checked = unparsed = 0
+        fabricated: list[str] = []
+        noncitation: list[str] = []
+        for result in suite["results"]:
+            if result.get("status") in ("truncated", "api_error"):
+                continue
+            audit = audit_answer(
+                declared, result.get("response") or "", aliases, titles
+            )
+            checked += (
+                len(audit["offline"]) + len(audit["live"]) + len(audit["fabricated"])
+            )
+            unparsed += len(audit["unparsed"])
+            fabricated.extend(audit["fabricated"])
+            noncitation.extend(audit.get("noncitation", ()))
+        recomputed = {"checked": checked, "unparsed": unparsed}
+        totals["recomputed"]["checked"] += checked
+        totals["recomputed"]["unparsed"] += unparsed
+        suites.append(
+            {
+                "master": suite["master"],
+                "status": "audited",
+                "recorded": recorded,
+                "recomputed": recomputed,
+                "fabricated": sorted(set(fabricated)),
+                # 判定为「不是引文」的【…】块。不计入覆盖率的分母,但必须
+                # 数出来:排除而不申报,和静默跳过没有区别。
+                "noncitation": sorted(set(noncitation)),
+            }
+        )
+    return {"meta": report.get("meta", {}), "suites": suites, "totals": totals}
+
+
+def _coverage(counts: dict | None) -> str:
+    if not counts:
+        return "n/a"
+    total = counts["checked"] + counts["unparsed"]
+    if not total:
+        return "n/a"
+    return f"{counts['checked']}/{total} {counts['checked'] / total:.0%}"
+
+
+def main(argv: list[str]) -> int:
+    if len(argv) != 2:
+        print(__doc__.strip().splitlines()[0])
+        print(f"usage: {Path(argv[0]).name} <eval/reports/*.json>")
+        return 2
+    path = Path(argv[1])
+    report = json.loads(path.read_text())
+    out = reaudit(report)
+
+    meta = out["meta"]
+    print(f"{path.name} — commit {meta.get('commit_short')}, model {meta.get('model')}")
+    print(f"\n{'skill':26} {'as recorded':>13}  {'re-audited':>13}   fabricated")
+    for suite in out["suites"]:
+        if suite["status"] == "unavailable":
+            print(f"{suite['master']:26} {_coverage(suite['recorded']):>13}  "
+                  f"{'(no meta.json)':>13}")
+            continue
+        moved = "  <--" if suite["recorded"] != suite["recomputed"] else ""
+        print(
+            f"{suite['master']:26} {_coverage(suite['recorded']):>13}  "
+            f"{_coverage(suite['recomputed']):>13}   "
+            f"{', '.join(suite['fabricated'])}{moved}"
+        )
+    print(
+        f"\ntotal coverage  {_coverage(out['totals']['recorded'])}"
+        f"  ->  {_coverage(out['totals']['recomputed'])}"
+    )
+    skipped = sorted({b for s in out["suites"] for b in s.get("noncitation", ())})
+    if skipped:
+        print(
+            f"\n{len(skipped)} 个【…】块判定为非引文，不计入上面的分母"
+            "（人格拿它当小标题或复述问题）："
+        )
+        for block in skipped:
+            print(f"    【{block}】")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv))

package/scripts/regrade-report.py

@@ -0,0 +1,157 @@
+#!/usr/bin/env python3
+"""Re-grade a committed run against the current judge and fixtures. Offline, free.
+
+`check_response` is deterministic, and every answer has been stored since PR
+#142. So any change to the judge or to the fixtures can be measured against an
+already-paid-for run for nothing, instead of claimed.
+
+That matters most for changes that *relax* something. Moving a requirement into
+`must_convey` — "the substring matcher cannot decide this" — is honest when the
+adjudication says so and is laundering when it does not. Re-grading the ¥3.89
+sweep shows exactly which cases move and whether anything moved the wrong way.
+
+    python3 scripts/regrade-report.py eval/reports/0.11.0-06b8142-deepseek.json
+
+Results are joined to fixtures **by question text**, never by position: a
+fixture added or removed shifts every index after it, and grading an answer
+against someone else's question would produce a confident, meaningless number.
+"""
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+
+import importlib.util
+
+ROOT = Path(__file__).resolve().parent.parent
+_spec = importlib.util.spec_from_file_location(
+    "_fidelity", ROOT / "scripts" / "test-fidelity.py"
+)
+_fidelity = importlib.util.module_from_spec(_spec)
+sys.modules["_fidelity"] = _fidelity
+_spec.loader.exec_module(_fidelity)
+
+from verify_citations import load_declared_ids, load_member_aliases  # noqa: E402
+
+
+def load_fixtures() -> dict[str, list[dict]]:
+    fixtures: dict[str, list[dict]] = {}
+    for path in sorted((ROOT / "prebuilt").glob("*/tests/fidelity.jsonl")):
+        fixtures[path.parent.parent.name] = [
+            json.loads(line) for line in path.read_text().splitlines() if line.strip()
+        ]
+    return fixtures
+
+
+def regrade(report: dict, fixtures: dict[str, list[dict]]) -> dict:
+    """Re-run the judge over every stored answer. Raises if the run stored none."""
+    if not any(
+        "response" in result
+        for suite in report["suites"]
+        for result in suite["results"]
+    ):
+        raise ValueError("no stored answers in this report — nothing to re-grade")
+
+    cases: list[dict] = []
+    graded_results: list[dict] = []
+    by_test_type: dict[str, dict[str, int]] = {}
+
+    for suite in report["suites"]:
+        master = suite["master"]
+        cases_for_master = fixtures.get(master, [])
+        try:
+            declared = load_declared_ids(master) or None
+            aliases = load_member_aliases(master) or None
+        except (FileNotFoundError, ValueError):
+            declared = None
+            aliases = None
+
+        for result in suite["results"]:
+            if result.get("status") in ("truncated", "api_error"):
+                continue
+            index = result["index"]
+            fixture = cases_for_master[index] if index < len(cases_for_master) else None
+            if fixture is None or fixture.get("q") != result["question"]:
+                raise ValueError(
+                    f"{master} #{index}: the fixture at this position does not match "
+                    f"the question that was graded. Fixtures moved; re-grading by "
+                    f"position would compare an answer with someone else's question."
+                )
+            check = _fidelity.check_response(
+                result.get("response") or "",
+                fixture,
+                declared_ids=declared,
+                member_aliases=aliases,
+            )
+            entry = _fidelity.result_entry(index, fixture, check, result.get("response") or "")
+            graded_results.append(entry)
+            was = result["status"]
+            now = "PASS" if check["passed"] else "FAIL"
+            bucket = by_test_type.setdefault(
+                result["test_type"], {"graded": 0, "was": 0, "now": 0}
+            )
+            bucket["graded"] += 1
+            bucket["was"] += was == "PASS"
+            bucket["now"] += now == "PASS"
+            cases.append(
+                {
+                    "master": master,
+                    "index": index,
+                    "test_type": result["test_type"],
+                    "was": was,
+                    "now": now,
+                    "needs_review": check["needs_review"],
+                }
+            )
+
+    return {
+        "meta": report.get("meta", {}),
+        "cases": cases,
+        "by_test_type": by_test_type,
+        "mentions": _fidelity.summarize_mentions(graded_results),
+        "needs_review": sum(1 for c in cases if c["needs_review"]),
+    }
+
+
+def main(argv: list[str]) -> int:
+    if len(argv) != 2:
+        print(f"usage: {Path(argv[0]).name} <eval/reports/*.json>")
+        return 2
+    path = Path(argv[1])
+    out = regrade(json.loads(path.read_text()), load_fixtures())
+    meta = out["meta"]
+    print(f"{path.name} — commit {meta.get('commit_short')}, model {meta.get('model')}")
+    print(f"\n{'test_type':12} {'graded':>7} {'as graded':>11} {'re-graded':>11}")
+    total = {"graded": 0, "was": 0, "now": 0}
+    for test_type, b in sorted(out["by_test_type"].items()):
+        for key in total:
+            total[key] += b[key]
+        print(f"{test_type:12} {b['graded']:>7} "
+              f"{b['was']:>6} {b['was']/b['graded']:>4.0%} "
+              f"{b['now']:>6} {b['now']/b['graded']:>4.0%}")
+    print(f"{'TOTAL':12} {total['graded']:>7} "
+          f"{total['was']:>6} {total['was']/total['graded']:>4.0%} "
+          f"{total['now']:>6} {total['now']/total['graded']:>4.0%}")
+
+    m = out["mentions"]
+    print(
+        f"\nmention coverage {m['mentions_decided']}/{m['mention_requirements']} "
+        f"= {m['mention_coverage']}  "
+        f"({m['mentions_unverified']} undecidable, "
+        f"{m['script_mismatches']} answers in the wrong script)"
+    )
+    print(f"cases needing adjudication: {out['needs_review']}")
+
+    moved_wrong = [c for c in out["cases"] if c["was"] == "PASS" and c["now"] == "FAIL"]
+    if moved_wrong:
+        print(f"\nWARNING: {len(moved_wrong)} case(s) that passed now fail:")
+        for c in moved_wrong:
+            print(f"  {c['master']} #{c['index']} ({c['test_type']})")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main(sys.argv))