master-skill 0.11.0 → 0.12.0

package/scripts/validate-citation-references.py

@@ -0,0 +1,150 @@
+#!/usr/bin/env python3
+"""Gate: a persona must not instruct a citation its own contract forbids.
+
+`validate-citation-contract.py` checks that `meta.json` declares the right
+*fields*. Nothing checked whether the persona's own SKILL.md, sources/ and
+references/ tell it to cite sources that `meta.json` never declares — and that
+is a shipped defect, not a hypothetical: `master-tsongkhapa`'s SKILL.md gives
+
+    印度大乘论典所引：`【月称《入中论》§第六章】（Toh 3861）`
+
+as the prescribed format while `meta.json` declares five sources, none of them
+`Toh:3861`. Every use of that instruction violates the B1 citation rule.
+
+That was found by a ¥3.89 graded run over 211 fixtures, which caught it only
+because one fixture happened to trigger it. This finds every instance of the
+class deterministically, for free, on every PR.
+"""
+from __future__ import annotations
+
+import json
+import re
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from verify_citations import audit_answer, load_declared_ids, load_member_aliases  # noqa: E402
+
+PREBUILT_DIR = Path(__file__).resolve().parent.parent / "prebuilt"
+
+_BLOCK = re.compile(r"【([^】]*)】")
+# Same attribution region verify_citations uses, so a tag documented after the
+# block (（BDRC: Wxxxxx）) is judged together with the block it belongs to.
+_LINK_WINDOW = 120
+
+# Documentation showing the citation *format* is not a citation. Every marker
+# here appears in shipped persona docs; none can occur in a real reference.
+_TEMPLATE_MARKERS = (
+    re.compile(r"\{"),            # 【《{title}》，{source_id}】
+    re.compile(r"[A-Za-z][xX]{3,}"),  # （BDRC: Wxxxxx） / Txxnxxxx
+    re.compile(r"卷N"),           # 【《法華玄義》卷N，T1716】
+    re.compile(r"典籍名|章节名"),   # 【《典籍名》§章节】
+)
+
+# Open findings, each awaiting a maintainer decision. This is a ratchet, not an
+# allowlist: an entry here is a defect that has been *seen*, not one that has
+# been permitted. Do not add to it to turn a red build green — that is exactly
+# the failure this gate exists to prevent.
+#
+# Empty as of 2026-09-03. Both findings this gate ever recorded were resolved
+# by declaring the source: `Toh:3861` in master-tsongkhapa/meta.json (月称《入
+# 中论》is a real Tengyur text Tsongkhapa's tradition treats as its own
+# foundation) and `J36nB348` in master-ouyi/meta.json (《灵峰宗论》is Ouyi's own
+# collected works). Neither needed a B1 contract change — both simply belonged
+# in the declared set. See CHANGELOG.md for the maintainer decision.
+KNOWN_UNDECLARED: dict[tuple[str, str], str] = {}
+
+
+def is_template_block(text: str) -> bool:
+    """Is this citation documenting the format rather than citing a source?"""
+    return any(marker.search(text) for marker in _TEMPLATE_MARKERS)
+
+
+def _strip_template_citations(text: str) -> str:
+    """Drop format-documentation citations, with their attribution regions."""
+    blocks = list(_BLOCK.finditer(text))
+    out: list[str] = []
+    prev = 0
+    for index, match in enumerate(blocks):
+        next_start = blocks[index + 1].start() if index + 1 < len(blocks) else len(text)
+        region_end = min(next_start, match.end() + _LINK_WINDOW)
+        context = match.group(1) + text[match.end():region_end]
+        if is_template_block(context):
+            out.append(text[prev:match.start()])
+            prev = region_end
+    out.append(text[prev:])
+    return "".join(out)
+
+
+@dataclass(frozen=True)
+class Finding:
+    master: str
+    citation: str
+    path: str
+
+
+def find_undeclared(prebuilt_dir: Path) -> list[Finding]:
+    """Every citation the personas' own material makes that meta.json omits."""
+    findings: list[Finding] = []
+    for persona in sorted(Path(prebuilt_dir).iterdir()):
+        meta_path = persona / "meta.json"
+        if not persona.is_dir() or not meta_path.is_file():
+            continue
+        # Found by an independent code-review pass (2026-09-03): this used to
+        # reimplement its own meta.json parsing instead of calling
+        # verify_citations.py's own loader — a real drift risk, since a future
+        # change to how sources/notes are parsed there would silently stop
+        # applying here. `persona.name` round-trips through resolve_master_dir
+        # (it already accepts the full `master-<slug>` form).
+        try:
+            declared = load_declared_ids(persona.name, base=str(prebuilt_dir))
+            aliases = load_member_aliases(persona.name, base=str(prebuilt_dir))
+        except (FileNotFoundError, ValueError):
+            continue  # meta.json exists (meta_path.is_file() above) but is unreadable
+        if not declared:
+            continue  # nothing to audit against; see master-debate
+        docs = [persona / "SKILL.md"]
+        docs += sorted(persona.glob("sources/*.md"))
+        docs += sorted(persona.glob("references/*.md"))
+        for doc in docs:
+            if not doc.is_file():
+                continue
+            text = _strip_template_citations(doc.read_text(encoding="utf-8"))
+            for citation in dict.fromkeys(audit_answer(declared, text, aliases)["fabricated"]):
+                findings.append(
+                    Finding(persona.name, citation, str(doc.relative_to(prebuilt_dir.parent)))
+                )
+    return findings
+
+
+def main() -> int:
+    findings = find_undeclared(PREBUILT_DIR)
+    known, new = [], []
+    for f in findings:
+        (known if (f.master, f.citation) in KNOWN_UNDECLARED else new).append(f)
+
+    if known:
+        print(f"Known undeclared citations ({len(known)}) — open findings, not permissions:")
+        for f in known:
+            print(f"  {f.master}: {f.citation}  ({f.path})")
+        for key in dict.fromkeys((f.master, f.citation) for f in known):
+            print(f"\n  {key[0]} / {key[1]}:\n    {KNOWN_UNDECLARED[key]}")
+
+    if new:
+        print(f"\nFAIL: {len(new)} citation(s) not declared in the persona's meta.json:")
+        for f in new:
+            print(f"  {f.master}: {f.citation}  ({f.path})")
+        print(
+            "\nThe persona's own material instructs a citation its citation contract "
+            "forbids.\nDeclare the source in meta.json, or stop citing it. Adding it to "
+            "KNOWN_UNDECLARED\nwithout a maintainer decision defeats the point of this gate."
+        )
+        return 1
+
+    print(f"\nOK: no new undeclared citations across {len(list(PREBUILT_DIR.iterdir()))} skills.")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/scripts/validate-citation-templates.py

@@ -0,0 +1,176 @@
+#!/usr/bin/env python3
+"""Gate: a skill must not demand verifiable citations and then show a template
+that cannot carry one.
+
+`compare-masters/SKILL.md` did exactly that. Line 223 required every doctrinal
+claim to "解析到所选 persona 的 meta.json.sources[]" and line 259 required a
+"可核验的来源引用" — while the output template two hundred lines earlier read
+
+    > 出处：【《经名》卷N】→ fojin.app 链接
+
+with no identifier in it. The model followed the template, and the 2026-09-12
+sweep measured the result: 18 answers, 49 citations, **zero checkable**, audit
+coverage 0%. The rule and the example were in the same file and contradicted
+each other; nothing was comparing them.
+
+The check is deliberately narrow. It does NOT require every citation format to
+carry an identifier — `master-buddhaghosa`, `master-mahasi-sayadaw` and
+`master-tsongkhapa` declare formats without one because PTS / SuttaCentral /
+BDRC are corpus-level references with no per-passage id, which the citation
+contract documents as a known boundary. Requiring an id there would force a
+fabricated one, which is worse than an unverifiable citation.
+
+What it requires is consistency: a skill that quotes *other* personas must
+defer to their declared `citation_format` rather than inventing a weaker shape
+of its own. Deferring is also what makes the boundary above work — quoting
+buddhaghosa inherits buddhaghosa's format, id-less and honest.
+
+The second rule was added after the first one turned out to examine 1 of the 3
+meta-skills and return early on the other two. `master-debate` requires "至少 1
+个本宗 citation" four times over and never says what one looks like; the same
+sweep shows it wrote 24 real sutra ids as `《坛经》（T48n2008）`. The auditor's
+`_CITATION_BLOCK` parses `【…】` only, so all 24 were invisible and the report
+read `audit_coverage: N/A` — the same cell `master-help`, which legitimately
+never cites, prints. Requiring citations and not naming an auditable shape
+produces citations nobody checks, which reads as clean.
+
+Usage:
+    python3 scripts/validate-citation-templates.py
+"""
+
+from __future__ import annotations
+
+import re
+import sys
+from pathlib import Path
+
+PREBUILT = Path(__file__).resolve().parent.parent / "prebuilt"
+
+# A literal citation example: 【…】 with no {placeholder} inside, i.e. a shape
+# the writer intended to be copied rather than filled from a declared format.
+_LITERAL_CITATION = re.compile(r"【《[^》{]*》[^】{]*】")
+
+# Phrases by which a file asserts its citations are checkable. Kept to the ones
+# this repo actually uses, so the check cannot drift into policing prose.
+_VERIFIABILITY_CLAIMS = ("可核验", "必须解析到", "可核对")
+
+# Skills that quote other personas rather than speaking as one.
+_META_SKILLS = ("compare-masters", "master-debate", "master-curriculum")
+
+# The only shape `verify_citations._CITATION_BLOCK` parses. A citation written
+# any other way is served to the reader and audited by nothing.
+_AUDITABLE_BLOCK = re.compile(r"【[^】]*】")
+
+# A file demanding citations of its output. Narrow on purpose: `master-help`
+# mentions the word without requiring any, and must not be caught.
+_REQUIRES_CITATION = (
+    re.compile(r"至少\s*\d+\s*(?:个|条)[^。\n]{0,12}citation"),
+    re.compile(r"引用必须"),
+    re.compile(r"必须附[^。\n]{0,12}引用"),
+    re.compile(r"引经必经查证"),
+)
+
+
+def _strip_html_comments(text: str) -> str:
+    """Drop <!-- … --> before scanning.
+
+    The first run of this check flagged the comment that explains what the bad
+    template used to look like — reading the note about the defect as the
+    defect. That is the same "the test matched the docstring" shape this repo
+    keeps producing, one level up. A comment is not an instruction to the
+    model: SKILL.md is rendered as markdown, and an HTML comment does not
+    appear in what the model is shown.
+
+    The newlines are kept, so a reported line number still points at the line a
+    maintainer will open. Deleting them outright shifts every later line and
+    sends the reader to the wrong place — a small lie in the same family.
+    """
+    return re.sub(
+        r"<!--.*?-->",
+        lambda m: "\n" * m.group().count("\n"),
+        text,
+        flags=re.S,
+    )
+
+
+def check_skill(path: Path) -> tuple[list[str], list[str]]:
+    """Return (problems, rules that actually examined this file).
+
+    The second element exists so `main` can print what was looked at. The
+    first version of this check returned early on two of the three meta-skills
+    and printed a clean line anyway; a gate that says nothing about its own
+    reach is how this repo has repeatedly shipped an inert one.
+    """
+    text = _strip_html_comments(path.read_text(encoding="utf-8"))
+    name = path.parent.name
+    problems: list[str] = []
+    examined: list[str] = []
+
+    declares_own_format = re.search(r"^citation_format:", text, re.M) is not None
+
+    # Rule 1 — a verifiability claim must not sit next to an id-less template.
+    claims = [c for c in _VERIFIABILITY_CLAIMS if c in text]
+    if claims:
+        examined.append("claims-vs-template")
+        for match in _LITERAL_CITATION.finditer(text):
+            # A persona with its own declared format may show an example of it.
+            if declares_own_format and name not in _META_SKILLS:
+                continue
+            line = text[: match.start()].count("\n") + 1
+            problems.append(
+                f"{name}/SKILL.md:{line}: the file claims citations are "
+                f"{claims[0]} but shows the literal template {match.group()!r}, "
+                "which carries no identifier. A skill that quotes personas must "
+                "defer to their declared `citation_format`."
+            )
+
+    # Rule 2 — requiring citations without naming an auditable shape produces
+    # citations the auditor cannot see, which reports as `N/A`, not as a miss.
+    demand = next((p for p in _REQUIRES_CITATION if p.search(text)), None)
+    if demand is not None:
+        examined.append("demand-vs-auditable-shape")
+        if not _AUDITABLE_BLOCK.search(text):
+            line = text[: demand.search(text).start()].count("\n") + 1
+            problems.append(
+                f"{name}/SKILL.md:{line}: the file requires citations "
+                f"(matched {demand.pattern!r}) but never shows the 【…】 form. "
+                "`verify_citations._CITATION_BLOCK` parses that shape and no "
+                "other, so whatever the model writes instead is served "
+                "unaudited and reports as `audit_coverage: N/A`."
+            )
+
+    return problems, examined
+
+
+def main() -> int:
+    skills = sorted(p for p in PREBUILT.glob("*/SKILL.md"))
+    if not skills:
+        print("no SKILL.md found under prebuilt/ — this check would pass vacuously")
+        return 1
+
+    problems: list[str] = []
+    reach: dict[str, list[str]] = {}
+    for path in skills:
+        found, examined = check_skill(path)
+        problems += found
+        for rule in examined:
+            reach.setdefault(rule, []).append(path.parent.name)
+
+    if problems:
+        print(f"✗ {len(problems)} citation-template problem(s):\n")
+        for problem in problems:
+            print(f"  - {problem}")
+        return 1
+
+    print(f"✓ citation templates ok — {len(skills)} skills scanned")
+    for rule in ("claims-vs-template", "demand-vs-auditable-shape"):
+        names = reach.get(rule, [])
+        if not names:
+            print(f"  ✗ {rule}: examined 0 skills — this rule decided nothing")
+            return 1
+        print(f"  {rule}: {len(names)} skill(s) — {', '.join(sorted(names))}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/scripts/validate-fixture-terms.py

@@ -0,0 +1,127 @@
+#!/usr/bin/env python3
+"""Gate: a requirement may only be declared undecidable with evidence behind it.
+
+`must_convey` is how a fixture says "the substring matcher cannot decide this" —
+the honest answer for 方便 when the answer says 「应病与药」, or for 不是虚无 when
+it says 「空非虚无」. It is also the easiest possible way to launder a failure:
+move the inconvenient requirement there and the build goes green while looking
+more rigorous than before.
+
+So a term may sit in `must_convey` only if a committed adjudication under
+`eval/reports/` ruled it an instrument artifact, on a quote that
+`verify-adjudication.py` proved is still in the answer it judges. A term the
+adjudication ruled `upheld` — a real failure — can never be moved there.
+
+Same ratchet as `KNOWN_UNDECLARED` in validate-citation-references.py: an entry
+records a decision that was made and shown, not a permission anyone can take.
+"""
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent.parent
+PREBUILT = ROOT / "prebuilt"
+REPORTS = ROOT / "eval" / "reports"
+
+# Verdicts that license moving a requirement out of hard grading.
+PERMITTING = {"instrument", "fixture"}
+
+
+def load_fixtures() -> dict[str, list[dict]]:
+    fixtures: dict[str, list[dict]] = {}
+    for path in sorted(PREBUILT.glob("*/tests/fidelity.jsonl")):
+        master = path.parent.parent.name
+        fixtures[master] = [
+            json.loads(line) for line in path.read_text().splitlines() if line.strip()
+        ]
+    return fixtures
+
+
+def load_adjudications() -> list[dict]:
+    return [json.loads(p.read_text()) for p in sorted(REPORTS.glob("adjudication-*.json"))]
+
+
+def _permitted(adjudications: list[dict]) -> set[tuple[str, int, str]]:
+    """(master, index, term) triples an adjudication ruled an instrument artifact.
+
+    Keyed by fixture index, not just (master, term) — a verdict on one fixture
+    must not license the same term string at a different, unadjudicated
+    fixture for the same master. `缘起` being paraphrased in one answer says
+    nothing about whether it is genuinely omitted in another; only the
+    specific case a verdict names has evidence behind it.
+    """
+    allowed: set[tuple[str, int, str]] = set()
+    for adjudication in adjudications:
+        for case in adjudication.get("cases", []):
+            for verdict in case.get("mention_verdicts", []):
+                if verdict.get("verdict") in PERMITTING:
+                    allowed.add((case["master"], case["index"], verdict["term"]))
+    return allowed
+
+
+def verify(fixtures: dict[str, list[dict]], adjudications: list[dict]) -> list[str]:
+    """Return every reason a fixture declares more than the evidence supports."""
+    problems: list[str] = []
+    allowed = _permitted(adjudications)
+
+    declares_any = any(
+        case.get("must_convey") for cases in fixtures.values() for case in cases
+    )
+    if declares_any and not adjudications:
+        problems.append(
+            "no adjudication under eval/reports/ — must_convey entries exist but "
+            "there is nothing to check them against"
+        )
+        return problems
+
+    for master, cases in fixtures.items():
+        for index, case in enumerate(cases):
+            convey = case.get("must_convey") or []
+            mention = set(case.get("must_mention") or [])
+            for term in convey:
+                where = f"{master} #{index}"
+                if term in mention:
+                    problems.append(
+                        f"{where}: {term!r} is in both must_mention and must_convey — "
+                        "a requirement is either graded or undecidable, not both"
+                    )
+                if (master, index, term) not in allowed:
+                    problems.append(
+                        f"{where}: {term!r} is declared undecidable but no adjudication "
+                        f"ruled it an instrument artifact for this specific fixture. A "
+                        "verdict on a different case does not transfer. Grade it, or "
+                        "adjudicate this exact case and show the evidence."
+                    )
+    return problems
+
+
+def main() -> int:
+    fixtures = load_fixtures()
+    adjudications = load_adjudications()
+    problems = verify(fixtures, adjudications)
+    convey = sum(
+        len(case.get("must_convey") or [])
+        for cases in fixtures.values()
+        for case in cases
+    )
+    graded = sum(
+        len(case.get("must_mention") or [])
+        for cases in fixtures.values()
+        for case in cases
+    )
+    if problems:
+        print(f"FAIL: {len(problems)} fixture term(s) declare more than the evidence supports:")
+        for problem in problems:
+            print(f"  {problem}")
+        return 1
+    print(
+        f"OK: {graded} graded mention requirements, {convey} declared undecidable, "
+        f"every one of the latter backed by an adjudicated verdict."
+    )
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())