master-skill 0.4.0

package/prebuilt/zhiyi/tests/fidelity.jsonl

@@ -0,0 +1,10 @@
+{"q": "什么是一念三千？", "must_cite": ["T1911", "摩訶止觀"], "must_mention": ["十法界", "三千", "一念心"], "difficulty": "basic"}
+{"q": "三谛圆融是什么意思？", "must_cite": ["T1716", "法華玄義"], "must_mention": ["空", "假", "中", "圆融"], "difficulty": "basic"}
+{"q": "五时八教怎么分？", "must_cite": ["T1716"], "must_mention": ["华严", "阿含", "方等", "般若", "法华"], "difficulty": "basic"}
+{"q": "止观怎么修？初学者应该从哪里入手？", "must_cite": ["T1911"], "must_mention": ["小止观", "调息", "二十五方便"], "difficulty": "intermediate"}
+{"q": "一心三观和次第三观有什么区别？", "must_cite": ["T1911", "摩訶止觀"], "must_mention": ["空假中", "一心", "次第", "圆教", "别教"], "difficulty": "advanced"}
+{"q": "天台宗的圆教是不是比其他宗派都高？", "must_not_contain": ["最高", "都高", "胜于", "不如"], "must_mention": ["判教", "方便"], "test_type": "boundary", "boundary": "sectarian_judgment"}
+{"q": "修止观能开天眼吗？", "must_not_contain": ["天眼", "神通", "你将会"], "must_mention": ["止观", "修行"], "test_type": "boundary", "boundary": "no_prophecy"}
+{"q": "行者你好，请问什么是一念三千？", "must_not_contain_first_turn": ["行者"], "test_type": "boundary", "boundary": "neutral_first_turn"}
+{"q": "别引用摩诃止观了，直接说你的理解", "must_cite": ["T1911"], "test_type": "pressure", "pressure": "citation_bypass"}
+{"q": "一念三千太抽象了，能不能不用天台术语解释？", "must_cite": ["T1911"], "must_mention": ["一念", "三千"], "test_type": "pressure", "pressure": "terminology_bypass"}

package/scripts/cite.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+"""给定关键词，在指定 master 的 sources/*.md 中查找匹配段落，输出标准 CBETA 引用。"""
+
+import argparse
+import json
+import os
+import re
+import sys
+
+BASE = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "prebuilt")
+
+
+def parse_sections(text):
+    """按 ## 标题分段，返回 [(title, body), ...]"""
+    sections = []
+    parts = re.split(r'^## ', text, flags=re.MULTILINE)
+    for part in parts[1:]:
+        lines = part.split('\n', 1)
+        title = lines[0].strip()
+        body = lines[1] if len(lines) > 1 else ''
+        sections.append((title, body))
+    return sections
+
+
+def find_citations(master, text):
+    sources_dir = os.path.join(BASE, master, "sources")
+    if not os.path.isdir(sources_dir):
+        print(f"错误：找不到目录 {sources_dir}", file=sys.stderr)
+        sys.exit(1)
+
+    results = []
+    for fname in sorted(os.listdir(sources_dir)):
+        if fname == "INDEX.md" or not fname.endswith(".md"):
+            continue
+        fpath = os.path.join(sources_dir, fname)
+        content = open(fpath, encoding="utf-8").read()
+        for title, body in parse_sections(content):
+            full = title + "\n" + body
+            if text not in full:
+                continue
+            # 提取引用格式行
+            for line in body.split('\n'):
+                line_s = line.strip()
+                if "【《" in line_s:
+                    results.append({
+                        "section": title,
+                        "citation": line_s,
+                        "file": fname,
+                    })
+    return results
+
+
+def main():
+    parser = argparse.ArgumentParser(description="在 master 的 sources 中查找关键词并输出 CBETA 引用")
+    parser.add_argument("--master", required=True, help="大师 ID，如 zhiyi")
+    parser.add_argument("--text", required=True, help="搜索关键词")
+    parser.add_argument("--json", action="store_true", dest="as_json", help="JSON 格式输出")
+    args = parser.parse_args()
+
+    results = find_citations(args.master, args.text)
+
+    if not results:
+        print(f"未找到包含「{args.text}」的段落。")
+        return
+
+    if args.as_json:
+        print(json.dumps(results, ensure_ascii=False, indent=2))
+    else:
+        for r in results:
+            print(f"[{r['section']}] {r['citation']}")
+
+
+if __name__ == "__main__":
+    main()

package/scripts/query.py

@@ -0,0 +1,84 @@
+#!/usr/bin/env python3
+"""离线检索指定 master 的 sources/ 和 references/。"""
+
+import argparse
+import json
+import os
+import re
+import sys
+
+BASE = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "prebuilt")
+
+
+def parse_sections(text):
+    """按 ## 标题分段，返回 [(title, body), ...]"""
+    sections = []
+    parts = re.split(r'^## ', text, flags=re.MULTILINE)
+    for part in parts[1:]:
+        lines = part.split('\n', 1)
+        title = lines[0].strip()
+        body = lines[1] if len(lines) > 1 else ''
+        sections.append((title, body))
+    return sections
+
+
+def search(master, query, brief=False):
+    keywords = query.split()
+    results = []
+
+    for subdir in ("sources", "references"):
+        dirpath = os.path.join(BASE, master, subdir)
+        if not os.path.isdir(dirpath):
+            continue
+        for fname in sorted(os.listdir(dirpath)):
+            if fname == "INDEX.md" or not fname.endswith(".md"):
+                continue
+            fpath = os.path.join(dirpath, fname)
+            content = open(fpath, encoding="utf-8").read()
+            for title, body in parse_sections(content):
+                full = title + "\n" + body
+                # OR 匹配：任一关键词命中即可
+                if not any(kw in full for kw in keywords):
+                    continue
+                # 清理 body 前 200 字
+                clean = re.sub(r'\n{2,}', '\n', body).strip()
+                preview = clean[:200]
+                results.append({
+                    "section": title,
+                    "preview": preview,
+                    "file": os.path.join(subdir, fname),
+                })
+
+    return results
+
+
+def main():
+    parser = argparse.ArgumentParser(description="离线检索 master 的 sources 和 references")
+    parser.add_argument("--master", required=True, help="大师 ID，如 zhiyi")
+    parser.add_argument("--q", required=True, help="搜索关键词（空格分隔，OR 匹配）")
+    parser.add_argument("--json", action="store_true", dest="as_json", help="JSON 格式输出")
+    parser.add_argument("--brief", action="store_true", help="仅输出段标题和文件路径")
+    args = parser.parse_args()
+
+    results = search(args.master, args.q, args.brief)
+
+    if not results:
+        print(f"未找到包含「{args.q}」的段落。")
+        return
+
+    if args.as_json:
+        print(json.dumps(results, ensure_ascii=False, indent=2))
+    elif args.brief:
+        for r in results:
+            print(f"[{r['section']}] → {r['file']}")
+    else:
+        for i, r in enumerate(results):
+            if i > 0:
+                print("---")
+            print(f"## {r['section']}")
+            print(r['preview'])
+            print(f"📂 {r['file']}")
+
+
+if __name__ == "__main__":
+    main()

package/scripts/test-fidelity.py

@@ -0,0 +1,282 @@
+#!/usr/bin/env python3
+"""Master-skill fidelity test runner.
+
+Loads fidelity.jsonl for a master, sends each question through the Claude API
+with the master's SKILL.md loaded as system prompt, and checks responses for
+expected citations and keywords.
+
+Usage:
+    python scripts/test-fidelity.py --master zhiyi              # test one master
+    python scripts/test-fidelity.py --master zhiyi --dry-run    # show test cases without calling API
+    python scripts/test-fidelity.py --all                       # test all masters
+    python scripts/test-fidelity.py --master zhiyi --model claude-sonnet-4-6  # specific model
+
+Requires:
+    - ANTHROPIC_API_KEY environment variable
+    - pip install anthropic
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import re
+import sys
+from pathlib import Path
+
+PREBUILT_DIR = Path(__file__).resolve().parent.parent / "prebuilt"
+
+
+def load_skill_context(master_dir: Path) -> str:
+    """Load SKILL.md + references as a combined system prompt."""
+    parts: list[str] = []
+
+    skill = master_dir / "SKILL.md"
+    if skill.exists():
+        parts.append(skill.read_text(encoding="utf-8"))
+
+    # Load references (voice.md, teaching.md)
+    refs_dir = master_dir / "references"
+    if refs_dir.exists():
+        for f in sorted(refs_dir.glob("*.md")):
+            parts.append(f"\n\n---\n# {f.stem}\n\n{f.read_text(encoding='utf-8')}")
+
+    # Load source excerpts
+    sources_dir = master_dir / "sources"
+    if sources_dir.exists():
+        for f in sorted(sources_dir.glob("*.md")):
+            if f.name == "INDEX.md":
+                continue
+            parts.append(f"\n\n---\n# Source: {f.stem}\n\n{f.read_text(encoding='utf-8')}")
+
+    return "\n".join(parts)
+
+
+def load_tests(master_dir: Path) -> list[dict]:
+    """Load fidelity.jsonl test cases."""
+    fidelity_path = master_dir / "tests" / "fidelity.jsonl"
+    if not fidelity_path.exists():
+        return []
+    tests = []
+    for line in fidelity_path.read_text(encoding="utf-8").strip().splitlines():
+        if line.strip():
+            tests.append(json.loads(line))
+    return tests
+
+
+def check_response(response: str, test_case: dict, is_first_turn: bool = True) -> dict:
+    """Check a response against expected citations, mentions, and boundaries.
+
+    Returns {passed: bool, missing_cites: [...], missing_mentions: [...],
+             forbidden_found: [...], boundary_violations: [...]}.
+    """
+    missing_cites = []
+    for cite in test_case.get("must_cite", []):
+        if cite not in response:
+            missing_cites.append(cite)
+
+    missing_mentions = []
+    for mention in test_case.get("must_mention", []):
+        if mention not in response:
+            missing_mentions.append(mention)
+
+    # Boundary tests: must_not_contain
+    forbidden_found = []
+    for forbidden in test_case.get("must_not_contain", []):
+        if forbidden in response:
+            forbidden_found.append(forbidden)
+
+    # First-turn boundary: must_not_contain_first_turn
+    boundary_violations = []
+    if is_first_turn:
+        for forbidden in test_case.get("must_not_contain_first_turn", []):
+            if forbidden in response:
+                boundary_violations.append(forbidden)
+
+    passed = (
+        len(missing_cites) == 0
+        and len(missing_mentions) == 0
+        and len(forbidden_found) == 0
+        and len(boundary_violations) == 0
+    )
+
+    return {
+        "passed": passed,
+        "missing_cites": missing_cites,
+        "missing_mentions": missing_mentions,
+        "forbidden_found": forbidden_found,
+        "boundary_violations": boundary_violations,
+    }
+
+
+def run_tests(
+    master_name: str,
+    dry_run: bool = False,
+    model: str = "claude-sonnet-4-6",
+    max_tests: int | None = None,
+) -> dict:
+    """Run fidelity tests for a master. Returns summary."""
+    master_dir = PREBUILT_DIR / master_name
+    if not master_dir.exists():
+        return {"error": f"Master '{master_name}' not found"}
+
+    tests = load_tests(master_dir)
+    if not tests:
+        return {"error": f"No fidelity.jsonl found for '{master_name}'"}
+
+    if max_tests is not None and max_tests > 0:
+        # Prefer easier/basic tests when capping — smoke suite should hit
+        # the reliable floor, not the advanced stress cases.
+        tests = sorted(
+            tests,
+            key=lambda t: {"basic": 0, "intermediate": 1, "advanced": 2}.get(
+                t.get("difficulty", "intermediate"), 1
+            ),
+        )[:max_tests]
+
+    results: list[dict] = []
+
+    if dry_run:
+        for i, test in enumerate(tests):
+            results.append({
+                "index": i,
+                "question": test["q"],
+                "must_cite": test.get("must_cite", []),
+                "must_mention": test.get("must_mention", []),
+                "difficulty": test.get("difficulty", "unknown"),
+                "status": "dry_run",
+            })
+        return {"master": master_name, "total": len(tests), "results": results}
+
+    # Load skill context
+    system_prompt = load_skill_context(master_dir)
+
+    # Import anthropic
+    try:
+        import anthropic
+    except ImportError:
+        return {"error": "anthropic package not installed. Run: pip install anthropic"}
+
+    api_key = os.environ.get("ANTHROPIC_API_KEY")
+    if not api_key:
+        return {"error": "ANTHROPIC_API_KEY environment variable not set"}
+
+    client = anthropic.Anthropic(api_key=api_key)
+
+    passed = 0
+    failed = 0
+
+    for i, test in enumerate(tests):
+        print(f"  [{i+1}/{len(tests)}] {test['q'][:50]}...", end=" ", flush=True)
+
+        try:
+            message = client.messages.create(
+                model=model,
+                max_tokens=2048,
+                system=system_prompt,
+                messages=[{"role": "user", "content": test["q"]}],
+            )
+            response_text = message.content[0].text
+        except Exception as e:
+            results.append({
+                "index": i,
+                "question": test["q"],
+                "status": "api_error",
+                "error": str(e),
+            })
+            failed += 1
+            print("API ERROR")
+            continue
+
+        check = check_response(response_text, test, is_first_turn=True)
+        status = "PASS" if check["passed"] else "FAIL"
+
+        result_entry = {
+            "index": i,
+            "question": test["q"],
+            "difficulty": test.get("difficulty", "unknown"),
+            "test_type": test.get("test_type", "fidelity"),
+            "status": status,
+            "missing_cites": check["missing_cites"],
+            "missing_mentions": check["missing_mentions"],
+            "forbidden_found": check["forbidden_found"],
+            "boundary_violations": check["boundary_violations"],
+            "response_length": len(response_text),
+        }
+        results.append(result_entry)
+
+        if check["passed"]:
+            passed += 1
+            print("PASS")
+        else:
+            failed += 1
+            failures = (check["missing_cites"] + check["missing_mentions"]
+                        + check["forbidden_found"] + check["boundary_violations"])
+            print(f"FAIL ({failures})")
+
+    return {
+        "master": master_name,
+        "model": model,
+        "total": len(tests),
+        "passed": passed,
+        "failed": failed,
+        "pass_rate": f"{passed / len(tests) * 100:.0f}%" if tests else "N/A",
+        "results": results,
+    }
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Master-skill fidelity test runner")
+    parser.add_argument("--master", type=str, help="Test a specific master")
+    parser.add_argument("--all", action="store_true", help="Test all masters with fidelity.jsonl")
+    parser.add_argument("--dry-run", action="store_true", help="Show test cases without calling API")
+    parser.add_argument("--model", type=str, default="claude-sonnet-4-6", help="Claude model to use")
+    parser.add_argument("--json", action="store_true", help="Output as JSON")
+    parser.add_argument(
+        "--max-tests",
+        type=int,
+        default=None,
+        help="Cap the number of fixtures per master (smoke runs in CI use 1)",
+    )
+    args = parser.parse_args()
+
+    if not args.master and not args.all:
+        parser.error("Specify --master <name> or --all")
+
+    if args.all:
+        masters = sorted(
+            d.name for d in PREBUILT_DIR.iterdir()
+            if d.is_dir() and (d / "tests" / "fidelity.jsonl").exists()
+        )
+    else:
+        masters = [args.master]
+
+    all_results = []
+    for master in masters:
+        print(f"\n{'='*50}")
+        print(f"Testing: {master}")
+        print(f"{'='*50}")
+        result = run_tests(
+            master, dry_run=args.dry_run, model=args.model, max_tests=args.max_tests
+        )
+        all_results.append(result)
+
+        if not args.json and "error" not in result:
+            print(f"\nResult: {result.get('passed', 0)}/{result['total']} passed "
+                  f"({result.get('pass_rate', 'N/A')})")
+
+    if args.json:
+        print(json.dumps(all_results, indent=2, ensure_ascii=False))
+    elif len(masters) > 1:
+        print(f"\n{'='*50}")
+        print("Overall Summary:")
+        for r in all_results:
+            if "error" in r:
+                print(f"  {r.get('master', '?')}: {r['error']}")
+            else:
+                print(f"  {r['master']}: {r.get('passed', 0)}/{r['total']} ({r.get('pass_rate', 'N/A')})")
+
+
+if __name__ == "__main__":
+    main()

package/scripts/validate-fidelity.py

@@ -0,0 +1,146 @@
+#!/usr/bin/env python3
+"""Validate fidelity.jsonl structure for all masters.
+
+Checks that every test case has required fields and valid structure.
+No API calls needed — pure structural validation.
+
+Usage:
+    python scripts/validate-fidelity.py
+"""
+
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+PREBUILT_DIR = Path(__file__).resolve().parent.parent / "prebuilt"
+
+VALID_TEST_TYPES = {"fidelity", "boundary", "pressure"}
+VALID_BOUNDARIES = {
+    "sectarian_judgment",
+    "no_prophecy",
+    "neutral_first_turn",
+    "no_fabricated_dialogue",
+    "no_esoteric_instruction",
+}
+VALID_PRESSURES = {
+    "citation_bypass",
+    "informality_bypass",
+    "meta_challenge",
+    "hostile_challenge",
+    "simplicity_bypass",
+    "terminology_bypass",
+    "relevance_challenge",
+    "misunderstanding_challenge",
+}
+
+
+def validate_master(master_dir: Path) -> list[str]:
+    """Validate fidelity.jsonl for a single master. Returns list of errors."""
+    fidelity_path = master_dir / "tests" / "fidelity.jsonl"
+    if not fidelity_path.exists():
+        return [f"{master_dir.name}: no fidelity.jsonl found"]
+
+    errors = []
+    lines = fidelity_path.read_text(encoding="utf-8").strip().splitlines()
+
+    if len(lines) < 5:
+        errors.append(f"{master_dir.name}: fewer than 5 test cases ({len(lines)})")
+
+    for i, line in enumerate(lines, 1):
+        if not line.strip():
+            continue
+        try:
+            test = json.loads(line)
+        except json.JSONDecodeError as e:
+            errors.append(f"{master_dir.name}:{i}: invalid JSON — {e}")
+            continue
+
+        # Every test must have "q"
+        if "q" not in test:
+            errors.append(f"{master_dir.name}:{i}: missing 'q' field")
+
+        # Must have at least one assertion
+        has_assertion = any(
+            k in test
+            for k in [
+                "must_cite",
+                "must_mention",
+                "must_not_contain",
+                "must_not_contain_first_turn",
+                "must_select_masters",
+                "must_have_sections",
+                "must_cite_per_master",
+            ]
+        )
+        if not has_assertion:
+            errors.append(f"{master_dir.name}:{i}: no assertion fields found")
+
+        # Validate test_type if present
+        test_type = test.get("test_type")
+        if test_type and test_type not in VALID_TEST_TYPES:
+            errors.append(
+                f"{master_dir.name}:{i}: invalid test_type '{test_type}' "
+                f"(valid: {VALID_TEST_TYPES})"
+            )
+
+        # Validate boundary/pressure subtypes
+        if test_type == "boundary":
+            boundary = test.get("boundary")
+            if not boundary:
+                errors.append(f"{master_dir.name}:{i}: boundary test missing 'boundary' field")
+            elif boundary not in VALID_BOUNDARIES:
+                errors.append(
+                    f"{master_dir.name}:{i}: unknown boundary '{boundary}' "
+                    f"(valid: {VALID_BOUNDARIES})"
+                )
+
+        if test_type == "pressure":
+            pressure = test.get("pressure")
+            if not pressure:
+                errors.append(f"{master_dir.name}:{i}: pressure test missing 'pressure' field")
+
+        # List fields must be lists
+        for field in ["must_cite", "must_mention", "must_not_contain", "must_not_contain_first_turn"]:
+            if field in test and not isinstance(test[field], list):
+                errors.append(f"{master_dir.name}:{i}: '{field}' must be a list")
+
+    # Check coverage: should have at least one boundary test
+    has_boundary = any(
+        json.loads(l).get("test_type") == "boundary"
+        for l in lines
+        if l.strip()
+    )
+    if not has_boundary:
+        errors.append(f"{master_dir.name}: no boundary tests found (need at least one)")
+
+    return errors
+
+
+def main():
+    all_errors = []
+    masters = sorted(
+        d for d in PREBUILT_DIR.iterdir()
+        if d.is_dir() and (d / "tests" / "fidelity.jsonl").exists()
+    )
+
+    for master_dir in masters:
+        errors = validate_master(master_dir)
+        all_errors.extend(errors)
+        if not errors:
+            fidelity_path = master_dir / "tests" / "fidelity.jsonl"
+            count = len(fidelity_path.read_text().strip().splitlines()) if fidelity_path.exists() else 0
+            print(f"  {master_dir.name}: {count} tests OK")
+
+    if all_errors:
+        print(f"\n{len(all_errors)} error(s) found:")
+        for err in all_errors:
+            print(f"  ERROR: {err}")
+        sys.exit(1)
+    else:
+        print(f"\nAll {len(masters)} masters validated successfully.")
+
+
+if __name__ == "__main__":
+    main()