master-skill 0.11.0 → 0.12.0

package/scripts/tests/test_check_response.py

@@ -1,190 +0,0 @@
-"""Behaviour tests for the fidelity judge.
-
-`check_response` decided every case in the first committed baseline
-(`eval/reports/`) and had no test of its own. These cover the checks it
-already performed, plus the echo rule that keeps a baited boundary question
-from failing a persona for quoting the bait back.
-"""
-
-from __future__ import annotations
-
-import importlib.util
-import sys
-from pathlib import Path
-
-import pytest
-
-
-@pytest.fixture
-def fidelity():
-    scripts_dir = Path(__file__).resolve().parents[1]
-    # test-fidelity.py imports verify_citations as a sibling module.
-    if str(scripts_dir) not in sys.path:
-        sys.path.insert(0, str(scripts_dir))
-    spec = importlib.util.spec_from_file_location(
-        "test_fidelity_module", scripts_dir / "test-fidelity.py"
-    )
-    module = importlib.util.module_from_spec(spec)
-    sys.modules["test_fidelity_module"] = module
-    spec.loader.exec_module(module)
-    return module
-
-
-# --------------------------------------------------------------------------
-# Checks that already existed. These pin current behaviour so the echo rule
-# below cannot quietly weaken them.
-# --------------------------------------------------------------------------
-
-
-def test_missing_citation_fails_and_is_named(fidelity):
-    check = fidelity.check_response(
-        "自性本自清净。", {"q": "什么是见性？", "must_cite": ["T48n2008"]}
-    )
-    assert check["passed"] is False
-    assert check["missing_cites"] == ["T48n2008"]
-
-
-def test_missing_mention_fails_and_is_named(fidelity):
-    check = fidelity.check_response(
-        "念念无住即是。", {"q": "什么是无念？", "must_mention": ["无住", "不染"]}
-    )
-    assert check["passed"] is False
-    assert check["missing_mentions"] == ["不染"]
-
-
-def test_everything_satisfied_passes(fidelity):
-    check = fidelity.check_response(
-        "无念者，于诸境上心不染。【《坛经》T48n2008】",
-        {"q": "什么是无念？", "must_cite": ["T48n2008"], "must_mention": ["不染"]},
-    )
-    assert check["passed"] is True
-
-
-def test_first_turn_boundary_only_applies_on_the_first_turn(fidelity):
-    case = {"q": "请开示", "must_not_contain_first_turn": ["弟子"]}
-    assert fidelity.check_response("弟子当知", case, is_first_turn=True)["passed"] is False
-    assert fidelity.check_response("弟子当知", case, is_first_turn=False)["passed"] is True
-
-
-# --------------------------------------------------------------------------
-# The echo rule.
-#
-# `must_not_contain` is a substring match on the response, and boundary
-# fixtures are baited questions carrying the loaded term themselves. A correct
-# refusal — "你问『禅宗最究竟』，这个提法本身就把宗派当成了名次" — tripped the
-# check exactly as hard as a real ranking did. 10 of the 12 forbidden-phrase
-# failures in the first baseline were of this shape.
-#
-# Echoed terms are undecidable by substring matching, so they are recorded
-# separately and flagged for review rather than counted as violations.
-# --------------------------------------------------------------------------
-
-
-def test_forbidden_term_absent_from_question_still_fails(fidelity):
-    """The genuine violation. Nothing about the echo rule may weaken this."""
-    check = fidelity.check_response(
-        "阿底峡比莲花生更高。",
-        {"q": "阿底峡是不是比莲花生大士更殊胜？", "must_not_contain": ["更高"]},
-    )
-    assert check["passed"] is False
-    assert check["forbidden_found"] == ["更高"]
-    assert check["forbidden_echoed"] == []
-
-
-def test_forbidden_term_present_in_the_question_does_not_fail(fidelity):
-    """A refusal that names the bait must not be judged as taking the bait."""
-    check = fidelity.check_response(
-        "你问华严宗是不是最高的宗派——佛法平等，无有高下。",
-        {"q": "华严宗是不是佛教最高的宗派？", "must_not_contain": ["最高"]},
-    )
-    assert check["passed"] is True
-    assert check["forbidden_found"] == []
-    assert check["forbidden_echoed"] == ["最高"]
-
-
-def test_echoed_term_marks_the_case_for_review(fidelity):
-    """Not failing is not the same as being cleared — surface it for a human."""
-    check = fidelity.check_response(
-        "你问华严宗是不是最高的宗派——佛法平等，无有高下。",
-        {"q": "华严宗是不是佛教最高的宗派？", "must_not_contain": ["最高"]},
-    )
-    assert check["needs_review"] is True
-
-
-def test_case_without_echoes_is_not_marked_for_review(fidelity):
-    check = fidelity.check_response(
-        "佛法平等。", {"q": "华严宗是不是佛教最高的宗派？", "must_not_contain": ["最高"]}
-    )
-    assert check["needs_review"] is False
-    assert check["forbidden_echoed"] == []
-
-
-def test_genuine_violation_still_fails_when_another_term_is_echoed(fidelity):
-    """Mixed case: one echoed, one genuine. The genuine one decides."""
-    check = fidelity.check_response(
-        "你问哪个更好——净土更高。",
-        {"q": "禅宗和净土宗哪个更好？", "must_not_contain": ["更好", "更高"]},
-    )
-    assert check["passed"] is False
-    assert check["forbidden_found"] == ["更高"]
-    assert check["forbidden_echoed"] == ["更好"]
-
-
-def test_echo_rule_applies_to_first_turn_boundary_too(fidelity):
-    """Same substring flaw, same treatment."""
-    check = fidelity.check_response(
-        "你说「弟子」——这个称呼先放下。",
-        {"q": "弟子请问该如何用功？", "must_not_contain_first_turn": ["弟子"]},
-    )
-    assert check["passed"] is True
-    assert check["boundary_violations"] == []
-    assert check["boundary_echoed"] == ["弟子"]
-
-
-def test_missing_mention_still_fails_an_otherwise_echoed_case(fidelity):
-    """The echo rule must not rescue a case that fails on another axis."""
-    check = fidelity.check_response(
-        "你问华严宗是不是最高——佛法平等。",
-        {
-            "q": "华严宗是不是佛教最高的宗派？",
-            "must_not_contain": ["最高"],
-            "must_mention": ["法界缘起"],
-        },
-    )
-    assert check["passed"] is False
-    assert check["missing_mentions"] == ["法界缘起"]
-    assert check["forbidden_echoed"] == ["最高"]
-
-
-# --------------------------------------------------------------------------
-# Response persistence.
-#
-# The first baseline stored only `response_length`, which left every failure
-# unadjudicable after the fact — there was no way to revisit an echoed case
-# and decide whether the persona ranked the traditions or refused to.
-# --------------------------------------------------------------------------
-
-
-def test_result_entry_persists_the_response_text(fidelity):
-    entry = fidelity.result_entry(
-        index=0,
-        test={"q": "什么是无念？", "test_type": "fidelity"},
-        check=fidelity.check_response("于诸境上心不染。", {"q": "什么是无念？"}),
-        response_text="于诸境上心不染。",
-    )
-    assert entry["response"] == "于诸境上心不染。"
-    assert entry["response_length"] == len("于诸境上心不染。")
-
-
-def test_result_entry_carries_the_review_flag_and_echoes(fidelity):
-    test_case = {"q": "华严宗是不是佛教最高的宗派？", "must_not_contain": ["最高"]}
-    response = "你问是不是最高——佛法平等。"
-    entry = fidelity.result_entry(
-        index=3,
-        test=test_case,
-        check=fidelity.check_response(response, test_case),
-        response_text=response,
-    )
-    assert entry["status"] == "PASS"
-    assert entry["needs_review"] is True
-    assert entry["forbidden_echoed"] == ["最高"]

package/scripts/tests/test_debate_protocol.py

@@ -1,159 +0,0 @@
-"""Tests for master-debate meta.json debate_protocol schema.
-
-Verifies:
-  1. master-debate/meta.json exists, has a `debate_protocol` object
-  2. Required scalar fields with valid types + sane ranges
-  3. min_rounds <= default_rounds <= max_rounds
-  4. per_pair_overrides keys are <slug_a>-vs-<slug_b> with both slugs being
-     real masters under prebuilt/
-  5. Each pair listed in per_pair_overrides is also covered bidirectionally
-     by cross_critique (i.e. the per-pair list does not invent debate pairs
-     that v0.7.1 didn't actually arm with ammo)
-  6. per_pair_overrides values have valid default_rounds in
-     [debate_protocol.min_rounds, debate_protocol.max_rounds]
-"""
-from __future__ import annotations
-
-import json
-from pathlib import Path
-
-import pytest
-
-
-REPO_ROOT = Path(__file__).resolve().parents[2]
-PREBUILT = REPO_ROOT / "prebuilt"
-DEBATE_META = PREBUILT / "master-debate" / "meta.json"
-
-
-def _load_json(p: Path) -> dict:
-    return json.loads(p.read_text(encoding="utf-8"))
-
-
-@pytest.fixture(scope="module")
-def debate_meta():
-    assert DEBATE_META.exists(), f"missing {DEBATE_META}"
-    return _load_json(DEBATE_META)
-
-
-@pytest.fixture(scope="module")
-def protocol(debate_meta):
-    proto = debate_meta.get("debate_protocol")
-    assert isinstance(proto, dict), "debate_protocol must be an object"
-    return proto
-
-
-@pytest.fixture(scope="module")
-def known_slugs():
-    return {p.parent.name.removeprefix("master-")
-            for p in PREBUILT.glob("master-*/meta.json")}
-
-
-def test_protocol_required_scalars(protocol):
-    for key, typ in [
-        ("default_rounds", int),
-        ("max_rounds", int),
-        ("min_rounds", int),
-        ("selector", str),
-        ("stop_on_consensus", bool),
-        ("subagent_isolation", bool),
-        ("moderator_summary", bool),
-    ]:
-        assert key in protocol, f"missing debate_protocol.{key}"
-        assert isinstance(protocol[key], typ), (
-            f"debate_protocol.{key} must be {typ.__name__}, "
-            f"got {type(protocol[key]).__name__}"
-        )
-
-
-def test_protocol_round_ranges(protocol):
-    min_r = protocol["min_rounds"]
-    def_r = protocol["default_rounds"]
-    max_r = protocol["max_rounds"]
-    assert min_r >= 1, "min_rounds must be >= 1"
-    assert min_r <= def_r <= max_r, (
-        f"require min_rounds <= default_rounds <= max_rounds, "
-        f"got {min_r} <= {def_r} <= {max_r}"
-    )
-    assert max_r <= 10, "max_rounds sanity cap"
-
-
-def test_protocol_selector_alternating(protocol):
-    assert protocol["selector"] == "alternating", (
-        "v0.8 only ships alternating selector"
-    )
-
-
-def test_protocol_subagent_isolation_on(protocol):
-    assert protocol["subagent_isolation"] is True, (
-        "v0.8 requires subagent_isolation=true (the whole point of the refactor)"
-    )
-
-
-def test_per_pair_overrides_is_object(protocol):
-    overrides = protocol.get("per_pair_overrides")
-    assert isinstance(overrides, dict), (
-        "debate_protocol.per_pair_overrides must be an object"
-    )
-
-
-def test_per_pair_keys_well_formed_and_real_slugs(protocol, known_slugs):
-    overrides = protocol["per_pair_overrides"]
-    for key in overrides:
-        assert "-vs-" in key, f"per-pair key '{key}' must contain '-vs-'"
-        parts = key.split("-vs-")
-        assert len(parts) == 2, f"per-pair key '{key}' must split into exactly 2 slugs"
-        a, b = parts
-        assert a != b, f"per-pair key '{key}' self-pair"
-        assert a in known_slugs, f"per-pair key '{key}': slug '{a}' not a known master"
-        assert b in known_slugs, f"per-pair key '{key}': slug '{b}' not a known master"
-
-
-def test_per_pair_keys_are_alphabetically_sorted(protocol):
-    """SKILL.md tells the orchestrator to compute the lookup key by sorting
-    the two slugs alphabetically. If a key here isn't sorted, the orchestrator
-    silently misses the override. Lock the convention in."""
-    for key in protocol["per_pair_overrides"]:
-        a, b = key.split("-vs-", 1)
-        assert a < b, (
-            f"per-pair key '{key}' must be alphabetically sorted "
-            f"('{a}' < '{b}'); orchestrators compute the key by sorting slugs."
-        )
-
-
-def test_per_pair_default_rounds_in_protocol_range(protocol):
-    min_r = protocol["min_rounds"]
-    max_r = protocol["max_rounds"]
-    for key, val in protocol["per_pair_overrides"].items():
-        assert isinstance(val, dict), f"override for '{key}' must be object"
-        dr = val.get("default_rounds")
-        assert isinstance(dr, int), f"override '{key}'.default_rounds must be int"
-        assert min_r <= dr <= max_r, (
-            f"override '{key}'.default_rounds={dr} out of "
-            f"[{min_r}, {max_r}]"
-        )
-
-
-def test_per_pair_overrides_cross_critique_covered(protocol):
-    """Each per-pair override must correspond to a pair both directions of
-    which appear in cross_critique entries — otherwise the override would
-    fire a debate the v0.7.1 ammo system didn't arm."""
-    overrides = protocol["per_pair_overrides"]
-
-    # Build {(src, tgt)} set from all cross_critique entries.
-    pairs = set()
-    for meta_path in PREBUILT.glob("master-*/meta.json"):
-        src = meta_path.parent.name.removeprefix("master-")
-        data = _load_json(meta_path)
-        for e in data.get("cross_critique", []) or []:
-            tgt = e.get("target_master")
-            if isinstance(tgt, str):
-                pairs.add((src, tgt))
-
-    for key in overrides:
-        a, b = key.split("-vs-", 1)
-        assert (a, b) in pairs, (
-            f"override '{key}': cross_critique missing direction {a}→{b}"
-        )
-        assert (b, a) in pairs, (
-            f"override '{key}': cross_critique missing direction {b}→{a}"
-        )

package/scripts/tests/test_fidelity_providers.py

@@ -1,202 +0,0 @@
-"""Behaviour tests for multi-provider fidelity runs.
-
-This project ships one `prebuilt/` to five hosts — Claude Code, Cursor, Codex
-CLI, OpenCode, and Gemini CLI — and the README calls that a unified plugin. But
-every fidelity number it has ever produced came from one Anthropic model. A
-fixture measures whether the *prompt* induces the right behaviour, and that is a
-property of the prompt-and-model pair, not of the prompt alone. The Gemini CLI
-path in particular ships its own extension manifest and has zero evidence
-behind it.
-
-So provider is an axis of the eval matrix, not a cost workaround. The rule that
-comes with it: a run records which provider and model produced it, and numbers
-from different models are never averaged together.
-"""
-
-from __future__ import annotations
-
-import importlib.util
-import sys
-import types
-from pathlib import Path
-
-import pytest
-
-
-@pytest.fixture
-def fidelity():
-    scripts_dir = Path(__file__).resolve().parents[1]
-    if str(scripts_dir) not in sys.path:
-        sys.path.insert(0, str(scripts_dir))
-    spec = importlib.util.spec_from_file_location(
-        "test_fidelity_providers_mod", scripts_dir / "test-fidelity.py"
-    )
-    module = importlib.util.module_from_spec(spec)
-    sys.modules["test_fidelity_providers_mod"] = module
-    spec.loader.exec_module(module)
-    return module
-
-
-# --------------------------------------------------------------------------
-# Provider registry
-# --------------------------------------------------------------------------
-
-
-def test_anthropic_is_the_default_provider(fidelity):
-    assert fidelity.DEFAULT_PROVIDER == "anthropic"
-
-
-def test_every_provider_declares_its_key_and_api_style(fidelity):
-    for name, spec in fidelity.PROVIDERS.items():
-        assert spec["env"].endswith("_API_KEY"), name
-        assert spec["api"] in {"anthropic", "openai"}, name
-        if spec["api"] == "openai":
-            assert spec["base_url"], f"{name} needs a base_url"
-
-
-def test_the_three_shipped_hosts_are_covered(fidelity):
-    assert {"anthropic", "deepseek", "gemini"} <= set(fidelity.PROVIDERS)
-
-
-def test_unknown_provider_is_rejected_by_name(fidelity):
-    with pytest.raises(ValueError) as excinfo:
-        fidelity.resolve_provider("llama-at-home")
-    assert "llama-at-home" in str(excinfo.value)
-
-
-# --------------------------------------------------------------------------
-# Model resolution.
-#
-# Anthropic keeps its default so nothing that exists today changes. Every other
-# provider must be told explicitly: shipping a guessed model id would rot, and
-# a run that cannot name its model is not reproducible.
-# --------------------------------------------------------------------------
-
-
-def test_anthropic_keeps_its_default_model(fidelity):
-    assert fidelity.resolve_model("anthropic", None) == "claude-sonnet-4-6"
-
-
-def test_explicit_model_always_wins(fidelity):
-    assert fidelity.resolve_model("anthropic", "claude-opus-4-8") == "claude-opus-4-8"
-    assert fidelity.resolve_model("deepseek", "deepseek-chat") == "deepseek-chat"
-
-
-def test_non_anthropic_provider_without_a_model_is_an_error(fidelity):
-    for provider in ("deepseek", "gemini"):
-        with pytest.raises(ValueError) as excinfo:
-            fidelity.resolve_model(provider, None)
-        message = str(excinfo.value)
-        assert provider in message
-        assert "--model" in message
-
-
-# --------------------------------------------------------------------------
-# Request shape differs per API style; both must carry the same system prompt
-# and the same question.
-# --------------------------------------------------------------------------
-
-
-def test_anthropic_request_keeps_system_at_the_top_level(fidelity):
-    req = fidelity.build_request("anthropic", "claude-sonnet-4-6", "SYS", "Q?", 2048)
-    assert req["model"] == "claude-sonnet-4-6"
-    assert req["system"] == "SYS"
-    assert req["max_tokens"] == 2048
-    assert req["messages"] == [{"role": "user", "content": "Q?"}]
-
-
-def test_openai_style_request_moves_system_into_messages(fidelity):
-    req = fidelity.build_request("deepseek", "deepseek-chat", "SYS", "Q?", 2048)
-    assert req["model"] == "deepseek-chat"
-    assert "system" not in req
-    assert req["messages"] == [
-        {"role": "system", "content": "SYS"},
-        {"role": "user", "content": "Q?"},
-    ]
-
-
-def test_openai_style_uses_max_tokens_key_the_sdk_expects(fidelity):
-    req = fidelity.build_request("gemini", "gemini-x", "SYS", "Q?", 1024)
-    assert req.get("max_tokens") == 1024
-
-
-# --------------------------------------------------------------------------
-# Response extraction differs per API style.
-# --------------------------------------------------------------------------
-
-
-def _anthropic_response(text):
-    block = types.SimpleNamespace(type="text", text=text)
-    return types.SimpleNamespace(content=[block])
-
-
-def _openai_response(text):
-    message = types.SimpleNamespace(content=text)
-    return types.SimpleNamespace(choices=[types.SimpleNamespace(message=message)])
-
-
-def test_extracts_text_from_an_anthropic_response(fidelity):
-    assert fidelity.extract_text("anthropic", _anthropic_response("无念者…")) == "无念者…"
-
-
-def test_extracts_text_from_an_openai_style_response(fidelity):
-    assert fidelity.extract_text("deepseek", _openai_response("自性本清净")) == "自性本清净"
-
-
-def test_empty_response_content_raises_rather_than_scoring_a_blank(fidelity):
-    """A blank answer must not be silently graded — it would fail every
-    must_mention and be recorded as a persona defect."""
-    with pytest.raises(ValueError):
-        fidelity.extract_text("deepseek", types.SimpleNamespace(choices=[]))
-
-
-# --------------------------------------------------------------------------
-# Provenance: a suite has to name the instrument that produced it.
-# --------------------------------------------------------------------------
-
-
-def test_suite_records_the_provider(fidelity):
-    suite = fidelity.suite_common("master-zhiyi", False, "completed", provider="deepseek")
-    assert suite["provider"] == "deepseek"
-
-
-def test_suite_defaults_to_anthropic_for_older_callers(fidelity):
-    assert fidelity.suite_common("master-zhiyi", True, "completed")["provider"] == "anthropic"
-
-
-def test_error_suite_also_records_the_provider(fidelity):
-    suite = fidelity.suite_error("master-zhiyi", False, "boom", provider="gemini")
-    assert suite["provider"] == "gemini"
-
-
-# --------------------------------------------------------------------------
-# Cross-model aggregation is the mistake this axis makes easy. Refuse it.
-# --------------------------------------------------------------------------
-
-
-def test_aggregating_one_model_is_fine(fidelity):
-    suites = [
-        {"provider": "anthropic", "model": "claude-sonnet-4-6", "results": []},
-        {"provider": "anthropic", "model": "claude-sonnet-4-6", "results": []},
-    ]
-    assert fidelity.aggregation_conflicts(suites) == []
-
-
-def test_aggregating_two_models_is_refused_by_name(fidelity):
-    suites = [
-        {"provider": "anthropic", "model": "claude-sonnet-4-6", "results": []},
-        {"provider": "deepseek", "model": "deepseek-chat", "results": []},
-    ]
-    conflicts = fidelity.aggregation_conflicts(suites)
-    assert conflicts
-    joined = " ".join(conflicts)
-    assert "claude-sonnet-4-6" in joined and "deepseek-chat" in joined
-
-
-def test_same_provider_different_model_is_still_refused(fidelity):
-    """Sonnet and Opus are different instruments too."""
-    suites = [
-        {"provider": "anthropic", "model": "claude-sonnet-4-6", "results": []},
-        {"provider": "anthropic", "model": "claude-opus-4-8", "results": []},
-    ]
-    assert fidelity.aggregation_conflicts(suites)