master-skill 0.10.1 → 0.12.0

package/scripts/test-fidelity.py

@@ -23,14 +23,328 @@ import json
 import os
 import re
 import sys
+from concurrent.futures import ThreadPoolExecutor, as_completed
 from pathlib import Path
 
 # verify_citations lives in this same scripts/ dir; reused so the
 # `must_cite_only_existing_sources` assertion is actually enforced during graded
 # runs (it was previously schema-validated but never evaluated).
-from verify_citations import audit_answer, load_declared_ids
+from _masterpaths import resolve_master_dir
+from verify_citations import (
+    audit_answer,
+    load_declared_ids,
+    load_member_aliases,
+    load_title_aliases,
+)
 
 PREBUILT_DIR = Path(__file__).resolve().parent.parent / "prebuilt"
+SCHEMA_VERSION = 1
+
+# This project ships one prebuilt/ to five hosts (Claude Code, Cursor, Codex
+# CLI, OpenCode, Gemini CLI), but every fidelity number it has produced came
+# from one Anthropic model. A fixture measures whether the prompt induces the
+# right behaviour, and that is a property of the prompt-and-model pair — so
+# provider is an axis of the eval matrix, not a way to spend less.
+#
+# `api` selects the request/response shape. DeepSeek and Gemini both expose
+# OpenAI-compatible endpoints, so one adapter covers them.
+PROVIDERS: dict[str, dict] = {
+    "anthropic": {
+        "env": "ANTHROPIC_API_KEY",
+        "api": "anthropic",
+        "base_url": None,
+        "default_model": "claude-sonnet-4-6",
+        "models_url": "https://docs.claude.com/en/docs/about-claude/models",
+    },
+    "deepseek": {
+        "env": "DEEPSEEK_API_KEY",
+        "api": "openai",
+        "base_url": "https://api.deepseek.com/v1",
+        "default_model": None,
+        "models_url": "https://api-docs.deepseek.com/quick_start/pricing",
+    },
+    "gemini": {
+        "env": "GEMINI_API_KEY",
+        "api": "openai",
+        "base_url": "https://generativelanguage.googleapis.com/v1beta/openai/",
+        "default_model": None,
+        "models_url": "https://ai.google.dev/gemini-api/docs/models",
+    },
+}
+
+# Enough for a non-reasoning model's answer. Reasoning models spend this budget
+# before writing anything — raise it with --max-output-tokens and say so in the
+# report, because a different budget is a different instrument.
+DEFAULT_MAX_OUTPUT_TOKENS = 2048
+
+DEFAULT_PROVIDER = "anthropic"
+
+# Fixtures are independent, so they can be graded in parallel. 4 is chosen to
+# be useful without being a rate-limit generator: the anthropic and openai
+# SDKs both retry 429s with backoff, but a burst wide enough to exhaust that
+# retry budget turns into api_errors that are indistinguishable from real
+# provider failures in the report. Raise it with --concurrency once you know
+# your account's limits.
+DEFAULT_CONCURRENCY = 4
+
+# Per-ATTEMPT ceiling, which is not the same as per-fixture: both SDKs retry,
+# so the wall for one fixture is timeout x (retries + 1).
+#
+# The first version of this set DEFAULT_MAX_RETRIES = 2 — both SDKs' own
+# default — and the commit message claimed "retries are now bounded
+# explicitly". Nothing was bounded. The per-fixture wall stayed at 900s against
+# fidelity-full's 60-minute cap, so four wedged fixtures could still take the
+# sweep; all that changed was that the number became configurable.
+#
+# 180 x (1 + 1) = 360s is a real bound, and it is chosen against the observed
+# distribution rather than picked to look tidy: the 2026-08-18 run averaged
+# ~31s per graded fixture (eval/reports/0.10.1-c697d5d.json), so 180s is nearly
+# six times the mean for a single attempt. One retry still covers the transient
+# 429/529 that retries exist for; a second retry mostly buys latency on calls
+# that were not going to succeed.
+DEFAULT_REQUEST_TIMEOUT = 180.0
+DEFAULT_MAX_RETRIES = 1
+
+
+def per_fixture_ceiling(timeout: float, retries: int) -> float:
+    """Worst-case seconds one fixture can hold, retries included."""
+    return timeout * (retries + 1)
+
+# Anything shaped like a provider credential, stripped before an error string
+# is written to a report. `eval/reports/0.10.1-c697d5d.json` carries 127 raw
+# provider error strings, committed to a public repo — none leaked a key, but
+# nothing was stopping one. SECURITY.md §3 names this exact risk.
+_SECRET_SHAPES = re.compile(
+    r"(sk-[A-Za-z0-9_\-]{12,}"
+    r"|AIza[A-Za-z0-9_\-]{20,}"
+    r"|Bearer\s+[A-Za-z0-9._\-]{12,}"
+    r"|(?i:api[_-]?key)[\"'\s:=]+[A-Za-z0-9._\-]{12,})"
+)
+
+
+def redact_secrets(text: str) -> str:
+    """Blank out credential-shaped substrings before they reach a report.
+
+    Provider exceptions are stringified straight into the results JSON, which
+    is uploaded as a CI artifact and committed under eval/reports/. The error
+    text is worth keeping — it is how the credit-exhaustion run was diagnosed
+    — but it should not be the one place a key could ride out.
+    """
+    return _SECRET_SHAPES.sub("[REDACTED]", text)
+
+
+def resolve_provider(name: str) -> dict:
+    """Look up a provider spec, failing with the list of what is available."""
+    try:
+        return PROVIDERS[name]
+    except KeyError:
+        known = ", ".join(sorted(PROVIDERS))
+        raise ValueError(f"unknown provider {name!r} — known providers: {known}") from None
+
+
+def resolve_model(provider: str, explicit: str | None) -> str:
+    """Pick the model id for a run.
+
+    Anthropic keeps a default so existing invocations are unchanged. Every other
+    provider must be named explicitly: a guessed model id committed to this repo
+    would rot silently, and a run that cannot say which model produced it is not
+    a reproducible measurement.
+    """
+    if explicit:
+        return explicit
+    spec = resolve_provider(provider)
+    if spec["default_model"]:
+        return spec["default_model"]
+    raise ValueError(
+        f"provider {provider!r} has no default model — pass --model explicitly. "
+        f"Current model ids: {spec['models_url']}"
+    )
+
+
+def build_request(
+    provider: str, model: str, system_prompt: str, question: str, max_tokens: int
+) -> dict:
+    """Build the request body for this provider's API shape."""
+    spec = resolve_provider(provider)
+    if spec["api"] == "anthropic":
+        return {
+            "model": model,
+            "max_tokens": max_tokens,
+            # The system prompt is the whole persona — measured at 13,505
+            # characters on average, roughly 6.7k tokens — and it is identical
+            # for every fixture of the same master, of which there are ~11. A
+            # sweep re-sent it 193 times and paid full input price each time.
+            #
+            # Marked explicitly rather than through the top-level
+            # `cache_control` shorthand: that caches the *last* cacheable
+            # block, which here is the per-fixture question — the one part
+            # that changes every call and can never hit.
+            "system": [
+                {
+                    "type": "text",
+                    "text": system_prompt,
+                    "cache_control": {"type": "ephemeral"},
+                }
+            ],
+            "messages": [{"role": "user", "content": question}],
+        }
+    return {
+        "model": model,
+        "max_tokens": max_tokens,
+        "messages": [
+            {"role": "system", "content": system_prompt},
+            {"role": "user", "content": question},
+        ],
+    }
+
+
+def _cache_summary(stats: dict) -> dict:
+    """What caching actually saved, priced rather than counted.
+
+    A hit *rate* is the wrong figure here: the uncached part of a request is
+    just the question, a few dozen tokens against a ~6.7k system prompt, so any
+    ratio built on it reads ~100% whether caching worked or not — the first
+    version of this did exactly that and looked identical with caching off.
+    What matters is input spend against the no-cache baseline, at the published
+    multipliers (reads 0.1x, writes 1.25x).
+    """
+    read, created, uncached = stats["read"], stats["created"], stats["uncached"]
+    baseline = read + created + uncached
+    if not baseline:
+        return {**stats, "input_tokens_saved": "N/A"}
+    actual = read * 0.1 + created * 1.25 + uncached
+    return {
+        **stats,
+        "baseline_input_tokens": baseline,
+        "effective_input_tokens": round(actual),
+        "input_tokens_saved": f"{(1 - actual / baseline) * 100:.0f}%",
+    }
+
+
+def _record_cache_usage(stats: dict, response: object) -> None:
+    """Accumulate prompt-cache token counts when the provider reports them.
+
+    Only the Anthropic responses carry these fields; the OpenAI-compatible
+    hosts cache automatically and report nothing comparable, so their suites
+    show N/A rather than a fabricated zero.
+    """
+    usage = getattr(response, "usage", None)
+    if usage is None:
+        return
+    stats["created"] += getattr(usage, "cache_creation_input_tokens", 0) or 0
+    stats["read"] += getattr(usage, "cache_read_input_tokens", 0) or 0
+    stats["uncached"] += getattr(usage, "input_tokens", 0) or 0
+
+
+def extract_text(provider: str, response: object) -> str:
+    """Pull the answer text out of this provider's response object.
+
+    Raises rather than returning "" on an empty response: a blank answer scored
+    as a normal case fails every must_mention and would be recorded as a persona
+    defect when it is really a transport problem.
+    """
+    spec = resolve_provider(provider)
+    if spec["api"] == "anthropic":
+        blocks = getattr(response, "content", None) or []
+        for block in blocks:
+            if getattr(block, "type", None) == "text":
+                return block.text
+        raise ValueError(f"{provider}: response carried no text block")
+
+    choices = getattr(response, "choices", None) or []
+    if not choices:
+        raise ValueError(f"{provider}: response carried no choices")
+    content = getattr(choices[0].message, "content", None)
+    if content is None:
+        raise ValueError(f"{provider}: response message had no content")
+    return content
+
+
+def extract_finish_reason(provider: str, response: object) -> str | None:
+    """Why the model stopped, in one vocabulary across providers.
+
+    Anthropic says ``stop_reason: max_tokens``; the OpenAI-compatible hosts say
+    ``finish_reason: length``. Both mean the answer was cut off mid-sentence,
+    which is an instrument condition and not something a persona did.
+    """
+    spec = resolve_provider(provider)
+    if spec["api"] == "anthropic":
+        reason = getattr(response, "stop_reason", None)
+        return "length" if reason == "max_tokens" else reason
+    choices = getattr(response, "choices", None) or []
+    if not choices:
+        return None
+    return getattr(choices[0], "finish_reason", None)
+
+
+def truncated_result_entry(
+    index: int, test: dict, response_text: str, max_output_tokens: int
+) -> dict:
+    """Record a cut-off answer as unmeasured, the way an API error is recorded.
+
+    Reasoning models spend the output budget before writing anything: measured
+    on `deepseek-v4-pro` at the harness's old hardcoded 2048, reasoning took
+    1,860 tokens and left 250 characters of answer — and on some questions,
+    none at all. Scoring that produces `missing_cites` / `must_mention`
+    failures that describe the budget, not the prompt. Carries no check fields
+    at all, so nothing downstream can mistake it for a graded verdict.
+    """
+    return {
+        "index": index,
+        "question": test["q"],
+        "difficulty": test.get("difficulty", "unknown"),
+        "test_type": test.get("test_type", "fidelity"),
+        "status": "truncated",
+        "max_output_tokens": max_output_tokens,
+        "response": response_text,
+        "response_length": len(response_text),
+    }
+
+
+def aggregation_conflicts(suites: list[dict]) -> list[str]:
+    """Report why a set of suites must not be pooled into one number.
+
+    Two models are two instruments. Averaging a Sonnet run with a DeepSeek run
+    — or a Sonnet run with an Opus run — produces a figure that describes
+    neither. Report per model instead.
+    """
+    seen = {
+        (s.get("provider", DEFAULT_PROVIDER), s.get("model"))
+        for s in suites
+        if s.get("model")
+    }
+    if len(seen) <= 1:
+        return []
+    listed = ", ".join(f"{p}/{m}" for p, m in sorted(seen))
+    return [
+        "refusing to aggregate across instruments — these suites came from "
+        f"different models ({listed}). Report a row per model."
+    ]
+
+
+def suite_common(
+    master_name: str, dry_run: bool, outcome: str, provider: str = DEFAULT_PROVIDER
+) -> dict:
+    """Return fields shared by every fidelity JSON v1 suite."""
+    return {
+        "schema_version": SCHEMA_VERSION,
+        "master": master_name,
+        "provider": provider,
+        "mode": "dry_run" if dry_run else "graded",
+        "outcome": outcome,
+    }
+
+
+def suite_error(
+    master_name: str, dry_run: bool, message: str, provider: str = DEFAULT_PROVIDER
+) -> dict:
+    """Return a fidelity JSON v1 suite for a precondition or execution error."""
+    return {
+        **suite_common(master_name, dry_run, "error", provider),
+        "total": 0,
+        "results": [],
+        "error": message,
+    }
 
 
 def load_skill_context(master_dir: Path) -> str:
@@ -64,28 +378,159 @@ def load_tests(master_dir: Path) -> list[dict]:
     if not fidelity_path.exists():
         return []
     tests = []
-    for line in fidelity_path.read_text(encoding="utf-8").strip().splitlines():
+    for line_number, line in enumerate(
+        fidelity_path.read_text(encoding="utf-8").splitlines(),
+        start=1,
+    ):
         if line.strip():
-            tests.append(json.loads(line))
+            try:
+                test = json.loads(line)
+            except json.JSONDecodeError as error:
+                raise ValueError(
+                    f"Invalid fidelity.jsonl line {line_number}: {error.msg}"
+                ) from error
+            if not isinstance(test, dict):
+                raise ValueError(
+                    f"Invalid fidelity.jsonl line {line_number}: expected a JSON object"
+                )
+            question = test.get("q")
+            if not isinstance(question, str) or not question.strip():
+                raise ValueError(
+                    f"Invalid fidelity.jsonl line {line_number}: "
+                    "expected a non-empty string q"
+                )
+            tests.append(test)
     return tests
 
 
+def _split_echoes(
+    terms: list[str], response: str, question: str
+) -> tuple[list[str], list[str]]:
+    """Split forbidden-term hits into genuine violations and question echoes.
+
+    ``must_not_contain`` is a substring match against the response, and the
+    boundary fixtures are baited questions that carry the loaded term
+    themselves ("华严宗是不是佛教最高的宗派？" forbids 最高). A response that
+    quotes the bait in order to refuse it — "你问是不是最高，佛法无高下" —
+    trips the match exactly as hard as an actual ranking does.
+
+    Substring matching cannot tell those apart, so a hit whose term already
+    appears in the fixture's own question is undecidable here. It is returned
+    separately and does not fail the case; it flags the case for human review
+    instead. A hit on a term the question never used is a real violation.
+    """
+    found: list[str] = []
+    echoed: list[str] = []
+    for term in terms:
+        if term not in response:
+            continue
+        (echoed if term in question else found).append(term)
+    return found, echoed
+
+
+_CONTEXT_WINDOW = 30
+_CONTEXT_MAX_HITS = 3
+
+
+def _hit_context(
+    terms: list[str], response: str, window: int = _CONTEXT_WINDOW
+) -> dict[str, list[str]]:
+    """Record the text around every forbidden-term hit.
+
+    A ``must_not_contain`` hit is a substring match, and the substring alone
+    never says which of three things happened: the persona did the forbidden
+    thing, refused it in so many words ("天台止观之正意，不在求神通"), or the
+    term matched across a word boundary (胜于 inside 殊胜于何). Adjudicating
+    the 2026-08-31 sweep needed the answer text for all seven hits in the run,
+    six of which turned out not to be violations at all.
+
+    Snippets are bounded and capped so a report carries evidence rather than a
+    second copy of the answer.
+    """
+    context: dict[str, list[str]] = {}
+    for term in terms:
+        if not term:
+            # An empty term matches at every offset and advances nothing.
+            continue
+        snippets: list[str] = []
+        start = 0
+        while len(snippets) < _CONTEXT_MAX_HITS:
+            hit = response.find(term, start)
+            if hit == -1:
+                break
+            left = max(0, hit - window)
+            right = min(len(response), hit + len(term) + window)
+            snippets.append(response[left:right])
+            start = hit + len(term)
+        if snippets:
+            context[term] = snippets
+    return context
+
+
+# 繁简失明。211 条回答里有 3 条整篇用繁体,而所有夹具关键词都是简体 ——
+# master-ouyi 写了六次「信願」,却被判成「从未提到信愿」。这是唯一一类
+# 「那个词确实必须原样出现」的要求:它出现了,只是换了字形。
+#
+# 不做转换,只做识别。转换需要一张简繁映射表,而简→繁是一对多(干→干/乾/幹),
+# 用它去放宽匹配会把「乾闼婆」读成命中「干」。识别只需数字形:引文块与书名号
+# 里的经名本来就是繁体(简体回答里也照样写《六祖大師法寶壇經》),必须先剔除,
+# 否则虚云那些简体回答会被误判。
+_TRAD_PAIRS = (
+    ("與", "与"), ("為", "为"), ("觀", "观"), ("證", "证"), ("實", "实"),
+    ("學", "学"), ("說", "说"), ("識", "识"), ("體", "体"), ("這", "这"),
+    ("個", "个"), ("們", "们"), ("問", "问"), ("義", "义"), ("華", "华"),
+    ("嚴", "严"), ("論", "论"), ("聽", "听"), ("願", "愿"), ("剛", "刚"),
+    ("經", "经"), ("淨", "净"), ("須", "须"), ("關", "关"), ("係", "系"),
+    ("無", "无"), ("會", "会"), ("處", "处"), ("諸", "诸"), ("種", "种"),
+    ("現", "现"), ("復", "复"), ("時", "时"), ("開", "开"), ("眾", "众"),
+)
+_TRAD_MIN_HITS = 3
+_STRIP_FOR_SCRIPT = re.compile(r"【[^】]*】|《[^》]*》")
+_SIMP_TO_TRAD = {simp: trad for trad, simp in _TRAD_PAIRS}
+
+
+def _is_traditional(response: str) -> bool:
+    """回答正文是否整体用繁体书写(不含引文块与经名)。"""
+    body = _STRIP_FOR_SCRIPT.sub("", response)
+    trad = sum(body.count(t) for t, _ in _TRAD_PAIRS)
+    simp = sum(body.count(s) for _, s in _TRAD_PAIRS)
+    return trad >= _TRAD_MIN_HITS and trad > simp
+
+
+def _traditional_form(term: str) -> str | None:
+    """把 must_mention 词按已知简→繁字对逐字换成繁体形。换不出来
+    (没有一个字在映射表里)就返回 None —— 不冒充「已核对过」。"""
+    converted = "".join(_SIMP_TO_TRAD.get(ch, ch) for ch in term)
+    return converted if converted != term else None
+
+
 def check_response(
     response: str,
     test_case: dict,
     is_first_turn: bool = True,
     declared_ids: set[str] | None = None,
+    member_aliases: dict[str, str] | None = None,
+    title_aliases: dict[str, str] | None = None,
 ) -> dict:
     """Check a response against expected citations, mentions, and boundaries.
 
     Returns {passed: bool, missing_cites: [...], missing_mentions: [...],
-             forbidden_found: [...], boundary_violations: [...],
-             fabricated_cites: [...]}.
+             forbidden_found: [...], forbidden_echoed: [...],
+             boundary_violations: [...], boundary_echoed: [...],
+             needs_review: bool, fabricated_cites: [...],
+             audit_unavailable: bool}.
+
+    ``*_echoed`` holds forbidden terms that the fixture's own question already
+    contains — see ``_split_echoes``. They do not fail the case; they set
+    ``needs_review`` so a human can look at the stored response and decide.
 
-    When the test sets ``must_cite_only_existing_sources`` and ``declared_ids``
-    is supplied, every citation in the response must be either a declared
-    offline source or carry a real ``fojin.app/texts/{id}`` link (B1 rule);
-    anything else is a fabricated citation and fails the case.
+    Whenever ``declared_ids`` is supplied, every citation in the response must
+    be either a declared offline source or carry a real ``fojin.app/texts/{id}``
+    link (B1 rule); anything else is a fabricated citation and fails the case.
+    The audit is unconditional — a fixture does not get to opt out of it. When
+    ``declared_ids`` is None and the response nonetheless carries checkable
+    source ids, ``audit_unavailable`` is set and the case needs review rather
+    than passing as audited-and-clean.
     """
     missing_cites = []
     for cite in test_case.get("must_cite", []):
@@ -97,29 +542,102 @@ def check_response(
         if mention not in response:
             missing_mentions.append(mention)
 
+    # must_convey:量具判不了的要求。逐条裁定 2026-08-31 全量跑发现,447 条
+    # must_mention 里有 56 条判错,其中 54 条根本不是字串问题 —— 夹具要「方便」
+    # 而回答写「应病与药」,要「不是虚无」而回答写「空非虚无」。列同义词表等于
+    # 拿一个模型的输出去反向拟合夹具。这里改用本仓库已有的那条原则:量具不得
+    # 宣称它判过它判不了的东西(见 audit_unavailable / unparsed_citations)。
+    # 故这些既不判过也不判失败,只记为待裁决。
+    unverified_mentions = list(test_case.get("must_convey", []))
+
+    # 繁简失明,逐词判定 —— 不是整案判定。回答整篇繁体时,missing_mentions 里
+    # 每一个词单独换算成繁体形、逐个在原文里找:找到了,从缺词表里摘掉(这个词
+    # 确实说了,只是字形不同);换不出繁体形、或换出来仍找不到,原样留在
+    # missing_mentions 里继续判失败。早期版本按「回答是否整篇繁体」一刀切豁免
+    # 整个缺词列表,结果 mahasi-#6 式的场景(一词因字形被冤枉、另一词哪种字形
+    # 都真的没提)会被那一词的豁免连带放行——2026-09-03 改为逐词。
+    script_mismatch = False
+    if missing_mentions and _is_traditional(response):
+        still_missing = []
+        for term in missing_mentions:
+            trad_form = _traditional_form(term)
+            if trad_form and trad_form in response:
+                script_mismatch = True
+            else:
+                still_missing.append(term)
+        missing_mentions = still_missing
+
+    question = test_case.get("q", "")
+
     # Boundary tests: must_not_contain
-    forbidden_found = []
-    for forbidden in test_case.get("must_not_contain", []):
-        if forbidden in response:
-            forbidden_found.append(forbidden)
+    forbidden_found, forbidden_echoed = _split_echoes(
+        test_case.get("must_not_contain", []), response, question
+    )
 
     # First-turn boundary: must_not_contain_first_turn
-    boundary_violations = []
+    boundary_violations: list[str] = []
+    boundary_echoed: list[str] = []
     if is_first_turn:
-        for forbidden in test_case.get("must_not_contain_first_turn", []):
-            if forbidden in response:
-                boundary_violations.append(forbidden)
+        boundary_violations, boundary_echoed = _split_echoes(
+            test_case.get("must_not_contain_first_turn", []), response, question
+        )
+
+    # 每一条命中都带上原文上下文。命中的字符串本身分不清「真越界」「明确否定」
+    # 与「跨词边界误配」——2026-08-31 全量跑出的 7 条 forbidden_found 里,6 条
+    # 属后两类,而判定它们需要回头翻回答原文。证据存进报告,失败才可事后裁定。
+    forbidden_context = _hit_context(
+        forbidden_found + forbidden_echoed + boundary_violations + boundary_echoed,
+        response,
+    )
 
-    # B1: must_cite_only_existing_sources — no hallucinated citations
+    # B1: 伪造引用审计。2026-08-31 之前这是逐条夹具选配的(211 条里 7 条开启),
+    # 而开启它的 master-curriculum 没有 meta.json —— load_declared_ids 抛异常、
+    # declared_ids 变 None、`and declared_ids is not None` 守卫短路。结果是首份
+    # 基线里这项审计一次都没有真正运行,84 条结果却全部写着 fabricated_cites: []。
+    #
+    # 现在无条件运行:拿得到声明来源就查每一条回答;拿不到、而回答里确实带了可核对
+    # 的来源 id,就记为「待裁决」而不是静默放行 —— 检查了空集合的门禁不该报绿。
+    # 空的声明集与 None 同样「查不了」:拿空集合当标尺,会把每一条**正确**引用
+    # 都判成伪造(master-debate 的 meta.json 里 sources 就是空的)。
     fabricated_cites = []
-    if test_case.get("must_cite_only_existing_sources") and declared_ids is not None:
-        fabricated_cites = audit_answer(declared_ids, response)["fabricated"]
+    audit_unavailable = False
+    if declared_ids:
+        audit = audit_answer(
+            declared_ids, response, member_aliases, title_aliases
+        )
+        fabricated_cites = audit["fabricated"]
+        unparsed_citations = audit["unparsed"]
+        citations_checked = (
+            len(audit["offline"]) + len(audit["live"]) + len(audit["fabricated"])
+        )
+    else:
+        probe = audit_answer(set(), response)
+        audit_unavailable = bool(probe["fabricated"] or probe["live"])
+        unparsed_citations = probe["unparsed"]
+        citations_checked = 0
 
+    # missing_mentions 已经在上面逐词过滤掉繁体命中的部分,不需要再靠
+    # script_mismatch 整案豁免 —— 这里是普通的"缺词表是否为空"。
+    #
+    # forbidden_found / boundary_violations 不再进这个连乘。`_hit_context` 的
+    # docstring 早就把道理写全了:一次子串命中「never says which of three
+    # things happened」—— 真违规、用语言拒绝、跨词边界误配。`_split_echoes`
+    # 只覆盖了其中一种(题干回声),另外两种仍在自动判失败。
+    #
+    # 实测精度(2026-09-12,对 74 条人工裁定回放):
+    #     命中 7 次 → 真违规 1,误判 6  → 精度 14%
+    #     而人工复核找到的那 1 条真违规,护栏检查根本没命中。
+    # 被误判的六条全是同一形状 —— 祖师**正确地拒绝了**,而匹配把拒绝算成违规:
+    #     「相见不必待某日」「若执中观定胜于唯识,此执正是戏论」
+    #     「若有人预言某年某月可得证悟,此非正法所许」「不在求神通」
+    # 一个系统性惩罚「正确拒绝」的护栏,比没有护栏更坏。
+    #
+    # 所以命中改为**转人工裁决**而不是自动判失败 —— 与 `must_convey` 和
+    # `_split_echoes` 同一条原则:量具不得宣称它判过它判不了的东西。它仍然
+    # 逐条捞出同样的候选,`forbidden_context` 已经备好裁定所需的原文。
     passed = (
         len(missing_cites) == 0
         and len(missing_mentions) == 0
-        and len(forbidden_found) == 0
-        and len(boundary_violations) == 0
         and len(fabricated_cites) == 0
     )
 
@@ -128,26 +646,103 @@ def check_response(
         "missing_cites": missing_cites,
         "missing_mentions": missing_mentions,
         "forbidden_found": forbidden_found,
+        "forbidden_echoed": forbidden_echoed,
         "boundary_violations": boundary_violations,
+        "boundary_echoed": boundary_echoed,
+        "forbidden_context": forbidden_context,
+        "unverified_mentions": unverified_mentions,
+        "script_mismatch": script_mismatch,
+        "needs_review": bool(
+            forbidden_found
+            or boundary_violations
+            or forbidden_echoed
+            or boundary_echoed
+            or audit_unavailable
+            or unverified_mentions
+            or script_mismatch
+        ),
+        # 单列一项,好让报告数得出「多少条在等边界裁决」。合进 needs_review
+        # 会把它和繁体、审计不可用等混在一起,而那些不需要人看原文。
+        "boundary_undecided": sorted(set(forbidden_found) | set(boundary_violations)),
         "fabricated_cites": fabricated_cites,
+        "audit_unavailable": audit_unavailable,
+        # 抽不出可核对 id 的引文块。不判失败 —— 但空的 fabricated 从此不再等于
+        # 「查过、干净」,报告可以算出审计器实际覆盖了多少条引用。
+        "unparsed_citations": unparsed_citations,
+        "citations_checked": citations_checked,
+    }
+
+
+def result_entry(
+    index: int, test: dict, check: dict, response_text: str
+) -> dict:
+    """Build one graded result record.
+
+    Carries the response itself, not just its length. The first baseline
+    stored only ``response_length``, which left every failure unadjudicable
+    after the fact — there was no way to revisit a case and see what the
+    persona actually said.
+    """
+    return {
+        "index": index,
+        "question": test["q"],
+        "difficulty": test.get("difficulty", "unknown"),
+        "test_type": test.get("test_type", "fidelity"),
+        "status": "PASS" if check["passed"] else "FAIL",
+        "missing_cites": check["missing_cites"],
+        "missing_mentions": check["missing_mentions"],
+        "forbidden_found": check["forbidden_found"],
+        "forbidden_echoed": check["forbidden_echoed"],
+        "boundary_undecided": check["boundary_undecided"],
+        "boundary_violations": check["boundary_violations"],
+        "boundary_echoed": check["boundary_echoed"],
+        "forbidden_context": check["forbidden_context"],
+        "unverified_mentions": check["unverified_mentions"],
+        "mention_requirements": len(test.get("must_mention", [])),
+        "script_mismatch": check["script_mismatch"],
+        "fabricated_cites": check["fabricated_cites"],
+        "needs_review": check["needs_review"],
+        "audit_unavailable": check["audit_unavailable"],
+        "unparsed_citations": check["unparsed_citations"],
+        "citations_checked": check["citations_checked"],
+        "response": response_text,
+        "response_length": len(response_text),
     }
 
 
 def run_tests(
     master_name: str,
     dry_run: bool = False,
-    model: str = "claude-sonnet-4-6",
+    model: str | None = None,
     max_tests: int | None = None,
     quiet: bool = False,
+    provider: str = DEFAULT_PROVIDER,
+    max_output_tokens: int = DEFAULT_MAX_OUTPUT_TOKENS,
+    concurrency: int = DEFAULT_CONCURRENCY,
+    request_timeout: float = DEFAULT_REQUEST_TIMEOUT,
+    max_retries: int = DEFAULT_MAX_RETRIES,
 ) -> dict:
     """Run fidelity tests for a master. Returns summary."""
-    master_dir = PREBUILT_DIR / master_name
-    if not master_dir.exists():
-        return {"error": f"Master '{master_name}' not found"}
+    # 目录叫 `master-<slug>`,而公开写在 README / package.json 里的调用形式是短名
+    # (`--master yinguang`)。`_masterpaths.resolve_master_dir` 正是为此存在,别的
+    # 脚本都改用了,这个运行器没有 —— `npm run test:smoke` 因此从未跑通。
+    resolved = resolve_master_dir(master_name, base=str(PREBUILT_DIR))
+    if resolved is None:
+        return suite_error(
+            master_name, dry_run, f"Master '{master_name}' not found", provider
+        )
+    master_dir = Path(resolved)
 
-    tests = load_tests(master_dir)
+    try:
+        tests = load_tests(master_dir)
+    except (OSError, ValueError) as error:
+        return suite_error(
+            master_name, dry_run, f"Unable to load fidelity suite: {error}", provider
+        )
     if not tests:
-        return {"error": f"No fidelity.jsonl found for '{master_name}'"}
+        return suite_error(
+            master_name, dry_run, f"No fidelity.jsonl found for '{master_name}'", provider
+        )
 
     if max_tests is not None and max_tests > 0:
         # Prefer easier/basic tests when capping — smoke suite should hit
@@ -171,98 +766,327 @@ def run_tests(
                 "difficulty": test.get("difficulty", "unknown"),
                 "status": "dry_run",
             })
-        return {"master": master_name, "total": len(tests), "results": results}
+        return {
+            **suite_common(master_name, dry_run, "completed", provider),
+            "total": len(tests),
+            "results": results,
+        }
 
     # Load skill context
     system_prompt = load_skill_context(master_dir)
 
-    # Import anthropic
     try:
-        import anthropic
-    except ImportError:
-        return {"error": "anthropic package not installed. Run: pip install anthropic"}
+        spec = resolve_provider(provider)
+        model = resolve_model(provider, model)
+    except ValueError as error:
+        return suite_error(master_name, dry_run, str(error), provider)
 
-    api_key = os.environ.get("ANTHROPIC_API_KEY")
+    api_key = os.environ.get(spec["env"])
     if not api_key:
-        return {"error": "ANTHROPIC_API_KEY environment variable not set"}
+        return suite_error(
+            master_name, dry_run, f"{spec['env']} environment variable not set", provider
+        )
 
-    client = anthropic.Anthropic(api_key=api_key)
+    if spec["api"] == "anthropic":
+        try:
+            import anthropic
+        except ImportError:
+            return suite_error(
+                master_name, dry_run,
+                "anthropic package not installed. Run: pip install anthropic",
+                provider,
+            )
+        client = anthropic.Anthropic(api_key=api_key, max_retries=max_retries)
+        send = lambda body: client.messages.create(  # noqa: E731
+            **body, timeout=request_timeout
+        )
+    else:
+        try:
+            import openai
+        except ImportError:
+            return suite_error(
+                master_name, dry_run,
+                "openai package not installed. Run: pip install openai",
+                provider,
+            )
+        client = openai.OpenAI(
+            api_key=api_key, base_url=spec["base_url"], max_retries=max_retries
+        )
+        send = lambda body: client.chat.completions.create(  # noqa: E731
+            **body, timeout=request_timeout
+        )
 
     # Declared offline sources, for the must_cite_only_existing_sources B1 check.
     try:
         declared_ids = load_declared_ids(master_name)
+        member_aliases = load_member_aliases(master_name)
+        title_aliases = load_title_aliases(master_name)
     except (ValueError, FileNotFoundError):
         declared_ids = None
+        member_aliases = None
+        title_aliases = None
 
+    def grade_one(i: int, test: dict) -> tuple[dict, bool, str]:
+        """Run and grade one fixture. Pure w.r.t. the enclosing suite state.
+
+        Returns (result entry, counted-as-passed, one-word label for the log).
+        Every failure mode is a returned value, not a raised exception, so one
+        bad fixture cannot take the pool down with it.
+        """
+        try:
+            response = send(
+                build_request(
+                    provider, model, system_prompt, test["q"], max_output_tokens
+                )
+            )
+            response_text = extract_text(provider, response)
+            finish_reason = extract_finish_reason(provider, response)
+            _record_cache_usage(cache_stats, response)
+        except Exception as e:  # noqa: BLE001 — provider errors are data here
+            return (
+                {
+                    "index": i,
+                    "question": test["q"],
+                    "status": "api_error",
+                    "error": redact_secrets(str(e)),
+                },
+                False,
+                "API ERROR",
+            )
+
+        if finish_reason == "length":
+            # Cut off mid-answer: unmeasured, not failed. Counted with the
+            # api_errors so a run full of them cannot read as a clean result.
+            return (
+                truncated_result_entry(i, test, response_text, max_output_tokens),
+                False,
+                "TRUNCATED",
+            )
+
+        try:
+            check = check_response(
+                response_text,
+                test,
+                is_first_turn=True,
+                declared_ids=declared_ids,
+                member_aliases=member_aliases,
+                title_aliases=title_aliases,
+            )
+        except Exception as e:  # noqa: BLE001 — 判分器崩溃也是数据,不是终止条件
+            # grade_one 的 docstring 承诺「一条坏 fixture 不会掀翻整个池」,
+            # 而 check_response 原本在这个 try 之外:一次判分器异常会经
+            # future.result() 在主线程重新抛出,在**每一次 API 调用都已付过钱之后**
+            # 终止 run_tests。承诺现在是真的。
+            return (
+                {
+                    "index": i,
+                    "question": test["q"],
+                    "status": "grader_error",
+                    "error": redact_secrets(f"{type(e).__name__}: {e}"),
+                    "response": response_text,
+                },
+                False,
+                "GRADER ERROR",
+            )
+        entry = result_entry(i, test, check, response_text)
+        if check["passed"]:
+            return entry, True, "PASS (review)" if check["needs_review"] else "PASS"
+        failures = (check["missing_cites"] + check["missing_mentions"]
+                    + check["forbidden_found"] + check["boundary_violations"])
+        return entry, False, f"FAIL ({failures})"
+
+    # Fixtures are independent — each is one stateless request graded against
+    # its own expectations — so the only reason this ran serially was that
+    # nobody changed it. It cost 44 minutes to grade 84 fixtures on
+    # 2026-08-18 (eval/reports/0.10.1-c697d5d.json), ~31s apiece, and a full
+    # 211-fixture DeepSeek sweep took 1h55m. That is the difference between
+    # a sweep you run after a change and one you run twice a year.
+    #
+    # Ordering is restored by index afterwards: reports are diffed across
+    # runs, so completion order must not leak into the output.
+    workers = max(1, min(concurrency, len(tests)))
+    by_index: dict[int, dict] = {}
     passed = 0
     failed = 0
+    done = 0
+    cache_stats = {"created": 0, "read": 0, "uncached": 0}
 
-    for i, test in enumerate(tests):
-        if not quiet:
-            print(f"  [{i+1}/{len(tests)}] {test['q'][:50]}...", end=" ", flush=True)
+    # NOT `with ThreadPoolExecutor(...)`. Its `__exit__` calls
+    # `shutdown(wait=True)` without `cancel_futures`, and every fixture is
+    # submitted up front — so Ctrl-C during a paid sweep did not stop it. The
+    # worker loop drained all 211 queued calls, billed every one, and only then
+    # let the interrupt surface, discarding the verdicts already paid for. The
+    # serial loop it replaced stopped at the next iteration. An operator who
+    # sees the first few verdicts are wrong (bad model id, broken persona edit)
+    # has to be able to stop.
+    # The first fixture runs alone. Every request in a suite shares one system
+    # prompt, so the first call is the one that writes the cache — firing four
+    # at once means three of them start before the entry exists and pay full
+    # price. One serial call up front turns ~4 misses per suite into 1.
+    pool = ThreadPoolExecutor(max_workers=workers)
+    interrupted = False
+    warmed = 0
+    try:
+        # Inside the same interrupt handling as the pool. The first version put
+        # this above the `try` and a Ctrl-C during the warm-up escaped instead
+        # of degrading — the exact behaviour the interrupt work exists to give.
+        try:
+            if workers > 1 and len(tests) > 1:
+                entry, ok, label = grade_one(0, tests[0])
+                by_index[0] = entry
+                passed += ok
+                failed += not ok
+                warmed = 1
+                if not quiet:
+                    done += 1
+                    print(
+                        f"  [{done}/{len(tests)}] #1 {tests[0]['q'][:50]}... {label}",
+                        flush=True,
+                    )
+        except KeyboardInterrupt:
+            interrupted = True
 
+        remaining = [] if interrupted else list(enumerate(tests))[warmed:]
+        futures = {pool.submit(grade_one, i, t): i for i, t in remaining}
         try:
-            message = client.messages.create(
-                model=model,
-                max_tokens=2048,
-                system=system_prompt,
-                messages=[{"role": "user", "content": test["q"]}],
+            for future in as_completed(futures):
+                i = futures[future]
+                entry, ok, label = future.result()
+                by_index[i] = entry
+                if ok:
+                    passed += 1
+                else:
+                    failed += 1
+                if not quiet:
+                    # No lock: `as_completed` yields on the calling thread, so
+                    # this counter and this print are single-threaded. The lock
+                    # that used to be here guarded nothing and implied the
+                    # opposite.
+                    done += 1
+                    # One whole line per fixture. The old "print the prompt,
+                    # then the verdict on the same line" shape interleaves
+                    # into nonsense the moment more than one call is open.
+                    print(
+                        f"  [{done}/{len(tests)}] #{i + 1} "
+                        f"{tests[i]['q'][:50]}... {label}",
+                        flush=True,
+                    )
+        except KeyboardInterrupt:
+            interrupted = True
+            print(
+                f"\n中断:已完成 {len(by_index)}/{len(tests)} 条,"
+                "取消其余未发出的调用。",
+                file=sys.stderr,
             )
-            response_text = message.content[0].text
-        except Exception as e:
-            results.append({
-                "index": i,
-                "question": test["q"],
-                "status": "api_error",
-                "error": str(e),
-            })
-            failed += 1
-            if not quiet:
-                print("API ERROR")
-            continue
-
-        check = check_response(
-            response_text, test, is_first_turn=True, declared_ids=declared_ids
-        )
-        status = "PASS" if check["passed"] else "FAIL"
-
-        result_entry = {
-            "index": i,
-            "question": test["q"],
-            "difficulty": test.get("difficulty", "unknown"),
-            "test_type": test.get("test_type", "fidelity"),
-            "status": status,
-            "missing_cites": check["missing_cites"],
-            "missing_mentions": check["missing_mentions"],
-            "forbidden_found": check["forbidden_found"],
-            "boundary_violations": check["boundary_violations"],
-            "fabricated_cites": check["fabricated_cites"],
-            "response_length": len(response_text),
-        }
-        results.append(result_entry)
+    finally:
+        pool.shutdown(wait=True, cancel_futures=True)
 
-        if check["passed"]:
-            passed += 1
-            if not quiet:
-                print("PASS")
-        else:
-            failed += 1
-            failures = (check["missing_cites"] + check["missing_mentions"]
-                        + check["forbidden_found"] + check["boundary_violations"])
-            if not quiet:
-                print(f"FAIL ({failures})")
+    results = [by_index[i] for i in sorted(by_index)]
 
     return {
-        "master": master_name,
+        **suite_common(master_name, dry_run, "completed", provider),
         "model": model,
         "total": len(tests),
         "passed": passed,
         "failed": failed,
         "pass_rate": f"{passed / len(tests) * 100:.0f}%" if tests else "N/A",
+        "audit": summarize_audit(results),
+        "boundary": summarize_boundary(results),
+        # Measured, not assumed. `cache_read_input_tokens` staying at zero
+        # across a suite is how a silent invalidator announces itself, and a
+        # caching change nobody verified is indistinguishable from no caching.
+        "cache": _cache_summary(cache_stats),
+        "mentions": summarize_mentions(results),
+        "max_output_tokens": max_output_tokens,
+        # Part of the instrument, so it is recorded with the reading: a
+        # concurrent run can meet rate limits a serial one never would, and
+        # those arrive as api_errors that look like nothing else.
+        "concurrency": workers,
+        # Recorded together: a reader comparing two runs needs to know the
+        # per-fixture wall, not just the per-attempt one.
+        "request_timeout": request_timeout,
+        "max_retries": max_retries,
+        "per_fixture_ceiling_s": per_fixture_ceiling(request_timeout, max_retries),
+        # 中断的运行必须能与完整运行区分 —— 否则一份跑了 12/211 的结果读起来
+        # 和跑完的一样,正是本仓一直在修的形状。
+        "interrupted": interrupted,
         "results": results,
     }
 
 
+def summarize_mentions(results: list[dict]) -> dict:
+    """Aggregate how many mention requirements the matcher could actually decide.
+
+    `must_mention` is a substring match, and 56 of the 447 requirements in the
+    2026-08-31 run were graded wrong by it. The ones that cannot be decided are
+    now declared rather than guessed — `must_convey` entries, and every
+    requirement in an answer written in the wrong script. A pass rate resting
+    partly on undecided requirements has to say so, the same way a fabrication
+    count has to carry `audit_coverage`.
+    """
+    total = decided = unverified = mismatches = 0
+    for result in results:
+        required = result.get("mention_requirements", 0)
+        undecidable = result.get("unverified_mentions") or []
+        total += required + len(undecidable)
+        unverified += len(undecidable)
+        if result.get("script_mismatch"):
+            mismatches += 1
+        else:
+            decided += required
+    return {
+        "mention_requirements": total,
+        "mentions_decided": decided,
+        "mentions_unverified": unverified,
+        "script_mismatches": mismatches,
+        "mention_coverage": f"{decided / total * 100:.0f}%" if total else "N/A",
+    }
+
+
+def summarize_boundary(results: list[dict]) -> dict:
+    """How many cases are waiting on a human boundary ruling, and on what.
+
+    A `must_not_contain` hit used to fail the case. Replaying the grader
+    against the 74 hand-adjudicated cases measured that rule at one real
+    violation in seven hits — and it missed the one violation a human found.
+    Hits are surfaced for a ruling now instead, which is only honest if the
+    count is loud: a suite reporting "12/13 passed" while three cases sit
+    undecided is the same silence this repo keeps finding in itself.
+    """
+    pending = [r for r in results if r.get("boundary_undecided")]
+    terms: dict[str, int] = {}
+    for r in pending:
+        for term in r["boundary_undecided"]:
+            terms[term] = terms.get(term, 0) + 1
+    return {
+        "cases_awaiting_ruling": len(pending),
+        "indices": sorted(r["index"] for r in pending),
+        "terms": dict(sorted(terms.items(), key=lambda kv: (-kv[1], kv[0]))),
+    }
+
+
+def summarize_audit(results: list[dict]) -> dict:
+    """Aggregate what the fabrication audit could and could not read.
+
+    A master's "zero fabricated citations" line means nothing on its own: it is
+    equally what you get when every citation was checked and clean, and when
+    none of them could be parsed. ``audit_coverage`` is the share of emitted
+    citations the auditor actually resolved, and it belongs beside any
+    fabrication count that gets reported.
+    """
+    checked = sum(r.get("citations_checked", 0) for r in results)
+    unparsed = sum(len(r.get("unparsed_citations", ())) for r in results)
+    total = checked + unparsed
+    return {
+        "citations_checked": checked,
+        "citations_unparsed": unparsed,
+        "citations_fabricated": sum(
+            len(r.get("fabricated_cites", ())) for r in results
+        ),
+        "audit_coverage": f"{checked / total * 100:.0f}%" if total else "N/A",
+    }
+
+
 def results_failed(results: list[dict], dry_run: bool) -> bool:
     """Return whether collected fidelity results require a failing exit status."""
     if dry_run:
@@ -271,7 +1095,7 @@ def results_failed(results: list[dict], dry_run: bool) -> bool:
         "error" in suite
         or suite.get("failed", 0) > 0
         or any(
-            case.get("status") in {"FAIL", "api_error"}
+            case.get("status") in {"FAIL", "api_error", "truncated"}
             for case in suite.get("results", [])
         )
         for suite in results
@@ -283,7 +1107,16 @@ def main() -> int:
     parser.add_argument("--master", type=str, help="Test a specific master")
     parser.add_argument("--all", action="store_true", help="Test all masters with fidelity.jsonl")
     parser.add_argument("--dry-run", action="store_true", help="Show test cases without calling API")
-    parser.add_argument("--model", type=str, default="claude-sonnet-4-6", help="Claude model to use")
+    parser.add_argument(
+        "--provider", type=str, default=DEFAULT_PROVIDER, choices=sorted(PROVIDERS),
+        help="Which API to grade against (default: anthropic). Non-anthropic "
+             "providers require --model.",
+    )
+    parser.add_argument(
+        "--model", type=str, default=None,
+        help="Model id. Defaults to claude-sonnet-4-6 for --provider anthropic; "
+             "required for every other provider.",
+    )
     parser.add_argument("--json", action="store_true", help="Output as JSON")
     parser.add_argument(
         "--max-tests",
@@ -291,8 +1124,58 @@ def main() -> int:
         default=None,
         help="Cap the number of fixtures per master (smoke runs in CI use 1)",
     )
+    parser.add_argument(
+        "--max-output-tokens",
+        type=int,
+        default=DEFAULT_MAX_OUTPUT_TOKENS,
+        help=(
+            "Output budget per answer (default %(default)s). Reasoning models "
+            "spend this on reasoning before writing: deepseek-v4-pro needs "
+            "~8192 or it stops mid-answer, and master-debate needs 16384 — "
+            "measured 2026-09-13 on deepseek-v4-flash, 3 of its 8 fixtures "
+            "truncated at 8192 on two consecutive runs and 0 of 8 at 16384. "
+            "Its four-round format is simply longer than a single answer. "
+            "A different budget is a different instrument — record it with "
+            "the run, and do not compare across one."
+        ),
+    )
+    parser.add_argument(
+        "--concurrency",
+        type=int,
+        default=DEFAULT_CONCURRENCY,
+        help=(
+            "Fixtures graded in parallel (default %(default)s). Recorded in "
+            "the report: a concurrent run can meet rate limits a serial one "
+            "never would."
+        ),
+    )
+    parser.add_argument(
+        "--request-timeout",
+        type=float,
+        default=DEFAULT_REQUEST_TIMEOUT,
+        help=(
+            "Seconds before one API ATTEMPT is abandoned (default %(default)s). "
+            "With --max-retries the per-fixture wall is timeout x (retries+1)."
+        ),
+    )
+    parser.add_argument(
+        "--max-retries",
+        type=int,
+        default=DEFAULT_MAX_RETRIES,
+        help="SDK retries per fixture (default %(default)s)",
+    )
     args = parser.parse_args()
 
+    if args.concurrency < 1:
+        parser.error("--concurrency must be at least 1")
+
+    if args.max_retries < 0:
+        # A negative count is accepted by the SDKs and would put a negative
+        # "worst-case seconds" into the report.
+        parser.error("--max-retries cannot be negative")
+    if args.request_timeout <= 0:
+        parser.error("--request-timeout must be positive")
+
     if not args.master and not args.all:
         parser.error("Specify --master <name> or --all")
 
@@ -317,18 +1200,38 @@ def main() -> int:
             master,
             dry_run=args.dry_run,
             model=args.model,
+            provider=args.provider,
             max_tests=args.max_tests,
             quiet=args.json,
+            max_output_tokens=args.max_output_tokens,
+            concurrency=args.concurrency,
+            request_timeout=args.request_timeout,
+            max_retries=args.max_retries,
         )
         all_results.append(result)
 
+        if not args.json and "error" in result:
+            # 出错时以前什么都不印:操作者只看到一行标题然后是沉默,读起来像卡住
+            # 而不是失败。付费跑分时错误必须看得见。
+            print(f"ERROR: {result['error']}", file=sys.stderr)
+
         if not args.json and "error" not in result:
             print(f"\nResult: {result.get('passed', 0)}/{result['total']} passed "
                   f"({result.get('pass_rate', 'N/A')})")
+            pending = (result.get("boundary") or {}).get("cases_awaiting_ruling", 0)
+            if pending:
+                # Printed next to the pass rate on purpose: a rate quoted
+                # without this reads as a verdict on cases nobody has ruled on.
+                terms = ", ".join((result["boundary"].get("terms") or {}))
+                print(f"  ⚠ {pending} case(s) await a boundary ruling — {terms}")
 
     if args.json:
         print(json.dumps(all_results, indent=2, ensure_ascii=False))
     elif len(masters) > 1:
+        conflicts = aggregation_conflicts(all_results)
+        for conflict in conflicts:
+            print(f"\n::warning:: {conflict}")
+
         print(f"\n{'='*50}")
         print("Overall Summary:")
         for r in all_results: