master-skill 0.11.0 → 0.12.0

package/scripts/test-fidelity.py

@@ -23,12 +23,19 @@ import json
 import os
 import re
 import sys
+from concurrent.futures import ThreadPoolExecutor, as_completed
 from pathlib import Path
 
 # verify_citations lives in this same scripts/ dir; reused so the
 # `must_cite_only_existing_sources` assertion is actually enforced during graded
 # runs (it was previously schema-validated but never evaluated).
-from verify_citations import audit_answer, load_declared_ids
+from _masterpaths import resolve_master_dir
+from verify_citations import (
+    audit_answer,
+    load_declared_ids,
+    load_member_aliases,
+    load_title_aliases,
+)
 
 PREBUILT_DIR = Path(__file__).resolve().parent.parent / "prebuilt"
 SCHEMA_VERSION = 1
@@ -65,8 +72,66 @@ PROVIDERS: dict[str, dict] = {
     },
 }
 
+# Enough for a non-reasoning model's answer. Reasoning models spend this budget
+# before writing anything — raise it with --max-output-tokens and say so in the
+# report, because a different budget is a different instrument.
+DEFAULT_MAX_OUTPUT_TOKENS = 2048
+
 DEFAULT_PROVIDER = "anthropic"
 
+# Fixtures are independent, so they can be graded in parallel. 4 is chosen to
+# be useful without being a rate-limit generator: the anthropic and openai
+# SDKs both retry 429s with backoff, but a burst wide enough to exhaust that
+# retry budget turns into api_errors that are indistinguishable from real
+# provider failures in the report. Raise it with --concurrency once you know
+# your account's limits.
+DEFAULT_CONCURRENCY = 4
+
+# Per-ATTEMPT ceiling, which is not the same as per-fixture: both SDKs retry,
+# so the wall for one fixture is timeout x (retries + 1).
+#
+# The first version of this set DEFAULT_MAX_RETRIES = 2 — both SDKs' own
+# default — and the commit message claimed "retries are now bounded
+# explicitly". Nothing was bounded. The per-fixture wall stayed at 900s against
+# fidelity-full's 60-minute cap, so four wedged fixtures could still take the
+# sweep; all that changed was that the number became configurable.
+#
+# 180 x (1 + 1) = 360s is a real bound, and it is chosen against the observed
+# distribution rather than picked to look tidy: the 2026-08-18 run averaged
+# ~31s per graded fixture (eval/reports/0.10.1-c697d5d.json), so 180s is nearly
+# six times the mean for a single attempt. One retry still covers the transient
+# 429/529 that retries exist for; a second retry mostly buys latency on calls
+# that were not going to succeed.
+DEFAULT_REQUEST_TIMEOUT = 180.0
+DEFAULT_MAX_RETRIES = 1
+
+
+def per_fixture_ceiling(timeout: float, retries: int) -> float:
+    """Worst-case seconds one fixture can hold, retries included."""
+    return timeout * (retries + 1)
+
+# Anything shaped like a provider credential, stripped before an error string
+# is written to a report. `eval/reports/0.10.1-c697d5d.json` carries 127 raw
+# provider error strings, committed to a public repo — none leaked a key, but
+# nothing was stopping one. SECURITY.md §3 names this exact risk.
+_SECRET_SHAPES = re.compile(
+    r"(sk-[A-Za-z0-9_\-]{12,}"
+    r"|AIza[A-Za-z0-9_\-]{20,}"
+    r"|Bearer\s+[A-Za-z0-9._\-]{12,}"
+    r"|(?i:api[_-]?key)[\"'\s:=]+[A-Za-z0-9._\-]{12,})"
+)
+
+
+def redact_secrets(text: str) -> str:
+    """Blank out credential-shaped substrings before they reach a report.
+
+    Provider exceptions are stringified straight into the results JSON, which
+    is uploaded as a CI artifact and committed under eval/reports/. The error
+    text is worth keeping — it is how the credit-exhaustion run was diagnosed
+    — but it should not be the one place a key could ride out.
+    """
+    return _SECRET_SHAPES.sub("[REDACTED]", text)
+
 
 def resolve_provider(name: str) -> dict:
     """Look up a provider spec, failing with the list of what is available."""
@@ -105,7 +170,22 @@ def build_request(
         return {
             "model": model,
             "max_tokens": max_tokens,
-            "system": system_prompt,
+            # The system prompt is the whole persona — measured at 13,505
+            # characters on average, roughly 6.7k tokens — and it is identical
+            # for every fixture of the same master, of which there are ~11. A
+            # sweep re-sent it 193 times and paid full input price each time.
+            #
+            # Marked explicitly rather than through the top-level
+            # `cache_control` shorthand: that caches the *last* cacheable
+            # block, which here is the per-fixture question — the one part
+            # that changes every call and can never hit.
+            "system": [
+                {
+                    "type": "text",
+                    "text": system_prompt,
+                    "cache_control": {"type": "ephemeral"},
+                }
+            ],
             "messages": [{"role": "user", "content": question}],
         }
     return {
@@ -118,6 +198,44 @@ def build_request(
     }
 
 
+def _cache_summary(stats: dict) -> dict:
+    """What caching actually saved, priced rather than counted.
+
+    A hit *rate* is the wrong figure here: the uncached part of a request is
+    just the question, a few dozen tokens against a ~6.7k system prompt, so any
+    ratio built on it reads ~100% whether caching worked or not — the first
+    version of this did exactly that and looked identical with caching off.
+    What matters is input spend against the no-cache baseline, at the published
+    multipliers (reads 0.1x, writes 1.25x).
+    """
+    read, created, uncached = stats["read"], stats["created"], stats["uncached"]
+    baseline = read + created + uncached
+    if not baseline:
+        return {**stats, "input_tokens_saved": "N/A"}
+    actual = read * 0.1 + created * 1.25 + uncached
+    return {
+        **stats,
+        "baseline_input_tokens": baseline,
+        "effective_input_tokens": round(actual),
+        "input_tokens_saved": f"{(1 - actual / baseline) * 100:.0f}%",
+    }
+
+
+def _record_cache_usage(stats: dict, response: object) -> None:
+    """Accumulate prompt-cache token counts when the provider reports them.
+
+    Only the Anthropic responses carry these fields; the OpenAI-compatible
+    hosts cache automatically and report nothing comparable, so their suites
+    show N/A rather than a fabricated zero.
+    """
+    usage = getattr(response, "usage", None)
+    if usage is None:
+        return
+    stats["created"] += getattr(usage, "cache_creation_input_tokens", 0) or 0
+    stats["read"] += getattr(usage, "cache_read_input_tokens", 0) or 0
+    stats["uncached"] += getattr(usage, "input_tokens", 0) or 0
+
+
 def extract_text(provider: str, response: object) -> str:
     """Pull the answer text out of this provider's response object.
 
@@ -142,6 +260,47 @@ def extract_text(provider: str, response: object) -> str:
     return content
 
 
+def extract_finish_reason(provider: str, response: object) -> str | None:
+    """Why the model stopped, in one vocabulary across providers.
+
+    Anthropic says ``stop_reason: max_tokens``; the OpenAI-compatible hosts say
+    ``finish_reason: length``. Both mean the answer was cut off mid-sentence,
+    which is an instrument condition and not something a persona did.
+    """
+    spec = resolve_provider(provider)
+    if spec["api"] == "anthropic":
+        reason = getattr(response, "stop_reason", None)
+        return "length" if reason == "max_tokens" else reason
+    choices = getattr(response, "choices", None) or []
+    if not choices:
+        return None
+    return getattr(choices[0], "finish_reason", None)
+
+
+def truncated_result_entry(
+    index: int, test: dict, response_text: str, max_output_tokens: int
+) -> dict:
+    """Record a cut-off answer as unmeasured, the way an API error is recorded.
+
+    Reasoning models spend the output budget before writing anything: measured
+    on `deepseek-v4-pro` at the harness's old hardcoded 2048, reasoning took
+    1,860 tokens and left 250 characters of answer — and on some questions,
+    none at all. Scoring that produces `missing_cites` / `must_mention`
+    failures that describe the budget, not the prompt. Carries no check fields
+    at all, so nothing downstream can mistake it for a graded verdict.
+    """
+    return {
+        "index": index,
+        "question": test["q"],
+        "difficulty": test.get("difficulty", "unknown"),
+        "test_type": test.get("test_type", "fidelity"),
+        "status": "truncated",
+        "max_output_tokens": max_output_tokens,
+        "response": response_text,
+        "response_length": len(response_text),
+    }
+
+
 def aggregation_conflicts(suites: list[dict]) -> list[str]:
     """Report why a set of suites must not be pooled into one number.
 
@@ -269,27 +428,109 @@ def _split_echoes(
     return found, echoed
 
 
+_CONTEXT_WINDOW = 30
+_CONTEXT_MAX_HITS = 3
+
+
+def _hit_context(
+    terms: list[str], response: str, window: int = _CONTEXT_WINDOW
+) -> dict[str, list[str]]:
+    """Record the text around every forbidden-term hit.
+
+    A ``must_not_contain`` hit is a substring match, and the substring alone
+    never says which of three things happened: the persona did the forbidden
+    thing, refused it in so many words ("天台止观之正意，不在求神通"), or the
+    term matched across a word boundary (胜于 inside 殊胜于何). Adjudicating
+    the 2026-08-31 sweep needed the answer text for all seven hits in the run,
+    six of which turned out not to be violations at all.
+
+    Snippets are bounded and capped so a report carries evidence rather than a
+    second copy of the answer.
+    """
+    context: dict[str, list[str]] = {}
+    for term in terms:
+        if not term:
+            # An empty term matches at every offset and advances nothing.
+            continue
+        snippets: list[str] = []
+        start = 0
+        while len(snippets) < _CONTEXT_MAX_HITS:
+            hit = response.find(term, start)
+            if hit == -1:
+                break
+            left = max(0, hit - window)
+            right = min(len(response), hit + len(term) + window)
+            snippets.append(response[left:right])
+            start = hit + len(term)
+        if snippets:
+            context[term] = snippets
+    return context
+
+
+# 繁简失明。211 条回答里有 3 条整篇用繁体,而所有夹具关键词都是简体 ——
+# master-ouyi 写了六次「信願」,却被判成「从未提到信愿」。这是唯一一类
+# 「那个词确实必须原样出现」的要求:它出现了,只是换了字形。
+#
+# 不做转换,只做识别。转换需要一张简繁映射表,而简→繁是一对多(干→干/乾/幹),
+# 用它去放宽匹配会把「乾闼婆」读成命中「干」。识别只需数字形:引文块与书名号
+# 里的经名本来就是繁体(简体回答里也照样写《六祖大師法寶壇經》),必须先剔除,
+# 否则虚云那些简体回答会被误判。
+_TRAD_PAIRS = (
+    ("與", "与"), ("為", "为"), ("觀", "观"), ("證", "证"), ("實", "实"),
+    ("學", "学"), ("說", "说"), ("識", "识"), ("體", "体"), ("這", "这"),
+    ("個", "个"), ("們", "们"), ("問", "问"), ("義", "义"), ("華", "华"),
+    ("嚴", "严"), ("論", "论"), ("聽", "听"), ("願", "愿"), ("剛", "刚"),
+    ("經", "经"), ("淨", "净"), ("須", "须"), ("關", "关"), ("係", "系"),
+    ("無", "无"), ("會", "会"), ("處", "处"), ("諸", "诸"), ("種", "种"),
+    ("現", "现"), ("復", "复"), ("時", "时"), ("開", "开"), ("眾", "众"),
+)
+_TRAD_MIN_HITS = 3
+_STRIP_FOR_SCRIPT = re.compile(r"【[^】]*】|《[^》]*》")
+_SIMP_TO_TRAD = {simp: trad for trad, simp in _TRAD_PAIRS}
+
+
+def _is_traditional(response: str) -> bool:
+    """回答正文是否整体用繁体书写(不含引文块与经名)。"""
+    body = _STRIP_FOR_SCRIPT.sub("", response)
+    trad = sum(body.count(t) for t, _ in _TRAD_PAIRS)
+    simp = sum(body.count(s) for _, s in _TRAD_PAIRS)
+    return trad >= _TRAD_MIN_HITS and trad > simp
+
+
+def _traditional_form(term: str) -> str | None:
+    """把 must_mention 词按已知简→繁字对逐字换成繁体形。换不出来
+    (没有一个字在映射表里)就返回 None —— 不冒充「已核对过」。"""
+    converted = "".join(_SIMP_TO_TRAD.get(ch, ch) for ch in term)
+    return converted if converted != term else None
+
+
 def check_response(
     response: str,
     test_case: dict,
     is_first_turn: bool = True,
     declared_ids: set[str] | None = None,
+    member_aliases: dict[str, str] | None = None,
+    title_aliases: dict[str, str] | None = None,
 ) -> dict:
     """Check a response against expected citations, mentions, and boundaries.
 
     Returns {passed: bool, missing_cites: [...], missing_mentions: [...],
              forbidden_found: [...], forbidden_echoed: [...],
              boundary_violations: [...], boundary_echoed: [...],
-             needs_review: bool, fabricated_cites: [...]}.
+             needs_review: bool, fabricated_cites: [...],
+             audit_unavailable: bool}.
 
     ``*_echoed`` holds forbidden terms that the fixture's own question already
     contains — see ``_split_echoes``. They do not fail the case; they set
     ``needs_review`` so a human can look at the stored response and decide.
 
-    When the test sets ``must_cite_only_existing_sources`` and ``declared_ids``
-    is supplied, every citation in the response must be either a declared
-    offline source or carry a real ``fojin.app/texts/{id}`` link (B1 rule);
-    anything else is a fabricated citation and fails the case.
+    Whenever ``declared_ids`` is supplied, every citation in the response must
+    be either a declared offline source or carry a real ``fojin.app/texts/{id}``
+    link (B1 rule); anything else is a fabricated citation and fails the case.
+    The audit is unconditional — a fixture does not get to opt out of it. When
+    ``declared_ids`` is None and the response nonetheless carries checkable
+    source ids, ``audit_unavailable`` is set and the case needs review rather
+    than passing as audited-and-clean.
     """
     missing_cites = []
     for cite in test_case.get("must_cite", []):
@@ -301,6 +542,31 @@ def check_response(
         if mention not in response:
             missing_mentions.append(mention)
 
+    # must_convey:量具判不了的要求。逐条裁定 2026-08-31 全量跑发现,447 条
+    # must_mention 里有 56 条判错,其中 54 条根本不是字串问题 —— 夹具要「方便」
+    # 而回答写「应病与药」,要「不是虚无」而回答写「空非虚无」。列同义词表等于
+    # 拿一个模型的输出去反向拟合夹具。这里改用本仓库已有的那条原则:量具不得
+    # 宣称它判过它判不了的东西(见 audit_unavailable / unparsed_citations)。
+    # 故这些既不判过也不判失败,只记为待裁决。
+    unverified_mentions = list(test_case.get("must_convey", []))
+
+    # 繁简失明,逐词判定 —— 不是整案判定。回答整篇繁体时,missing_mentions 里
+    # 每一个词单独换算成繁体形、逐个在原文里找:找到了,从缺词表里摘掉(这个词
+    # 确实说了,只是字形不同);换不出繁体形、或换出来仍找不到,原样留在
+    # missing_mentions 里继续判失败。早期版本按「回答是否整篇繁体」一刀切豁免
+    # 整个缺词列表,结果 mahasi-#6 式的场景(一词因字形被冤枉、另一词哪种字形
+    # 都真的没提)会被那一词的豁免连带放行——2026-09-03 改为逐词。
+    script_mismatch = False
+    if missing_mentions and _is_traditional(response):
+        still_missing = []
+        for term in missing_mentions:
+            trad_form = _traditional_form(term)
+            if trad_form and trad_form in response:
+                script_mismatch = True
+            else:
+                still_missing.append(term)
+        missing_mentions = still_missing
+
     question = test_case.get("q", "")
 
     # Boundary tests: must_not_contain
@@ -316,16 +582,62 @@ def check_response(
             test_case.get("must_not_contain_first_turn", []), response, question
         )
 
-    # B1: must_cite_only_existing_sources — no hallucinated citations
-    fabricated_cites = []
-    if test_case.get("must_cite_only_existing_sources") and declared_ids is not None:
-        fabricated_cites = audit_answer(declared_ids, response)["fabricated"]
+    # 每一条命中都带上原文上下文。命中的字符串本身分不清「真越界」「明确否定」
+    # 与「跨词边界误配」——2026-08-31 全量跑出的 7 条 forbidden_found 里,6 条
+    # 属后两类,而判定它们需要回头翻回答原文。证据存进报告,失败才可事后裁定。
+    forbidden_context = _hit_context(
+        forbidden_found + forbidden_echoed + boundary_violations + boundary_echoed,
+        response,
+    )
 
+    # B1: 伪造引用审计。2026-08-31 之前这是逐条夹具选配的(211 条里 7 条开启),
+    # 而开启它的 master-curriculum 没有 meta.json —— load_declared_ids 抛异常、
+    # declared_ids 变 None、`and declared_ids is not None` 守卫短路。结果是首份
+    # 基线里这项审计一次都没有真正运行,84 条结果却全部写着 fabricated_cites: []。
+    #
+    # 现在无条件运行:拿得到声明来源就查每一条回答;拿不到、而回答里确实带了可核对
+    # 的来源 id,就记为「待裁决」而不是静默放行 —— 检查了空集合的门禁不该报绿。
+    # 空的声明集与 None 同样「查不了」:拿空集合当标尺,会把每一条**正确**引用
+    # 都判成伪造(master-debate 的 meta.json 里 sources 就是空的)。
+    fabricated_cites = []
+    audit_unavailable = False
+    if declared_ids:
+        audit = audit_answer(
+            declared_ids, response, member_aliases, title_aliases
+        )
+        fabricated_cites = audit["fabricated"]
+        unparsed_citations = audit["unparsed"]
+        citations_checked = (
+            len(audit["offline"]) + len(audit["live"]) + len(audit["fabricated"])
+        )
+    else:
+        probe = audit_answer(set(), response)
+        audit_unavailable = bool(probe["fabricated"] or probe["live"])
+        unparsed_citations = probe["unparsed"]
+        citations_checked = 0
+
+    # missing_mentions 已经在上面逐词过滤掉繁体命中的部分,不需要再靠
+    # script_mismatch 整案豁免 —— 这里是普通的"缺词表是否为空"。
+    #
+    # forbidden_found / boundary_violations 不再进这个连乘。`_hit_context` 的
+    # docstring 早就把道理写全了:一次子串命中「never says which of three
+    # things happened」—— 真违规、用语言拒绝、跨词边界误配。`_split_echoes`
+    # 只覆盖了其中一种(题干回声),另外两种仍在自动判失败。
+    #
+    # 实测精度(2026-09-12,对 74 条人工裁定回放):
+    #     命中 7 次 → 真违规 1,误判 6  → 精度 14%
+    #     而人工复核找到的那 1 条真违规,护栏检查根本没命中。
+    # 被误判的六条全是同一形状 —— 祖师**正确地拒绝了**,而匹配把拒绝算成违规:
+    #     「相见不必待某日」「若执中观定胜于唯识,此执正是戏论」
+    #     「若有人预言某年某月可得证悟,此非正法所许」「不在求神通」
+    # 一个系统性惩罚「正确拒绝」的护栏,比没有护栏更坏。
+    #
+    # 所以命中改为**转人工裁决**而不是自动判失败 —— 与 `must_convey` 和
+    # `_split_echoes` 同一条原则:量具不得宣称它判过它判不了的东西。它仍然
+    # 逐条捞出同样的候选,`forbidden_context` 已经备好裁定所需的原文。
     passed = (
         len(missing_cites) == 0
         and len(missing_mentions) == 0
-        and len(forbidden_found) == 0
-        and len(boundary_violations) == 0
         and len(fabricated_cites) == 0
     )
 
@@ -337,8 +649,27 @@ def check_response(
         "forbidden_echoed": forbidden_echoed,
         "boundary_violations": boundary_violations,
         "boundary_echoed": boundary_echoed,
-        "needs_review": bool(forbidden_echoed or boundary_echoed),
+        "forbidden_context": forbidden_context,
+        "unverified_mentions": unverified_mentions,
+        "script_mismatch": script_mismatch,
+        "needs_review": bool(
+            forbidden_found
+            or boundary_violations
+            or forbidden_echoed
+            or boundary_echoed
+            or audit_unavailable
+            or unverified_mentions
+            or script_mismatch
+        ),
+        # 单列一项,好让报告数得出「多少条在等边界裁决」。合进 needs_review
+        # 会把它和繁体、审计不可用等混在一起,而那些不需要人看原文。
+        "boundary_undecided": sorted(set(forbidden_found) | set(boundary_violations)),
         "fabricated_cites": fabricated_cites,
+        "audit_unavailable": audit_unavailable,
+        # 抽不出可核对 id 的引文块。不判失败 —— 但空的 fabricated 从此不再等于
+        # 「查过、干净」,报告可以算出审计器实际覆盖了多少条引用。
+        "unparsed_citations": unparsed_citations,
+        "citations_checked": citations_checked,
     }
 
 
@@ -362,10 +693,18 @@ def result_entry(
         "missing_mentions": check["missing_mentions"],
         "forbidden_found": check["forbidden_found"],
         "forbidden_echoed": check["forbidden_echoed"],
+        "boundary_undecided": check["boundary_undecided"],
         "boundary_violations": check["boundary_violations"],
         "boundary_echoed": check["boundary_echoed"],
+        "forbidden_context": check["forbidden_context"],
+        "unverified_mentions": check["unverified_mentions"],
+        "mention_requirements": len(test.get("must_mention", [])),
+        "script_mismatch": check["script_mismatch"],
         "fabricated_cites": check["fabricated_cites"],
         "needs_review": check["needs_review"],
+        "audit_unavailable": check["audit_unavailable"],
+        "unparsed_citations": check["unparsed_citations"],
+        "citations_checked": check["citations_checked"],
         "response": response_text,
         "response_length": len(response_text),
     }
@@ -378,13 +717,21 @@ def run_tests(
     max_tests: int | None = None,
     quiet: bool = False,
     provider: str = DEFAULT_PROVIDER,
+    max_output_tokens: int = DEFAULT_MAX_OUTPUT_TOKENS,
+    concurrency: int = DEFAULT_CONCURRENCY,
+    request_timeout: float = DEFAULT_REQUEST_TIMEOUT,
+    max_retries: int = DEFAULT_MAX_RETRIES,
 ) -> dict:
     """Run fidelity tests for a master. Returns summary."""
-    master_dir = PREBUILT_DIR / master_name
-    if not master_dir.exists():
+    # 目录叫 `master-<slug>`,而公开写在 README / package.json 里的调用形式是短名
+    # (`--master yinguang`)。`_masterpaths.resolve_master_dir` 正是为此存在,别的
+    # 脚本都改用了,这个运行器没有 —— `npm run test:smoke` 因此从未跑通。
+    resolved = resolve_master_dir(master_name, base=str(PREBUILT_DIR))
+    if resolved is None:
         return suite_error(
             master_name, dry_run, f"Master '{master_name}' not found", provider
         )
+    master_dir = Path(resolved)
 
     try:
         tests = load_tests(master_dir)
@@ -449,8 +796,10 @@ def run_tests(
                 "anthropic package not installed. Run: pip install anthropic",
                 provider,
             )
-        client = anthropic.Anthropic(api_key=api_key)
-        send = lambda body: client.messages.create(**body)  # noqa: E731
+        client = anthropic.Anthropic(api_key=api_key, max_retries=max_retries)
+        send = lambda body: client.messages.create(  # noqa: E731
+            **body, timeout=request_timeout
+        )
     else:
         try:
             import openai
@@ -460,54 +809,179 @@ def run_tests(
                 "openai package not installed. Run: pip install openai",
                 provider,
             )
-        client = openai.OpenAI(api_key=api_key, base_url=spec["base_url"])
-        send = lambda body: client.chat.completions.create(**body)  # noqa: E731
+        client = openai.OpenAI(
+            api_key=api_key, base_url=spec["base_url"], max_retries=max_retries
+        )
+        send = lambda body: client.chat.completions.create(  # noqa: E731
+            **body, timeout=request_timeout
+        )
 
     # Declared offline sources, for the must_cite_only_existing_sources B1 check.
     try:
         declared_ids = load_declared_ids(master_name)
+        member_aliases = load_member_aliases(master_name)
+        title_aliases = load_title_aliases(master_name)
     except (ValueError, FileNotFoundError):
         declared_ids = None
+        member_aliases = None
+        title_aliases = None
 
-    passed = 0
-    failed = 0
-
-    for i, test in enumerate(tests):
-        if not quiet:
-            print(f"  [{i+1}/{len(tests)}] {test['q'][:50]}...", end=" ", flush=True)
+    def grade_one(i: int, test: dict) -> tuple[dict, bool, str]:
+        """Run and grade one fixture. Pure w.r.t. the enclosing suite state.
 
+        Returns (result entry, counted-as-passed, one-word label for the log).
+        Every failure mode is a returned value, not a raised exception, so one
+        bad fixture cannot take the pool down with it.
+        """
         try:
             response = send(
-                build_request(provider, model, system_prompt, test["q"], 2048)
+                build_request(
+                    provider, model, system_prompt, test["q"], max_output_tokens
+                )
             )
             response_text = extract_text(provider, response)
-        except Exception as e:
-            results.append({
-                "index": i,
-                "question": test["q"],
-                "status": "api_error",
-                "error": str(e),
-            })
-            failed += 1
-            if not quiet:
-                print("API ERROR")
-            continue
+            finish_reason = extract_finish_reason(provider, response)
+            _record_cache_usage(cache_stats, response)
+        except Exception as e:  # noqa: BLE001 — provider errors are data here
+            return (
+                {
+                    "index": i,
+                    "question": test["q"],
+                    "status": "api_error",
+                    "error": redact_secrets(str(e)),
+                },
+                False,
+                "API ERROR",
+            )
 
-        check = check_response(
-            response_text, test, is_first_turn=True, declared_ids=declared_ids
-        )
-        results.append(result_entry(i, test, check, response_text))
+        if finish_reason == "length":
+            # Cut off mid-answer: unmeasured, not failed. Counted with the
+            # api_errors so a run full of them cannot read as a clean result.
+            return (
+                truncated_result_entry(i, test, response_text, max_output_tokens),
+                False,
+                "TRUNCATED",
+            )
 
+        try:
+            check = check_response(
+                response_text,
+                test,
+                is_first_turn=True,
+                declared_ids=declared_ids,
+                member_aliases=member_aliases,
+                title_aliases=title_aliases,
+            )
+        except Exception as e:  # noqa: BLE001 — 判分器崩溃也是数据,不是终止条件
+            # grade_one 的 docstring 承诺「一条坏 fixture 不会掀翻整个池」,
+            # 而 check_response 原本在这个 try 之外:一次判分器异常会经
+            # future.result() 在主线程重新抛出,在**每一次 API 调用都已付过钱之后**
+            # 终止 run_tests。承诺现在是真的。
+            return (
+                {
+                    "index": i,
+                    "question": test["q"],
+                    "status": "grader_error",
+                    "error": redact_secrets(f"{type(e).__name__}: {e}"),
+                    "response": response_text,
+                },
+                False,
+                "GRADER ERROR",
+            )
+        entry = result_entry(i, test, check, response_text)
         if check["passed"]:
-            passed += 1
-            if not quiet:
-                print("PASS (review)" if check["needs_review"] else "PASS")
-        else:
-            failed += 1
-            failures = (check["missing_cites"] + check["missing_mentions"]
-                        + check["forbidden_found"] + check["boundary_violations"])
-            if not quiet:
-                print(f"FAIL ({failures})")
+            return entry, True, "PASS (review)" if check["needs_review"] else "PASS"
+        failures = (check["missing_cites"] + check["missing_mentions"]
+                    + check["forbidden_found"] + check["boundary_violations"])
+        return entry, False, f"FAIL ({failures})"
+
+    # Fixtures are independent — each is one stateless request graded against
+    # its own expectations — so the only reason this ran serially was that
+    # nobody changed it. It cost 44 minutes to grade 84 fixtures on
+    # 2026-08-18 (eval/reports/0.10.1-c697d5d.json), ~31s apiece, and a full
+    # 211-fixture DeepSeek sweep took 1h55m. That is the difference between
+    # a sweep you run after a change and one you run twice a year.
+    #
+    # Ordering is restored by index afterwards: reports are diffed across
+    # runs, so completion order must not leak into the output.
+    workers = max(1, min(concurrency, len(tests)))
+    by_index: dict[int, dict] = {}
+    passed = 0
+    failed = 0
+    done = 0
+    cache_stats = {"created": 0, "read": 0, "uncached": 0}
+
+    # NOT `with ThreadPoolExecutor(...)`. Its `__exit__` calls
+    # `shutdown(wait=True)` without `cancel_futures`, and every fixture is
+    # submitted up front — so Ctrl-C during a paid sweep did not stop it. The
+    # worker loop drained all 211 queued calls, billed every one, and only then
+    # let the interrupt surface, discarding the verdicts already paid for. The
+    # serial loop it replaced stopped at the next iteration. An operator who
+    # sees the first few verdicts are wrong (bad model id, broken persona edit)
+    # has to be able to stop.
+    # The first fixture runs alone. Every request in a suite shares one system
+    # prompt, so the first call is the one that writes the cache — firing four
+    # at once means three of them start before the entry exists and pay full
+    # price. One serial call up front turns ~4 misses per suite into 1.
+    pool = ThreadPoolExecutor(max_workers=workers)
+    interrupted = False
+    warmed = 0
+    try:
+        # Inside the same interrupt handling as the pool. The first version put
+        # this above the `try` and a Ctrl-C during the warm-up escaped instead
+        # of degrading — the exact behaviour the interrupt work exists to give.
+        try:
+            if workers > 1 and len(tests) > 1:
+                entry, ok, label = grade_one(0, tests[0])
+                by_index[0] = entry
+                passed += ok
+                failed += not ok
+                warmed = 1
+                if not quiet:
+                    done += 1
+                    print(
+                        f"  [{done}/{len(tests)}] #1 {tests[0]['q'][:50]}... {label}",
+                        flush=True,
+                    )
+        except KeyboardInterrupt:
+            interrupted = True
+
+        remaining = [] if interrupted else list(enumerate(tests))[warmed:]
+        futures = {pool.submit(grade_one, i, t): i for i, t in remaining}
+        try:
+            for future in as_completed(futures):
+                i = futures[future]
+                entry, ok, label = future.result()
+                by_index[i] = entry
+                if ok:
+                    passed += 1
+                else:
+                    failed += 1
+                if not quiet:
+                    # No lock: `as_completed` yields on the calling thread, so
+                    # this counter and this print are single-threaded. The lock
+                    # that used to be here guarded nothing and implied the
+                    # opposite.
+                    done += 1
+                    # One whole line per fixture. The old "print the prompt,
+                    # then the verdict on the same line" shape interleaves
+                    # into nonsense the moment more than one call is open.
+                    print(
+                        f"  [{done}/{len(tests)}] #{i + 1} "
+                        f"{tests[i]['q'][:50]}... {label}",
+                        flush=True,
+                    )
+        except KeyboardInterrupt:
+            interrupted = True
+            print(
+                f"\n中断:已完成 {len(by_index)}/{len(tests)} 条,"
+                "取消其余未发出的调用。",
+                file=sys.stderr,
+            )
+    finally:
+        pool.shutdown(wait=True, cancel_futures=True)
+
+    results = [by_index[i] for i in sorted(by_index)]
 
     return {
         **suite_common(master_name, dry_run, "completed", provider),
@@ -516,10 +990,103 @@ def run_tests(
         "passed": passed,
         "failed": failed,
         "pass_rate": f"{passed / len(tests) * 100:.0f}%" if tests else "N/A",
+        "audit": summarize_audit(results),
+        "boundary": summarize_boundary(results),
+        # Measured, not assumed. `cache_read_input_tokens` staying at zero
+        # across a suite is how a silent invalidator announces itself, and a
+        # caching change nobody verified is indistinguishable from no caching.
+        "cache": _cache_summary(cache_stats),
+        "mentions": summarize_mentions(results),
+        "max_output_tokens": max_output_tokens,
+        # Part of the instrument, so it is recorded with the reading: a
+        # concurrent run can meet rate limits a serial one never would, and
+        # those arrive as api_errors that look like nothing else.
+        "concurrency": workers,
+        # Recorded together: a reader comparing two runs needs to know the
+        # per-fixture wall, not just the per-attempt one.
+        "request_timeout": request_timeout,
+        "max_retries": max_retries,
+        "per_fixture_ceiling_s": per_fixture_ceiling(request_timeout, max_retries),
+        # 中断的运行必须能与完整运行区分 —— 否则一份跑了 12/211 的结果读起来
+        # 和跑完的一样,正是本仓一直在修的形状。
+        "interrupted": interrupted,
         "results": results,
     }
 
 
+def summarize_mentions(results: list[dict]) -> dict:
+    """Aggregate how many mention requirements the matcher could actually decide.
+
+    `must_mention` is a substring match, and 56 of the 447 requirements in the
+    2026-08-31 run were graded wrong by it. The ones that cannot be decided are
+    now declared rather than guessed — `must_convey` entries, and every
+    requirement in an answer written in the wrong script. A pass rate resting
+    partly on undecided requirements has to say so, the same way a fabrication
+    count has to carry `audit_coverage`.
+    """
+    total = decided = unverified = mismatches = 0
+    for result in results:
+        required = result.get("mention_requirements", 0)
+        undecidable = result.get("unverified_mentions") or []
+        total += required + len(undecidable)
+        unverified += len(undecidable)
+        if result.get("script_mismatch"):
+            mismatches += 1
+        else:
+            decided += required
+    return {
+        "mention_requirements": total,
+        "mentions_decided": decided,
+        "mentions_unverified": unverified,
+        "script_mismatches": mismatches,
+        "mention_coverage": f"{decided / total * 100:.0f}%" if total else "N/A",
+    }
+
+
+def summarize_boundary(results: list[dict]) -> dict:
+    """How many cases are waiting on a human boundary ruling, and on what.
+
+    A `must_not_contain` hit used to fail the case. Replaying the grader
+    against the 74 hand-adjudicated cases measured that rule at one real
+    violation in seven hits — and it missed the one violation a human found.
+    Hits are surfaced for a ruling now instead, which is only honest if the
+    count is loud: a suite reporting "12/13 passed" while three cases sit
+    undecided is the same silence this repo keeps finding in itself.
+    """
+    pending = [r for r in results if r.get("boundary_undecided")]
+    terms: dict[str, int] = {}
+    for r in pending:
+        for term in r["boundary_undecided"]:
+            terms[term] = terms.get(term, 0) + 1
+    return {
+        "cases_awaiting_ruling": len(pending),
+        "indices": sorted(r["index"] for r in pending),
+        "terms": dict(sorted(terms.items(), key=lambda kv: (-kv[1], kv[0]))),
+    }
+
+
+def summarize_audit(results: list[dict]) -> dict:
+    """Aggregate what the fabrication audit could and could not read.
+
+    A master's "zero fabricated citations" line means nothing on its own: it is
+    equally what you get when every citation was checked and clean, and when
+    none of them could be parsed. ``audit_coverage`` is the share of emitted
+    citations the auditor actually resolved, and it belongs beside any
+    fabrication count that gets reported.
+    """
+    checked = sum(r.get("citations_checked", 0) for r in results)
+    unparsed = sum(len(r.get("unparsed_citations", ())) for r in results)
+    total = checked + unparsed
+    return {
+        "citations_checked": checked,
+        "citations_unparsed": unparsed,
+        "citations_fabricated": sum(
+            len(r.get("fabricated_cites", ())) for r in results
+        ),
+        "audit_coverage": f"{checked / total * 100:.0f}%" if total else "N/A",
+    }
+
+
 def results_failed(results: list[dict], dry_run: bool) -> bool:
     """Return whether collected fidelity results require a failing exit status."""
     if dry_run:
@@ -528,7 +1095,7 @@ def results_failed(results: list[dict], dry_run: bool) -> bool:
         "error" in suite
         or suite.get("failed", 0) > 0
         or any(
-            case.get("status") in {"FAIL", "api_error"}
+            case.get("status") in {"FAIL", "api_error", "truncated"}
             for case in suite.get("results", [])
         )
         for suite in results
@@ -557,8 +1124,58 @@ def main() -> int:
         default=None,
         help="Cap the number of fixtures per master (smoke runs in CI use 1)",
     )
+    parser.add_argument(
+        "--max-output-tokens",
+        type=int,
+        default=DEFAULT_MAX_OUTPUT_TOKENS,
+        help=(
+            "Output budget per answer (default %(default)s). Reasoning models "
+            "spend this on reasoning before writing: deepseek-v4-pro needs "
+            "~8192 or it stops mid-answer, and master-debate needs 16384 — "
+            "measured 2026-09-13 on deepseek-v4-flash, 3 of its 8 fixtures "
+            "truncated at 8192 on two consecutive runs and 0 of 8 at 16384. "
+            "Its four-round format is simply longer than a single answer. "
+            "A different budget is a different instrument — record it with "
+            "the run, and do not compare across one."
+        ),
+    )
+    parser.add_argument(
+        "--concurrency",
+        type=int,
+        default=DEFAULT_CONCURRENCY,
+        help=(
+            "Fixtures graded in parallel (default %(default)s). Recorded in "
+            "the report: a concurrent run can meet rate limits a serial one "
+            "never would."
+        ),
+    )
+    parser.add_argument(
+        "--request-timeout",
+        type=float,
+        default=DEFAULT_REQUEST_TIMEOUT,
+        help=(
+            "Seconds before one API ATTEMPT is abandoned (default %(default)s). "
+            "With --max-retries the per-fixture wall is timeout x (retries+1)."
+        ),
+    )
+    parser.add_argument(
+        "--max-retries",
+        type=int,
+        default=DEFAULT_MAX_RETRIES,
+        help="SDK retries per fixture (default %(default)s)",
+    )
     args = parser.parse_args()
 
+    if args.concurrency < 1:
+        parser.error("--concurrency must be at least 1")
+
+    if args.max_retries < 0:
+        # A negative count is accepted by the SDKs and would put a negative
+        # "worst-case seconds" into the report.
+        parser.error("--max-retries cannot be negative")
+    if args.request_timeout <= 0:
+        parser.error("--request-timeout must be positive")
+
     if not args.master and not args.all:
         parser.error("Specify --master <name> or --all")
 
@@ -586,12 +1203,27 @@ def main() -> int:
             provider=args.provider,
             max_tests=args.max_tests,
             quiet=args.json,
+            max_output_tokens=args.max_output_tokens,
+            concurrency=args.concurrency,
+            request_timeout=args.request_timeout,
+            max_retries=args.max_retries,
         )
         all_results.append(result)
 
+        if not args.json and "error" in result:
+            # 出错时以前什么都不印:操作者只看到一行标题然后是沉默,读起来像卡住
+            # 而不是失败。付费跑分时错误必须看得见。
+            print(f"ERROR: {result['error']}", file=sys.stderr)
+
         if not args.json and "error" not in result:
             print(f"\nResult: {result.get('passed', 0)}/{result['total']} passed "
                   f"({result.get('pass_rate', 'N/A')})")
+            pending = (result.get("boundary") or {}).get("cases_awaiting_ruling", 0)
+            if pending:
+                # Printed next to the pass rate on purpose: a rate quoted
+                # without this reads as a verdict on cases nobody has ruled on.
+                terms = ", ".join((result["boundary"].get("terms") or {}))
+                print(f"  ⚠ {pending} case(s) await a boundary ruling — {terms}")
 
     if args.json:
         print(json.dumps(all_results, indent=2, ensure_ascii=False))