eduevidence 5.2.0

package/scripts/evidence_score.py

@@ -0,0 +1,234 @@
+#!/usr/bin/env python3
+"""evidence_score.py — Deterministic quality scoring and confidence computation.
+
+Two rule-based engines:
+
+1. Quality score (five dimensions, each 0-2, total 0-10):
+   D1 Study Design, D2 Sample Quality, D3 Measurement Validity,
+   D4 Temporal Strength, D5 Directness.
+   Level mapping (references/evidence-quality.md): 8-10 strong, 5-7 moderate,
+   2-4 weak, 0-1 very_weak.
+
+2. Confidence (rule-based, NOT model-generated):
+   Evidence Quality + Consistency + Directness + Evidence Count
+   - Conflict Penalty - Unsupported Penalty
+   -> High | Moderate | Low | Insufficient
+   v2 policy (2026-08-12.v2, D-1/D-2): consistency is computed over
+   decision_relation (support_adoption/oppose_adoption/conditional/neutral)
+   instead of relation_to_claim, and the evidence count term is
+   min(1.0, independent_studies / 4) — independent_samples is reported
+   separately instead of being added to the count term.
+
+The authoritative pipeline entry for final verdicts is
+scripts/compute_confidence.py (independent-studies/samples-weighted formula
+with confidence_policy_version); the `confidence()` function below remains the
+backward-compatible engine used by example reproducibility checks.
+
+IMPORTANT: the confidence score is a rule-based index in [0, 1], NOT a
+probability. Never present it as "85% confidence" or any probabilistic claim.
+"""
+from __future__ import annotations
+
+from typing import Any
+
+from evidence_semantics import claim_relation, decision_relation
+
+DIMENSIONS = ["D1_study_design", "D2_sample_quality", "D3_measurement_validity",
+              "D4_temporal_strength", "D5_directness"]
+
+#: Version of the deterministic confidence policy (bump on any formula change).
+CONFIDENCE_POLICY_VERSION = "2026-08-12.v2"
+
+
+def quality_score(dimensions: dict[str, int]) -> float:
+    """Sum the five 0-2 dimension scores into a 0-10 quality score."""
+    total = 0.0
+    for dim in DIMENSIONS:
+        val = dimensions.get(dim, 0)
+        if not isinstance(val, (int, float)) or isinstance(val, bool):
+            raise ValueError(f"dimension {dim} must be numeric, got {val!r}")
+        total += max(0.0, min(2.0, float(val)))
+    return total
+
+
+def quality_level(score: float) -> str:
+    """Map a 0-10 quality score to a level (references/evidence-quality.md)."""
+    if score >= 8:
+        return "strong"
+    if score >= 5:
+        return "moderate"
+    if score >= 2:
+        return "weak"
+    return "very_weak"
+
+
+def direction_of(evidence: dict[str, Any]) -> str:
+    """Legacy wrapper for the claim relation of one evidence object.
+
+    Delegates to ``evidence_semantics.claim_relation`` so the interpretation
+    of ``relation_to_claim`` / legacy ``direction`` stays centralized (A-1).
+    Kept for backward-compatible callers; new code should use
+    ``evidence_semantics.claim_relation`` directly.
+    """
+    return claim_relation(evidence)
+
+
+def independent_studies(evidence_list: list[dict[str, Any]]) -> int:
+    """Number of distinct studies behind the evidence.
+
+    Counts unique non-empty ``study_id`` values; falls back to unique
+    ``source_id`` values when ``study_id`` is absent.
+    """
+    ids = [e.get("study_id") or e.get("source_id") for e in evidence_list]
+    return len({i for i in ids if i})
+
+
+def independent_samples(evidence_list: list[dict[str, Any]]) -> int:
+    """Number of distinct samples behind the evidence.
+
+    Counts unique non-empty ``sample_id`` values. When ``sample_id`` is missing
+    entirely, falls back to study-level ids (each study contributes at least one
+    sample), so the count stays a deterministic lower-bound estimate.
+    """
+    ids = [e.get("sample_id") or e.get("study_id") or e.get("source_id") for e in evidence_list]
+    return len({i for i in ids if i})
+
+
+def consistency_score(directions: list[str]) -> float:
+    """Consistency in [0,1]: proportion of non-neutral evidence agreeing with the majority direction."""
+    non_neutral = [d for d in directions if d in ("support", "contradict")]
+    if not non_neutral:
+        return 0.0
+    majority = max(non_neutral.count("support"), non_neutral.count("contradict"))
+    return majority / len(non_neutral)
+
+
+def decision_consistency_score(relations: list[str]) -> float:
+    """Decision-level consistency in [0,1] (D-2).
+
+    Computed over ``decision_relation`` values instead of ``relation_to_claim``:
+    claim-level evidence is usually extracted precisely because it supports its
+    claim, so relation-level consistency overstates agreement about the final
+    teaching decision. Decisive relations are support_adoption (for) and
+    oppose_adoption (against); conditional and neutral are non-committal.
+    """
+    decisive = [r for r in relations if r in ("support_adoption", "oppose_adoption")]
+    if not decisive:
+        return 0.0
+    majority = max(decisive.count("support_adoption"), decisive.count("oppose_adoption"))
+    return majority / len(decisive)
+
+
+def directness_score(evidence_list: list[dict[str, Any]]) -> float:
+    """Average D5 Directness (0-2) across evidence; 0 if empty."""
+    if not evidence_list:
+        return 0.0
+    values = [e.get("quality_dimensions", {}).get("D5_directness", 0) for e in evidence_list]
+    return sum(float(v) for v in values) / len(values)
+
+
+def confidence(evidence_list: list[dict[str, Any]], *, target_outcome: str | None = None) -> dict[str, Any]:
+    """Rule-based confidence computation per plan section 13.
+
+    Normalized implementation (weights sum to 1.0 so the score stays in [0, 1]):
+
+        score = 0.30*Evidence Quality + 0.25*Consistency + 0.20*Directness
+                + 0.25*Evidence Count - Conflict Penalty - Unsupported Penalty
+
+    where Evidence Count = min(1.0, independent_studies / 4) (D-1),
+    Conflict Penalty = 0.15 if any evidence opposes adoption
+    (decision_relation == oppose_adoption, D-2), and
+    Unsupported Penalty = min(0.20, 0.05 * n_unsupported). The raw quality /
+    consistency / directness terms are scaled to [0, 1] first.
+
+    Returns breakdown dict plus final label (High | Moderate | Low | Insufficient).
+    """
+    if not evidence_list:
+        return {"confidence": "Insufficient",
+                "confidence_policy_version": CONFIDENCE_POLICY_VERSION,
+                "confidence_breakdown": {"evidence_count": 0}}
+
+    # 1. Evidence Quality (mean of quality scores, scaled to 0-1 range)
+    quality_values = [e.get("quality_score") for e in evidence_list]
+    numeric = [q for q in quality_values if isinstance(q, (int, float))]
+    avg_quality = sum(numeric) / len(numeric) if numeric else 0.0
+    quality_term = avg_quality / 10.0  # 0-1
+
+    # 2. Consistency (decision_relation based, D-2): claim-level evidence is
+    #    usually extracted to support its claim, so relation_to_claim would
+    #    overstate agreement about the final teaching decision.
+    decisions = [decision_relation(e) for e in evidence_list]
+    consistency = decision_consistency_score(decisions)
+
+    # 3. Directness (0-2 -> 0-1)
+    directness = directness_score(evidence_list) / 2.0
+
+    # 4. Evidence count (capped at 1) — 独立研究计权（D-1）：count term 只用
+    #    independent_studies，不再与 independent_samples 相加（一个普通研究
+    #    通常 1 study + 1 sample，相加会让同一研究贡献两次）。independent_samples
+    #    单独展示在 Provenance / Evidence Summary。
+    n_studies = independent_studies(evidence_list)
+    n_samples = independent_samples(evidence_list)
+    count_term = min(1.0, n_studies / 4.0)
+
+    # 5. Conflict penalty (0.15 when any evidence opposes adoption)
+    conflict_penalty = 0.15 if "oppose_adoption" in decisions else 0.0
+
+    # 6. Unsupported penalty (capped at 0.20)
+    unsupported = [e for e in evidence_list if e.get("status") in ("UNSUPPORTED", "DOWNGRADE_CONFIDENCE")]
+    unsupported_penalty = min(0.20, len(unsupported) * 0.05)
+
+    score = (0.30 * quality_term + 0.25 * consistency + 0.20 * directness
+             + 0.25 * count_term - conflict_penalty - unsupported_penalty)
+    score = max(0.0, min(1.0, score))
+
+    if target_outcome:
+        relevant = [e for e in evidence_list if e.get("outcome_type") == target_outcome]
+        if relevant:
+            score *= 0.5 + 0.5 * (len(relevant) / max(1, len(evidence_list)))
+
+    label = _confidence_label(score)
+    return {
+        "confidence": label,
+        "confidence_score": round(score, 3),
+        "confidence_policy_version": CONFIDENCE_POLICY_VERSION,
+        "independent_studies": n_studies,
+        "independent_samples": n_samples,
+        "confidence_breakdown": {
+            "score": round(score, 3),
+            "evidence_quality": round(quality_term, 3),
+            "consistency": round(consistency, 3),
+            "directness": round(directness, 3),
+            "evidence_count": len(evidence_list),
+            "independent_studies": n_studies,
+            "independent_samples": n_samples,
+            "count_term": round(count_term, 3),
+            "conflict_penalty": conflict_penalty,
+            "unsupported_penalty": round(unsupported_penalty, 3),
+        },
+    }
+
+
+def _confidence_label(score: float) -> str:
+    if score >= 0.72:
+        return "High"
+    if score >= 0.45:
+        return "Moderate"
+    if score >= 0.2:
+        return "Low"
+    return "Insufficient"
+
+
+if __name__ == "__main__":
+    import json
+    import sys
+
+    if len(sys.argv) != 2:
+        print("usage: evidence_score.py <evidence.jsonl>", file=sys.stderr)
+        sys.exit(2)
+    with open(sys.argv[1], encoding="utf-8") as fh:
+        evs = [json.loads(line) for line in fh if line.strip()]
+    for ev in evs:
+        if ev.get("quality_dimensions") and ev.get("quality_score") is None:
+            ev["quality_score"] = quality_score(ev["quality_dimensions"])
+    print(json.dumps(confidence(evs), ensure_ascii=False, indent=2))

package/scripts/evidence_semantics.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python3
+"""evidence_semantics.py — Centralized Evidence semantic helpers (A-1).
+
+The three evidence "direction" semantics are deliberately distinct; every
+consumer must go through these helpers instead of re-interpreting the raw
+fields, so the semantics cannot drift across files again:
+
+    claim_relation(ev)     support | contradict | neutral
+        Does this evidence support the specific claim it is bound to?
+        (field: relation_to_claim; legacy fallback: direction)
+
+    effect_direction(ev)   positive | negative | null
+        What did the study observe? (field: effect_direction)
+
+    decision_relation(ev)  support_adoption | oppose_adoption | conditional | neutral
+        What does this evidence mean for the final teaching decision?
+        (field: decision_relation; deterministic fallback derived from the
+        claim relation when the field is absent)
+
+Consumers:
+
+    Claim Audit                    -> claim_relation
+    Outcome Visualization          -> effect_direction
+    Cross-study Consistency        -> decision_relation
+
+V2 note: the Research Engine's V2 semantics live in `engine/semantics.py`
+(finding_effect / claim_relation / decision_implication) and operate on the
+versioned graph entities. This module keeps its legacy flat-evidence API for
+V1 callers; the two layers are structurally different and must not be mixed.
+"""
+from __future__ import annotations
+
+from typing import Any
+
+CLAIM_RELATIONS = ("support", "contradict", "neutral")
+EFFECT_DIRECTIONS = ("positive", "negative", "null")
+DECISION_RELATIONS = ("support_adoption", "oppose_adoption", "conditional", "neutral")
+
+_CLAIM_TO_DECISION = {
+    "support": "support_adoption",
+    "contradict": "oppose_adoption",
+    "neutral": "neutral",
+}
+
+
+def claim_relation(evidence: dict[str, Any]) -> str:
+    """Relation of one evidence object to the claim it is bound to.
+
+    Reads the new-contract field ``relation_to_claim`` (support | contradict |
+    neutral); falls back to the legacy ``direction`` field for backward
+    compatibility. Returns 'neutral' when neither is present/valid.
+    """
+    relation = evidence.get("relation_to_claim")
+    if relation in CLAIM_RELATIONS:
+        return relation
+    legacy = evidence.get("direction")
+    if legacy in CLAIM_RELATIONS:
+        return legacy
+    return "neutral"
+
+
+def effect_direction(evidence: dict[str, Any]) -> str:
+    """Observed effect direction of one evidence object.
+
+    Reads ``effect_direction`` (positive | negative | null). Returns 'null'
+    when the field is absent or invalid — a missing measurement is treated as
+    no measurable effect, never as an invented one.
+    """
+    effect = evidence.get("effect_direction")
+    if effect in EFFECT_DIRECTIONS:
+        return effect
+    return "null"
+
+
+def decision_relation(evidence: dict[str, Any]) -> str:
+    """Relation of one evidence object to the final teaching decision.
+
+    Reads ``decision_relation`` (support_adoption | oppose_adoption |
+    conditional | neutral). When the field is absent, derives a deterministic
+    fallback from the claim relation (support -> support_adoption,
+    contradict -> oppose_adoption, neutral -> neutral) so legacy data keeps
+    working without silent semantic drift.
+    """
+    decision = evidence.get("decision_relation")
+    if decision in DECISION_RELATIONS:
+        return decision
+    return _CLAIM_TO_DECISION[claim_relation(evidence)]

package/scripts/fetch_benchmark.py

@@ -0,0 +1,177 @@
+#!/usr/bin/env python3
+"""fetch_benchmark.py — Fetch Reliability Benchmark (Smart Web Fetch v3 §19).
+
+Compares built-in fetch vs Smart Web Fetch over benchmarks/fetch/urls.jsonl
+(30 URLs covering paper landing / Chinese web / university sites / dynamic
+pages / failure simulations).
+
+Metrics:
+    Fetch Success Rate      FETCH_VALID+PARTIAL proportion
+    Useful Content Recall   content captured for expected-title URLs
+    Noise Ratio             1 - clean_size/raw_size average
+    Latency                 seconds per fetch
+    Fallback Rate           proportion of attempts that used a fallback provider
+
+Offline mode (--dry-run): uses a deterministic local simulator so the
+benchmark runs without network — useful for CI and reproducibility. Live mode
+(--live) actually hits the network.
+
+Usage:
+    python3 scripts/fetch_benchmark.py --urls benchmarks/fetch/urls.jsonl --dry-run
+    python3 scripts/fetch_benchmark.py --urls benchmarks/fetch/urls.jsonl --live --out benchmarks/fetch/results/summary.json
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import random
+import sys
+import time
+from pathlib import Path
+from typing import Any
+
+
+def _simulate(url: str, category: str, expect_title: str, seed: int = 42) -> dict[str, Any]:
+    """Deterministic offline simulator mirroring real fetch outcomes."""
+    rng = random.Random(seed + hash(url) % 100000)
+
+    # service failure simulations
+    if category == "service_failure_sim":
+        return {"url": url, "fetch_status": "FETCH_FAILED", "fetch_provider": "builtin",
+                "latency_s": 0.05, "fallback_used": False, "fallback_chain": ["builtin:error"],
+                "raw_size": 0, "clean_size": 0, "compression_ratio": 0.0}
+
+    # login-walled / dynamic pages degrade to PARTIAL at best
+    if category in ("dynamic_page", "complex_nav"):
+        status = "FETCH_PARTIAL" if rng.random() < 0.6 else "FETCH_FAILED"
+        provider = "jina_reader" if status == "FETCH_PARTIAL" else "builtin"
+        return {"url": url, "fetch_status": status, "fetch_provider": provider,
+                "latency_s": round(0.3 + rng.random() * 0.5, 2), "fallback_used": status == "FETCH_PARTIAL",
+                "fallback_chain": ["builtin:error", "jina_reader:ok"] if status == "FETCH_PARTIAL" else ["builtin:error"],
+                "raw_size": 80_000, "clean_size": 6_000 if status == "FETCH_PARTIAL" else 0,
+                "compression_ratio": 0.9 if status == "FETCH_PARTIAL" else 0.0}
+
+    # short page
+    if category == "short_page":
+        return {"url": url, "fetch_status": "FETCH_PARTIAL", "fetch_provider": "builtin",
+                "latency_s": 0.1, "fallback_used": False, "fallback_chain": ["builtin"],
+                "raw_size": 1_200, "clean_size": 120, "compression_ratio": 0.9}
+
+    # paper landing / journal / institution: reliable
+    if category in ("paper_landing", "journal_article", "institution_org", "government_org",
+                    "international_org", "report", "dataset_page", "wiki", "long_page",
+                    "professional_institution"):
+        status = "FETCH_VALID" if rng.random() < 0.85 else "FETCH_PARTIAL"
+        provider = "builtin" if status == "FETCH_VALID" else "jina_reader"
+        clean = 18_000 if status == "FETCH_VALID" else 4_000
+        return {"url": url, "fetch_status": status, "fetch_provider": provider,
+                "latency_s": round(0.2 + rng.random() * 0.6, 2), "fallback_used": status == "FETCH_PARTIAL",
+                "fallback_chain": ["builtin"] if status == "FETCH_VALID" else ["builtin:error", "jina_reader:ok"],
+                "raw_size": 120_000, "clean_size": clean, "compression_ratio": 1 - clean / 120_000}
+
+    # university / chinese / news sites: mixed reliability
+    content_ok = rng.random() < 0.7
+    status = "FETCH_VALID" if content_ok else "FETCH_PARTIAL"
+    provider = "builtin" if status == "FETCH_VALID" else "markdown_new"
+    return {"url": url, "fetch_status": status, "fetch_provider": provider,
+            "latency_s": round(0.2 + rng.random() * 0.7, 2), "fallback_used": status == "FETCH_PARTIAL",
+            "fallback_chain": ["builtin"] if status == "FETCH_VALID" else ["builtin:error", "markdown_new:ok"],
+            "raw_size": 90_000, "clean_size": 10_000 if status == "FETCH_VALID" else 3_000,
+            "compression_ratio": 1 - (10_000 if status == "FETCH_VALID" else 3_000) / 90_000}
+
+
+def _live_fetch(url: str, timeout: int = 20) -> dict[str, Any]:
+    """Live fetch through the Smart Web Fetch chain (network required)."""
+    sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+    from integrations.smart_web_fetch import smart_fetch
+
+    start = time.monotonic()
+    result = smart_fetch(url, timeout=timeout)
+    elapsed = time.monotonic() - start
+    return {
+        "url": url,
+        "fetch_status": result["fetch_status"],
+        "fetch_provider": result["fetch_provider"],
+        "latency_s": round(elapsed, 2),
+        "fallback_used": result["fallback_used"],
+        "fallback_chain": result["fallback_chain"],
+        "raw_size": result["raw_size"],
+        "clean_size": result["clean_size"],
+        "compression_ratio": result["compression_ratio"],
+    }
+
+
+def run_benchmark(urls: list[dict[str, Any]], *, live: bool, timeout: int = 20) -> dict[str, Any]:
+    results: list[dict[str, Any]] = []
+    for row in urls:
+        if live:
+            res = _live_fetch(row["url"], timeout=timeout)
+        else:
+            res = _simulate(row["url"], row.get("category", ""), row.get("expect_title", ""))
+        res["id"] = row["id"]
+        res["category"] = row.get("category", "")
+        res["expect_title"] = row.get("expect_title", "")
+        results.append(res)
+
+    total = len(results)
+    valid = [r for r in results if r["fetch_status"] == "FETCH_VALID"]
+    partial = [r for r in results if r["fetch_status"] == "FETCH_PARTIAL"]
+    failed = [r for r in results if r["fetch_status"] == "FETCH_FAILED"]
+
+    latencies = [r["latency_s"] for r in results if r.get("latency_s")]
+    ratios = [r["compression_ratio"] for r in results if r.get("compression_ratio")]
+    fallback = [r for r in results if r.get("fallback_used")]
+
+    # Useful content recall: expected-title URLs that returned usable content
+    expect_title_urls = [r for r in results if r.get("expect_title")]
+    recall_hits = [r for r in expect_title_urls
+                   if r["fetch_status"] in ("FETCH_VALID", "FETCH_PARTIAL")]
+    recall = len(recall_hits) / len(expect_title_urls) if expect_title_urls else 0.0
+
+    summary = {
+        "total_urls": total,
+        "fetch_success_rate": round(len(valid) / total, 3),
+        "partial_rate": round(len(partial) / total, 3),
+        "failure_rate": round(len(failed) / total, 3),
+        "useful_content_recall": round(recall, 3),
+        "avg_noise_ratio": round(sum(ratios) / len(ratios), 3) if ratios else 0.0,
+        "avg_latency_s": round(sum(latencies) / len(latencies), 3) if latencies else 0.0,
+        "fallback_rate": round(len(fallback) / total, 3),
+        "mode": "live" if live else "dry_run",
+        "results": results,
+    }
+    return summary
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="EduEvidence Fetch Reliability Benchmark")
+    parser.add_argument("--urls", required=True, help="benchmarks/fetch/urls.jsonl")
+    parser.add_argument("--live", action="store_true", help="actually hit the network")
+    parser.add_argument("--dry-run", dest="dry_run", action="store_true",
+                        help="explicitly run the offline simulator (default; mutually exclusive with --live)")
+    parser.add_argument("--timeout", type=int, default=20)
+    parser.add_argument("--out", help="optional JSON output path")
+    args = parser.parse_args()
+
+    if args.live and args.dry_run:
+        parser.error("--live and --dry-run are mutually exclusive")
+
+    urls = [json.loads(line) for line in Path(args.urls).read_text(encoding="utf-8").splitlines()
+            if line.strip()]
+    summary = run_benchmark(urls, live=args.live, timeout=args.timeout)
+
+    if args.out:
+        out_path = Path(args.out)
+        out_path.parent.mkdir(parents=True, exist_ok=True)
+        out_path.write_text(json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8")
+        print(f"wrote {args.out}")
+
+    keys = ("total_urls", "fetch_success_rate", "partial_rate", "failure_rate",
+            "useful_content_recall", "avg_noise_ratio", "avg_latency_s", "fallback_rate", "mode")
+    for k in keys:
+        print(f"{k}: {summary[k]}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/scripts/generate_metrics.py

@@ -0,0 +1,99 @@
+#!/usr/bin/env python3
+"""generate_metrics.py — Single source of truth for headline project metrics.
+
+Docs must stop hard-coding counts that drift (plan R5). This script derives
+the authoritative numbers from the repository itself and writes
+docs/metrics.json. Run with --check (CI) to fail when a doc-facing metric
+changed without regenerating this file.
+
+Metrics:
+- engine_version          from engine/versions.py (the version authority)
+- test_functions          grep 'def test_' across tests/
+- test_files              number of collected test modules in tests/
+- schema_count            schemas/*.json at root + v2/ + v3/ + v4/
+- reference_doc_count     references/*.md
+- gold_annotation_count   benchmarks/annotations/gold-Q*.json
+- example_packs           examples/*/ directories shipping result.json
+
+Usage:
+    python3 scripts/generate_metrics.py            # regenerate docs/metrics.json
+    python3 scripts/generate_metrics.py --check    # exit 1 if file is stale
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import sys
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+
+
+def collect() -> dict:
+    versions_py = (REPO_ROOT / "engine" / "versions.py").read_text(encoding="utf-8")
+    m = re.search(r'^ENGINE_VERSION\s*=\s*"(\d+\.\d+\.\d+)"', versions_py, re.M)
+    engine_version = m.group(1) if m else "unknown"
+
+    test_functions = test_files = 0
+    tests_dir = REPO_ROOT / "tests"
+    if tests_dir.is_dir():
+        for path in tests_dir.glob("test_*.py"):
+            test_files += 1
+            test_functions += len(re.findall(r"^\s*def test_", path.read_text(encoding="utf-8"), re.M))
+
+    schema_count = 0
+    schemas_dir = REPO_ROOT / "schemas"
+    if schemas_dir.is_dir():
+        schema_count = len(list(schemas_dir.rglob("*.json")))
+
+    reference_doc_count = len(list((REPO_ROOT / "references").glob("*.md"))) \
+        if (REPO_ROOT / "references").is_dir() else 0
+
+    gold_annotation_count = len(list((REPO_ROOT / "benchmarks" / "annotations").glob("gold-Q*.json"))) \
+        if (REPO_ROOT / "benchmarks" / "annotations").is_dir() else 0
+
+    example_packs = sorted(
+        p.name for p in (REPO_ROOT / "examples").iterdir()
+        if p.is_dir() and (p / "result.json").exists()
+    ) if (REPO_ROOT / "examples").is_dir() else []
+
+    return {
+        "engine_version": engine_version,
+        "test_functions": test_functions,
+        "test_files": test_files,
+        "schema_count": schema_count,
+        "reference_doc_count": reference_doc_count,
+        "gold_annotation_count": gold_annotation_count,
+        "example_packs": example_packs,
+    }
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--check", action="store_true",
+                        help="fail if docs/metrics.json is stale")
+    args = parser.parse_args()
+
+    metrics = collect()
+    out_path = REPO_ROOT / "docs" / "metrics.json"
+    current = json.loads(out_path.read_text(encoding="utf-8")) if out_path.exists() else None
+
+    if args.check:
+        if current != metrics:
+            print("FAIL: docs/metrics.json is stale; rerun scripts/generate_metrics.py")
+            print(f"expected: {json.dumps(metrics, ensure_ascii=False)}")
+            print(f"found:    {json.dumps(current, ensure_ascii=False)}")
+            return 1
+        print(f"metrics OK ({metrics['engine_version']}, "
+              f"{metrics['test_functions']} tests, {metrics['schema_count']} schemas)")
+        return 0
+
+    out_path.write_text(json.dumps(metrics, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+    print(json.dumps(metrics, ensure_ascii=False, indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())