@ngocsangairvds/vsaf 3.2.14 → 3.2.16

package/tools/vds-scripts/audit_orchestrator/scripts/benchmark_dspy.py

@@ -1,513 +0,0 @@
-#!/usr/bin/env python3
-"""Benchmark legacy vs DSPy evaluation paths for rollout-gate decisions."""
-
-from __future__ import annotations
-
-import argparse
-import importlib.util
-import json
-import sys
-import time
-from collections import Counter
-from pathlib import Path
-from typing import Any
-
-
-def _load_optimize_module():
-    module_path = Path(__file__).resolve().parent / "optimize_audit.py"
-    spec = importlib.util.spec_from_file_location("optimize_audit", module_path)
-    if spec is None or spec.loader is None:
-        raise RuntimeError("Cannot load optimize_audit.py")
-    module = importlib.util.module_from_spec(spec)
-    sys.modules[spec.name] = module
-    spec.loader.exec_module(module)
-    return module
-
-
-def _load_dataset(path: Path) -> dict[str, Any]:
-    payload = json.loads(path.read_text(encoding="utf-8"))
-    if not isinstance(payload, dict):
-        raise ValueError("Invalid dataset payload")
-    metadata = payload.get("metadata")
-    if not isinstance(metadata, dict):
-        raise ValueError("Dataset missing metadata wrapper")
-    required_metadata = (
-        "dataset_version",
-        "dataset_hash",
-        "sample_count",
-        "created_at",
-        "label_distribution",
-    )
-    missing_metadata = [key for key in required_metadata if key not in metadata]
-    if missing_metadata:
-        raise ValueError(f"Dataset metadata missing keys: {missing_metadata}")
-    samples = payload.get("samples")
-    if not isinstance(samples, dict):
-        raise ValueError("Dataset missing samples split payload")
-    for split in ("train", "dev", "test"):
-        if not isinstance(samples.get(split), list):
-            raise ValueError(f"Dataset samples.{split} must be a list")
-    return payload
-
-
-def _dataset_hash(dataset: dict[str, Any]) -> str:
-    return str(dataset["metadata"]["dataset_hash"])
-
-
-def _dataset_samples(dataset: dict[str, Any]) -> dict[str, Any]:
-    return dataset["samples"]
-
-
-def _status_from_score(score_percent: float) -> str:
-    if score_percent >= 90:
-        return "PASS"
-    if score_percent >= 40:
-        return "PARTIAL"
-    return "FAIL"
-
-
-def _manual_predict(row: dict[str, Any]) -> str:
-    return _status_from_score(float(row.get("score_percent") or 0.0))
-
-
-def _dspy_predict(row: dict[str, Any], artifact: dict[str, Any] | None) -> str:
-    if artifact is None:
-        return _manual_predict(row)
-    rulebook = artifact.get("rulebook", {})
-    category = str(row.get("requirement_category") or "uncategorized")
-    return str(rulebook.get(category) or "PARTIAL")
-
-
-def _confusion(gold: list[str], pred: list[str]) -> dict[str, dict[str, int]]:
-    statuses = sorted(set(gold) | set(pred) | {"PASS", "PARTIAL", "FAIL", "ERROR", "N/A"})
-    matrix: dict[str, dict[str, int]] = {label: dict.fromkeys(statuses, 0) for label in statuses}
-    for g, p in zip(gold, pred, strict=False):
-        matrix[g][p] += 1
-    return matrix
-
-
-def _accuracy(gold: list[str], pred: list[str]) -> float:
-    if not gold:
-        return 0.0
-    ok = sum(1 for g, p in zip(gold, pred, strict=False) if g == p)
-    return round(ok / len(gold), 4)
-
-
-def _token_estimate(rows: list[dict[str, Any]]) -> int:
-    total_chars = sum(len(str(r.get("requirement_text") or "")) + len(str(r.get("reason_vn") or "")) for r in rows)
-    return max(1, total_chars // 4)
-
-
-def run_proxy_benchmark(dataset: dict[str, Any], artifact: dict[str, Any] | None) -> dict[str, Any]:
-    """Legacy proxy benchmark (rulebook/manual) kept for fallback compatibility."""
-    heldout = _dataset_samples(dataset)["test"]
-
-    gold = [str(row.get("status") or "PARTIAL") for row in heldout if isinstance(row, dict)]
-    rows = [row for row in heldout if isinstance(row, dict)]
-
-    t0 = time.perf_counter()
-    manual = [_manual_predict(row) for row in rows]
-    manual_latency = (time.perf_counter() - t0) * 1000
-
-    t1 = time.perf_counter()
-    dspy = [_dspy_predict(row, artifact) for row in rows]
-    dspy_latency = (time.perf_counter() - t1) * 1000
-
-    return {
-        "kind": "manual_vs_dspy_metrics",
-        "benchmark_mode": "rulebook_proxy",
-        "dataset_hash": _dataset_hash(dataset),
-        "rows_evaluated": len(rows),
-        "manual": {
-            "accuracy": _accuracy(gold, manual),
-            "latency_ms": round(manual_latency, 3),
-            "token_estimate": _token_estimate(rows),
-        },
-        "dspy": {
-            "accuracy": _accuracy(gold, dspy),
-            "latency_ms": round(dspy_latency, 3),
-            "token_estimate": _token_estimate(rows),
-            "compiled_artifact": bool(artifact),
-        },
-        "quality_metrics": {
-            "guidance_adherence": {"manual": None, "dspy": None},
-            "anchor_grounding_quality": {"manual": None, "dspy": None},
-            "stability": {"manual_status_agreement": None, "dspy_status_agreement": None},
-        },
-        "fallback_reason_counts": {},
-        "confusion_matrix": {
-            "manual": _confusion(gold, manual),
-            "dspy": _confusion(gold, dspy),
-        },
-        "sampled_vn_quality_review": [
-            {
-                "row_id": row.get("row_id"),
-                "status": row.get("status"),
-                "reason_vn": row.get("reason_vn"),
-                "finding_vn": row.get("finding_vn"),
-            }
-            for row in rows[: min(10, len(rows))]
-        ],
-        "rollout_gate": {
-            "rule": "dspy_accuracy >= manual_accuracy and dspy_latency_ms <= manual_latency_ms * 1.2",
-            "passed": _accuracy(gold, dspy) >= _accuracy(gold, manual)
-            and round(dspy_latency, 3) <= round(manual_latency, 3) * 1.2,
-            "status_distribution_test": dict(Counter(gold)),
-        },
-    }
-
-
-def _compute_guidance_adherence(rows: list[dict[str, Any]], reasons: list[str], findings: list[str]) -> float:
-    if not rows:
-        return 0.0
-    matched = 0
-    for row, reason, finding in zip(rows, reasons, findings, strict=False):
-        category = str(row.get("requirement_category") or "").strip()
-        guidance_terms = [segment.strip().lower() for segment in category.split("||") if segment.strip()]
-        if not guidance_terms:
-            continue
-        text = f"{reason}\n{finding}".lower()
-        if any(term in text for term in guidance_terms):
-            matched += 1
-    return round(matched / len(rows), 4)
-
-
-def _compute_anchor_quality(anchor_counts: list[int], excerpt_counts: list[int]) -> float:
-    if not anchor_counts:
-        return 0.0
-    total = 0.0
-    for anchors, excerpts in zip(anchor_counts, excerpt_counts, strict=False):
-        if anchors <= 0:
-            continue
-        if excerpts > 0:
-            total += 1.0
-        else:
-            total += 0.5
-    return round(total / len(anchor_counts), 4)
-
-
-def _stability_agreement(base: list[str], reruns: list[list[str]]) -> float:
-    if not base:
-        return 0.0
-    if not reruns:
-        return 1.0
-    per_run = []
-    for run in reruns:
-        ok = sum(1 for b, r in zip(base, run, strict=False) if b == r)
-        per_run.append(ok / len(base))
-    return round(sum(per_run) / len(per_run), 4)
-
-
-def _build_runtime_inputs(rows: list[dict[str, Any]]) -> tuple[Any, list[Any], dict[str, Any]]:
-    """Create minimal template/check/evidence map for row evaluator runtime."""
-    from vds_audit_orchestrator.models.evidence import Evidence, EvidenceBundle, EvidenceStatus, compute_content_hash
-    from vds_audit_orchestrator.models.template import AuditCheck, AuditSection, AuditTemplate
-
-    checks: list[AuditCheck] = []
-    evidence_map: dict[str, Evidence] = {}
-    confluence_evidence: list[Evidence] = []
-
-    for index, row in enumerate(rows, start=1):
-        row_id = str(row.get("row_id") or f"row-{index}")
-        requirement_text = str(row.get("requirement_text") or row.get("reason_vn") or "Yêu cầu kiểm tra").strip()
-        requirement_category = str(row.get("requirement_category") or "uncategorized").strip()
-        guidance_hint = str(row.get("reason_vn") or "").strip()
-        evidence_hint = str(row.get("finding_vn") or "").strip()
-        check_id = f"BENCH-{index:04d}"
-
-        check = AuditCheck(
-            id=check_id,
-            name=requirement_text,
-            description=requirement_text,
-            section_id="bench",
-            check_type="semantic_requirements",
-            requirement_category=requirement_category,
-            check_config={
-                "confluence_main": requirement_category,
-                "confluence_eval_guide": guidance_hint,
-                "confluence_evidence_hint": evidence_hint,
-            },
-        )
-        checks.append(check)
-
-        content = "\n".join(
-            part
-            for part in [
-                str(row.get("reason_vn") or "").strip(),
-                str(row.get("finding_vn") or "").strip(),
-            ]
-            if part
-        ).strip()
-        if not content:
-            content = requirement_text
-
-        evidence = Evidence(
-            source="confluence_page",
-            source_id=f"dataset://{row_id}",
-            source_url=f"dataset://{row_id}",
-            title=f"Dataset row {row_id}",
-            content=content,
-            content_hash=compute_content_hash(content),
-            status=EvidenceStatus.FOUND,
-            metadata={"dataset_row_id": row_id, "requirement_category": requirement_category},
-        )
-        confluence_evidence.append(evidence)
-        evidence_map[check_id] = evidence
-
-    section = AuditSection(
-        id="bench",
-        name="DSPy Runtime Benchmark",
-        description="Synthetic runtime benchmark section",
-        weight=100.0,
-        checks=checks,
-    )
-    template = AuditTemplate(name="dspy-runtime-benchmark", version="1.0.0", sections=[section])
-    bundle = EvidenceBundle(repository="dspy-runtime-benchmark", confluence_evidence=confluence_evidence)
-    return template, checks, evidence_map | {"__bundle__": bundle}
-
-
-def _runtime_eval(
-    *,
-    rows: list[dict[str, Any]],
-    use_dspy: bool,
-    protocol: str | None,
-    model: str | None,
-    llm_mode: str,
-) -> dict[str, Any]:
-    from vds_audit_orchestrator.engine.row_evaluator import create_row_evaluator
-
-    template, checks, evidence_payload = _build_runtime_inputs(rows)
-    bundle = evidence_payload.pop("__bundle__")
-
-    evaluator = create_row_evaluator(
-        template=template,
-        evidence_bundle=bundle,
-        mode=llm_mode,
-        protocol=protocol,
-        model=model,
-        use_dspy=use_dspy,
-    )
-
-    statuses: list[str] = []
-    reasons: list[str] = []
-    findings: list[str] = []
-    anchor_counts: list[int] = []
-    excerpt_counts: list[int] = []
-    protocol_by_row: list[str] = []
-
-    t0 = time.perf_counter()
-    for idx, check in enumerate(checks):
-        result = evaluator.evaluate_row(
-            check=check,
-            evidence=evidence_payload.get(check.id),
-            row_id=f"bench-row-{idx}",
-            row_index=idx,
-        )
-        statuses.append(str(result.status.value))
-        reasons.append(result.reason)
-        findings.append(result.finding)
-        anchor_counts.append(len(result.evidence_anchors))
-        excerpt_counts.append(sum(1 for anchor in result.evidence_anchors if anchor.excerpt))
-        protocol_by_row.append(str(result.provenance.protocol or ""))
-
-    latency_ms = round((time.perf_counter() - t0) * 1000, 3)
-    return {
-        "statuses": statuses,
-        "reasons": reasons,
-        "findings": findings,
-        "latency_ms": latency_ms,
-        "token_estimate": _token_estimate(rows),
-        "guidance_adherence": _compute_guidance_adherence(rows, reasons, findings),
-        "anchor_grounding_quality": _compute_anchor_quality(anchor_counts, excerpt_counts),
-        "protocol_distribution": dict(Counter(protocol_by_row)),
-    }
-
-
-def run_runtime_benchmark(
-    dataset: dict[str, Any],
-    artifact: dict[str, Any] | None,
-    *,
-    protocol: str | None,
-    model: str | None,
-    llm_mode: str,
-    max_rows: int,
-    stability_runs: int,
-) -> dict[str, Any]:
-    heldout = _dataset_samples(dataset)["test"]
-    if not isinstance(heldout, list):
-        raise ValueError("Dataset missing test split list")
-
-    rows = [row for row in heldout if isinstance(row, dict)][: max(1, max_rows)]
-    gold = [str(row.get("status") or "PARTIAL") for row in rows]
-
-    manual_eval = _runtime_eval(rows=rows, use_dspy=False, protocol=protocol, model=model, llm_mode=llm_mode)
-    dspy_eval = _runtime_eval(rows=rows, use_dspy=True, protocol=protocol, model=model, llm_mode=llm_mode)
-
-    manual_reruns = [
-        _runtime_eval(rows=rows, use_dspy=False, protocol=protocol, model=model, llm_mode=llm_mode)["statuses"]
-        for _ in range(max(0, stability_runs - 1))
-    ]
-    dspy_reruns = [
-        _runtime_eval(rows=rows, use_dspy=True, protocol=protocol, model=model, llm_mode=llm_mode)["statuses"]
-        for _ in range(max(0, stability_runs - 1))
-    ]
-
-    manual_statuses = list(manual_eval["statuses"])
-    dspy_statuses = list(dspy_eval["statuses"])
-
-    fallback_reason_counts: dict[str, int] = {}
-    if dspy_eval.get("protocol_distribution"):
-        fallback_reason_counts = {
-            "legacy_provider_rows": sum(
-                count
-                for provider_name, count in dspy_eval["protocol_distribution"].items()
-                if provider_name and provider_name not in {"dspy", "none"}
-            )
-        }
-
-    return {
-        "kind": "manual_vs_dspy_metrics",
-        "benchmark_mode": "row_evaluator_ab",
-        "dataset_hash": _dataset_hash(dataset),
-        "rows_evaluated": len(rows),
-        "manual": {
-            "accuracy": _accuracy(gold, manual_statuses),
-            "latency_ms": manual_eval["latency_ms"],
-            "token_estimate": manual_eval["token_estimate"],
-        },
-        "dspy": {
-            "accuracy": _accuracy(gold, dspy_statuses),
-            "latency_ms": dspy_eval["latency_ms"],
-            "token_estimate": dspy_eval["token_estimate"],
-            "compiled_artifact": bool(artifact),
-        },
-        "quality_metrics": {
-            "guidance_adherence": {
-                "manual": manual_eval["guidance_adherence"],
-                "dspy": dspy_eval["guidance_adherence"],
-            },
-            "anchor_grounding_quality": {
-                "manual": manual_eval["anchor_grounding_quality"],
-                "dspy": dspy_eval["anchor_grounding_quality"],
-            },
-            "stability": {
-                "manual_status_agreement": _stability_agreement(manual_statuses, manual_reruns),
-                "dspy_status_agreement": _stability_agreement(dspy_statuses, dspy_reruns),
-                "runs": max(1, stability_runs),
-            },
-        },
-        "fallback_reason_counts": fallback_reason_counts,
-        "confusion_matrix": {
-            "manual": _confusion(gold, manual_statuses),
-            "dspy": _confusion(gold, dspy_statuses),
-        },
-        "sampled_vn_quality_review": [
-            {
-                "row_id": row.get("row_id"),
-                "status": row.get("status"),
-                "manual_reason": manual_eval["reasons"][idx],
-                "dspy_reason": dspy_eval["reasons"][idx],
-                "manual_finding": manual_eval["findings"][idx],
-                "dspy_finding": dspy_eval["findings"][idx],
-            }
-            for idx, row in enumerate(rows[: min(10, len(rows))])
-        ],
-        "rollout_gate": {
-            "rule": "dspy_accuracy >= manual_accuracy and dspy_latency_ms <= manual_latency_ms * 1.2",
-            "passed": _accuracy(gold, dspy_statuses) >= _accuracy(gold, manual_statuses)
-            and float(dspy_eval["latency_ms"]) <= float(manual_eval["latency_ms"]) * 1.2,
-            "status_distribution_test": dict(Counter(gold)),
-        },
-    }
-
-
-def _parse_args() -> argparse.Namespace:
-    parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument("--dataset", type=Path, default=Path("reports/dspy_gold_dataset.json"))
-    parser.add_argument(
-        "--compiled-artifact",
-        type=Path,
-        default=Path("reports/dspy_compiled/audit_judge_optimized.json"),
-    )
-    parser.add_argument("--output-dir", type=Path, default=Path("reports/dspy_benchmark"))
-    parser.add_argument(
-        "--benchmark-mode",
-        choices=["row_evaluator_ab", "rulebook_proxy"],
-        default="row_evaluator_ab",
-        help="row_evaluator_ab uses real row evaluator path; rulebook_proxy keeps legacy synthetic benchmark.",
-    )
-    parser.add_argument("--protocol", default=None, help="Override LLM protocol for runtime benchmark mode.")
-    parser.add_argument("--model", default=None, help="Override LLM model for runtime benchmark mode.")
-    parser.add_argument("--llm-mode", choices=["off", "selective", "per-row"], default="per-row")
-    parser.add_argument("--max-rows", type=int, default=25)
-    parser.add_argument("--stability-runs", type=int, default=2)
-    parser.add_argument(
-        "--allow-proxy-fallback",
-        action="store_true",
-        help="If runtime benchmark fails, fallback to rulebook_proxy output instead of exiting non-zero.",
-    )
-    return parser.parse_args()
-
-
-def main() -> int:
-    args = _parse_args()
-    dataset = _load_dataset(args.dataset)
-
-    artifact: dict[str, Any] | None = None
-    if args.compiled_artifact.exists():
-        optimize_module = _load_optimize_module()
-        artifact = optimize_module.load_compiled_artifact(args.compiled_artifact)
-
-    if args.max_rows < 1:
-        raise ValueError("--max-rows must be >= 1")
-    if args.stability_runs < 1:
-        raise ValueError("--stability-runs must be >= 1")
-
-    try:
-        if args.benchmark_mode == "row_evaluator_ab":
-            result = run_runtime_benchmark(
-                dataset,
-                artifact,
-                protocol=args.protocol,
-                model=args.model,
-                llm_mode=args.llm_mode,
-                max_rows=args.max_rows,
-                stability_runs=args.stability_runs,
-            )
-        else:
-            result = run_proxy_benchmark(dataset, artifact)
-    except Exception as exc:
-        if not args.allow_proxy_fallback:
-            raise
-        result = run_proxy_benchmark(dataset, artifact)
-        result["runtime_error"] = f"{type(exc).__name__}: {exc}"
-
-    args.output_dir.mkdir(parents=True, exist_ok=True)
-
-    metrics_path = args.output_dir / "manual_vs_dspy_metrics.json"
-    confusion_path = args.output_dir / "confusion_matrix.json"
-    review_path = args.output_dir / "sampled_vn_quality_review.json"
-
-    metrics_path.write_text(json.dumps(result, ensure_ascii=False, indent=2), encoding="utf-8")
-    confusion_path.write_text(json.dumps(result["confusion_matrix"], ensure_ascii=False, indent=2), encoding="utf-8")
-    review_path.write_text(
-        json.dumps(result["sampled_vn_quality_review"], ensure_ascii=False, indent=2), encoding="utf-8"
-    )
-
-    print(
-        json.dumps(
-            {
-                "metrics": str(metrics_path),
-                "confusion": str(confusion_path),
-                "vn_review": str(review_path),
-                "benchmark_mode": result.get("benchmark_mode"),
-                "rollout_gate_passed": result["rollout_gate"]["passed"],
-            },
-            ensure_ascii=False,
-        )
-    )
-    return 0
-
-
-if __name__ == "__main__":
-    raise SystemExit(main())

package/tools/vds-scripts/audit_orchestrator/scripts/benchmark_nlp_accuracy.py

@@ -1,138 +0,0 @@
-#!/usr/bin/env python3
-"""Offline benchmark for Phase 119 exemplar-corpus accuracy.
-
-The benchmark stays deterministic and offline while approximating the
-classification strength of the expanded corpus using leave-one-out nearest
-neighbor matching over bilingual exemplar text.
-"""
-
-from __future__ import annotations
-
-import argparse
-import json
-import unicodedata
-from collections import Counter
-from dataclasses import asdict, dataclass
-from pathlib import Path
-from typing import Any
-
-TARGET_OVERALL_ACCURACY = 0.90
-TARGET_PER_CATEGORY_ACCURACY = 0.80
-
-
-def _repo_root() -> Path:
-    return Path(__file__).resolve().parent.parent
-
-
-def _default_corpus_path() -> Path:
-    return _repo_root() / "data" / "requirement_exemplars.json"
-
-
-def _normalize_tokens(text: str) -> Counter[str]:
-    normalized = unicodedata.normalize("NFC", text).lower()
-    chars = [ch if ch.isalnum() or ch.isspace() else " " for ch in normalized]
-    return Counter(token for token in "".join(chars).split() if len(token) > 1)
-
-
-def _load_exemplars(path: Path) -> list[dict[str, Any]]:
-    payload = json.loads(path.read_text(encoding="utf-8"))
-    exemplars = payload.get("exemplars", [])
-    if not isinstance(exemplars, list):
-        raise ValueError("Corpus payload must contain an 'exemplars' list")
-    return [item for item in exemplars if isinstance(item, dict)]
-
-
-def _label(exemplar: dict[str, Any]) -> tuple[Any, ...]:
-    ground_truth = exemplar.get("ground_truth", {})
-    return (
-        ground_truth.get("analysis_mode"),
-        tuple(ground_truth.get("evidence_targets", [])),
-        ground_truth.get("project_scope_required"),
-        ground_truth.get("finalization_policy"),
-    )
-
-
-def _candidate_text(exemplar: dict[str, Any]) -> str:
-    return f"{exemplar.get('requirement_text', '')} {exemplar.get('requirement_en', '')}".strip()
-
-
-def _similarity_score(lhs: dict[str, Any], rhs: dict[str, Any]) -> float:
-    left = _normalize_tokens(_candidate_text(lhs))
-    right = _normalize_tokens(_candidate_text(rhs))
-    if not left and not right:
-        return 1.0
-    intersection = sum(min(left[token], right[token]) for token in left)
-    denominator = sum(left.values()) + sum(right.values())
-    return (2 * intersection / denominator) if denominator else 0.0
-
-
-@dataclass(frozen=True, slots=True)
-class BenchmarkSummary:
-    total_exemplars: int
-    correct_predictions: int
-    overall_accuracy: float
-    per_category_accuracy: dict[str, float]
-    target_overall_accuracy: float = TARGET_OVERALL_ACCURACY
-    target_per_category_accuracy: float = TARGET_PER_CATEGORY_ACCURACY
-
-    @property
-    def passes_threshold(self) -> bool:
-        return self.overall_accuracy >= self.target_overall_accuracy and all(
-            accuracy >= self.target_per_category_accuracy for accuracy in self.per_category_accuracy.values()
-        )
-
-
-def benchmark_exemplar_accuracy(exemplars: list[dict[str, Any]]) -> BenchmarkSummary:
-    correct_predictions = 0
-    by_category_total: Counter[str] = Counter()
-    by_category_correct: Counter[str] = Counter()
-
-    for exemplar in exemplars:
-        category = str(exemplar.get("category") or "Unknown")
-        by_category_total[category] += 1
-        peers = [
-            candidate
-            for candidate in exemplars
-            if candidate.get("id") != exemplar.get("id") and str(candidate.get("category") or "Unknown") == category
-        ]
-        if not peers:
-            peers = [candidate for candidate in exemplars if candidate.get("id") != exemplar.get("id")]
-        if not peers:
-            continue
-        predicted = max(peers, key=lambda candidate: _similarity_score(exemplar, candidate))
-        if _label(predicted) == _label(exemplar):
-            correct_predictions += 1
-            by_category_correct[category] += 1
-
-    per_category_accuracy = {
-        category: (by_category_correct.get(category, 0) / total if total else 0.0)
-        for category, total in sorted(by_category_total.items())
-    }
-    total_exemplars = len(exemplars)
-    overall_accuracy = (correct_predictions / total_exemplars) if total_exemplars else 0.0
-    return BenchmarkSummary(
-        total_exemplars=total_exemplars,
-        correct_predictions=correct_predictions,
-        overall_accuracy=overall_accuracy,
-        per_category_accuracy=per_category_accuracy,
-    )
-
-
-def _build_parser() -> argparse.ArgumentParser:
-    parser = argparse.ArgumentParser(description="Benchmark Phase 119 exemplar corpus accuracy")
-    parser.add_argument("--corpus-path", type=Path, default=_default_corpus_path())
-    parser.add_argument("--json-only", action="store_true")
-    return parser
-
-
-def main(argv: list[str] | None = None) -> int:
-    args = _build_parser().parse_args(argv)
-    summary = benchmark_exemplar_accuracy(_load_exemplars(args.corpus_path))
-    payload = asdict(summary)
-    payload["passes_threshold"] = summary.passes_threshold
-    print(json.dumps(payload, ensure_ascii=False, indent=2))
-    return 0 if summary.passes_threshold else 1
-
-
-if __name__ == "__main__":
-    raise SystemExit(main())