@rpamis/comet 0.4.0-beta.1 → 0.4.0-beta.11

package/eval/local/tests/scaffold/test_compare_baselines.py

@@ -0,0 +1,999 @@
+"""Unit tests for baseline comparison reporting."""
+
+import json
+from pathlib import Path
+
+from local.scripts.compare_baselines import build_report, main
+from scaffold.python.report_outputs import render_markdown_html
+
+
+def _write_report(reports_dir: Path, name: str, tokens: int, cost: float, passed: bool = True):
+    report = {
+        "name": f"comet-fix-median-{name}",
+        "passed": passed,
+        "checks_passed": [
+            "[RUBRIC] main_flow: 1.00 - ok",
+            "[RUBRIC] gate_guard: 1.00 - ok",
+            "[RUBRIC] skill_invocation: 1.00 - ok",
+            "[RUBRIC] spec_drift: 1.00 - ok",
+            "[RUBRIC] business_completion: 1.00 - ok",
+            "[RUBRIC] workflow_completion: 1.00 - ok",
+            "[RUBRIC] efficiency: 1.00 - ok",
+            "[RUBRIC] decision_point_compliance: 1.00 - ok",
+            "[RUBRIC] artifact_quality: 1.00 - ok",
+        ],
+        "checks_failed": [] if passed else ["validator failed"],
+        "events_summary": {
+            "total_tokens": tokens,
+            "total_cost_usd": cost,
+        },
+    }
+    (reports_dir / f"{name.lower()}_report.json").write_text(json.dumps(report))
+
+
+def _write_quality_report(
+    reports_dir: Path,
+    filename: str,
+    *,
+    name: str,
+    weighted_score: float,
+    tokens: int,
+    cost: float,
+    sample_quality: dict,
+    passed: bool = True,
+):
+    report = {
+        "name": name,
+        "passed": passed,
+        "run_id": filename,
+        "checks_passed": [f"[RUBRIC] weighted_score: {weighted_score:.2f}"],
+        "checks_failed": [] if passed else ["validator failed"],
+        "sample_quality": sample_quality,
+        "events_summary": {
+            "total_tokens": tokens,
+            "total_cost_usd": cost,
+            "artifact_references": {"report": f"reports/{filename}.json"},
+        },
+    }
+    (reports_dir / f"{filename}.json").write_text(json.dumps(report), encoding="utf-8")
+
+
+def test_compare_report_includes_spend_summary(tmp_path: Path):
+    experiment = tmp_path / "experiment"
+    reports = experiment / "reports"
+    reports.mkdir(parents=True)
+    _write_report(reports, "CONTROL", 100, 0.01)
+    _write_report(reports, "COMET_FULL_040_BETA", 200, 0.02)
+    _write_report(reports, "COMET_FULL_039", 300, 0.03)
+
+    report = build_report(experiment)
+
+    assert "## Spend summary" in report
+    assert "| Treatment | Runs | Tokens | Cost | Avg Tokens/Run | Avg Cost/Run |" in report
+    assert "| COMET_FULL_040_BETA | 1 | 200 | $0.0200 | 200 | $0.0200 |" in report
+
+
+def test_compare_report_reads_utf8_report_json(tmp_path: Path):
+    experiment = tmp_path / "experiment"
+    reports = experiment / "reports"
+    reports.mkdir(parents=True)
+    report = {
+        "name": "comet-fix-median-COMET_FULL_040_BETA",
+        "passed": True,
+        "checks_passed": ["[RUBRIC] weighted_score: 1.00 - 通过"],
+        "checks_failed": [],
+        "events_summary": {
+            "total_tokens": 200,
+            "total_cost_usd": 0.02,
+            "artifact_references": {"report": "报告路径"},
+        },
+    }
+    (reports / "utf8_report.json").write_text(
+        json.dumps(report, ensure_ascii=False),
+        encoding="utf-8",
+    )
+
+    output = build_report(experiment)
+
+    assert "报告路径" in output
+
+
+def test_compare_report_handles_partial_llm_judge_overlay_scores(tmp_path: Path):
+    experiment = tmp_path / "experiment"
+    reports = experiment / "reports"
+    reports.mkdir(parents=True)
+    _write_report(reports, "CONTROL", 100, 0.01)
+    _write_report(reports, "COMET_FULL_039", 300, 0.03)
+    workflow = {
+        "name": "comet-fix-median-COMET_FULL_040_BETA",
+        "passed": True,
+        "checks_passed": ["[RUBRIC-JUDGE] artifact_quality: 0.75 - ok"],
+        "checks_failed": [],
+        "events_summary": {},
+    }
+    (reports / "workflow_judge_only.json").write_text(
+        json.dumps(workflow),
+        encoding="utf-8",
+    )
+
+    report = build_report(experiment)
+
+    assert "## LLM-judge overlay (rule vs judge)" in report
+    assert "| artifact_quality | COMET_FULL_040_BETA | — | 0.75 | — |" in report
+
+
+def test_compare_report_includes_metric_guide_and_runtime_summary(tmp_path: Path):
+    experiment = tmp_path / "experiment"
+    reports = experiment / "reports"
+    reports.mkdir(parents=True)
+    _write_report(reports, "CONTROL", 100, 0.01)
+    _write_report(reports, "COMET_FULL_040_BETA", 200, 0.02)
+    _write_report(reports, "COMET_FULL_039", 300, 0.03)
+
+    for path in reports.glob("*.json"):
+        report = json.loads(path.read_text(encoding="utf-8"))
+        report["events_summary"].update(
+            {
+                "num_turns": 6,
+                "duration_seconds": 120,
+                "tool_calls": 18,
+            }
+        )
+        path.write_text(json.dumps(report), encoding="utf-8")
+
+    report = build_report(experiment)
+
+    assert "## Metric guide" in report
+    assert "| Metric | Meaning | Source | Report section |" in report
+    assert "| `raw runs` | All discovered report JSON files before quality filtering. | report files | Data quality summary |" in report
+    assert "| `business_completion` | Business validator pass rate; CONTROL is evaluated on this without requiring Comet workflow artifacts. | `[RUBRIC] business_completion` | Rubric dimensions / pass@k |" in report
+    assert "| `workflow_completion` | Comet workflow validator pass rate; `/` means not applicable for CONTROL. | `[RUBRIC] workflow_completion` | Rubric dimensions / pass@k |" in report
+    assert "## Runtime summary" in report
+    assert "| Treatment | Runs | Turns | Duration | Tool Calls | Avg Turns/Run | Avg Duration/Run | Avg Tool Calls/Run |" in report
+    assert "| COMET_FULL_040_BETA | 1 | 6 | 120s | 18 | 6.0 | 120s | 18.0 |" in report
+
+
+def test_compare_report_includes_rubric_dimension_guide(tmp_path: Path):
+    experiment = tmp_path / "experiment"
+    reports = experiment / "reports"
+    reports.mkdir(parents=True)
+    _write_report(reports, "CONTROL", 100, 0.01)
+    _write_report(reports, "COMET_FULL_040_BETA", 200, 0.02)
+    _write_report(reports, "COMET_FULL_039", 300, 0.03)
+
+    report = build_report(experiment)
+    html = render_markdown_html(report, title="Comet Baseline Comparison Report")
+
+    assert "### Dimension guide" in report
+    assert "| main_flow | Completion of the expected Comet workflow phases. |" in report
+    assert "| business_completion | Business validator pass rate for the requested task behavior. |" in report
+    assert "<h3>维度说明</h3>" in html
+    assert "业务 validator 对用户请求行为的通过率" in html
+    assert "Workflow validator 对 Comet workflow 产物的通过率" in html
+
+
+def test_compare_report_normalizes_repeated_run_suffixes(tmp_path: Path):
+    experiment = tmp_path / "experiment"
+    reports = experiment / "reports"
+    reports.mkdir(parents=True)
+    report = {
+        "name": "comet-fix-median-COMET_FULL_040_BETA-r1",
+        "passed": True,
+        "checks_passed": ["[RUBRIC] weighted_score: 1.00"],
+        "checks_failed": [],
+        "events_summary": {"total_tokens": 200, "total_cost_usd": 0.02},
+    }
+    (reports / "workflow_report.json").write_text(json.dumps(report))
+
+    output = build_report(experiment)
+
+    assert "| COMET_FULL_040_BETA | 1 |" in output
+    assert "| comet-fix-median | — | PASS | — |" in output
+    html = render_markdown_html(output, title="Comet Baseline Comparison Report")
+    assert "Task outcome matrix" in html
+    assert "N/A" in html
+
+
+def test_compare_report_includes_task_outcome_table(tmp_path: Path):
+    experiment = tmp_path / "experiment"
+    reports = experiment / "reports"
+    reports.mkdir(parents=True)
+    _write_report(reports, "CONTROL", 100, 0.01)
+    _write_report(reports, "COMET_FULL_040_BETA", 200, 0.02)
+    _write_report(reports, "COMET_FULL_039", 300, 0.03, passed=False)
+
+    report = build_report(experiment)
+
+    assert "## Task outcomes" in report
+    assert "| Task | CONTROL | COMET_FULL_040_BETA | COMET_FULL_039 |" in report
+    assert "| comet-fix-median | PASS | PASS | FAIL |" in report
+
+
+def test_compare_report_splits_pass_metrics_by_business_and_workflow(tmp_path: Path):
+    experiment = tmp_path / "experiment"
+    reports = experiment / "reports"
+    reports.mkdir(parents=True)
+
+    def write_run(
+        filename: str,
+        *,
+        treatment: str,
+        business_score: float,
+        workflow_score: float | None,
+        passed: bool,
+    ):
+        checks = [f"[RUBRIC] business_completion: {business_score:.2f} - demo"]
+        if workflow_score is None:
+            checks.append("[RUBRIC] workflow_completion: N/A - CONTROL business-only baseline")
+        else:
+            checks.append(f"[RUBRIC] workflow_completion: {workflow_score:.2f} - demo")
+        report = {
+            "name": f"comet-fix-median-{treatment}",
+            "passed": passed,
+            "checks_passed": checks,
+            "checks_failed": [] if passed else ["validator failed"],
+            "events_summary": {},
+        }
+        (reports / filename).write_text(json.dumps(report), encoding="utf-8")
+
+    write_run(
+        "control_1.json",
+        treatment="CONTROL",
+        business_score=1.0,
+        workflow_score=None,
+        passed=True,
+    )
+    write_run(
+        "control_2.json",
+        treatment="CONTROL",
+        business_score=0.0,
+        workflow_score=None,
+        passed=False,
+    )
+    write_run(
+        "workflow_1.json",
+        treatment="COMET_FULL_040_BETA",
+        business_score=1.0,
+        workflow_score=1.0,
+        passed=True,
+    )
+    write_run(
+        "workflow_2.json",
+        treatment="COMET_FULL_040_BETA",
+        business_score=1.0,
+        workflow_score=0.0,
+        passed=False,
+    )
+    write_run(
+        "baseline_1.json",
+        treatment="COMET_FULL_039",
+        business_score=1.0,
+        workflow_score=1.0,
+        passed=True,
+    )
+    write_run(
+        "baseline_2.json",
+        treatment="COMET_FULL_039",
+        business_score=1.0,
+        workflow_score=1.0,
+        passed=True,
+    )
+
+    report = build_report(experiment)
+
+    assert "| Metric | Treatment | pass@1 | pass@2 | pass^1 | pass^2 | pass/fail |" in report
+    assert "| overall | COMET_FULL_040_BETA | 0.50 | 1.00 | 0 | 0 | 1/2 |" in report
+    assert "| business | COMET_FULL_040_BETA | 1.00 | 1.00 | 1 | 1 | 2/2 |" in report
+    assert "| workflow | CONTROL | / | / | / | / | / |" in report
+    assert "| workflow | COMET_FULL_040_BETA | 0.50 | 1.00 | 0 | 0 | 1/2 |" in report
+
+
+def test_compare_report_includes_pass_at_3_when_three_runs_exist(tmp_path: Path):
+    experiment = tmp_path / "experiment"
+    reports = experiment / "reports"
+    reports.mkdir(parents=True)
+
+    for rep, passed in enumerate((True, False, False), start=1):
+        _write_report(reports, f"CONTROL-r{rep}", 100, 0.01, passed=passed)
+    for rep, passed in enumerate((True, True, False), start=1):
+        _write_report(reports, f"COMET_FULL_040_BETA-r{rep}", 200, 0.02, passed=passed)
+    for rep, passed in enumerate((True, True, True), start=1):
+        _write_report(reports, f"COMET_FULL_039-r{rep}", 300, 0.03, passed=passed)
+
+    report = build_report(experiment)
+
+    assert (
+        "| Metric | Treatment | pass@1 | pass@2 | pass@3 | pass^1 | pass^2 | pass^3 | pass/fail |"
+        in report
+    )
+    assert "| overall | CONTROL | 0.33 | 0.67 | 1.00 | 0 | 0 | 0 | 1/3 |" in report
+    assert "| overall | COMET_FULL_040_BETA | 0.67 | 1.00 | 1.00 | 0 | 0 | 0 | 2/3 |" in report
+    assert "| overall | COMET_FULL_039 | 1.00 | 1.00 | 1.00 | 1 | 1 | 1 | 3/3 |" in report
+
+
+def test_compare_report_honors_html_report_output_config(monkeypatch, tmp_path: Path):
+    monkeypatch.setenv("BENCH_LOGS_DIR", str(tmp_path))
+    experiment = tmp_path / "experiments" / "exp1"
+    reports = experiment / "reports"
+    reports.mkdir(parents=True)
+    _write_report(reports, "CONTROL", 100, 0.01)
+    _write_report(reports, "COMET_FULL_040_BETA", 200, 0.02)
+    _write_report(reports, "COMET_FULL_039", 300, 0.03)
+    config = tmp_path / "report-config.json"
+    config.write_text(json.dumps({"report_outputs": {"markdown": False, "html": True}}))
+
+    result = main(["--experiment", "exp1", "--report-config", str(config)])
+
+    assert result == 0
+    assert not (experiment / "comparison_report.md").exists()
+    html_report = experiment / "comparison_report.html"
+    assert html_report.exists()
+    html = html_report.read_text(encoding="utf-8")
+    assert "Comet Baseline Comparison Report" in html
+    assert "paper-figure" in html
+    assert 'data-chart-backend="python"' in html
+    assert "Rubric dimension deltas" in html
+    assert "Task outcome matrix" in html
+    assert "<svg" in html
+    assert "matplotlib" not in html.lower()
+    assert "chart.js" not in html.lower()
+
+
+def test_html_report_falls_back_when_python_chart_backend_disabled(monkeypatch, tmp_path: Path):
+    monkeypatch.setenv("COMET_EVAL_REPORT_CHART_BACKEND", "inline")
+    experiment = tmp_path / "experiment"
+    reports = experiment / "reports"
+    reports.mkdir(parents=True)
+    _write_report(reports, "CONTROL", 100, 0.01)
+    _write_report(reports, "COMET_FULL_040_BETA", 200, 0.02)
+    _write_report(reports, "COMET_FULL_039", 300, 0.03)
+
+    markdown = build_report(experiment)
+    html = render_markdown_html(markdown, title="Comet Baseline Comparison Report")
+
+    assert 'data-chart-backend="inline-svg"' in html
+    assert "Task outcome matrix" in html
+
+
+def test_compare_report_uses_structured_failure_attribution(tmp_path: Path):
+    experiment = tmp_path / "experiment"
+    reports = experiment / "reports"
+    reports.mkdir(parents=True)
+    _write_report(reports, "CONTROL", 100, 0.01)
+    _write_report(reports, "COMET_FULL_039", 300, 0.03)
+    workflow = {
+        "name": "comet-full-workflow-COMET_FULL_040_BETA",
+        "passed": False,
+        "checks_passed": [],
+        "checks_failed": ["Required skill not invoked: comet"],
+        "events_summary": {
+            "total_tokens": 200,
+            "total_cost_usd": 0.02,
+            "failure_attribution": [
+                {
+                    "bucket": "harness",
+                    "check": "Required skill not invoked: comet",
+                    "reason": "target Skill was never invoked",
+                }
+            ],
+        },
+    }
+    (reports / "COMET_FULL_040_BETA_report.json").write_text(json.dumps(workflow))
+
+    report = build_report(experiment)
+
+    assert "**harness**" in report
+    assert "[harness] target Skill was never invoked" in report
+
+
+def test_compare_report_groups_repeated_failure_attribution(tmp_path: Path):
+    experiment = tmp_path / "experiment"
+    reports = experiment / "reports"
+    reports.mkdir(parents=True)
+    _write_report(reports, "CONTROL", 100, 0.01)
+    _write_report(reports, "COMET_FULL_039", 300, 0.03)
+    for index in range(2):
+        workflow = {
+            "name": f"comet-full-workflow-COMET_FULL_040_BETA-r{index}",
+            "passed": False,
+            "checks_passed": [],
+            "checks_failed": ["Required Superpowers dependency skill not invoked"],
+            "events_summary": {
+                "total_tokens": 200,
+                "total_cost_usd": 0.02,
+                "failure_attribution": [
+                    {
+                        "bucket": "workflow",
+                        "check": "Required Superpowers dependency skill not invoked",
+                        "reason": "Skill invocation contract failed",
+                    }
+                ],
+            },
+        }
+        (reports / f"COMET_FULL_040_BETA_{index}_report.json").write_text(
+            json.dumps(workflow)
+        )
+
+    report = build_report(experiment)
+
+    assert "- **workflow** (2):" in report
+    assert (
+        "x2 Required Superpowers dependency skill not invoked  ->  "
+        "[workflow] Skill invocation contract failed"
+    ) in report
+
+
+def test_compare_report_labels_failure_attribution_as_run_level_checks(tmp_path: Path):
+    experiment = tmp_path / "experiment"
+    reports = experiment / "reports"
+    reports.mkdir(parents=True)
+    _write_report(reports, "CONTROL", 100, 0.01)
+    _write_report(reports, "COMET_FULL_039", 300, 0.03)
+    _write_report(reports, "COMET_FULL_040_BETA", 200, 0.02, passed=False)
+
+    report = build_report(experiment)
+    html = render_markdown_html(report, title="Comet Baseline Comparison Report")
+
+    assert "## Run-level failed checks" in report
+    assert (
+        "These are sample-level `checks_failed` entries. They can coexist with "
+        "`workflow_completion == 1.00`, `pass@k == 1.00`, or a passing task outcome"
+        in report
+    )
+    assert "<h2>样本级失败检查</h2>" in html
+    assert "它们可以与 <code>workflow_completion == 1.00</code>、<code>pass@k == 1.00</code>" in html
+
+
+def test_compare_report_lists_source_evidence(tmp_path: Path):
+    experiment = tmp_path / "experiment"
+    reports = experiment / "reports"
+    reports.mkdir(parents=True)
+    _write_report(reports, "CONTROL", 100, 0.01)
+    _write_report(reports, "COMET_FULL_039", 300, 0.03)
+    workflow = {
+        "name": "comet-full-workflow-COMET_FULL_040_BETA",
+        "passed": True,
+        "run_id": "run-123",
+        "checks_passed": ["[RUBRIC] weighted_score: 1.00 - ok"],
+        "checks_failed": [],
+        "events_summary": {
+            "profile": "comet-workflow",
+            "skill_sources": [{"name": "comet", "hash": "sha256:abc"}],
+            "eval_manifest": "demo/comet/eval.yaml",
+            "artifact_references": {"report": "reports/COMET_FULL_040_BETA_report.json"},
+            "total_tokens": 200,
+            "total_cost_usd": 0.02,
+        },
+    }
+    (reports / "COMET_FULL_040_BETA_report.json").write_text(json.dumps(workflow))
+
+    report = build_report(experiment)
+
+    assert "## Source evidence" in report
+    assert "Use this section to trace each aggregate metric back to the raw run artifacts." in report
+    assert "`Quality` is the sample-quality status used by the analysis-set filter." in report
+    assert "`Skill sources` records installed Skill identity or hash evidence when the run provides it." in report
+    assert "`run-123`" in report
+    assert "comet-workflow" in report
+    assert "sha256:abc" in report
+    assert "reports/COMET_FULL_040_BETA_report.json" in report
+
+
+def test_compare_report_overall_uses_weighted_score(tmp_path: Path):
+    experiment = tmp_path / "experiment"
+    reports = experiment / "reports"
+    reports.mkdir(parents=True)
+
+    workflow = {
+        "name": "comet-full-workflow-COMET_FULL_040_BETA",
+        "passed": True,
+        "checks_passed": [
+            "[RUBRIC] main_flow: 1.00 - ok",
+            "[RUBRIC] gate_guard: 1.00 - ok",
+            "[RUBRIC] skill_invocation: 1.00 - ok",
+            "[RUBRIC] spec_drift: 1.00 - ok",
+            "[RUBRIC] business_completion: 1.00 - ok",
+            "[RUBRIC] workflow_completion: 1.00 - ok",
+            "[RUBRIC] efficiency: 1.00 - ok",
+            "[RUBRIC] decision_point_compliance: 1.00 - ok",
+            "[RUBRIC] artifact_quality: 1.00 - ok",
+            "[RUBRIC] recovery_resilience: 1.00 - ok",
+            "[RUBRIC] weighted_score: 0.25",
+        ],
+        "checks_failed": [],
+        "events_summary": {},
+    }
+    baseline = {
+        "name": "comet-full-workflow-COMET_FULL_039",
+        "passed": True,
+        "checks_passed": [
+            "[RUBRIC] main_flow: 0.00 - missing",
+            "[RUBRIC] gate_guard: 0.00 - missing",
+            "[RUBRIC] skill_invocation: 0.00 - missing",
+            "[RUBRIC] spec_drift: 0.00 - missing",
+            "[RUBRIC] business_completion: 0.00 - missing",
+            "[RUBRIC] workflow_completion: 0.00 - missing",
+            "[RUBRIC] efficiency: 0.00 - missing",
+            "[RUBRIC] decision_point_compliance: 0.00 - missing",
+            "[RUBRIC] artifact_quality: 0.00 - missing",
+            "[RUBRIC] recovery_resilience: 0.00 - missing",
+            "[RUBRIC] weighted_score: 0.75",
+        ],
+        "checks_failed": [],
+        "events_summary": {},
+    }
+    (reports / "workflow_report.json").write_text(json.dumps(workflow))
+    (reports / "baseline_report.json").write_text(json.dumps(baseline))
+
+    report = build_report(experiment)
+
+    assert "| **Overall** | — | 0.25 | 0.75 | -0.50 |" in report
+
+
+def test_compare_report_marks_control_workflow_dimensions_with_slash(tmp_path: Path):
+    experiment = tmp_path / "experiment"
+    reports = experiment / "reports"
+    reports.mkdir(parents=True)
+
+    control = {
+        "name": "comet-full-workflow-CONTROL",
+        "passed": True,
+        "checks_passed": [
+            "[RUBRIC] main_flow: N/A - CONTROL business-only baseline",
+            "[RUBRIC] gate_guard: N/A - CONTROL business-only baseline",
+            "[RUBRIC] skill_invocation: N/A - CONTROL business-only baseline",
+            "[RUBRIC] spec_drift: N/A - CONTROL business-only baseline",
+            "[RUBRIC] business_completion: 1.00 - ok",
+            "[RUBRIC] workflow_completion: N/A - CONTROL business-only baseline",
+            "[RUBRIC] efficiency: 1.00 - ok",
+            "[RUBRIC] weighted_score: 1.00",
+        ],
+        "checks_failed": [],
+        "events_summary": {},
+    }
+    workflow = {
+        "name": "comet-full-workflow-COMET_FULL_040_BETA",
+        "passed": True,
+        "checks_passed": [
+            "[RUBRIC] main_flow: 1.00 - ok",
+            "[RUBRIC] gate_guard: 1.00 - ok",
+            "[RUBRIC] skill_invocation: 1.00 - ok",
+            "[RUBRIC] spec_drift: 1.00 - ok",
+            "[RUBRIC] business_completion: 1.00 - ok",
+            "[RUBRIC] workflow_completion: 1.00 - ok",
+            "[RUBRIC] efficiency: 1.00 - ok",
+            "[RUBRIC] weighted_score: 1.00",
+        ],
+        "checks_failed": [],
+        "events_summary": {},
+    }
+    baseline = dict(workflow, name="comet-full-workflow-COMET_FULL_039")
+    (reports / "control_report.json").write_text(json.dumps(control))
+    (reports / "workflow_report.json").write_text(json.dumps(workflow))
+    (reports / "baseline_report.json").write_text(json.dumps(baseline))
+
+    report = build_report(experiment)
+
+    assert "| main_flow | / | 1.00 | 1.00 | +0.00 |" in report
+    assert "| skill_invocation | / | 1.00 | 1.00 | +0.00 |" in report
+    assert "| business_completion | 1.00 | 1.00 | 1.00 | +0.00 |" in report
+    assert "| workflow_completion | / | 1.00 | 1.00 | +0.00 |" in report
+    assert "| completion |" not in report
+
+
+def test_compare_report_excludes_hard_noise_from_analysis_metrics(tmp_path: Path):
+    experiment = tmp_path / "experiment"
+    reports = experiment / "reports"
+    reports.mkdir(parents=True)
+    _write_report(reports, "CONTROL", 100, 0.01)
+    _write_quality_report(
+        reports,
+        "workflow_clean",
+        name="comet-fix-median-COMET_FULL_040_BETA",
+        weighted_score=1.0,
+        tokens=200,
+        cost=0.02,
+        sample_quality={
+            "status": "included",
+            "reason_code": "valid_signal",
+            "reason": "clean",
+            "include_in_analysis": True,
+            "confidence": "high",
+            "evidence": ["result event present"],
+        },
+    )
+    _write_quality_report(
+        reports,
+        "workflow_timeout",
+        name="comet-fix-median-COMET_FULL_040_BETA-r1",
+        weighted_score=0.0,
+        tokens=999,
+        cost=9.99,
+        sample_quality={
+            "status": "excluded",
+            "reason_code": "api_timeout",
+            "reason": "timeout",
+            "include_in_analysis": False,
+            "confidence": "high",
+            "evidence": ["stderr timeout"],
+        },
+        passed=False,
+    )
+    _write_report(reports, "COMET_FULL_039", 300, 0.03)
+
+    report = build_report(experiment)
+
+    assert "## Data quality summary" in report
+    assert "| COMET_FULL_040_BETA | 2 | 1 | 0 | 1 |" in report
+    assert "| COMET_FULL_040_BETA | 1 |" in report
+    assert "| COMET_FULL_040_BETA | 1 | 200 | $0.0200 | 200 | $0.0200 |" in report
+    assert "workflow_timeout" in report
+    assert "api_timeout" in report
+    assert "999" not in report.split("## Spend summary", 1)[1].split("## Source evidence", 1)[0]
+
+
+def test_compare_report_lists_flagged_runs_without_excluding_them(tmp_path: Path):
+    experiment = tmp_path / "experiment"
+    reports = experiment / "reports"
+    reports.mkdir(parents=True)
+    _write_report(reports, "CONTROL", 100, 0.01)
+    _write_report(reports, "COMET_FULL_039", 300, 0.03)
+    _write_quality_report(
+        reports,
+        "workflow_flagged",
+        name="comet-full-workflow-COMET_FULL_040_BETA",
+        weighted_score=0.5,
+        tokens=200,
+        cost=0.02,
+        sample_quality={
+            "status": "flagged",
+            "reason_code": "harness_trigger_suspect",
+            "reason": "target Skill may not have been triggered",
+            "include_in_analysis": True,
+            "confidence": "medium",
+            "evidence": ["target Skill was never invoked"],
+        },
+    )
+
+    report = build_report(experiment)
+
+    assert "## Flagged runs" in report
+    assert "harness_trigger_suspect" in report
+    assert "| Treatment | Raw runs | Analysis set | Flagged | Excluded |" in report
+    assert "| COMET_FULL_040_BETA | 1 | 1 | 1 | 0 |" in report
+    assert "| COMET_FULL_040_BETA | 1 | 200 | $0.0200 | 200 | $0.0200 |" in report
+
+
+def test_compare_report_summarizes_multiline_loop_evidence(tmp_path: Path):
+    experiment = tmp_path / "experiment"
+    reports = experiment / "reports"
+    reports.mkdir(parents=True)
+    _write_report(reports, "CONTROL", 100, 0.01)
+    _write_report(reports, "COMET_FULL_039", 300, 0.03)
+    _write_quality_report(
+        reports,
+        "workflow_flagged",
+        name="comet-full-workflow-COMET_FULL_040_BETA",
+        weighted_score=0.5,
+        tokens=200,
+        cost=0.02,
+        sample_quality={
+            "status": "flagged",
+            "reason_code": "completed_run_mentions_outer_failure",
+            "reason": "completed with suspicious runner output",
+            "include_in_analysis": True,
+            "confidence": "medium",
+            "evidence": [
+                "[loop] turn 1/12\n"
+                "[loop] decision point detected; simulating user reply\n"
+                "[loop] simulated reply (42 chars)\n"
+                "[loop] turn 2/12\n"
+                "[loop] workflow appears complete; ending\n"
+                "[loop] finished after 2 turns"
+            ],
+        },
+    )
+
+    report = build_report(experiment)
+
+    assert "loop trace: 2 turns; 1 simulated decision reply; see report JSON for full evidence" in report
+    flagged_section = report.split("## Flagged runs", 1)[1].split("## Raw vs analysis", 1)[0]
+    assert "[loop] decision point detected" not in flagged_section
+
+
+def test_compare_report_reports_insufficient_clean_data(tmp_path: Path):
+    experiment = tmp_path / "experiment"
+    reports = experiment / "reports"
+    reports.mkdir(parents=True)
+    _write_report(reports, "CONTROL", 100, 0.01)
+    _write_report(reports, "COMET_FULL_039", 300, 0.03)
+    _write_quality_report(
+        reports,
+        "workflow_timeout",
+        name="comet-full-workflow-COMET_FULL_040_BETA",
+        weighted_score=0.0,
+        tokens=999,
+        cost=9.99,
+        sample_quality={
+            "status": "excluded",
+            "reason_code": "runner_timeout",
+            "reason": "timeout",
+            "include_in_analysis": False,
+            "confidence": "high",
+            "evidence": ["Timeout after 600s"],
+        },
+        passed=False,
+    )
+
+    report = build_report(experiment)
+
+    assert "Insufficient clean data" in report
+    assert "COMET_FULL_040_BETA" in report
+
+
+def test_compare_report_keeps_data_quality_sections_when_all_key_runs_are_excluded(tmp_path: Path):
+    experiment = tmp_path / "experiment"
+    reports = experiment / "reports"
+    reports.mkdir(parents=True)
+    _write_quality_report(
+        reports,
+        "workflow_timeout",
+        name="comet-full-workflow-COMET_FULL_040_BETA",
+        weighted_score=0.0,
+        tokens=999,
+        cost=9.99,
+        sample_quality={
+            "status": "excluded",
+            "reason_code": "runner_timeout",
+            "reason": "timeout",
+            "include_in_analysis": False,
+            "confidence": "high",
+            "evidence": ["Timeout after 600s"],
+        },
+        passed=False,
+    )
+    _write_quality_report(
+        reports,
+        "baseline_timeout",
+        name="comet-full-workflow-COMET_FULL_039",
+        weighted_score=0.0,
+        tokens=888,
+        cost=8.88,
+        sample_quality={
+            "status": "excluded",
+            "reason_code": "api_timeout",
+            "reason": "timeout",
+            "include_in_analysis": False,
+            "confidence": "high",
+            "evidence": ["stderr timeout"],
+        },
+        passed=False,
+    )
+
+    report = build_report(experiment)
+
+    assert "## Data quality summary" in report
+    assert "## Excluded runs" in report
+    assert "## Source evidence" in report
+    assert "Insufficient clean data" in report
+    assert "No report data found" not in report
+
+
+def test_html_report_includes_data_quality_summary(tmp_path: Path):
+    experiment = tmp_path / "experiment"
+    reports = experiment / "reports"
+    reports.mkdir(parents=True)
+    _write_report(reports, "CONTROL", 100, 0.01)
+    _write_report(reports, "COMET_FULL_040_BETA", 200, 0.02)
+    _write_report(reports, "COMET_FULL_039", 300, 0.03)
+
+    markdown = build_report(experiment)
+    html = render_markdown_html(markdown, title="Comet Baseline Comparison Report")
+
+    assert "Data quality summary" in html
+    assert "paper-figure" in html
+
+
+def test_html_report_wraps_wide_evidence_tables(tmp_path: Path):
+    experiment = tmp_path / "experiment"
+    reports = experiment / "reports"
+    reports.mkdir(parents=True)
+    _write_report(reports, "CONTROL", 100, 0.01)
+    _write_report(reports, "COMET_FULL_039", 300, 0.03)
+    _write_quality_report(
+        reports,
+        "workflow_flagged",
+        name="comet-full-workflow-COMET_FULL_040_BETA",
+        weighted_score=0.5,
+        tokens=200,
+        cost=0.02,
+        sample_quality={
+            "status": "flagged",
+            "reason_code": "completed_run_mentions_outer_failure",
+            "reason": "completed with suspicious runner output",
+            "include_in_analysis": True,
+            "confidence": "medium",
+            "evidence": ["a-very-long-evidence-token-without-natural-breaks-" * 8],
+        },
+    )
+
+    markdown = build_report(experiment)
+    html = render_markdown_html(markdown, title="Comet Baseline Comparison Report")
+
+    assert 'class="table-scroll"' in html
+    assert 'class="col-evidence"' in html
+    assert 'class="col-report"' in html
+    assert "overflow-wrap: anywhere" in html
+
+
+def test_html_report_includes_bilingual_language_toggle(tmp_path: Path):
+    experiment = tmp_path / "experiment"
+    reports = experiment / "reports"
+    reports.mkdir(parents=True)
+    _write_report(reports, "CONTROL", 100, 0.01)
+    _write_report(reports, "COMET_FULL_040_BETA", 200, 0.02)
+    _write_report(reports, "COMET_FULL_039", 300, 0.03)
+
+    markdown = build_report(experiment)
+    html = render_markdown_html(markdown, title="Comet Baseline Comparison Report")
+
+    assert '<html lang="zh-CN" data-lang="zh">' in html
+    assert 'class="language-toggle"' in html
+    assert 'data-set-lang="zh"' in html
+    assert 'data-set-lang="en"' in html
+    assert 'data-locale="zh"' in html
+    assert 'data-locale="en"' in html
+    assert "Comet 基线对比报告" in html
+    assert "Comet Baseline Comparison Report" in html
+    assert "pass@k / pass^k — 能力上限 vs 可靠性下限" in html
+    assert "指标说明" in html
+    assert "运行摘要" in html
+    assert "业务 validator 通过率" in html
+
+
+def test_html_report_uses_conference_paper_layout(tmp_path: Path):
+    experiment = tmp_path / "experiment"
+    reports = experiment / "reports"
+    reports.mkdir(parents=True)
+    _write_report(reports, "CONTROL", 100, 0.01)
+    _write_report(reports, "COMET_FULL_040_BETA", 200, 0.02)
+    _write_report(reports, "COMET_FULL_039", 300, 0.03)
+
+    markdown = build_report(experiment)
+    html = render_markdown_html(markdown, title="Comet Baseline Comparison Report")
+
+    assert "--page-width: 980px;" in html
+    assert "linear-gradient(90deg" not in html
+    assert "box-shadow: none;" in html
+    assert "border-top: 2px solid var(--ink);" in html
+    assert html.index("<h1>Comet 基线对比报告</h1>") < html.index('class="paper-figures"')
+
+
+def test_html_report_includes_paper_abstract_and_centers_figures(tmp_path: Path):
+    experiment = tmp_path / "experiment"
+    reports = experiment / "reports"
+    reports.mkdir(parents=True)
+    _write_report(reports, "CONTROL", 100, 0.01)
+    _write_report(reports, "COMET_FULL_040_BETA", 200, 0.02)
+    _write_report(reports, "COMET_FULL_039", 300, 0.03)
+
+    markdown = build_report(experiment)
+    html = render_markdown_html(markdown, title="Comet Baseline Comparison Report")
+
+    assert '<section class="paper-abstract">' in html
+    assert "<strong>Abstract.</strong>" in html
+    assert "<strong>摘要。</strong>" in html
+    assert "justify-items: center;" in html
+    assert "margin-left: auto;" in html
+    assert "margin-right: auto;" in html
+    assert html.index("<h1>Comet 基线对比报告</h1>") < html.index('<section class="paper-abstract">')
+    assert html.index('<section class="paper-abstract">') < html.index('class="paper-figures"')
+
+
+def test_html_report_centers_markdown_tables(tmp_path: Path):
+    experiment = tmp_path / "experiment"
+    reports = experiment / "reports"
+    reports.mkdir(parents=True)
+    _write_report(reports, "CONTROL", 100, 0.01)
+    _write_report(reports, "COMET_FULL_040_BETA", 200, 0.02)
+    _write_report(reports, "COMET_FULL_039", 300, 0.03)
+
+    markdown = build_report(experiment)
+    html = render_markdown_html(markdown, title="Comet Baseline Comparison Report")
+
+    assert "width: max-content;" in html
+    assert "max-width: 100%;" in html
+    assert "margin: 0 auto 1rem;" in html
+    assert "margin: 0 auto;" in html
+    assert "text-align: center;" in html
+    assert ".col-task," in html
+
+
+def test_html_report_renders_localized_emphasis_without_literal_underscore_markers(tmp_path: Path):
+    experiment = tmp_path / "experiment"
+    reports = experiment / "reports"
+    reports.mkdir(parents=True)
+    _write_report(reports, "CONTROL", 100, 0.01)
+    _write_report(reports, "COMET_FULL_040_BETA", 200, 0.02)
+    _write_report(reports, "COMET_FULL_039", 300, 0.03)
+
+    markdown = build_report(experiment)
+    html = render_markdown_html(markdown, title="Comet Baseline Comparison Report")
+
+    assert "<p>_分数是二值通过率" not in html
+    assert "权重见下方）。_</p>" not in html
+    assert "<p><em>分数是二值通过率" in html
+    assert "weighted_score" in html
+
+
+def test_html_report_keeps_chinese_chart_text_utf8(tmp_path: Path):
+    experiment = tmp_path / "experiment"
+    reports = experiment / "reports"
+    reports.mkdir(parents=True)
+    _write_report(reports, "CONTROL", 100, 0.01)
+    _write_report(reports, "COMET_FULL_040_BETA", 200, 0.02)
+    _write_report(reports, "COMET_FULL_039", 300, 0.03)
+
+    markdown = build_report(experiment)
+    html = render_markdown_html(markdown, title="Comet Baseline Comparison Report")
+
+    assert "图 1. Rubric 维度差异" in html
+    assert "图 3. 任务结果矩阵" in html
+    assert "ͼ 1." not in html
+    assert "����" not in html
+
+
+def test_html_report_localizes_failure_attribution_in_chinese_layer(tmp_path: Path):
+    experiment = tmp_path / "experiment"
+    reports = experiment / "reports"
+    reports.mkdir(parents=True)
+    _write_report(reports, "COMET_FULL_039", 300, 0.03)
+    control = {
+        "name": "comet-fix-median-CONTROL",
+        "passed": False,
+        "checks_passed": [],
+        "checks_failed": ["business validator failed"],
+        "events_summary": {},
+    }
+    workflow = {
+        "name": "comet-fix-median-COMET_FULL_040_BETA",
+        "passed": False,
+        "checks_passed": [],
+        "checks_failed": ["workflow validator failed"],
+        "events_summary": {},
+    }
+    (reports / "control_report.json").write_text(json.dumps(control), encoding="utf-8")
+    (reports / "workflow_report.json").write_text(json.dumps(workflow), encoding="utf-8")
+
+    markdown = build_report(experiment)
+    html = render_markdown_html(markdown, title="Comet Baseline Comparison Report")
+    zh = html.split('<section class="localized" data-locale="zh"', 1)[1].split(
+        '<section class="localized" data-locale="en"',
+        1,
+    )[0]
+
+    assert "CONTROL（1 个失败）" in zh
+    assert "business/业务" in zh
+    assert "业务实现未通过" in zh
+    assert "workflow/流程" in zh
+    assert "Workflow 验证未通过" in zh
+    assert "model/模型" not in zh
+    assert "unclassified failure" not in zh
+    assert "failure(s)" not in zh
+
+
+def test_html_report_renders_markdown_emphasis_without_literal_underscores(tmp_path: Path):
+    experiment = tmp_path / "experiment"
+    reports = experiment / "reports"
+    reports.mkdir(parents=True)
+    _write_report(reports, "CONTROL", 100, 0.01)
+    _write_report(reports, "COMET_FULL_040_BETA", 200, 0.02)
+    _write_report(reports, "COMET_FULL_039", 300, 0.03)
+
+    markdown = build_report(experiment)
+    html = render_markdown_html(markdown, title="Comet Baseline Comparison Report")
+
+    assert "<em>仅统计分析集；已排除的硬噪声运行不会进入统计。</em>" in html
+    assert "<em>Analysis set only; excluded hard-noise runs are omitted.</em>" in html
+    assert "<p>_仅统计分析集" not in html
+    assert "<p>_Analysis set only" not in html