@rpamis/comet 0.4.0-beta.1 → 0.4.0-beta.11

package/eval/langsmith/tests/conftest.py

@@ -0,0 +1,187 @@
+"""LangSmith eval suite pytest configuration.
+
+This suite reuses the local task corpus by default and writes reports under
+eval/langsmith/logs. LangSmith-specific environment checks live here so local
+runs stay free of LangSmith requirements.
+"""
+
+import importlib.util
+import os
+import subprocess
+from pathlib import Path
+
+import pytest
+from dotenv import load_dotenv
+from scaffold.python.utils import _to_bash_path
+
+LANGSMITH_ROOT = Path(__file__).resolve().parents[1]
+EVAL_ROOT = LANGSMITH_ROOT.parent
+LOCAL_ROOT = EVAL_ROOT / "local"
+DEFAULT_LANGSMITH_PLUGIN_DIR = EVAL_ROOT / ".cache" / "langsmith-cc-plugin"
+
+os.environ.setdefault("BENCH_SUITE_ROOT", str(LANGSMITH_ROOT))
+os.environ.setdefault("BENCH_TASKS_DIR", str(LOCAL_ROOT / "tasks"))
+os.environ.setdefault("BENCH_TREATMENTS_DIR", str(LOCAL_ROOT / "treatments"))
+os.environ.setdefault("BENCH_SKILLS_DIR", str(LOCAL_ROOT / "skills"))
+os.environ.setdefault("BENCH_LOGS_DIR", str(LANGSMITH_ROOT / "logs"))
+# Group every run in this suite under one LangSmith dataset/test-suite by default.
+os.environ.setdefault("LANGSMITH_TEST_SUITE", "comet-skill-eval")
+
+load_dotenv(EVAL_ROOT / ".env")
+load_dotenv(LANGSMITH_ROOT / ".env", override=True)
+
+
+def configure_langsmith_environment() -> None:
+    """Derive Claude Code plugin env from the eval suite's LangSmith config."""
+    os.environ.setdefault("LANGSMITH_TRACING", "true")
+    os.environ.setdefault(
+        "TRACE_TO_LANGSMITH",
+        "true" if os.environ.get("LANGSMITH_TRACING", "").lower() == "true" else "false",
+    )
+    os.environ.setdefault("LANGSMITH_PROJECT", "comet-skill-eval")
+
+    api_key = os.environ.get("LANGSMITH_API_KEY", "")
+    if api_key:
+        os.environ.setdefault("CC_LANGSMITH_API_KEY", api_key)
+    os.environ.setdefault("CC_LANGSMITH_PROJECT", os.environ["LANGSMITH_PROJECT"])
+
+
+configure_langsmith_environment()
+
+_LOCAL_CONFTEST = LOCAL_ROOT / "tests" / "conftest.py"
+_spec = importlib.util.spec_from_file_location("_comet_local_conftest", _LOCAL_CONFTEST)
+_local_conftest = importlib.util.module_from_spec(_spec)
+assert _spec and _spec.loader
+_spec.loader.exec_module(_local_conftest)
+
+for _name in dir(_local_conftest):
+    if not _name.startswith("__") and _name not in globals():
+        globals()[_name] = getattr(_local_conftest, _name)
+
+
+@pytest.fixture(scope="session", autouse=True)
+def verify_langsmith_environment(request):
+    """Require LangSmith credentials only for non-unit LangSmith eval runs."""
+    if _local_conftest._is_unit_tests_only(request.config):
+        return
+    if not os.environ.get("LANGSMITH_API_KEY"):
+        pytest.skip("LANGSMITH_API_KEY not set")
+
+
+def _build_default_langsmith_plugin(target_dir: Path) -> bool:
+    """Build the official Claude Code LangSmith plugin into eval/.cache."""
+    target_dir.parent.mkdir(parents=True, exist_ok=True)
+    mount_arg = f"{target_dir.parent}:/out"
+    command = (
+        "set -e; "
+        "cd /tmp; "
+        "git clone --depth 1 https://github.com/langchain-ai/langsmith-claude-code-plugins; "
+        "cd langsmith-claude-code-plugins; "
+        "corepack enable; "
+        "pnpm install; "
+        "pnpm build; "
+        "rm -rf /out/langsmith-cc-plugin; "
+        "cp -r . /out/langsmith-cc-plugin"
+    )
+    result = subprocess.run(
+        ["docker", "run", "--rm", "-v", mount_arg, "node:20", "sh", "-c", command],
+        capture_output=True,
+        text=True,
+        timeout=600,
+    )
+    if result.returncode == 0:
+        return True
+    print("[langsmith] failed to auto-build Claude Code tracing plugin.")
+    if result.stdout.strip():
+        print(result.stdout.strip())
+    if result.stderr.strip():
+        print(result.stderr.strip())
+    return False
+
+
+def _plugin_dir_env_value(plugin_dir: Path) -> str:
+    """Return a plugin path value that docker.sh can test and mount from bash."""
+    return _to_bash_path(plugin_dir)
+
+
+def provision_langsmith_plugin_dir() -> Path | None:
+    """Resolve or auto-build the official Claude Code LangSmith plugin directory."""
+    configure_langsmith_environment()
+    if os.environ.get("TRACE_TO_LANGSMITH", "").lower() != "true":
+        return None
+
+    explicit_dir = os.environ.get("CC_LANGSMITH_PLUGIN_DIR", "").strip()
+    if explicit_dir:
+        plugin_dir = Path(explicit_dir)
+        if plugin_dir.is_dir():
+            os.environ["CC_LANGSMITH_PLUGIN_DIR"] = _plugin_dir_env_value(plugin_dir)
+            return plugin_dir
+        print(
+            f"[langsmith] CC_LANGSMITH_PLUGIN_DIR not a directory ({plugin_dir}); "
+            "falling back to the eval plugin cache."
+        )
+        os.environ.pop("CC_LANGSMITH_PLUGIN_DIR", None)
+
+    plugin_dir = DEFAULT_LANGSMITH_PLUGIN_DIR
+    if plugin_dir.is_dir():
+        os.environ["CC_LANGSMITH_PLUGIN_DIR"] = _plugin_dir_env_value(plugin_dir)
+        return plugin_dir
+
+    if os.environ.get("CC_LANGSMITH_PLUGIN_AUTO_BUILD", "true").lower() in {
+        "0",
+        "false",
+        "no",
+    }:
+        print(
+            "[langsmith] trajectory tracing disabled: "
+            "CC_LANGSMITH_PLUGIN_AUTO_BUILD=false and no plugin directory was provided."
+        )
+        return None
+
+    print(
+        "[langsmith] building Claude Code tracing plugin into "
+        f"{plugin_dir} via node:20..."
+    )
+    if _build_default_langsmith_plugin(plugin_dir) and plugin_dir.is_dir():
+        os.environ["CC_LANGSMITH_PLUGIN_DIR"] = _plugin_dir_env_value(plugin_dir)
+        return plugin_dir
+
+    os.environ.pop("CC_LANGSMITH_PLUGIN_DIR", None)
+    return None
+
+
+@pytest.fixture(scope="session", autouse=True)
+def provision_langsmith_tracing(request):
+    """Wire the official langsmith-tracing Claude Code plugin, best-effort.
+
+    Results logging (rubric scores + treatment comparison) only needs a
+    LANGSMITH_API_KEY. Trajectory tracing additionally needs the prebuilt
+    ``langsmith-tracing`` plugin, provided via ``CC_LANGSMITH_PLUGIN_DIR``
+    (host path). When it is missing we log once and continue without the
+    nested Claude Code trajectory instead of failing the run.
+    """
+    if _local_conftest._is_unit_tests_only(request.config):
+        return
+
+    plugin_dir = provision_langsmith_plugin_dir()
+    if plugin_dir:
+        print(f"[langsmith] trajectory tracing enabled via plugin: {plugin_dir}")
+    else:
+        print(
+            "[langsmith] trajectory tracing disabled "
+            "(rubric + comparison still logged)."
+        )
+
+
+@pytest.fixture(scope="session")
+def langsmith_experiment_metadata():
+    """Attach suite metadata to each LangSmith experiment (project).
+
+    Read by the LangSmith pytest plugin (requires langsmith>=0.7.13); ignored by
+    older versions.
+    """
+    return {
+        "suite": "comet-skill-eval",
+        "cc_model": os.environ.get("BENCH_CC_MODEL", "cli-default"),
+        "cc_version": os.environ.get("BENCH_CC_VERSION", "latest"),
+    }

package/eval/langsmith/tests/tasks/test_tasks.py

@@ -0,0 +1,287 @@
+"""LangSmith suite task runner.
+
+Wraps the local task runner with LangSmith's pytest integration. Each
+``(task, treatment)`` run syncs to a LangSmith dataset example + experiment:
+
+- ``log_inputs`` / ``log_reference_outputs`` capture the task prompt and the
+  ground-truth expectations (expected artifacts, required skills, rubric).
+- ``log_outputs`` captures the run's efficiency metrics and invoked skills.
+- ``log_feedback`` reports each rubric dimension score (and the check pass rate),
+  so CONTROL vs skill-injected treatments compare directly in the experiment view.
+- Trajectory tracing (official ``langsmith-tracing`` Claude Code plugin) nests
+  under the same run via ``CC_LANGSMITH_PARENT_DOTTED_ORDER`` when a prebuilt
+  plugin dir is provided through ``CC_LANGSMITH_PLUGIN_DIR``.
+
+The local suite stays completely free of any LangSmith import; this module only
+adds a thin logging wrapper around the unchanged local runner.
+"""
+
+import importlib.util
+import json
+import os
+import sys
+from pathlib import Path
+
+import pytest
+
+from scaffold.python.tasks import load_task
+from scaffold.python.logging import TreatmentResult
+
+# ---------------------------------------------------------------------------
+# Load the local runner as a module (do NOT re-export its test_task_treatment;
+# we wrap it below). pytest_generate_tests and any non-task unit tests are
+# re-exported unchanged so parametrization and coverage stay identical.
+# ---------------------------------------------------------------------------
+_LOCAL_TEST_TASKS = (
+    Path(__file__).resolve().parents[3] / "local" / "tests" / "tasks" / "test_tasks.py"
+)
+_spec = importlib.util.spec_from_file_location("_comet_local_test_tasks", _LOCAL_TEST_TASKS)
+_local_test_tasks = importlib.util.module_from_spec(_spec)
+assert _spec and _spec.loader
+sys.modules[_spec.name] = _local_test_tasks
+_spec.loader.exec_module(_local_test_tasks)
+
+pytest_generate_tests = _local_test_tasks.pytest_generate_tests
+for _name in dir(_local_test_tasks):
+    if _name.startswith("test_") and _name != "test_task_treatment":
+        globals()[_name] = getattr(_local_test_tasks, _name)
+
+PYTEST_TIMEOUT = _local_test_tasks.PYTEST_TIMEOUT
+_run_local_task_treatment = _local_test_tasks.test_task_treatment
+
+# LangSmith helpers are optional; degrade to a plain pass-through when the
+# ``langsmith`` extra is not installed so collection never crashes.
+try:
+    from langsmith import testing as ls_testing
+    from langsmith import get_current_run_tree
+    from langsmith.testing import _internal as ls_testing_internal
+
+    _LANGSMITH_AVAILABLE = True
+except Exception:  # pragma: no cover - import guard
+    ls_testing = None
+    get_current_run_tree = None
+    ls_testing_internal = None
+    _LANGSMITH_AVAILABLE = False
+
+try:
+    from scaffold.python.logging import _rubric_scores as _extract_rubric_scores
+except Exception:  # pragma: no cover - fallback if private helper moves
+    import re as _re
+
+    _RUBRIC_RE = _re.compile(r"\[RUBRIC\]\s+(\S+):\s*([0-9.]+)")
+
+    def _extract_rubric_scores(result):
+        scores = {}
+        for check in getattr(result, "checks_passed", []):
+            match = _RUBRIC_RE.search(check)
+            if match:
+                try:
+                    scores[match.group(1)] = float(match.group(2))
+                except ValueError:
+                    continue
+        return scores
+
+
+# =============================================================================
+# LANGSMITH LOGGING HELPERS (all best-effort; never break the eval run)
+# =============================================================================
+
+
+def _safe(fn, label=None):
+    """Call a LangSmith logging fn, swallowing errors so tracing never fails a run."""
+    try:
+        fn()
+    except Exception as exc:  # pragma: no cover - defensive
+        if label:
+            print(f"[langsmith] {label} failed: {exc}", file=sys.stderr)
+
+
+def _install_failed_end_run_outputs_patch():
+    """Preserve log_outputs payloads when LangSmith ends a failed pytest run."""
+    if not _LANGSMITH_AVAILABLE or ls_testing_internal is None:
+        return
+    test_case_cls = getattr(ls_testing_internal, "_TestCase", None)
+    if test_case_cls is None:
+        return
+    original = getattr(test_case_cls, "end_run", None)
+    if original is None or getattr(original, "_comet_preserve_logged_outputs", False):
+        return
+
+    def end_run(self, run_tree, outputs):
+        if outputs is None and getattr(self, "_logged_outputs", None) is not None:
+            outputs = self._logged_outputs
+        return original(self, run_tree, outputs)
+
+    end_run._comet_preserve_logged_outputs = True
+    end_run._comet_original = original
+    test_case_cls.end_run = end_run
+
+
+_install_failed_end_run_outputs_patch()
+
+
+def _log_inputs_and_reference(task_name, treatment_name):
+    if not _LANGSMITH_AVAILABLE:
+        return
+    try:
+        task = load_task(task_name)
+        evaluation = task.config.evaluation
+        _safe(
+            lambda: ls_testing.log_inputs(
+                {
+                    "task": task_name,
+                    "treatment": treatment_name,
+                    "difficulty": getattr(task.config.metadata, "difficulty", None),
+                }
+            )
+        )
+        _safe(
+            lambda: ls_testing.log_reference_outputs(
+                {
+                    "expected_artifacts": list(evaluation.expected_artifacts or []),
+                    "required_skills": list(evaluation.required_skills or []),
+                    "rubric_criteria": list(evaluation.rubric_criteria or []),
+                }
+            )
+        )
+    except Exception:  # pragma: no cover - defensive
+        pass
+
+
+def _set_parent_run_env():
+    """Point the Claude Code plugin at this test's run so its trajectory nests here."""
+    if not _LANGSMITH_AVAILABLE or get_current_run_tree is None:
+        return None
+    try:
+        run_tree = get_current_run_tree()
+        dotted_order = getattr(run_tree, "dotted_order", None) if run_tree else None
+    except Exception:  # pragma: no cover - defensive
+        dotted_order = None
+    if dotted_order:
+        os.environ["CC_LANGSMITH_PARENT_DOTTED_ORDER"] = dotted_order
+    return dotted_order
+
+
+def _result_from_logger(logger, treatment_name):
+    runs = logger.results.get(treatment_name) or []
+    if runs:
+        return runs[-1]
+
+    parametrized_suffix = f"-{treatment_name}-"
+    for result_name, result_runs in reversed(list(logger.results.items())):
+        if result_name == treatment_name or parametrized_suffix in result_name:
+            return result_runs[-1] if result_runs else None
+    return None
+
+
+def _iter_result_loggers():
+    seen = set()
+    candidates = [
+        getattr(_local_test_tasks, "conftest", None),
+        sys.modules.get("conftest"),
+        sys.modules.get("_comet_local_conftest"),
+    ]
+    for module in candidates:
+        plugin = getattr(module, "_plugin", None) if module else None
+        logger = getattr(plugin, "logger", None) if plugin else None
+        if logger is not None and id(logger) not in seen:
+            seen.add(id(logger))
+            yield logger
+
+
+def _latest_report_result(treatment_name):
+    logs_dir = Path(os.environ.get("BENCH_LOGS_DIR", ""))
+    if not logs_dir:
+        return None
+    reports_root = logs_dir / "experiments"
+    if not reports_root.exists():
+        return None
+    parametrized_suffix = f"-{treatment_name}-"
+    report_files = sorted(
+        reports_root.glob("*/reports/*_report.json"),
+        key=lambda path: path.stat().st_mtime,
+        reverse=True,
+    )
+    for report_file in report_files:
+        try:
+            report = json.loads(report_file.read_text())
+        except Exception:
+            continue
+        result_name = report.get("name", "")
+        if result_name != treatment_name and parametrized_suffix not in result_name:
+            continue
+        return TreatmentResult(
+            name=result_name,
+            passed=report.get("passed", False),
+            checks_passed=report.get("checks_passed", []),
+            checks_failed=report.get("checks_failed", []),
+            events_summary=report.get("events_summary", {}),
+            run_id=report.get("run_id", ""),
+        )
+    return None
+
+
+def _latest_result(treatment_name):
+    for logger in _iter_result_loggers():
+        result = _result_from_logger(logger, treatment_name)
+        if result is not None:
+            return result
+    return _latest_report_result(treatment_name)
+
+
+def _log_outputs_and_feedback(result):
+    if not _LANGSMITH_AVAILABLE:
+        return
+    if result is None:
+        print("[langsmith] no local eval result found; outputs/feedback not logged.", file=sys.stderr)
+        return
+    summary = getattr(result, "events_summary", {}) or {}
+    outputs = {
+        "run_id": getattr(result, "run_id", ""),
+        "passed": getattr(result, "passed", None),
+        "checks_passed": len(getattr(result, "checks_passed", [])),
+        "checks_failed": len(getattr(result, "checks_failed", [])),
+        "num_turns": summary.get("num_turns"),
+        "tool_calls": summary.get("tool_calls"),
+        "duration_seconds": summary.get("duration_seconds"),
+        "total_tokens": summary.get("total_tokens"),
+        "total_cost_usd": summary.get("total_cost_usd"),
+        "skills_invoked": summary.get("skills_invoked", []),
+    }
+    _safe(
+        lambda: ls_testing.log_outputs(outputs)
+    )
+
+    passed = len(getattr(result, "checks_passed", []))
+    failed = len(getattr(result, "checks_failed", []))
+    total = passed + failed
+    if total:
+        _safe(
+            lambda: ls_testing.log_feedback(key="checks_pass_rate", score=passed / total),
+            label="log_feedback checks_pass_rate",
+        )
+
+    for dim, score in _extract_rubric_scores(result).items():
+        _safe(
+            lambda d=dim, s=score: ls_testing.log_feedback(key=f"rubric.{d}", score=s),
+            label=f"log_feedback rubric.{dim}",
+        )
+
+
+# =============================================================================
+# TEST
+# =============================================================================
+
+
+@pytest.mark.timeout(PYTEST_TIMEOUT)
+@pytest.mark.langsmith
+def test_task_treatment(task_name, treatment_name):
+    """Run a task+treatment via the local runner and log results to LangSmith."""
+    _log_inputs_and_reference(task_name, treatment_name)
+    _set_parent_run_env()
+    try:
+        _run_local_task_treatment(task_name, treatment_name)
+    finally:
+        _log_outputs_and_feedback(_latest_result(treatment_name))
+        os.environ.pop("CC_LANGSMITH_PARENT_DOTTED_ORDER", None)
+

package/eval/langsmith/treatments/README.md

@@ -0,0 +1,5 @@
+# LangSmith Treatments
+
+这个套件当前通过 `BENCH_TREATMENTS_DIR` 复用 `../../local/treatments`。
+
+如果需要不同的 Skill 组合方式，可以在这里添加 LangSmith 专用 treatment。

package/eval/local/README.md

@@ -0,0 +1,130 @@
+# 本地评估套件
+
+这个套件用于在本机运行 Comet 和任意本地 Skill 的 benchmark，并将日志与 Docker 验证结果保存到本地。
+它不需要 LangSmith 凭证。
+
+从 `eval/` 目录运行：
+
+```bash
+uv run pytest local/tests/tasks/test_tasks.py --task=comet-fix-median --treatment=COMET_FULL_040_BETA -v
+uv run pytest local/tests/tasks/test_tasks.py --task=comet-full-workflow --treatment=CONTROL,COMET_FULL_040_BETA -v
+uv run pytest local/tests/tasks/test_tasks.py -v
+```
+
+快速运行通用 smoke 任务：
+
+```bash
+uv run pytest local/tests/tasks/test_tasks.py --task=generic-skill-smoke --treatment=CONTROL -v
+```
+
+运行只安装 Comet Native、自包含且不注入 OpenSpec / Superpowers 的完整流程评估：
+
+```bash
+uv run pytest local/tests/tasks/test_tasks.py \
+  --task=comet-native-workflow \
+  --treatment=COMET_NATIVE_PHASE1 -v
+```
+
+将 Native 与 0.4.0 基线按相同的 16 个 canonical Comet 任务、每任务 3 次对齐运行：
+
+```bash
+uv run pytest local/tests/tasks/test_tasks.py::test_task_treatment \
+  --treatment=COMET_NATIVE_PHASE1 --count=3 -n 4 \
+  -k "not comet-native and not generic-skill-smoke and not authoring-skill-smoke and not workflow-route-conformance and not workflow-overlay-contract" -v
+```
+
+对齐评估保留每个任务原有的业务 validator，但会把 Classic/OpenSpec 专属流程检查替换为 Native 的 Skill 隔离、配置、终态 change、验证归档和安全 trajectory 检查。调用 `/comet` 或其他外部 Skill、留下活动 change、缺少完整规格/验证证据，都会作为真实 Native workflow failure 进入 pass@k，而不会被当成环境噪声排除。
+
+运行生成 Skill 的 authoring smoke 任务：
+
+```bash
+uv run pytest local/tests/tasks/test_tasks.py \
+  --task=authoring-skill-smoke \
+  --eval-manifest=/path/to/generated-skill/comet/eval.yaml \
+  --profile=authoring-skill -v
+```
+
+运行生成 Skill 的路由一致性验证：
+
+```bash
+uv run pytest local/tests/tasks/test_tasks.py \
+  --task=workflow-route-conformance \
+  --eval-manifest=/path/to/generated-skill/comet/eval.yaml \
+  --profile=authoring-skill -v
+```
+
+当 `--eval-manifest` 指向 `/comet-any` 生成包时，runner 会先把入口 Skill 包和生成的内部 Node Skills 复制到隔离的 eval 工作区，再执行 Docker 验证。
+
+`comet-workflow` profile 会把 `skills/benchmarks/dependency/claude-md/comet-workflow/CLAUDE.md` 注入到隔离工作区根目录，用项目级指令强制 agent 先触发 `/comet`。如果需要调整这段提示词，直接改这个资产文件，不需要改 Python runner。
+
+Comet baseline 和它依赖的 OpenSpec / Superpowers Skill 都会按完整 Skill 包安装到 `.claude/skills/`，包括 `rules/`、`reference/`、`runtime/`、`scripts/` 以及其他随包文件。安装 canonical `comet` Skill 时，runner 会根据 baseline 自动配置 Claude Code `PreToolUse` hook：`COMET_FULL_039` 使用 `comet-hook-guard.sh`，`COMET_FULL_040_BETA` 使用 `comet-hook-guard.mjs`。
+
+报告中的 `Skills invoked` 和 `skill_invocation` rubric 只统计 Claude Code 事件里真实观测到的 Skill 工具调用，不会根据产物反推。`comet-workflow` profile 会把嵌套 Comet 阶段 Skill、OpenSpec 依赖 Skill、Superpowers 依赖 Skill 都作为关键证据；如果只调用主 `comet` 但没有调用这些依赖，run 会被标记为工作流契约失败。rubric 会按 `full`、`hotfix`、`tweak` 三类 workflow 分别评分：`full` 要求深度设计和完整阶段证据，`hotfix` / `tweak` 使用预设路径的轻量产物和连续执行决策点口径。
+
+容器内 Claude 实验工作区的 ASCII 目录结构见 `../README.md` 的“容器中的 Claude 实验工作区”。
+
+任务索引：
+
+```text
+tasks/index.yaml
+```
+
+报告和产物写入：
+
+```text
+logs/experiments/
+```
+
+报告默认只输出 Markdown（`summary.md`）。如需启用可浏览的 HTML，或关闭 Markdown 输出，传入 JSON/YAML 报告配置：
+
+```json
+{
+  "report_outputs": {
+    "markdown": true,
+    "html": true
+  }
+}
+```
+
+```bash
+uv run pytest local/tests/tasks/test_tasks.py --report-config report-config.json -v
+uv run python local/scripts/compare_baselines.py --report-config report-config.json
+```
+
+Native 与 0.4.0 的 pass@3 来自两个独立 experiment 时，使用严格对齐模式：
+
+```bash
+uv run python local/scripts/compare_baselines.py \
+  --candidate-experiment local/logs/experiments/experiment_NATIVE \
+  --candidate-treatment COMET_NATIVE_PHASE1 \
+  --baseline-experiment langsmith/logs/experiments/experiment_040_PASS3 \
+  --baseline-treatment COMET_FULL_040_BETA \
+  --ks 1,2,3
+```
+
+pytest 会在模型运行前把收集到的 `task + treatment + repetition` 写入 experiment 的
+expected case matrix；严格比较按两侧目标 treatment 的矩阵检查 coverage，因此即使某个
+样本在两侧都没有留下 report，也会被列为缺失，而不是从报告并集中无声消失。对 v2
+报告，只有 `task + repetition + case_hash` 全部一致才会配对；`case_hash` 同时绑定任务、
+验证器、runner/controller 源码、不可变 Docker image、Claude 工具版本以及 model /
+interaction 配置。某个 task 缺少 repetition 时，报告会降低该 k 的 coverage 并列出
+缺失项，不会把 pass@3 偷换成 pass@2。
+
+历史耗时会从 raw stdout 按当前的累加 parser 重算；缺少 raw duration 的运行只计入
+缺失 coverage，不会和已有耗时混合平均。没有 expected matrix 的旧 experiment 会明确
+标为 observed-report fallback；v1 或缺失 case manifest 的运行最多只能按任务核心 hash
+兼容比较，并明确标注无法证明当时的 runner、image、tool、model 或 interaction 身份，
+不会把比较时一致误写成历史 checkout 与执行环境已被精确证明一致。
+
+严格对齐报告还会从 raw stdout 重算成对效率指标，包括模型启动/恢复次数、Agent 轮次、
+工具调用、累计耗时、输入/输出/cache token、总 token、模型成本和上下文压力。主要视图只
+统计两侧都通过的同一 `task + repetition + case_hash`，避免把未完成任务的低消耗误当成
+效率提升。多次恢复产生的顶层 `result` 按调用累加；流式 assistant 事件按 message id 去重。
+上下文指标只有在两侧都保存逐消息 usage 时才可比较，coverage 过低时报告会明确提示，不能
+据此宣传 workflow 间的上下文改进。
+
+也可以设置 `COMET_EVAL_REPORT_CONFIG=/path/to/report-config.json`。
+
+每次运行的报告都会包含 profile、Skill 来源元数据、run id、产物引用，以及结构化失败归因。归因桶包括 `harness`、`workflow`、`task` 和 `model`。
+
+比较报告还会显示 `Data quality summary`。报告保留所有 raw runs 供审计，但 headline 指标、pass@k/pass^k、成本统计、图表和 verdict 默认使用 analysis set。`excluded` 表示明确的环境或运行器噪声（例如 API timeout、rate limit、Docker 启动失败），不会进入主统计；`flagged` 表示可疑 harness/task 噪声，仍进入主统计但会在报告中单独标出；真实 workflow/model/task 失败会保留为 `included`，不会因为分数低被过滤。

package/eval/local/regression_baseline.json

@@ -0,0 +1,13 @@
+{
+  "dimensions": {
+    "main_flow": 1.0,
+    "gate_guard": 1.0,
+    "skill_invocation": 1.0,
+    "spec_drift": 0.25,
+    "completion": 0.8,
+    "efficiency": 0.73,
+    "decision_point_compliance": 0.5,
+    "artifact_quality": 1.0
+  },
+  "source": "regression_check.py"
+}

package/eval/local/report-style-demo-assets/quality_cost.en.svg

@@ -0,0 +1,41 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 820 460" role="img" aria-label="Figure 2. Quality-cost frontier">
+<style>
+text { font-family: 'Times New Roman', SimSun, 'Songti SC', 'Noto Serif CJK SC', serif; }
+.title { fill: #1B1A18; font-size: 18px; font-weight: 700; }
+.subtitle, .tick { fill: #66615A; font-size: 12px; }
+.label { fill: #1B1A18; font-size: 13px; }
+.value { fill: #1B1A18; font-size: 12px; font-family: 'Times New Roman', SimSun, 'Songti SC', 'Noto Serif CJK SC', serif; }
+.grid { stroke: #D8D2C7; stroke-width: 1; shape-rendering: crispEdges; }
+.axis { stroke: #1B1A18; stroke-width: 1; shape-rendering: crispEdges; }
+</style>
+<rect width="820" height="460" fill="#FFFFFF" />
+<text class="title" x="32" y="32">Figure 2. Quality-cost frontier</text>
+<text class="subtitle" x="32" y="54">Y-axis is rubric average, x-axis is total tokens; upper-left is better.</text>
+<line class="grid" x1="86.0" y1="72" x2="86.0" y2="372" />
+<text class="tick" x="86.0" y="402" text-anchor="middle">0M</text>
+<line class="grid" x1="292.7" y1="72" x2="292.7" y2="372" />
+<text class="tick" x="292.7" y="402" text-anchor="middle">4M</text>
+<line class="grid" x1="499.3" y1="72" x2="499.3" y2="372" />
+<text class="tick" x="499.3" y="402" text-anchor="middle">8M</text>
+<line class="grid" x1="706.0" y1="72" x2="706.0" y2="372" />
+<text class="tick" x="706.0" y="402" text-anchor="middle">12M</text>
+<line class="grid" x1="86" y1="372.0" x2="706" y2="372.0" />
+<text class="tick" x="60" y="376.0" text-anchor="end">0.3</text>
+<line class="grid" x1="86" y1="272.0" x2="706" y2="272.0" />
+<text class="tick" x="60" y="276.0" text-anchor="end">0.5</text>
+<line class="grid" x1="86" y1="172.0" x2="706" y2="172.0" />
+<text class="tick" x="60" y="176.0" text-anchor="end">0.7</text>
+<line class="grid" x1="86" y1="72.0" x2="706" y2="72.0" />
+<text class="tick" x="60" y="76.0" text-anchor="end">0.9</text>
+<line class="axis" x1="86" y1="372" x2="706" y2="372" />
+<line class="axis" x1="86" y1="72" x2="86" y2="372" />
+<line x1="372.8" y1="252.0" x2="677.4" y2="162.0" stroke="#9B9488" stroke-width="1.5" stroke-dasharray="5 5" />
+<circle cx="677.4" cy="162.0" r="8" fill="#1F5F8B" />
+<text class="value" x="663.4" y="152.0" text-anchor="end">Current workflow</text>
+<text class="tick" x="663.4" y="171.0" text-anchor="end">0.72 / $7.59</text>
+<circle cx="372.8" cy="252.0" r="8" fill="#9F3D2F" />
+<text class="value" x="386.8" y="242.0" text-anchor="start">0.3.9 baseline</text>
+<text class="tick" x="386.8" y="261.0" text-anchor="start">0.54 / $3.97</text>
+<text class="label" x="396.0" y="438" text-anchor="middle">Total tokens</text>
+<text class="label" x="24" y="222.0" transform="rotate(-90 24 222.0)" text-anchor="middle">Rubric average</text>
+</svg>