@rpamis/comet 0.4.0-beta.1 → 0.4.0-beta.10

package/eval/local/tests/scaffold/test_evidence.py

@@ -0,0 +1,15 @@
+from pathlib import Path
+
+from scaffold.python.evidence import build_eval_artifact_references
+
+
+def test_build_eval_artifact_references_uses_stable_paths(tmp_path: Path):
+    refs = build_eval_artifact_references(tmp_path, "COMET-FULL", 2)
+
+    assert refs == {
+        "events": str(tmp_path / "events" / "COMET_FULL_040_BETA_rep2.json"),
+        "raw_stdout": str(tmp_path / "raw" / "COMET_FULL_040_BETA_rep2_stdout.json"),
+        "raw_stderr": str(tmp_path / "raw" / "COMET_FULL_040_BETA_rep2_stderr.txt"),
+        "report": str(tmp_path / "reports" / "COMET_FULL_040_BETA_rep2_report.json"),
+        "artifacts": str(tmp_path / "artifacts" / "COMET_FULL_040_BETA_rep2"),
+    }

package/eval/local/tests/scaffold/test_langsmith_conftest.py

@@ -0,0 +1,244 @@
+"""Unit tests for LangSmith eval suite configuration helpers."""
+
+from __future__ import annotations
+
+import inspect
+import importlib.util
+import sys
+from pathlib import Path
+from types import SimpleNamespace
+import json
+
+
+def _load_langsmith_conftest():
+    eval_root = Path(__file__).resolve().parents[3]
+    conftest_path = eval_root / "langsmith" / "tests" / "conftest.py"
+    spec = importlib.util.spec_from_file_location("_test_langsmith_conftest", conftest_path)
+    module = importlib.util.module_from_spec(spec)
+    assert spec and spec.loader
+    spec.loader.exec_module(module)
+    return module
+
+
+def _load_langsmith_task_tests():
+    eval_root = Path(__file__).resolve().parents[3]
+    task_tests_path = eval_root / "langsmith" / "tests" / "tasks" / "test_tasks.py"
+    sys.modules.pop("_test_langsmith_task_tests", None)
+    sys.modules.pop("_comet_local_test_tasks", None)
+    spec = importlib.util.spec_from_file_location("_test_langsmith_task_tests", task_tests_path)
+    module = importlib.util.module_from_spec(spec)
+    assert spec and spec.loader
+    spec.loader.exec_module(module)
+    return module
+
+
+def test_langsmith_env_derives_claude_code_plugin_settings(monkeypatch):
+    module = _load_langsmith_conftest()
+    for key in (
+        "LANGSMITH_API_KEY",
+        "LANGSMITH_PROJECT",
+        "LANGSMITH_TRACING",
+        "TRACE_TO_LANGSMITH",
+        "CC_LANGSMITH_API_KEY",
+        "CC_LANGSMITH_PROJECT",
+    ):
+        monkeypatch.delenv(key, raising=False)
+    monkeypatch.setenv("LANGSMITH_API_KEY", "lsv2_pt_test")
+    monkeypatch.setenv("LANGSMITH_PROJECT", "comet-tests")
+
+    module.configure_langsmith_environment()
+
+    assert module.os.environ["LANGSMITH_TRACING"] == "true"
+    assert module.os.environ["TRACE_TO_LANGSMITH"] == "true"
+    assert module.os.environ["CC_LANGSMITH_API_KEY"] == "lsv2_pt_test"
+    assert module.os.environ["CC_LANGSMITH_PROJECT"] == "comet-tests"
+
+
+def test_langsmith_env_preserves_explicit_claude_code_overrides(monkeypatch):
+    module = _load_langsmith_conftest()
+    monkeypatch.setenv("LANGSMITH_API_KEY", "lsv2_pt_eval")
+    monkeypatch.setenv("LANGSMITH_PROJECT", "eval-project")
+    monkeypatch.setenv("LANGSMITH_TRACING", "false")
+    monkeypatch.setenv("TRACE_TO_LANGSMITH", "custom")
+    monkeypatch.setenv("CC_LANGSMITH_API_KEY", "lsv2_pt_plugin")
+    monkeypatch.setenv("CC_LANGSMITH_PROJECT", "plugin-project")
+
+    module.configure_langsmith_environment()
+
+    assert module.os.environ["LANGSMITH_TRACING"] == "false"
+    assert module.os.environ["TRACE_TO_LANGSMITH"] == "custom"
+    assert module.os.environ["CC_LANGSMITH_API_KEY"] == "lsv2_pt_plugin"
+    assert module.os.environ["CC_LANGSMITH_PROJECT"] == "plugin-project"
+
+
+def test_langsmith_tracing_auto_builds_default_plugin_cache(monkeypatch, tmp_path):
+    module = _load_langsmith_conftest()
+    plugin_dir = tmp_path / "langsmith-cc-plugin"
+    calls = []
+
+    monkeypatch.setattr(module, "DEFAULT_LANGSMITH_PLUGIN_DIR", plugin_dir)
+    monkeypatch.delenv("CC_LANGSMITH_PLUGIN_DIR", raising=False)
+    monkeypatch.setenv("LANGSMITH_TRACING", "true")
+    monkeypatch.setenv("TRACE_TO_LANGSMITH", "true")
+
+    def fake_build(target):
+        calls.append(target)
+        target.mkdir(parents=True)
+        return True
+
+    monkeypatch.setattr(module, "_build_default_langsmith_plugin", fake_build)
+
+    resolved = module.provision_langsmith_plugin_dir()
+
+    assert resolved == plugin_dir
+    assert calls == [plugin_dir]
+    assert module.os.environ["CC_LANGSMITH_PLUGIN_DIR"] == module._plugin_dir_env_value(plugin_dir)
+
+
+def test_langsmith_task_wrapper_registers_reexported_local_module():
+    _load_langsmith_task_tests()
+
+    assert "_comet_local_test_tasks" in sys.modules
+
+
+def test_langsmith_task_wrapper_does_not_require_pytest_request_fixture():
+    module = _load_langsmith_task_tests()
+
+    assert "request" not in inspect.signature(module.test_task_treatment).parameters
+
+
+def test_langsmith_task_wrapper_finds_parametrized_local_result():
+    module = _load_langsmith_task_tests()
+    result = object()
+    module._local_test_tasks.conftest._plugin = SimpleNamespace(
+        logger=SimpleNamespace(
+            results={
+                "comet-fix-median-CONTROL-r1": [result],
+            }
+        )
+    )
+
+    assert module._latest_result("CONTROL") is result
+
+
+def test_langsmith_task_wrapper_finds_result_from_loaded_conftest(monkeypatch):
+    module = _load_langsmith_task_tests()
+    result = object()
+    monkeypatch.setattr(module._local_test_tasks.conftest, "_plugin", None, raising=False)
+    monkeypatch.setitem(
+        sys.modules,
+        "conftest",
+        SimpleNamespace(
+            _plugin=SimpleNamespace(
+                logger=SimpleNamespace(
+                    results={
+                        "comet-fix-median-CONTROL-r1": [result],
+                    }
+                )
+            )
+        ),
+    )
+
+    assert module._latest_result("CONTROL") is result
+
+
+def test_langsmith_task_wrapper_falls_back_to_latest_report(monkeypatch, tmp_path):
+    module = _load_langsmith_task_tests()
+    monkeypatch.setattr(module._local_test_tasks.conftest, "_plugin", None, raising=False)
+    monkeypatch.setenv("BENCH_LOGS_DIR", str(tmp_path))
+    reports_dir = tmp_path / "experiments" / "experiment_20260703_220136" / "reports"
+    reports_dir.mkdir(parents=True)
+    report = {
+        "name": "comet-fix-median-CONTROL-r1",
+        "passed": False,
+        "run_id": "local-run-1",
+        "checks_passed": ["[RUBRIC] weighted_score: 0.35"],
+        "checks_failed": ["openspec_artifacts: missing"],
+        "events_summary": {"total_tokens": 123},
+    }
+    (reports_dir / "comet_fix_median_CONTROL_r1_rep1_report.json").write_text(
+        json.dumps(report)
+    )
+
+    result = module._latest_result("CONTROL")
+
+    assert result.run_id == "local-run-1"
+    assert result.checks_failed == ["openspec_artifacts: missing"]
+    assert result.events_summary["total_tokens"] == 123
+
+
+def test_langsmith_task_wrapper_patches_failed_end_run_outputs():
+    module = _load_langsmith_task_tests()
+
+    assert getattr(
+        module.ls_testing_internal._TestCase.end_run,
+        "_comet_preserve_logged_outputs",
+        False,
+    )
+
+
+def test_langsmith_task_wrapper_logs_outputs_and_rubric_feedback(monkeypatch):
+    module = _load_langsmith_task_tests()
+    calls = []
+    monkeypatch.setattr(
+        module,
+        "ls_testing",
+        SimpleNamespace(
+            log_outputs=lambda outputs: calls.append(("outputs", outputs)),
+            log_feedback=lambda **kwargs: calls.append(("feedback", kwargs)),
+        ),
+    )
+    result = SimpleNamespace(
+        run_id="local-run-1",
+        passed=False,
+        checks_passed=[
+            "tests_written: ok",
+            "[RUBRIC] skill_invocation: 0.20 - missing deps",
+            "[RUBRIC] weighted_score: 0.35",
+        ],
+        checks_failed=["openspec_artifacts: missing"],
+        events_summary={"total_tokens": 123, "skills_invoked": ["comet"]},
+    )
+
+    module._log_outputs_and_feedback(result)
+
+    assert calls == [
+        (
+            "outputs",
+            {
+                "run_id": "local-run-1",
+                "passed": False,
+                "checks_passed": 3,
+                "checks_failed": 1,
+                "num_turns": None,
+                "tool_calls": None,
+                "duration_seconds": None,
+                "total_tokens": 123,
+                "total_cost_usd": None,
+                "skills_invoked": ["comet"],
+            },
+        ),
+        ("feedback", {"key": "checks_pass_rate", "score": 0.75}),
+        ("feedback", {"key": "rubric.skill_invocation", "score": 0.2}),
+        ("feedback", {"key": "rubric.weighted_score", "score": 0.35}),
+    ]
+
+
+def test_langsmith_task_wrapper_keeps_plugin_on_configured_project(monkeypatch):
+    module = _load_langsmith_task_tests()
+    monkeypatch.delenv("CC_LANGSMITH_PARENT_DOTTED_ORDER", raising=False)
+    monkeypatch.setenv("CC_LANGSMITH_PROJECT", "comet-skill-eval")
+    monkeypatch.setattr(
+        module,
+        "get_current_run_tree",
+        lambda: SimpleNamespace(
+            dotted_order="20260703T000000Z00000000000000000000000000",
+            session_name="comet-skill-eval:abc12345",
+        ),
+    )
+
+    dotted_order = module._set_parent_run_env()
+
+    assert dotted_order == "20260703T000000Z00000000000000000000000000"
+    assert module.os.environ["CC_LANGSMITH_PARENT_DOTTED_ORDER"] == dotted_order
+    assert module.os.environ["CC_LANGSMITH_PROJECT"] == "comet-skill-eval"

package/eval/local/tests/scaffold/test_logging.py

@@ -0,0 +1,377 @@
+"""Unit tests for eval stream parsing and experiment summaries."""
+
+import json
+from pathlib import Path
+
+import pytest
+
+from scaffold.python.logging import (
+    ExperimentLogger,
+    TreatmentResult,
+    extract_events,
+    parse_output,
+    rubric_columns,
+    save_events,
+    save_raw,
+    save_report,
+)
+from scaffold.python.report_outputs import ReportOutputConfig
+
+
+def test_save_artifacts_excludes_nested_git_metadata(tmp_path: Path):
+    from conftest import _save_artifacts
+
+    workspace = tmp_path / "workspace"
+    workspace.mkdir()
+    (workspace / "result.md").write_text("ok", encoding="utf-8")
+    git_dir = workspace / ".git"
+    git_dir.mkdir()
+    (git_dir / "config").write_text("[core]\n", encoding="utf-8")
+
+    _save_artifacts(tmp_path, "COMET_FULL_040_BETA", 1, workspace)
+
+    snapshot = tmp_path / "artifacts" / "comet_full_040_beta_rep1" / "claude"
+    assert (snapshot / "result.md").read_text(encoding="utf-8") == "ok"
+    assert not (snapshot / ".git").exists()
+
+
+def test_save_artifacts_excludes_controller_cli_snapshot(tmp_path: Path):
+    from conftest import _save_artifacts
+
+    workspace = tmp_path / "workspace"
+    (workspace / "_eval_current_comet/dist").mkdir(parents=True)
+    (workspace / "_eval_current_comet/dist/index.js").write_text("export {};\n", encoding="utf-8")
+    (workspace / "result.md").write_text("ok", encoding="utf-8")
+
+    _save_artifacts(tmp_path, "COMET_NATIVE_PHASE1", 1, workspace)
+
+    snapshot = tmp_path / "artifacts/comet_native_phase1_rep1/claude"
+    assert (snapshot / "result.md").is_file()
+    assert not (snapshot / "_eval_current_comet").exists()
+
+
+def test_extract_events_captures_token_usage_and_cost():
+    stdout = "\n".join(
+        [
+            json.dumps({"type": "assistant", "message": {"content": []}}),
+            json.dumps(
+                {
+                    "type": "result",
+                    "duration_ms": 1200,
+                    "num_turns": 3,
+                    "total_cost_usd": 0.123456,
+                    "usage": {
+                        "input_tokens": 100,
+                        "output_tokens": 25,
+                        "cache_read_input_tokens": 300,
+                        "cache_creation_input_tokens": 50,
+                    },
+                    "modelUsage": {
+                        "mimo-v2.5-pro": {
+                            "inputTokens": 100,
+                            "outputTokens": 25,
+                            "cacheReadInputTokens": 300,
+                            "cacheCreationInputTokens": 50,
+                            "costUSD": 0.123456,
+                        }
+                    },
+                }
+            ),
+        ]
+    )
+
+    events = extract_events(parse_output(stdout))
+
+    assert events["duration_seconds"] == 1.2
+    assert events["num_turns"] == 3
+    assert events["input_tokens"] == 100
+    assert events["output_tokens"] == 25
+    assert events["cache_read_input_tokens"] == 300
+    assert events["cache_creation_input_tokens"] == 50
+    assert events["total_tokens"] == 475
+    assert events["total_cost_usd"] == 0.123456
+    assert events["model_usage"]["mimo-v2.5-pro"]["costUSD"] == 0.123456
+
+
+def test_extract_events_accumulates_duration_across_results():
+    stdout = "\n".join(
+        [
+            json.dumps(
+                {
+                    "type": "result",
+                    "duration_ms": 1200,
+                    "num_turns": 2,
+                    "total_cost_usd": 0.1,
+                    "usage": {"input_tokens": 100, "output_tokens": 20},
+                }
+            ),
+            json.dumps(
+                {
+                    "type": "result",
+                    "duration_ms": 800,
+                    "num_turns": 4,
+                    "total_cost_usd": 0.2,
+                    "usage": {"input_tokens": 200, "output_tokens": 40},
+                }
+            ),
+        ]
+    )
+
+    events = extract_events(parse_output(stdout))
+
+    assert events["duration_seconds"] == 2.0
+    assert events["subject_invocations"] == 2
+    assert events["num_turns"] == 6
+    assert events["input_tokens"] == 300
+    assert events["output_tokens"] == 60
+    assert events["total_tokens"] == 360
+    assert events["total_cost_usd"] == pytest.approx(0.3)
+
+
+def test_extract_events_reports_deduplicated_context_pressure():
+    assistant = {
+        "type": "assistant",
+        "message": {
+            "id": "message-1",
+            "model": "mimo-v2.5-pro",
+            "content": [],
+            "usage": {
+                "input_tokens": 2_000,
+                "cache_read_input_tokens": 8_000,
+                "cache_creation_input_tokens": 0,
+            },
+        },
+    }
+    stdout = "\n".join(
+        [
+            json.dumps(assistant),
+            json.dumps(assistant),
+            json.dumps(
+                {
+                    "type": "assistant",
+                    "message": {
+                        "id": "message-2",
+                        "model": "mimo-v2.5-pro",
+                        "content": [],
+                        "usage": {
+                            "input_tokens": 1_000,
+                            "cache_read_input_tokens": 4_000,
+                        },
+                    },
+                }
+            ),
+            json.dumps(
+                {
+                    "type": "result",
+                    "modelUsage": {
+                        "mimo-v2.5-pro": {
+                            "contextWindow": 200_000,
+                            "inputTokens": 3_000,
+                        }
+                    },
+                }
+            ),
+        ]
+    )
+
+    events = extract_events(parse_output(stdout))
+
+    assert events["peak_context_input_tokens"] == 10_000
+    assert events["p95_context_input_tokens"] == 10_000
+    assert events["average_context_input_tokens"] == 7_500
+    assert events["peak_context_window_tokens"] == 200_000
+    assert events["peak_context_occupancy_pct"] == 5
+
+
+def test_extract_events_ignores_missing_result_duration():
+    stdout = "\n".join(
+        [
+            json.dumps({"type": "result", "duration_ms": 1200}),
+            json.dumps({"type": "result"}),
+        ]
+    )
+
+    events = extract_events(parse_output(stdout))
+
+    assert events["duration_seconds"] == 1.2
+
+
+def test_extract_events_keeps_duration_missing_without_observed_value():
+    events = extract_events(parse_output(json.dumps({"type": "result"})))
+
+    assert events["duration_seconds"] is None
+
+
+def test_extract_events_normalizes_openspec_skill_aliases():
+    stdout = "\n".join(
+        [
+            json.dumps(
+                {
+                    "type": "assistant",
+                    "message": {
+                        "content": [
+                            {
+                                "type": "tool_use",
+                                "id": "call_1",
+                                "name": "Skill",
+                                "input": {"skill": "opsx:new"},
+                            },
+                            {
+                                "type": "tool_use",
+                                "id": "call_2",
+                                "name": "Skill",
+                                "input": {"skill": "openspec-new-change"},
+                            },
+                        ]
+                    },
+                }
+            )
+        ]
+    )
+
+    events = extract_events(parse_output(stdout))
+
+    assert events["skills_invoked"] == ["openspec-new-change"]
+
+
+def test_experiment_summary_includes_token_and_cost_columns(monkeypatch, tmp_path: Path):
+    monkeypatch.setenv("BENCH_LOGS_DIR", str(tmp_path))
+    logger = ExperimentLogger(experiment_name="token-cost")
+    logger.add_result(
+        "COMET_FULL_040_BETA",
+        TreatmentResult(
+            name="COMET_FULL_040_BETA",
+            passed=True,
+            checks_passed=["baseline"],
+            checks_failed=[],
+            events_summary={
+                "num_turns": 2,
+                "duration_seconds": 12,
+                "tool_calls": 3,
+                "total_tokens": 475,
+                "total_cost_usd": 0.123456,
+            },
+        ),
+    )
+
+    summary = logger.generate_summary()
+
+    assert "| Treatment | Checks |" in summary
+    assert "Tokens" in summary
+    assert "Cost" in summary
+    assert "| COMET_FULL_040_BETA | 1/1 (100%) |" in summary
+    assert "475" in summary
+    assert "$0.1235" in summary
+
+
+def test_experiment_finalize_honors_html_report_output_config(monkeypatch, tmp_path: Path):
+    monkeypatch.setenv("BENCH_LOGS_DIR", str(tmp_path))
+    logger = ExperimentLogger(
+        experiment_name="html-summary",
+        report_outputs=ReportOutputConfig(markdown=False, html=True),
+    )
+    logger.add_result(
+        "COMET_FULL_040_BETA",
+        TreatmentResult(
+            name="COMET_FULL_040_BETA",
+            passed=True,
+            checks_passed=["baseline"],
+            checks_failed=[],
+        ),
+    )
+
+    output_path = logger.finalize()
+    summary_md = logger.base_dir / "summary.md"
+    summary_html = logger.base_dir / "summary.html"
+    metadata = json.loads((logger.base_dir / "metadata.json").read_text())
+
+    assert output_path == summary_html
+    assert not summary_md.exists()
+    summary = summary_html.read_text(encoding="utf-8")
+    assert "<html" in summary.lower()
+    assert "Experiment Results Summary" in summary
+    assert metadata["report_outputs"]["html"].endswith("summary.html")
+    assert "markdown" not in metadata["report_outputs"]
+
+
+def test_save_raw_writes_utf8_output(tmp_path: Path):
+    save_raw(tmp_path, "COMET_FULL_040_BETA", 1, '{"text":"中文 �"}\n', "stderr 中文")
+
+    stdout_path = tmp_path / "raw" / "COMET_FULL_040_BETA_rep1_stdout.json"
+    stderr_path = tmp_path / "raw" / "COMET_FULL_040_BETA_rep1_stderr.txt"
+
+    assert "中文" in stdout_path.read_text(encoding="utf-8")
+    assert "stderr 中文" == stderr_path.read_text(encoding="utf-8")
+
+
+def test_save_artifacts_preserve_stable_treatment_filenames(tmp_path: Path):
+    save_events(tmp_path, "COMET_FULL_040_BETA", 2, {"ok": True})
+    save_raw(tmp_path, "COMET_FULL_040_BETA", 2, "{}", "stderr")
+    save_report(tmp_path, "COMET_FULL_040_BETA", 2, {"ok": True})
+
+    assert sorted(path.name for path in (tmp_path / "events").iterdir()) == [
+        "COMET_FULL_040_BETA_rep2.json"
+    ]
+    assert sorted(path.name for path in (tmp_path / "raw").iterdir()) == [
+        "COMET_FULL_040_BETA_rep2_stderr.txt",
+        "COMET_FULL_040_BETA_rep2_stdout.json",
+    ]
+    assert sorted(path.name for path in (tmp_path / "reports").iterdir()) == [
+        "COMET_FULL_040_BETA_rep2_report.json"
+    ]
+
+
+def test_rubric_columns_accept_profile_dimensions():
+    columns = rubric_columns(("completion", "skill_invocation", "weighted_score"))
+
+    assert [column.name for column in columns] == [
+        "completion",
+        "skill_invocation",
+        "weighted_score",
+        "RubricAvg",
+    ]
+
+
+def test_rubric_average_excludes_weighted_score():
+    columns = rubric_columns(("completion", "skill_invocation", "weighted_score"))
+    avg_column = next(column for column in columns if column.name == "RubricAvg")
+    result = TreatmentResult(
+        name="COMET_FULL_040_BETA",
+        passed=True,
+        checks_passed=[
+            "[RUBRIC] completion: 0.00 - failed",
+            "[RUBRIC] skill_invocation: 1.00 - ok",
+            "[RUBRIC] weighted_score: 1.00",
+        ],
+        checks_failed=[],
+    )
+
+    assert avg_column.get_value(result) == "0.50"
+
+
+def test_treatment_result_exposes_eval_metadata():
+    result = TreatmentResult(
+        name="DYNAMIC_SKILL",
+        passed=True,
+        checks_passed=[],
+        checks_failed=[],
+        events_summary={
+            "profile": "generic",
+            "skill_sources": [{"name": "demo", "hash": "sha256:abc"}],
+            "eval_manifest": "demo/comet/eval.yaml",
+            "interaction": {"mode": "none"},
+            "artifact_references": {"report": "logs/reports/demo_report.json"},
+            "failure_attribution": [
+                {
+                    "bucket": "task",
+                    "check": "validator missing",
+                    "reason": "task or validator path assumption failed",
+                }
+            ],
+        },
+    )
+
+    assert result.events_summary["profile"] == "generic"
+    assert result.events_summary["skill_sources"][0]["hash"] == "sha256:abc"
+    assert result.events_summary["artifact_references"]["report"].endswith("demo_report.json")
+    assert result.events_summary["failure_attribution"][0]["bucket"] == "task"