@rpamis/comet 0.4.0-beta.3 → 0.4.0-beta.5

package/eval/local/tests/scaffold/test_langsmith_conftest.py

@@ -0,0 +1,244 @@
+"""Unit tests for LangSmith eval suite configuration helpers."""
+
+from __future__ import annotations
+
+import inspect
+import importlib.util
+import sys
+from pathlib import Path
+from types import SimpleNamespace
+import json
+
+
+def _load_langsmith_conftest():
+    eval_root = Path(__file__).resolve().parents[3]
+    conftest_path = eval_root / "langsmith" / "tests" / "conftest.py"
+    spec = importlib.util.spec_from_file_location("_test_langsmith_conftest", conftest_path)
+    module = importlib.util.module_from_spec(spec)
+    assert spec and spec.loader
+    spec.loader.exec_module(module)
+    return module
+
+
+def _load_langsmith_task_tests():
+    eval_root = Path(__file__).resolve().parents[3]
+    task_tests_path = eval_root / "langsmith" / "tests" / "tasks" / "test_tasks.py"
+    sys.modules.pop("_test_langsmith_task_tests", None)
+    sys.modules.pop("_comet_local_test_tasks", None)
+    spec = importlib.util.spec_from_file_location("_test_langsmith_task_tests", task_tests_path)
+    module = importlib.util.module_from_spec(spec)
+    assert spec and spec.loader
+    spec.loader.exec_module(module)
+    return module
+
+
+def test_langsmith_env_derives_claude_code_plugin_settings(monkeypatch):
+    module = _load_langsmith_conftest()
+    for key in (
+        "LANGSMITH_API_KEY",
+        "LANGSMITH_PROJECT",
+        "LANGSMITH_TRACING",
+        "TRACE_TO_LANGSMITH",
+        "CC_LANGSMITH_API_KEY",
+        "CC_LANGSMITH_PROJECT",
+    ):
+        monkeypatch.delenv(key, raising=False)
+    monkeypatch.setenv("LANGSMITH_API_KEY", "lsv2_pt_test")
+    monkeypatch.setenv("LANGSMITH_PROJECT", "comet-tests")
+
+    module.configure_langsmith_environment()
+
+    assert module.os.environ["LANGSMITH_TRACING"] == "true"
+    assert module.os.environ["TRACE_TO_LANGSMITH"] == "true"
+    assert module.os.environ["CC_LANGSMITH_API_KEY"] == "lsv2_pt_test"
+    assert module.os.environ["CC_LANGSMITH_PROJECT"] == "comet-tests"
+
+
+def test_langsmith_env_preserves_explicit_claude_code_overrides(monkeypatch):
+    module = _load_langsmith_conftest()
+    monkeypatch.setenv("LANGSMITH_API_KEY", "lsv2_pt_eval")
+    monkeypatch.setenv("LANGSMITH_PROJECT", "eval-project")
+    monkeypatch.setenv("LANGSMITH_TRACING", "false")
+    monkeypatch.setenv("TRACE_TO_LANGSMITH", "custom")
+    monkeypatch.setenv("CC_LANGSMITH_API_KEY", "lsv2_pt_plugin")
+    monkeypatch.setenv("CC_LANGSMITH_PROJECT", "plugin-project")
+
+    module.configure_langsmith_environment()
+
+    assert module.os.environ["LANGSMITH_TRACING"] == "false"
+    assert module.os.environ["TRACE_TO_LANGSMITH"] == "custom"
+    assert module.os.environ["CC_LANGSMITH_API_KEY"] == "lsv2_pt_plugin"
+    assert module.os.environ["CC_LANGSMITH_PROJECT"] == "plugin-project"
+
+
+def test_langsmith_tracing_auto_builds_default_plugin_cache(monkeypatch, tmp_path):
+    module = _load_langsmith_conftest()
+    plugin_dir = tmp_path / "langsmith-cc-plugin"
+    calls = []
+
+    monkeypatch.setattr(module, "DEFAULT_LANGSMITH_PLUGIN_DIR", plugin_dir)
+    monkeypatch.delenv("CC_LANGSMITH_PLUGIN_DIR", raising=False)
+    monkeypatch.setenv("LANGSMITH_TRACING", "true")
+    monkeypatch.setenv("TRACE_TO_LANGSMITH", "true")
+
+    def fake_build(target):
+        calls.append(target)
+        target.mkdir(parents=True)
+        return True
+
+    monkeypatch.setattr(module, "_build_default_langsmith_plugin", fake_build)
+
+    resolved = module.provision_langsmith_plugin_dir()
+
+    assert resolved == plugin_dir
+    assert calls == [plugin_dir]
+    assert module.os.environ["CC_LANGSMITH_PLUGIN_DIR"] == module._plugin_dir_env_value(plugin_dir)
+
+
+def test_langsmith_task_wrapper_registers_reexported_local_module():
+    _load_langsmith_task_tests()
+
+    assert "_comet_local_test_tasks" in sys.modules
+
+
+def test_langsmith_task_wrapper_does_not_require_pytest_request_fixture():
+    module = _load_langsmith_task_tests()
+
+    assert "request" not in inspect.signature(module.test_task_treatment).parameters
+
+
+def test_langsmith_task_wrapper_finds_parametrized_local_result():
+    module = _load_langsmith_task_tests()
+    result = object()
+    module._local_test_tasks.conftest._plugin = SimpleNamespace(
+        logger=SimpleNamespace(
+            results={
+                "comet-fix-median-CONTROL-r1": [result],
+            }
+        )
+    )
+
+    assert module._latest_result("CONTROL") is result
+
+
+def test_langsmith_task_wrapper_finds_result_from_loaded_conftest(monkeypatch):
+    module = _load_langsmith_task_tests()
+    result = object()
+    monkeypatch.setattr(module._local_test_tasks.conftest, "_plugin", None, raising=False)
+    monkeypatch.setitem(
+        sys.modules,
+        "conftest",
+        SimpleNamespace(
+            _plugin=SimpleNamespace(
+                logger=SimpleNamespace(
+                    results={
+                        "comet-fix-median-CONTROL-r1": [result],
+                    }
+                )
+            )
+        ),
+    )
+
+    assert module._latest_result("CONTROL") is result
+
+
+def test_langsmith_task_wrapper_falls_back_to_latest_report(monkeypatch, tmp_path):
+    module = _load_langsmith_task_tests()
+    monkeypatch.setattr(module._local_test_tasks.conftest, "_plugin", None, raising=False)
+    monkeypatch.setenv("BENCH_LOGS_DIR", str(tmp_path))
+    reports_dir = tmp_path / "experiments" / "experiment_20260703_220136" / "reports"
+    reports_dir.mkdir(parents=True)
+    report = {
+        "name": "comet-fix-median-CONTROL-r1",
+        "passed": False,
+        "run_id": "local-run-1",
+        "checks_passed": ["[RUBRIC] weighted_score: 0.35"],
+        "checks_failed": ["openspec_artifacts: missing"],
+        "events_summary": {"total_tokens": 123},
+    }
+    (reports_dir / "comet_fix_median_CONTROL_r1_rep1_report.json").write_text(
+        json.dumps(report)
+    )
+
+    result = module._latest_result("CONTROL")
+
+    assert result.run_id == "local-run-1"
+    assert result.checks_failed == ["openspec_artifacts: missing"]
+    assert result.events_summary["total_tokens"] == 123
+
+
+def test_langsmith_task_wrapper_patches_failed_end_run_outputs():
+    module = _load_langsmith_task_tests()
+
+    assert getattr(
+        module.ls_testing_internal._TestCase.end_run,
+        "_comet_preserve_logged_outputs",
+        False,
+    )
+
+
+def test_langsmith_task_wrapper_logs_outputs_and_rubric_feedback(monkeypatch):
+    module = _load_langsmith_task_tests()
+    calls = []
+    monkeypatch.setattr(
+        module,
+        "ls_testing",
+        SimpleNamespace(
+            log_outputs=lambda outputs: calls.append(("outputs", outputs)),
+            log_feedback=lambda **kwargs: calls.append(("feedback", kwargs)),
+        ),
+    )
+    result = SimpleNamespace(
+        run_id="local-run-1",
+        passed=False,
+        checks_passed=[
+            "tests_written: ok",
+            "[RUBRIC] skill_invocation: 0.20 - missing deps",
+            "[RUBRIC] weighted_score: 0.35",
+        ],
+        checks_failed=["openspec_artifacts: missing"],
+        events_summary={"total_tokens": 123, "skills_invoked": ["comet"]},
+    )
+
+    module._log_outputs_and_feedback(result)
+
+    assert calls == [
+        (
+            "outputs",
+            {
+                "run_id": "local-run-1",
+                "passed": False,
+                "checks_passed": 3,
+                "checks_failed": 1,
+                "num_turns": None,
+                "tool_calls": None,
+                "duration_seconds": None,
+                "total_tokens": 123,
+                "total_cost_usd": None,
+                "skills_invoked": ["comet"],
+            },
+        ),
+        ("feedback", {"key": "checks_pass_rate", "score": 0.75}),
+        ("feedback", {"key": "rubric.skill_invocation", "score": 0.2}),
+        ("feedback", {"key": "rubric.weighted_score", "score": 0.35}),
+    ]
+
+
+def test_langsmith_task_wrapper_keeps_plugin_on_configured_project(monkeypatch):
+    module = _load_langsmith_task_tests()
+    monkeypatch.delenv("CC_LANGSMITH_PARENT_DOTTED_ORDER", raising=False)
+    monkeypatch.setenv("CC_LANGSMITH_PROJECT", "comet-skill-eval")
+    monkeypatch.setattr(
+        module,
+        "get_current_run_tree",
+        lambda: SimpleNamespace(
+            dotted_order="20260703T000000Z00000000000000000000000000",
+            session_name="comet-skill-eval:abc12345",
+        ),
+    )
+
+    dotted_order = module._set_parent_run_env()
+
+    assert dotted_order == "20260703T000000Z00000000000000000000000000"
+    assert module.os.environ["CC_LANGSMITH_PARENT_DOTTED_ORDER"] == dotted_order
+    assert module.os.environ["CC_LANGSMITH_PROJECT"] == "comet-skill-eval"

package/eval/local/tests/scaffold/test_logging.py

@@ -0,0 +1,243 @@
+"""Unit tests for eval stream parsing and experiment summaries."""
+
+import json
+from pathlib import Path
+
+from scaffold.python.logging import (
+    ExperimentLogger,
+    TreatmentResult,
+    extract_events,
+    parse_output,
+    rubric_columns,
+    save_events,
+    save_raw,
+    save_report,
+)
+from scaffold.python.report_outputs import ReportOutputConfig
+
+
+def test_save_artifacts_excludes_nested_git_metadata(tmp_path: Path):
+    from conftest import _save_artifacts
+
+    workspace = tmp_path / "workspace"
+    workspace.mkdir()
+    (workspace / "result.md").write_text("ok", encoding="utf-8")
+    git_dir = workspace / ".git"
+    git_dir.mkdir()
+    (git_dir / "config").write_text("[core]\n", encoding="utf-8")
+
+    _save_artifacts(tmp_path, "COMET_FULL_040_BETA", 1, workspace)
+
+    snapshot = tmp_path / "artifacts" / "comet_full_040_beta_rep1" / "claude"
+    assert (snapshot / "result.md").read_text(encoding="utf-8") == "ok"
+    assert not (snapshot / ".git").exists()
+
+
+def test_extract_events_captures_token_usage_and_cost():
+    stdout = "\n".join(
+        [
+            json.dumps({"type": "assistant", "message": {"content": []}}),
+            json.dumps(
+                {
+                    "type": "result",
+                    "duration_ms": 1200,
+                    "num_turns": 3,
+                    "total_cost_usd": 0.123456,
+                    "usage": {
+                        "input_tokens": 100,
+                        "output_tokens": 25,
+                        "cache_read_input_tokens": 300,
+                        "cache_creation_input_tokens": 50,
+                    },
+                    "modelUsage": {
+                        "mimo-v2.5-pro": {
+                            "inputTokens": 100,
+                            "outputTokens": 25,
+                            "cacheReadInputTokens": 300,
+                            "cacheCreationInputTokens": 50,
+                            "costUSD": 0.123456,
+                        }
+                    },
+                }
+            ),
+        ]
+    )
+
+    events = extract_events(parse_output(stdout))
+
+    assert events["input_tokens"] == 100
+    assert events["output_tokens"] == 25
+    assert events["cache_read_input_tokens"] == 300
+    assert events["cache_creation_input_tokens"] == 50
+    assert events["total_tokens"] == 475
+    assert events["total_cost_usd"] == 0.123456
+    assert events["model_usage"]["mimo-v2.5-pro"]["costUSD"] == 0.123456
+
+
+def test_extract_events_normalizes_openspec_skill_aliases():
+    stdout = "\n".join(
+        [
+            json.dumps(
+                {
+                    "type": "assistant",
+                    "message": {
+                        "content": [
+                            {
+                                "type": "tool_use",
+                                "id": "call_1",
+                                "name": "Skill",
+                                "input": {"skill": "opsx:new"},
+                            },
+                            {
+                                "type": "tool_use",
+                                "id": "call_2",
+                                "name": "Skill",
+                                "input": {"skill": "openspec-new-change"},
+                            },
+                        ]
+                    },
+                }
+            )
+        ]
+    )
+
+    events = extract_events(parse_output(stdout))
+
+    assert events["skills_invoked"] == ["openspec-new-change"]
+
+
+def test_experiment_summary_includes_token_and_cost_columns(monkeypatch, tmp_path: Path):
+    monkeypatch.setenv("BENCH_LOGS_DIR", str(tmp_path))
+    logger = ExperimentLogger(experiment_name="token-cost")
+    logger.add_result(
+        "COMET_FULL_040_BETA",
+        TreatmentResult(
+            name="COMET_FULL_040_BETA",
+            passed=True,
+            checks_passed=["baseline"],
+            checks_failed=[],
+            events_summary={
+                "num_turns": 2,
+                "duration_seconds": 12,
+                "tool_calls": 3,
+                "total_tokens": 475,
+                "total_cost_usd": 0.123456,
+            },
+        ),
+    )
+
+    summary = logger.generate_summary()
+
+    assert "| Treatment | Checks |" in summary
+    assert "Tokens" in summary
+    assert "Cost" in summary
+    assert "| COMET_FULL_040_BETA | 1/1 (100%) |" in summary
+    assert "475" in summary
+    assert "$0.1235" in summary
+
+
+def test_experiment_finalize_honors_html_report_output_config(monkeypatch, tmp_path: Path):
+    monkeypatch.setenv("BENCH_LOGS_DIR", str(tmp_path))
+    logger = ExperimentLogger(
+        experiment_name="html-summary",
+        report_outputs=ReportOutputConfig(markdown=False, html=True),
+    )
+    logger.add_result(
+        "COMET_FULL_040_BETA",
+        TreatmentResult(
+            name="COMET_FULL_040_BETA",
+            passed=True,
+            checks_passed=["baseline"],
+            checks_failed=[],
+        ),
+    )
+
+    output_path = logger.finalize()
+    summary_md = logger.base_dir / "summary.md"
+    summary_html = logger.base_dir / "summary.html"
+    metadata = json.loads((logger.base_dir / "metadata.json").read_text())
+
+    assert output_path == summary_html
+    assert not summary_md.exists()
+    summary = summary_html.read_text(encoding="utf-8")
+    assert "<html" in summary.lower()
+    assert "Experiment Results Summary" in summary
+    assert metadata["report_outputs"]["html"].endswith("summary.html")
+    assert "markdown" not in metadata["report_outputs"]
+
+
+def test_save_raw_writes_utf8_output(tmp_path: Path):
+    save_raw(tmp_path, "COMET_FULL_040_BETA", 1, '{"text":"中文 �"}\n', "stderr 中文")
+
+    stdout_path = tmp_path / "raw" / "COMET_FULL_040_BETA_rep1_stdout.json"
+    stderr_path = tmp_path / "raw" / "COMET_FULL_040_BETA_rep1_stderr.txt"
+
+    assert "中文" in stdout_path.read_text(encoding="utf-8")
+    assert "stderr 中文" == stderr_path.read_text(encoding="utf-8")
+
+
+def test_save_artifacts_preserve_stable_treatment_filenames(tmp_path: Path):
+    save_events(tmp_path, "COMET_FULL_040_BETA", 2, {"ok": True})
+    save_raw(tmp_path, "COMET_FULL_040_BETA", 2, "{}", "stderr")
+    save_report(tmp_path, "COMET_FULL_040_BETA", 2, {"ok": True})
+
+    assert sorted(path.name for path in (tmp_path / "events").iterdir()) == [
+        "COMET_FULL_040_BETA_rep2.json"
+    ]
+    assert sorted(path.name for path in (tmp_path / "raw").iterdir()) == [
+        "COMET_FULL_040_BETA_rep2_stderr.txt",
+        "COMET_FULL_040_BETA_rep2_stdout.json",
+    ]
+    assert sorted(path.name for path in (tmp_path / "reports").iterdir()) == [
+        "COMET_FULL_040_BETA_rep2_report.json"
+    ]
+
+
+def test_rubric_columns_accept_profile_dimensions():
+    columns = rubric_columns(("completion", "skill_invocation", "weighted_score"))
+
+    assert [column.name for column in columns] == [
+        "completion",
+        "skill_invocation",
+        "weighted_score",
+        "RubricAvg",
+    ]
+
+
+def test_rubric_average_excludes_weighted_score():
+    columns = rubric_columns(("completion", "skill_invocation", "weighted_score"))
+    avg_column = next(column for column in columns if column.name == "RubricAvg")
+    result = TreatmentResult(
+        name="COMET_FULL_040_BETA",
+        passed=True,
+        checks_passed=[
+            "[RUBRIC] completion: 0.00 - failed",
+            "[RUBRIC] skill_invocation: 1.00 - ok",
+            "[RUBRIC] weighted_score: 1.00",
+        ],
+        checks_failed=[],
+    )
+
+    assert avg_column.get_value(result) == "0.50"
+
+
+def test_treatment_result_exposes_eval_metadata():
+    result = TreatmentResult(
+        name="DYNAMIC_SKILL",
+        passed=True,
+        checks_passed=[],
+        checks_failed=[],
+        events_summary={
+            "profile": "generic",
+            "skill_sources": [{"name": "demo", "hash": "sha256:abc"}],
+            "eval_manifest": "demo/comet/eval.yaml",
+            "interaction": {"mode": "none"},
+            "artifact_references": {"report": "logs/reports/demo_report.json"},
+            "failure_attribution": [{"bucket": "task", "check": "validator missing", "reason": "task or validator path assumption failed"}],
+        },
+    )
+
+    assert result.events_summary["profile"] == "generic"
+    assert result.events_summary["skill_sources"][0]["hash"] == "sha256:abc"
+    assert result.events_summary["artifact_references"]["report"].endswith("demo_report.json")
+    assert result.events_summary["failure_attribution"][0]["bucket"] == "task"

package/eval/local/tests/scaffold/test_manifests.py

@@ -0,0 +1,132 @@
+"""Tests for generated Skill eval manifest loading."""
+
+from pathlib import Path
+
+import pytest
+
+from scaffold.python.manifests import load_eval_manifest
+
+
+def test_load_eval_manifest_parses_skill_package_metadata(tmp_path: Path):
+    package = tmp_path / "my-skill"
+    package.mkdir()
+    (package / "SKILL.md").write_text("---\nname: my-skill\n---\n\nBody.", encoding="utf-8")
+    comet_dir = package / "comet"
+    comet_dir.mkdir()
+    manifest_path = comet_dir / "eval.yaml"
+    manifest_path.write_text(
+        """
+apiVersion: comet.eval/v1alpha1
+kind: SkillEvalManifest
+metadata:
+  name: my-skill
+  description: Demo manifest
+skill:
+  name: my-skill
+  source: ..
+  profile: generic
+evaluation:
+  recommendedTasks:
+    - generic-skill-smoke
+    - workflow-route-conformance
+  baselineTreatments:
+    - CONTROL
+  qualityGates:
+    minWeightedScore: 0.8
+  requiredOutputSchemas:
+    - result.schema.v1
+  expectedEvidence:
+    - node: open
+      check: output-schema:open.result.schema.v1.summary
+  requiredSkills:
+    - my-skill
+  expectedArtifacts:
+    - result.md
+  generatedNodeSkills:
+    - my-skill-open
+    - my-skill-build
+  routeConformance:
+    task: workflow-route-conformance
+    expectedNodeOrder:
+      - open
+      - build
+interaction:
+  mode: auto_user
+  maxTurns: 8
+  simulatorPrompt: Answer concisely.
+""",
+        encoding="utf-8",
+    )
+
+    manifest = load_eval_manifest(manifest_path)
+
+    assert manifest.name == "my-skill"
+    assert manifest.skill_name == "my-skill"
+    assert manifest.skill_path == package.resolve()
+    assert manifest.profile == "generic"
+    assert manifest.recommended_tasks == ["generic-skill-smoke", "workflow-route-conformance"]
+    assert manifest.baseline_treatments == ["CONTROL"]
+    assert manifest.quality_gates == {"minWeightedScore": 0.8}
+    assert manifest.required_output_schemas == ["result.schema.v1"]
+    assert manifest.expected_evidence == [
+        {"node": "open", "check": "output-schema:open.result.schema.v1.summary"}
+    ]
+    assert manifest.required_skills == ["my-skill"]
+    assert manifest.expected_artifacts == ["result.md"]
+    assert manifest.generated_node_skills == ["my-skill-open", "my-skill-build"]
+    assert manifest.route_conformance_task == "workflow-route-conformance"
+    assert manifest.route_conformance_expected_node_order == ["open", "build"]
+    assert manifest.interaction.mode == "auto_user"
+    assert manifest.interaction.max_turns == 8
+    assert manifest.interaction.simulator_prompt == "Answer concisely."
+
+
+def test_load_eval_manifest_rejects_wrong_kind(tmp_path: Path):
+    manifest_path = tmp_path / "eval.yaml"
+    manifest_path.write_text(
+        "apiVersion: comet.eval/v1alpha1\nkind: Other\nmetadata:\n  name: bad\n",
+        encoding="utf-8",
+    )
+
+    with pytest.raises(ValueError, match="SkillEvalManifest"):
+        load_eval_manifest(manifest_path)
+
+
+@pytest.mark.parametrize(
+    ("field_yaml", "message"),
+    [
+        (
+            "metadata:\n  name: bad\n  draftHash: not-a-hash\nskill:\n  name: bad\n",
+            "metadata.draftHash",
+        ),
+        (
+            "metadata:\n  name: bad\nskill:\n  name: bad\nevaluation:\n  baselineTreatments: CONTROL\n",
+            "evaluation.baselineTreatments",
+        ),
+        (
+            "metadata:\n  name: bad\nskill:\n  name: bad\nevaluation:\n  requiredOutputSchemas:\n    - 42\n",
+            "evaluation.requiredOutputSchemas",
+        ),
+        (
+            "metadata:\n  name: bad\nskill:\n  name: bad\nevaluation:\n  qualityGates:\n    - bad\n",
+            "evaluation.qualityGates",
+        ),
+        (
+            "metadata:\n  name: bad\nskill:\n  name: bad\nevaluation:\n  expectedEvidence:\n    - missing-node\n",
+            "evaluation.expectedEvidence",
+        ),
+    ],
+)
+def test_load_eval_manifest_rejects_malformed_structured_fields(
+    tmp_path: Path,
+    field_yaml: str,
+    message: str,
+):
+    manifest_path = tmp_path / "eval.yaml"
+    manifest_path.write_text(
+        f"apiVersion: comet.eval/v1alpha1\nkind: SkillEvalManifest\n{field_yaml}",
+        encoding="utf-8",
+    )
+
+    with pytest.raises(ValueError, match=message):
+        load_eval_manifest(manifest_path)