@rpamis/comet 0.4.0-beta.2 → 0.4.0-beta.4

package/eval/local/tasks/comet-noise-distractor/task.toml

@@ -0,0 +1,20 @@
+[metadata]
+name = "comet-noise-distractor"
+description = "Fix invoice tax calculation while leaving an unrelated distracting file untouched"
+difficulty = "medium"
+category = "comet"
+tags = ["comet", "workflow", "noise", "scope-control"]
+default_treatments = ["CONTROL", "COMET_FULL_040_BETA", "COMET_FULL_039"]
+
+[template]
+required = []
+
+[environment]
+description = "Python invoice calculator plus a distracting unrelated file. Claude must keep scope tight."
+dockerfile = "environment/Dockerfile"
+timeout_sec = 900
+
+[validation]
+test_scripts = "test_noise_distractor.py"
+target_artifacts = []
+timeout = 120

package/eval/local/tasks/comet-noise-distractor/validation/test_noise_distractor.py

@@ -0,0 +1,20 @@
+import subprocess
+import sys
+
+from comet_checks import run_comet_checks, write_results, _passed, _failed, WORKSPACE
+
+
+def check_invoice_scope() -> dict:
+    result = subprocess.run(
+        [sys.executable, "-m", "pytest", "test_invoice.py", "-q"],
+        capture_output=True,
+        text=True,
+        cwd=str(WORKSPACE),
+    )
+    if result.returncode != 0:
+        return _failed("invoice_fix_scope_control", (result.stdout + result.stderr)[-600:])
+    return _passed("invoice_fix_scope_control", "tax calculation fixed and distractor untouched")
+
+
+if __name__ == "__main__":
+    write_results([*run_comet_checks(), check_invoice_scope()])

package/eval/local/tasks/comet-observability-env-template/environment/Dockerfile

@@ -0,0 +1,15 @@
+FROM python:3.12-slim
+
+WORKDIR /workspace
+
+RUN apt-get update && apt-get install -y --no-install-recommends bash git curl jq ca-certificates nodejs npm && rm -rf /var/lib/apt/lists/*
+RUN pip install --no-cache-dir pyyaml pytest
+RUN npm config set registry https://registry.npmmirror.com && npm install -g @fission-ai/openspec@1.3.1 && npm install -g @anthropic-ai/claude-code@latest && npm config delete registry
+COPY . /workspace/
+RUN mkdir -p openspec/changes
+RUN useradd --create-home --shell /bin/bash agent && chown -R agent:agent /workspace
+USER agent
+ENV HOME=/home/agent
+ENV PYTHONPATH=/workspace
+RUN git config --global user.email "agent@eval.local" && git config --global user.name "Comet Eval Agent" && git config --global init.defaultBranch master
+CMD ["bash"]

package/eval/local/tasks/comet-observability-env-template/environment/env_writer.py

@@ -0,0 +1,17 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+
+def write_env_example(path: str | Path = ".env.example") -> Path:
+    target = Path(path)
+    target.write_text(
+        "ANTHROPIC_API_KEY=\n"
+        "LANGSMITH_API_KEY=\n",
+        encoding="utf-8",
+    )
+    return target
+
+
+if __name__ == "__main__":
+    write_env_example()

package/eval/local/tasks/comet-observability-env-template/environment/test_env_writer.py

@@ -0,0 +1,17 @@
+from __future__ import annotations
+
+import subprocess
+import sys
+from pathlib import Path
+
+
+def test_env_example_contains_simplified_langsmith_config(tmp_path):
+    script = Path.cwd() / "env_writer.py"
+    result = subprocess.run([sys.executable, str(script)], cwd=tmp_path, capture_output=True, text=True)
+    assert result.returncode == 0
+    content = (tmp_path / ".env.example").read_text(encoding="utf-8")
+    assert "ANTHROPIC_API_KEY=" in content
+    assert "LANGSMITH_API_KEY=" in content
+    assert "LANGSMITH_PROJECT=agent-template" in content
+    assert "LANGSMITH_TRACING=true" in content
+    assert "TRACE_TO_LANGSMITH" not in content

package/eval/local/tasks/comet-observability-env-template/instruction.md

@@ -0,0 +1,20 @@
+You are working on a Python project called "agent-template".
+
+Your task: Use the comet workflow to fix the `.env.example` generator for LangSmith observability.
+
+This task is adapted from `skills-benchmarks/ls-lang-tracing`.
+
+## Requirements
+
+Running `python env_writer.py` should create `.env.example` with:
+
+- `ANTHROPIC_API_KEY=`
+- `LANGSMITH_API_KEY=`
+- `LANGSMITH_PROJECT=agent-template`
+- `LANGSMITH_TRACING=true`
+
+Do not include `TRACE_TO_LANGSMITH` in the normal template; the harness derives Claude Code plugin variables from `LANGSMITH_*`.
+
+Run `python -m pytest test_env_writer.py -q`, follow the comet workflow, and archive the completed change.
+
+When the workflow asks for confirmation, assume "yes, proceed with the recommended option".

package/eval/local/tasks/comet-observability-env-template/task.toml

@@ -0,0 +1,20 @@
+[metadata]
+name = "comet-observability-env-template"
+description = "Generate a LangSmith-ready .env.example with simplified LANGSMITH_* configuration"
+difficulty = "medium"
+category = "comet"
+tags = ["comet", "workflow", "config", "observability"]
+default_treatments = ["CONTROL", "COMET_FULL_040_BETA", "COMET_FULL_039"]
+
+[template]
+required = []
+
+[environment]
+description = "Python project with an incomplete env template generator."
+dockerfile = "environment/Dockerfile"
+timeout_sec = 900
+
+[validation]
+test_scripts = "test_observability_env_template.py"
+target_artifacts = []
+timeout = 120

package/eval/local/tasks/comet-observability-env-template/validation/test_observability_env_template.py

@@ -0,0 +1,46 @@
+import subprocess
+import sys
+
+from comet_checks import WORKSPACE, _failed, _passed, run_comet_checks, write_results
+
+
+def check_langsmith_env_template() -> dict:
+    result = subprocess.run(
+        [sys.executable, "-m", "pytest", "test_env_writer.py", "-q"],
+        capture_output=True,
+        text=True,
+        cwd=str(WORKSPACE),
+    )
+    if result.returncode != 0:
+        return _failed("langsmith_env_template", (result.stdout + result.stderr)[-600:])
+
+    env_example = WORKSPACE / ".env.example"
+    if not env_example.exists():
+        run_result = subprocess.run(
+            [sys.executable, "env_writer.py"],
+            capture_output=True,
+            text=True,
+            cwd=str(WORKSPACE),
+        )
+        if run_result.returncode != 0:
+            return _failed(
+                "langsmith_env_template",
+                (run_result.stdout + run_result.stderr)[-600:],
+            )
+    content = env_example.read_text(encoding="utf-8")
+    expected = [
+        "ANTHROPIC_API_KEY=",
+        "LANGSMITH_API_KEY=",
+        "LANGSMITH_PROJECT=agent-template",
+        "LANGSMITH_TRACING=true",
+    ]
+    missing = [item for item in expected if item not in content]
+    if missing:
+        return _failed("langsmith_env_template", f"missing entries: {missing}")
+    if "TRACE_TO_LANGSMITH" in content:
+        return _failed("langsmith_env_template", "TRACE_TO_LANGSMITH should not be emitted")
+    return _passed("langsmith_env_template", "LangSmith .env.example template is correct")
+
+
+if __name__ == "__main__":
+    write_results([*run_comet_checks(), check_langsmith_env_template()])

package/eval/local/tasks/comet-perf-dedupe/environment/Dockerfile

@@ -0,0 +1,39 @@
+FROM python:3.12-slim
+
+WORKDIR /workspace
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    bash git curl jq ca-certificates \
+    nodejs npm && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN pip install --no-cache-dir pyyaml pytest
+
+RUN npm config set registry https://registry.npmmirror.com && \
+    npm install -g @fission-ai/openspec@1.3.1 && \
+    npm config delete registry
+
+ARG CLAUDE_CODE_VERSION=latest
+RUN npm config set registry https://registry.npmmirror.com && \
+    npm install -g @anthropic-ai/claude-code@${CLAUDE_CODE_VERSION} && \
+    npm config delete registry
+
+COPY dedupe.py /workspace/dedupe.py
+COPY test_dedupe.py /workspace/test_dedupe.py
+
+RUN mkdir -p wordcount-cli && \
+    echo '# Dedupe Tool' > wordcount-cli/README.md
+RUN mkdir -p openspec/changes
+
+RUN useradd --create-home --shell /bin/bash agent && \
+    chown -R agent:agent /workspace
+USER agent
+ENV HOME=/home/agent
+# Validator scripts import scaffold.*; bake PYTHONPATH so it never crosses the
+# MSYS path-conversion boundary on the docker run command line.
+ENV PYTHONPATH=/workspace
+RUN git config --global user.email "agent@eval.local" && \
+    git config --global user.name "Comet Eval Agent" && \
+    git config --global init.defaultBranch master
+
+CMD ["bash"]

package/eval/local/tasks/comet-perf-dedupe/environment/dedupe.py

@@ -0,0 +1,22 @@
+"""dedupe-tool: remove duplicates from a sequence while preserving order.
+
+The current implementation is O(n²) — it scans the whole output list for each
+element. This is correct but far too slow for large inputs (10k+ items).
+"""
+
+from __future__ import annotations
+
+from typing import Hashable, List, Sequence
+
+
+def dedupe(items: Sequence[Hashable]) -> List[Hashable]:
+    """Remove duplicates, preserving first-occurrence order.
+
+    BUG (performance): O(n²) because `in result` is a linear scan. For a list
+    of 10,000 items this takes several seconds; it should be near-instant.
+    """
+    result: List[Hashable] = []
+    for item in items:
+        if item not in result:  # <-- O(n) scan, making the whole thing O(n²)
+            result.append(item)
+    return result

package/eval/local/tasks/comet-perf-dedupe/environment/test_dedupe.py

@@ -0,0 +1,43 @@
+"""Tests for dedupe-tool. All pass currently; the issue is performance, not correctness."""
+
+import time
+
+import pytest
+
+from dedupe import dedupe
+
+
+class TestCorrectness:
+    def test_basic(self):
+        assert dedupe([1, 2, 2, 3, 3, 3]) == [1, 2, 3]
+
+    def test_preserves_order(self):
+        assert dedupe([3, 1, 3, 2, 1]) == [3, 1, 2]
+
+    def test_empty(self):
+        assert dedupe([]) == []
+
+    def test_all_same(self):
+        assert dedupe([5, 5, 5, 5]) == [5]
+
+    def test_no_duplicates(self):
+        assert dedupe([1, 2, 3]) == [1, 2, 3]
+
+    def test_strings(self):
+        assert dedupe(["a", "b", "a", "c"]) == ["a", "b", "c"]
+
+
+class TestPerformance:
+    def test_large_input_is_fast(self):
+        """A 30k-element list should dedupe in well under 1 second.
+
+        The current O(n^2) implementation takes ~2s on a fast machine and
+        longer in a Docker container; an O(n) implementation is near-instant.
+        """
+        data = list(range(15000)) * 2  # 30k items, 15k unique
+        start = time.perf_counter()
+        result = dedupe(data)
+        elapsed = time.perf_counter() - start
+
+        assert result == list(range(15000)), "correctness broke"
+        assert elapsed < 1.0, f"too slow: {elapsed:.2f}s (O(n²)? expected O(n))"

package/eval/local/tasks/comet-perf-dedupe/instruction.md

@@ -0,0 +1,25 @@
+You are working on a Python project called "dedupe-tool" - a utility for removing duplicates from sequences.
+
+Your task: Use the comet workflow to **optimize** the `dedupe()` function from O(n²) to O(n).
+
+## The problem
+
+`dedupe()` in `dedupe.py` is correct but uses `if item not in result` where `result` is a list — a linear scan that makes the whole function O(n²). For a 10,000-element input this takes several seconds.
+
+The performance test `TestPerformance::test_large_input_is_fast` currently FAILS because of this.
+
+## Goal
+
+Rewrite `dedupe()` to run in **O(n)** time while preserving:
+1. **Correctness** — same output for all inputs (first-occurrence order preserved)
+2. **The signature** — `dedupe(items: Sequence[Hashable]) -> List[Hashable]`
+
+The standard approach is to track seen items in a `set` (O(1) lookup) alongside the result list. Consider whether `dict.fromkeys()` (which preserves insertion order in Python 3.7+) is appropriate, and document the tradeoff in your design.
+
+## What to do
+
+1. Run the tests to confirm the perf test fails: `python -m pytest test_dedupe.py -v`
+2. Follow the comet workflow (Open → Design → Build → Verify → Archive).
+3. After optimizing, confirm ALL tests pass including the performance test.
+
+Start by detecting the current phase and following the comet workflow. When you reach a decision point that asks for confirmation, assume "yes, proceed with the recommended option".

package/eval/local/tasks/comet-perf-dedupe/task.toml

@@ -0,0 +1,24 @@
+[metadata]
+name = "comet-perf-dedupe"
+description = "Optimize an O(n^2) deduplication function to O(n) using a set/dict while preserving order"
+difficulty = "medium"
+category = "comet"
+tags = ["comet", "workflow", "performance", "algorithm"]
+default_treatments = [
+    "CONTROL",
+    "COMET_FULL_040_BETA",
+    "COMET_FULL_039",
+]
+
+[template]
+required = []
+
+[environment]
+description = "Python dedupe tool with an O(n^2) implementation. Claude must optimize to O(n)."
+dockerfile = "environment/Dockerfile"
+timeout_sec = 900
+
+[validation]
+test_scripts = "test_perf_dedupe.py"
+target_artifacts = []
+timeout = 120

package/eval/local/tasks/comet-perf-dedupe/validation/test_perf_dedupe.py

@@ -0,0 +1,40 @@
+"""Validation for the comet-perf-dedupe task.
+
+Checks: comet workflow followed + all tests pass (correctness + performance).
+"""
+
+import subprocess
+import sys
+
+from comet_checks import (
+    run_comet_checks,
+    write_results,
+    _passed,
+    _failed,
+    WORKSPACE,
+)
+
+
+def check_dedupe_optimized() -> dict:
+    """All dedupe tests pass, including the performance test."""
+    result = subprocess.run(
+        [sys.executable, "-m", "pytest", "test_dedupe.py", "-v", "-q"],
+        capture_output=True, text=True, cwd=str(WORKSPACE),
+    )
+    if result.returncode != 0:
+        # Distinguish perf failure from correctness failure.
+        out = result.stdout + result.stderr
+        if "too slow" in out or "O(n" in out:
+            return _failed("dedupe_optimized", f"still O(n²): {out[-300:]}")
+        return _failed("dedupe_optimized", f"tests failing:\n{out[-400:]}")
+    return _passed("dedupe_optimized", "correctness + performance tests pass")
+
+
+def run_all() -> list[dict]:
+    results = run_comet_checks()
+    results.append(check_dedupe_optimized())
+    return results
+
+
+if __name__ == "__main__":
+    write_results(run_all())

package/eval/local/tasks/comet-persistence-threading/environment/Dockerfile

@@ -0,0 +1,15 @@
+FROM python:3.12-slim
+
+WORKDIR /workspace
+
+RUN apt-get update && apt-get install -y --no-install-recommends bash git curl jq ca-certificates nodejs npm && rm -rf /var/lib/apt/lists/*
+RUN pip install --no-cache-dir pyyaml pytest
+RUN npm config set registry https://registry.npmmirror.com && npm install -g @fission-ai/openspec@1.3.1 && npm install -g @anthropic-ai/claude-code@latest && npm config delete registry
+COPY . /workspace/
+RUN mkdir -p openspec/changes
+RUN useradd --create-home --shell /bin/bash agent && chown -R agent:agent /workspace
+USER agent
+ENV HOME=/home/agent
+ENV PYTHONPATH=/workspace
+RUN git config --global user.email "agent@eval.local" && git config --global user.name "Comet Eval Agent" && git config --global init.defaultBranch master
+CMD ["bash"]

package/eval/local/tasks/comet-persistence-threading/environment/session_store.py

@@ -0,0 +1,15 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+
+class SessionStore:
+    def __init__(self, path: str | Path):
+        self.path = Path(path)
+        self._state: dict[str, str] = {}
+
+    def save(self, thread_id: str, key: str, value: str) -> None:
+        self._state[key] = value
+
+    def load(self, thread_id: str, key: str, default: str | None = None) -> str | None:
+        return self._state.get(key, default)

package/eval/local/tasks/comet-persistence-threading/environment/test_session_store.py

@@ -0,0 +1,21 @@
+from __future__ import annotations
+
+from session_store import SessionStore
+
+
+def test_thread_ids_are_isolated(tmp_path):
+    store = SessionStore(tmp_path / "state.json")
+    store.save("thread-a", "status", "draft")
+    store.save("thread-b", "status", "approved")
+
+    assert store.load("thread-a", "status") == "draft"
+    assert store.load("thread-b", "status") == "approved"
+
+
+def test_state_persists_across_store_instances(tmp_path):
+    path = tmp_path / "state.json"
+    first = SessionStore(path)
+    first.save("thread-a", "checkpoint", "ready")
+
+    second = SessionStore(path)
+    assert second.load("thread-a", "checkpoint") == "ready"

package/eval/local/tasks/comet-persistence-threading/instruction.md

@@ -0,0 +1,14 @@
+You are working on a Python project called "session-store".
+
+Your task: Use the comet workflow to fix state isolation and persistence.
+
+This task is adapted from `skills-benchmarks/oss-fix-lg-persistence`.
+
+## Bugs
+
+- Values written for one `thread_id` leak into another thread.
+- Creating a new `SessionStore` pointed at the same data file loses previously saved state.
+
+Run `python -m pytest test_session_store.py -q`, follow the comet workflow, and keep the `SessionStore` API compatible.
+
+When the workflow asks for confirmation, assume "yes, proceed with the recommended option".

package/eval/local/tasks/comet-persistence-threading/task.toml

@@ -0,0 +1,20 @@
+[metadata]
+name = "comet-persistence-threading"
+description = "Make session state isolate by thread_id and persist across store instances"
+difficulty = "hard"
+category = "comet"
+tags = ["comet", "workflow", "persistence", "threading"]
+default_treatments = ["CONTROL", "COMET_FULL_040_BETA", "COMET_FULL_039"]
+
+[template]
+required = []
+
+[environment]
+description = "Python session store with thread isolation and persistence bugs."
+dockerfile = "environment/Dockerfile"
+timeout_sec = 900
+
+[validation]
+test_scripts = "test_persistence_threading.py"
+target_artifacts = []
+timeout = 120

package/eval/local/tasks/comet-persistence-threading/validation/test_persistence_threading.py

@@ -0,0 +1,20 @@
+import subprocess
+import sys
+
+from comet_checks import run_comet_checks, write_results, _passed, _failed, WORKSPACE
+
+
+def check_persistence_threading() -> dict:
+    result = subprocess.run(
+        [sys.executable, "-m", "pytest", "test_session_store.py", "-q"],
+        capture_output=True,
+        text=True,
+        cwd=str(WORKSPACE),
+    )
+    if result.returncode != 0:
+        return _failed("persistence_threading", (result.stdout + result.stderr)[-600:])
+    return _passed("persistence_threading", "thread state is isolated and persisted")
+
+
+if __name__ == "__main__":
+    write_results([*run_comet_checks(), check_persistence_threading()])

package/eval/local/tasks/comet-refactor-counter/environment/Dockerfile

@@ -0,0 +1,39 @@
+FROM python:3.12-slim
+
+WORKDIR /workspace
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    bash git curl jq ca-certificates \
+    nodejs npm && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN pip install --no-cache-dir pyyaml pytest
+
+RUN npm config set registry https://registry.npmmirror.com && \
+    npm install -g @fission-ai/openspec@1.3.1 && \
+    npm config delete registry
+
+ARG CLAUDE_CODE_VERSION=latest
+RUN npm config set registry https://registry.npmmirror.com && \
+    npm install -g @anthropic-ai/claude-code@${CLAUDE_CODE_VERSION} && \
+    npm config delete registry
+
+COPY text_processor.py /workspace/text_processor.py
+COPY test_text_processor.py /workspace/test_text_processor.py
+
+RUN mkdir -p wordcount-cli && \
+    echo '# Text Processor' > wordcount-cli/README.md
+RUN mkdir -p openspec/changes
+
+RUN useradd --create-home --shell /bin/bash agent && \
+    chown -R agent:agent /workspace
+USER agent
+ENV HOME=/home/agent
+# Validator scripts import scaffold.*; bake PYTHONPATH so it never crosses the
+# MSYS path-conversion boundary on the docker run command line.
+ENV PYTHONPATH=/workspace
+RUN git config --global user.email "agent@eval.local" && \
+    git config --global user.name "Comet Eval Agent" && \
+    git config --global init.defaultBranch master
+
+CMD ["bash"]

package/eval/local/tasks/comet-refactor-counter/environment/test_text_processor.py

@@ -0,0 +1,57 @@
+"""Tests for text-processor. All pass; refactoring must keep them passing."""
+
+import pytest
+
+from text_processor import count_words, count_lines, count_chars, analyze
+
+
+class TestCountWords:
+    def test_basic(self):
+        assert count_words("hello world foo") == 3
+
+    def test_empty(self):
+        assert count_words("") == 0
+
+    def test_whitespace_only(self):
+        assert count_words("   \t  ") == 0
+
+    def test_single(self):
+        assert count_words("hello") == 1
+
+    def test_multiple_spaces(self):
+        assert count_words("a  b   c") == 3
+
+
+class TestCountLines:
+    def test_basic(self):
+        assert count_lines("line1\nline2\nline3") == 3
+
+    def test_empty(self):
+        assert count_lines("") == 0
+
+    def test_single_no_newline(self):
+        assert count_lines("hello") == 1
+
+    def test_trailing_newline(self):
+        assert count_lines("a\nb\n") == 2
+
+
+class TestCountChars:
+    def test_basic(self):
+        assert count_chars("hello") == 5
+
+    def test_empty(self):
+        assert count_chars("") == 0
+
+    def test_with_spaces(self):
+        assert count_chars("a b c") == 5
+
+
+class TestAnalyze:
+    def test_combined(self):
+        text = "hello world\nfoo bar"
+        result = analyze(text)
+        assert result == {"words": 4, "lines": 2, "chars": 19}
+
+    def test_empty(self):
+        assert analyze("") == {"words": 0, "lines": 0, "chars": 0}

package/eval/local/tasks/comet-refactor-counter/environment/text_processor.py

@@ -0,0 +1,49 @@
+"""text-processor: text analysis utilities.
+
+This module has three near-identical counting functions with duplicated logic.
+They should be refactored into a single `count(text, unit)` dispatcher while
+preserving the exact same behavior (and the public names as thin wrappers).
+"""
+
+from __future__ import annotations
+
+
+def count_words(text: str) -> int:
+    """Count words (whitespace-separated tokens)."""
+    if not text:
+        return 0
+    words = text.split()
+    count = 0
+    for word in words:
+        count += 1
+    return count
+
+
+def count_lines(text: str) -> int:
+    """Count lines (newline-separated)."""
+    if not text:
+        return 0
+    lines = text.splitlines()
+    count = 0
+    for line in lines:
+        count += 1
+    return count
+
+
+def count_chars(text: str) -> int:
+    """Count characters."""
+    if not text:
+        return 0
+    count = 0
+    for char in text:
+        count += 1
+    return count
+
+
+def analyze(text: str) -> dict:
+    """Return a summary of all counts."""
+    return {
+        "words": count_words(text),
+        "lines": count_lines(text),
+        "chars": count_chars(text),
+    }