@rpamis/comet 0.4.0-beta.3 → 0.4.0-beta.5

package/eval/local/tasks/comet-dependency-confusion/validation/test_dependency_confusion.py

@@ -0,0 +1,24 @@
+import subprocess
+import sys
+
+from comet_checks import run_comet_checks, write_results, _passed, _failed, WORKSPACE
+
+
+def _tail(result: subprocess.CompletedProcess) -> str:
+    return "\n".join(part.strip() for part in (result.stdout, result.stderr) if part and part.strip())[-600:]
+
+
+def check_dependency_compatibility() -> dict:
+    result = subprocess.run(
+        [sys.executable, "-m", "pytest", "test_settings_service.py", "-q"],
+        capture_output=True,
+        text=True,
+        cwd=str(WORKSPACE),
+    )
+    if result.returncode != 0:
+        return _failed("dependency_compatibility", _tail(result))
+    return _passed("dependency_compatibility", "old and new settings imports both work")
+
+
+if __name__ == "__main__":
+    write_results([*run_comet_checks(), check_dependency_compatibility()])

package/eval/local/tasks/comet-fix-median/environment/Dockerfile

@@ -0,0 +1,50 @@
+FROM python:3.12-slim
+
+WORKDIR /workspace
+
+# Runtime tools for both comet baselines:
+# - bash: 0.3.9 classic shell scripts (bash 4+ arrays, [[ ]]; python:3.12-slim ships only dash)
+# - nodejs npm: 0.4.0-beta.1 comet (independent .mjs command scripts) and OpenSpec CLI
+# - git curl jq: comet scripts and state inspection
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    bash git curl jq ca-certificates \
+    nodejs npm && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN pip install --no-cache-dir pyyaml pytest
+
+# OpenSpec CLI (comet drives openspec new/list/status/archive).
+RUN npm config set registry https://registry.npmmirror.com && \
+    npm install -g @fission-ai/openspec@1.3.1 && \
+    npm config delete registry
+
+# Claude Code CLI (the agent runtime; version overridable via BENCH_CC_VERSION).
+ARG CLAUDE_CODE_VERSION=latest
+RUN npm config set registry https://registry.npmmirror.com && \
+    npm install -g @anthropic-ai/claude-code@${CLAUDE_CODE_VERSION} && \
+    npm config delete registry
+
+# The stats library with the median bug
+COPY stats.py /workspace/stats.py
+COPY test_stats.py /workspace/test_stats.py
+
+RUN mkdir -p wordcount-cli && \
+    echo '# Stats Lib\nA small statistics library.' > wordcount-cli/README.md
+
+# OpenSpec working tree (comet-open writes under openspec/changes/)
+RUN mkdir -p openspec/changes
+
+# Create a non-root user: claude --dangerously-skip-permissions refuses to run
+# as root, and most comet scripts also expect a normal HOME for git config etc.
+RUN useradd --create-home --shell /bin/bash agent && \
+    chown -R agent:agent /workspace
+USER agent
+ENV HOME=/home/agent
+# Validator scripts import `scaffold.*`; bake PYTHONPATH so it never crosses the
+# MSYS path-conversion boundary on the docker run command line.
+ENV PYTHONPATH=/workspace
+RUN git config --global user.email "agent@eval.local" && \
+    git config --global user.name "Comet Eval Agent" && \
+    git config --global init.defaultBranch master
+
+CMD ["bash"]

package/eval/local/tasks/comet-fix-median/environment/stats.py

@@ -0,0 +1,54 @@
+"""stats-lib: a small statistics calculation library.
+
+Provides mean, median, mode, and standard deviation. There is a known bug in
+`median()` for even-length inputs that needs fixing.
+"""
+
+from __future__ import annotations
+
+from collections import Counter
+from typing import Sequence
+
+
+def mean(values: Sequence[float]) -> float:
+    """Arithmetic mean. Raises ValueError on empty input."""
+    if not values:
+        raise ValueError("mean() requires at least one value")
+    return sum(values) / len(values)
+
+
+def median(values: Sequence[float]) -> float:
+    """Median (middle value for odd length, average of two middles for even).
+
+    BUG: for even-length inputs this returns the lower middle only instead of
+    averaging the two middle values.
+    """
+    if not values:
+        raise ValueError("median() requires at least one value")
+    ordered = sorted(values)
+    n = len(ordered)
+    mid = n // 2
+    if n % 2 == 1:
+        return ordered[mid]
+    # BUG: should be (ordered[mid - 1] + ordered[mid]) / 2
+    return ordered[mid - 1]
+
+
+def mode(values: Sequence[float]) -> float:
+    """Most common value. Raises ValueError on empty input or ties."""
+    if not values:
+        raise ValueError("mode() requires at least one value")
+    counts = Counter(values)
+    top = counts.most_common(2)
+    if len(top) > 1 and top[0][1] == top[1][1]:
+        raise ValueError("mode() is ambiguous: multiple values share the top count")
+    return top[0][0]
+
+
+def stdev(values: Sequence[float]) -> float:
+    """Population standard deviation. Raises ValueError on empty input."""
+    if not values:
+        raise ValueError("stdev() requires at least one value")
+    avg = mean(values)
+    variance = sum((x - avg) ** 2 for x in values) / len(values)
+    return variance ** 0.5

package/eval/local/tasks/comet-fix-median/environment/test_stats.py

@@ -0,0 +1,68 @@
+"""Tests for stats-lib.
+
+The median tests for even-length input currently FAIL — that is the bug to fix.
+All other tests pass.
+"""
+
+import pytest
+
+from stats import mean, median, mode, stdev
+
+
+class TestMean:
+    def test_basic(self):
+        assert mean([1, 2, 3, 4]) == 2.5
+
+    def test_single(self):
+        assert mean([42]) == 42
+
+    def test_empty_raises(self):
+        with pytest.raises(ValueError):
+            mean([])
+
+
+class TestMedian:
+    def test_odd(self):
+        assert median([3, 1, 2]) == 2
+
+    def test_single(self):
+        assert median([7]) == 7
+
+    def test_even(self):
+        # This test currently FAILS due to the bug.
+        assert median([1, 2, 3, 4]) == 2.5
+
+    def test_even_unsorted(self):
+        # This test currently FAILS due to the bug.
+        assert median([4, 1, 3, 2]) == 2.5
+
+    def test_negative(self):
+        assert median([-5, -1, -3]) == -3
+
+    def test_empty_raises(self):
+        with pytest.raises(ValueError):
+            median([])
+
+
+class TestMode:
+    def test_basic(self):
+        assert mode([1, 2, 2, 3]) == 2
+
+    def test_all_same(self):
+        assert mode([5, 5, 5]) == 5
+
+    def test_tie_raises(self):
+        with pytest.raises(ValueError):
+            mode([1, 2])
+
+
+class TestStdev:
+    def test_basic(self):
+        assert stdev([2, 4, 4, 4, 5, 5, 7, 9]) == 2.0
+
+    def test_single_zero(self):
+        assert stdev([3]) == 0.0
+
+    def test_empty_raises(self):
+        with pytest.raises(ValueError):
+            stdev([])

package/eval/local/tasks/comet-fix-median/instruction.md

@@ -0,0 +1,25 @@
+You are working on a Python project called "stats-lib" - a small statistics calculation library.
+
+Your task: Use the comet workflow to **fix a bug** in the `median()` function.
+
+## The bug
+
+`median()` returns the wrong result for even-length lists. For example, `median([1, 2, 3, 4])` returns `2` but should return `2.5` (the average of the two middle values).
+
+The existing tests in `test_stats.py` make this explicit — two tests in `TestMedian` currently fail:
+- `test_even`: `median([1, 2, 3, 4])` should be `2.5`
+- `test_even_unsorted`: `median([4, 1, 3, 2])` should be `2.5`
+
+All other tests pass and must continue to pass.
+
+## What to do
+
+1. First, run the existing tests to confirm the failure: `python -m pytest test_stats.py -v`
+2. Follow the comet workflow phases:
+   - **Open**: Create a proposal describing the bug and fix scope
+   - **Design**: Investigate the root cause and design the fix
+   - **Build**: Implement the fix in `stats.py`
+   - **Verify**: Run the full test suite to confirm all tests pass (including previously-failing ones) and no regressions
+   - **Archive**: Archive the completed change
+
+Start by detecting the current phase and following the comet workflow. When you reach a decision point that asks for confirmation, assume "yes, proceed with the recommended option".

package/eval/local/tasks/comet-fix-median/task.toml

@@ -0,0 +1,24 @@
+[metadata]
+name = "comet-fix-median"
+description = "Fix a bug in the median() function of a stats library (even-length lists return the wrong value)"
+difficulty = "medium"
+category = "comet"
+tags = ["comet", "workflow", "bugfix", "debugging"]
+default_treatments = [
+    "CONTROL",
+    "COMET_FULL_040_BETA",
+    "COMET_FULL_039",
+]
+
+[template]
+required = []
+
+[environment]
+description = "Python stats library with a median bug. Claude must diagnose, fix, and verify via the comet workflow."
+dockerfile = "environment/Dockerfile"
+timeout_sec = 900
+
+[validation]
+test_scripts = "test_fix_median.py"
+target_artifacts = []
+timeout = 120

package/eval/local/tasks/comet-fix-median/validation/test_fix_median.py

@@ -0,0 +1,60 @@
+"""Validation script for the comet-fix-median task.
+
+Runs inside Docker. Checks the comet workflow was followed AND the median bug
+is fixed (even-length lists) with no regressions.
+"""
+
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+from comet_checks import (
+    run_comet_checks,
+    write_results,
+    _passed,
+    _failed,
+    WORKSPACE,
+)
+
+
+def _subprocess_output(result: subprocess.CompletedProcess) -> str:
+    output = "\n".join(
+        part.strip()
+        for part in (result.stdout or "", result.stderr or "")
+        if part and part.strip()
+    )
+    return output[-600:]
+
+
+def check_median_fix() -> dict:
+    """median() returns 2.5 for [1,2,3,4] and all existing tests pass."""
+    # Import the (presumably fixed) stats module and check the even case.
+    result = subprocess.run(
+        [sys.executable, "-c", "from stats import median; print(median([1,2,3,4]))"],
+        capture_output=True, text=True, cwd=str(WORKSPACE),
+    )
+    if result.returncode != 0:
+        return _failed("median_fix", f"import/run error: {result.stderr.strip()[:150]}")
+    val = result.stdout.strip()
+    if val != "2.5":
+        return _failed("median_fix", f"median([1,2,3,4]) == {val}, expected 2.5")
+
+    # Run the full existing test suite — all must pass now.
+    test_result = subprocess.run(
+        [sys.executable, "-m", "pytest", "test_stats.py", "-q"],
+        capture_output=True, text=True, cwd=str(WORKSPACE),
+    )
+    if test_result.returncode != 0:
+        return _failed("median_fix", f"existing tests still failing:\n{_subprocess_output(test_result)}")
+    return _passed("median_fix", "median([1,2,3,4])==2.5, all tests pass")
+
+
+def run_all() -> list[dict]:
+    results = run_comet_checks()
+    results.append(check_median_fix())
+    return results
+
+
+if __name__ == "__main__":
+    write_results(run_all())

package/eval/local/tasks/comet-framework-selection/environment/Dockerfile

@@ -0,0 +1,28 @@
+FROM python:3.12-slim
+
+WORKDIR /workspace
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    bash git curl jq ca-certificates nodejs npm && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN pip install --no-cache-dir pyyaml pytest
+
+RUN npm config set registry https://registry.npmmirror.com && \
+    npm install -g @fission-ai/openspec@1.3.1 && \
+    npm install -g @anthropic-ai/claude-code@latest && \
+    npm config delete registry
+
+COPY . /workspace/
+RUN mkdir -p openspec/changes
+
+RUN useradd --create-home --shell /bin/bash agent && \
+    chown -R agent:agent /workspace
+USER agent
+ENV HOME=/home/agent
+ENV PYTHONPATH=/workspace
+RUN git config --global user.email "agent@eval.local" && \
+    git config --global user.name "Comet Eval Agent" && \
+    git config --global init.defaultBranch master
+
+CMD ["bash"]

package/eval/local/tasks/comet-framework-selection/environment/architecture.py

@@ -0,0 +1,12 @@
+def select_framework(requirements):
+    return "single-tool-agent"
+
+
+def build_document_assistant():
+    return {
+        "orchestrator": "create_react_agent",
+        "memory": "short_term_only",
+        "tools": ["edit_document", "extract_fields"],
+        "subworkflow": None,
+        "retry_policy": 0,
+    }

package/eval/local/tasks/comet-framework-selection/environment/test_architecture.py

@@ -0,0 +1,30 @@
+from pathlib import Path
+
+import architecture
+
+
+def test_selects_hybrid_framework_for_long_session_documents():
+    requirements = [
+        "long_session_memory",
+        "multi_document_editing",
+        "deterministic_extraction",
+        "retry_on_failure",
+    ]
+
+    assert architecture.select_framework(requirements) == "hybrid-deep-agent-langgraph"
+
+
+def test_blueprint_uses_deep_orchestrator_and_compiled_subworkflow():
+    blueprint = architecture.build_document_assistant()
+
+    assert blueprint["orchestrator"] == "create_deep_agent"
+    assert blueprint["subworkflow"] == "CompiledSubAgent"
+    assert blueprint["memory"] == "persistent_session"
+    assert blueprint["retry_policy"] >= 2
+
+
+def test_source_contains_framework_selection_markers():
+    source = Path("architecture.py").read_text(encoding="utf-8")
+
+    assert "create_deep_agent" in source
+    assert "CompiledSubAgent" in source

package/eval/local/tasks/comet-framework-selection/instruction.md

@@ -0,0 +1,22 @@
+You are working on a Python project called "document-architect".
+
+Your task: Use the comet workflow to choose and encode the right framework architecture.
+
+This task is adapted from `skills-benchmarks/lc-framework-hybrid-pipeline`.
+
+## Requirements
+
+Build a blueprint for an assistant that:
+
+1. Manages and edits multiple documents across a long session.
+2. Delegates structured extraction for specific documents to a deterministic retry-on-failure pipeline.
+3. Uses a deep-agent style orchestrator plus a compiled sub-workflow, rather than a single generic tool agent.
+
+## What to do
+
+1. Run `python -m pytest test_architecture.py -q` to confirm the failures.
+2. Follow the comet workflow phases from open through archive.
+3. Fix `architecture.py`; do not add external package dependencies.
+4. Verify that the selected architecture and blueprint clearly represent the hybrid orchestrator/sub-workflow design.
+
+When the workflow asks for confirmation, assume "yes, proceed with the recommended option".

package/eval/local/tasks/comet-framework-selection/task.toml

@@ -0,0 +1,24 @@
+[metadata]
+name = "comet-framework-selection"
+description = "Select a hybrid deep-agent plus deterministic sub-workflow architecture for document editing"
+difficulty = "medium"
+category = "comet"
+tags = ["comet", "workflow", "architecture", "framework-selection"]
+default_treatments = [
+    "CONTROL",
+    "COMET_FULL_040_BETA",
+    "COMET_FULL_039",
+]
+
+[template]
+required = []
+
+[environment]
+description = "Python workspace with an incomplete architecture blueprint for a long-session document assistant."
+dockerfile = "environment/Dockerfile"
+timeout_sec = 900
+
+[validation]
+test_scripts = "test_framework_selection.py"
+target_artifacts = []
+timeout = 120

package/eval/local/tasks/comet-framework-selection/validation/test_framework_selection.py

@@ -0,0 +1,24 @@
+import subprocess
+import sys
+
+from comet_checks import WORKSPACE, _failed, _passed, run_comet_checks, write_results
+
+
+def _tail(result: subprocess.CompletedProcess) -> str:
+    return "\n".join(part.strip() for part in (result.stdout, result.stderr) if part and part.strip())[-600:]
+
+
+def check_framework_selection() -> dict:
+    result = subprocess.run(
+        [sys.executable, "-m", "pytest", "test_architecture.py", "-q"],
+        capture_output=True,
+        text=True,
+        cwd=str(WORKSPACE),
+    )
+    if result.returncode != 0:
+        return _failed("framework_selection", _tail(result))
+    return _passed("framework_selection", "hybrid deep-agent plus compiled sub-workflow blueprint passes")
+
+
+if __name__ == "__main__":
+    write_results([*run_comet_checks(), check_framework_selection()])

package/eval/local/tasks/comet-full-workflow/environment/Dockerfile

@@ -0,0 +1,53 @@
+FROM python:3.12-slim
+
+WORKDIR /workspace
+
+# Runtime tools for both comet baselines:
+# - bash: 0.3.9 classic shell scripts (bash 4+ arrays, [[ ]]; python:3.12-slim ships only dash)
+# - nodejs npm: 0.4.0-beta.1 comet (independent .mjs command scripts) and OpenSpec CLI
+# - git curl jq: comet scripts and state inspection
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    bash git curl jq ca-certificates \
+    nodejs npm && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN pip install --no-cache-dir pyyaml
+
+# OpenSpec CLI (comet drives openspec new/list/status/archive).
+# Use npmmirror for reproducible builds behind GFW.
+RUN npm config set registry https://registry.npmmirror.com && \
+    npm install -g @fission-ai/openspec@1.3.1 && \
+    npm config delete registry
+
+# Claude Code CLI (the agent runtime; version overridable via BENCH_CC_VERSION).
+ARG CLAUDE_CODE_VERSION=latest
+RUN npm config set registry https://registry.npmmirror.com && \
+    npm install -g @anthropic-ai/claude-code@${CLAUDE_CODE_VERSION} && \
+    npm config delete registry
+
+# Sample project structure
+RUN mkdir -p wordcount-cli && \
+    echo '# Word Count CLI\nA simple word counting tool.' > wordcount-cli/README.md
+
+# Existing wordcount code (extended during the task)
+COPY wordcount.py /workspace/wordcount.py
+COPY test_wordcount.py /workspace/test_wordcount.py
+
+# OpenSpec working tree (comet-open writes under openspec/changes/)
+RUN mkdir -p openspec/changes
+
+# Create a non-root user: claude --dangerously-skip-permissions refuses to run
+# as root, and most comet scripts also expect a normal HOME for git config etc.
+RUN useradd --create-home --shell /bin/bash agent && \
+    chown -R agent:agent /workspace
+USER agent
+ENV HOME=/home/agent
+# Validator scripts import scaffold.*; bake PYTHONPATH so it never crosses the
+# MSYS path-conversion boundary on the docker run command line.
+ENV PYTHONPATH=/workspace
+# Minimal git identity so comet commits don't fail.
+RUN git config --global user.email "agent@eval.local" && \
+    git config --global user.name "Comet Eval Agent" && \
+    git config --global init.defaultBranch master
+
+CMD ["bash"]

package/eval/local/tasks/comet-full-workflow/environment/test_wordcount.py

@@ -0,0 +1,29 @@
+"""Tests for wordcount CLI."""
+
+import subprocess
+import sys
+
+
+def test_count_words():
+    result = subprocess.run(
+        [sys.executable, "wordcount.py"],
+        input="hello world foo",
+        capture_output=True, text=True
+    )
+    assert "Words: 3" in result.stdout
+
+
+def test_count_lines():
+    result = subprocess.run(
+        [sys.executable, "wordcount.py", "--lines"],
+        input="hello\nworld\nfoo",
+        capture_output=True, text=True
+    )
+    assert "Words: 3" in result.stdout
+    assert "Lines: 3" in result.stdout
+
+
+if __name__ == "__main__":
+    test_count_words()
+    test_count_lines()
+    print("All tests passed")

package/eval/local/tasks/comet-full-workflow/environment/wordcount.py

@@ -0,0 +1,40 @@
+"""Simple word count CLI tool."""
+
+import argparse
+import sys
+
+
+def count_words(text: str) -> int:
+    """Count words in text."""
+    return len(text.split())
+
+
+def count_lines(text: str) -> int:
+    """Count lines in text."""
+    return len(text.splitlines())
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Count words and lines in text")
+    parser.add_argument("file", nargs="?", help="Input file (reads stdin if omitted)")
+    parser.add_argument("--lines", action="store_true", help="Also count lines")
+    parser.add_argument("--words", action="store_true", default=True, help="Count words (default)")
+
+    args = parser.parse_args()
+
+    if args.file:
+        with open(args.file) as f:
+            text = f.read()
+    else:
+        text = sys.stdin.read()
+
+    word_count = count_words(text)
+    print(f"Words: {word_count}")
+
+    if args.lines:
+        line_count = count_lines(text)
+        print(f"Lines: {line_count}")
+
+
+if __name__ == "__main__":
+    main()

package/eval/local/tasks/comet-full-workflow/instruction.md

@@ -0,0 +1,18 @@
+You are working on a Python project called "wordcount-cli" - a simple command-line word counting tool.
+
+Your task: Use the comet workflow to add a new feature: **sentence counting** (`--sentences` flag).
+
+The feature requirements:
+- Add a `--sentences` flag to the CLI
+- Count sentences by splitting on `.`, `!`, `?`
+- Output format: `Sentences: N`
+- Handle edge cases (empty input, no punctuation)
+
+Follow the comet workflow phases:
+1. **Open**: Create proposal, design outline, and task list
+2. **Design**: Deep design with brainstorming (create a Design Doc)
+3. **Build**: Plan implementation and write the code
+4. **Verify**: Run tests and verify the implementation
+5. **Archive**: Archive the completed change
+
+Start by detecting the current phase and following the comet workflow.

package/eval/local/tasks/comet-full-workflow/task.toml

@@ -0,0 +1,24 @@
+[metadata]
+name = "comet-full-workflow"
+description = "Test Claude's ability to follow the full comet workflow (open → design → build → verify → archive)"
+difficulty = "hard"
+category = "comet"
+tags = ["comet", "workflow", "openspec", "full-cycle"]
+default_treatments = [
+    "CONTROL",
+    "COMET_FULL_040_BETA",
+    "COMET_FULL_039",
+]
+
+[template]
+required = []
+
+[environment]
+description = "Python workspace with openspec CLI and comet scripts pre-installed. Claude must execute a full comet workflow for a sample feature request."
+dockerfile = "environment/Dockerfile"
+timeout_sec = 900
+
+[validation]
+test_scripts = "test_full_workflow.py"
+target_artifacts = []
+timeout = 120