@rpamis/comet 0.4.0-beta.1 → 0.4.0-beta.11

package/eval/local/tasks/comet-full-workflow/environment/test_wordcount.py

@@ -0,0 +1,29 @@
+"""Tests for wordcount CLI."""
+
+import subprocess
+import sys
+
+
+def test_count_words():
+    result = subprocess.run(
+        [sys.executable, "wordcount.py"],
+        input="hello world foo",
+        capture_output=True, text=True
+    )
+    assert "Words: 3" in result.stdout
+
+
+def test_count_lines():
+    result = subprocess.run(
+        [sys.executable, "wordcount.py", "--lines"],
+        input="hello\nworld\nfoo",
+        capture_output=True, text=True
+    )
+    assert "Words: 3" in result.stdout
+    assert "Lines: 3" in result.stdout
+
+
+if __name__ == "__main__":
+    test_count_words()
+    test_count_lines()
+    print("All tests passed")

package/eval/local/tasks/comet-full-workflow/environment/wordcount.py

@@ -0,0 +1,40 @@
+"""Simple word count CLI tool."""
+
+import argparse
+import sys
+
+
+def count_words(text: str) -> int:
+    """Count words in text."""
+    return len(text.split())
+
+
+def count_lines(text: str) -> int:
+    """Count lines in text."""
+    return len(text.splitlines())
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Count words and lines in text")
+    parser.add_argument("file", nargs="?", help="Input file (reads stdin if omitted)")
+    parser.add_argument("--lines", action="store_true", help="Also count lines")
+    parser.add_argument("--words", action="store_true", default=True, help="Count words (default)")
+
+    args = parser.parse_args()
+
+    if args.file:
+        with open(args.file) as f:
+            text = f.read()
+    else:
+        text = sys.stdin.read()
+
+    word_count = count_words(text)
+    print(f"Words: {word_count}")
+
+    if args.lines:
+        line_count = count_lines(text)
+        print(f"Lines: {line_count}")
+
+
+if __name__ == "__main__":
+    main()

package/eval/local/tasks/comet-full-workflow/instruction.md

@@ -0,0 +1,18 @@
+You are working on a Python project called "wordcount-cli" - a simple command-line word counting tool.
+
+Your task: Use the comet workflow to add a new feature: **sentence counting** (`--sentences` flag).
+
+The feature requirements:
+- Add a `--sentences` flag to the CLI
+- Count sentences by splitting on `.`, `!`, `?`
+- Output format: `Sentences: N`
+- Handle edge cases (empty input, no punctuation)
+
+Follow the comet workflow phases:
+1. **Open**: Create proposal, design outline, and task list
+2. **Design**: Deep design with brainstorming (create a Design Doc)
+3. **Build**: Plan implementation and write the code
+4. **Verify**: Run tests and verify the implementation
+5. **Archive**: Archive the completed change
+
+Start by detecting the current phase and following the comet workflow.

package/eval/local/tasks/comet-full-workflow/task.toml

@@ -0,0 +1,24 @@
+[metadata]
+name = "comet-full-workflow"
+description = "Test Claude's ability to follow the full comet workflow (open → design → build → verify → archive)"
+difficulty = "hard"
+category = "comet"
+tags = ["comet", "workflow", "openspec", "full-cycle"]
+default_treatments = [
+    "CONTROL",
+    "COMET_FULL_040_BETA",
+    "COMET_FULL_039",
+]
+
+[template]
+required = []
+
+[environment]
+description = "Python workspace with openspec CLI and comet scripts pre-installed. Claude must execute a full comet workflow for a sample feature request."
+dockerfile = "environment/Dockerfile"
+timeout_sec = 900
+
+[validation]
+test_scripts = "test_full_workflow.py"
+target_artifacts = []
+timeout = 120

package/eval/local/tasks/comet-full-workflow/validation/test_full_workflow.py

@@ -0,0 +1,171 @@
+"""Validation script for comet full workflow task.
+
+Runs inside Docker. Checks that Claude followed the comet workflow
+and implemented the sentence counting feature correctly.
+"""
+
+import json
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+WORKSPACE = Path("/workspace")
+RESULTS_FILE = os.environ.get("BENCH_TEST_RESULTS", "_test_results.json")
+
+
+def passed(name: str):
+    return {"check": name, "status": "passed"}
+
+
+def failed(name: str, reason: str):
+    return {"check": name, "status": "failed", "reason": reason}
+
+
+def check_openspec_artifacts():
+    """Check that OpenSpec artifacts were created (proposal, design, tasks).
+
+    Looks for a change directory either directly under openspec/changes/ (active)
+    or under openspec/changes/archive/ (archived). Accepts the first change dir
+    that actually contains proposal.md + tasks.md.
+    """
+    changes_dir = WORKSPACE / "openspec" / "changes"
+    if not changes_dir.exists():
+        return failed("openspec_artifacts", "openspec/changes/ directory not found")
+
+    # Candidate change dirs: direct children of openspec/changes/ (excluding the
+    # archive/ container itself) plus children of openspec/changes/archive/.
+    candidates = []
+    for d in changes_dir.iterdir():
+        if not d.is_dir():
+            continue
+        if d.name == "archive":
+            for sub in d.iterdir():
+                if sub.is_dir():
+                    candidates.append(sub)
+        else:
+            candidates.append(d)
+
+    if not candidates:
+        return failed("openspec_artifacts", "No change directories found in openspec/changes/")
+
+    for change_dir in candidates:
+        if (change_dir / "proposal.md").exists() and (change_dir / "tasks.md").exists():
+            return passed("openspec_artifacts")
+
+    first = candidates[0]
+    return failed("openspec_artifacts", f"proposal.md/tasks.md not found together in any change dir (checked {len(candidates)}; e.g. {first})")
+
+
+def check_sentence_feature():
+    """Check that the sentence counting feature was implemented."""
+    wordcount = WORKSPACE / "wordcount.py"
+    if not wordcount.exists():
+        return failed("sentence_feature", "wordcount.py not found")
+
+    content = wordcount.read_text()
+
+    # Check for --sentences flag
+    if "--sentences" not in content:
+        return failed("sentence_feature", "--sentences flag not found in wordcount.py")
+
+    # Check that it actually works
+    try:
+        result = subprocess.run(
+            [sys.executable, str(wordcount), "--sentences"],
+            input="Hello world. How are you? Fine!",
+            capture_output=True, text=True, timeout=10
+        )
+        if result.returncode != 0:
+            return failed("sentence_feature", f"wordcount.py --sentences failed: {result.stderr}")
+
+        # Should count 3 sentences
+        if "3" not in result.stdout:
+            return failed("sentence_feature", f"Expected 3 sentences, got: {result.stdout}")
+
+    except Exception as e:
+        return failed("sentence_feature", f"Error running wordcount.py: {e}")
+
+    return passed("sentence_feature")
+
+
+def check_tests_exist():
+    """Check that tests were written for the new feature."""
+    # Look for test files
+    test_files = list(WORKSPACE.glob("test_*.py"))
+    if not test_files:
+        return failed("tests_exist", "No test files found")
+
+    # Check if any test file has sentence-related tests
+    for tf in test_files:
+        content = tf.read_text()
+        if "sentence" in content.lower() or "sentences" in content.lower():
+            return passed("tests_exist")
+
+    return failed("tests_exist", "No sentence-related tests found")
+
+
+def check_comet_state():
+    """Check that .comet.yaml state file was created."""
+    # Look for .comet.yaml anywhere in the workspace
+    comet_files = list(WORKSPACE.rglob(".comet.yaml"))
+    if not comet_files:
+        # Also check for openspec status files
+        status_files = list(WORKSPACE.rglob("*.yaml"))
+        comet_like = [f for f in status_files if "comet" in f.name.lower()]
+        if not comet_like:
+            return failed("comet_state", "No .comet.yaml or comet state files found")
+        return passed("comet_state")
+
+    return passed("comet_state")
+
+
+def check_workflow_phases():
+    """Check that multiple workflow phases were evident."""
+    # Look for evidence of multiple phases in any markdown files
+    md_files = list(WORKSPACE.rglob("*.md"))
+    all_content = " ".join(f.read_text() for f in md_files if f.exists())
+
+    phases_found = []
+    phase_keywords = {
+        "open": ["proposal", "design outline", "task list"],
+        "design": ["design doc", "brainstorming", "technical design"],
+        "build": ["implementation", "plan", "code"],
+        "verify": ["verification", "test results", "passed"],
+    }
+
+    for phase, keywords in phase_keywords.items():
+        if any(kw in all_content.lower() for kw in keywords):
+            phases_found.append(phase)
+
+    if len(phases_found) >= 2:
+        return passed("workflow_phases")
+
+    return failed("workflow_phases", f"Only found evidence of phases: {phases_found}")
+
+
+def main():
+    results = []
+
+    results.append(check_openspec_artifacts())
+    results.append(check_sentence_feature())
+    results.append(check_tests_exist())
+    results.append(check_comet_state())
+    results.append(check_workflow_phases())
+
+    passed_list = [r["check"] for r in results if r["status"] == "passed"]
+    failed_list = [f'{r["check"]}: {r.get("reason", "")}' for r in results if r["status"] == "failed"]
+
+    output = {"passed": passed_list, "failed": failed_list}
+
+    # Write results file
+    (WORKSPACE / RESULTS_FILE).write_text(json.dumps(output, indent=2))
+
+    # Also print for stdout capture
+    print(json.dumps(output))
+
+    return 0 if not failed_list else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/eval/local/tasks/comet-graph-execution-review/environment/Dockerfile

@@ -0,0 +1,25 @@
+FROM python:3.12-slim
+
+WORKDIR /workspace
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    bash git curl jq ca-certificates nodejs npm && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN pip install --no-cache-dir pyyaml pytest
+
+RUN npm install -g @fission-ai/openspec@1.3.1 &&  npm install -g @anthropic-ai/claude-code@latest
+
+COPY . /workspace/
+RUN mkdir -p openspec/changes
+
+RUN useradd --create-home --shell /bin/bash agent && \
+    chown -R agent:agent /workspace
+USER agent
+ENV HOME=/home/agent
+ENV PYTHONPATH=/workspace
+RUN git config --global user.email "agent@eval.local" && \
+    git config --global user.name "Comet Eval Agent" && \
+    git config --global init.defaultBranch master
+
+CMD ["bash"]

package/eval/local/tasks/comet-graph-execution-review/environment/pipeline.py

@@ -0,0 +1,19 @@
+THREADS = {}
+
+
+def _process(task: str) -> str:
+    return f"processed:{task}"
+
+
+def run_pipeline(tasks, thread_id="default", require_review=True):
+    selected = list(tasks)[:1]
+    results = [_process(task) for task in selected]
+    summary = f"finalized {len(results)} task(s)"
+    THREADS[thread_id] = {"tasks": selected, "results": results, "status": "finalized"}
+    return {"status": "finalized", "results": results, "summary": summary}
+
+
+def resume_after_review(thread_id="default"):
+    state = THREADS.get(thread_id, {"tasks": [], "results": []})
+    THREADS[thread_id] = {"tasks": [], "results": [], "status": "approved"}
+    return {"status": "approved", "results": [], "summary": "approved 0 task(s)"}

package/eval/local/tasks/comet-graph-execution-review/environment/test_pipeline.py

@@ -0,0 +1,53 @@
+import pipeline
+
+
+def setup_function():
+    pipeline.THREADS.clear()
+
+
+def test_fan_out_processes_every_task_without_review():
+    result = pipeline.run_pipeline(
+        ["alpha", "beta", "gamma"],
+        thread_id="fanout",
+        require_review=False,
+    )
+
+    assert result["status"] == "finalized"
+    assert len(result["results"]) == 3
+    assert "alpha" in " ".join(result["results"])
+    assert "beta" in " ".join(result["results"])
+    assert "gamma" in " ".join(result["results"])
+
+
+def test_multi_task_run_interrupts_for_review_before_summary():
+    result = pipeline.run_pipeline(["draft", "verify"], thread_id="review")
+
+    assert "__interrupt__" in result
+    assert result["status"] == "waiting_for_review"
+    assert not result.get("summary")
+
+
+def test_resume_after_review_continues_same_thread():
+    first = pipeline.run_pipeline(["extract", "transform", "load"], thread_id="resume")
+
+    assert "__interrupt__" in first
+
+    resumed = pipeline.resume_after_review(thread_id="resume")
+
+    assert resumed["status"] == "approved"
+    assert len(resumed["results"]) == 3
+    assert "3" in resumed["summary"]
+    assert "extract" in " ".join(resumed["results"])
+
+
+def test_thread_state_is_isolated_by_thread_id():
+    pipeline.run_pipeline(["a1", "a2"], thread_id="thread-a")
+    pipeline.run_pipeline(["b1", "b2", "b3"], thread_id="thread-b")
+
+    resumed_a = pipeline.resume_after_review(thread_id="thread-a")
+    resumed_b = pipeline.resume_after_review(thread_id="thread-b")
+
+    assert len(resumed_a["results"]) == 2
+    assert len(resumed_b["results"]) == 3
+    assert "a1" in " ".join(resumed_a["results"])
+    assert "b1" in " ".join(resumed_b["results"])

package/eval/local/tasks/comet-graph-execution-review/instruction.md

@@ -0,0 +1,21 @@
+You are working on a Python project called "review-pipeline".
+
+Your task: Use the comet workflow to fix a graph-style data processing pipeline.
+
+This task is adapted from `skills-benchmarks/oss-fix-lg-execution`.
+
+## Bugs
+
+1. `run_pipeline()` only processes part of the submitted task list.
+2. Multi-task runs finalize immediately instead of pausing for human review.
+3. `resume_after_review()` restarts or loses state instead of continuing the reviewed thread.
+4. Thread state must be isolated by `thread_id`.
+
+## What to do
+
+1. Run `python -m pytest test_pipeline.py -q` to confirm the failures.
+2. Follow the comet workflow phases from open through archive.
+3. Fix the pipeline while keeping the public function names unchanged.
+4. Verify that fan-out, review interruption, resume, and thread isolation all pass.
+
+When the workflow asks for confirmation, assume "yes, proceed with the recommended option".

package/eval/local/tasks/comet-graph-execution-review/task.toml

@@ -0,0 +1,24 @@
+[metadata]
+name = "comet-graph-execution-review"
+description = "Fix parallel fan-out, human review, and resume behavior in a graph-style processing pipeline"
+difficulty = "hard"
+category = "comet"
+tags = ["comet", "workflow", "graph", "review", "resume"]
+default_treatments = [
+    "CONTROL",
+    "COMET_FULL_040_BETA",
+    "COMET_FULL_039",
+]
+
+[template]
+required = []
+
+[environment]
+description = "Python workspace with a graph-style pipeline that drops fan-out results and finalizes before review."
+dockerfile = "environment/Dockerfile"
+timeout_sec = 900
+
+[validation]
+test_scripts = "test_graph_execution_review.py"
+target_artifacts = []
+timeout = 120

package/eval/local/tasks/comet-graph-execution-review/validation/test_graph_execution_review.py

@@ -0,0 +1,24 @@
+import subprocess
+import sys
+
+from comet_checks import WORKSPACE, _failed, _passed, run_comet_checks, write_results
+
+
+def _tail(result: subprocess.CompletedProcess) -> str:
+    return "\n".join(part.strip() for part in (result.stdout, result.stderr) if part and part.strip())[-600:]
+
+
+def check_graph_execution_review() -> dict:
+    result = subprocess.run(
+        [sys.executable, "-m", "pytest", "test_pipeline.py", "-q"],
+        capture_output=True,
+        text=True,
+        cwd=str(WORKSPACE),
+    )
+    if result.returncode != 0:
+        return _failed("graph_execution_review", _tail(result))
+    return _passed("graph_execution_review", "fan-out, review interruption, resume, and thread isolation pass")
+
+
+if __name__ == "__main__":
+    write_results([*run_comet_checks(), check_graph_execution_review()])

package/eval/local/tasks/comet-human-approval-flow/environment/Dockerfile

@@ -0,0 +1,15 @@
+FROM python:3.12-slim
+
+WORKDIR /workspace
+
+RUN apt-get update && apt-get install -y --no-install-recommends bash git curl jq ca-certificates nodejs npm && rm -rf /var/lib/apt/lists/*
+RUN pip install --no-cache-dir pyyaml pytest
+RUN npm install -g @fission-ai/openspec@1.3.1 && npm install -g @anthropic-ai/claude-code@latest
+COPY . /workspace/
+RUN mkdir -p openspec/changes
+RUN useradd --create-home --shell /bin/bash agent && chown -R agent:agent /workspace
+USER agent
+ENV HOME=/home/agent
+ENV PYTHONPATH=/workspace
+RUN git config --global user.email "agent@eval.local" && git config --global user.name "Comet Eval Agent" && git config --global init.defaultBranch master
+CMD ["bash"]

package/eval/local/tasks/comet-human-approval-flow/environment/approvals.py

@@ -0,0 +1,17 @@
+from __future__ import annotations
+
+
+DOCUMENTS = {"a": "Alpha", "b": "Beta"}
+
+
+def read_document(doc_id: str) -> str | None:
+    return DOCUMENTS.get(doc_id)
+
+
+def delete_document(doc_id: str) -> dict:
+    DOCUMENTS.pop(doc_id, None)
+    return {"status": "deleted", "doc_id": doc_id}
+
+
+def approve(approval_id: str) -> dict:
+    return {"status": "missing", "approval_id": approval_id}

package/eval/local/tasks/comet-human-approval-flow/environment/test_approvals.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+import approvals
+
+
+def setup_function():
+    approvals.DOCUMENTS.clear()
+    approvals.DOCUMENTS.update({"a": "Alpha", "b": "Beta"})
+    if hasattr(approvals, "PENDING_APPROVALS"):
+        approvals.PENDING_APPROVALS.clear()
+
+
+def test_delete_returns_pending_approval_without_deleting():
+    result = approvals.delete_document("a")
+    assert result["status"] == "pending_approval"
+    assert result["doc_id"] == "a"
+    assert "approval_id" in result
+    assert approvals.read_document("a") == "Alpha"
+
+
+def test_approve_executes_pending_delete_once():
+    pending = approvals.delete_document("a")
+    approved = approvals.approve(pending["approval_id"])
+    assert approved == {"status": "deleted", "doc_id": "a"}
+    assert approvals.read_document("a") is None
+    assert approvals.approve(pending["approval_id"])["status"] == "missing"
+
+
+def test_read_stays_immediate():
+    assert approvals.read_document("b") == "Beta"

package/eval/local/tasks/comet-human-approval-flow/instruction.md

@@ -0,0 +1,15 @@
+You are working on a Python project called "doc-ops".
+
+Your task: Use the comet workflow to add a human approval step before dangerous document operations execute.
+
+This task is adapted from `skills-benchmarks/oss-fix-lc-hitl`.
+
+## Requirements
+
+- `delete_document(doc_id)` should return a pending approval record, not delete immediately.
+- `approve(approval_id)` should execute the pending delete exactly once.
+- Safe operations such as `read_document(doc_id)` should remain immediate.
+
+Run `python -m pytest test_approvals.py -q`, follow the comet workflow, and archive the completed change.
+
+When the workflow asks for confirmation, assume "yes, proceed with the recommended option".

package/eval/local/tasks/comet-human-approval-flow/task.toml

@@ -0,0 +1,20 @@
+[metadata]
+name = "comet-human-approval-flow"
+description = "Add a pending approval flow before dangerous document operations execute"
+difficulty = "hard"
+category = "comet"
+tags = ["comet", "workflow", "approval", "hitl"]
+default_treatments = ["CONTROL", "COMET_FULL_040_BETA", "COMET_FULL_039"]
+
+[template]
+required = []
+
+[environment]
+description = "Python document operations module missing human approval for dangerous actions."
+dockerfile = "environment/Dockerfile"
+timeout_sec = 900
+
+[validation]
+test_scripts = "test_human_approval_flow.py"
+target_artifacts = []
+timeout = 120

package/eval/local/tasks/comet-human-approval-flow/validation/test_human_approval_flow.py

@@ -0,0 +1,20 @@
+import subprocess
+import sys
+
+from comet_checks import run_comet_checks, write_results, _passed, _failed, WORKSPACE
+
+
+def check_approval_flow() -> dict:
+    result = subprocess.run(
+        [sys.executable, "-m", "pytest", "test_approvals.py", "-q"],
+        capture_output=True,
+        text=True,
+        cwd=str(WORKSPACE),
+    )
+    if result.returncode != 0:
+        return _failed("human_approval_flow", (result.stdout + result.stderr)[-600:])
+    return _passed("human_approval_flow", "dangerous deletes require pending approval")
+
+
+if __name__ == "__main__":
+    write_results([*run_comet_checks(), check_approval_flow()])

package/eval/local/tasks/comet-layered-streaming-fix/environment/Dockerfile

@@ -0,0 +1,17 @@
+FROM python:3.12-slim
+
+WORKDIR /workspace
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    bash git curl jq ca-certificates nodejs npm && \
+    rm -rf /var/lib/apt/lists/*
+RUN pip install --no-cache-dir pyyaml pytest
+RUN npm install -g @fission-ai/openspec@1.3.1 &&  npm install -g @anthropic-ai/claude-code@latest
+COPY . /workspace/
+RUN mkdir -p openspec/changes
+RUN useradd --create-home --shell /bin/bash agent && chown -R agent:agent /workspace
+USER agent
+ENV HOME=/home/agent
+ENV PYTHONPATH=/workspace
+RUN git config --global user.email "agent@eval.local" && git config --global user.name "Comet Eval Agent" && git config --global init.defaultBranch master
+CMD ["bash"]

package/eval/local/tasks/comet-layered-streaming-fix/environment/chat_app.py

@@ -0,0 +1,19 @@
+from __future__ import annotations
+
+
+def stream_response(tokens: list[str]):
+    buffer = ""
+    for token in tokens:
+        buffer += token
+    yield buffer
+
+
+def lookup_order(order_id: str) -> str:
+    return {"A100": "shipped", "B200": "processing"}.get(order_id, "unknown")
+
+
+def handle_tool_call(tool_name: str, argument: str) -> str:
+    if tool_name == "lookup_order":
+        lookup_order(argument)
+        return "Tool call completed."
+    return "Unsupported tool."

package/eval/local/tasks/comet-layered-streaming-fix/environment/test_chat_app.py

@@ -0,0 +1,15 @@
+from __future__ import annotations
+
+from chat_app import handle_tool_call, stream_response
+
+
+def test_stream_response_yields_each_token():
+    assert list(stream_response(["Hel", "lo", "!"])) == ["Hel", "lo", "!"]
+
+
+def test_tool_result_is_included_in_response():
+    assert handle_tool_call("lookup_order", "A100") == "Order A100 is shipped."
+
+
+def test_unknown_order_is_reported():
+    assert handle_tool_call("lookup_order", "Z999") == "Order Z999 is unknown."

package/eval/local/tasks/comet-layered-streaming-fix/instruction.md

@@ -0,0 +1,14 @@
+You are working on a Python project called "chat-stream".
+
+Your task: Use the comet workflow to fix two related chat application bugs.
+
+This task is adapted from `skills-benchmarks/oss-fix-lc-streaming`.
+
+## Bugs
+
+1. `stream_response()` buffers all tokens and emits one combined string instead of streaming token chunks.
+2. `handle_tool_call()` ignores the tool result and returns a placeholder response.
+
+Run `python -m pytest test_chat_app.py -q` first, then follow the comet workflow through archive. Keep the public function names unchanged.
+
+When the workflow asks for confirmation, assume "yes, proceed with the recommended option".