@rpamis/comet 0.4.0-beta.1 → 0.4.0-beta.11

package/eval/local/tasks/comet-agent-memory-routing/environment/agent_system.py

@@ -0,0 +1,40 @@
+class StateBackend:
+    persistent = False
+
+
+class StoreBackend:
+    persistent = True
+
+
+class MemorySaver:
+    pass
+
+
+def resolve_backend(routes, path):
+    matches = [prefix for prefix in routes if path.startswith(prefix)]
+    if not matches:
+        return None
+    prefix = max(matches, key=len)
+    return routes[prefix]
+
+
+def create_agent_config():
+    routes = {
+        "/memory": StoreBackend(),
+        "/memory/preferences": StateBackend(),
+        "/session": StateBackend(),
+    }
+    return {
+        "backend_routes": routes,
+        "checkpointer": None,
+        "subagents": [
+            {
+                "name": "researcher",
+                "description": "Research project docs",
+            },
+            {
+                "name": "deployer",
+                "description": "Deploy services",
+            },
+        ],
+    }

package/eval/local/tasks/comet-agent-memory-routing/environment/test_agent_system.py

@@ -0,0 +1,29 @@
+import agent_system
+
+
+def test_preferences_route_to_persistent_backend():
+    config = agent_system.create_agent_config()
+    backend = agent_system.resolve_backend(
+        config["backend_routes"],
+        "/memory/preferences/theme",
+    )
+
+    assert backend is not None
+    assert backend.persistent is True
+
+
+def test_researcher_subagent_has_explicit_doc_skill():
+    config = agent_system.create_agent_config()
+    researcher = next(sub for sub in config["subagents"] if sub["name"] == "researcher")
+
+    assert "skills" in researcher
+    assert "project-docs" in researcher["skills"]
+
+
+def test_deployment_interrupt_has_checkpointer():
+    config = agent_system.create_agent_config()
+    deployer = next(sub for sub in config["subagents"] if sub["name"] == "deployer")
+
+    assert config["checkpointer"] is not None
+    assert "interrupt_on" in deployer
+    assert "deploy_production" in deployer["interrupt_on"]

package/eval/local/tasks/comet-agent-memory-routing/instruction.md

@@ -0,0 +1,20 @@
+You are working on a Python project called "doc-agent-system".
+
+Your task: Use the comet workflow to fix a multi-agent configuration.
+
+This task is adapted from `skills-benchmarks/oss-fix-da-memory`.
+
+## Bug reports
+
+1. User preferences disappear after restarting because the preferences path routes to ephemeral state.
+2. The researcher subagent cannot read project documentation because it does not receive explicit skills.
+3. Production deployments execute without human approval because interrupt handling is incomplete.
+
+## What to do
+
+1. Run `python -m pytest test_agent_system.py -q` to confirm the failures.
+2. Follow the comet workflow phases from open through archive.
+3. Keep the simple in-repo backend classes; do not add heavyweight external dependencies.
+4. Verify that preference routing, researcher skills, and approval checkpoint configuration all pass.
+
+When the workflow asks for confirmation, assume "yes, proceed with the recommended option".

package/eval/local/tasks/comet-agent-memory-routing/task.toml

@@ -0,0 +1,24 @@
+[metadata]
+name = "comet-agent-memory-routing"
+description = "Fix persistent preference routing, subagent skill access, and deployment approval checkpoints"
+difficulty = "hard"
+category = "comet"
+tags = ["comet", "workflow", "memory", "subagents", "approval"]
+default_treatments = [
+    "CONTROL",
+    "COMET_FULL_040_BETA",
+    "COMET_FULL_039",
+]
+
+[template]
+required = []
+
+[environment]
+description = "Python workspace with a multi-agent config whose memory, subagent, and approval settings are subtly wrong."
+dockerfile = "environment/Dockerfile"
+timeout_sec = 900
+
+[validation]
+test_scripts = "test_agent_memory_routing.py"
+target_artifacts = []
+timeout = 120

package/eval/local/tasks/comet-agent-memory-routing/validation/test_agent_memory_routing.py

@@ -0,0 +1,24 @@
+import subprocess
+import sys
+
+from comet_checks import WORKSPACE, _failed, _passed, run_comet_checks, write_results
+
+
+def _tail(result: subprocess.CompletedProcess) -> str:
+    return "\n".join(part.strip() for part in (result.stdout, result.stderr) if part and part.strip())[-600:]
+
+
+def check_agent_memory_routing() -> dict:
+    result = subprocess.run(
+        [sys.executable, "-m", "pytest", "test_agent_system.py", "-q"],
+        capture_output=True,
+        text=True,
+        cwd=str(WORKSPACE),
+    )
+    if result.returncode != 0:
+        return _failed("agent_memory_routing", _tail(result))
+    return _passed("agent_memory_routing", "preferences persist, researcher has docs, and deployment approval is checkpointed")
+
+
+if __name__ == "__main__":
+    write_results([*run_comet_checks(), check_agent_memory_routing()])

package/eval/local/tasks/comet-api-cache-ttl/environment/Dockerfile

@@ -0,0 +1,35 @@
+FROM python:3.12-slim
+
+WORKDIR /workspace
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    bash git curl jq ca-certificates \
+    nodejs npm && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN pip install --no-cache-dir pyyaml pytest
+
+RUN npm install -g @fission-ai/openspec@1.3.1
+
+ARG CLAUDE_CODE_VERSION=latest
+RUN npm install -g @anthropic-ai/claude-code@${CLAUDE_CODE_VERSION}
+
+COPY cache.py /workspace/cache.py
+COPY test_cache.py /workspace/test_cache.py
+
+RUN mkdir -p wordcount-cli && \
+    echo '# Cache Lib' > wordcount-cli/README.md
+RUN mkdir -p openspec/changes
+
+RUN useradd --create-home --shell /bin/bash agent && \
+    chown -R agent:agent /workspace
+USER agent
+ENV HOME=/home/agent
+# Validator scripts import scaffold.*; bake PYTHONPATH so it never crosses the
+# MSYS path-conversion boundary on the docker run command line.
+ENV PYTHONPATH=/workspace
+RUN git config --global user.email "agent@eval.local" && \
+    git config --global user.name "Comet Eval Agent" && \
+    git config --global init.defaultBranch master
+
+CMD ["bash"]

package/eval/local/tasks/comet-api-cache-ttl/environment/cache.py

@@ -0,0 +1,39 @@
+"""cache-lib: a simple in-memory key-value cache.
+
+Currently supports set/get/delete with no expiration. The task is to add
+optional TTL (time-to-live) support while keeping the existing API backward
+compatible.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Optional
+
+
+class Cache:
+    """A simple in-memory cache. Thread-safety is out of scope."""
+
+    def __init__(self) -> None:
+        self._store: dict[str, Any] = {}
+
+    def set(self, key: str, value: Any) -> None:
+        """Store a value. Overwrites any existing value (no expiry)."""
+        self._store[key] = value
+
+    def get(self, key: str, default: Any = None) -> Any:
+        """Retrieve a value, or `default` if the key is absent."""
+        return self._store.get(key, default)
+
+    def delete(self, key: str) -> bool:
+        """Remove a key. Returns True if it existed, False otherwise."""
+        return self._store.pop(key, None) is not None
+
+    def clear(self) -> None:
+        """Remove all entries."""
+        self._store.clear()
+
+    def __len__(self) -> int:
+        return len(self._store)
+
+    def __contains__(self, key: object) -> bool:
+        return key in self._store

package/eval/local/tasks/comet-api-cache-ttl/environment/test_cache.py

@@ -0,0 +1,81 @@
+"""Tests for cache-lib.
+
+The TTL tests currently FAIL because TTL is not yet implemented.
+The basic tests all pass and must continue to pass (backward compatibility).
+"""
+
+import time
+
+import pytest
+
+from cache import Cache
+
+
+class TestBasic:
+    def test_set_get(self):
+        c = Cache()
+        c.set("k", "v")
+        assert c.get("k") == "v"
+
+    def test_get_missing_default(self):
+        c = Cache()
+        assert c.get("nope") is None
+        assert c.get("nope", "fallback") == "fallback"
+
+    def test_overwrite(self):
+        c = Cache()
+        c.set("k", 1)
+        c.set("k", 2)
+        assert c.get("k") == 2
+
+    def test_delete(self):
+        c = Cache()
+        c.set("k", "v")
+        assert c.delete("k") is True
+        assert c.delete("k") is False
+        assert c.get("k") is None
+
+    def test_clear(self):
+        c = Cache()
+        c.set("a", 1)
+        c.set("b", 2)
+        c.clear()
+        assert len(c) == 0
+
+    def test_contains(self):
+        c = Cache()
+        c.set("k", "v")
+        assert "k" in c
+        assert "x" not in c
+
+
+class TestTTL:
+    def test_set_with_ttl_expires(self):
+        """A key set with a TTL should return the default after expiry."""
+        c = Cache()
+        c.set("temp", "value", ttl=0.1)
+        assert c.get("temp") == "value"  # immediately available
+        time.sleep(0.2)
+        assert c.get("temp") is None  # expired
+
+    def test_no_ttl_does_not_expire(self):
+        """A key set without TTL must not expire (backward compat)."""
+        c = Cache()
+        c.set("perm", "value")
+        time.sleep(0.1)
+        assert c.get("perm") == "value"
+
+    def test_ttl_overwrite_without_ttl(self):
+        """Overwriting a TTL key without TTL should remove the expiry."""
+        c = Cache()
+        c.set("k", 1, ttl=0.05)
+        c.set("k", 2)  # no ttl -> permanent
+        time.sleep(0.1)
+        assert c.get("k") == 2
+
+    def test_ttl_zero_means_no_expiry(self):
+        """ttl=0 (or omitted) means the key never expires."""
+        c = Cache()
+        c.set("k", "v", ttl=0)
+        time.sleep(0.05)
+        assert c.get("k") == "v"

package/eval/local/tasks/comet-api-cache-ttl/instruction.md

@@ -0,0 +1,33 @@
+You are working on a Python project called "cache-lib" - a simple in-memory key-value cache.
+
+Your task: Use the comet workflow to **add TTL (time-to-live) support** to the `Cache` class.
+
+## The requirement
+
+Add an optional `ttl` parameter to `set()`:
+
+```python
+def set(self, key: str, value: Any, ttl: Optional[float] = None) -> None:
+    """Store a value. If ttl is given (seconds), the key expires after that long.
+    ttl=0 or None means the key never expires."""
+```
+
+After a key's TTL elapses, `get()` must return the default (as if the key were absent). The existing behavior (no TTL) must not change.
+
+## Design decisions to make (use brainstorming)
+
+- **Lazy vs active eviction**: check expiry on `get()` (lazy, simple) vs a background sweep (active, complex). Lazy is recommended for this scope.
+- **Overwrite semantics**: setting a key again without TTL should clear any prior expiry.
+- **`__contains__` / `__len__`**: should expired keys count? (Expired keys should not appear present.)
+
+## Tests
+
+`test_cache.py` has a `TestTTL` class with 4 tests that currently FAIL. Plus 6 basic tests that must keep passing.
+
+## What to do
+
+1. Run tests to confirm: `python -m pytest test_cache.py -v`
+2. Follow the comet workflow (Open → Design → Build → Verify → Archive).
+3. Confirm all 10 tests pass after implementation.
+
+Start by detecting the current phase and following the comet workflow. When you reach a decision point that asks for confirmation, assume "yes, proceed with the recommended option".

package/eval/local/tasks/comet-api-cache-ttl/task.toml

@@ -0,0 +1,24 @@
+[metadata]
+name = "comet-api-cache-ttl"
+description = "Add optional TTL (time-to-live) expiration to an in-memory cache while keeping the API backward compatible"
+difficulty = "hard"
+category = "comet"
+tags = ["comet", "workflow", "api-design", "backward-compat"]
+default_treatments = [
+    "CONTROL",
+    "COMET_FULL_040_BETA",
+    "COMET_FULL_039",
+]
+
+[template]
+required = []
+
+[environment]
+description = "Python in-memory cache. Claude must add TTL support via the comet workflow."
+dockerfile = "environment/Dockerfile"
+timeout_sec = 900
+
+[validation]
+test_scripts = "test_api_cache_ttl.py"
+target_artifacts = []
+timeout = 120

package/eval/local/tasks/comet-api-cache-ttl/validation/test_api_cache_ttl.py

@@ -0,0 +1,58 @@
+"""Validation for the comet-api-cache-ttl task.
+
+Checks: comet workflow followed + set() accepts ttl + TTL tests pass + backward compat.
+"""
+
+import inspect
+import subprocess
+import sys
+
+from comet_checks import (
+    run_comet_checks,
+    write_results,
+    _passed,
+    _failed,
+    WORKSPACE,
+)
+
+
+def _subprocess_output(result: subprocess.CompletedProcess) -> str:
+    output = "\n".join(
+        part.strip()
+        for part in (result.stdout or "", result.stderr or "")
+        if part and part.strip()
+    )
+    return output[-600:]
+
+
+def check_ttl_implementation() -> dict:
+    """set() has a ttl param and all tests pass (TTL + backward compat)."""
+    # set() must accept a ttl parameter.
+    sig_result = subprocess.run(
+        [sys.executable, "-c",
+         "import inspect,cache; sig=inspect.signature(cache.Cache.set); print(list(sig.parameters))"],
+        capture_output=True, text=True, cwd=str(WORKSPACE),
+    )
+    if sig_result.returncode != 0:
+        return _failed("ttl_implementation", f"cannot inspect cache.Cache.set: {sig_result.stderr[:150]}")
+    if "ttl" not in sig_result.stdout:
+        return _failed("ttl_implementation", "Cache.set() has no 'ttl' parameter")
+
+    # All tests must pass (TTL + backward compat).
+    test_result = subprocess.run(
+        [sys.executable, "-m", "pytest", "test_cache.py", "-q"],
+        capture_output=True, text=True, cwd=str(WORKSPACE),
+    )
+    if test_result.returncode != 0:
+        return _failed("ttl_implementation", f"tests failing:\n{_subprocess_output(test_result)}")
+    return _passed("ttl_implementation", "set(ttl=...) works, all 10 tests pass")
+
+
+def run_all() -> list[dict]:
+    results = run_comet_checks()
+    results.append(check_ttl_implementation())
+    return results
+
+
+if __name__ == "__main__":
+    write_results(run_all())

package/eval/local/tasks/comet-classic-layout-lifecycle/environment/.include-current-comet-cli

@@ -0,0 +1 @@
+include

package/eval/local/tasks/comet-classic-layout-lifecycle/environment/Dockerfile

@@ -0,0 +1,45 @@
+FROM node:22-bookworm-slim
+
+WORKDIR /workspace
+
+# Runtime tools for current Classic plus frozen 0.3.9/0.4.0 treatments.
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    bash git curl jq ca-certificates \
+    python3 python3-pip python-is-python3 && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN pip install --break-system-packages --no-cache-dir pyyaml
+
+# Match Comet's supported OpenSpec floor. The current-checkout CLI and its
+# dependencies remain separate so the benchmark does not depend on a published
+# Comet package.
+RUN npm install -g @fission-ai/openspec@1.5.0
+
+COPY current-comet-package.json /opt/comet-cli/package.json
+RUN npm install --prefix /opt/comet-cli --omit=dev --ignore-scripts
+
+# Claude Code CLI (the agent runtime; version overridable via BENCH_CC_VERSION).
+ARG CLAUDE_CODE_VERSION=latest
+RUN npm install -g @anthropic-ai/claude-code@${CLAUDE_CODE_VERSION}
+
+# Existing wordcount code (extended during the task).
+COPY wordcount.py /workspace/wordcount.py
+COPY test_wordcount.py /workspace/test_wordcount.py
+COPY current-comet.sh /usr/local/bin/comet
+RUN chmod 0755 /usr/local/bin/comet
+
+# Do not pre-create openspec/. Each treatment owns its selected artifact
+# layout; OpenSpec creates the configured root on first use.
+
+# Create a non-root user: claude --dangerously-skip-permissions refuses to run
+# as root, and Comet expects a normal HOME for Git/config discovery.
+RUN useradd --create-home --shell /bin/bash agent && \
+    chown -R agent:agent /workspace
+USER agent
+ENV HOME=/home/agent
+ENV PYTHONPATH=/workspace
+RUN git config --global user.email "agent@eval.local" && \
+    git config --global user.name "Comet Eval Agent" && \
+    git config --global init.defaultBranch master
+
+CMD ["bash"]

package/eval/local/tasks/comet-classic-layout-lifecycle/environment/current-comet-package.json

@@ -0,0 +1,18 @@
+{
+  "private": true,
+  "type": "module",
+  "dependencies": {
+    "@fission-ai/openspec": "^1.5.0",
+    "@inquirer/ansi": "^2.0.5",
+    "@inquirer/core": "^11.1.10",
+    "@inquirer/figures": "^2.0.5",
+    "@inquirer/prompts": "^8.4.3",
+    "@inquirer/type": "^4.0.5",
+    "commander": "^14.0.0",
+    "dompurify": "^3.4.12",
+    "highlight.js": "^11.11.1",
+    "marked": "^18.0.4",
+    "mermaid": "11.15.0",
+    "yaml": "^2.9.0"
+  }
+}

package/eval/local/tasks/comet-classic-layout-lifecycle/environment/current-comet.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+snapshot="/workspace/_eval_current_comet"
+if [[ ! -f "$snapshot/bin/comet.js" || ! -d "$snapshot/dist" || ! -f "$snapshot/assets/manifest.json" ]]; then
+    echo "Current Comet CLI snapshot is unavailable" >&2
+    exit 2
+fi
+
+runtime="$(mktemp -d)"
+trap 'rm -rf "$runtime"' EXIT
+cp -a "$snapshot/." "$runtime/"
+ln -s /opt/comet-cli/node_modules "$runtime/node_modules"
+node "$runtime/bin/comet.js" "$@"

package/eval/local/tasks/comet-classic-layout-lifecycle/environment/test_wordcount.py

@@ -0,0 +1,29 @@
+"""Tests for wordcount CLI."""
+
+import subprocess
+import sys
+
+
+def test_count_words():
+    result = subprocess.run(
+        [sys.executable, "wordcount.py"],
+        input="hello world foo",
+        capture_output=True, text=True
+    )
+    assert "Words: 3" in result.stdout
+
+
+def test_count_lines():
+    result = subprocess.run(
+        [sys.executable, "wordcount.py", "--lines"],
+        input="hello\nworld\nfoo",
+        capture_output=True, text=True
+    )
+    assert "Words: 3" in result.stdout
+    assert "Lines: 3" in result.stdout
+
+
+if __name__ == "__main__":
+    test_count_words()
+    test_count_lines()
+    print("All tests passed")

package/eval/local/tasks/comet-classic-layout-lifecycle/environment/wordcount.py

@@ -0,0 +1,40 @@
+"""Simple word count CLI tool."""
+
+import argparse
+import sys
+
+
+def count_words(text: str) -> int:
+    """Count words in text."""
+    return len(text.split())
+
+
+def count_lines(text: str) -> int:
+    """Count lines in text."""
+    return len(text.splitlines())
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Count words and lines in text")
+    parser.add_argument("file", nargs="?", help="Input file (reads stdin if omitted)")
+    parser.add_argument("--lines", action="store_true", help="Also count lines")
+    parser.add_argument("--words", action="store_true", default=True, help="Count words (default)")
+
+    args = parser.parse_args()
+
+    if args.file:
+        with open(args.file) as f:
+            text = f.read()
+    else:
+        text = sys.stdin.read()
+
+    word_count = count_words(text)
+    print(f"Words: {word_count}")
+
+    if args.lines:
+        line_count = count_lines(text)
+        print(f"Lines: {line_count}")
+
+
+if __name__ == "__main__":
+    main()

package/eval/local/tasks/comet-classic-layout-lifecycle/instruction.md

@@ -0,0 +1,18 @@
+You are working on a Python project called "wordcount-cli" - a simple command-line word counting tool.
+
+Your task: Use the comet workflow to add a new feature: **sentence counting** (`--sentences` flag).
+
+The feature requirements:
+- Add a `--sentences` flag to the CLI
+- Count sentences by splitting on `.`, `!`, `?`
+- Output format: `Sentences: N`
+- Handle edge cases (empty input, no punctuation)
+
+Follow the comet workflow phases:
+1. **Open**: Create proposal, design outline, and task list
+2. **Design**: Deep design with brainstorming (create a Design Doc)
+3. **Build**: Plan implementation and write the code
+4. **Verify**: Run tests and verify the implementation
+5. **Archive**: Archive the completed change
+
+Start by detecting the current phase and following the comet workflow.

package/eval/local/tasks/comet-classic-layout-lifecycle/task.toml

@@ -0,0 +1,23 @@
+[metadata]
+name = "comet-classic-layout-lifecycle"
+description = "Exercise the current Comet Classic lifecycle against both docs and legacy artifact layouts"
+difficulty = "hard"
+category = "comet"
+tags = ["comet", "classic", "workflow", "openspec", "layout", "full-cycle"]
+default_treatments = [
+    "COMET_CLASSIC_DOCS_LAYOUT",
+    "COMET_CLASSIC_LEGACY_LAYOUT",
+]
+
+[template]
+required = []
+
+[environment]
+description = "Current-checkout Comet CLI with OpenSpec 1.5.0. Claude must complete and archive a full Classic workflow in the treatment-selected layout."
+dockerfile = "environment/Dockerfile"
+timeout_sec = 900
+
+[validation]
+test_scripts = "test_classic_layout_lifecycle.py"
+target_artifacts = []
+timeout = 120