@rpamis/comet 0.4.0-beta.3 → 0.4.0-beta.5

package/eval/local/tasks/comet-agent-memory-routing/environment/agent_system.py

@@ -0,0 +1,40 @@
+class StateBackend:
+    persistent = False
+
+
+class StoreBackend:
+    persistent = True
+
+
+class MemorySaver:
+    pass
+
+
+def resolve_backend(routes, path):
+    matches = [prefix for prefix in routes if path.startswith(prefix)]
+    if not matches:
+        return None
+    prefix = max(matches, key=len)
+    return routes[prefix]
+
+
+def create_agent_config():
+    routes = {
+        "/memory": StoreBackend(),
+        "/memory/preferences": StateBackend(),
+        "/session": StateBackend(),
+    }
+    return {
+        "backend_routes": routes,
+        "checkpointer": None,
+        "subagents": [
+            {
+                "name": "researcher",
+                "description": "Research project docs",
+            },
+            {
+                "name": "deployer",
+                "description": "Deploy services",
+            },
+        ],
+    }

package/eval/local/tasks/comet-agent-memory-routing/environment/test_agent_system.py

@@ -0,0 +1,29 @@
+import agent_system
+
+
+def test_preferences_route_to_persistent_backend():
+    config = agent_system.create_agent_config()
+    backend = agent_system.resolve_backend(
+        config["backend_routes"],
+        "/memory/preferences/theme",
+    )
+
+    assert backend is not None
+    assert backend.persistent is True
+
+
+def test_researcher_subagent_has_explicit_doc_skill():
+    config = agent_system.create_agent_config()
+    researcher = next(sub for sub in config["subagents"] if sub["name"] == "researcher")
+
+    assert "skills" in researcher
+    assert "project-docs" in researcher["skills"]
+
+
+def test_deployment_interrupt_has_checkpointer():
+    config = agent_system.create_agent_config()
+    deployer = next(sub for sub in config["subagents"] if sub["name"] == "deployer")
+
+    assert config["checkpointer"] is not None
+    assert "interrupt_on" in deployer
+    assert "deploy_production" in deployer["interrupt_on"]

package/eval/local/tasks/comet-agent-memory-routing/instruction.md

@@ -0,0 +1,20 @@
+You are working on a Python project called "doc-agent-system".
+
+Your task: Use the comet workflow to fix a multi-agent configuration.
+
+This task is adapted from `skills-benchmarks/oss-fix-da-memory`.
+
+## Bug reports
+
+1. User preferences disappear after restarting because the preferences path routes to ephemeral state.
+2. The researcher subagent cannot read project documentation because it does not receive explicit skills.
+3. Production deployments execute without human approval because interrupt handling is incomplete.
+
+## What to do
+
+1. Run `python -m pytest test_agent_system.py -q` to confirm the failures.
+2. Follow the comet workflow phases from open through archive.
+3. Keep the simple in-repo backend classes; do not add heavyweight external dependencies.
+4. Verify that preference routing, researcher skills, and approval checkpoint configuration all pass.
+
+When the workflow asks for confirmation, assume "yes, proceed with the recommended option".

package/eval/local/tasks/comet-agent-memory-routing/task.toml

@@ -0,0 +1,24 @@
+[metadata]
+name = "comet-agent-memory-routing"
+description = "Fix persistent preference routing, subagent skill access, and deployment approval checkpoints"
+difficulty = "hard"
+category = "comet"
+tags = ["comet", "workflow", "memory", "subagents", "approval"]
+default_treatments = [
+    "CONTROL",
+    "COMET_FULL_040_BETA",
+    "COMET_FULL_039",
+]
+
+[template]
+required = []
+
+[environment]
+description = "Python workspace with a multi-agent config whose memory, subagent, and approval settings are subtly wrong."
+dockerfile = "environment/Dockerfile"
+timeout_sec = 900
+
+[validation]
+test_scripts = "test_agent_memory_routing.py"
+target_artifacts = []
+timeout = 120

package/eval/local/tasks/comet-agent-memory-routing/validation/test_agent_memory_routing.py

@@ -0,0 +1,24 @@
+import subprocess
+import sys
+
+from comet_checks import WORKSPACE, _failed, _passed, run_comet_checks, write_results
+
+
+def _tail(result: subprocess.CompletedProcess) -> str:
+    return "\n".join(part.strip() for part in (result.stdout, result.stderr) if part and part.strip())[-600:]
+
+
+def check_agent_memory_routing() -> dict:
+    result = subprocess.run(
+        [sys.executable, "-m", "pytest", "test_agent_system.py", "-q"],
+        capture_output=True,
+        text=True,
+        cwd=str(WORKSPACE),
+    )
+    if result.returncode != 0:
+        return _failed("agent_memory_routing", _tail(result))
+    return _passed("agent_memory_routing", "preferences persist, researcher has docs, and deployment approval is checkpointed")
+
+
+if __name__ == "__main__":
+    write_results([*run_comet_checks(), check_agent_memory_routing()])

package/eval/local/tasks/comet-api-cache-ttl/environment/Dockerfile

@@ -0,0 +1,39 @@
+FROM python:3.12-slim
+
+WORKDIR /workspace
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    bash git curl jq ca-certificates \
+    nodejs npm && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN pip install --no-cache-dir pyyaml pytest
+
+RUN npm config set registry https://registry.npmmirror.com && \
+    npm install -g @fission-ai/openspec@1.3.1 && \
+    npm config delete registry
+
+ARG CLAUDE_CODE_VERSION=latest
+RUN npm config set registry https://registry.npmmirror.com && \
+    npm install -g @anthropic-ai/claude-code@${CLAUDE_CODE_VERSION} && \
+    npm config delete registry
+
+COPY cache.py /workspace/cache.py
+COPY test_cache.py /workspace/test_cache.py
+
+RUN mkdir -p wordcount-cli && \
+    echo '# Cache Lib' > wordcount-cli/README.md
+RUN mkdir -p openspec/changes
+
+RUN useradd --create-home --shell /bin/bash agent && \
+    chown -R agent:agent /workspace
+USER agent
+ENV HOME=/home/agent
+# Validator scripts import scaffold.*; bake PYTHONPATH so it never crosses the
+# MSYS path-conversion boundary on the docker run command line.
+ENV PYTHONPATH=/workspace
+RUN git config --global user.email "agent@eval.local" && \
+    git config --global user.name "Comet Eval Agent" && \
+    git config --global init.defaultBranch master
+
+CMD ["bash"]

package/eval/local/tasks/comet-api-cache-ttl/environment/cache.py

@@ -0,0 +1,39 @@
+"""cache-lib: a simple in-memory key-value cache.
+
+Currently supports set/get/delete with no expiration. The task is to add
+optional TTL (time-to-live) support while keeping the existing API backward
+compatible.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Optional
+
+
+class Cache:
+    """A simple in-memory cache. Thread-safety is out of scope."""
+
+    def __init__(self) -> None:
+        self._store: dict[str, Any] = {}
+
+    def set(self, key: str, value: Any) -> None:
+        """Store a value. Overwrites any existing value (no expiry)."""
+        self._store[key] = value
+
+    def get(self, key: str, default: Any = None) -> Any:
+        """Retrieve a value, or `default` if the key is absent."""
+        return self._store.get(key, default)
+
+    def delete(self, key: str) -> bool:
+        """Remove a key. Returns True if it existed, False otherwise."""
+        return self._store.pop(key, None) is not None
+
+    def clear(self) -> None:
+        """Remove all entries."""
+        self._store.clear()
+
+    def __len__(self) -> int:
+        return len(self._store)
+
+    def __contains__(self, key: object) -> bool:
+        return key in self._store

package/eval/local/tasks/comet-api-cache-ttl/environment/test_cache.py

@@ -0,0 +1,81 @@
+"""Tests for cache-lib.
+
+The TTL tests currently FAIL because TTL is not yet implemented.
+The basic tests all pass and must continue to pass (backward compatibility).
+"""
+
+import time
+
+import pytest
+
+from cache import Cache
+
+
+class TestBasic:
+    def test_set_get(self):
+        c = Cache()
+        c.set("k", "v")
+        assert c.get("k") == "v"
+
+    def test_get_missing_default(self):
+        c = Cache()
+        assert c.get("nope") is None
+        assert c.get("nope", "fallback") == "fallback"
+
+    def test_overwrite(self):
+        c = Cache()
+        c.set("k", 1)
+        c.set("k", 2)
+        assert c.get("k") == 2
+
+    def test_delete(self):
+        c = Cache()
+        c.set("k", "v")
+        assert c.delete("k") is True
+        assert c.delete("k") is False
+        assert c.get("k") is None
+
+    def test_clear(self):
+        c = Cache()
+        c.set("a", 1)
+        c.set("b", 2)
+        c.clear()
+        assert len(c) == 0
+
+    def test_contains(self):
+        c = Cache()
+        c.set("k", "v")
+        assert "k" in c
+        assert "x" not in c
+
+
+class TestTTL:
+    def test_set_with_ttl_expires(self):
+        """A key set with a TTL should return the default after expiry."""
+        c = Cache()
+        c.set("temp", "value", ttl=0.1)
+        assert c.get("temp") == "value"  # immediately available
+        time.sleep(0.2)
+        assert c.get("temp") is None  # expired
+
+    def test_no_ttl_does_not_expire(self):
+        """A key set without TTL must not expire (backward compat)."""
+        c = Cache()
+        c.set("perm", "value")
+        time.sleep(0.1)
+        assert c.get("perm") == "value"
+
+    def test_ttl_overwrite_without_ttl(self):
+        """Overwriting a TTL key without TTL should remove the expiry."""
+        c = Cache()
+        c.set("k", 1, ttl=0.05)
+        c.set("k", 2)  # no ttl -> permanent
+        time.sleep(0.1)
+        assert c.get("k") == 2
+
+    def test_ttl_zero_means_no_expiry(self):
+        """ttl=0 (or omitted) means the key never expires."""
+        c = Cache()
+        c.set("k", "v", ttl=0)
+        time.sleep(0.05)
+        assert c.get("k") == "v"

package/eval/local/tasks/comet-api-cache-ttl/instruction.md

@@ -0,0 +1,33 @@
+You are working on a Python project called "cache-lib" - a simple in-memory key-value cache.
+
+Your task: Use the comet workflow to **add TTL (time-to-live) support** to the `Cache` class.
+
+## The requirement
+
+Add an optional `ttl` parameter to `set()`:
+
+```python
+def set(self, key: str, value: Any, ttl: Optional[float] = None) -> None:
+    """Store a value. If ttl is given (seconds), the key expires after that long.
+    ttl=0 or None means the key never expires."""
+```
+
+After a key's TTL elapses, `get()` must return the default (as if the key were absent). The existing behavior (no TTL) must not change.
+
+## Design decisions to make (use brainstorming)
+
+- **Lazy vs active eviction**: check expiry on `get()` (lazy, simple) vs a background sweep (active, complex). Lazy is recommended for this scope.
+- **Overwrite semantics**: setting a key again without TTL should clear any prior expiry.
+- **`__contains__` / `__len__`**: should expired keys count? (Expired keys should not appear present.)
+
+## Tests
+
+`test_cache.py` has a `TestTTL` class with 4 tests that currently FAIL. Plus 6 basic tests that must keep passing.
+
+## What to do
+
+1. Run tests to confirm: `python -m pytest test_cache.py -v`
+2. Follow the comet workflow (Open → Design → Build → Verify → Archive).
+3. Confirm all 10 tests pass after implementation.
+
+Start by detecting the current phase and following the comet workflow. When you reach a decision point that asks for confirmation, assume "yes, proceed with the recommended option".

package/eval/local/tasks/comet-api-cache-ttl/task.toml

@@ -0,0 +1,24 @@
+[metadata]
+name = "comet-api-cache-ttl"
+description = "Add optional TTL (time-to-live) expiration to an in-memory cache while keeping the API backward compatible"
+difficulty = "hard"
+category = "comet"
+tags = ["comet", "workflow", "api-design", "backward-compat"]
+default_treatments = [
+    "CONTROL",
+    "COMET_FULL_040_BETA",
+    "COMET_FULL_039",
+]
+
+[template]
+required = []
+
+[environment]
+description = "Python in-memory cache. Claude must add TTL support via the comet workflow."
+dockerfile = "environment/Dockerfile"
+timeout_sec = 900
+
+[validation]
+test_scripts = "test_api_cache_ttl.py"
+target_artifacts = []
+timeout = 120

package/eval/local/tasks/comet-api-cache-ttl/validation/test_api_cache_ttl.py

@@ -0,0 +1,58 @@
+"""Validation for the comet-api-cache-ttl task.
+
+Checks: comet workflow followed + set() accepts ttl + TTL tests pass + backward compat.
+"""
+
+import inspect
+import subprocess
+import sys
+
+from comet_checks import (
+    run_comet_checks,
+    write_results,
+    _passed,
+    _failed,
+    WORKSPACE,
+)
+
+
+def _subprocess_output(result: subprocess.CompletedProcess) -> str:
+    output = "\n".join(
+        part.strip()
+        for part in (result.stdout or "", result.stderr or "")
+        if part and part.strip()
+    )
+    return output[-600:]
+
+
+def check_ttl_implementation() -> dict:
+    """set() has a ttl param and all tests pass (TTL + backward compat)."""
+    # set() must accept a ttl parameter.
+    sig_result = subprocess.run(
+        [sys.executable, "-c",
+         "import inspect,cache; sig=inspect.signature(cache.Cache.set); print(list(sig.parameters))"],
+        capture_output=True, text=True, cwd=str(WORKSPACE),
+    )
+    if sig_result.returncode != 0:
+        return _failed("ttl_implementation", f"cannot inspect cache.Cache.set: {sig_result.stderr[:150]}")
+    if "ttl" not in sig_result.stdout:
+        return _failed("ttl_implementation", "Cache.set() has no 'ttl' parameter")
+
+    # All tests must pass (TTL + backward compat).
+    test_result = subprocess.run(
+        [sys.executable, "-m", "pytest", "test_cache.py", "-q"],
+        capture_output=True, text=True, cwd=str(WORKSPACE),
+    )
+    if test_result.returncode != 0:
+        return _failed("ttl_implementation", f"tests failing:\n{_subprocess_output(test_result)}")
+    return _passed("ttl_implementation", "set(ttl=...) works, all 10 tests pass")
+
+
+def run_all() -> list[dict]:
+    results = run_comet_checks()
+    results.append(check_ttl_implementation())
+    return results
+
+
+if __name__ == "__main__":
+    write_results(run_all())

package/eval/local/tasks/comet-cross-file-refactor/environment/Dockerfile

@@ -0,0 +1,15 @@
+FROM python:3.12-slim
+
+WORKDIR /workspace
+
+RUN apt-get update && apt-get install -y --no-install-recommends bash git curl jq ca-certificates nodejs npm && rm -rf /var/lib/apt/lists/*
+RUN pip install --no-cache-dir pyyaml pytest
+RUN npm config set registry https://registry.npmmirror.com && npm install -g @fission-ai/openspec@1.3.1 && npm install -g @anthropic-ai/claude-code@latest && npm config delete registry
+COPY . /workspace/
+RUN mkdir -p openspec/changes
+RUN useradd --create-home --shell /bin/bash agent && chown -R agent:agent /workspace
+USER agent
+ENV HOME=/home/agent
+ENV PYTHONPATH=/workspace
+RUN git config --global user.email "agent@eval.local" && git config --global user.name "Comet Eval Agent" && git config --global init.defaultBranch master
+CMD ["bash"]

package/eval/local/tasks/comet-cross-file-refactor/environment/cli.py

@@ -0,0 +1,13 @@
+from __future__ import annotations
+
+from counting import count_chars, count_lines, count_words
+
+
+def run(text: str, unit: str) -> int:
+    if unit == "words":
+        return count_words(text)
+    if unit == "lines":
+        return count_lines(text)
+    if unit == "chars":
+        return count_chars(text)
+    raise ValueError(f"unsupported unit: {unit}")

package/eval/local/tasks/comet-cross-file-refactor/environment/counting.py

@@ -0,0 +1,13 @@
+from __future__ import annotations
+
+
+def count_words(text: str) -> int:
+    return len(text.split())
+
+
+def count_lines(text: str) -> int:
+    return len(text.splitlines())
+
+
+def count_chars(text: str) -> int:
+    return len(text)

package/eval/local/tasks/comet-cross-file-refactor/environment/test_textkit.py

@@ -0,0 +1,21 @@
+from __future__ import annotations
+
+import cli
+import counting
+
+
+def test_new_dispatcher_counts_all_units():
+    assert counting.count("hello world", "words") == 2
+    assert counting.count("a\nb", "lines") == 2
+    assert counting.count("abc", "chars") == 3
+
+
+def test_wrappers_remain_compatible():
+    assert counting.count_words("hello world") == 2
+    assert counting.count_lines("a\nb") == 2
+    assert counting.count_chars("abc") == 3
+
+
+def test_cli_uses_dispatcher():
+    assert cli.run("hello world", "words") == 2
+    assert cli.run("abc", "chars") == 3

package/eval/local/tasks/comet-cross-file-refactor/instruction.md

@@ -0,0 +1,16 @@
+You are working on a Python package called "textkit".
+
+Your task: Use the comet workflow to refactor duplicated counting functions into a shared dispatcher.
+
+This task is adapted from `skills-benchmarks/lc-basic`.
+
+## Requirements
+
+- Add `count(text, unit)` supporting `"words"`, `"lines"`, and `"chars"`.
+- Keep existing wrappers `count_words`, `count_lines`, and `count_chars`.
+- Update the CLI helper to use the shared dispatcher.
+- Preserve existing behavior and tests.
+
+Run `python -m pytest test_textkit.py -q`, follow the comet workflow, and archive the completed change.
+
+When the workflow asks for confirmation, assume "yes, proceed with the recommended option".

package/eval/local/tasks/comet-cross-file-refactor/task.toml

@@ -0,0 +1,20 @@
+[metadata]
+name = "comet-cross-file-refactor"
+description = "Add a shared count(text, unit) dispatcher while keeping wrappers and CLI behavior compatible"
+difficulty = "hard"
+category = "comet"
+tags = ["comet", "workflow", "refactor", "cross-file"]
+default_treatments = ["CONTROL", "COMET_FULL_040_BETA", "COMET_FULL_039"]
+
+[template]
+required = []
+
+[environment]
+description = "Python package split across counting and CLI modules. Claude must refactor without breaking compatibility."
+dockerfile = "environment/Dockerfile"
+timeout_sec = 900
+
+[validation]
+test_scripts = "test_cross_file_refactor.py"
+target_artifacts = []
+timeout = 120

package/eval/local/tasks/comet-cross-file-refactor/validation/test_cross_file_refactor.py

@@ -0,0 +1,20 @@
+import subprocess
+import sys
+
+from comet_checks import run_comet_checks, write_results, _passed, _failed, WORKSPACE
+
+
+def check_cross_file_refactor() -> dict:
+    result = subprocess.run(
+        [sys.executable, "-m", "pytest", "test_textkit.py", "-q"],
+        capture_output=True,
+        text=True,
+        cwd=str(WORKSPACE),
+    )
+    if result.returncode != 0:
+        return _failed("cross_file_count_dispatcher", (result.stdout + result.stderr)[-600:])
+    return _passed("cross_file_count_dispatcher", "dispatcher, wrappers, and CLI all work")
+
+
+if __name__ == "__main__":
+    write_results([*run_comet_checks(), check_cross_file_refactor()])

package/eval/local/tasks/comet-dependency-confusion/environment/Dockerfile

@@ -0,0 +1,28 @@
+FROM python:3.12-slim
+
+WORKDIR /workspace
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    bash git curl jq ca-certificates nodejs npm && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN pip install --no-cache-dir pyyaml pytest
+
+RUN npm config set registry https://registry.npmmirror.com && \
+    npm install -g @fission-ai/openspec@1.3.1 && \
+    npm install -g @anthropic-ai/claude-code@latest && \
+    npm config delete registry
+
+COPY . /workspace/
+RUN mkdir -p openspec/changes
+
+RUN useradd --create-home --shell /bin/bash agent && \
+    chown -R agent:agent /workspace
+USER agent
+ENV HOME=/home/agent
+ENV PYTHONPATH=/workspace
+RUN git config --global user.email "agent@eval.local" && \
+    git config --global user.name "Comet Eval Agent" && \
+    git config --global init.defaultBranch master
+
+CMD ["bash"]

package/eval/local/tasks/comet-dependency-confusion/environment/app.py

@@ -0,0 +1,8 @@
+from __future__ import annotations
+
+from legacy_settings import load_settings
+
+
+def connection_url(overrides: dict[str, str] | None = None) -> str:
+    settings = load_settings(overrides)
+    return f"http://{settings['host']}:{settings['port']}"

package/eval/local/tasks/comet-dependency-confusion/environment/settings_loader.py

@@ -0,0 +1,14 @@
+from __future__ import annotations
+
+
+DEFAULTS = {
+    "host": "127.0.0.1",
+    "port": "8080",
+}
+
+
+def load_config(overrides: dict[str, str] | None = None) -> dict[str, str]:
+    config = dict(DEFAULTS)
+    if overrides:
+        config.update(overrides)
+    return config

package/eval/local/tasks/comet-dependency-confusion/environment/test_settings_service.py

@@ -0,0 +1,19 @@
+from __future__ import annotations
+
+import importlib
+
+from app import connection_url
+
+
+def test_old_import_path_still_loads_settings():
+    legacy = importlib.import_module("legacy_settings")
+    assert legacy.load_settings({"port": "9000"})["port"] == "9000"
+
+
+def test_new_import_path_still_loads_config():
+    loader = importlib.import_module("settings_loader")
+    assert loader.load_config({"host": "localhost"})["host"] == "localhost"
+
+
+def test_app_connection_url_uses_compatible_loader():
+    assert connection_url({"host": "localhost", "port": "5000"}) == "http://localhost:5000"

package/eval/local/tasks/comet-dependency-confusion/instruction.md

@@ -0,0 +1,18 @@
+You are working on a Python project called "settings-service".
+
+Your task: Use the comet workflow to fix a compatibility regression caused by a module rename.
+
+This task is adapted from `skills-benchmarks/lc-deps-tavily`.
+
+## Problem
+
+The project used to expose `legacy_settings.load_settings()`, but the implementation moved to `settings_loader.py`. Existing callers still import the old module name, and the app also imports a stale helper name. The current tests in `test_settings_service.py` fail.
+
+## What to do
+
+1. Run `python -m pytest test_settings_service.py -q` to confirm the failure.
+2. Follow the comet workflow phases from open through archive.
+3. Restore compatibility without deleting the new `settings_loader.py` implementation.
+4. Verify that both old and new import paths work and all tests pass.
+
+When the workflow asks for confirmation, assume "yes, proceed with the recommended option".

package/eval/local/tasks/comet-dependency-confusion/task.toml

@@ -0,0 +1,24 @@
+[metadata]
+name = "comet-dependency-confusion"
+description = "Repair stale module names and compatibility imports in a small settings loader"
+difficulty = "medium"
+category = "comet"
+tags = ["comet", "workflow", "dependencies", "compatibility"]
+default_treatments = [
+    "CONTROL",
+    "COMET_FULL_040_BETA",
+    "COMET_FULL_039",
+]
+
+[template]
+required = []
+
+[environment]
+description = "Python workspace with stale imports after a module rename. Claude must use the comet workflow to restore compatibility."
+dockerfile = "environment/Dockerfile"
+timeout_sec = 900
+
+[validation]
+test_scripts = "test_dependency_confusion.py"
+target_artifacts = []
+timeout = 120