@rpamis/comet 0.3.9 → 0.4.0-beta.10

package/eval/local/tasks/comet-layered-streaming-fix/task.toml

@@ -0,0 +1,20 @@
+[metadata]
+name = "comet-layered-streaming-fix"
+description = "Fix both streaming chunk behavior and tool-result handling in a chat application"
+difficulty = "hard"
+category = "comet"
+tags = ["comet", "workflow", "streaming", "layered-bugs"]
+default_treatments = ["CONTROL", "COMET_FULL_040_BETA", "COMET_FULL_039"]
+
+[template]
+required = []
+
+[environment]
+description = "Python chat app with layered streaming and tool-result bugs."
+dockerfile = "environment/Dockerfile"
+timeout_sec = 900
+
+[validation]
+test_scripts = "test_layered_streaming_fix.py"
+target_artifacts = []
+timeout = 120

package/eval/local/tasks/comet-layered-streaming-fix/validation/test_layered_streaming_fix.py

@@ -0,0 +1,20 @@
+import subprocess
+import sys
+
+from comet_checks import run_comet_checks, write_results, _passed, _failed, WORKSPACE
+
+
+def check_chat_fixes() -> dict:
+    result = subprocess.run(
+        [sys.executable, "-m", "pytest", "test_chat_app.py", "-q"],
+        capture_output=True,
+        text=True,
+        cwd=str(WORKSPACE),
+    )
+    if result.returncode != 0:
+        return _failed("chat_streaming_and_tool_results", (result.stdout + result.stderr)[-600:])
+    return _passed("chat_streaming_and_tool_results", "streaming chunks and tool results are correct")
+
+
+if __name__ == "__main__":
+    write_results([*run_comet_checks(), check_chat_fixes()])

package/eval/local/tasks/comet-native-clarification/environment/Dockerfile

@@ -0,0 +1,25 @@
+FROM python:3.12-slim
+
+WORKDIR /workspace
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    git nodejs npm ca-certificates && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN npm install -g @anthropic-ai/claude-code@latest
+
+RUN pip install --no-cache-dir pyyaml pytest
+
+COPY wordcount.py /workspace/wordcount.py
+COPY test_wordcount.py /workspace/test_wordcount.py
+
+RUN useradd --create-home --shell /bin/bash agent && \
+    chown -R agent:agent /workspace
+USER agent
+ENV HOME=/home/agent
+ENV PYTHONPATH=/workspace
+RUN git config --global user.email "agent@eval.local" && \
+    git config --global user.name "Comet Native Clarification Eval Agent" && \
+    git config --global init.defaultBranch master
+
+CMD ["bash"]

package/eval/local/tasks/comet-native-clarification/environment/test_wordcount.py

@@ -0,0 +1,25 @@
+"""Baseline tests for the word-count CLI."""
+
+import subprocess
+import sys
+
+
+def run_cli(*args: str, input_text: str) -> str:
+    result = subprocess.run(
+        [sys.executable, "wordcount.py", *args],
+        input=input_text,
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+    return result.stdout
+
+
+def test_count_words():
+    assert "Words: 3" in run_cli(input_text="hello world foo")
+
+
+def test_count_lines():
+    output = run_cli("--lines", input_text="hello\nworld\nfoo")
+    assert "Words: 3" in output
+    assert "Lines: 3" in output

package/eval/local/tasks/comet-native-clarification/environment/wordcount.py

@@ -0,0 +1,36 @@
+"""Simple word count CLI tool."""
+
+import argparse
+import sys
+
+
+def count_words(text: str) -> int:
+    """Count words in text."""
+    return len(text.split())
+
+
+def count_lines(text: str) -> int:
+    """Count lines in text."""
+    return len(text.splitlines())
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Count words and lines in text")
+    parser.add_argument("file", nargs="?", help="Input file (reads stdin if omitted)")
+    parser.add_argument("--lines", action="store_true", help="Also count lines")
+    parser.add_argument("--words", action="store_true", default=True, help="Count words (default)")
+    args = parser.parse_args()
+
+    if args.file:
+        with open(args.file) as source:
+            text = source.read()
+    else:
+        text = sys.stdin.read()
+
+    print(f"Words: {count_words(text)}")
+    if args.lines:
+        print(f"Lines: {count_lines(text)}")
+
+
+if __name__ == "__main__":
+    main()

package/eval/local/tasks/comet-native-clarification/instruction.md

@@ -0,0 +1,7 @@
+You are working on a Python project named `wordcount-cli`.
+
+Begin by invoking the `/comet-native` Skill. Initialize Native under `docs/comet/`, create a change, and add a `--sentences` CLI flag that prints `Sentences: N` while preserving the existing word and line counters.
+
+Sentence terminators include `.`, `!`, and `?`. Product requirements intentionally do not specify how abbreviations such as `e.g.` and `Dr.` affect sentence boundaries. Do not guess how abbreviations should behave. This decision changes user-visible counts, so ask the user before finalizing Shape. Ask only the single highest-value question, with a recommendation and practical impact. Do not begin implementation before the user answers.
+
+After the answer, record it in the brief, use Native's explicit confirmation path, write a complete target specification, implement focused tests including abbreviations, verify, and archive. Use no other workflow Skill.

package/eval/local/tasks/comet-native-clarification/task.toml

@@ -0,0 +1,40 @@
+[metadata]
+name = "comet-native-clarification"
+description = "Test the Native one-question clarification boundary and confirmed resume"
+difficulty = "hard"
+category = "comet-native"
+tags = ["comet-native", "clarification", "confirmation", "multi-turn"]
+default_treatments = ["COMET_NATIVE_PHASE1"]
+
+[template]
+required = []
+
+[environment]
+description = "Python word-count workspace with only the self-contained Comet Native Skill available."
+dockerfile = "environment/Dockerfile"
+timeout_sec = 900
+
+[validation]
+test_scripts = "test_native_clarification.py"
+target_artifacts = ["docs/comet/archive", "docs/comet/specs"]
+timeout = 120
+
+[evaluation]
+profile = "generic"
+required_skills = ["comet-native"]
+expected_artifacts = [".comet/config.yaml", "docs/comet/archive", "docs/comet/specs"]
+require_skill_invocation = true
+rubric_criteria = [
+  "Before implementation, asks exactly one highest-value question about abbreviation behavior",
+  "Does not ask the user for repository facts or implementation-method choices",
+  "Records the supplied abbreviation decision in the brief and leaves Shape with explicit confirmation",
+  "Completes Build, Verify, and Archive using only the Comet Native Skill and bundled runtime",
+]
+
+[interaction]
+mode = "auto_user"
+max_turns = 4
+simulator_prompt = """You are the developer user in a Comet Native eval. The assistant should ask one highest-value question about how abbreviations affect sentence boundaries. Answer that question in one sentence: Abbreviations such as e.g. and Dr. do not end a sentence; use a small explicit abbreviation list. If the assistant asks multiple questions, answer only the abbreviation question and remind it to handle one decision at a time. Do not write code or choose implementation tooling."""
+decision_patterns = ["abbreviation", "e.g.", "Dr.", "sentence boundary"]
+decision_reply = "Abbreviations such as e.g. and Dr. do not end a sentence; use a small explicit abbreviation list."
+continue_prompt = "Please verify the Native status from disk and continue until the same change is actually archived with its canonical specification."

package/eval/local/tasks/comet-native-clarification/validation/test_native_clarification.py

@@ -0,0 +1,193 @@
+"""Validate Native clarification, confirmation, implementation, and archive artifacts."""
+
+import json
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+import yaml
+
+
+WORKSPACE = Path("/workspace")
+RESULTS_FILE = os.environ.get("BENCH_TEST_RESULTS", "_test_results.json")
+
+
+def passed(name: str):
+    return {"check": name, "status": "passed"}
+
+
+def failed(name: str, reason: str):
+    return {"check": name, "status": "failed", "reason": reason}
+
+
+def archive_directories():
+    root = WORKSPACE / "docs" / "comet" / "archive"
+    return sorted(path for path in root.glob("*-*") if path.is_dir())
+
+
+def archive_directory():
+    candidates = archive_directories()
+    return candidates[-1] if candidates else None
+
+
+def canonical_spec_files():
+    root = WORKSPACE / "docs" / "comet" / "specs"
+    return sorted(root.glob("*/spec.md"))
+
+
+def active_change_directories():
+    root = WORKSPACE / "docs" / "comet" / "changes"
+    return sorted(path for path in root.glob("*") if path.is_dir())
+
+
+def read_interaction():
+    context = WORKSPACE / "_test_context.json"
+    if not context.is_file():
+        return {}
+    return json.loads(context.read_text(encoding="utf-8")).get("interaction") or {}
+
+
+def check_behavior():
+    try:
+        subprocess.run(
+            [sys.executable, "-m", "pytest", "-q"],
+            cwd=WORKSPACE,
+            capture_output=True,
+            text=True,
+            timeout=30,
+            check=True,
+        )
+        result = subprocess.run(
+            [sys.executable, "wordcount.py", "--sentences"],
+            cwd=WORKSPACE,
+            input="Use e.g. examples. Ask Dr. Smith!",
+            capture_output=True,
+            text=True,
+            timeout=10,
+            check=True,
+        )
+    except Exception as error:
+        return failed("clarified_behavior", str(error))
+    if "Sentences: 2" not in result.stdout:
+        return failed("clarified_behavior", f"Expected Sentences: 2, got {result.stdout!r}")
+    return passed("clarified_behavior")
+
+
+def check_confirmed_archive():
+    active = active_change_directories()
+    if active:
+        return failed(
+            "confirmed_archive",
+            f"Expected no active change after archive, found {len(active)}",
+        )
+    archives = archive_directories()
+    if len(archives) != 1:
+        return failed(
+            "confirmed_archive",
+            f"Expected exactly one Native archive for the clarified change, found {len(archives)}",
+    )
+    archived = archives[0]
+    state_file = archived / "comet-state.yaml"
+    if not state_file.is_file():
+        return failed("confirmed_archive", "Archived comet-state.yaml is missing")
+    state = yaml.safe_load(state_file.read_text(encoding="utf-8")) or {}
+    if state.get("phase") != "archive" or state.get("archived") is not True:
+        return failed("confirmed_archive", "Archived state is not terminal")
+    if state.get("approval") != "confirmed":
+        return failed("confirmed_archive", "Shape did not record explicit confirmation")
+    if state.get("verification_result") != "pass":
+        return failed("confirmed_archive", "Verify did not record a passing result")
+    brief_file = archived / "brief.md"
+    if not brief_file.is_file():
+        return failed("confirmed_archive", "Archived brief is missing")
+    brief = brief_file.read_text(encoding="utf-8").lower()
+    if not all(marker in brief for marker in ("abbreviation", "e.g.", "dr.")):
+        return failed("confirmed_archive", "The confirmed abbreviation decision is missing")
+
+    spec_changes = state.get("spec_changes") or []
+    if len(spec_changes) != 1 or not isinstance(spec_changes[0], dict):
+        return failed("confirmed_archive", "Expected exactly one linked capability change")
+    spec_change = spec_changes[0]
+    capability = spec_change.get("capability")
+    expected_source = f"specs/{capability}/spec.md"
+    if not capability or spec_change.get("source") != expected_source:
+        return failed("confirmed_archive", "Archived spec change is not linked to its capability")
+
+    canonical_specs = canonical_spec_files()
+    canonical_path = WORKSPACE / "docs" / "comet" / "specs" / capability / "spec.md"
+    if canonical_specs != [canonical_path]:
+        return failed(
+            "confirmed_archive",
+            "Canonical specification does not match the archived capability",
+        )
+    archived_spec = archived / expected_source
+    if not archived_spec.is_file():
+        return failed("confirmed_archive", "Archived target specification is missing")
+    canonical_source = canonical_path.read_text(encoding="utf-8")
+    if canonical_source != archived_spec.read_text(encoding="utf-8"):
+        return failed("confirmed_archive", "Archived and canonical specifications differ")
+    canonical = canonical_source.lower()
+    if "--sentences" not in canonical or not any(
+        marker in canonical for marker in ("abbreviation", "e.g.", "dr.")
+    ):
+        return failed(
+            "confirmed_archive",
+            "The canonical target specification does not cover sentence counting and abbreviations",
+        )
+
+    report_name = state.get("verification_report")
+    report_file = archived / str(report_name or "")
+    if report_name != "verification.md" or not report_file.is_file():
+        return failed("confirmed_archive", "Verification evidence is missing")
+    report = report_file.read_text(encoding="utf-8").lower()
+    if "pass" not in report or "pytest" not in report:
+        return failed("confirmed_archive", "Verification evidence does not record passing tests")
+
+    trajectory_file = archived / "runtime" / "trajectory.jsonl"
+    if not trajectory_file.is_file():
+        return failed("confirmed_archive", "Native trajectory is missing")
+    transitions = []
+    for line in trajectory_file.read_text(encoding="utf-8").splitlines():
+        event = json.loads(line)
+        if event.get("type") == "state_transitioned":
+            data = event.get("data") or {}
+            transitions.append((data.get("previousPhase"), data.get("nextPhase")))
+    expected_transitions = [
+        ("shape", "build"),
+        ("build", "verify"),
+        ("verify", "archive"),
+        ("archive", None),
+    ]
+    if transitions != expected_transitions:
+        return failed("confirmed_archive", "Native trajectory does not cover every phase exactly once")
+
+    interaction = read_interaction()
+    if interaction.get("mode") != "auto_user":
+        return failed("confirmed_archive", "Clarification interaction metadata is missing")
+    if interaction.get("decision_points") != 1 or interaction.get("deterministic_replies") != 1:
+        return failed("confirmed_archive", "Expected exactly one deterministic clarification reply")
+    turns = interaction.get("actual_turns")
+    maximum = interaction.get("max_turns")
+    if not isinstance(turns, int) or not isinstance(maximum, int) or not (2 <= turns <= maximum):
+        return failed("confirmed_archive", "Clarification driver turn count is invalid")
+    return passed("confirmed_archive")
+
+
+def main():
+    results = [check_behavior(), check_confirmed_archive()]
+    output = {
+        "passed": [result["check"] for result in results if result["status"] == "passed"],
+        "failed": [
+            f'{result["check"]}: {result.get("reason", "")}'
+            for result in results
+            if result["status"] == "failed"
+        ],
+    }
+    (WORKSPACE / RESULTS_FILE).write_text(json.dumps(output, indent=2), encoding="utf-8")
+    print(json.dumps(output))
+    return 0 if not output["failed"] else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/eval/local/tasks/comet-native-clarification-depth/environment/Dockerfile

@@ -0,0 +1,25 @@
+FROM python:3.12-slim
+
+WORKDIR /workspace
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    git nodejs npm ca-certificates && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN npm install -g @anthropic-ai/claude-code@latest
+
+RUN pip install --no-cache-dir pyyaml pytest
+
+COPY wordcount.py /workspace/wordcount.py
+COPY test_wordcount.py /workspace/test_wordcount.py
+
+RUN useradd --create-home --shell /bin/bash agent && \
+    chown -R agent:agent /workspace
+USER agent
+ENV HOME=/home/agent
+ENV PYTHONPATH=/workspace
+RUN git config --global user.email "agent@eval.local" && \
+    git config --global user.name "Comet Native Clarification Depth Eval Agent" && \
+    git config --global init.defaultBranch master
+
+CMD ["bash"]

package/eval/local/tasks/comet-native-clarification-depth/environment/test_wordcount.py

@@ -0,0 +1,25 @@
+"""Baseline tests for the word-count CLI."""
+
+import subprocess
+import sys
+
+
+def run_cli(*args: str, input_text: str) -> str:
+    result = subprocess.run(
+        [sys.executable, "wordcount.py", *args],
+        input=input_text,
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+    return result.stdout
+
+
+def test_count_words():
+    assert "Words: 3" in run_cli(input_text="hello world foo")
+
+
+def test_count_lines():
+    output = run_cli("--lines", input_text="hello\nworld\nfoo")
+    assert "Words: 3" in output
+    assert "Lines: 3" in output

package/eval/local/tasks/comet-native-clarification-depth/environment/wordcount.py

@@ -0,0 +1,36 @@
+"""Simple word count CLI tool."""
+
+import argparse
+import sys
+
+
+def count_words(text: str) -> int:
+    """Count words in text."""
+    return len(text.split())
+
+
+def count_lines(text: str) -> int:
+    """Count lines in text."""
+    return len(text.splitlines())
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Count words and lines in text")
+    parser.add_argument("file", nargs="?", help="Input file (reads stdin if omitted)")
+    parser.add_argument("--lines", action="store_true", help="Also count lines")
+    parser.add_argument("--words", action="store_true", default=True, help="Count words (default)")
+    args = parser.parse_args()
+
+    if args.file:
+        with open(args.file) as source:
+            text = source.read()
+    else:
+        text = sys.stdin.read()
+
+    print(f"Words: {count_words(text)}")
+    if args.lines:
+        print(f"Lines: {count_lines(text)}")
+
+
+if __name__ == "__main__":
+    main()

package/eval/local/tasks/comet-native-clarification-depth/instruction.md

@@ -0,0 +1,17 @@
+You are working on a Python project named `wordcount-cli`.
+
+Begin by invoking the `/comet-native` Skill. Initialize Native under `docs/comet/`, create one change named `add-sentence-counting`, and add an opt-in `--sentences` CLI flag that prints `Sentences: N` after any requested `Lines: N` output while preserving the existing word and line counters.
+
+The fixed interface contract is:
+
+- only the ASCII terminal characters `.`, `!`, and `?` can create sentence boundaries;
+- non-empty input with no terminal character counts as one sentence;
+- when `--sentences` is present, it prints independently of `--lines`; the flag behavior and output order described above are not open product decisions.
+
+The input domain is ordinary ASCII prose. Numbers, URLs, file paths, code, markup, quoted data, and Unicode-specific segmentation are out of scope.
+
+The contract deliberately leaves three areas unresolved: empty or whitespace-only input, consecutive terminal characters, and terminal-looking characters embedded inside ordinary prose tokens. Resolve them in that listed order through the configured clarification protocol. An answer about whether an embedded character creates a boundary may introduce later decisions about how the recognized token collection is maintained and its exact contents; handle each dependent decision immediately after its parent.
+
+The requested interface is intentionally underspecified. Follow the already configured Native Sequential clarification protocol and resolve every undefined user-visible behavior before implementation. Investigate repository-owned facts yourself, and do not change the configured clarification mode.
+
+Complete Shape, Build, Verify, and Archive using only the Comet Native Skill and its bundled runtime.

package/eval/local/tasks/comet-native-clarification-depth/task.toml

@@ -0,0 +1,47 @@
+[metadata]
+name = "comet-native-clarification-depth"
+description = "Measure whether Native Sequential discovers hidden user-owned decisions before implementation"
+difficulty = "hard"
+category = "comet-native"
+tags = ["comet-native", "clarification", "sequential", "multi-turn", "decision-depth"]
+default_treatments = ["COMET_NATIVE_SEQUENTIAL"]
+
+[template]
+required = []
+
+[environment]
+description = "Python word-count workspace with only the self-contained Comet Native Skill available."
+dockerfile = "environment/Dockerfile"
+timeout_sec = 2400
+
+[validation]
+test_scripts = "test_native_clarification_depth.py"
+target_artifacts = [".comet/config.yaml", "docs/comet/archive", "docs/comet/specs"]
+timeout = 120
+
+[evaluation]
+profile = "generic"
+required_skills = ["comet-native"]
+expected_artifacts = [".comet/config.yaml", "docs/comet/archive", "docs/comet/specs"]
+require_skill_invocation = true
+rubric_criteria = [
+  "Investigates repository-owned facts without asking the user",
+  "Discovers all hidden user-owned behavior decisions one at a time",
+  "Provides a recommendation and practical impact for each decision",
+  "Obtains explicit shared-understanding confirmation before Build",
+  "Completes Build, Verify, and Archive using only the Comet Native Skill and bundled runtime",
+]
+
+[interaction]
+mode = "auto_user"
+max_turns = 8
+decision_replies = [
+  "It prints Sentences: 0.",
+  "They count as one sentence boundary.",
+  "Recognized abbreviations such as e.g. and Dr. always suppress their periods, including at end-of-input; every other occurrence of ., !, or ? remains a sentence boundary, and the non-empty fallback still makes an abbreviation-only input one sentence.",
+  "Use a small explicit list. Ask me for the exact entries separately.",
+  "Use exactly Dr., Mr., Mrs., Ms., Prof., Sr., Jr., e.g., i.e., etc., vs., Inc., Ltd., Corp., St., and Ave., matched case-sensitively exactly as written.",
+  "confirmed",
+]
+decision_patterns = ["Question:", "Recommendation:", "Impact:", "shared understanding", "Please confirm"]
+continue_prompt = "Please read the persisted Native state and continue the same confirmed change through verification and archive."