@rpamis/comet 0.4.0-beta.5 → 0.4.0-beta.7

package/eval/local/tasks/comet-native-clarification-modes/environment/wordcount.py

@@ -0,0 +1,36 @@
+"""Simple word count CLI tool."""
+
+import argparse
+import sys
+
+
+def count_words(text: str) -> int:
+    """Count words in text."""
+    return len(text.split())
+
+
+def count_lines(text: str) -> int:
+    """Count lines in text."""
+    return len(text.splitlines())
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Count words and lines in text")
+    parser.add_argument("file", nargs="?", help="Input file (reads stdin if omitted)")
+    parser.add_argument("--lines", action="store_true", help="Also count lines")
+    parser.add_argument("--words", action="store_true", default=True, help="Count words (default)")
+    args = parser.parse_args()
+
+    if args.file:
+        with open(args.file) as source:
+            text = source.read()
+    else:
+        text = sys.stdin.read()
+
+    print(f"Words: {count_words(text)}")
+    if args.lines:
+        print(f"Lines: {count_lines(text)}")
+
+
+if __name__ == "__main__":
+    main()

package/eval/local/tasks/comet-native-clarification-modes/instruction.md

@@ -0,0 +1,13 @@
+You are working on a Python project named `wordcount-cli`.
+
+Begin by invoking the `/comet-native` Skill. Initialize Native under `docs/comet/`, create one change named `add-sentence-counting`, and add a `--sentences` CLI flag that prints `Sentences: N` while preserving the existing word and line counters.
+
+Before Shape is complete, resolve these three independent product decisions with the user:
+
+1. Whether abbreviations such as `e.g.` and `Dr.` end a sentence.
+2. What sentence count empty input should print.
+3. Whether consecutive terminators such as `?!` count as one boundary or multiple boundaries.
+
+These decisions do not depend on one another, and the repository does not determine them. Follow the `native.clarification_mode` already selected by the treatment. Do not choose the clarification mode yourself or change it. Give a recommendation and practical impact for every question, persist all answers in the brief and complete target specification, and satisfy any confirmation required by the configured mode before implementation.
+
+After clarification, write a complete target specification using Native's `## Requirement:` and `### Scenario:` structure. Then implement focused tests for all three decisions, verify the result, and archive the change. Use no other workflow Skill.

package/eval/local/tasks/comet-native-clarification-modes/task.toml

@@ -0,0 +1,39 @@
+[metadata]
+name = "comet-native-clarification-modes"
+description = "Compare Native Sequential and Batch clarification over the same independent product decisions"
+difficulty = "hard"
+category = "comet-native"
+tags = ["comet-native", "clarification", "batch", "sequential", "multi-turn"]
+default_treatments = ["COMET_NATIVE_SEQUENTIAL", "COMET_NATIVE_BATCH"]
+
+[template]
+required = []
+
+[environment]
+description = "Python word-count workspace with only the self-contained Comet Native Skill available."
+dockerfile = "environment/Dockerfile"
+timeout_sec = 2400
+
+[validation]
+test_scripts = "test_native_clarification_modes.py"
+target_artifacts = [".comet/config.yaml", "docs/comet/archive", "docs/comet/specs"]
+timeout = 120
+
+[evaluation]
+profile = "generic"
+required_skills = ["comet-native"]
+expected_artifacts = [".comet/config.yaml", "docs/comet/archive", "docs/comet/specs"]
+require_skill_invocation = true
+rubric_criteria = [
+  "Preserves the treatment-selected clarification mode for the entire run",
+  "Asks only user-owned product decisions and groups them according to Sequential or Batch semantics",
+  "Records abbreviation, empty-input, and consecutive-terminator decisions in the brief and complete target specification",
+  "Completes Build, Verify, and Archive using only the Comet Native Skill and bundled runtime",
+]
+
+[interaction]
+mode = "auto_user"
+max_turns = 6
+simulator_prompt = """You are the developer user in a paired Comet Native clarification eval. Answer only the questions present in the assistant's current message; never volunteer answers to questions it has not asked. Use these fixed product decisions: (1) abbreviations such as e.g. and Dr. do not end a sentence, using a small explicit abbreviation list; (2) empty input prints Sentences: 0; (3) consecutive terminators such as ?! count as one sentence boundary. If the assistant presents a final shared-understanding summary, confirm it explicitly. Do not choose or change the clarification mode, write code, select implementation tooling, or reject the task."""
+decision_patterns = ["abbreviation", "empty input", "consecutive terminators", "shared understanding", "confirm"]
+continue_prompt = "Please read the persisted Native state and continue the same change through verification and archive."

package/eval/local/tasks/comet-native-clarification-modes/validation/test_native_clarification_modes.py

@@ -0,0 +1,235 @@
+"""Validate paired Native clarification-mode behavior and terminal artifacts."""
+
+import json
+import os
+import re
+import subprocess
+import sys
+from pathlib import Path
+
+import yaml
+
+
+WORKSPACE = Path("/workspace")
+RESULTS_FILE = os.environ.get("BENCH_TEST_RESULTS", "_test_results.json")
+EXPECTED_MODES = {
+    "COMET_NATIVE_SEQUENTIAL": "sequential",
+    "COMET_NATIVE_BATCH": "batch",
+}
+
+
+def passed(name: str):
+    return {"check": name, "status": "passed"}
+
+
+def failed(name: str, reason: str):
+    return {"check": name, "status": "failed", "reason": reason}
+
+
+def missing_decisions(text: str) -> list[str]:
+    """Return user decisions that are not preserved semantically in an artifact."""
+    normalized = " ".join(text.lower().split())
+    missing = []
+    abbreviation = "abbrevi" in normalized and bool(
+        re.search(
+            r"(?:do|does|should)\s+not\s+(?:end|count)|"
+            r"(?:ignored|filtered|excluded)\s+as\s+(?:a\s+)?sentence\s+boundar|"
+            r"abbrevi.{0,240}not.{0,80}(?:sentence\s+(?:ending|boundar)|false\s+boundar)",
+            normalized,
+        )
+    )
+    if not abbreviation:
+        missing.append("abbreviation behavior")
+
+    empty = "empty" in normalized and bool(
+        re.search(
+            r"empty.{0,180}(?:returns?|prints?|result(?:\s+is)?|sentences:)"
+            r"[^0-9]{0,24}0\b",
+            normalized,
+        )
+    )
+    if not empty:
+        missing.append("empty-input count")
+
+    terminators = bool(
+        re.search(
+            r"(?:consecutive|contiguous|run\s+of).{0,100}(?:terminator|punctuation)", normalized
+        )
+        and re.search(r"(?:one|single|exactly\s+one).{0,40}(?:sentence\s+)?boundar", normalized)
+    )
+    if not terminators:
+        missing.append("terminator-run behavior")
+    return missing
+
+
+def approval_is_valid(mode: str | None, approval: str | None) -> bool:
+    if mode == "sequential":
+        return approval in {"implicit", "confirmed"}
+    return approval == "confirmed"
+
+
+def read_context() -> dict:
+    path = WORKSPACE / "_test_context.json"
+    if not path.is_file():
+        return {}
+    return json.loads(path.read_text(encoding="utf-8"))
+
+
+def archive_directories():
+    root = WORKSPACE / "docs" / "comet" / "archive"
+    return sorted(path for path in root.glob("*-*") if path.is_dir())
+
+
+def check_behavior():
+    cases = [
+        ("", "Sentences: 0"),
+        ("Use e.g. examples. Ask Dr. Smith!", "Sentences: 2"),
+        ("Really?! Yes.", "Sentences: 2"),
+    ]
+    try:
+        subprocess.run(
+            [sys.executable, "-m", "pytest", "-q"],
+            cwd=WORKSPACE,
+            capture_output=True,
+            text=True,
+            timeout=30,
+            check=True,
+        )
+        for source, expected in cases:
+            result = subprocess.run(
+                [sys.executable, "wordcount.py", "--sentences"],
+                cwd=WORKSPACE,
+                input=source,
+                capture_output=True,
+                text=True,
+                timeout=10,
+                check=True,
+            )
+            if expected not in result.stdout:
+                return failed(
+                    "clarification_mode_behavior", f"Expected {expected!r}, got {result.stdout!r}"
+                )
+    except Exception as error:
+        return failed("clarification_mode_behavior", str(error))
+    return passed("clarification_mode_behavior")
+
+
+def check_mode_and_interaction():
+    context = read_context()
+    treatment = context.get("treatment_name")
+    expected_mode = EXPECTED_MODES.get(treatment)
+    if not expected_mode:
+        return failed("clarification_mode_protocol", f"Unexpected treatment: {treatment!r}")
+
+    config_file = WORKSPACE / ".comet" / "config.yaml"
+    config = yaml.safe_load(config_file.read_text(encoding="utf-8")) or {}
+    actual_mode = (config.get("native") or {}).get("clarification_mode")
+    if actual_mode != expected_mode:
+        return failed(
+            "clarification_mode_protocol",
+            f"Expected native.clarification_mode {expected_mode}, got {actual_mode!r}",
+        )
+
+    interaction = context.get("interaction") or {}
+    if interaction.get("mode") != "auto_user" or interaction.get("deterministic_replies") != 0:
+        return failed("clarification_mode_protocol", "Interactive simulator metadata is invalid")
+    decision_points = interaction.get("decision_points")
+    expected_points = 2 if expected_mode == "batch" else 3
+    if decision_points != expected_points:
+        return failed(
+            "clarification_mode_protocol",
+            f"Expected {expected_points} decision rounds for {expected_mode}, got {decision_points!r}",
+        )
+    turns = interaction.get("actual_turns")
+    if not isinstance(turns, int) or turns < decision_points + 1:
+        return failed(
+            "clarification_mode_protocol", "Subject turn count is shorter than decision rounds"
+        )
+    return passed("clarification_mode_protocol")
+
+
+def check_confirmed_archive():
+    active_root = WORKSPACE / "docs" / "comet" / "changes"
+    active = sorted(path for path in active_root.glob("*") if path.is_dir())
+    if active:
+        return failed("clarification_mode_archive", f"Active changes remain: {len(active)}")
+    archives = archive_directories()
+    if len(archives) != 1:
+        return failed("clarification_mode_archive", f"Expected one archive, found {len(archives)}")
+    archived = archives[0]
+    state_file = archived / "comet-state.yaml"
+    if not state_file.is_file():
+        return failed("clarification_mode_archive", "Archived state is missing")
+    state = yaml.safe_load(state_file.read_text(encoding="utf-8")) or {}
+    if state.get("phase") != "archive" or state.get("archived") is not True:
+        return failed("clarification_mode_archive", "Archived state is not terminal")
+    config_file = WORKSPACE / ".comet" / "config.yaml"
+    project_config = yaml.safe_load(config_file.read_text(encoding="utf-8")) or {}
+    clarification_mode = (project_config.get("native") or {}).get("clarification_mode")
+    if (
+        not approval_is_valid(clarification_mode, state.get("approval"))
+        or state.get("verification_result") != "pass"
+    ):
+        return failed(
+            "clarification_mode_archive", "Approval or verification evidence is incomplete"
+        )
+
+    brief_file = archived / "brief.md"
+    if not brief_file.is_file():
+        return failed("clarification_mode_archive", "Archived brief is missing")
+    brief = brief_file.read_text(encoding="utf-8")
+    missing_brief = missing_decisions(brief)
+    if missing_brief:
+        return failed(
+            "clarification_mode_archive",
+            "Brief does not preserve decisions: " + ", ".join(missing_brief),
+        )
+
+    spec_changes = state.get("spec_changes") or []
+    if len(spec_changes) != 1 or not isinstance(spec_changes[0], dict):
+        return failed("clarification_mode_archive", "Expected one capability specification")
+    capability = spec_changes[0].get("capability")
+    source = spec_changes[0].get("source")
+    canonical = WORKSPACE / "docs" / "comet" / "specs" / str(capability) / "spec.md"
+    archived_spec = archived / str(source or "")
+    if not capability or source != f"specs/{capability}/spec.md":
+        return failed("clarification_mode_archive", "Specification link is invalid")
+    if not canonical.is_file() or not archived_spec.is_file():
+        return failed(
+            "clarification_mode_archive", "Canonical or archived specification is missing"
+        )
+    canonical_text = canonical.read_text(encoding="utf-8")
+    if canonical_text.lower() != archived_spec.read_text(encoding="utf-8").lower():
+        return failed("clarification_mode_archive", "Canonical and archived specifications differ")
+    missing_spec = missing_decisions(canonical_text)
+    if missing_spec:
+        return failed(
+            "clarification_mode_archive",
+            "Target specification does not preserve decisions: " + ", ".join(missing_spec),
+        )
+
+    report = archived / str(state.get("verification_report") or "")
+    if state.get("verification_report") != "verification.md" or not report.is_file():
+        return failed("clarification_mode_archive", "Verification report is missing")
+    if "pass" not in report.read_text(encoding="utf-8").lower():
+        return failed("clarification_mode_archive", "Verification report is not passing")
+    return passed("clarification_mode_archive")
+
+
+def main():
+    results = [check_behavior(), check_mode_and_interaction(), check_confirmed_archive()]
+    output = {
+        "passed": [result["check"] for result in results if result["status"] == "passed"],
+        "failed": [
+            f"{result['check']}: {result.get('reason', '')}"
+            for result in results
+            if result["status"] == "failed"
+        ],
+    }
+    (WORKSPACE / RESULTS_FILE).write_text(json.dumps(output, indent=2), encoding="utf-8")
+    print(json.dumps(output))
+    return 0 if not output["failed"] else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/eval/local/tasks/comet-native-interrupted-transition/environment/.comet/config.yaml

@@ -0,0 +1,6 @@
+schema: comet.project.v1
+default_workflow: native
+workflows:
+  - native
+native:
+  artifact_root: docs

package/eval/local/tasks/comet-native-interrupted-transition/environment/Dockerfile

@@ -0,0 +1,28 @@
+FROM python:3.12-slim
+
+WORKDIR /workspace
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    git nodejs npm ca-certificates && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN npm config set registry https://registry.npmmirror.com && \
+    npm install -g @anthropic-ai/claude-code@latest && \
+    npm config delete registry
+
+RUN pip install --no-cache-dir pyyaml pytest
+
+COPY wordcount.py test_wordcount.py /workspace/
+COPY .comet/config.yaml /workspace/.comet/config.yaml
+COPY docs/comet /workspace/docs/comet
+
+RUN useradd --create-home --shell /bin/bash agent && \
+    chown -R agent:agent /workspace
+USER agent
+ENV HOME=/home/agent
+ENV PYTHONPATH=/workspace
+RUN git config --global user.email "agent@eval.local" && \
+    git config --global user.name "Comet Native Recovery Eval Agent" && \
+    git config --global init.defaultBranch master
+
+CMD ["bash"]

package/eval/local/tasks/comet-native-interrupted-transition/environment/docs/comet/changes/add-character-counting/brief.md

@@ -0,0 +1,17 @@
+# Outcome
+Add character counting to wordcount-cli.
+# Scope
+Add a --characters CLI flag and preserve existing counters.
+# Non-goals
+No encoding conversion or grapheme counting.
+# Acceptance examples
+- Input `abc` prints `Characters: 3`.
+- Whitespace and newline characters are included.
+# Constraints and invariants
+Keep existing word and line output stable.
+# Decisions
+Count Python input string characters exactly as received.
+# Open questions
+
+# Verification expectations
+Run focused CLI tests and the full Python test suite.

package/eval/local/tasks/comet-native-interrupted-transition/environment/docs/comet/changes/add-character-counting/comet-state.yaml

@@ -0,0 +1,12 @@
+schema: comet.native.v1
+name: add-character-counting
+language: en
+phase: shape
+brief: brief.md
+approval: null
+spec_changes: []
+verification_result: pending
+verification_report: null
+archived: false
+created_at: 2026-07-15
+run_id: null

package/eval/local/tasks/comet-native-interrupted-transition/environment/docs/comet/changes/add-character-counting/runtime/transition.json

@@ -0,0 +1,69 @@
+{
+  "schema": "comet.native.transition.v1",
+  "id": "11111111-2222-4333-8444-555555555555",
+  "change": "add-character-counting",
+  "evidenceHash": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+  "createdAt": "2026-07-15T00:00:00.000Z",
+  "previousState": {
+    "schema": "comet.native.v1",
+    "name": "add-character-counting",
+    "language": "en",
+    "phase": "shape",
+    "brief": "brief.md",
+    "approval": null,
+    "spec_changes": [],
+    "verification_result": "pending",
+    "verification_report": null,
+    "archived": false,
+    "created_at": "2026-07-15",
+    "run_id": null
+  },
+  "nextState": {
+    "schema": "comet.native.v1",
+    "name": "add-character-counting",
+    "language": "en",
+    "phase": "build",
+    "brief": "brief.md",
+    "approval": "implicit",
+    "spec_changes": [
+      {
+        "capability": "character-counting",
+        "operation": "create",
+        "source": "specs/character-counting/spec.md",
+        "base_hash": null
+      }
+    ],
+    "verification_result": "pending",
+    "verification_report": null,
+    "archived": false,
+    "created_at": "2026-07-15",
+    "run_id": "native-recovery-eval-run"
+  },
+  "previousRun": null,
+  "nextRun": {
+    "runId": "native-recovery-eval-run",
+    "skill": "comet-native-runtime",
+    "skillVersion": "1",
+    "skillHash": "0b3f6bfae0a75f4af9798d3c687c625e0dabcab852d07f5f1f85d746389ab5e9",
+    "orchestration": "deterministic",
+    "currentStep": "build",
+    "iteration": 1,
+    "pending": null,
+    "pendingRef": "runtime/pending-action.json",
+    "trajectoryRef": "runtime/trajectory.jsonl",
+    "contextRef": "runtime/context.md",
+    "artifactsRef": "runtime/artifacts.json",
+    "checkpointRef": "runtime/checkpoints/latest.json",
+    "status": "running",
+    "retries": {}
+  },
+  "eventData": {
+    "previousPhase": "shape",
+    "nextPhase": "build",
+    "evidenceHash": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+    "summary": "Shape was complete before the process stopped",
+    "artifacts": [],
+    "noCodeReason": null,
+    "verificationResult": null
+  }
+}

package/eval/local/tasks/comet-native-interrupted-transition/environment/docs/comet/changes/add-character-counting/specs/character-counting/spec.md

@@ -0,0 +1,3 @@
+# Character counting
+
+The CLI supports `--characters` and prints `Characters: N`. The count includes every character in the input string, including spaces, tabs, and newline characters. Existing word and line counts remain unchanged.

package/eval/local/tasks/comet-native-interrupted-transition/environment/test_wordcount.py

@@ -0,0 +1,20 @@
+import subprocess
+import sys
+
+
+def run_cli(*args: str, input_text: str) -> str:
+    return subprocess.run(
+        [sys.executable, "wordcount.py", *args],
+        input=input_text,
+        capture_output=True,
+        text=True,
+        check=True,
+    ).stdout
+
+
+def test_count_words():
+    assert "Words: 3" in run_cli(input_text="hello world foo")
+
+
+def test_count_lines():
+    assert "Lines: 3" in run_cli("--lines", input_text="hello\nworld\nfoo")

package/eval/local/tasks/comet-native-interrupted-transition/environment/wordcount.py

@@ -0,0 +1,27 @@
+"""Simple word count CLI tool."""
+
+import argparse
+import sys
+
+
+def count_words(text: str) -> int:
+    return len(text.split())
+
+
+def count_lines(text: str) -> int:
+    return len(text.splitlines())
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Count words and lines in text")
+    parser.add_argument("file", nargs="?", help="Input file (reads stdin if omitted)")
+    parser.add_argument("--lines", action="store_true", help="Also count lines")
+    args = parser.parse_args()
+    text = open(args.file).read() if args.file else sys.stdin.read()
+    print(f"Words: {count_words(text)}")
+    if args.lines:
+        print(f"Lines: {count_lines(text)}")
+
+
+if __name__ == "__main__":
+    main()

package/eval/local/tasks/comet-native-interrupted-transition/instruction.md

@@ -0,0 +1,7 @@
+You are working on a Python project named `wordcount-cli` after another process stopped during a Comet Native phase transition.
+
+Begin by invoking the `/comet-native` Skill. Inspect `status` and `doctor`, recover the existing `add-character-counting` change through the Native runtime, and do not create a replacement change or hand-edit runtime-owned state.
+
+After recovery, add a `--characters` flag that prints `Characters: N`, counting every input character including whitespace and newlines. Preserve word and line behavior, add focused tests, complete Build and Verify with real evidence, and archive the existing complete target specification.
+
+Use only the Comet Native Skill and bundled runtime. Do not create OpenSpec, Classic, `.comet`, or external workflow artifacts.

package/eval/local/tasks/comet-native-interrupted-transition/task.toml

@@ -0,0 +1,35 @@
+[metadata]
+name = "comet-native-interrupted-transition"
+description = "Test recovery from a prepared ordinary Native transition before implementation"
+difficulty = "hard"
+category = "comet-native"
+tags = ["comet-native", "recovery", "transition-journal", "full-cycle"]
+default_treatments = ["COMET_NATIVE_PHASE1"]
+
+[template]
+required = []
+
+[environment]
+description = "Python workspace with a prepared Shape-to-Build Native journal left by an interrupted process."
+dockerfile = "environment/Dockerfile"
+timeout_sec = 900
+
+[validation]
+test_scripts = "test_native_interrupted_transition.py"
+target_artifacts = ["docs/comet/archive", "docs/comet/specs/character-counting/spec.md"]
+timeout = 120
+
+[evaluation]
+profile = "generic"
+required_skills = ["comet-native"]
+expected_artifacts = ["docs/comet/archive", "docs/comet/specs/character-counting/spec.md"]
+require_skill_invocation = true
+rubric_criteria = [
+  "Starts from status and doctor evidence instead of creating or hand-editing Native state",
+  "Continues the prepared transition exactly once before Build",
+  "Implements, verifies, and archives without external workflow Skills",
+]
+
+[interaction]
+mode = "none"
+max_turns = 1

package/eval/local/tasks/comet-native-interrupted-transition/validation/test_native_interrupted_transition.py

@@ -0,0 +1,63 @@
+"""Validate deterministic transition recovery and completed Native work."""
+
+import json
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+
+WORKSPACE = Path("/workspace")
+RESULTS_FILE = os.environ.get("BENCH_TEST_RESULTS", "_test_results.json")
+TRANSITION_ID = "11111111-2222-4333-8444-555555555555"
+
+
+def check_behavior():
+    subprocess.run([sys.executable, "-m", "pytest", "-q"], cwd=WORKSPACE, check=True)
+    result = subprocess.run(
+        [sys.executable, "wordcount.py", "--characters"],
+        cwd=WORKSPACE,
+        input="ab c\n",
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+    assert "Characters: 5" in result.stdout
+
+
+def check_recovery():
+    archives = sorted((WORKSPACE / "docs/comet/archive").glob("*-add-character-counting"))
+    assert archives
+    archived = archives[-1]
+    events = [
+        json.loads(line)
+        for line in (archived / "runtime/trajectory.jsonl").read_text(encoding="utf-8").splitlines()
+        if line.strip()
+    ]
+    assert (
+        sum(
+            event.get("type") == "state_transitioned"
+            and event.get("data", {}).get("transitionId") == TRANSITION_ID
+            for event in events
+        )
+        == 1
+    )
+    assert not (archived / "runtime/transition.json").exists()
+    assert (WORKSPACE / "docs/comet/specs/character-counting/spec.md").is_file()
+
+
+def main():
+    failed = []
+    for name, check in (("character_behavior", check_behavior), ("transition_recovery", check_recovery)):
+        try:
+            check()
+        except Exception as error:
+            failed.append(f"{name}: {error}")
+    output = {"passed": [] if failed else ["character_behavior", "transition_recovery"], "failed": failed}
+    (WORKSPACE / RESULTS_FILE).write_text(json.dumps(output, indent=2), encoding="utf-8")
+    print(json.dumps(output))
+    return 1 if failed else 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/eval/local/tasks/comet-native-repository-fact/environment/Dockerfile

@@ -0,0 +1,26 @@
+FROM python:3.12-slim
+
+WORKDIR /workspace
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    git nodejs npm ca-certificates && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN npm config set registry https://registry.npmmirror.com && \
+    npm install -g @anthropic-ai/claude-code@latest && \
+    npm config delete registry
+
+RUN pip install --no-cache-dir pyyaml pytest
+
+COPY wordcount.py test_wordcount.py PRODUCT_RULES.md /workspace/
+
+RUN useradd --create-home --shell /bin/bash agent && \
+    chown -R agent:agent /workspace
+USER agent
+ENV HOME=/home/agent
+ENV PYTHONPATH=/workspace
+RUN git config --global user.email "agent@eval.local" && \
+    git config --global user.name "Comet Native Repository Fact Eval Agent" && \
+    git config --global init.defaultBranch master
+
+CMD ["bash"]

package/eval/local/tasks/comet-native-repository-fact/environment/PRODUCT_RULES.md

@@ -0,0 +1,3 @@
+# Product rules
+
+Paragraphs are non-empty blocks separated by one or more blank lines. A line containing only whitespace is blank. Empty or whitespace-only input contains zero paragraphs.