@rpamis/comet 0.4.0-beta.6 → 0.4.0-beta.8

package/eval/local/README.md

@@ -17,6 +17,24 @@ uv run pytest local/tests/tasks/test_tasks.py -v
 uv run pytest local/tests/tasks/test_tasks.py --task=generic-skill-smoke --treatment=CONTROL -v
 ```
 
+运行只安装 Comet Native、自包含且不注入 OpenSpec / Superpowers 的完整流程评估：
+
+```bash
+uv run pytest local/tests/tasks/test_tasks.py \
+  --task=comet-native-workflow \
+  --treatment=COMET_NATIVE_PHASE1 -v
+```
+
+将 Native 与 0.4.0 基线按相同的 16 个 canonical Comet 任务、每任务 3 次对齐运行：
+
+```bash
+uv run pytest local/tests/tasks/test_tasks.py::test_task_treatment \
+  --treatment=COMET_NATIVE_PHASE1 --count=3 -n 4 \
+  -k "not comet-native and not generic-skill-smoke and not authoring-skill-smoke and not workflow-route-conformance and not workflow-overlay-contract" -v
+```
+
+对齐评估保留每个任务原有的业务 validator，但会把 Classic/OpenSpec 专属流程检查替换为 Native 的 Skill 隔离、配置、终态 change、验证归档和安全 trajectory 检查。调用 `/comet` 或其他外部 Skill、留下活动 change、缺少完整规格/验证证据，都会作为真实 Native workflow failure 进入 pass@k，而不会被当成环境噪声排除。
+
 运行生成 Skill 的 authoring smoke 任务：
 
 ```bash
@@ -73,6 +91,38 @@ uv run pytest local/tests/tasks/test_tasks.py --report-config report-config.json
 uv run python local/scripts/compare_baselines.py --report-config report-config.json
 ```
 
+Native 与 0.4.0 的 pass@3 来自两个独立 experiment 时，使用严格对齐模式：
+
+```bash
+uv run python local/scripts/compare_baselines.py \
+  --candidate-experiment local/logs/experiments/experiment_NATIVE \
+  --candidate-treatment COMET_NATIVE_PHASE1 \
+  --baseline-experiment langsmith/logs/experiments/experiment_040_PASS3 \
+  --baseline-treatment COMET_FULL_040_BETA \
+  --ks 1,2,3
+```
+
+pytest 会在模型运行前把收集到的 `task + treatment + repetition` 写入 experiment 的
+expected case matrix；严格比较按两侧目标 treatment 的矩阵检查 coverage，因此即使某个
+样本在两侧都没有留下 report，也会被列为缺失，而不是从报告并集中无声消失。对 v2
+报告，只有 `task + repetition + case_hash` 全部一致才会配对；`case_hash` 同时绑定任务、
+验证器、runner/controller 源码、不可变 Docker image、Claude 工具版本以及 model /
+interaction 配置。某个 task 缺少 repetition 时，报告会降低该 k 的 coverage 并列出
+缺失项，不会把 pass@3 偷换成 pass@2。
+
+历史耗时会从 raw stdout 按当前的累加 parser 重算；缺少 raw duration 的运行只计入
+缺失 coverage，不会和已有耗时混合平均。没有 expected matrix 的旧 experiment 会明确
+标为 observed-report fallback；v1 或缺失 case manifest 的运行最多只能按任务核心 hash
+兼容比较，并明确标注无法证明当时的 runner、image、tool、model 或 interaction 身份，
+不会把比较时一致误写成历史 checkout 与执行环境已被精确证明一致。
+
+严格对齐报告还会从 raw stdout 重算成对效率指标，包括模型启动/恢复次数、Agent 轮次、
+工具调用、累计耗时、输入/输出/cache token、总 token、模型成本和上下文压力。主要视图只
+统计两侧都通过的同一 `task + repetition + case_hash`，避免把未完成任务的低消耗误当成
+效率提升。多次恢复产生的顶层 `result` 按调用累加；流式 assistant 事件按 message id 去重。
+上下文指标只有在两侧都保存逐消息 usage 时才可比较，coverage 过低时报告会明确提示，不能
+据此宣传 workflow 间的上下文改进。
+
 也可以设置 `COMET_EVAL_REPORT_CONFIG=/path/to/report-config.json`。
 
 每次运行的报告都会包含 profile、Skill 来源元数据、run id、产物引用，以及结构化失败归因。归因桶包括 `harness`、`workflow`、`task` 和 `model`。

package/eval/local/scripts/compare_baselines.py

@@ -1,3 +1,4 @@
+# ruff: noqa: E402
 """Compare comet baseline treatments across the rubric dimensions.
 
 Reads experiment reports from ``local/logs/experiments/<id>/reports/*.json`` and
@@ -31,8 +32,10 @@ if hasattr(sys.stdout, "reconfigure"):
 if hasattr(sys.stderr, "reconfigure"):
     sys.stderr.reconfigure(encoding="utf-8")
 
+from scaffold.python.aligned_comparison import build_aligned_report
 from scaffold.python.paths import get_logs_dir
 from scaffold.python.report_outputs import (
+    localize_eval_markdown,
     load_report_output_config,
     write_report_outputs,
 )
@@ -59,6 +62,7 @@ RUBRIC_DIMENSION_GUIDE = {
 # Treatments we compare. CONTROL is included for context but not used in the
 # pass/fail decision.
 TREATMENTS = ("CONTROL", "COMET_FULL_040_BETA", "COMET_FULL_039")
+KNOWN_TREATMENTS = (*TREATMENTS, "COMET_NATIVE_PHASE1")
 WORKFLOW = "COMET_FULL_040_BETA"
 BASELINE = "COMET_FULL_039"
 
@@ -72,7 +76,7 @@ class ReportPartitions:
 
 
 def _treatment_from_report_name(raw_name: str) -> str:
-    for treatment in sorted(TREATMENTS, key=len, reverse=True):
+    for treatment in sorted(KNOWN_TREATMENTS, key=len, reverse=True):
         if re.search(rf"(?:^|-){re.escape(treatment)}(?:-r\d+)?$", raw_name):
             return treatment
     return raw_name
@@ -86,6 +90,35 @@ def _latest_experiment(logs: Path) -> Path | None:
     return dirs[-1] if dirs else None
 
 
+def _resolve_experiment(value: str, logs: Path) -> Path:
+    """Resolve an experiment id or an explicit local/langsmith experiment path."""
+    supplied = Path(value).expanduser()
+    candidates = [
+        supplied,
+        EVAL_ROOT / supplied,
+        logs / "experiments" / value,
+        EVAL_ROOT / "local" / "logs" / "experiments" / value,
+        EVAL_ROOT / "langsmith" / "logs" / "experiments" / value,
+    ]
+    matches: list[Path] = []
+    seen: set[Path] = set()
+    for candidate in candidates:
+        try:
+            resolved = candidate.resolve()
+        except OSError:
+            continue
+        if resolved in seen or not (resolved / "reports").is_dir():
+            continue
+        seen.add(resolved)
+        matches.append(resolved)
+    if not matches:
+        raise FileNotFoundError(f"Experiment not found: {value}")
+    if len(matches) > 1:
+        rendered = ", ".join(str(path) for path in matches)
+        raise ValueError(f"Experiment id is ambiguous; pass an explicit path: {rendered}")
+    return matches[0]
+
+
 def _run_passed(report: dict) -> bool:
     """A run 'passes' when its task-level validator has zero failures.
 
@@ -984,9 +1017,102 @@ def main(argv: list[str] | None = None) -> int:
         help="Output path (defaults to <experiment>/comparison_report.md)",
     )
     parser.add_argument("--report-config", default=None, help="JSON/YAML config for report outputs")
+    parser.add_argument(
+        "--language",
+        choices=("en", "zh"),
+        default="en",
+        help="Language used by generated Markdown and HTML report bodies",
+    )
+    parser.add_argument(
+        "--candidate-experiment",
+        default=None,
+        help="Candidate experiment id or path for strict two-experiment alignment",
+    )
+    parser.add_argument(
+        "--baseline-experiment",
+        default=None,
+        help="Baseline experiment id or path for strict two-experiment alignment",
+    )
+    parser.add_argument(
+        "--candidate-treatment",
+        default="COMET_NATIVE_PHASE1",
+        help="Treatment selected from the candidate experiment",
+    )
+    parser.add_argument(
+        "--baseline-treatment",
+        default="COMET_FULL_040_BETA",
+        help="Treatment selected from the baseline experiment",
+    )
+    parser.add_argument(
+        "--ks",
+        default="1,2,3",
+        help="Comma-separated k values for strict task-macro pass metrics",
+    )
     args = parser.parse_args(argv)
 
     logs = get_logs_dir()
+    aligned_mode = bool(args.candidate_experiment or args.baseline_experiment)
+    if aligned_mode:
+        if not args.candidate_experiment or not args.baseline_experiment:
+            print(
+                "--candidate-experiment and --baseline-experiment must be provided together",
+                file=sys.stderr,
+            )
+            return 2
+        if args.experiment:
+            print(
+                "--experiment cannot be combined with two-experiment alignment",
+                file=sys.stderr,
+            )
+            return 2
+        try:
+            ks = tuple(int(item.strip()) for item in args.ks.split(",") if item.strip())
+        except ValueError:
+            print("--ks must be a comma-separated list of integers", file=sys.stderr)
+            return 2
+        if not ks or any(k < 1 for k in ks):
+            print("--ks must contain positive integers", file=sys.stderr)
+            return 2
+        try:
+            candidate_dir = _resolve_experiment(args.candidate_experiment, logs)
+            baseline_dir = _resolve_experiment(args.baseline_experiment, logs)
+        except (FileNotFoundError, ValueError) as exc:
+            print(str(exc), file=sys.stderr)
+            return 1
+        try:
+            report = build_aligned_report(
+                candidate_dir,
+                baseline_dir,
+                candidate_treatment=args.candidate_treatment,
+                baseline_treatment=args.baseline_treatment,
+                tasks_dir=EVAL_ROOT / "local" / "tasks",
+                ks=ks,
+            )
+        except ValueError as exc:
+            print(str(exc), file=sys.stderr)
+            return 1
+        html_report = report
+        if args.language == "zh":
+            report = localize_eval_markdown(report)
+        out_path = Path(args.out) if args.out else candidate_dir / "aligned_comparison_report.md"
+        outputs = write_report_outputs(
+            report,
+            out_path,
+            load_report_output_config(args.report_config),
+            title=(
+                "Comet 对齐实验对比报告"
+                if args.language == "zh"
+                else "Comet Aligned Experiment Comparison Report"
+            ),
+            html_markdown=html_report,
+        )
+        print(report)
+        if outputs:
+            print("\nWrote: " + ", ".join(str(path) for path in outputs.values()))
+        else:
+            print("\nReport outputs disabled by report config")
+        return 0
+
     if args.experiment:
         experiment_dir = logs / "experiments" / args.experiment
         if not experiment_dir.exists():

package/eval/local/scripts/regression_check.py

@@ -33,9 +33,9 @@ EVAL_ROOT = Path(__file__).resolve().parents[2]
 if str(EVAL_ROOT) not in sys.path:
     sys.path.insert(0, str(EVAL_ROOT))
 
-from scaffold.python.paths import get_logs_dir
-from scaffold.python.validation.rubric import RUBRIC_DIMENSIONS
-from local.scripts.compare_baselines import (
+from scaffold.python.paths import get_logs_dir  # noqa: E402
+from scaffold.python.validation.rubric import RUBRIC_DIMENSIONS  # noqa: E402
+from local.scripts.compare_baselines import (  # noqa: E402
     BASELINE,
     WORKFLOW,
     _aggregate,
@@ -85,7 +85,6 @@ def _run_eval(count: int, treatments: str) -> str:
         f"--count={count}",
         "-v",
     ]
-    env_runner = "BENCH_LOOP_MAX_TURNS"
     print(f"[regression] running eval: {' '.join(cmd)}", file=sys.stderr)
     result = subprocess.run(cmd, capture_output=False)
     if result.returncode != 0:

package/eval/local/tasks/comet-native-clarification/environment/Dockerfile

@@ -0,0 +1,27 @@
+FROM python:3.12-slim
+
+WORKDIR /workspace
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    git nodejs npm ca-certificates && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN npm config set registry https://registry.npmmirror.com && \
+    npm install -g @anthropic-ai/claude-code@latest && \
+    npm config delete registry
+
+RUN pip install --no-cache-dir pyyaml pytest
+
+COPY wordcount.py /workspace/wordcount.py
+COPY test_wordcount.py /workspace/test_wordcount.py
+
+RUN useradd --create-home --shell /bin/bash agent && \
+    chown -R agent:agent /workspace
+USER agent
+ENV HOME=/home/agent
+ENV PYTHONPATH=/workspace
+RUN git config --global user.email "agent@eval.local" && \
+    git config --global user.name "Comet Native Clarification Eval Agent" && \
+    git config --global init.defaultBranch master
+
+CMD ["bash"]

package/eval/local/tasks/comet-native-clarification/environment/test_wordcount.py

@@ -0,0 +1,25 @@
+"""Baseline tests for the word-count CLI."""
+
+import subprocess
+import sys
+
+
+def run_cli(*args: str, input_text: str) -> str:
+    result = subprocess.run(
+        [sys.executable, "wordcount.py", *args],
+        input=input_text,
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+    return result.stdout
+
+
+def test_count_words():
+    assert "Words: 3" in run_cli(input_text="hello world foo")
+
+
+def test_count_lines():
+    output = run_cli("--lines", input_text="hello\nworld\nfoo")
+    assert "Words: 3" in output
+    assert "Lines: 3" in output

package/eval/local/tasks/comet-native-clarification/environment/wordcount.py

@@ -0,0 +1,36 @@
+"""Simple word count CLI tool."""
+
+import argparse
+import sys
+
+
+def count_words(text: str) -> int:
+    """Count words in text."""
+    return len(text.split())
+
+
+def count_lines(text: str) -> int:
+    """Count lines in text."""
+    return len(text.splitlines())
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Count words and lines in text")
+    parser.add_argument("file", nargs="?", help="Input file (reads stdin if omitted)")
+    parser.add_argument("--lines", action="store_true", help="Also count lines")
+    parser.add_argument("--words", action="store_true", default=True, help="Count words (default)")
+    args = parser.parse_args()
+
+    if args.file:
+        with open(args.file) as source:
+            text = source.read()
+    else:
+        text = sys.stdin.read()
+
+    print(f"Words: {count_words(text)}")
+    if args.lines:
+        print(f"Lines: {count_lines(text)}")
+
+
+if __name__ == "__main__":
+    main()

package/eval/local/tasks/comet-native-clarification/instruction.md

@@ -0,0 +1,7 @@
+You are working on a Python project named `wordcount-cli`.
+
+Begin by invoking the `/comet-native` Skill. Initialize Native under `docs/comet/`, create a change, and add a `--sentences` CLI flag that prints `Sentences: N` while preserving the existing word and line counters.
+
+Sentence terminators include `.`, `!`, and `?`. Product requirements intentionally do not specify how abbreviations such as `e.g.` and `Dr.` affect sentence boundaries. Do not guess how abbreviations should behave. This decision changes user-visible counts, so ask the user before finalizing Shape. Ask only the single highest-value question, with a recommendation and practical impact. Do not begin implementation before the user answers.
+
+After the answer, record it in the brief, use Native's explicit confirmation path, write a complete target specification, implement focused tests including abbreviations, verify, and archive. Use no other workflow Skill.

package/eval/local/tasks/comet-native-clarification/task.toml

@@ -0,0 +1,40 @@
+[metadata]
+name = "comet-native-clarification"
+description = "Test the Native one-question clarification boundary and confirmed resume"
+difficulty = "hard"
+category = "comet-native"
+tags = ["comet-native", "clarification", "confirmation", "multi-turn"]
+default_treatments = ["COMET_NATIVE_PHASE1"]
+
+[template]
+required = []
+
+[environment]
+description = "Python word-count workspace with only the self-contained Comet Native Skill available."
+dockerfile = "environment/Dockerfile"
+timeout_sec = 900
+
+[validation]
+test_scripts = "test_native_clarification.py"
+target_artifacts = ["docs/comet/archive", "docs/comet/specs"]
+timeout = 120
+
+[evaluation]
+profile = "generic"
+required_skills = ["comet-native"]
+expected_artifacts = [".comet/config.yaml", "docs/comet/archive", "docs/comet/specs"]
+require_skill_invocation = true
+rubric_criteria = [
+  "Before implementation, asks exactly one highest-value question about abbreviation behavior",
+  "Does not ask the user for repository facts or implementation-method choices",
+  "Records the supplied abbreviation decision in the brief and leaves Shape with explicit confirmation",
+  "Completes Build, Verify, and Archive using only the Comet Native Skill and bundled runtime",
+]
+
+[interaction]
+mode = "auto_user"
+max_turns = 4
+simulator_prompt = """You are the developer user in a Comet Native eval. The assistant should ask one highest-value question about how abbreviations affect sentence boundaries. Answer that question in one sentence: Abbreviations such as e.g. and Dr. do not end a sentence; use a small explicit abbreviation list. If the assistant asks multiple questions, answer only the abbreviation question and remind it to handle one decision at a time. Do not write code or choose implementation tooling."""
+decision_patterns = ["abbreviation", "e.g.", "Dr.", "sentence boundary"]
+decision_reply = "Abbreviations such as e.g. and Dr. do not end a sentence; use a small explicit abbreviation list."
+continue_prompt = "Please verify the Native status from disk and continue until the same change is actually archived with its canonical specification."

package/eval/local/tasks/comet-native-clarification/validation/test_native_clarification.py

@@ -0,0 +1,193 @@
+"""Validate Native clarification, confirmation, implementation, and archive artifacts."""
+
+import json
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+import yaml
+
+
+WORKSPACE = Path("/workspace")
+RESULTS_FILE = os.environ.get("BENCH_TEST_RESULTS", "_test_results.json")
+
+
+def passed(name: str):
+    return {"check": name, "status": "passed"}
+
+
+def failed(name: str, reason: str):
+    return {"check": name, "status": "failed", "reason": reason}
+
+
+def archive_directories():
+    root = WORKSPACE / "docs" / "comet" / "archive"
+    return sorted(path for path in root.glob("*-*") if path.is_dir())
+
+
+def archive_directory():
+    candidates = archive_directories()
+    return candidates[-1] if candidates else None
+
+
+def canonical_spec_files():
+    root = WORKSPACE / "docs" / "comet" / "specs"
+    return sorted(root.glob("*/spec.md"))
+
+
+def active_change_directories():
+    root = WORKSPACE / "docs" / "comet" / "changes"
+    return sorted(path for path in root.glob("*") if path.is_dir())
+
+
+def read_interaction():
+    context = WORKSPACE / "_test_context.json"
+    if not context.is_file():
+        return {}
+    return json.loads(context.read_text(encoding="utf-8")).get("interaction") or {}
+
+
+def check_behavior():
+    try:
+        subprocess.run(
+            [sys.executable, "-m", "pytest", "-q"],
+            cwd=WORKSPACE,
+            capture_output=True,
+            text=True,
+            timeout=30,
+            check=True,
+        )
+        result = subprocess.run(
+            [sys.executable, "wordcount.py", "--sentences"],
+            cwd=WORKSPACE,
+            input="Use e.g. examples. Ask Dr. Smith!",
+            capture_output=True,
+            text=True,
+            timeout=10,
+            check=True,
+        )
+    except Exception as error:
+        return failed("clarified_behavior", str(error))
+    if "Sentences: 2" not in result.stdout:
+        return failed("clarified_behavior", f"Expected Sentences: 2, got {result.stdout!r}")
+    return passed("clarified_behavior")
+
+
+def check_confirmed_archive():
+    active = active_change_directories()
+    if active:
+        return failed(
+            "confirmed_archive",
+            f"Expected no active change after archive, found {len(active)}",
+        )
+    archives = archive_directories()
+    if len(archives) != 1:
+        return failed(
+            "confirmed_archive",
+            f"Expected exactly one Native archive for the clarified change, found {len(archives)}",
+    )
+    archived = archives[0]
+    state_file = archived / "comet-state.yaml"
+    if not state_file.is_file():
+        return failed("confirmed_archive", "Archived comet-state.yaml is missing")
+    state = yaml.safe_load(state_file.read_text(encoding="utf-8")) or {}
+    if state.get("phase") != "archive" or state.get("archived") is not True:
+        return failed("confirmed_archive", "Archived state is not terminal")
+    if state.get("approval") != "confirmed":
+        return failed("confirmed_archive", "Shape did not record explicit confirmation")
+    if state.get("verification_result") != "pass":
+        return failed("confirmed_archive", "Verify did not record a passing result")
+    brief_file = archived / "brief.md"
+    if not brief_file.is_file():
+        return failed("confirmed_archive", "Archived brief is missing")
+    brief = brief_file.read_text(encoding="utf-8").lower()
+    if not all(marker in brief for marker in ("abbreviation", "e.g.", "dr.")):
+        return failed("confirmed_archive", "The confirmed abbreviation decision is missing")
+
+    spec_changes = state.get("spec_changes") or []
+    if len(spec_changes) != 1 or not isinstance(spec_changes[0], dict):
+        return failed("confirmed_archive", "Expected exactly one linked capability change")
+    spec_change = spec_changes[0]
+    capability = spec_change.get("capability")
+    expected_source = f"specs/{capability}/spec.md"
+    if not capability or spec_change.get("source") != expected_source:
+        return failed("confirmed_archive", "Archived spec change is not linked to its capability")
+
+    canonical_specs = canonical_spec_files()
+    canonical_path = WORKSPACE / "docs" / "comet" / "specs" / capability / "spec.md"
+    if canonical_specs != [canonical_path]:
+        return failed(
+            "confirmed_archive",
+            "Canonical specification does not match the archived capability",
+        )
+    archived_spec = archived / expected_source
+    if not archived_spec.is_file():
+        return failed("confirmed_archive", "Archived target specification is missing")
+    canonical_source = canonical_path.read_text(encoding="utf-8")
+    if canonical_source != archived_spec.read_text(encoding="utf-8"):
+        return failed("confirmed_archive", "Archived and canonical specifications differ")
+    canonical = canonical_source.lower()
+    if "--sentences" not in canonical or not any(
+        marker in canonical for marker in ("abbreviation", "e.g.", "dr.")
+    ):
+        return failed(
+            "confirmed_archive",
+            "The canonical target specification does not cover sentence counting and abbreviations",
+        )
+
+    report_name = state.get("verification_report")
+    report_file = archived / str(report_name or "")
+    if report_name != "verification.md" or not report_file.is_file():
+        return failed("confirmed_archive", "Verification evidence is missing")
+    report = report_file.read_text(encoding="utf-8").lower()
+    if "pass" not in report or "pytest" not in report:
+        return failed("confirmed_archive", "Verification evidence does not record passing tests")
+
+    trajectory_file = archived / "runtime" / "trajectory.jsonl"
+    if not trajectory_file.is_file():
+        return failed("confirmed_archive", "Native trajectory is missing")
+    transitions = []
+    for line in trajectory_file.read_text(encoding="utf-8").splitlines():
+        event = json.loads(line)
+        if event.get("type") == "state_transitioned":
+            data = event.get("data") or {}
+            transitions.append((data.get("previousPhase"), data.get("nextPhase")))
+    expected_transitions = [
+        ("shape", "build"),
+        ("build", "verify"),
+        ("verify", "archive"),
+        ("archive", None),
+    ]
+    if transitions != expected_transitions:
+        return failed("confirmed_archive", "Native trajectory does not cover every phase exactly once")
+
+    interaction = read_interaction()
+    if interaction.get("mode") != "auto_user":
+        return failed("confirmed_archive", "Clarification interaction metadata is missing")
+    if interaction.get("decision_points") != 1 or interaction.get("deterministic_replies") != 1:
+        return failed("confirmed_archive", "Expected exactly one deterministic clarification reply")
+    turns = interaction.get("actual_turns")
+    maximum = interaction.get("max_turns")
+    if not isinstance(turns, int) or not isinstance(maximum, int) or not (2 <= turns <= maximum):
+        return failed("confirmed_archive", "Clarification driver turn count is invalid")
+    return passed("confirmed_archive")
+
+
+def main():
+    results = [check_behavior(), check_confirmed_archive()]
+    output = {
+        "passed": [result["check"] for result in results if result["status"] == "passed"],
+        "failed": [
+            f'{result["check"]}: {result.get("reason", "")}'
+            for result in results
+            if result["status"] == "failed"
+        ],
+    }
+    (WORKSPACE / RESULTS_FILE).write_text(json.dumps(output, indent=2), encoding="utf-8")
+    print(json.dumps(output))
+    return 0 if not output["failed"] else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/eval/local/tasks/comet-native-clarification-modes/environment/Dockerfile

@@ -0,0 +1,27 @@
+FROM python:3.12-slim
+
+WORKDIR /workspace
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    git nodejs npm ca-certificates && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN npm config set registry https://registry.npmmirror.com && \
+    npm install -g @anthropic-ai/claude-code@latest && \
+    npm config delete registry
+
+RUN pip install --no-cache-dir pyyaml pytest
+
+COPY wordcount.py /workspace/wordcount.py
+COPY test_wordcount.py /workspace/test_wordcount.py
+
+RUN useradd --create-home --shell /bin/bash agent && \
+    chown -R agent:agent /workspace
+USER agent
+ENV HOME=/home/agent
+ENV PYTHONPATH=/workspace
+RUN git config --global user.email "agent@eval.local" && \
+    git config --global user.name "Comet Native Clarification Modes Eval Agent" && \
+    git config --global init.defaultBranch master
+
+CMD ["bash"]

package/eval/local/tasks/comet-native-clarification-modes/environment/test_wordcount.py

@@ -0,0 +1,25 @@
+"""Baseline tests for the word-count CLI."""
+
+import subprocess
+import sys
+
+
+def run_cli(*args: str, input_text: str) -> str:
+    result = subprocess.run(
+        [sys.executable, "wordcount.py", *args],
+        input=input_text,
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+    return result.stdout
+
+
+def test_count_words():
+    assert "Words: 3" in run_cli(input_text="hello world foo")
+
+
+def test_count_lines():
+    output = run_cli("--lines", input_text="hello\nworld\nfoo")
+    assert "Words: 3" in output
+    assert "Lines: 3" in output