@rpamis/comet 0.4.0-beta.5 → 0.4.0-beta.7

package/eval/local/tasks/comet-native-wave-e-parallel-safety/validation/test_native_wave_e_parallel_safety.py

@@ -0,0 +1,309 @@
+"""Validate early Native conflict/workspace status and one-winner CAS."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+import shutil
+import subprocess
+import sys
+import tempfile
+import time
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+from scaffold.python.validation.native_wave import (
+    active_changes,
+    archive_changes,
+    check_checkpoint_cas_envelopes,
+    check_native_isolation,
+    check_pytest,
+    check_runtime_envelopes,
+    failed,
+    passed,
+    read_json,
+    write_results,
+)
+
+
+WORKSPACE = Path("/workspace")
+EVIDENCE = Path(".cache/comet-native-eval")
+EXPECTED_CHANGES = {"normalize-case", "preserve-acronyms"}
+FORBIDDEN_STATUS_FIELDS = {
+    "worktreeId",
+    "commonDirId",
+    "sessionHash",
+    "workspaceIdentityHash",
+    "projectPrefix",
+    "projectRootId",
+    "nativeRootId",
+}
+
+BARRIER_WORKER = r"""
+import pathlib, subprocess, sys, time
+ready = pathlib.Path(sys.argv[1])
+release = pathlib.Path(sys.argv[2])
+ready.write_text("ready", encoding="utf-8")
+deadline = time.monotonic() + 10
+while not release.exists():
+    if time.monotonic() >= deadline:
+        raise SystemExit(124)
+    time.sleep(0.005)
+result = subprocess.run(sys.argv[3:], capture_output=True, text=True, check=False)
+sys.stdout.write(result.stdout)
+sys.stderr.write(result.stderr)
+raise SystemExit(result.returncode)
+"""
+
+
+def _finding_codes(value: Any) -> set[str]:
+    found: set[str] = set()
+    if isinstance(value, dict):
+        for key, child in value.items():
+            if key in {"code", "codes", "findingCodes"}:
+                if isinstance(child, str):
+                    found.add(child)
+                elif isinstance(child, list):
+                    found.update(item for item in child if isinstance(item, str))
+            found.update(_finding_codes(child))
+    elif isinstance(value, list):
+        for child in value:
+            found.update(_finding_codes(child))
+    return found
+
+
+def check_parallel_safety() -> dict[str, str]:
+    check = "parallel_safety"
+    active = {path.name for path in active_changes(WORKSPACE)}
+    if active != EXPECTED_CHANGES:
+        return failed(
+            check, f"Expected active changes {sorted(EXPECTED_CHANGES)}, found {sorted(active)}"
+        )
+    archived_names = {
+        (yaml.safe_load((path / "comet-state.yaml").read_text(encoding="utf-8")) or {}).get(
+            "name"
+        )
+        for path in archive_changes(WORKSPACE)
+    }
+    if archived_names & EXPECTED_CHANGES:
+        return failed(check, "Conflicting changes must remain active")
+
+    evidence = WORKSPACE / EVIDENCE
+    try:
+        status = read_json(evidence / "conflict-status.json")
+    except Exception as error:
+        return failed(check, f"Invalid conflict status: {error}")
+    if (
+        not isinstance(status, dict)
+        or status.get("command") != "status"
+        or status.get("exitCode") != 0
+        or not isinstance(status.get("data"), dict)
+        or status["data"].get("name") != "normalize-case"
+        or status["data"].get("phase") == "archive"
+    ):
+        return failed(check, "Conflict evidence is not an early exact Native status envelope")
+    codes = _finding_codes(status)
+    missing_codes = {"native-change-conflict"} - codes
+    if missing_codes:
+        return failed(check, f"Early status is missing findings: {sorted(missing_codes)}")
+    serialized = json.dumps(status, sort_keys=True)
+    leaked = sorted(field for field in FORBIDDEN_STATUS_FIELDS if field in serialized)
+    if leaked:
+        return failed(check, f"Status exposed raw workspace identity fields: {leaked}")
+
+    return passed(check)
+
+
+def _native_runtime_path() -> Path:
+    oracle_root = WORKSPACE / "_eval_trusted_oracles"
+    identity_path = oracle_root / "native-runtime-identity.json"
+    runtime = oracle_root / "comet-native-runtime.mjs"
+    if oracle_root.is_symlink() or not oracle_root.is_dir():
+        raise FileNotFoundError("The controller-trusted Native oracle is unavailable")
+    if identity_path.is_symlink() or not identity_path.is_file():
+        raise FileNotFoundError("The controller-trusted Native oracle identity is unavailable")
+    if runtime.is_symlink() or not runtime.is_file():
+        raise FileNotFoundError("The controller-trusted Native runtime is unavailable")
+    identity = read_json(identity_path)
+    expected_keys = {"schema", "runtimeFile", "runtimeHash"}
+    if not isinstance(identity, dict) or set(identity) != expected_keys:
+        raise ValueError("The controller-trusted Native oracle identity is invalid")
+    if (
+        identity.get("schema") != "comet.eval.trusted-native-runtime.v1"
+        or identity.get("runtimeFile") != runtime.name
+        or not re.fullmatch(r"[a-f0-9]{64}", str(identity.get("runtimeHash", "")))
+        or hashlib.sha256(runtime.read_bytes()).hexdigest() != identity["runtimeHash"]
+    ):
+        raise ValueError("The controller-trusted Native runtime does not match its identity")
+    return runtime
+
+
+def run_barrier_commands(
+    commands: list[list[str]], root: Path
+) -> list[subprocess.CompletedProcess]:
+    """Release independent workers together, then collect their exact process results."""
+    if len(commands) != 2 or any(not command for command in commands):
+        raise ValueError("The CAS validator requires exactly two commands")
+    barrier = Path(tempfile.mkdtemp(prefix="native-cas-barrier-", dir=root))
+    release = barrier / "release"
+    workers: list[subprocess.Popen] = []
+    try:
+        for index, command in enumerate(commands):
+            ready = barrier / f"ready-{index}"
+            workers.append(
+                subprocess.Popen(
+                    [sys.executable, "-c", BARRIER_WORKER, str(ready), str(release), *command],
+                    cwd=WORKSPACE,
+                    stdout=subprocess.PIPE,
+                    stderr=subprocess.PIPE,
+                    text=True,
+                )
+            )
+        deadline = time.monotonic() + 10
+        while not all((barrier / f"ready-{index}").is_file() for index in range(2)):
+            if any(worker.poll() is not None for worker in workers):
+                raise RuntimeError("A CAS worker exited before reaching the barrier")
+            if time.monotonic() >= deadline:
+                raise TimeoutError("CAS workers did not reach the barrier")
+            time.sleep(0.005)
+        release.write_text("release", encoding="utf-8")
+        results = []
+        for worker in workers:
+            stdout, stderr = worker.communicate(timeout=30)
+            results.append(
+                subprocess.CompletedProcess(worker.args, worker.returncode, stdout, stderr)
+            )
+        return results
+    finally:
+        for worker in workers:
+            if worker.poll() is None:
+                worker.kill()
+                worker.wait(timeout=5)
+        shutil.rmtree(barrier, ignore_errors=True)
+
+
+def check_live_concurrent_cas() -> dict[str, str]:
+    check = "live_concurrent_cas"
+    state_file = WORKSPACE / "docs/comet/changes/normalize-case/comet-state.yaml"
+    try:
+        state = yaml.safe_load(state_file.read_text(encoding="utf-8")) or {}
+        expected = state.get("revision")
+        if isinstance(expected, bool) or not isinstance(expected, int) or expected < 1:
+            raise ValueError("normalize-case has no positive current revision")
+        runtime = _native_runtime_path()
+        node = shutil.which("node")
+        if not node:
+            raise FileNotFoundError("Node.js is unavailable to the CAS validator")
+        commands = [
+            [
+                node,
+                str(runtime),
+                "checkpoint",
+                "normalize-case",
+                "--summary",
+                f"Validator concurrent checkpoint {label}",
+                "--next-action",
+                f"Inspect concurrent result {label}",
+                "--expect-revision",
+                str(expected),
+                "--project-root",
+                str(WORKSPACE),
+                "--json",
+            ]
+            for label in ("A", "B")
+        ]
+        results = run_barrier_commands(commands, WORKSPACE / EVIDENCE)
+        paths = []
+        for label, result in zip(("a", "b"), results):
+            payload = json.loads(result.stdout)
+            if payload.get("exitCode") != result.returncode:
+                raise ValueError("CAS process exit code differs from its runtime envelope")
+            target = WORKSPACE / EVIDENCE / f"_validator-checkpoint-{label}.json"
+            target.write_text(json.dumps(payload, indent=2), encoding="utf-8")
+            paths.append(target)
+        race = check_checkpoint_cas_envelopes(paths)
+        if race["status"] != "passed":
+            raise ValueError(race.get("reason", "Concurrent CAS evidence is invalid"))
+        final_state = yaml.safe_load(state_file.read_text(encoding="utf-8")) or {}
+        if final_state.get("revision") != expected + 1:
+            raise ValueError("Concurrent CAS did not commit exactly one revision")
+    except Exception as error:
+        return failed(check, str(error))
+    return passed(check)
+
+
+def check_live_conflict_and_workspace() -> dict[str, str]:
+    check = "live_conflict_and_workspace"
+    canonical = WORKSPACE / "docs/comet/specs/word-normalization/spec.md"
+    if not canonical.is_file():
+        return failed(check, "Canonical specification is missing")
+    canonical_hash = hashlib.sha256(canonical.read_bytes()).hexdigest()
+    proposed_hashes: set[str] = set()
+    for name in EXPECTED_CHANGES:
+        root = WORKSPACE / "docs/comet/changes" / name
+        state_file = root / "comet-state.yaml"
+        proposed = root / "specs/word-normalization/spec.md"
+        workspace_file = root / "runtime/workspace.json"
+        if not all(path.is_file() for path in (state_file, proposed, workspace_file)):
+            return failed(check, f"{name} lacks live state, proposed spec, or workspace identity")
+        try:
+            state = yaml.safe_load(state_file.read_text(encoding="utf-8")) or {}
+            workspace = read_json(workspace_file)
+        except (OSError, yaml.YAMLError, json.JSONDecodeError) as error:
+            return failed(check, f"Invalid live metadata for {name}: {error}")
+        matching = [
+            item
+            for item in state.get("spec_changes", [])
+            if isinstance(item, dict) and item.get("capability") == "word-normalization"
+        ]
+        if len(matching) != 1:
+            return failed(check, f"{name} does not declare exactly one word-normalization change")
+        change = matching[0]
+        if (
+            state.get("name") != name
+            or state.get("phase") == "archive"
+            or change.get("operation") != "replace"
+            or change.get("source") != "specs/word-normalization/spec.md"
+            or change.get("base_hash") != canonical_hash
+            or not proposed.read_text(encoding="utf-8").strip()
+        ):
+            return failed(check, f"{name} is not a real same-base replacement conflict")
+        if (
+            not isinstance(workspace, dict)
+            or workspace.get("schema") != "comet.native.workspace.v2"
+            or workspace.get("nativeRootRef") != "docs/comet"
+            or not re.fullmatch(r"[a-f0-9]{64}", str(workspace.get("projectRootId", "")))
+            or not re.fullmatch(r"[a-f0-9]{64}", str(workspace.get("nativeRootId", "")))
+        ):
+            return failed(check, f"{name} does not have a process-free Native workspace identity")
+        proposed_hashes.add(hashlib.sha256(proposed.read_bytes()).hexdigest())
+    if len(proposed_hashes) != 2:
+        return failed(check, "The two conflicting changes do not propose distinct outcomes")
+
+    return passed(check)
+
+
+def main() -> int:
+    evidence = WORKSPACE / EVIDENCE
+    results = [
+        check_pytest(WORKSPACE),
+        check_runtime_envelopes(
+            [
+                evidence / "conflict-status.json",
+                evidence / "checkpoint-attempt-a.json",
+                evidence / "checkpoint-attempt-b.json",
+            ]
+        ),
+        check_parallel_safety(),
+        check_live_conflict_and_workspace(),
+        check_live_concurrent_cas(),
+        check_native_isolation(WORKSPACE),
+    ]
+    return write_results(results, WORKSPACE)
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/eval/local/tasks/comet-native-wave-f-dashboard-readonly/environment/.include-current-comet-cli

@@ -0,0 +1 @@
+The eval harness replaces this marker with a snapshot of the current checkout's bin/ and dist/.

package/eval/local/tasks/comet-native-wave-f-dashboard-readonly/environment/Dockerfile

@@ -0,0 +1,31 @@
+FROM node:22-bookworm-slim
+
+WORKDIR /workspace
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    git python3 python3-pip python-is-python3 ca-certificates && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN npm config set registry https://registry.npmmirror.com && \
+    npm install -g @anthropic-ai/claude-code@latest && \
+    npm config delete registry
+
+COPY current-comet-package.json /opt/comet-cli/package.json
+RUN npm install --prefix /opt/comet-cli --omit=dev --ignore-scripts
+
+RUN pip install --break-system-packages --no-cache-dir pyyaml pytest
+
+COPY wordcount.py /workspace/wordcount.py
+COPY test_wordcount.py /workspace/test_wordcount.py
+COPY current-comet.sh /workspace/current-comet.sh
+
+RUN useradd --create-home --shell /bin/bash agent && \
+    chown -R agent:agent /workspace
+USER agent
+ENV HOME=/home/agent
+ENV PYTHONPATH=/workspace
+RUN git config --global user.email "agent@eval.local" && \
+    git config --global user.name "Comet Native Eval Agent" && \
+    git config --global init.defaultBranch master
+
+CMD ["bash"]

package/eval/local/tasks/comet-native-wave-f-dashboard-readonly/environment/current-comet-package.json

@@ -0,0 +1,18 @@
+{
+  "private": true,
+  "type": "module",
+  "dependencies": {
+    "@fission-ai/openspec": "^1.5.0",
+    "@inquirer/ansi": "^2.0.5",
+    "@inquirer/core": "^11.1.10",
+    "@inquirer/figures": "^2.0.5",
+    "@inquirer/prompts": "^8.4.3",
+    "@inquirer/type": "^4.0.5",
+    "commander": "^14.0.0",
+    "dompurify": "^3.4.12",
+    "highlight.js": "^11.11.1",
+    "marked": "^18.0.4",
+    "mermaid": "11.15.0",
+    "yaml": "^2.9.0"
+  }
+}

package/eval/local/tasks/comet-native-wave-f-dashboard-readonly/environment/current-comet.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+snapshot="/workspace/_eval_current_comet"
+if [[ ! -f "$snapshot/bin/comet.js" || ! -d "$snapshot/dist" ]]; then
+    echo "Current Comet CLI snapshot is unavailable" >&2
+    exit 2
+fi
+
+runtime="$(mktemp -d)"
+trap 'rm -rf "$runtime"' EXIT
+cp -a "$snapshot/." "$runtime/"
+ln -s /opt/comet-cli/node_modules "$runtime/node_modules"
+node "$runtime/bin/comet.js" "$@"

package/eval/local/tasks/comet-native-wave-f-dashboard-readonly/environment/test_wordcount.py

@@ -0,0 +1,27 @@
+"""Baseline tests for the word-count CLI."""
+
+import subprocess
+import sys
+from pathlib import Path
+
+
+def run_cli(*args: str, input_text: str) -> str:
+    result = subprocess.run(
+        [sys.executable, "wordcount.py", *args],
+        input=input_text,
+        capture_output=True,
+        text=True,
+        check=True,
+        cwd=Path(__file__).parent,
+    )
+    return result.stdout
+
+
+def test_count_words():
+    assert "Words: 3" in run_cli(input_text="hello world foo")
+
+
+def test_count_lines():
+    output = run_cli("--lines", input_text="hello\nworld\nfoo")
+    assert "Words: 3" in output
+    assert "Lines: 3" in output

package/eval/local/tasks/comet-native-wave-f-dashboard-readonly/environment/wordcount.py

@@ -0,0 +1,36 @@
+"""Simple word count CLI tool."""
+
+import argparse
+import sys
+
+
+def count_words(text: str) -> int:
+    """Count words in text."""
+    return len(text.split())
+
+
+def count_lines(text: str) -> int:
+    """Count lines in text."""
+    return len(text.splitlines())
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Count words and lines in text")
+    parser.add_argument("file", nargs="?", help="Input file (reads stdin if omitted)")
+    parser.add_argument("--lines", action="store_true", help="Also count lines")
+    parser.add_argument("--words", action="store_true", default=True, help="Count words (default)")
+    args = parser.parse_args()
+
+    if args.file:
+        with open(args.file) as source:
+            text = source.read()
+    else:
+        text = sys.stdin.read()
+
+    print(f"Words: {count_words(text)}")
+    if args.lines:
+        print(f"Lines: {count_lines(text)}")
+
+
+if __name__ == "__main__":
+    main()

package/eval/local/tasks/comet-native-wave-f-dashboard-readonly/instruction.md

@@ -0,0 +1,7 @@
+You are working on a Python project named `wordcount-cli` and evaluating Dashboard's Native projection. Do not implement or archive a product feature.
+
+Begin by invoking the `/comet-native` Skill. Initialize Native under `docs/comet/`, create one active Shape-phase change named `dashboard-visible-change`, and give it a valid brief plus target specification for capability `dashboard-visible-change`.
+
+Save the exact `status dashboard-visible-change --json` Native CLI envelope before collection to `.cache/comet-native-eval/cli-before.json`. Produce a deterministic JSON manifest mapping every relative file under `docs/comet/` to its SHA-256 digest and save it as `.cache/comet-native-eval/native-tree-before.json`. Run `bash /workspace/current-comet.sh dashboard . --json` to execute the controller's immutable, source-built current-checkout Dashboard CLI and save its exact output to `.cache/comet-native-eval/dashboard.json`; do not alter `_eval_current_comet` or its build identity. Then capture the same exact Native status envelope and tree manifest as `.cache/comet-native-eval/cli-after.json` and `.cache/comet-native-eval/native-tree-after.json`. The validator independently reruns that source build against the live tree and two fresh read-only copies, so a one-time write is still a failure.
+
+The Dashboard and CLI must agree on change name, phase, revision, next command, continuation, verification result, findings, and Archive readiness. Dashboard's `native` subtree must contain only its bounded read-only projection: no raw paths, reports, evidence envelopes, conflict signals, operation details, or workspace identity hashes. Both tree manifests must be identical. Leave the change active. Do not fabricate, paraphrase, or hand-author CLI or Dashboard output; the tree manifests must be computed from disk. Use only the Comet Native Skill and bundled runtime; do not create OpenSpec, Classic, `.comet`, or external workflow artifacts.

package/eval/local/tasks/comet-native-wave-f-dashboard-readonly/task.toml

@@ -0,0 +1,36 @@
+[metadata]
+name = "comet-native-wave-f-dashboard-readonly"
+description = "Test Dashboard parity with Native CLI state and prove read-only collection"
+difficulty = "hard"
+category = "comet-native"
+tags = ["comet-native", "dashboard", "projection", "read-only", "wave-f"]
+default_treatments = ["COMET_NATIVE_PHASE1"]
+
+[template]
+required = []
+
+[environment]
+description = "Passing baseline Python workspace with an active Native change visible to Dashboard."
+dockerfile = "environment/Dockerfile"
+timeout_sec = 900
+
+[validation]
+test_scripts = "test_native_wave_f_dashboard_readonly.py"
+target_artifacts = [".cache/comet-native-eval/cli-before.json", ".cache/comet-native-eval/cli-after.json", ".cache/comet-native-eval/dashboard.json", ".cache/comet-native-eval/native-tree-before.json", ".cache/comet-native-eval/native-tree-after.json"]
+timeout = 120
+
+[evaluation]
+profile = "generic"
+required_skills = ["comet-native"]
+expected_artifacts = [".cache/comet-native-eval/cli-before.json", ".cache/comet-native-eval/cli-after.json", ".cache/comet-native-eval/dashboard.json", ".cache/comet-native-eval/native-tree-before.json", ".cache/comet-native-eval/native-tree-after.json"]
+require_skill_invocation = true
+rubric_criteria = [
+  "Creates one active Native change with a stable CLI projection",
+  "Captures the bounded Dashboard Native snapshot and matches Runtime state and continuation fields",
+  "Hashes the Native artifact tree before and after Dashboard collection",
+  "Demonstrates that Dashboard collection changes no Native-owned file and exposes no raw evidence",
+]
+
+[interaction]
+mode = "none"
+max_turns = 1