@rpamis/comet 0.4.0-beta.6 → 0.4.0-beta.8

package/eval/scaffold/python/validation/native_workflow.py

@@ -0,0 +1,252 @@
+"""Hard checks for self-contained Comet Native workflow completion."""
+
+from __future__ import annotations
+
+import json
+import re
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+
+def _failure(name: str, reason: str) -> str:
+    return f"{name}: {reason}"
+
+
+def _terminal_archive(archive_root: Path) -> tuple[Path | None, dict[str, Any] | None]:
+    if not archive_root.is_dir():
+        return None, None
+    for candidate in sorted(
+        (path for path in archive_root.iterdir() if path.is_dir()),
+        reverse=True,
+    ):
+        state_file = candidate / "comet-state.yaml"
+        if not state_file.is_file():
+            continue
+        try:
+            state = yaml.safe_load(state_file.read_text(encoding="utf-8")) or {}
+        except (OSError, yaml.YAMLError):
+            continue
+        if state.get("phase") == "archive" and state.get("archived") is True:
+            return candidate, state
+    return None, None
+
+
+def _spec_changes_are_archived(
+    native_root: Path,
+    archive: Path,
+    state: dict[str, Any],
+) -> bool:
+    spec_changes = state.get("spec_changes") or []
+    if not isinstance(spec_changes, list) or not spec_changes:
+        return False
+    for change in spec_changes:
+        if not isinstance(change, dict):
+            return False
+        capability = change.get("capability")
+        operation = change.get("operation")
+        if not isinstance(capability, str) or not re.fullmatch(r"[a-z0-9][a-z0-9-]*", capability):
+            return False
+        canonical = native_root / "specs" / capability / "spec.md"
+        if operation == "remove":
+            if change.get("source") is not None or canonical.exists():
+                return False
+            continue
+        expected_source = f"specs/{capability}/spec.md"
+        archived = archive / expected_source
+        if operation not in {"create", "replace"} or change.get("source") != expected_source:
+            return False
+        if not archived.is_file() or not canonical.is_file():
+            return False
+        if not archived.read_text(encoding="utf-8").strip():
+            return False
+        if archived.read_bytes() != canonical.read_bytes():
+            return False
+    return True
+
+
+def validate_native_workflow(
+    test_dir: Path,
+    outputs: dict[str, Any],
+    terminal_mode: str = "archive",
+) -> tuple[list[str], list[str]]:
+    """Validate Native invocation, terminal artifacts, trajectory, and isolation."""
+    passed: list[str] = []
+    failed: list[str] = []
+
+    invoked = (outputs.get("events") or {}).get("skills_invoked", []) or []
+    unexpected_skills = sorted({skill for skill in invoked if skill != "comet-native"})
+    if "comet-native" in invoked and not unexpected_skills:
+        passed.append("native_skill_invocation")
+    elif unexpected_skills:
+        failed.append(
+            _failure(
+                "native_skill_invocation",
+                f"unexpected Skills were invoked: {', '.join(unexpected_skills)}",
+            )
+        )
+    else:
+        failed.append(_failure("native_skill_invocation", "comet-native was not invoked"))
+
+    config_file = test_dir / ".comet" / "config.yaml"
+    config: dict[str, Any] = {}
+    if config_file.is_file():
+        try:
+            config = yaml.safe_load(config_file.read_text(encoding="utf-8")) or {}
+        except (OSError, yaml.YAMLError):
+            config = {}
+    artifact_root = (config.get("native") or {}).get("artifact_root")
+    project_root = test_dir.resolve()
+    artifact_root_path = Path(artifact_root) if isinstance(artifact_root, str) else None
+    candidate_native_root = (
+        (project_root / artifact_root_path / "comet").resolve()
+        if artifact_root_path is not None
+        else project_root / "comet"
+    )
+    config_valid = (
+        config.get("schema") == "comet.project.v1"
+        and config.get("default_workflow") == "native"
+        and "native" in (config.get("workflows") or [config.get("default_workflow")])
+        and isinstance(artifact_root, str)
+        and artifact_root.strip()
+        and artifact_root_path is not None
+        and not artifact_root_path.is_absolute()
+        and candidate_native_root.is_relative_to(project_root)
+    )
+    if config_valid:
+        passed.append("native_artifacts")
+        native_root = candidate_native_root
+    else:
+        failed.append(_failure("native_artifacts", "valid .comet/config.yaml is missing"))
+        native_root = project_root / "comet"
+
+    changes_root = native_root / "changes"
+    active_changes = (
+        [path for path in changes_root.iterdir() if path.is_dir()]
+        if changes_root.is_dir()
+        else []
+    )
+    archive, state = _terminal_archive(native_root / "archive")
+    trajectory_files: list[Path] = []
+    require_hard_stop = terminal_mode == "active-blocked"
+    require_active = terminal_mode in {"active", "active-blocked"}
+    if require_hard_stop:
+        active_state: dict[str, Any] = {}
+        if len(active_changes) == 1:
+            try:
+                active_state = (
+                    yaml.safe_load(
+                        (active_changes[0] / "comet-state.yaml").read_text(encoding="utf-8")
+                    )
+                    or {}
+                )
+            except (OSError, yaml.YAMLError):
+                active_state = {}
+        if archive is not None:
+            failed.append(_failure("native_state", "blocked Native task was unexpectedly archived"))
+        elif (
+            len(active_changes) != 1
+            or active_state.get("phase") != "build"
+            or active_state.get("verification_result") != "fail"
+            or active_state.get("archived") is not False
+        ):
+            failed.append(_failure("native_state", "expected one active failed Native change"))
+        else:
+            passed.append("native_state")
+            trajectory_files.append(active_changes[0] / "runtime" / "trajectory.jsonl")
+    elif require_active:
+        active_states: list[dict[str, Any]] = []
+        try:
+            active_states = [
+                yaml.safe_load((change / "comet-state.yaml").read_text(encoding="utf-8")) or {}
+                for change in active_changes
+            ]
+        except (OSError, yaml.YAMLError):
+            active_states = []
+        if not active_states or any(
+            state.get("phase") == "archive" or state.get("archived") is not False
+            for state in active_states
+        ):
+            failed.append(_failure("native_state", "expected active non-archived Native changes"))
+        else:
+            passed.append("native_state")
+            trajectory_files.extend(
+                change / "runtime" / "trajectory.jsonl" for change in active_changes
+            )
+    elif archive is None or state is None:
+        failed.append(_failure("native_state", "no terminal Native archive exists"))
+    elif active_changes:
+        failed.append(_failure("native_state", "active Native changes remain after archive"))
+    elif state.get("verification_result") != "pass":
+        failed.append(_failure("native_state", "archived change is not verified as pass"))
+    else:
+        report_name = state.get("verification_report")
+        required_files = [archive / "brief.md", archive / str(report_name or "")]
+        if (
+            report_name != "verification.md"
+            or not all(path.is_file() and path.stat().st_size > 0 for path in required_files)
+            or not _spec_changes_are_archived(native_root, archive, state)
+        ):
+            failed.append(
+                _failure("native_state", "brief, specification, or verification evidence is incomplete")
+            )
+        else:
+            passed.append("native_state")
+
+    if not trajectory_files and archive:
+        trajectory_files.append(archive / "runtime" / "trajectory.jsonl")
+    phases: set[str] = set()
+    hidden_reasoning = False
+    hard_stop = False
+    for trajectory_file in trajectory_files:
+        if not trajectory_file.is_file():
+            continue
+        for line in trajectory_file.read_text(encoding="utf-8").splitlines():
+            if not line.strip():
+                continue
+            try:
+                event = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+            data = event.get("data") or {}
+            phases.update(
+                value
+                for value in (data.get("phase"), data.get("previousPhase"), data.get("nextPhase"))
+                if isinstance(value, str)
+            )
+            serialized = json.dumps(event).lower()
+            hidden_reasoning = hidden_reasoning or any(
+                marker in serialized
+                for marker in ("chain_of_thought", "reasoning_content", "hidden_reasoning")
+            )
+            repair = data.get("repairStagnation") or {}
+            hard_stop = hard_stop or repair.get("disposition") == "hard-stop"
+    required_phases = (
+        {"shape", "build", "verify"}
+        if require_hard_stop
+        else {"shape", "build"}
+        if require_active
+        else {"shape", "build", "verify", "archive"}
+    )
+    if (
+        required_phases.issubset(phases)
+        and not hidden_reasoning
+        and (hard_stop if require_hard_stop else True)
+    ):
+        passed.append("native_trajectory")
+    else:
+        failed.append(_failure("native_trajectory", "complete safe phase trajectory is missing"))
+
+    comet_config_dir = test_dir / ".comet"
+    hidden_entries = (
+        {path.name for path in comet_config_dir.iterdir()}
+        if comet_config_dir.is_dir()
+        else set()
+    )
+    if (test_dir / "openspec").exists() or hidden_entries - {"config.yaml"}:
+        failed.append(_failure("native_isolation", "Classic or hidden workflow artifacts exist"))
+    else:
+        passed.append("native_isolation")
+
+    return passed, failed

package/eval/scaffold/python/validation/rubric.py

@@ -39,8 +39,6 @@ import re
 from pathlib import Path
 from typing import Any
 
-from scaffold.python.validation.core import ValidatorFn
-
 # All nine rubric dimensions, in display order.
 RUBRIC_DIMENSIONS = (
     "main_flow",

package/eval/scaffold/shell/completion-point.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+set -uo pipefail
+
+TEXT="${1:-}"
+shopt -s nocasematch
+
+NEGATED_PATTERN='(archive|workflow|change)([[:space:]]+is)?[[:space:]]+not[[:space:]]+(complete|completed|archived)|not[[:space:]]+(yet[[:space:]]+)?(complete|completed|archived)|not[[:space:]]+completed[[:space:]]+through[[:space:]]+archive|not[[:space:]]+([^[:space:]]+[[:space:]]+){0,3}archived'
+if [[ "$TEXT" =~ $NEGATED_PATTERN ]]; then
+    exit 1
+fi
+
+COMPLETION_PATTERN='archive(d)?([[:space:]]+is)?[[:space:]]+(complete|completed)|change([[:space:]]+is)?[[:space:]]+archived|native[[:space:]]+change[^[:cntrl:]]*archived|archived[[:space:]]+(at|to)|completed[[:space:]]+through[[:space:]]+archive|completed[[:space:]]+through[[:space:]]+all[[:space:]]+phases[[:space:]]+and[[:space:]]+archived|terminal[[:space:]]+archived[[:space:]]+state|fully[[:space:]]+archived|workflow([[:space:]]+is)?[[:space:]]+(complete|completed)|all[[:space:]]+(5|five)[[:space:]]+phases([[:space:]]+are)?[[:space:]]+(complete|completed|recorded)'
+if [[ "$TEXT" =~ $COMPLETION_PATTERN ]]; then
+    exit 0
+fi
+
+exit 1

package/eval/scaffold/shell/decision-point.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+set -uo pipefail
+
+TEXT="${1:-}"
+shift || true
+shopt -s nocasematch
+
+QUESTION_PATTERN='[?？][[:space:]]*$'
+if [[ "$TEXT" =~ $QUESTION_PATTERN ]]; then
+    exit 0
+fi
+
+INTERROGATIVE_PATTERN='(^|[[:space:]])(how|what|which|would|could|can|should|is|are|do|does|will|where|when|who)[[:space:]]+[^?？]*[?？]|(是否|怎样|如何|哪个|哪种|要不要)[^？]*？'
+LABELLED_QUESTION_PATTERN='(question|问题)[*[:space:]]*:[*[:space:]]*(whether|how|what|which|would|could|can|should|is|are|do|does|will|where|when|who|是否|怎样|如何|哪个|哪种|要不要)'
+REQUEST_PATTERN='(^|[.!。！][[:space:]]*)(please[[:space:]]+)?(confirm|choose|approve|select|provide|enter)[[:space:]]|(^|[.!。！][[:space:]]*)(would you|could you|can you|shall we|do you want|which (option|approach|name))|(please[[:space:]]+)?(reply|respond)[^.!?。！？]*(confirm|approve)'
+
+# Evaluate question and request syntax one line at a time. A completed summary can
+# contain an interrogative-looking decision label on one line and punctuation
+# examples such as `?!` on a later line; treating the whole response as one regex
+# span incorrectly turns that summary into another simulated user round.
+while IFS= read -r line; do
+    if [[ "$line" =~ $INTERROGATIVE_PATTERN || "$line" =~ $LABELLED_QUESTION_PATTERN || "$line" =~ $REQUEST_PATTERN ]]; then
+        exit 0
+    fi
+done <<< "$TEXT"
+
+UNRESOLVED_PATTERN='(^|[^[:alnum:]_])(unresolved|blocking|need your (input|answer|decision|preference)|waiting for your (input|answer|decision|preference))([^[:alnum:]_]|$)'
+for pattern in "$@"; do
+    if [[ -n "$pattern" && "${TEXT,,}" == *"${pattern,,}"* && "$TEXT" =~ $UNRESOLVED_PATTERN ]]; then
+        exit 0
+    fi
+done
+
+exit 1

package/eval/scaffold/shell/docker.sh

@@ -148,6 +148,65 @@ image_exists() {
     docker images -q "$image_name" 2>/dev/null | grep -q .
 }
 
+sha256_text() {
+    if command -v sha256sum &> /dev/null; then
+        printf '%s' "$1" | sha256sum | cut -d' ' -f1
+    elif command -v shasum &> /dev/null; then
+        printf '%s' "$1" | shasum -a 256 | cut -d' ' -f1
+    else
+        echo "ERROR: sha256sum or shasum is required for execution identity" >&2
+        return 1
+    fi
+}
+
+resolve_runtime_image() {
+    local dir="$1"
+    local expected_image_id="${2:-}"
+    if [[ -n "$expected_image_id" ]]; then
+        if [[ ! "$expected_image_id" =~ ^sha256:[0-9a-f]{64}$ ]]; then
+            echo "ERROR: Invalid immutable Docker image ID" >&2
+            return 1
+        fi
+        if ! docker image inspect "$expected_image_id" &> /dev/null; then
+            echo "ERROR: Expected Docker image is no longer available" >&2
+            return 1
+        fi
+        printf '%s' "$expected_image_id"
+        return 0
+    fi
+
+    local image_name image_id
+    image_name=$(docker_build "$dir") || return 1
+    image_id=$(docker image inspect --format '{{.Id}}' "$image_name") || return 1
+    if [[ ! "$image_id" =~ ^sha256:[0-9a-f]{64}$ ]]; then
+        echo "ERROR: Docker returned an invalid immutable image ID" >&2
+        return 1
+    fi
+    printf '%s' "$image_id"
+}
+
+docker_execution_identity() {
+    local dir="$1"
+    local image_name image_id repo_digests claude_version
+    image_name=$(docker_build "$dir") || return 1
+    image_id=$(docker image inspect --format '{{.Id}}' "$image_name") || return 1
+    if [[ ! "$image_id" =~ ^sha256:[0-9a-f]{64}$ ]]; then
+        echo "ERROR: Docker returned an invalid immutable image ID" >&2
+        return 1
+    fi
+    repo_digests=$(docker image inspect --format '{{json .RepoDigests}}' "$image_id") || return 1
+    claude_version=$(docker run --rm "$image_id" claude --version 2>/dev/null) || {
+        echo "ERROR: Cannot verify Claude CLI version in benchmark image" >&2
+        return 1
+    }
+    printf '{"schema":"comet.eval.execution-identity.v1","runtime_image_id":"%s","image_id_hash":"sha256:%s","image_repo_digests_hash":"sha256:%s","image_ref_hash":"sha256:%s","claude_tool_version_hash":"sha256:%s"}\n' \
+        "$image_id" \
+        "$(sha256_text "$image_id")" \
+        "$(sha256_text "$repo_digests")" \
+        "$(sha256_text "$image_name")" \
+        "$(sha256_text "$claude_version")"
+}
+
 # =============================================================================
 # DOCKER BUILD
 # =============================================================================
@@ -233,6 +292,27 @@ build_plugin_args() {
     PLUGIN_CLI_ARGS=("--plugin-dir" "//opt/langsmith-cc-plugin")
 }
 
+# Keep controller-built oracle snapshots immutable inside both agent and validator containers.
+# Nested read-only binds override the writable /workspace parent mount when present.
+build_trusted_oracle_mount_args() {
+    local dir="$1"
+    TRUSTED_ORACLE_MOUNT_ARGS=()
+    if [[ -d "$dir/_eval_current_comet" ]]; then
+        local current_comet_host
+        current_comet_host=$(_winpath "$dir/_eval_current_comet")
+        TRUSTED_ORACLE_MOUNT_ARGS+=(
+            "-v" "$current_comet_host://workspace/_eval_current_comet:ro"
+        )
+    fi
+    if [[ -d "$dir/_eval_trusted_oracles" ]]; then
+        local native_oracles_host
+        native_oracles_host=$(_winpath "$dir/_eval_trusted_oracles")
+        TRUSTED_ORACLE_MOUNT_ARGS+=(
+            "-v" "$native_oracles_host://workspace/_eval_trusted_oracles:ro"
+        )
+    fi
+}
+
 # Run command in Docker container
 # Usage: docker_run <directory> <command...>
 docker_run() {
@@ -244,12 +324,14 @@ docker_run() {
     image_name=$(docker_build "$dir") || return 1
 
     build_env_args
+    build_trusted_oracle_mount_args "$dir"
 
     local windir
     windir=$(_winpath "$dir")
 
     docker run --rm \
         -v "$windir://workspace" \
+        ${TRUSTED_ORACLE_MOUNT_ARGS[@]+"${TRUSTED_ORACLE_MOUNT_ARGS[@]}"} \
         -w //workspace \
         "${ENV_ARGS[@]}" \
         "$image_name" \
@@ -290,6 +372,7 @@ docker_run_claude() {
 
     local model=""
     local timeout="300"
+    local expected_image_id=""
 
     while [[ $# -gt 0 ]]; do
         case "$1" in
@@ -301,17 +384,22 @@ docker_run_claude() {
                 timeout="$2"
                 shift 2
                 ;;
+            --image-id)
+                expected_image_id="$2"
+                shift 2
+                ;;
             *)
                 shift
                 ;;
         esac
     done
 
-    local image_name
-    image_name=$(docker_build "$dir") || return 1
+    local image_id
+    image_id=$(resolve_runtime_image "$dir" "$expected_image_id") || return 1
 
     build_env_args
     build_plugin_args
+    build_trusted_oracle_mount_args "$dir"
 
     local cmd=(
         claude -p "$prompt"
@@ -334,18 +422,20 @@ docker_run_claude() {
     if [[ -n "$TIMEOUT_CMD" ]]; then
         $TIMEOUT_CMD "$timeout" docker run --rm \
             -v "$windir://workspace" \
+            ${TRUSTED_ORACLE_MOUNT_ARGS[@]+"${TRUSTED_ORACLE_MOUNT_ARGS[@]}"} \
             ${PLUGIN_MOUNT_ARGS[@]+"${PLUGIN_MOUNT_ARGS[@]}"} \
             -w //workspace \
             "${ENV_ARGS[@]}" \
-            "$image_name" \
+            "$image_id" \
             "${cmd[@]}"
     else
         docker run --rm \
             -v "$windir://workspace" \
+            ${TRUSTED_ORACLE_MOUNT_ARGS[@]+"${TRUSTED_ORACLE_MOUNT_ARGS[@]}"} \
             ${PLUGIN_MOUNT_ARGS[@]+"${PLUGIN_MOUNT_ARGS[@]}"} \
             -w //workspace \
             "${ENV_ARGS[@]}" \
-            "$image_name" \
+            "$image_id" \
             "${cmd[@]}"
     fi
 }
@@ -358,28 +448,63 @@ docker_run_claude_loop() {
     local prompt="$2"
     shift 2
 
-    local image_name
-    image_name=$(docker_build "$dir") || return 1
+    local expected_image_id=""
+    local loop_args=()
+    while [[ $# -gt 0 ]]; do
+        case "$1" in
+            --image-id)
+                expected_image_id="$2"
+                shift 2
+                ;;
+            --max-turns|--model|--simulator-prompt-file|--decision-reply|--continue-prompt|--decision-pattern|--fresh-resume-marker)
+                loop_args+=("$1" "$2")
+                shift 2
+                ;;
+            *)
+                loop_args+=("$1")
+                shift
+                ;;
+        esac
+    done
+
+    local image_id
+    image_id=$(resolve_runtime_image "$dir" "$expected_image_id") || return 1
 
     build_env_args
     build_plugin_args
+    build_trusted_oracle_mount_args "$dir"
 
     local windir shell_dir
     windir=$(_winpath "$dir")
     shell_dir=$(_winpath "$SCRIPT_DIR")
+    local container_name
+    container_name="comet-eval-loop-$(sha256_text "$dir" | cut -c1-24)"
+
+    # A previous host-side timeout may have terminated bash before Docker could
+    # process --rm. The name is scoped to this unique pytest workspace, so it is
+    # safe to remove only that stale container before retrying the same case.
+    docker rm -f "$container_name" &> /dev/null || true
 
     # Mount the scaffold shell scripts read-only so the loop driver is available
     # at /opt/scaffold-shell/ inside the container.
-    docker run --rm \
+    docker run --rm --name "$container_name" \
         -v "$windir://workspace" \
+        ${TRUSTED_ORACLE_MOUNT_ARGS[@]+"${TRUSTED_ORACLE_MOUNT_ARGS[@]}"} \
         -v "$shell_dir://opt/scaffold-shell:ro" \
         ${PLUGIN_MOUNT_ARGS[@]+"${PLUGIN_MOUNT_ARGS[@]}"} \
         -w //workspace \
         "${ENV_ARGS[@]}" \
-        "$image_name" \
+        "$image_id" \
         bash //opt/scaffold-shell/run-claude-loop.sh "$prompt" \
             ${PLUGIN_CLI_ARGS[@]+"${PLUGIN_CLI_ARGS[@]}"} \
-            "$@"
+            "${loop_args[@]}"
+}
+
+cleanup_claude_loop() {
+    local dir="$1"
+    local container_name
+    container_name="comet-eval-loop-$(sha256_text "$dir" | cut -c1-24)"
+    docker rm -f "$container_name" &> /dev/null || true
 }
 
 # =============================================================================
@@ -406,6 +531,13 @@ if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
         fi
         docker_build "$(realpath "$dir")" "$force"
         ;;
+    execution-identity)
+        dir="${1:-}"
+        if [[ -z "$dir" ]]; then
+            die "Usage: $0 execution-identity <directory>"
+        fi
+        docker_execution_identity "$(realpath "$dir")"
+        ;;
     run)
         dir="${1:-}"
         if [[ -z "$dir" ]]; then
@@ -450,6 +582,13 @@ if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
         shift 2
         docker_run_claude_loop "$(realpath "$dir")" "$prompt" "$@"
         ;;
+    cleanup-claude-loop)
+        dir="${1:-}"
+        if [[ -z "$dir" ]]; then
+            die "Usage: $0 cleanup-claude-loop <directory>"
+        fi
+        cleanup_claude_loop "$(realpath "$dir")"
+        ;;
     help|*)
         cat <<EOF
 Docker utilities for skill benchmarks
@@ -459,6 +598,7 @@ Usage: $0 <command> [args...]
 Commands:
   check                              Check if Docker is available
   build <dir> [--force]              Build image (cached by Dockerfile hash)
+  execution-identity <dir>           Hash the immutable image and Claude CLI version
   run <dir> <cmd...>                 Run command in container
   run-python <dir> <script> [args]   Run Python script in container
   run-node <dir> <script> [args]     Run Node.js/TypeScript in container
@@ -467,6 +607,7 @@ Commands:
 Options for run-claude:
   --model MODEL      Model to use
   --timeout SECONDS  Timeout (default: 300)
+  --image-id SHA256  Run the controller-verified immutable image
 
 Environment variables passed to containers:
   OPENAI_API_KEY, ANTHROPIC_API_KEY, LANGSMITH_API_KEY,