@rpamis/comet 0.4.0-beta.6 → 0.4.0-beta.7

package/eval/local/tests/scaffold/test_utils.py

@@ -1,11 +1,29 @@
 """Unit tests for eval scaffold utilities."""
 
+import importlib
+import os
+import subprocess
 from pathlib import Path
 
+import dotenv
+
 from scaffold.python import utils
 from scaffold.python.skill_parser import load_skill_content, parse_skill_md
 
 
+def test_import_does_not_read_dotenv(monkeypatch):
+    calls = []
+    monkeypatch.setattr(dotenv, "load_dotenv", lambda *args, **kwargs: calls.append((args, kwargs)))
+
+    importlib.reload(utils)
+
+    assert calls == []
+    utils.load_eval_environment()
+    assert len(calls) == 2
+    monkeypatch.undo()
+    importlib.reload(utils)
+
+
 def test_execution_validator_converts_structured_checks(monkeypatch, tmp_path: Path):
     def fake_run_eval_in_docker(*_args, **_kwargs):
         return {
@@ -33,6 +51,25 @@ def test_execution_validator_converts_structured_checks(monkeypatch, tmp_path: P
     assert failed == ["tests_written: missing assertions"]
 
 
+def test_docker_script_failure_preserves_stderr(monkeypatch, tmp_path: Path):
+    monkeypatch.setattr(utils, "check_docker_available", lambda: True)
+    monkeypatch.setattr(
+        utils,
+        "run_shell",
+        lambda *_args, **_kwargs: subprocess.CompletedProcess(
+            args=["docker.sh"],
+            returncode=1,
+            stdout="",
+            stderr="validator import failed",
+        ),
+    )
+
+    success, output = utils.run_python_in_docker(tmp_path, "validation/check.py")
+
+    assert success is False
+    assert "validator import failed" in output
+
+
 def test_run_shell_decodes_subprocess_output_as_utf8(monkeypatch):
     captured = {}
 
@@ -55,6 +92,24 @@ def test_to_bash_path_uses_msys_drive_prefix_for_git_bash(monkeypatch):
     assert utils._to_bash_path(r"D:\Project\Comet\eval") == "/d/Project/Comet/eval"
 
 
+def test_resolve_bash_prefers_git_bash_when_path_bash_is_wsl(monkeypatch):
+    monkeypatch.delenv("GIT_BASH", raising=False)
+    monkeypatch.setattr(
+        utils.shutil,
+        "which",
+        lambda name: {"bash": r"C:\Windows\System32\bash.exe", "git": r"D:\Git\cmd\git.exe"}.get(
+            name
+        ),
+    )
+    monkeypatch.setattr(
+        utils.os.path,
+        "isfile",
+        lambda path: str(path).replace("/", "\\").lower() == r"d:\git\bin\bash.exe",
+    )
+
+    assert utils._resolve_bash(os_name="nt") == r"D:\Git\bin\bash.exe"
+
+
 def test_to_bash_path_uses_wsl_mount_prefix_for_windowsapps_bash(monkeypatch):
     monkeypatch.setattr(utils.os, "name", "nt")
     monkeypatch.setattr(
@@ -122,10 +177,40 @@ def test_docker_loop_passes_langsmith_plugin_args_to_loop_driver():
     assert "CC_LANGSMITH_LOG_FILE" in docker_sh
 
 
+def test_claude_loop_timeout_force_removes_its_named_container(monkeypatch, tmp_path: Path):
+    calls = []
+
+    def fake_run_shell(script, *args, **kwargs):
+        calls.append((script, args, kwargs))
+        if args[0] == "run-claude-loop":
+            raise subprocess.TimeoutExpired([script, *args], timeout=42)
+        return subprocess.CompletedProcess([script, *args], 0, "", "")
+
+    monkeypatch.setattr(utils, "run_shell", fake_run_shell)
+
+    result = utils.run_claude_loop_in_docker(tmp_path, ["prompt"], timeout=42)
+
+    assert result.returncode == 124
+    assert calls[0][1][0] == "run-claude-loop"
+    assert calls[1][1] == ("cleanup-claude-loop", tmp_path)
+
+    docker_sh = (utils.SHELL_DIR / "docker.sh").read_text(encoding="utf-8")
+    assert '--name "$container_name"' in docker_sh
+    assert "cleanup-claude-loop" in docker_sh
+
+
+def test_docker_subject_run_uses_controller_verified_immutable_image_identity():
+    docker_sh = (utils.SHELL_DIR / "docker.sh").read_text(encoding="utf-8")
+
+    assert "docker_execution_identity" in docker_sh
+    assert "claude --version" in docker_sh
+    assert "runtime_image_id" in docker_sh
+    assert 'image_id=$(resolve_runtime_image "$dir" "$expected_image_id")' in docker_sh
+    assert '"$image_id"' in docker_sh
+
+
 def test_run_claude_fixture_defaults_langsmith_hook_log_path():
-    conftest_py = (Path(__file__).resolve().parents[1] / "conftest.py").read_text(
-        encoding="utf-8"
-    )
+    conftest_py = (Path(__file__).resolve().parents[1] / "conftest.py").read_text(encoding="utf-8")
 
     assert 'os.environ["CC_LANGSMITH_LOG_FILE"] = "/workspace/langsmith-hook.log"' in conftest_py
 
@@ -134,6 +219,194 @@ def test_claude_loop_applies_plugin_args_to_subject_turns_only():
     loop_sh = (utils.SHELL_DIR / "run-claude-loop.sh").read_text(encoding="utf-8")
 
     assert "PLUGIN_ARGS=()" in loop_sh
-    assert 'claude -p "$PROMPT" "${PLUGIN_ARGS[@]}"' in loop_sh
+    assert "shopt -s nocasematch" in loop_sh
+    assert "DECISION_REPLY=" in loop_sh
+    assert 'USER_REPLY="$DECISION_REPLY"' in loop_sh
+    assert 'bash "$SCRIPT_DIR/decision-point.sh" "$RESULT_TEXT"' in loop_sh
+    assert 'bash "$SCRIPT_DIR/completion-point.sh" "$RESULT_TEXT"' in loop_sh
+    assert loop_sh.index("workflow completion detected") < loop_sh.index(
+        'bash "$SCRIPT_DIR/decision-point.sh" "$RESULT_TEXT"'
+    )
+    assert 'SUBJECT_PROMPT="${FRESH_PROMPT:-$PROMPT}"' in loop_sh
+    assert 'claude -p "$SUBJECT_PROMPT" "${PLUGIN_ARGS[@]}"' in loop_sh
     assert 'claude -p "$USER_REPLY" "${PLUGIN_ARGS[@]}"' in loop_sh
+    assert "fresh resume boundary detected" in loop_sh
     assert 'claude -p "$sim_prompt" "${PLUGIN_ARGS[@]}"' not in loop_sh
+
+
+def test_claude_loop_surfaces_subject_resume_failure(tmp_path: Path):
+    fake_bin = tmp_path / "bin"
+    fake_bin.mkdir()
+    fake_claude = fake_bin / "claude"
+    fake_claude.write_text(
+        """#!/usr/bin/env bash
+for arg in "$@"; do
+  if [[ "$arg" == "--resume" ]]; then
+    echo "resume failed stdout diagnostic"
+    echo "resume failed diagnostic" >&2
+    exit 42
+  fi
+done
+printf '%s\n' '{"type":"system","session_id":"session-1"}'
+printf '%s\n' '{"type":"result","subtype":"success","session_id":"session-1","result":"Should abbreviations end a sentence?"}'
+""",
+        encoding="utf-8",
+        newline="\n",
+    )
+    fake_claude.chmod(0o755)
+
+    env = os.environ.copy()
+    env["PATH"] = f"{utils._to_bash_path(fake_bin)}:{env.get('PATH', '')}"
+    result = subprocess.run(
+        [
+            utils.BASH_EXEC,
+            utils._to_bash_path(utils.SHELL_DIR / "run-claude-loop.sh"),
+            "Implement the requested change.",
+            "--max-turns",
+            "2",
+            "--decision-pattern",
+            "abbreviation",
+            "--decision-reply",
+            "Use an explicit abbreviation list.",
+        ],
+        capture_output=True,
+        text=True,
+        encoding="utf-8",
+        errors="replace",
+        timeout=10,
+        check=False,
+        env=env,
+    )
+
+    assert result.returncode != 0
+    assert "resume failed stdout diagnostic" in result.stderr
+    assert "resume failed diagnostic" in result.stderr
+    assert "subject turn 2 failed" in result.stderr
+
+
+def test_decision_point_detector_rejects_completion_statements():
+    statement = "Implementation is complete and the artifacts provide the requested evidence."
+    punctuation_summary = "Done. The counter recognizes ., !, and ? terminators."
+    question = "Please confirm whether abbreviations should end a sentence."
+
+    statement_result = utils.run_shell("decision-point.sh", statement, check=False)
+    punctuation_result = utils.run_shell("decision-point.sh", punctuation_summary, check=False)
+    question_result = utils.run_shell("decision-point.sh", question, check=False)
+
+    assert statement_result.returncode == 1
+    assert punctuation_result.returncode == 1
+    assert question_result.returncode == 0
+
+
+def test_decision_point_detector_rejects_multiline_archived_decision_summary():
+    result = utils.run_shell(
+        "decision-point.sh",
+        """Change archived successfully.
+1. Should abbreviations end sentences — No.
+2. Consecutive terminators (`?!`) count as one boundary.
+All verification checks passed.""",
+        check=False,
+    )
+
+    assert result.returncode == 1
+
+
+def test_decision_point_patterns_require_an_unresolved_decision_signal():
+    ordinary = utils.run_shell(
+        "decision-point.sh",
+        "The brief records abbreviation behavior.",
+        "abbreviation",
+        check=False,
+    )
+    unresolved = utils.run_shell(
+        "decision-point.sh",
+        "Abbreviation behavior is unresolved; I need your decision before continuing.",
+        "abbreviation",
+        check=False,
+    )
+
+    assert ordinary.returncode == 1
+    assert unresolved.returncode == 0
+
+
+def test_decision_point_detector_accepts_question_followed_by_recommendation_and_impact():
+    result = utils.run_shell(
+        "decision-point.sh",
+        """How should abbreviations affect sentence boundaries?
+
+Recommendation: Ignore a small explicit list.
+Impact: Counts remain intuitive, but the list needs maintenance.""",
+        check=False,
+    )
+
+    assert result.returncode == 0
+
+
+def test_decision_point_detector_accepts_batch_labels_and_reply_confirmation():
+    labelled_question = utils.run_shell(
+        "decision-point.sh",
+        """**1. Question:** Whether abbreviations such as `e.g.` end a sentence.
+**Recommendation:** No.
+**Impact:** Counts stay intuitive.""",
+        check=False,
+    )
+    reply_confirmation = utils.run_shell(
+        "decision-point.sh",
+        'Please reply **"confirmed"** to approve this shared understanding.',
+        check=False,
+    )
+
+    assert labelled_question.returncode == 0
+    assert reply_confirmation.returncode == 0
+
+
+def test_completion_point_detector_requires_explicit_non_negated_workflow_completion():
+    archived = utils.run_shell(
+        "completion-point.sh",
+        "Change archived at docs/comet/archive/2026-07-15-add-counting/.",
+        check=False,
+    )
+    archive_complete = utils.run_shell(
+        "completion-point.sh", "The archive is complete and verified from disk.", check=False
+    )
+    completed_through_archive = utils.run_shell(
+        "completion-point.sh", "Change add-counting completed through Archive.", check=False
+    )
+    archived_to = utils.run_shell(
+        "completion-point.sh", "- **Archived to**: docs/comet/archive/2026-07-19-add-counting/", check=False
+    )
+    completed_all_phases = utils.run_shell(
+        "completion-point.sh",
+        "The change has been completed through all phases and archived.",
+        check=False,
+    )
+    terminal_archived = utils.run_shell(
+        "completion-point.sh",
+        "The change is already in its terminal archived state.",
+        check=False,
+    )
+    archived_heading = utils.run_shell(
+        "completion-point.sh",
+        "**Native change `add-sentence-counting` is archived**",
+        check=False,
+    )
+    phase_done = utils.run_shell(
+        "completion-point.sh", "Shape is done; Build remains pending.", check=False
+    )
+    negated = utils.run_shell(
+        "completion-point.sh", "The workflow is not complete yet.", check=False
+    )
+    negated_through_archive = utils.run_shell(
+        "completion-point.sh", "The change is not completed through Archive.", check=False
+    )
+
+    assert archived.returncode == 0
+    assert archive_complete.returncode == 0
+    assert completed_through_archive.returncode == 0
+    assert archived_to.returncode == 0
+    assert completed_all_phases.returncode == 0
+    assert terminal_archived.returncode == 0
+    assert archived_heading.returncode == 0
+    assert phase_done.returncode == 1
+    assert negated.returncode == 1
+    assert negated_through_archive.returncode == 1

package/eval/local/tests/tasks/__init__.py

@@ -0,0 +1 @@
+"""Unit tests for Eval task definitions and validators."""

package/eval/local/tests/tasks/test_tasks.py

@@ -21,25 +21,25 @@ import pytest
 import conftest
 from conftest import get_fixtures
 
-from scaffold import NoiseTask, Treatment
+from scaffold import Treatment
 from scaffold.python import extract_events, parse_output
+from scaffold.python.aligned_comparison import build_case_manifest, case_manifest_payload
+from scaffold.python.native_eval import (
+    adapt_checks_for_native,
+    adapt_prompt_for_native,
+    filter_control_workflow_checks as _filter_control_workflow_checks,
+    is_control_business_only_run as _is_control_business_only_run,
+    split_comet_completion_checks as _split_comet_completion_checks,
+)
 from scaffold.python.profiles import resolve_profile_name, run_profile_rubric
 from scaffold.python.tasks import list_tasks, load_task
 from scaffold.python.treatments import TreatmentConfig, build_treatment_skills, load_treatments
 from scaffold.python.validation import run_validators
 
 # Timeouts
-CLAUDE_TIMEOUT = 1500  # 25 minutes for Claude to complete task (multi-turn loop)
-PYTEST_TIMEOUT = 1800  # 30 minutes total including setup/teardown
+CLAUDE_TIMEOUT = 1500  # Default floor for Claude to complete a multi-turn task
+PYTEST_TIMEOUT = 3000  # 50 minutes total including task-specific runtime and teardown
 MANIFEST_DYNAMIC_ONLY_TASKS = {"workflow-overlay-contract"}
-CONTROL_BUSINESS_ONLY_TREATMENTS = {"CONTROL"}
-COMET_WORKFLOW_ONLY_CHECK_PREFIXES = (
-    "openspec_artifacts",
-    "comet_state",
-    "workflow_phases",
-    "tests_written",
-    "tests_exist",
-)
 
 
 # =============================================================================
@@ -98,9 +98,7 @@ def generate_test_params(task_filter: str | None, treatment_filter: str | None,
         treatment_list = expand_treatment_patterns(patterns, all_treatments)
     elif dynamic and manifest_tasks:
         treatment_list = [
-            treatment
-            for treatment in manifest_baseline_treatments
-            if treatment in all_treatments
+            treatment for treatment in manifest_baseline_treatments if treatment in all_treatments
         ]
         if dynamic.name not in treatment_list:
             treatment_list.append(dynamic.name)
@@ -130,52 +128,6 @@ def generate_test_params(task_filter: str | None, treatment_filter: str | None,
     return params
 
 
-def _is_control_business_only_run(profile_name: str, treatment_name: str) -> bool:
-    return profile_name == "comet-workflow" and treatment_name in CONTROL_BUSINESS_ONLY_TREATMENTS
-
-
-def _filter_control_workflow_checks(
-    profile_name: str,
-    treatment_name: str,
-    passed: list[str],
-    failed: list[str],
-) -> tuple[list[str], list[str]]:
-    """For CONTROL, evaluate comet tasks by business outcome only.
-
-    CONTROL intentionally has no Skill mounted. OpenSpec/Comet state, phase
-    evidence, and workflow-test discipline are reported by the rubric as
-    non-applicable instead of counted as task failures.
-    """
-    if not _is_control_business_only_run(profile_name, treatment_name):
-        return passed, failed
-
-    def keep(check: str) -> bool:
-        return not any(check.startswith(prefix) for prefix in COMET_WORKFLOW_ONLY_CHECK_PREFIXES)
-
-    return [check for check in passed if keep(check)], [check for check in failed if keep(check)]
-
-
-def _split_comet_completion_checks(
-    passed: list[str],
-    failed: list[str],
-) -> dict[str, dict[str, list[str]]]:
-    """Split validator results into business and workflow completion buckets."""
-
-    def is_workflow_check(check: str) -> bool:
-        return any(check.startswith(prefix) for prefix in COMET_WORKFLOW_ONLY_CHECK_PREFIXES)
-
-    return {
-        "business_completion": {
-            "passed": [check for check in passed if not is_workflow_check(check)],
-            "failed": [check for check in failed if not is_workflow_check(check)],
-        },
-        "workflow_completion": {
-            "passed": [check for check in passed if is_workflow_check(check)],
-            "failed": [check for check in failed if is_workflow_check(check)],
-        },
-    }
-
-
 def test_eval_manifest_baselines_extend_dynamic_treatment_list(tmp_path, monkeypatch):
     package = tmp_path / "manifest-skill"
     package.mkdir()
@@ -215,7 +167,9 @@ interaction:
         "load_treatments",
         lambda: {
             "CONTROL": TreatmentConfig(name="CONTROL", description="Control"),
-            "COMET_FULL_040_BETA": TreatmentConfig(name="COMET_FULL_040_BETA", description="Comet full"),
+            "COMET_FULL_040_BETA": TreatmentConfig(
+                name="COMET_FULL_040_BETA", description="Comet full"
+            ),
         },
     )
 
@@ -242,13 +196,23 @@ def test_control_comet_workflow_filters_workflow_only_checks():
     passed, failed = _filter_control_workflow_checks(
         "comet-workflow",
         "CONTROL",
-        ["sentence_feature", "tests_written: ok", "workflow_phases: 5/5", "tests_exist"],
+        [
+            "sentence_feature",
+            "tests_written: ok",
+            "workflow_phases: 5/5",
+            "tests_exist",
+            "native_skill_invocation",
+            "native_artifacts",
+            "native_trajectory",
+        ],
         [
             "openspec_artifacts: openspec/changes/ directory not found",
             "comet_state: No .comet.yaml found",
             "workflow_phases: Only 1/5 phases",
             "tests_written: No test files written by the agent",
             "tests_exist: No test files found",
+            "native_state: no terminal Native archive exists",
+            "native_isolation: Classic or hidden workflow artifacts exist",
             "sentence_feature: --sentences flag not found",
         ],
     )
@@ -259,10 +223,19 @@ def test_control_comet_workflow_filters_workflow_only_checks():
 
 def test_split_comet_completion_checks_separates_business_and_workflow():
     completion = _split_comet_completion_checks(
-        ["sentence_feature", "tests_exist", "workflow_phases: 5/5"],
+        [
+            "sentence_feature",
+            "tests_exist",
+            "workflow_phases: 5/5",
+            "native_skill_invocation",
+            "native_artifacts",
+            "native_trajectory",
+        ],
         [
             "openspec_artifacts: missing",
             "comet_state: missing",
+            "native_state: incomplete",
+            "native_isolation: forbidden artifacts",
             "business_rule: failed",
         ],
     )
@@ -272,8 +245,19 @@ def test_split_comet_completion_checks_separates_business_and_workflow():
         "failed": ["business_rule: failed"],
     }
     assert completion["workflow_completion"] == {
-        "passed": ["tests_exist", "workflow_phases: 5/5"],
-        "failed": ["openspec_artifacts: missing", "comet_state: missing"],
+        "passed": [
+            "tests_exist",
+            "workflow_phases: 5/5",
+            "native_skill_invocation",
+            "native_artifacts",
+            "native_trajectory",
+        ],
+        "failed": [
+            "openspec_artifacts: missing",
+            "comet_state: missing",
+            "native_state: incomplete",
+            "native_isolation: forbidden artifacts",
+        ],
     }
 
 
@@ -300,13 +284,19 @@ def pytest_generate_tests(metafunc):
         treatment_filter = metafunc.config.getoption("--treatment")
         count = int(metafunc.config.getoption("--count") or 1)
         base_params = generate_test_params(task_filter, treatment_filter, metafunc.config)
-        # pytest ids stay (task, treatment); the rep number is tracked separately
-        # by the experiment plugin's get_rep_number per treatment. To force N
-        # distinct test invocations we append a rep suffix to the param id.
+        # The explicit marker is the controller-owned repetition identity used
+        # to persist a complete matrix before any model run begins.
         params = []
         for rep in range(count):
             for task_name, treatment_name in base_params:
-                params.append(pytest.param(task_name, treatment_name, id=f"{task_name}-{treatment_name}-r{rep+1}"))
+                params.append(
+                    pytest.param(
+                        task_name,
+                        treatment_name,
+                        id=f"{task_name}-{treatment_name}-r{rep + 1}",
+                        marks=pytest.mark.eval_case(repetition=rep + 1),
+                    )
+                )
         metafunc.parametrize("task_name,treatment_name", params)
 
 
@@ -352,7 +342,11 @@ def test_task_treatment(task_name, treatment_name):
     for var_name, var_template in task.config.setup.template_vars.items():
         template_vars[var_name] = var_template.format(run_id=run_id)
 
-    prompt = task.render_prompt(**template_vars)
+    prompt = adapt_prompt_for_native(
+        task.render_prompt(**template_vars),
+        treatment_name,
+        terminal_mode=task.config.evaluation.native_terminal,
+    )
     target_profile = None
     if treatment_cfg.skills:
         target_profile = treatment_cfg.skills[0].get("profile")
@@ -361,20 +355,39 @@ def test_task_treatment(task_name, treatment_name):
         override=fixtures.request_config.getoption("--profile"),
         target_profile=target_profile,
     )
+    interaction = conftest._resolve_interaction_config(task, profile_name, fixtures.request_config)
     fixtures.setup_test_context(
         skills=treatment.skills,
         claude_md=conftest._build_eval_claude_md(profile_name, treatment.claude_md),
         environment_dir=task.environment_dir,
     )
-    interaction = conftest._resolve_interaction_config(task, profile_name, fixtures.request_config)
-    skill_package_path = (
-        conftest._snapshot_dynamic_skill_package(fixtures.test_dir, skill_hints)
-        or skill_hints.get("path")
+    selected_model = conftest._selected_claude_model()
+    captured_execution = conftest._capture_execution_identity(
+        fixtures.test_dir,
+        model=selected_model,
+        interaction=interaction,
+    )
+    case_manifest = case_manifest_payload(
+        build_case_manifest(
+            task.name,
+            task.path.parent,
+            execution_identity=captured_execution.report_identity,
+        )
+    )
+    skill_package_path = conftest._snapshot_dynamic_skill_package(
+        fixtures.test_dir, skill_hints
+    ) or skill_hints.get("path")
+
+    result = fixtures.run_claude(
+        prompt,
+        timeout=max(CLAUDE_TIMEOUT, task.config.timeout_sec),
+        model=selected_model,
+        interaction=interaction,
+        image_id=captured_execution.runtime_image_id,
     )
-
-    result = fixtures.run_claude(prompt, timeout=CLAUDE_TIMEOUT, interaction=interaction)
 
     events = extract_events(parse_output(result.stdout))
+    loop_interaction = conftest._extract_loop_interaction(result.stderr)
     outputs = {
         "run_id": run_id,
         "treatment_name": treatment_name,
@@ -386,8 +399,10 @@ def test_task_treatment(task_name, treatment_name):
         or task.config.evaluation.required_skills,
         "expected_artifacts": skill_hints.get("expected_artifacts")
         or task.config.evaluation.expected_artifacts,
-        "require_skill_invocation": task.config.evaluation.require_skill_invocation,
+        "require_skill_invocation": skill_hints.get("require_skill_invocation")
+        or task.config.evaluation.require_skill_invocation,
         "rubric_criteria": task.config.evaluation.rubric_criteria,
+        "native_terminal": task.config.evaluation.native_terminal,
         "skill_package_path": skill_package_path,
         "generated_node_skills": skill_hints.get("generated_node_skills") or [],
         "route_conformance_task": skill_hints.get("route_conformance_task"),
@@ -402,14 +417,23 @@ def test_task_treatment(task_name, treatment_name):
         "interaction": {
             "mode": interaction.mode,
             "max_turns": interaction.max_turns,
+            **loop_interaction,
         },
+        "case_manifest": case_manifest,
     }
     events["profile"] = outputs["profile"]
     events["skill_sources"] = outputs["skill_sources"]
     events["eval_manifest"] = outputs["eval_manifest"]
     events["interaction"] = outputs["interaction"]
+    events["case_manifest"] = case_manifest
 
     passed, failed = run_validators(validators, fixtures.test_dir, outputs)
+    passed, failed = adapt_checks_for_native(
+        fixtures.test_dir,
+        outputs,
+        passed,
+        failed,
+    )
     completion_slices = _split_comet_completion_checks(passed, failed)
     passed, failed = _filter_control_workflow_checks(
         profile_name,
@@ -427,7 +451,9 @@ def test_task_treatment(task_name, treatment_name):
         rubric_outputs.update(completion_slices)
         if _is_control_business_only_run(profile_name, treatment_name):
             rubric_outputs["workflow_completion"] = {"passed": [], "failed": []}
-    rubric_passed, rubric_failed = run_profile_rubric(profile_name, fixtures.test_dir, rubric_outputs)
+    rubric_passed, rubric_failed = run_profile_rubric(
+        profile_name, fixtures.test_dir, rubric_outputs
+    )
     passed = passed + rubric_passed
     failed = failed + rubric_failed
 

package/eval/local/tests/tasks/test_validation_scripts.py

@@ -3,6 +3,8 @@
 from __future__ import annotations
 
 import importlib.util
+import os
+import subprocess
 import sys
 import types
 from pathlib import Path
@@ -11,6 +13,41 @@ from pathlib import Path
 ROOT = Path(__file__).resolve().parents[3]
 
 
+def test_native_wave_validator_import_does_not_load_host_docker_helpers(tmp_path: Path):
+    from scaffold.python import utils
+
+    stale_package = tmp_path / "scaffold/python"
+    stale_package.mkdir(parents=True)
+    (tmp_path / "scaffold/__init__.py").write_text("", encoding="utf-8")
+    (stale_package / "__init__.py").write_text(
+        "raise RuntimeError('host scaffold package leaked into validator')\n",
+        encoding="utf-8",
+    )
+    utils._copy_scaffold_to_docker(tmp_path)
+    env = os.environ.copy()
+    env["PYTHONPATH"] = str(tmp_path)
+    result = subprocess.run(
+        [
+            sys.executable,
+            "-c",
+            (
+                "import sys; "
+                "import scaffold.python.validation.native_wave; "
+                "assert 'scaffold.python.validation.docker' not in sys.modules"
+            ),
+        ],
+        cwd=tmp_path,
+        capture_output=True,
+        text=True,
+        encoding="utf-8",
+        errors="replace",
+        check=False,
+        env=env,
+    )
+
+    assert result.returncode == 0, result.stderr
+
+
 def _load_validator(path: Path, workspace: Path):
     fake_checks = types.ModuleType("comet_checks")
     fake_checks.WORKSPACE = workspace