@rpamis/comet 0.4.0-beta.6 → 0.4.0-beta.7

package/eval/local/tests/scaffold/test_profiles.py

@@ -576,6 +576,23 @@ def test_generic_rubric_with_required_skills_scores_numeric(tmp_path: Path):
     assert any("[RUBRIC] artifact_presence: N/A -" in msg for msg in passed)
 
 
+def test_generic_interaction_compliance_uses_driver_turns(tmp_path: Path):
+    outputs = {
+        "completion": {"passed": ["ok"], "failed": []},
+        "events": {"num_turns": 28, "tool_calls": [], "duration_seconds": 5, "commands_run": []},
+        "required_skills": [],
+        "expected_artifacts": [],
+        "interaction": {"mode": "auto_user", "max_turns": 4, "actual_turns": 4},
+    }
+
+    passed, _ = run_profile_rubric("generic", tmp_path, outputs)
+
+    assert any(
+        "[RUBRIC] interaction_compliance: 1.00 - turns=4, max=4" in message
+        for message in passed
+    )
+
+
 def test_comet_control_marks_workflow_dimensions_not_applicable(tmp_path: Path):
     """CONTROL should score business completion without requiring Comet Skill use."""
     outputs = {
@@ -697,6 +714,22 @@ def test_generic_llm_judge_collects_workspace_files(tmp_path: Path):
     assert ".git" not in artifacts
 
 
+def test_generic_llm_judge_ignores_harness_transport_files(tmp_path: Path):
+    """The judge should use adapted completion output, not stale container transport."""
+    from scaffold.python.generic_llm_judge import _collect_workspace_artifacts
+
+    (tmp_path / "result.md").write_text("# Result\nDone")
+    (tmp_path / "_test_context.json").write_text('{"treatment_name":"legacy"}')
+    (tmp_path / "_test_results.json").write_text('{"failed":["openspec missing"]}')
+
+    artifacts = _collect_workspace_artifacts(tmp_path)
+
+    assert "result.md" in artifacts
+    assert "_test_context.json" not in artifacts
+    assert "_test_results.json" not in artifacts
+    assert "openspec missing" not in artifacts
+
+
 def test_judge_env_requires_explicit_judge_model(monkeypatch):
     """LLM judge must not silently reuse the subject model."""
     from scaffold.python.judge_config import build_judge_invocation

package/eval/local/tests/scaffold/test_sample_quality.py

@@ -33,6 +33,37 @@ def test_rate_limit_is_excluded_from_analysis():
     assert quality.include_in_analysis is False
 
 
+def test_structured_connection_failure_outranks_rate_limit_wording_in_prompt():
+    stderr = "\n".join(
+        [
+            json.dumps(
+                {
+                    "type": "user",
+                    "message": {"content": "Document rate limit behavior."},
+                }
+            ),
+            json.dumps(
+                {
+                    "type": "result",
+                    "is_error": True,
+                    "terminal_reason": "api_error",
+                    "result": "API Error: Connection closed mid-response.",
+                }
+            ),
+        ]
+    )
+
+    quality = infer_sample_quality(
+        checks_failed=["Required skill not invoked"],
+        stderr=stderr,
+        returncode=1,
+    )
+
+    assert quality.status == "excluded"
+    assert quality.reason_code == "network_failure"
+    assert quality.include_in_analysis is False
+
+
 def test_container_failure_is_excluded_from_analysis():
     quality = infer_sample_quality(stderr="ERROR: Docker daemon not running")
 
@@ -93,6 +124,32 @@ def test_completed_run_with_container_wording_in_logs_stays_in_analysis():
     assert quality.include_in_analysis is True
 
 
+def test_completed_run_with_container_wording_only_in_result_is_valid_signal():
+    quality = infer_sample_quality(
+        events={
+            "duration_seconds": 42,
+            "total_tokens": 1000,
+            "total_cost_usd": 0.12,
+        },
+        checks_failed=[],
+        stdout=(
+            json.dumps(
+                {
+                    "type": "result",
+                    "duration_ms": 42000,
+                    "result": "The Docker container workflow completed successfully.",
+                }
+            )
+            + "\n"
+        ),
+        stderr="[loop] workflow completion detected; ending\n",
+        returncode=0,
+    )
+
+    assert quality.status == "included"
+    assert quality.reason_code == "valid_signal"
+
+
 def test_validator_failure_with_observable_run_is_included():
     quality = infer_sample_quality(
         events={

package/eval/local/tests/scaffold/test_tasks.py

@@ -1,5 +1,9 @@
 """Unit tests for the comet eval task loader."""
 
+import importlib.util
+import json
+import shutil
+import subprocess
 from pathlib import Path
 
 import pytest
@@ -76,13 +80,16 @@ profile = "generic"
 required_skills = ["target-skill"]
 expected_artifacts = ["result.json"]
 require_skill_invocation = true
+native_terminal = "active-blocked"
 
 [interaction]
 mode = "auto_user"
 max_turns = 7
 simulator_prompt = "Answer as a concise developer user."
 decision_patterns = ["confirm", "choose"]
+decision_reply = "Use the recommended option."
 continue_prompt = "Please continue."
+fresh_resume_marker = "COLD_RESUME_READY"
 """
     )
 
@@ -92,11 +99,14 @@ continue_prompt = "Please continue."
     assert task.config.evaluation.required_skills == ["target-skill"]
     assert task.config.evaluation.expected_artifacts == ["result.json"]
     assert task.config.evaluation.require_skill_invocation is True
+    assert task.config.evaluation.native_terminal == "active-blocked"
     assert task.config.interaction.mode == "auto_user"
     assert task.config.interaction.max_turns == 7
     assert task.config.interaction.simulator_prompt == "Answer as a concise developer user."
     assert task.config.interaction.decision_patterns == ["confirm", "choose"]
+    assert task.config.interaction.decision_reply == "Use the recommended option."
     assert task.config.interaction.continue_prompt == "Please continue."
+    assert task.config.interaction.fresh_resume_marker == "COLD_RESUME_READY"
 
 
 def test_comet_tasks_default_to_comet_workflow_profile():
@@ -143,7 +153,7 @@ def test_comet_task_index_lists_real_tasks():
     index = yaml.safe_load(index_path.read_text(encoding="utf-8"))
     names = [task["name"] for task in index["tasks"]]
     assert sorted(names) == sorted(list_tasks())
-    assert len(names) == 20
+    assert len(names) == 30
     assert set(names) == {
         "authoring-skill-smoke",
         "comet-agent-memory-routing",
@@ -161,8 +171,374 @@ def test_comet_task_index_lists_real_tasks():
         "comet-refactor-counter",
         "comet-robust-config",
         "comet-noise-distractor",
+        "comet-native-workflow",
+        "comet-native-clarification",
+        "comet-native-clarification-modes",
+        "comet-native-repository-fact",
+        "comet-native-interrupted-transition",
+        "comet-native-wave-b-decision-resume",
+        "comet-native-wave-c-verification-integrity",
+        "comet-native-wave-d-stagnation-stop",
+        "comet-native-wave-e-parallel-safety",
+        "comet-native-wave-f-dashboard-readonly",
         "comet-observability-env-template",
         "generic-skill-smoke",
         "workflow-overlay-contract",
         "workflow-route-conformance",
     }
+
+
+def test_native_task_uses_its_own_skill_contract():
+    task = load_task("comet-native-workflow")
+
+    assert task.config.evaluation.profile == "generic"
+    assert task.config.evaluation.required_skills == ["comet-native"]
+    assert task.config.evaluation.require_skill_invocation is True
+    assert task.config.interaction.mode == "none"
+    prompt = task.render_prompt()
+    assert prompt.startswith("You are working on a Python project")
+    assert "Begin by invoking the `/comet-native` Skill" in prompt
+    assert "/comet` Skill/slash command" not in prompt
+
+
+def test_native_clarification_task_requires_one_decision_and_confirmed_resume():
+    task = load_task("comet-native-clarification")
+
+    assert task.config.evaluation.profile == "generic"
+    assert task.config.evaluation.required_skills == ["comet-native"]
+    assert task.config.interaction.mode == "auto_user"
+    assert task.config.interaction.max_turns == 4
+    assert task.config.interaction.decision_reply == (
+        "Abbreviations such as e.g. and Dr. do not end a sentence; use a small explicit abbreviation list."
+    )
+    assert "docs/comet/specs/sentence-counting/spec.md" not in (
+        task.config.evaluation.expected_artifacts
+    )
+    assert "verify the Native status from disk" in task.config.interaction.continue_prompt
+    assert "one highest-value question" in task.config.interaction.simulator_prompt
+    prompt = task.render_prompt()
+    assert "Do not guess how abbreviations should behave" in prompt
+    assert "Do not begin implementation before the user answers" in prompt
+
+
+def test_native_clarification_modes_task_compares_batch_and_sequential():
+    task = load_task("comet-native-clarification-modes")
+
+    assert task.default_treatments == ["COMET_NATIVE_SEQUENTIAL", "COMET_NATIVE_BATCH"]
+    assert task.config.evaluation.profile == "generic"
+    assert task.config.evaluation.required_skills == ["comet-native"]
+    assert task.config.interaction.mode == "auto_user"
+    assert task.config.interaction.max_turns == 6
+    assert task.config.timeout_sec == 2400
+    assert task.config.interaction.decision_reply is None
+    assert "answer only the questions present" in task.config.interaction.simulator_prompt.lower()
+    prompt = task.render_prompt()
+    assert "three independent product decisions" in prompt
+    assert "Do not choose the clarification mode" in prompt
+
+
+@pytest.mark.parametrize(
+    ("treatment", "mode", "decision_points"),
+    [
+        ("COMET_NATIVE_SEQUENTIAL", "sequential", 3),
+        ("COMET_NATIVE_BATCH", "batch", 2),
+    ],
+)
+def test_native_clarification_modes_validator_accepts_expected_rounds(
+    tmp_path: Path,
+    treatment: str,
+    mode: str,
+    decision_points: int,
+):
+    validator_path = (
+        get_tasks_dir()
+        / "comet-native-clarification-modes"
+        / "validation"
+        / "test_native_clarification_modes.py"
+    )
+    spec = importlib.util.spec_from_file_location(
+        "native_clarification_modes_validator", validator_path
+    )
+    assert spec and spec.loader
+    validator = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(validator)
+    validator.WORKSPACE = tmp_path
+
+    config = tmp_path / ".comet" / "config.yaml"
+    config.parent.mkdir(parents=True)
+    config.write_text(
+        f"schema: comet.project.v1\nnative:\n  clarification_mode: {mode}\n",
+        encoding="utf-8",
+    )
+    (tmp_path / "_test_context.json").write_text(
+        json.dumps(
+            {
+                "treatment_name": treatment,
+                "interaction": {
+                    "mode": "auto_user",
+                    "actual_turns": decision_points + 1,
+                    "decision_points": decision_points,
+                    "deterministic_replies": 0,
+                },
+            }
+        ),
+        encoding="utf-8",
+    )
+
+    assert validator.check_mode_and_interaction()["status"] == "passed"
+
+
+def test_native_clarification_modes_validator_accepts_semantic_decision_wording_and_approval():
+    validator_path = (
+        get_tasks_dir()
+        / "comet-native-clarification-modes"
+        / "validation"
+        / "test_native_clarification_modes.py"
+    )
+    spec = importlib.util.spec_from_file_location(
+        "native_clarification_modes_semantic_validator", validator_path
+    )
+    assert spec and spec.loader
+    validator = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(validator)
+
+    text = """
+### Scenario: Empty input
+Given input `""`, the function returns `0`.
+### Scenario: Abbreviations do not end sentences
+Known abbreviations such as `Dr.` are ignored as sentence boundaries.
+### Scenario: Punctuation runs
+A contiguous run of terminal punctuation (`.`, `!`, `?`) counts as exactly one boundary.
+"""
+
+    assert validator.missing_decisions(text) == []
+    assert validator.approval_is_valid("sequential", "implicit") is True
+    assert validator.approval_is_valid("sequential", "confirmed") is True
+    assert validator.approval_is_valid("batch", "confirmed") is True
+    assert validator.approval_is_valid("batch", "implicit") is False
+
+
+def test_native_clarification_validator_rejects_multiple_archives(tmp_path: Path):
+    validator_path = (
+        get_tasks_dir() / "comet-native-clarification" / "validation" / "test_native_clarification.py"
+    )
+    spec = importlib.util.spec_from_file_location("native_clarification_validator", validator_path)
+    assert spec and spec.loader
+    validator = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(validator)
+    validator.WORKSPACE = tmp_path
+
+    archive_root = tmp_path / "docs" / "comet" / "archive"
+    (archive_root / "2026-07-15-first-change").mkdir(parents=True)
+    current = archive_root / "2026-07-15-second-change"
+    current.mkdir()
+    (current / "comet-state.yaml").write_text("approval: confirmed\n", encoding="utf-8")
+    (current / "brief.md").write_text("Abbreviation decision confirmed.\n", encoding="utf-8")
+    canonical = tmp_path / "docs" / "comet" / "specs" / "sentence-counting" / "spec.md"
+    canonical.parent.mkdir(parents=True)
+    canonical.write_text("# Sentence counting\n", encoding="utf-8")
+
+    result = validator.check_confirmed_archive()
+
+    assert result["status"] == "failed"
+    assert "exactly one Native archive" in result["reason"]
+
+
+def test_native_clarification_validator_accepts_one_semantic_canonical_spec(tmp_path: Path):
+    validator_path = (
+        get_tasks_dir() / "comet-native-clarification" / "validation" / "test_native_clarification.py"
+    )
+    spec = importlib.util.spec_from_file_location("native_clarification_validator", validator_path)
+    assert spec and spec.loader
+    validator = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(validator)
+    validator.WORKSPACE = tmp_path
+
+    archived = tmp_path / "docs" / "comet" / "archive" / "2026-07-15-add-sentences"
+    archived.mkdir(parents=True)
+    (archived / "comet-state.yaml").write_text(
+        """schema: comet.native.v1
+name: add-sentences
+phase: archive
+approval: confirmed
+spec_changes:
+  - capability: sentences
+    operation: create
+    source: specs/sentences/spec.md
+    base_hash: null
+verification_result: pass
+verification_report: verification.md
+archived: true
+""",
+        encoding="utf-8",
+    )
+    (archived / "brief.md").write_text(
+        "# Decisions\nAbbreviations such as e.g. and Dr. do not end a sentence.\n",
+        encoding="utf-8",
+    )
+    canonical = tmp_path / "docs" / "comet" / "specs" / "sentences" / "spec.md"
+    canonical.parent.mkdir(parents=True)
+    canonical.write_text(
+        "# Sentence counting\nThe --sentences flag ignores e.g. and Dr. boundaries.\n",
+        encoding="utf-8",
+    )
+    archived_spec = archived / "specs" / "sentences" / "spec.md"
+    archived_spec.parent.mkdir(parents=True)
+    archived_spec.write_text(canonical.read_text(encoding="utf-8"), encoding="utf-8")
+    (archived / "verification.md").write_text(
+        "# Commands and results\npytest: 24 passed\n# Conclusion\npass\n",
+        encoding="utf-8",
+    )
+    trajectory = archived / "runtime" / "trajectory.jsonl"
+    trajectory.parent.mkdir(parents=True)
+    trajectory.write_text(
+        "\n".join(
+            json.dumps(
+                {
+                    "type": "state_transitioned",
+                    "data": {"previousPhase": previous, "nextPhase": following},
+                }
+            )
+            for previous, following in [
+                ("shape", "build"),
+                ("build", "verify"),
+                ("verify", "archive"),
+                ("archive", None),
+            ]
+        ),
+        encoding="utf-8",
+    )
+    (tmp_path / "_test_context.json").write_text(
+        json.dumps(
+            {
+                "interaction": {
+                    "mode": "auto_user",
+                    "actual_turns": 2,
+                    "max_turns": 4,
+                    "decision_points": 1,
+                    "deterministic_replies": 1,
+                    "completion_signals": 1,
+                }
+            }
+        ),
+        encoding="utf-8",
+    )
+
+    result = validator.check_confirmed_archive()
+
+    assert result["status"] == "passed"
+
+
+def test_native_clarification_validator_rejects_leftover_active_change(tmp_path: Path):
+    validator_path = (
+        get_tasks_dir() / "comet-native-clarification" / "validation" / "test_native_clarification.py"
+    )
+    spec = importlib.util.spec_from_file_location("native_clarification_validator", validator_path)
+    assert spec and spec.loader
+    validator = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(validator)
+    validator.WORKSPACE = tmp_path
+
+    archive = tmp_path / "docs" / "comet" / "archive" / "2026-07-15-add-sentences"
+    archive.mkdir(parents=True)
+    active = tmp_path / "docs" / "comet" / "changes" / "add-sentences"
+    active.mkdir(parents=True)
+
+    result = validator.check_confirmed_archive()
+
+    assert result["status"] == "failed"
+    assert "active change" in result["reason"]
+
+
+def test_native_repository_fact_task_requires_investigation_without_interaction():
+    task = load_task("comet-native-repository-fact")
+
+    assert task.config.evaluation.required_skills == ["comet-native"]
+    assert task.config.interaction.mode == "none"
+    prompt = task.render_prompt()
+    assert "already documented somewhere in this repository" in prompt
+    assert "do not ask the user to repeat repository facts" in prompt
+
+
+def test_native_interrupted_transition_task_requires_runtime_recovery():
+    task = load_task("comet-native-interrupted-transition")
+
+    assert task.config.evaluation.required_skills == ["comet-native"]
+    assert task.config.interaction.mode == "none"
+    prompt = task.render_prompt()
+    assert "recover the existing `add-character-counting` change" in prompt
+    assert "do not create a replacement change" in prompt
+
+
+def test_native_interrupted_transition_fixture_is_recovered_by_current_runtime(tmp_path: Path):
+    task = load_task("comet-native-interrupted-transition")
+    workspace = tmp_path / "workspace"
+    shutil.copytree(task.environment_dir, workspace)
+    runtime = get_tasks_dir().parents[2] / "assets/skills/comet-native/scripts/comet-native-runtime.mjs"
+    change = workspace / "docs/comet/changes/add-character-counting"
+
+    def run_native(*args: str, expected_exit: int = 0) -> dict:
+        result = subprocess.run(
+            [
+                "node",
+                str(runtime),
+                *args,
+                "--json",
+                "--project-root",
+                str(workspace),
+            ],
+            check=False,
+            capture_output=True,
+            text=True,
+            timeout=30,
+        )
+        assert result.returncode == expected_exit, result.stdout or result.stderr
+        return json.loads(result.stdout)
+
+    status = run_native("status", "add-character-counting")
+
+    assert status["data"]["name"] == "add-character-counting"
+    assert status["data"]["phase"] == "shape"
+    assert status["data"]["migrationRequired"] is True
+
+    repaired = run_native(
+        "doctor",
+        "add-character-counting",
+        "--repair",
+        "--strategy",
+        "continue",
+        expected_exit=65,
+    )
+
+    assert {finding["code"] for finding in repaired["data"]["findings"]} >= {
+        "schema-migrated",
+        "transition-recovered",
+        "contract-changed-after-approval",
+    }
+    state = yaml.safe_load((change / "comet-state.yaml").read_text(encoding="utf-8"))
+    assert state["phase"] == "build"
+    assert state["run_id"] == "native-recovery-eval-run"
+    assert not (change / "runtime/transition.json").exists()
+
+    run_native(
+        "doctor",
+        "add-character-counting",
+        "--repair",
+        "--strategy",
+        "continue",
+        expected_exit=65,
+    )
+
+    events = [
+        json.loads(line)
+        for line in (change / "runtime/trajectory.jsonl").read_text(encoding="utf-8").splitlines()
+        if line.strip()
+    ]
+    recovered = [
+        event
+        for event in events
+        if event.get("type") == "state_transitioned"
+        and event.get("data", {}).get("transitionId")
+        == "11111111-2222-4333-8444-555555555555"
+    ]
+    assert len(recovered) == 1

package/eval/local/tests/scaffold/test_treatments.py

@@ -92,15 +92,20 @@ def test_build_treatment_skills_rejects_path_without_skill_md(tmp_path: Path):
     skill_dir.mkdir()
 
     with pytest.raises(FileNotFoundError, match="SKILL.md"):
-        build_treatment_skills(
-            [{"name": "broken-skill", "source": "path", "path": str(skill_dir)}]
-        )
+        build_treatment_skills([{"name": "broken-skill", "source": "path", "path": str(skill_dir)}])
 
 
 def test_load_treatments_keeps_comet_core_categories_only():
     treatments = load_treatments()
 
-    assert set(treatments) == {"CONTROL", "COMET_FULL_040_BETA", "COMET_FULL_039"}
+    assert set(treatments) == {
+        "CONTROL",
+        "COMET_FULL_040_BETA",
+        "COMET_FULL_039",
+        "COMET_NATIVE_BATCH",
+        "COMET_NATIVE_PHASE1",
+        "COMET_NATIVE_SEQUENTIAL",
+    }
     assert all(isinstance(treatment, TreatmentConfig) for treatment in treatments.values())
 
 
@@ -149,14 +154,52 @@ def test_comet_full_039_includes_same_dependency_snapshot():
     assert names.issuperset(_benchmark_child_names("dependency", "superpowers"))
 
 
+def test_comet_native_phase1_is_self_contained():
+    treatment = load_treatments()["COMET_NATIVE_PHASE1"]
+
+    assert {skill["name"] for skill in treatment.skills} == {"comet-native"}
+    assert treatment.skills[0]["source"] == "path"
+    assert "assets/skills/comet-native" in treatment.skills[0]["path"]
+    assert treatment.skills[0]["profile"] == "generic"
+    assert treatment.skills[0]["required_skills"] == ["comet-native"]
+    assert treatment.skills[0]["require_skill_invocation"] is True
+    assert "Shape, Build, Verify, and Archive" in treatment.claude_md
+    assert "OpenSpec" in treatment.claude_md
+    normalized_claude_md = " ".join(treatment.claude_md.split())
+    assert "continue automatically while no user decision is unresolved" in normalized_claude_md
+    assert "explicit execution boundary" in normalized_claude_md
+    assert "takes priority over automatic progression" in normalized_claude_md
+    assert "material user decision" not in normalized_claude_md
+
+    skills = build_treatment_skills(treatment.skills)
+    assert set(skills) == {"comet-native"}
+    assert skills["comet-native"]["scripts_dir"].name == "scripts"
+    assert skills["comet-native"]["source_dir"].name == "comet-native"
+
+
+@pytest.mark.parametrize(
+    ("name", "mode"),
+    [
+        ("COMET_NATIVE_SEQUENTIAL", "sequential"),
+        ("COMET_NATIVE_BATCH", "batch"),
+    ],
+)
+def test_comet_native_clarification_treatments_are_self_contained(name: str, mode: str):
+    treatment = load_treatments()[name]
+
+    assert {skill["name"] for skill in treatment.skills} == {"comet-native"}
+    assert treatment.skills[0]["source"] == "path"
+    assert treatment.skills[0]["profile"] == "generic"
+    assert f"clarification_mode `{mode}`" in treatment.claude_md
+    assert "Do not change the configured clarification mode" in treatment.claude_md
+
+
 def test_comet_treatments_point_at_versioned_comet_snapshots():
     COMET_FULL_040_BETA = load_treatments()["COMET_FULL_040_BETA"]
     comet_039 = load_treatments()["COMET_FULL_039"]
 
     assert {
-        skill["skill"]
-        for skill in COMET_FULL_040_BETA.skills
-        if skill["name"].startswith("comet")
+        skill["skill"] for skill in COMET_FULL_040_BETA.skills if skill["name"].startswith("comet")
     } == {
         "040-beta/comet",
         "040-beta/comet-open",
@@ -167,11 +210,7 @@ def test_comet_treatments_point_at_versioned_comet_snapshots():
         "040-beta/comet-hotfix",
         "040-beta/comet-tweak",
     }
-    assert {
-        skill["skill"]
-        for skill in comet_039.skills
-        if skill["name"].startswith("comet")
-    } == {
+    assert {skill["skill"] for skill in comet_039.skills if skill["name"].startswith("comet")} == {
         "039-release/comet-classic-039",
         "039-release/comet-classic-039-open",
         "039-release/comet-classic-039-design",
@@ -241,4 +280,11 @@ def test_comet_full_040_beta_dependency_paths_are_loadable():
 
 
 def test_list_treatments_is_sorted_for_stable_cli_output():
-    assert list_treatments() == ["COMET_FULL_039", "COMET_FULL_040_BETA", "CONTROL"]
+    assert list_treatments() == [
+        "COMET_FULL_039",
+        "COMET_FULL_040_BETA",
+        "COMET_NATIVE_BATCH",
+        "COMET_NATIVE_PHASE1",
+        "COMET_NATIVE_SEQUENTIAL",
+        "CONTROL",
+    ]