@rpamis/comet 0.3.9 → 0.4.0-beta.10

package/eval/local/treatments/comet/comet_classic_docs_layout.yaml

@@ -0,0 +1,121 @@
+# Current Classic treatment for the unified docs catalogue. Frozen 039-release
+# and 040-beta snapshots remain unchanged.
+COMET_CLASSIC_DOCS_LAYOUT:
+  description: 'Current Comet Classic workflow with docs/openspec and docs/superpowers roots'
+  claude_md: |
+    Begin by invoking /comet-classic. Initialize a new Classic project with
+    classic.artifact_layout `docs`. Resolve paths with `comet classic root show`
+    and execute every OpenSpec command through `comet classic openspec --`.
+    The terminal project must use docs/openspec and docs/superpowers without
+    creating a second root-level openspec directory.
+  skills: &current_classic_skills
+    - name: comet
+      source: path
+      path: ../assets/skills/comet
+    - name: comet-classic
+      source: path
+      path: ../assets/skills/comet-classic
+    - name: comet-open
+      source: path
+      path: ../assets/skills/comet-open
+    - name: comet-design
+      source: path
+      path: ../assets/skills/comet-design
+    - name: comet-build
+      source: path
+      path: ../assets/skills/comet-build
+    - name: comet-verify
+      source: path
+      path: ../assets/skills/comet-verify
+    - name: comet-archive
+      source: path
+      path: ../assets/skills/comet-archive
+    - name: comet-hotfix
+      source: path
+      path: ../assets/skills/comet-hotfix
+    - name: comet-tweak
+      source: path
+      path: ../assets/skills/comet-tweak
+    - name: openspec-explore
+      skill: dependency/openspec/openspec-explore
+      variant: all
+      base: benchmarks
+    - name: openspec-new-change
+      skill: dependency/openspec/openspec-new-change
+      variant: all
+      base: benchmarks
+    - name: openspec-continue-change
+      skill: dependency/openspec/openspec-continue-change
+      variant: all
+      base: benchmarks
+    - name: openspec-apply-change
+      skill: dependency/openspec/openspec-apply-change
+      variant: all
+      base: benchmarks
+    - name: openspec-sync-specs
+      skill: dependency/openspec/openspec-sync-specs
+      variant: all
+      base: benchmarks
+    - name: openspec-archive-change
+      skill: dependency/openspec/openspec-archive-change
+      variant: all
+      base: benchmarks
+    - name: openspec-verify-change
+      skill: dependency/openspec/openspec-verify-change
+      variant: all
+      base: benchmarks
+    - name: brainstorming
+      skill: dependency/superpowers/brainstorming
+      variant: all
+      base: benchmarks
+    - name: writing-plans
+      skill: dependency/superpowers/writing-plans
+      variant: all
+      base: benchmarks
+    - name: executing-plans
+      skill: dependency/superpowers/executing-plans
+      variant: all
+      base: benchmarks
+    - name: subagent-driven-development
+      skill: dependency/superpowers/subagent-driven-development
+      variant: all
+      base: benchmarks
+    - name: test-driven-development
+      skill: dependency/superpowers/test-driven-development
+      variant: all
+      base: benchmarks
+    - name: systematic-debugging
+      skill: dependency/superpowers/systematic-debugging
+      variant: all
+      base: benchmarks
+    - name: verification-before-completion
+      skill: dependency/superpowers/verification-before-completion
+      variant: all
+      base: benchmarks
+    - name: using-git-worktrees
+      skill: dependency/superpowers/using-git-worktrees
+      variant: all
+      base: benchmarks
+    - name: requesting-code-review
+      skill: dependency/superpowers/requesting-code-review
+      variant: all
+      base: benchmarks
+    - name: receiving-code-review
+      skill: dependency/superpowers/receiving-code-review
+      variant: all
+      base: benchmarks
+    - name: finishing-a-development-branch
+      skill: dependency/superpowers/finishing-a-development-branch
+      variant: all
+      base: benchmarks
+
+COMET_CLASSIC_LEGACY_LAYOUT:
+  description: 'Current Comet Classic workflow with the explicit root-level legacy OpenSpec layout'
+  claude_md: |
+    Begin by invoking /comet-classic. This is the current legacy-compatibility
+    scenario: initialize the project with an explicit
+    classic.artifact_layout `legacy`, resolve paths with
+    `comet classic root show`, and execute every OpenSpec command through
+    `comet classic openspec --`. The terminal project must use the root-level
+    openspec directory and docs/superpowers without creating docs/openspec.
+  skills: *current_classic_skills

package/eval/local/treatments/comet/comet_full_039.yaml

@@ -0,0 +1,141 @@
+# Full comet skill treatment (0.3.9 baseline, .sh runtime)
+# Injects: the 0.3.9 Comet snapshot plus the same benchmark dependency
+# snapshots used by the 0.4.0 treatment. Source dirs are prefixed
+# comet-classic-039* to avoid colliding with the 0.4.0 sources, but the
+# injected skill names are restored to comet/comet-open/etc. so the 0.3.9
+# boilerplate `find ... */comet/scripts/comet-env.sh` resolves.
+COMET_FULL_039:
+  description: "0.3.9 baseline comet skill (shell runtime) with Comet, OpenSpec, and Superpowers dependencies"
+  skills:
+    # Main dispatcher (carries scripts/ with the frozen 0.3.9 .sh launchers)
+    - name: comet
+      skill: 039-release/comet-classic-039
+      variant: all
+      base: benchmarks
+    # Phase sub-skills — source is the 0.3.9 snapshot, injected under canonical names
+    - name: comet-open
+      skill: 039-release/comet-classic-039-open
+      variant: all
+      base: benchmarks
+    - name: comet-design
+      skill: 039-release/comet-classic-039-design
+      variant: all
+      base: benchmarks
+    - name: comet-build
+      skill: 039-release/comet-classic-039-build
+      variant: all
+      base: benchmarks
+    - name: comet-verify
+      skill: 039-release/comet-classic-039-verify
+      variant: all
+      base: benchmarks
+    - name: comet-archive
+      skill: 039-release/comet-classic-039-archive
+      variant: all
+      base: benchmarks
+    - name: comet-hotfix
+      skill: 039-release/comet-classic-039-hotfix
+      variant: all
+      base: benchmarks
+    - name: comet-tweak
+      skill: 039-release/comet-classic-039-tweak
+      variant: all
+      base: benchmarks
+    # OpenSpec workflow dependencies (benchmark dependency snapshots).
+    - name: openspec-apply-change
+      skill: dependency/openspec/openspec-apply-change
+      variant: all
+      base: benchmarks
+    - name: openspec-archive-change
+      skill: dependency/openspec/openspec-archive-change
+      variant: all
+      base: benchmarks
+    - name: openspec-bulk-archive-change
+      skill: dependency/openspec/openspec-bulk-archive-change
+      variant: all
+      base: benchmarks
+    - name: openspec-continue-change
+      skill: dependency/openspec/openspec-continue-change
+      variant: all
+      base: benchmarks
+    - name: openspec-explore
+      skill: dependency/openspec/openspec-explore
+      variant: all
+      base: benchmarks
+    - name: openspec-ff-change
+      skill: dependency/openspec/openspec-ff-change
+      variant: all
+      base: benchmarks
+    - name: openspec-new-change
+      skill: dependency/openspec/openspec-new-change
+      variant: all
+      base: benchmarks
+    - name: openspec-onboard
+      skill: dependency/openspec/openspec-onboard
+      variant: all
+      base: benchmarks
+    - name: openspec-propose
+      skill: dependency/openspec/openspec-propose
+      variant: all
+      base: benchmarks
+    - name: openspec-sync-specs
+      skill: dependency/openspec/openspec-sync-specs
+      variant: all
+      base: benchmarks
+    - name: openspec-verify-change
+      skill: dependency/openspec/openspec-verify-change
+      variant: all
+      base: benchmarks
+    # Superpowers dependencies used by Comet phase guidance and recovery paths.
+    - name: brainstorming
+      skill: dependency/superpowers/brainstorming
+      variant: all
+      base: benchmarks
+    - name: dispatching-parallel-agents
+      skill: dependency/superpowers/dispatching-parallel-agents
+      variant: all
+      base: benchmarks
+    - name: executing-plans
+      skill: dependency/superpowers/executing-plans
+      variant: all
+      base: benchmarks
+    - name: finishing-a-development-branch
+      skill: dependency/superpowers/finishing-a-development-branch
+      variant: all
+      base: benchmarks
+    - name: receiving-code-review
+      skill: dependency/superpowers/receiving-code-review
+      variant: all
+      base: benchmarks
+    - name: requesting-code-review
+      skill: dependency/superpowers/requesting-code-review
+      variant: all
+      base: benchmarks
+    - name: subagent-driven-development
+      skill: dependency/superpowers/subagent-driven-development
+      variant: all
+      base: benchmarks
+    - name: systematic-debugging
+      skill: dependency/superpowers/systematic-debugging
+      variant: all
+      base: benchmarks
+    - name: test-driven-development
+      skill: dependency/superpowers/test-driven-development
+      variant: all
+      base: benchmarks
+    - name: using-git-worktrees
+      skill: dependency/superpowers/using-git-worktrees
+      variant: all
+      base: benchmarks
+    - name: verification-before-completion
+      skill: dependency/superpowers/verification-before-completion
+      variant: all
+      base: benchmarks
+    - name: writing-plans
+      skill: dependency/superpowers/writing-plans
+      variant: all
+      base: benchmarks
+    - name: writing-skills
+      skill: dependency/superpowers/writing-skills
+      variant: all
+      base: benchmarks

package/eval/local/treatments/comet/comet_full_040_beta.yaml

@@ -0,0 +1,140 @@
+# Full comet skill treatment (workflow 0.4.0-beta.1, .mjs runtime)
+# Injects: the 0.4.0 beta Comet snapshot plus the benchmark dependency
+# snapshots under dependency/openspec and dependency/superpowers.
+# OpenSpec CLI is still provided by the task Docker image; OpenSpec Skills are
+# injected so slash/Skill invocation behavior matches a real installation.
+COMET_FULL_040_BETA:
+  description: "Full comet skill (workflow 0.4.0-beta.1) with Comet, OpenSpec, and Superpowers dependencies"
+  skills:
+    # Main dispatcher (carries scripts/ with independent .mjs command bundles and runtime/classic control package)
+    - name: comet
+      skill: 040-beta/comet
+      variant: all
+      base: benchmarks
+    # Phase sub-skills (slash-invoked as /comet-open etc.)
+    - name: comet-open
+      skill: 040-beta/comet-open
+      variant: all
+      base: benchmarks
+    - name: comet-design
+      skill: 040-beta/comet-design
+      variant: all
+      base: benchmarks
+    - name: comet-build
+      skill: 040-beta/comet-build
+      variant: all
+      base: benchmarks
+    - name: comet-verify
+      skill: 040-beta/comet-verify
+      variant: all
+      base: benchmarks
+    - name: comet-archive
+      skill: 040-beta/comet-archive
+      variant: all
+      base: benchmarks
+    - name: comet-hotfix
+      skill: 040-beta/comet-hotfix
+      variant: all
+      base: benchmarks
+    - name: comet-tweak
+      skill: 040-beta/comet-tweak
+      variant: all
+      base: benchmarks
+    # OpenSpec workflow dependencies (benchmark dependency snapshots).
+    - name: openspec-apply-change
+      skill: dependency/openspec/openspec-apply-change
+      variant: all
+      base: benchmarks
+    - name: openspec-archive-change
+      skill: dependency/openspec/openspec-archive-change
+      variant: all
+      base: benchmarks
+    - name: openspec-bulk-archive-change
+      skill: dependency/openspec/openspec-bulk-archive-change
+      variant: all
+      base: benchmarks
+    - name: openspec-continue-change
+      skill: dependency/openspec/openspec-continue-change
+      variant: all
+      base: benchmarks
+    - name: openspec-explore
+      skill: dependency/openspec/openspec-explore
+      variant: all
+      base: benchmarks
+    - name: openspec-ff-change
+      skill: dependency/openspec/openspec-ff-change
+      variant: all
+      base: benchmarks
+    - name: openspec-new-change
+      skill: dependency/openspec/openspec-new-change
+      variant: all
+      base: benchmarks
+    - name: openspec-onboard
+      skill: dependency/openspec/openspec-onboard
+      variant: all
+      base: benchmarks
+    - name: openspec-propose
+      skill: dependency/openspec/openspec-propose
+      variant: all
+      base: benchmarks
+    - name: openspec-sync-specs
+      skill: dependency/openspec/openspec-sync-specs
+      variant: all
+      base: benchmarks
+    - name: openspec-verify-change
+      skill: dependency/openspec/openspec-verify-change
+      variant: all
+      base: benchmarks
+    # Superpowers dependencies used by Comet phase guidance and recovery paths.
+    - name: brainstorming
+      skill: dependency/superpowers/brainstorming
+      variant: all
+      base: benchmarks
+    - name: dispatching-parallel-agents
+      skill: dependency/superpowers/dispatching-parallel-agents
+      variant: all
+      base: benchmarks
+    - name: executing-plans
+      skill: dependency/superpowers/executing-plans
+      variant: all
+      base: benchmarks
+    - name: finishing-a-development-branch
+      skill: dependency/superpowers/finishing-a-development-branch
+      variant: all
+      base: benchmarks
+    - name: receiving-code-review
+      skill: dependency/superpowers/receiving-code-review
+      variant: all
+      base: benchmarks
+    - name: requesting-code-review
+      skill: dependency/superpowers/requesting-code-review
+      variant: all
+      base: benchmarks
+    - name: subagent-driven-development
+      skill: dependency/superpowers/subagent-driven-development
+      variant: all
+      base: benchmarks
+    - name: systematic-debugging
+      skill: dependency/superpowers/systematic-debugging
+      variant: all
+      base: benchmarks
+    - name: test-driven-development
+      skill: dependency/superpowers/test-driven-development
+      variant: all
+      base: benchmarks
+    - name: using-git-worktrees
+      skill: dependency/superpowers/using-git-worktrees
+      variant: all
+      base: benchmarks
+    - name: verification-before-completion
+      skill: dependency/superpowers/verification-before-completion
+      variant: all
+      base: benchmarks
+    - name: writing-plans
+      skill: dependency/superpowers/writing-plans
+      variant: all
+      base: benchmarks
+    - name: writing-skills
+      skill: dependency/superpowers/writing-skills
+      variant: all
+      base: benchmarks

package/eval/local/treatments/comet/comet_native_phase1.yaml

@@ -0,0 +1,66 @@
+# Self-contained Comet Native treatment. It loads only the current repository's
+# Native Skill package; no Classic, OpenSpec, Superpowers, or dependency snapshot
+# is injected.
+COMET_NATIVE_PHASE1:
+  description: 'Self-contained Comet Native Phase 1 workflow for strong coding models'
+  claude_md: |
+    Begin by invoking the /comet-native Skill and use only its bundled runtime.
+    Never invoke /comet or any Skill other than /comet-native, even when the task
+    text says "use the comet workflow" or names legacy Comet phases.
+    When the task describes the legacy Comet Open, Design, Build, Verify, and
+    Archive phases, preserve the business requirements but execute them as the
+    Native Shape, Build, Verify, and Archive workflow. Initialize Native with
+    artifact_root `docs`, continue automatically while no user decision
+    is unresolved, and leave a terminal verified Native archive. An explicit
+    execution boundary in the task, including a request to stop at a phase or
+    switch sessions, takes priority over automatic progression: persist the
+    requested boundary, stop there, and continue toward the terminal archive
+    only in the resumed session. Do not create OpenSpec, Classic, Superpowers,
+    or hidden `.comet` workflow artifacts.
+  skills:
+    - name: comet-native
+      source: path
+      path: ../assets/skills/comet-native
+      profile: generic
+      required_skills:
+        - comet-native
+      require_skill_invocation: true
+
+COMET_NATIVE_SEQUENTIAL:
+  description: 'Self-contained Comet Native workflow with sequential clarification'
+  claude_md: |
+    Begin by invoking the /comet-native Skill and use only its bundled runtime.
+    Initialize Native under `docs/comet/`, then set native.clarification_mode `sequential`
+    in `.comet/config.yaml` before asking any product question.
+    Do not change the configured clarification mode. Ask one most-upstream user
+    decision per round, persist every answer, and continue through Shape, Build,
+    Verify, and Archive. Do not create OpenSpec, Classic, Superpowers, or hidden
+    `.comet` workflow artifacts.
+  skills:
+    - name: comet-native
+      source: path
+      path: ../assets/skills/comet-native
+      profile: generic
+      required_skills:
+        - comet-native
+      require_skill_invocation: true
+
+COMET_NATIVE_BATCH:
+  description: 'Self-contained Comet Native workflow with batch clarification'
+  claude_md: |
+    Begin by invoking the /comet-native Skill and use only its bundled runtime.
+    Initialize Native under `docs/comet/`, then set native.clarification_mode `batch`
+    in `.comet/config.yaml` before asking any product question.
+    Do not change the configured clarification mode. Ask every currently answerable
+    independent user decision in one numbered round, persist every answer, then
+    present and persist the required final shared-understanding confirmation
+    before continuing through Build, Verify, and Archive. Do not create OpenSpec,
+    Classic, Superpowers, or hidden `.comet` workflow artifacts.
+  skills:
+    - name: comet-native
+      source: path
+      path: ../assets/skills/comet-native
+      profile: generic
+      required_skills:
+        - comet-native
+      require_skill_invocation: true

package/eval/local/treatments/common/control.yaml

@@ -0,0 +1,4 @@
+# Control treatment: no skills injected (baseline)
+CONTROL:
+  description: "No skills baseline - Claude has no comet workflow guidance"
+  skills: []

package/eval/pyproject.toml

@@ -0,0 +1,41 @@
+[project]
+name = "comet-skill-eval"
+version = "0.1.0"
+description = "Benchmark evaluation for Comet skill (OpenSpec + Superpowers workflow)"
+requires-python = ">=3.11"
+dependencies = [
+    "pytest>=8.0",
+    "pytest-timeout>=2.2",
+    "pytest-xdist>=3.5",
+    "python-dotenv>=1.0",
+    "pyyaml>=6.0",
+    "pydantic>=2.0",
+    "requests>=2.31",
+    "tomli>=2.0; python_version < '3.11'",
+]
+
+[project.optional-dependencies]
+# LangSmith tracing + pytest integration; install with: uv sync --extra langsmith
+# The LangSmith suite marks tests with @pytest.mark.langsmith and logs rubric
+# scores + treatment comparison via langsmith.testing (requires the pytest extra).
+langsmith = [
+    "langsmith[pytest]>=0.3.4",
+    "langchain>=0.3",
+    "langchain-openai>=0.2",
+]
+dev = [
+    "ruff>=0.4",
+    "pre-commit>=3.7",
+    "matplotlib>=3.8",
+]
+
+[tool.pytest.ini_options]
+testpaths = ["local/tests"]
+pythonpath = ["."]
+markers = [
+    "langsmith: marks test as using LangSmith tracing",
+]
+
+[tool.ruff]
+line-length = 100
+target-version = "py311"

package/eval/report-html-config.json

@@ -0,0 +1,6 @@
+{
+  "report_outputs": {
+    "markdown": true,
+    "html": true
+  }
+}

package/eval/scaffold/__init__.py

@@ -0,0 +1,67 @@
+"""Testing scaffold for Comet skill benchmarks.
+
+Re-exports from scaffold.python for convenience.
+"""
+
+from .python import (
+    NoiseTask,
+    Treatment,
+    ValidatorFn,
+    build_docker_image,
+    check_claude_available,
+    check_code_execution,
+    check_docker_available,
+    check_file_exists,
+    check_no_pattern,
+    check_pattern,
+    check_skill_invoked,
+    compose_validators,
+    make_execution_validator,
+    run_claude_in_docker,
+    run_eval_in_docker,
+    run_node_in_docker,
+    run_python_in_docker,
+    run_shell,
+    run_validators,
+)
+from .python.tasks import Task, TaskConfig, list_tasks, load_task
+from .python.treatments import (
+    TreatmentConfig,
+    build_treatment_skills,
+    list_treatments,
+    load_treatment,
+    load_treatments,
+    load_treatments_yaml,
+)
+
+__all__ = [
+    "Task",
+    "TaskConfig",
+    "load_task",
+    "list_tasks",
+    "TreatmentConfig",
+    "build_treatment_skills",
+    "list_treatments",
+    "load_treatment",
+    "load_treatments",
+    "load_treatments_yaml",
+    "NoiseTask",
+    "Treatment",
+    "ValidatorFn",
+    "compose_validators",
+    "run_validators",
+    "check_file_exists",
+    "check_pattern",
+    "check_no_pattern",
+    "check_skill_invoked",
+    "check_code_execution",
+    "run_shell",
+    "check_docker_available",
+    "check_claude_available",
+    "build_docker_image",
+    "run_python_in_docker",
+    "make_execution_validator",
+    "run_eval_in_docker",
+    "run_node_in_docker",
+    "run_claude_in_docker",
+]

package/eval/scaffold/python/__init__.py

@@ -0,0 +1,95 @@
+"""Python-specific scaffold components for comet skill benchmarks."""
+
+from .logging import (
+    ExperimentLogger,
+    ReportColumn,
+    TreatmentResult,
+    bool_column,
+    default_columns,
+    extract_events,
+    parse_output,
+    quality_column,
+    save_events,
+    save_raw,
+    save_report,
+    strip_ansi,
+)
+from .manifests import SkillEvalManifest, load_eval_manifest
+from .profiles import (
+    AUTHORING_SKILL_PROFILE,
+    COMET_WORKFLOW_PROFILE,
+    GENERIC_PROFILE,
+    ProfileSpec,
+    all_rubric_dimensions,
+    get_profile,
+    list_profiles,
+    resolve_profile_name,
+    run_profile_rubric,
+)
+from .evidence import EvalArtifactReference, build_eval_artifact_references
+from .attribution import FailureAttribution, classify_failures
+from .sample_quality import (
+    SampleQuality,
+    include_in_analysis,
+    infer_sample_quality,
+    quality_from_report,
+    sample_quality_dict,
+)
+from .report_outputs import (
+    ReportOutputConfig,
+    load_report_output_config,
+    preferred_report_path,
+    render_markdown_html,
+    write_report_outputs,
+)
+from .schema import NoiseTask, Treatment
+from .utils import (
+    build_docker_image,
+    check_claude_available,
+    check_docker_available,
+    get_field,
+    get_nested_field,
+    make_execution_validator,
+    normalize_score,
+    read_json_file,
+    retry_with_backoff,
+    run_claude_in_docker,
+    run_eval_in_docker,
+    run_node_in_docker,
+    run_python_in_docker,
+    run_shell,
+)
+from .validation import (
+    ValidatorFn,
+    check_code_execution,
+    check_file_exists,
+    check_no_pattern,
+    check_pattern,
+    check_python_execution,
+    check_skill_invoked,
+    check_typescript_execution,
+    compose_validators,
+    run_validators,
+)
+
+__all__ = [
+    "NoiseTask", "Treatment", "ValidatorFn", "compose_validators", "run_validators",
+    "check_file_exists", "check_pattern", "check_no_pattern", "check_skill_invoked",
+    "check_code_execution", "check_python_execution", "check_typescript_execution",
+    "run_shell", "check_docker_available", "check_claude_available", "build_docker_image",
+    "run_python_in_docker", "make_execution_validator", "run_eval_in_docker",
+    "run_node_in_docker", "run_claude_in_docker", "retry_with_backoff", "read_json_file",
+    "get_field", "get_nested_field", "normalize_score", "parse_output", "extract_events",
+    "strip_ansi", "ExperimentLogger", "TreatmentResult", "save_events", "save_raw",
+    "save_report", "ReportColumn", "bool_column", "quality_column", "default_columns",
+    "EvalArtifactReference", "build_eval_artifact_references",
+    "FailureAttribution", "classify_failures",
+    "SkillEvalManifest", "load_eval_manifest",
+    "AUTHORING_SKILL_PROFILE", "COMET_WORKFLOW_PROFILE", "GENERIC_PROFILE",
+    "ProfileSpec", "all_rubric_dimensions", "get_profile", "list_profiles",
+    "resolve_profile_name", "run_profile_rubric",
+    "ReportOutputConfig", "load_report_output_config", "preferred_report_path",
+    "render_markdown_html", "write_report_outputs",
+    "SampleQuality", "infer_sample_quality", "quality_from_report",
+    "sample_quality_dict", "include_in_analysis",
+]