ace-test-runner-e2e 0.29.0

data/lib/ace/test/end_to_end_runner/cli/commands/run_test.rb

@@ -0,0 +1,156 @@
+# frozen_string_literal: true
+
+require "ace/support/cli"
+require "stringio"
+require "ace/support/cli"
+
+module Ace
+  module Test
+    module EndToEndRunner
+      module CLI
+        module Commands
+          # CLI command for running E2E tests
+          #
+          # Supports running a single test by ID or all tests in a package.
+          # Tests are executed via LLM and results are written to standard
+          # report locations.
+          class RunTest < Ace::Support::Cli::Command
+            include Ace::Support::Cli::Base
+
+            desc <<~DESC.strip
+              Run E2E tests via LLM execution
+
+              Discovers and executes TS-* test scenarios in a package's test/e2e/ directory.
+              Tests are sent to an LLM provider which executes the test steps and returns
+              structured results.
+
+              Output:
+                Exit codes: 0 (all pass), 1 (any fail/error)
+                Reports written to: .ace-local/test-e2e/{timestamp}-{pkg}-{id}-reports/
+            DESC
+
+            example [
+              "ace-lint TS-LINT-001          # Run specific test",
+              "ace-lint                      # Run all tests in package",
+              "ace-lint --provider gemini:flash  # Use specific provider",
+              "ace-lint --provider glite     # Use API provider (predict mode)",
+              "ace-lint --tags smoke         # Run only smoke-tagged scenarios",
+              "ace-lint TS-LINT-003 --dry-run  # Preview scenarios that would run"
+            ]
+
+            argument :package, required: true, desc: "Package name (e.g., ace-lint)"
+            argument :test_id, required: false, desc: "Test ID (e.g., TS-LINT-001)"
+
+            option :provider, type: :string, default: Molecules::ConfigLoader.default_provider,
+              desc: "LLM provider:model (e.g., claude:sonnet, gemini:flash)"
+            option :cli_args, type: :string,
+              desc: "Extra args for CLI-based LLM providers"
+            option :timeout, type: :string, default: Molecules::ConfigLoader.default_timeout.to_s,
+              desc: "Timeout per test in seconds"
+            option :parallel, type: :string, default: Molecules::ConfigLoader.default_parallel.to_s,
+              desc: "Number of tests to run in parallel (1 = sequential)"
+            option :progress, type: :boolean, desc: "Enable live animated display"
+            option :run_id, type: :string,
+              desc: "Pre-generated run ID for deterministic report paths"
+            option :report_dir, type: :string,
+              desc: "Explicit report directory path (overrides computed path)"
+            option :dry_run, type: :boolean,
+              desc: "Preview which scenarios would run without executing"
+            option :tags, type: :string,
+              desc: "Comma-separated scenario tags to include"
+            option :verify, type: :boolean,
+              desc: "Run independent verifier pass after runner execution"
+            option :quiet, type: :boolean, aliases: %w[-q], desc: "Suppress non-essential output"
+            option :verbose, type: :boolean, aliases: %w[-v], desc: "Show verbose output"
+            option :debug, type: :boolean, aliases: %w[-d], desc: "Show debug output"
+
+            def call(package:, test_id: nil, **options)
+              options = coerce_types(options, timeout: :integer, parallel: :integer)
+              output = quiet?(options) ? StringIO.new : $stdout
+
+              # Handle dry-run mode
+              if options[:dry_run]
+                return handle_dry_run(package, test_id, output, tags: parse_tags(options[:tags]))
+              end
+
+              orchestrator = Organisms::TestOrchestrator.new(
+                provider: options[:provider],
+                timeout: options[:timeout],
+                parallel: options[:parallel],
+                progress: options[:progress]
+              )
+
+              results = orchestrator.run(
+                package: package,
+                test_id: test_id,
+                verify: options[:verify],
+                tags: parse_tags(options[:tags]),
+                cli_args: options[:cli_args],
+                run_id: options[:run_id],
+                report_dir: options[:report_dir],
+                output: output
+              )
+
+              if results.empty?
+                raise Ace::Support::Cli::Error.new(
+                  "No tests found for package '#{package}'" +
+                  (test_id ? " with ID '#{test_id}'" : "")
+                )
+              end
+
+              # Exit with error if any test failed (excluding skips)
+              if results.any?(&:failed?)
+                failed = results.select(&:failed?)
+                failed_ids = failed.map(&:test_id).join(", ")
+                raise Ace::Support::Cli::Error.new(
+                  "#{failed.size} test(s) failed: #{failed_ids}"
+                )
+              end
+            end
+
+            private
+
+            # Handle dry-run mode: preview which scenarios would run
+            #
+            # @param package [String] Package name
+            # @param test_id [String, nil] Test ID
+            # @param output [IO] Output stream
+            def handle_dry_run(package, test_id, output, tags: [])
+              discoverer = Molecules::TestDiscoverer.new
+              loader = Molecules::ScenarioLoader.new
+
+              files = discoverer.find_tests(
+                package: package,
+                test_id: test_id,
+                tags: tags,
+                base_dir: Dir.pwd
+              )
+              if files.empty?
+                raise Ace::Support::Cli::Error.new(
+                  "No tests found for package '#{package}'" +
+                  (test_id ? " with ID '#{test_id}'" : "")
+                )
+              end
+
+              output.puts "Dry run: preview of scenarios to execute"
+              output.puts ""
+
+              files.each do |file|
+                scenario = loader.load(File.dirname(file))
+                output.puts "#{scenario.test_id}: #{scenario.title}"
+                output.puts "  [run] full scenario (#{scenario.test_case_ids.size} test cases)"
+                output.puts ""
+              end
+            end
+
+            def parse_tags(raw)
+              return [] if raw.nil? || raw.strip.empty?
+
+              raw.split(",").map(&:strip).reject(&:empty?).map(&:downcase)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ace/test/end_to_end_runner/models/test_case.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Ace
+  module Test
+    module EndToEndRunner
+      module Models
+        # Data model representing a single test case definition.
+        #
+        # Contains parsed frontmatter metadata and the full markdown body
+        # from an independent test case file within a scenario directory.
+        class TestCase
+          attr_reader :tc_id, :title, :content, :file_path, :pending, :goal_format
+
+          # @param tc_id [String] Test case identifier (e.g., "TC-001")
+          # @param title [String] Test case title from frontmatter
+          # @param content [String] Full markdown body (below frontmatter)
+          # @param file_path [String] Absolute path to the source test file
+          # @param pending [String, nil] Pending reason (presence = pending, value = reason)
+          # @param goal_format [String, nil] Test case source format ("standalone")
+          def initialize(tc_id:, title:, content:, file_path:, pending: nil, goal_format: nil)
+            @tc_id = tc_id
+            @title = title
+            @content = content
+            @file_path = file_path
+            @pending = pending
+            @goal_format = goal_format
+          end
+
+          # Whether this test case is pending (should be skipped)
+          # @return [Boolean]
+          def pending?
+            !pending.nil?
+          end
+
+          # Generate short test case ID for directory naming
+          # @return [String] Short ID (e.g., "tc001" from "TC-001", "tc001a" from "TC-001a")
+          def short_id
+            match = tc_id.match(/TC-(\d+[a-z]*)/i)
+            return "tc#{match[1]}" if match
+
+            tc_id.downcase.gsub(/[^a-z0-9]/, "")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ace/test/end_to_end_runner/models/test_result.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+module Ace
+  module Test
+    module EndToEndRunner
+      module Models
+        # Data model representing the result of an E2E test execution
+        #
+        # Contains the status, individual test case results, and metadata
+        # from executing a test scenario via LLM.
+        class TestResult
+          attr_reader :test_id, :status, :test_cases, :summary,
+            :started_at, :completed_at, :report_dir, :error
+
+          # @param test_id [String] Test identifier
+          # @param status [String] Overall status: "pass", "fail", "partial", "error"
+          # @param test_cases [Array<Hash>] Individual test case results
+          # @param summary [String] Brief execution summary
+          # @param started_at [Time] When execution started
+          # @param completed_at [Time] When execution completed
+          # @param report_dir [String, nil] Path to the reports directory
+          # @param error [String, nil] Error message if execution failed
+          def initialize(test_id:, status:, test_cases: [], summary: "",
+            started_at: nil, completed_at: nil, report_dir: nil, error: nil)
+            @test_id = test_id
+            @status = status
+            @test_cases = test_cases
+            @summary = summary
+            @started_at = started_at || Time.now
+            @completed_at = completed_at || Time.now
+            @report_dir = report_dir
+            @error = error
+          end
+
+          # Check if the test passed
+          # @return [Boolean]
+          def success?
+            status == "pass"
+          end
+
+          # Check if the test failed (non-pass, non-skip status)
+          # @return [Boolean]
+          def failed?
+            !success? && !skipped?
+          end
+
+          # Check if the test was skipped
+          # @return [Boolean]
+          def skipped?
+            status == "skip"
+          end
+
+          # Count of passed test cases
+          # @return [Integer]
+          def passed_count
+            test_cases.count { |tc| tc[:status] == "pass" }
+          end
+
+          # Count of failed test cases
+          # @return [Integer]
+          def failed_count
+            test_cases.count { |tc| tc[:status] == "fail" }
+          end
+
+          # Total number of test cases
+          # @return [Integer]
+          def total_count
+            test_cases.size
+          end
+
+          # IDs of failed test cases
+          # @return [Array<String>] List of test case IDs with "fail" status
+          def failed_test_case_ids
+            test_cases
+              .select { |tc| tc[:status] == "fail" }
+              .map { |tc| tc[:id] }
+          end
+
+          # Duration in seconds
+          # @return [Float]
+          def duration
+            (completed_at - started_at).to_f
+          end
+
+          # Return a copy with the report_dir set
+          # @param dir [String] Path to the reports directory
+          # @return [TestResult] New result with report_dir
+          def with_report_dir(dir)
+            TestResult.new(
+              test_id: test_id,
+              status: status,
+              test_cases: test_cases,
+              summary: summary,
+              started_at: started_at,
+              completed_at: completed_at,
+              report_dir: dir,
+              error: error
+            )
+          end
+
+          # Human-readable duration string
+          # @return [String]
+          def duration_display
+            d = duration
+            if d < 60
+              "#{d.round(1)}s"
+            else
+              "#{(d / 60).floor}m #{(d % 60).round(0)}s"
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ace/test/end_to_end_runner/models/test_scenario.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+module Ace
+  module Test
+    module EndToEndRunner
+      module Models
+        # Data model representing a parsed E2E test scenario (TS-*/scenario.yml directory)
+        #
+        # Contains all information extracted from a test scenario including
+        # scenario.yml metadata, test cases, and setup steps.
+        class TestScenario
+          attr_reader :test_id, :title, :area, :package, :priority, :duration,
+            :requires, :file_path, :content, :timeout,
+            :setup_steps, :dir_path, :fixture_path, :test_cases,
+            :tags, :tool_under_test, :sandbox_layout
+
+          # @param test_id [String] Test identifier (e.g., "TS-LINT-001")
+          # @param title [String] Test title
+          # @param area [String] Test area (e.g., "lint")
+          # @param package [String] Package name (e.g., "ace-lint")
+          # @param priority [String] Priority level (default: "medium")
+          # @param duration [String] Expected duration (default: "~5min")
+          # @param requires [Hash] Required tools and versions
+          # @param file_path [String] Absolute path to the scenario directory
+          # @param content [String] Full markdown content of the scenario
+          # @param timeout [Integer, nil] Optional per-scenario timeout in seconds
+          # @param setup_steps [Array] Declarative setup steps from scenario.yml
+          # @param dir_path [String, nil] Path to the scenario directory
+          # @param fixture_path [String, nil] Path to the fixtures/ directory
+          # @param test_cases [Array<Models::TestCase>] Independent test case files
+          # @param tags [Array<String>] Scenario-level tags for discovery-time filtering
+          # @param tool_under_test [String, nil] Primary tool under test
+          # @param sandbox_layout [Hash] Declared sandbox artifact layout
+          def initialize(test_id:, title:, area:, package:, file_path:, content:,
+            priority: "medium", duration: "~5min", requires: {},
+            setup_steps: [], dir_path: nil, fixture_path: nil, test_cases: [],
+            timeout: nil, tags: [], tool_under_test: nil,
+            sandbox_layout: {})
+            @test_id = test_id
+            @title = title
+            @area = area
+            @package = package
+            @priority = priority
+            @duration = duration
+            @requires = requires
+            @file_path = file_path
+            @content = content
+            @timeout = timeout
+            @setup_steps = setup_steps
+            @dir_path = dir_path
+            @fixture_path = fixture_path
+            @test_cases = test_cases
+            @tags = tags
+            @tool_under_test = tool_under_test
+            @sandbox_layout = sandbox_layout
+          end
+
+          # Generate short package name (without ace- prefix)
+          # @return [String] Short package name (e.g., "lint" from "ace-lint")
+          def short_package
+            package.sub(/\Aace-/, "")
+          end
+
+          # Generate short test ID for directory naming
+          # @return [String] Short ID (e.g., "ts001" from "TS-LINT-001")
+          def short_id
+            match = test_id.match(/TS-[A-Z0-9]+-(\d+[a-z]*)/)
+            return "ts#{match[1]}" if match
+
+            test_id.downcase.gsub(/[^a-z0-9]/, "")
+          end
+
+          # Extract test case IDs from the test_cases array
+          #
+          # @return [Array<String>] List of test case IDs (e.g., ["TC-001", "TC-002"])
+          def test_case_ids
+            @test_case_ids ||= test_cases.map(&:tc_id)
+          end
+
+          # Build a directory name for sandbox/reports
+          # @param timestamp [String] Timestamp ID (7-char Base36)
+          # @return [String] Directory name (e.g., "8xyz12-lint-ts001")
+          def dir_name(timestamp)
+            "#{timestamp}-#{short_package}-#{short_id}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ace/test/end_to_end_runner/molecules/affected_detector.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+require "open3"
+require "pathname"
+
+module Ace
+  module Test
+    module EndToEndRunner
+      module Molecules
+        # Detects packages affected by recent changes
+        #
+        # Analyzes git diff to determine which packages have changed
+        # since the last commit, allowing selective test execution.
+        class AffectedDetector
+          # Default git reference to compare against
+          DEFAULT_REF = "HEAD~1"
+
+          # Detect packages that have changed
+          #
+          # @param base_dir [String] Base directory for git operations
+          # @param ref [String] Git reference to compare against (default: HEAD~1)
+          # @return [Array<String>] List of affected package names
+          def detect(base_dir: Dir.pwd, ref: DEFAULT_REF)
+            diff_files = get_changed_files(base_dir, ref)
+            return [] if diff_files.empty?
+
+            diff_files.map { |file| extract_package(file, base_dir) }
+              .compact
+              .uniq
+              .sort
+          end
+
+          private
+
+          # Get list of changed files from git
+          #
+          # @param base_dir [String] Base directory
+          # @param ref [String] Git reference
+          # @return [Array<String>] Changed file paths
+          def get_changed_files(base_dir, ref)
+            # Run git diff to get changed files using array-based command for security
+            output, status = Open3.capture2("git", "diff", "--name-only", ref, "--",
+              chdir: base_dir)
+
+            return [] unless status.success?
+
+            output.lines.map(&:strip).reject(&:empty?)
+          rescue => e
+            # If git command fails, warn and return empty array
+            # This can happen in shallow clones or if ref doesn't exist
+            warn "Warning: git detection failed: #{e.message}"
+            []
+          end
+
+          # Extract package name from file path
+          #
+          # @param file_path [String] Path to changed file
+          # @param base_dir [String] Base directory
+          # @return [String, nil] Package name or nil if not in a package
+          def extract_package(file_path, base_dir)
+            # Make path relative to base directory
+            relative_path = if file_path.start_with?("/")
+              begin
+                Pathname.new(file_path).relative_path_from(Pathname.new(base_dir)).to_s
+              rescue ArgumentError
+                # file_path is not under base_dir, use original path
+                file_path
+              end
+            else
+              file_path
+            end
+
+            # Split path and check first component
+            parts = relative_path.split("/")
+
+            # Check if first part looks like a package name (starts with "ace-")
+            package = parts.first
+            return nil unless package
+            return nil unless package.start_with?("ace-")
+            return nil if package == "." # Skip current dir
+
+            # Verify it's actually a directory (a package)
+            package_dir = File.join(base_dir, package)
+            return nil unless File.directory?(package_dir)
+
+            package
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ace/test/end_to_end_runner/molecules/config_loader.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+require "ace/support/config"
+
+module Ace
+  module Test
+    module EndToEndRunner
+      module Molecules
+        # Load configuration using Ace::Support::Config.create() API
+        # Follows ADR-022: Configuration Default and Override Pattern
+        #
+        # Configuration priority (highest to lowest):
+        # 1. CLI options (handled by callers)
+        # 2. Project config: .ace/e2e-runner/config.yml
+        # 3. Gem defaults: ace-test-runner-e2e/.ace-defaults/e2e-runner/config.yml
+        class ConfigLoader
+          # Load and return merged config hash
+          # @return [Hash] Configuration with string keys
+          def self.load
+            new.load
+          end
+
+          # @return [String] Default provider from config
+          def self.default_provider
+            config = load
+            config.dig("execution", "provider") || "claude:sonnet"
+          end
+
+          # @return [Integer] Default timeout from config
+          def self.default_timeout
+            config = load
+            config.dig("execution", "timeout") || 300
+          end
+
+          # @return [Integer] Default parallel count from config
+          def self.default_parallel
+            config = load
+            config.dig("execution", "parallel") || 3
+          end
+
+          # @return [Array<String>] CLI provider names
+          def self.cli_providers
+            config = load
+            config.dig("providers", "cli") || %w[claude gemini codex codexoss opencode pi]
+          end
+
+          # @param provider_name [String] Provider name (e.g., "claude")
+          # @return [String, nil] Required CLI args for provider
+          def self.cli_args_for(provider_name)
+            config = load
+            config.dig("providers", "cli_args", provider_name)
+          end
+
+          # Load merged config from cascade
+          # @return [Hash] Configuration with string keys
+          def load
+            gem_root = Gem.loaded_specs["ace-test-runner-e2e"]&.gem_dir ||
+              File.expand_path("../../../../..", __dir__)
+
+            resolver = Ace::Support::Config.create(
+              config_dir: ".ace",
+              defaults_dir: ".ace-defaults",
+              gem_path: gem_root
+            )
+
+            resolver.resolve_namespace("e2e-runner").data
+          rescue => e
+            warn "Warning: Could not load e2e-runner config: #{e.message}" if ENV["DEBUG"]
+            {}
+          end
+        end
+      end
+    end
+  end
+end