ace-test-runner-e2e 0.29.0

data/lib/ace/test/end_to_end_runner/molecules/test_discoverer.rb

@@ -0,0 +1,136 @@
+# frozen_string_literal: true
+
+require "pathname"
+
+module Ace
+  module Test
+    module EndToEndRunner
+      module Molecules
+        # Discovers E2E test scenario directories (TS-*/scenario.yml) in packages
+        #
+        # Finds test scenarios in the TS-format directory structure:
+        #   {package}/test/e2e/TS-*/scenario.yml
+        #
+        # Note: This is a Molecule (not an Atom) because it performs filesystem
+        # I/O via Dir.glob.
+        class TestDiscoverer
+          TEST_DIR = "test/e2e"
+          SCENARIO_FILE = "scenario.yml"
+          SCENARIO_DIR_PATTERN = "TS-*"
+
+          # Find E2E test scenario files matching criteria
+          #
+          # @param package [String] Package name (e.g., "ace-lint")
+          # @param test_id [String, nil] Optional specific test ID (e.g., "TS-LINT-001")
+          # @param tags [Array<String>, String, nil] Scenario tags to include (OR semantics)
+          # @param exclude_tags [Array<String>, String, nil] Scenario tags to exclude (OR semantics)
+          # @param base_dir [String] Base directory to search from (default: current dir)
+          # @return [Array<String>] Sorted list of matching scenario.yml file paths
+          def find_tests(package:, test_id: nil, tags: nil, exclude_tags: nil, base_dir: Dir.pwd)
+            test_ids = test_id ? test_id.split(",").map(&:strip) : [nil]
+            scenario_files = test_ids
+              .flat_map { |id| Dir.glob(build_scenario_pattern(package, id, base_dir)) }
+              .uniq
+              .sort
+
+            return scenario_files if no_filters?(tags, exclude_tags)
+
+            loader = ScenarioLoader.new
+            scenarios = scenario_files.map do |yml_path|
+              loader.load(File.dirname(yml_path))
+            end
+
+            filter_scenarios(
+              scenarios,
+              tags: normalize_tags(tags),
+              exclude_tags: normalize_tags(exclude_tags)
+            ).map(&:file_path).sort
+          end
+
+          # Find TS-format scenario directories and load them as TestScenario models
+          #
+          # @param package [String] Package name
+          # @param test_id [String, nil] Optional test ID to filter
+          # @param tags [Array<String>, String, nil] Scenario tags to include (OR semantics)
+          # @param exclude_tags [Array<String>, String, nil] Scenario tags to exclude (OR semantics)
+          # @param base_dir [String] Base directory to search from
+          # @return [Array<Models::TestScenario>] Loaded scenario models with test_cases
+          def find_scenarios(package:, test_id: nil, tags: nil, exclude_tags: nil, base_dir: Dir.pwd)
+            test_dir = File.join(base_dir, package, TEST_DIR)
+            pattern = File.join(test_dir, SCENARIO_DIR_PATTERN, SCENARIO_FILE)
+            scenario_files = Dir.glob(pattern).sort
+
+            loader = ScenarioLoader.new
+            scenarios = scenario_files.map do |yml_path|
+              scenario_dir = File.dirname(yml_path)
+              loader.load(scenario_dir)
+            end
+
+            if test_id
+              scenarios = scenarios.select { |s| s.test_id == test_id }
+            end
+
+            filter_scenarios(
+              scenarios,
+              tags: normalize_tags(tags),
+              exclude_tags: normalize_tags(exclude_tags)
+            )
+          end
+
+          # List all packages that have E2E tests
+          #
+          # @param base_dir [String] Base directory to search from
+          # @return [Array<String>] Sorted list of package names
+          def list_packages(base_dir: Dir.pwd)
+            pattern = File.join(base_dir, "*/#{TEST_DIR}/#{SCENARIO_DIR_PATTERN}/#{SCENARIO_FILE}")
+
+            base = Pathname.new(base_dir)
+
+            Dir.glob(pattern)
+              .map { |f| Pathname.new(f).relative_path_from(base).each_filename.first }
+              .uniq
+              .sort
+          end
+
+          private
+
+          # Build glob pattern for finding TS-format scenario.yml files
+          def build_scenario_pattern(package, test_id, base_dir)
+            test_dir = File.join(base_dir, package, TEST_DIR)
+
+            if test_id
+              File.join(test_dir, "*#{test_id}*", SCENARIO_FILE)
+            else
+              File.join(test_dir, SCENARIO_DIR_PATTERN, SCENARIO_FILE)
+            end
+          end
+
+          def no_filters?(tags, exclude_tags)
+            normalize_tags(tags).empty? && normalize_tags(exclude_tags).empty?
+          end
+
+          def normalize_tags(raw)
+            return [] if raw.nil?
+
+            values = raw.is_a?(Array) ? raw : raw.to_s.split(",")
+            values.map(&:to_s).map(&:strip).reject(&:empty?).map(&:downcase)
+          end
+
+          def filter_scenarios(scenarios, tags:, exclude_tags:)
+            filtered = scenarios
+
+            unless tags.empty?
+              filtered = filtered.select { |scenario| !(scenario.tags & tags).empty? }
+            end
+
+            unless exclude_tags.empty?
+              filtered = filtered.reject { |scenario| !(scenario.tags & exclude_tags).empty? }
+            end
+
+            filtered
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ace/test/end_to_end_runner/molecules/test_executor.rb

@@ -0,0 +1,332 @@
+# frozen_string_literal: true
+
+require "ace/llm"
+require "ace/llm/query_interface"
+
+module Ace
+  module Test
+    module EndToEndRunner
+      module Molecules
+        # Executes a single E2E test scenario via LLM
+        #
+        # Routes execution through two paths based on provider type:
+        # - CLI providers (claude, gemini, codex): deterministic standalone pipeline
+        # - API providers (google, anthropic): prompt-based prediction mode
+        class TestExecutor
+          # @param provider [String] LLM provider:model string
+          # @param timeout [Integer] Request timeout in seconds
+          # @param config [Hash] Configuration hash (string keys) from ConfigLoader
+          def initialize(provider: nil, timeout: nil, config: nil)
+            config ||= Molecules::ConfigLoader.load
+            @provider = provider || config.dig("execution", "provider") || "claude:sonnet"
+            @timeout = timeout || config.dig("execution", "timeout") || 300
+            @prompt_builder = Atoms::PromptBuilder.new
+            @cli_provider_adapter = Atoms::CliProviderAdapter.new(config)
+          end
+
+          # Execute a single test scenario via LLM
+          #
+          # @param scenario [Models::TestScenario] The test scenario to execute
+          # @param cli_args [String, nil] Extra args for CLI providers
+          # @param run_id [String, nil] Pre-generated run ID for deterministic report paths
+          # @param test_cases [Array<String>, nil] Optional test case IDs to filter
+          # @param sandbox_path [String, nil] Path to pre-populated sandbox (skips LLM setup)
+          # @param env_vars [Hash, nil] Environment variables from setup execution
+          # @param report_dir [String, nil] Explicit report directory path (overrides computed path)
+          # @return [Models::TestResult] Test execution result
+          def execute(scenario, cli_args: nil, run_id: nil, test_cases: nil, sandbox_path: nil,
+            env_vars: nil, report_dir: nil, timeout: nil, verify: false)
+            resolved_timeout = timeout || @timeout
+            if Atoms::CliProviderAdapter.cli_provider?(@provider)
+              execute_via_pipeline(
+                scenario,
+                cli_args: cli_args,
+                run_id: run_id,
+                test_cases: test_cases,
+                sandbox_path: sandbox_path,
+                env_vars: env_vars,
+                report_dir: report_dir,
+                timeout: resolved_timeout
+              )
+            else
+              execute_via_prompt(scenario, cli_args: cli_args, test_cases: test_cases, timeout: resolved_timeout)
+            end
+          end
+
+          # Execute a single test case via LLM in a pre-populated sandbox
+          #
+          # @param test_case [Models::TestCase] The single test case to execute
+          # @param sandbox_path [String] Path to the pre-populated sandbox
+          # @param scenario [Models::TestScenario] The parent scenario for metadata
+          # @param cli_args [String, nil] Extra args for CLI providers
+          # @param run_id [String, nil] Pre-generated run ID
+          # @param env_vars [Hash, nil] Environment variables from setup execution
+          # @return [Models::TestResult] Test execution result
+          def execute_tc(test_case:, sandbox_path:, scenario:, cli_args: nil, run_id: nil, env_vars: nil)
+            if Atoms::CliProviderAdapter.cli_provider?(@provider)
+              execute_tc_via_skill(test_case, sandbox_path, scenario, cli_args: cli_args, run_id: run_id, env_vars: env_vars)
+            else
+              execute_tc_via_prompt(test_case, sandbox_path, scenario, cli_args: cli_args)
+            end
+          end
+
+          private
+
+          # Execute standalone scenarios with the deterministic pipeline.
+          def execute_via_pipeline(scenario, cli_args: nil, run_id: nil, test_cases: nil, sandbox_path: nil,
+            env_vars: nil, report_dir: nil, timeout: nil)
+            started_at = Time.now
+            resolved_report_dir = report_dir || default_report_dir_for(scenario, run_id)
+            resolved_sandbox_path = sandbox_path || resolve_sandbox_path(nil, resolved_report_dir)
+
+            if resolved_report_dir.nil? || resolved_sandbox_path.nil?
+              return Models::TestResult.new(
+                test_id: scenario.test_id,
+                status: "error",
+                summary: "Execution pipeline requires run_id/report_dir",
+                error: "Could not resolve deterministic sandbox/report paths",
+                started_at: started_at,
+                completed_at: Time.now
+              )
+            end
+
+            pipeline_executor(timeout: timeout).execute(
+              scenario: scenario,
+              cli_args: cli_args,
+              sandbox_path: resolved_sandbox_path,
+              report_dir: resolved_report_dir,
+              env_vars: env_vars,
+              test_cases: test_cases
+            )
+          end
+
+          def resolve_sandbox_path(sandbox_path, report_dir)
+            return sandbox_path if sandbox_path && !sandbox_path.empty?
+            return nil unless report_dir
+
+            report_dir.sub(/-reports\z/, "")
+          end
+
+          def default_report_dir_for(scenario, run_id)
+            return nil unless run_id && !run_id.to_s.empty?
+
+            File.join(Dir.pwd, ".ace-local", "test-e2e", "#{scenario.dir_name(run_id)}-reports")
+          end
+
+          # Execute via prompt for API providers (original behavior)
+          #
+          # @param scenario [Models::TestScenario] The test scenario
+          # @param cli_args [String, nil] Extra args
+          # @param test_cases [Array<String>, nil] Optional test case IDs to filter
+          # @return [Models::TestResult]
+          def execute_via_prompt(scenario, cli_args: nil, test_cases: nil, timeout: nil)
+            started_at = Time.now
+
+            prompt = @prompt_builder.build(scenario, test_cases: test_cases)
+
+            response = Ace::LLM::QueryInterface.query(
+              @provider,
+              prompt,
+              system: Atoms::PromptBuilder::SYSTEM_PROMPT,
+              cli_args: cli_args,
+              timeout: timeout || @timeout,
+              fallback: false
+            )
+
+            parsed = Atoms::ResultParser.parse(response[:text])
+            completed_at = Time.now
+
+            # Validate TC fidelity: ensure agent executed the expected test cases
+            fidelity = Atoms::TcFidelityValidator.validate(
+              parsed, scenario, filtered_tc_ids: test_cases
+            )
+            if fidelity
+              return Models::TestResult.new(
+                test_id: scenario.test_id,
+                status: "error",
+                test_cases: parsed[:test_cases],
+                summary: fidelity[:error],
+                error: fidelity[:error],
+                started_at: started_at,
+                completed_at: completed_at
+              )
+            end
+
+            Models::TestResult.new(
+              test_id: scenario.test_id,
+              status: parsed[:status],
+              test_cases: parsed[:test_cases],
+              summary: parsed[:summary],
+              started_at: started_at,
+              completed_at: completed_at
+            )
+          rescue Atoms::ResultParser::ParseError => e
+            Models::TestResult.new(
+              test_id: scenario.test_id,
+              status: "error",
+              summary: "Failed to parse LLM response",
+              error: e.message,
+              started_at: started_at,
+              completed_at: Time.now
+            )
+          rescue Ace::LLM::Error => e
+            Models::TestResult.new(
+              test_id: scenario.test_id,
+              status: "error",
+              summary: "LLM execution failed",
+              error: e.message,
+              started_at: started_at || Time.now,
+              completed_at: Time.now
+            )
+          rescue => e
+            Models::TestResult.new(
+              test_id: scenario.test_id,
+              status: "error",
+              summary: "Unexpected execution error",
+              error: "#{e.class}: #{e.message}",
+              started_at: started_at || Time.now,
+              completed_at: Time.now
+            )
+          end
+
+          # Execute TC via skill invocation for CLI providers
+          def execute_tc_via_skill(test_case, sandbox_path, scenario, cli_args: nil, run_id: nil, env_vars: nil)
+            with_tc_error_handling(scenario) do |started_at|
+              prompt = @cli_provider_adapter.build_tc_skill_prompt(
+                test_case: test_case, scenario: scenario,
+                sandbox_path: sandbox_path, run_id: run_id, env_vars: env_vars
+              )
+
+              response = Ace::LLM::QueryInterface.query(
+                @provider, prompt,
+                system: nil, cli_args: cli_args,
+                timeout: @timeout, fallback: false,
+                working_dir: sandbox_path,
+                subprocess_env: env_vars
+              )
+
+              invocation_error = detect_skill_invocation_error(response[:text])
+              if invocation_error
+                return Models::TestResult.new(
+                  test_id: scenario.test_id,
+                  status: "error",
+                  test_cases: [],
+                  summary: "TC skill invocation failed before test execution",
+                  error: invocation_error,
+                  started_at: started_at,
+                  completed_at: Time.now
+                )
+              end
+
+              parsed = Atoms::SkillResultParser.parse_tc(response[:text])
+              completed_at = Time.now
+
+              Models::TestResult.new(
+                test_id: scenario.test_id,
+                status: parsed[:status],
+                test_cases: parsed[:test_cases],
+                summary: parsed[:summary],
+                started_at: started_at,
+                completed_at: completed_at
+              )
+            end
+          end
+
+          # Execute TC via prompt for API providers
+          def execute_tc_via_prompt(test_case, sandbox_path, scenario, cli_args: nil)
+            with_tc_error_handling(scenario) do |started_at|
+              prompt = @prompt_builder.build_tc(
+                test_case: test_case, scenario: scenario, sandbox_path: sandbox_path
+              )
+
+              response = Ace::LLM::QueryInterface.query(
+                @provider, prompt,
+                system: Atoms::PromptBuilder::TC_SYSTEM_PROMPT,
+                cli_args: cli_args, timeout: @timeout, fallback: false
+              )
+
+              parsed = Atoms::ResultParser.parse_tc(response[:text])
+              completed_at = Time.now
+
+              Models::TestResult.new(
+                test_id: scenario.test_id,
+                status: parsed[:status],
+                test_cases: parsed[:test_cases],
+                summary: parsed[:summary],
+                started_at: started_at,
+                completed_at: completed_at
+              )
+            end
+          end
+
+          # Shared error handling for TC execution methods
+          def with_tc_error_handling(scenario)
+            started_at = Time.now
+            yield started_at
+          rescue Atoms::ResultParser::ParseError => e
+            Models::TestResult.new(
+              test_id: scenario.test_id, status: "error",
+              summary: "Failed to parse TC response",
+              error: e.message, started_at: started_at, completed_at: Time.now
+            )
+          rescue Ace::LLM::Error => e
+            Models::TestResult.new(
+              test_id: scenario.test_id, status: "error",
+              summary: "TC execution failed",
+              error: e.message, started_at: started_at, completed_at: Time.now
+            )
+          rescue => e
+            Models::TestResult.new(
+              test_id: scenario.test_id, status: "error",
+              summary: "Unexpected TC execution error",
+              error: "#{e.class}: #{e.message}",
+              started_at: started_at, completed_at: Time.now
+            )
+          end
+
+          # Detect common failure modes where the agent did not execute the
+          # /as-e2e-run skill correctly.
+          #
+          # @param text [String] Raw LLM response text
+          # @return [String, nil] Error message when a known failure is detected
+          def detect_skill_invocation_error(text)
+            return nil if text.nil? || text.strip.empty?
+
+            checks = [
+              [/\/as-e2e-run.*command not found/i, "The slash command was executed in a shell instead of chat."],
+              [/exit code 127.*\/as-e2e-run|\/as-e2e-run.*exit code 127/im, "The slash command failed with shell exit code 127."],
+              [/No tests found for package/i, "The test command ran in the wrong context or with invalid arguments."],
+              [/\bace-test\s+e2e\b/i, "An invalid command (`ace-test e2e`) was attempted instead of `ace-test-e2e`."],
+              [/slash commands are unavailable/i, "The agent reported slash commands are unavailable in this environment."]
+            ]
+
+            checks.each do |pattern, message|
+              next unless text.match?(pattern)
+
+              detail = extract_matching_line(text, pattern)
+              return "#{message} Detected output: #{detail}"
+            end
+
+            nil
+          end
+
+          def extract_matching_line(text, pattern)
+            line = text.to_s.lines.find { |candidate| candidate.match?(pattern) }
+            return line.strip if line && !line.strip.empty?
+
+            text.to_s.strip.split(/\s+/).first(30).join(" ")
+          end
+
+          def pipeline_executor(timeout: nil)
+            timeout ||= @timeout
+            @pipeline_executors ||= {}
+            @pipeline_executors[timeout] ||= Molecules::PipelineExecutor.new(
+              provider: @provider,
+              timeout: timeout
+            )
+          end
+        end
+      end
+    end
+  end
+end