ace-test-runner-e2e 0.29.0

data/lib/ace/test/end_to_end_runner/molecules/failure_finder.rb

@@ -0,0 +1,203 @@
+# frozen_string_literal: true
+
+require "yaml"
+
+module Ace
+  module Test
+    module EndToEndRunner
+      module Molecules
+        # Scans cache for failed test cases from previous E2E test runs
+        #
+        # Reads metadata.yml files from .ace-local/test-e2e/*-reports/ directories
+        # and extracts failed_test_cases arrays. Used by --only-failures CLI flag
+        # to re-run only tests that failed previously.
+        #
+        # Note: This is a Molecule (not an Atom) because it performs filesystem
+        # I/O via Dir.glob and YAML file reading.
+        class FailureFinder
+          CACHE_DIR = ".ace-local/test-e2e"
+          METADATA_FILE = "metadata.yml"
+          REPORTS_SUFFIX = "-reports"
+
+          # Find failed test cases for a specific package
+          #
+          # Scans cache directory for the most recent metadata.yml per test-id
+          # within the specified package, returning aggregated failed test case IDs.
+          #
+          # @param package [String] Package name (e.g., "ace-lint")
+          # @param base_dir [String] Base directory to search from (default: current dir)
+          # @return [Array<String>] Failed test case IDs (e.g., ["TC-001", "TC-003"])
+          def find_failures(package:, base_dir: Dir.pwd)
+            metadata_files = discover_metadata_files(base_dir)
+            return [] if metadata_files.empty?
+
+            # Filter to package and get most recent per test-id
+            package_metadata = filter_by_package(metadata_files, package)
+            most_recent = most_recent_per_test(package_metadata)
+
+            extract_all_failed_ids(most_recent)
+          end
+
+          # Find failed test cases across all packages
+          #
+          # Scans cache directory for the most recent metadata.yml per test-id
+          # across all packages, returning aggregated failed test case IDs.
+          #
+          # @param base_dir [String] Base directory to search from (default: current dir)
+          # @return [Array<String>] Failed test case IDs
+          def find_all_failures(base_dir: Dir.pwd)
+            metadata_files = discover_metadata_files(base_dir)
+            return [] if metadata_files.empty?
+
+            most_recent = most_recent_per_test(metadata_files)
+            extract_all_failed_ids(most_recent)
+          end
+
+          # Find failed test cases grouped by package
+          #
+          # Scans cache directory for the most recent metadata.yml per test-id
+          # within each package, returning a hash mapping package names to their
+          # failed test case IDs.
+          #
+          # @param packages [Array<String>] Package names to scan
+          # @param base_dir [String] Base directory to search from (default: current dir)
+          # @return [Hash{String => Array<String>}] Package name to failed test case IDs
+          def find_failures_by_package(packages:, base_dir: Dir.pwd)
+            metadata_files = discover_metadata_files(base_dir)
+            return {} if metadata_files.empty?
+
+            result = {}
+            packages.each do |package|
+              package_metadata = filter_by_package(metadata_files, package)
+              most_recent = most_recent_per_test(package_metadata)
+              failed_ids = extract_all_failed_ids(most_recent)
+              result[package] = failed_ids unless failed_ids.empty?
+            end
+            result
+          end
+
+          # Find failed test scenarios grouped by package and scenario (test-id)
+          #
+          # Like find_failures_by_package but preserves per-scenario granularity.
+          # Callers can use this to re-run full failed scenarios.
+          #
+          # @param packages [Array<String>] Package names to scan
+          # @param base_dir [String] Base directory to search from (default: current dir)
+          # @return [Hash{String => Hash{String => Array<String>}}]
+          #   Package name => { test-id => failed TC IDs }
+          def find_failures_by_scenario(packages:, base_dir: Dir.pwd)
+            metadata_files = discover_metadata_files(base_dir)
+            return {} if metadata_files.empty?
+
+            result = {}
+            packages.each do |package|
+              package_metadata = filter_by_package(metadata_files, package)
+              most_recent = most_recent_per_test(package_metadata)
+
+              scenario_failures = {}
+              most_recent.each do |entry|
+                test_id = entry[:data]["test-id"]
+                failed_ids = extract_failed_test_cases(entry[:data])
+                scenario_failures[test_id] = failed_ids unless failed_ids.empty?
+              end
+
+              result[package] = scenario_failures unless scenario_failures.empty?
+            end
+            result
+          end
+
+          private
+
+          # Discover all metadata.yml files in the cache directory
+          #
+          # @param base_dir [String] Base directory
+          # @return [Array<Hash>] Parsed metadata entries with :path and :data keys
+          def discover_metadata_files(base_dir)
+            cache_path = File.join(base_dir, CACHE_DIR)
+            return [] unless Dir.exist?(cache_path)
+
+            pattern = File.join(cache_path, "*#{REPORTS_SUFFIX}", METADATA_FILE)
+            Dir.glob(pattern).filter_map { |path| load_metadata(path) }
+          end
+
+          # Safely load and parse a metadata.yml file
+          #
+          # @param path [String] Absolute path to metadata.yml
+          # @return [Hash, nil] Hash with :path and :data keys, or nil on error
+          def load_metadata(path)
+            data = YAML.safe_load_file(path, permitted_classes: [Date])
+            return nil unless data.is_a?(Hash)
+
+            {path: path, data: data}
+          rescue => e
+            warn "Warning: Could not parse #{path}: #{e.message}" if ENV["DEBUG"]
+            nil
+          end
+
+          # Filter metadata entries by package name
+          #
+          # @param entries [Array<Hash>] Metadata entries
+          # @param package [String] Package name to filter by
+          # @return [Array<Hash>] Filtered entries
+          def filter_by_package(entries, package)
+            entries.select { |entry| entry[:data]["package"] == package }
+          end
+
+          # Get the most recent metadata entry per test-id
+          #
+          # Uses the report directory name (which contains a timestamp prefix)
+          # to determine recency. Later timestamps sort higher alphabetically.
+          #
+          # @param entries [Array<Hash>] Metadata entries
+          # @return [Array<Hash>] Most recent entry per test-id
+          def most_recent_per_test(entries)
+            grouped = entries.group_by { |entry| entry[:data]["test-id"] }
+            grouped.map do |_test_id, group|
+              # Sort by directory name (timestamp prefix ensures chronological order)
+              group.max_by { |entry| File.basename(File.dirname(entry[:path])) }
+            end
+          end
+
+          # Extract failed test case IDs from metadata entries
+          #
+          # Checks both the `failed_test_cases` array (from task 259.03 ReportWriter)
+          # and falls back to checking `status: "fail"` for older metadata formats.
+          #
+          # @param entries [Array<Hash>] Most recent metadata entries
+          # @return [Array<String>] Aggregated failed test case IDs
+          def extract_all_failed_ids(entries)
+            entries.flat_map { |entry| extract_failed_test_cases(entry[:data]) }.uniq
+          end
+
+          # Extract failed test case IDs from a single metadata hash
+          #
+          # Returns specific TC IDs when available, or ["*"] as a wildcard
+          # when metadata indicates failure but lacks granular test case data
+          # (common in legacy/CLI-agent-written metadata).
+          #
+          # @param data [Hash] Parsed metadata.yml data
+          # @return [Array<String>] Failed test case IDs, ["*"] for wildcard, or []
+          def extract_failed_test_cases(data)
+            # TC-first schema: failed: [{tc: "TC-001", ...}, ...]
+            failed_entries = data["failed"]
+            if failed_entries.is_a?(Array) && !failed_entries.empty?
+              tc_ids = failed_entries.filter_map { |entry| entry.is_a?(Hash) ? entry["tc"] : nil }.compact
+              return tc_ids unless tc_ids.empty?
+            end
+
+            # Primary: use failed_test_cases array (written by ReportWriter or workflow template)
+            failed_ids = data["failed_test_cases"]
+            return Array(failed_ids) if failed_ids.is_a?(Array) && !failed_ids.empty?
+
+            # Fallback: metadata has failures but no specific test case IDs.
+            # Return wildcard to signal "re-run entire test scenario".
+            status = data["status"]
+            return ["*"] if %w[fail partial error incomplete].include?(status)
+
+            []
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ace/test/end_to_end_runner/molecules/fixture_copier.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require "fileutils"
+
+module Ace
+  module Test
+    module EndToEndRunner
+      module Molecules
+        # Copies fixture files from a scenario's fixtures/ directory into a sandbox
+        #
+        # Preserves the full directory tree structure. Used by SetupExecutor
+        # to populate sandboxes with test data files.
+        #
+        # Note: This is a Molecule (not an Atom) because it performs filesystem
+        # I/O via FileUtils.cp_r and Dir.glob.
+        class FixtureCopier
+          # Copy fixture tree into target directory
+          #
+          # @param source_dir [String] Path to the fixtures/ directory
+          # @param target_dir [String] Path to the sandbox directory
+          # @return [Array<String>] Relative paths of copied files and directories
+          # @raise [ArgumentError] If source_dir does not exist
+          def copy(source_dir:, target_dir:)
+            raise ArgumentError, "Fixture source directory not found: #{source_dir}" unless Dir.exist?(source_dir)
+
+            FileUtils.mkdir_p(target_dir)
+            FileUtils.cp_r("#{source_dir}/.", target_dir)
+
+            Dir.glob("**/*", base: target_dir).sort
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ace/test/end_to_end_runner/molecules/pipeline_executor.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+require "ace/llm"
+require "ace/llm/query_interface"
+
+module Ace
+  module Test
+    module EndToEndRunner
+      module Molecules
+        # Executes standalone scenarios using the deterministic pipeline.
+        class PipelineExecutor
+          # @param provider [String]
+          # @param timeout [Integer]
+          # @param sandbox_builder [Molecules::PipelineSandboxBuilder]
+          # @param prompt_bundler [Molecules::PipelinePromptBundler]
+          # @param report_generator [Molecules::PipelineReportGenerator]
+          def initialize(provider:, timeout:, sandbox_builder: nil, prompt_bundler: nil, report_generator: nil)
+            @provider = provider
+            @timeout = timeout
+            @sandbox_builder = sandbox_builder || PipelineSandboxBuilder.new
+            @prompt_bundler = prompt_bundler || PipelinePromptBundler.new
+            @report_generator = report_generator || PipelineReportGenerator.new
+          end
+
+          # @param scenario [Models::TestScenario]
+          # @param cli_args [String, nil]
+          # @param sandbox_path [String]
+          # @param report_dir [String]
+          # @param env_vars [Hash, nil]
+          # @param test_cases [Array<String>, nil]
+          # @return [Models::TestResult]
+          def execute(scenario:, cli_args:, sandbox_path:, report_dir:, env_vars: nil, test_cases: nil)
+            started_at = Time.now
+
+            build_env = @sandbox_builder.build(
+              scenario: scenario,
+              sandbox_path: sandbox_path,
+              test_cases: test_cases
+            )
+            merged_env = (env_vars || {}).merge(build_env)
+
+            runner = @prompt_bundler.prepare_runner(
+              scenario: scenario,
+              sandbox_path: sandbox_path,
+              test_cases: test_cases
+            )
+            run_llm(
+              prompt_path: runner[:prompt_path],
+              system_path: runner[:system_path],
+              output_path: runner[:output_path],
+              cli_args: cli_args,
+              env_vars: merged_env
+            )
+
+            verifier = @prompt_bundler.prepare_verifier(
+              scenario: scenario,
+              sandbox_path: sandbox_path,
+              test_cases: test_cases
+            )
+            verifier_response = run_llm(
+              prompt_path: verifier[:prompt_path],
+              system_path: verifier[:system_path],
+              output_path: verifier[:output_path],
+              cli_args: cli_args,
+              env_vars: merged_env
+            )
+
+            @report_generator.generate(
+              scenario: scenario,
+              verifier_output: verifier_response[:text],
+              report_dir: report_dir,
+              provider: @provider,
+              started_at: started_at,
+              completed_at: Time.now
+            )
+          rescue => e
+            begin
+              @report_generator.write_failure_report(
+                scenario: scenario,
+                report_dir: report_dir,
+                provider: @provider,
+                started_at: started_at || Time.now,
+                completed_at: Time.now,
+                error_message: "#{e.class}: #{e.message}"
+              )
+            rescue => write_error
+              Models::TestResult.new(
+                test_id: scenario.test_id,
+                status: "error",
+                summary: "Execution pipeline failed",
+                error: "#{e.class}: #{e.message}; failed to write error report: #{write_error.class}: #{write_error.message}",
+                started_at: started_at || Time.now,
+                completed_at: Time.now
+              )
+            end
+          end
+
+          private
+
+          def run_llm(prompt_path:, system_path:, output_path:, cli_args:, env_vars:)
+            prompt = File.read(prompt_path)
+            system = File.read(system_path)
+            working_dir = env_vars["PROJECT_ROOT_PATH"] || env_vars[:PROJECT_ROOT_PATH]
+
+            Ace::LLM::QueryInterface.query(
+              @provider,
+              prompt,
+              system: system,
+              cli_args: cli_args,
+              timeout: @timeout,
+              fallback: false,
+              output: output_path,
+              working_dir: working_dir,
+              subprocess_env: env_vars
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ace/test/end_to_end_runner/molecules/pipeline_prompt_bundler.rb

@@ -0,0 +1,182 @@
+# frozen_string_literal: true
+
+require "date"
+require "fileutils"
+require "yaml"
+
+module Ace
+  module Test
+    module EndToEndRunner
+      module Molecules
+        # Prepares deterministic runner/verifier prompt files for pipeline execution.
+        class PipelinePromptBundler
+          RUNNER_SYSTEM_PROMPT = <<~PROMPT
+            You are an E2E test executor working in a sandbox directory.
+
+            Rules:
+            - Execute each goal in order
+            - Save all artifacts to results/tc/{NN}/ directories as specified
+            - Treat the initial working directory as SANDBOX_ROOT; if a goal needs commands in a created worktree, cd there for execution but keep artifact writes under SANDBOX_ROOT/results
+            - Do not fabricate output - all artifacts must come from real tool execution
+            - If a goal fails, note the failure and continue to the next goal
+            - After all goals, output a brief summary of what you produced for each goal
+          PROMPT
+
+          VERIFIER_SYSTEM_PROMPT = <<~PROMPT
+            You are an E2E test verifier. You inspect artifacts and render PASS/FAIL verdicts.
+
+            Rules:
+            - Evaluate each goal independently based solely on the artifacts provided
+            - Do not speculate about what the runner did - only judge what exists
+            - For each failed goal, include a category:
+              test-spec-error | tool-bug | runner-error | infrastructure-error
+            - For each goal, cite specific evidence (filenames, content snippets)
+            - Follow the output format exactly
+          PROMPT
+
+          # @param scenario [Models::TestScenario]
+          # @param sandbox_path [String]
+          # @param test_cases [Array<String>, nil]
+          # @return [Hash]
+          def prepare_runner(scenario:, sandbox_path:, test_cases: nil)
+            cache_dir = ensure_cache_dir(sandbox_path)
+            system_path = File.join(cache_dir, "runner-system.md")
+            prompt_path = File.join(cache_dir, "runner-prompt.md")
+
+            File.write(system_path, RUNNER_SYSTEM_PROMPT)
+
+            bundled = bundle_markdown_file(File.join(scenario.dir_path, "runner.yml.md"), test_cases: test_cases)
+            bundled = bundled.gsub("Workspace root: (current directory)", "Workspace root: #{File.expand_path(sandbox_path)}")
+            File.write(prompt_path, bundled)
+
+            {
+              system_path: system_path,
+              prompt_path: prompt_path,
+              output_path: File.join(cache_dir, "runner-output.md")
+            }
+          end
+
+          # @param scenario [Models::TestScenario]
+          # @param sandbox_path [String]
+          # @param test_cases [Array<String>, nil]
+          # @return [Hash]
+          def prepare_verifier(scenario:, sandbox_path:, test_cases: nil)
+            cache_dir = ensure_cache_dir(sandbox_path)
+            system_path = File.join(cache_dir, "verifier-system.md")
+            prompt_path = File.join(cache_dir, "verifier-prompt.md")
+
+            File.write(system_path, VERIFIER_SYSTEM_PROMPT)
+
+            artifacts = build_artifact_section(sandbox_path)
+            criteria = bundle_markdown_file(File.join(scenario.dir_path, "verifier.yml.md"), test_cases: test_cases)
+            File.write(prompt_path, [artifacts, criteria].join("\n\n---\n\n"))
+
+            {
+              system_path: system_path,
+              prompt_path: prompt_path,
+              output_path: File.join(cache_dir, "verifier-output.md")
+            }
+          end
+
+          private
+
+          def ensure_cache_dir(sandbox_path)
+            cache_dir = File.join(File.expand_path(sandbox_path), ".ace-local", "e2e")
+            FileUtils.mkdir_p(cache_dir)
+            cache_dir
+          end
+
+          def bundle_markdown_file(path, test_cases: nil)
+            raw = File.read(path)
+            frontmatter, body = split_frontmatter(raw)
+            bundle_files = parse_bundle_files(frontmatter, path)
+            selected_ids = normalize_selected_ids(test_cases)
+
+            included_paths = bundle_files.select do |entry|
+              include_bundle_entry?(entry, selected_ids)
+            end
+
+            sections = [body.rstrip]
+            included_paths.each do |entry|
+              full_path = File.expand_path(entry, File.dirname(path))
+              sections << File.read(full_path).rstrip
+            end
+            sections.reject(&:empty?).join("\n\n---\n\n")
+          end
+
+          def split_frontmatter(raw)
+            match = raw.match(/\A---\s*\r?\n(.*?)\r?\n---\s*\r?\n(.*)\z/m)
+            return [{}, raw] unless match
+
+            parsed = YAML.safe_load(match[1], permitted_classes: [Date]) || {}
+            [parsed, match[2]]
+          end
+
+          def parse_bundle_files(frontmatter, path)
+            files = frontmatter.dig("bundle", "files")
+            return [] unless files.is_a?(Array)
+
+            files.map(&:to_s).reject(&:empty?)
+          rescue Psych::SyntaxError => e
+            raise ArgumentError, "Invalid YAML frontmatter in #{path}: #{e.message}"
+          end
+
+          def normalize_selected_ids(test_cases)
+            return nil unless test_cases && !test_cases.empty?
+
+            test_cases.map { |tc| tc.to_s.upcase }.to_set
+          end
+
+          def include_bundle_entry?(entry, selected_ids)
+            return true unless selected_ids
+
+            tc_id = extract_tc_id(entry)
+            return true if tc_id.nil?
+
+            selected_ids.include?(tc_id)
+          end
+
+          def extract_tc_id(path)
+            match = File.basename(path).match(/\A(TC-\d+[a-z]*)/i)
+            match ? match[1].upcase : nil
+          end
+
+          def build_artifact_section(sandbox_path)
+            sandbox_path = File.expand_path(sandbox_path)
+            files = Dir.glob(File.join(sandbox_path, "results", "**", "*")).select { |f| File.file?(f) }.sort
+            tree_entries = files.map { |f| relative_path(f, sandbox_path) }
+
+            parts = []
+            parts << "# Sandbox Artifacts"
+            parts << ""
+            parts << "## Directory tree"
+            parts << "```"
+            parts.concat(tree_entries)
+            parts << "```"
+            parts << ""
+            parts << "## File contents"
+            parts << ""
+
+            files.each do |file|
+              parts << "### `#{relative_path(file, sandbox_path)}`"
+              parts << "```"
+              parts << safe_read(file)
+              parts << "```"
+              parts << ""
+            end
+
+            parts.join("\n").rstrip
+          end
+
+          def relative_path(path, root)
+            File.expand_path(path).sub("#{File.expand_path(root)}/", "")
+          end
+
+          def safe_read(path)
+            File.binread(path).encode("UTF-8", invalid: :replace, undef: :replace, replace: "?")
+          end
+        end
+      end
+    end
+  end
+end