ace-test-runner-e2e 0.29.0

data/lib/ace/test/end_to_end_runner/molecules/pipeline_report_generator.rb

@@ -0,0 +1,321 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "yaml"
+
+module Ace
+  module Test
+    module EndToEndRunner
+      module Molecules
+        # Generates TC-first reports from standalone verifier output.
+        class PipelineReportGenerator
+          FAILURE_CATEGORIES = %w[test-spec-error tool-bug runner-error infrastructure-error].freeze
+
+          # @param report_writer [Molecules::ReportWriter]
+          def initialize(report_writer: nil)
+            @report_writer = report_writer || Molecules::ReportWriter.new
+          end
+
+          # @param scenario [Models::TestScenario]
+          # @param verifier_output [String]
+          # @param report_dir [String]
+          # @param provider [String]
+          # @param started_at [Time]
+          # @param completed_at [Time]
+          # @return [Models::TestResult]
+          def generate(scenario:, verifier_output:, report_dir:, provider:, started_at:, completed_at:)
+            parsed = parse_verifier_output(verifier_output, scenario)
+
+            result = Models::TestResult.new(
+              test_id: scenario.test_id,
+              status: parsed[:status],
+              test_cases: parsed[:test_cases],
+              summary: parsed[:summary],
+              error: parsed[:error],
+              started_at: started_at,
+              completed_at: completed_at
+            )
+
+            FileUtils.mkdir_p(report_dir)
+            @report_writer.write(result, scenario, report_dir: report_dir)
+            write_goal_report(
+              path: File.join(report_dir, "report.md"),
+              scenario: scenario,
+              provider: provider,
+              result: result
+            )
+            result.with_report_dir(report_dir)
+          end
+
+          # Write deterministic error reports when pipeline execution fails before
+          # normal verifier parsing/report generation can complete.
+          #
+          # @param scenario [Models::TestScenario]
+          # @param report_dir [String]
+          # @param provider [String]
+          # @param started_at [Time]
+          # @param completed_at [Time]
+          # @param error_message [String]
+          # @return [Models::TestResult]
+          def write_failure_report(scenario:, report_dir:, provider:, started_at:, completed_at:, error_message:)
+            result = Models::TestResult.new(
+              test_id: scenario.test_id,
+              status: "error",
+              test_cases: [],
+              summary: "Execution pipeline failed",
+              error: error_message,
+              started_at: started_at,
+              completed_at: completed_at
+            )
+
+            FileUtils.mkdir_p(report_dir)
+            @report_writer.write(result, scenario, report_dir: report_dir)
+            write_goal_report(
+              path: File.join(report_dir, "report.md"),
+              scenario: scenario,
+              provider: provider,
+              result: result
+            )
+            result.with_report_dir(report_dir)
+          end
+
+          private
+
+          def parse_verifier_output(text, scenario)
+            goals = parse_goal_sections(text, scenario)
+            return build_result_from_goals(goals) unless goals.empty?
+
+            parsed = Atoms::SkillResultParser.parse_verifier(text)
+            {
+              status: parsed[:status],
+              test_cases: parsed[:test_cases],
+              summary: parsed[:summary],
+              error: parsed[:observations]
+            }
+          rescue Atoms::ResultParser::ParseError => e
+            issue = summarize_unstructured_verifier_output(text)
+            {
+              status: "error",
+              test_cases: [],
+              summary: "Verifier returned unstructured output",
+              error: issue || e.message
+            }
+          end
+
+          def parse_goal_sections(text, scenario)
+            lines = text.to_s.lines
+            headers = []
+            lines.each_with_index do |line, idx|
+              match = line.match(/^\#{2,3}\s+Goal\s+(\d+)\s*[—-]\s*(.+?)\s*$/i)
+              headers << [idx, match[1].to_i, match[2].strip] if match
+            end
+            return [] if headers.empty?
+
+            scenario_test_cases = scenario.test_cases || []
+
+            headers.each_with_index.map do |(start_idx, goal_number, title), index|
+              end_idx = (index + 1 < headers.size) ? headers[index + 1][0] : lines.size
+              block = lines[start_idx...end_idx].join
+
+              verdict = normalize_verdict(extract_field_token(block, %w[Verdict Status]))
+              evidence = extract_evidence(block)
+              next if verdict.nil?
+
+              tc_id = scenario_test_cases[goal_number - 1]&.tc_id || format("TC-%03d", goal_number)
+              category = extract_category(block, evidence)
+
+              {
+                id: tc_id,
+                description: title,
+                status: (verdict == "PASS") ? "pass" : "fail",
+                notes: evidence,
+                category: ((verdict == "FAIL") ? category : nil)
+              }
+            end.compact
+          end
+
+          def extract_value(block, field)
+            match = block.match(/^\s*[-*]?\s*\*\*#{Regexp.escape(field)}\*\*:\s*(.+?)\s*$/im)
+            return nil unless match
+
+            match[1].strip
+          end
+
+          def extract_evidence(block)
+            lines = block.to_s.lines
+            marker_index = nil
+            inline_value = nil
+
+            lines.each_with_index do |line, idx|
+              match = line.match(/^\s*[-*]?\s*\*\*Evidence(?:\s+of\s+failure)?\*\*:\s*(.*)$/i)
+              next unless match
+
+              marker_index = idx
+              inline_value = match[1].to_s.strip
+              break
+            end
+
+            return inline_value unless marker_index
+
+            collected = []
+            collected << inline_value unless inline_value.empty?
+
+            lines[(marker_index + 1)..]&.each do |line|
+              break if line.match?(/^\s*[-*]?\s*\*\*(Category|Verdict)\*\*:/i)
+              break if line.match?(/^\#{2,3}\s+Goal\s+\d+/i)
+              break if line.match?(/^\s*\*\*Results/i)
+              break if line.strip == "---"
+
+              text = line.rstrip
+              next if text.strip.empty?
+
+              text = text.sub(/^\s*[-*]\s+/, "")
+              collected << text.strip
+            end
+
+            collected.join(" ").strip
+          end
+
+          def extract_category(block, evidence)
+            explicit = extract_field_token(block, %w[Category])
+            return normalize_category(explicit) if explicit
+
+            inline = block.to_s.match(/`(test-spec-error|tool-bug|runner-error|infrastructure-error)`/i)
+            return normalize_category(inline[1]) if inline
+
+            paren = block.to_s.match(/\((test-spec-error|tool-bug|runner-error|infrastructure-error)\)/i)
+            return normalize_category(paren[1]) if paren
+
+            normalize_category("#{block}\n#{evidence}")
+          end
+
+          def normalize_category(value)
+            category = value.to_s.strip.downcase
+            match = category.match(/\b(test-spec-error|tool-bug|runner-error|infrastructure-error)\b/)
+            return match[1] if match
+
+            "runner-error"
+          end
+
+          def normalize_verdict(value)
+            raw = value.to_s.strip
+            return nil if raw.empty?
+
+            token = raw.gsub(/[*_`]/, "").upcase.match(/\b(PASS|FAIL)\b/)
+            return token[1] if token
+
+            nil
+          end
+
+          def extract_field_token(block, fields)
+            fields.each do |field|
+              direct = extract_value(block, field)
+              return direct if direct && !direct.empty?
+
+              bold_inline = block.match(/\*\*#{Regexp.escape(field)}\s*:\s*([^*\n]+)\*\*/i)
+              return bold_inline[1].strip if bold_inline
+
+              plain = block.match(/^\s*(?:[-*]\s+)?#{Regexp.escape(field)}\s*:\s*(.+?)\s*$/im)
+              return plain[1].strip if plain
+            end
+
+            nil
+          end
+
+          def build_result_from_goals(goals)
+            passed = goals.count { |goal| goal[:status] == "pass" }
+            total = goals.size
+            status = if passed == total
+              "pass"
+            elsif passed.zero?
+              "fail"
+            else
+              "partial"
+            end
+
+            {
+              status: status,
+              test_cases: goals,
+              summary: "#{passed}/#{total} passed"
+            }
+          end
+
+          def summarize_unstructured_verifier_output(text)
+            summary = text.to_s.lines.map(&:strip).reject(&:empty?).first(3).join(" ")
+            return nil if summary.empty?
+
+            (summary.length > 240) ? "#{summary[0, 237]}..." : summary
+          end
+
+          def write_goal_report(path:, scenario:, provider:, result:)
+            passed = result.passed_count
+            failed = result.failed_count
+            total = result.total_count
+            score = total.zero? ? 0.0 : (passed.to_f / total).round(3)
+            verdict = if result.status == "error"
+              "fail"
+            elsif failed.zero?
+              "pass"
+            elsif passed.zero?
+              "fail"
+            else
+              "partial"
+            end
+
+            frontmatter = {
+              "test-id" => scenario.test_id,
+              "title" => scenario.title,
+              "package" => scenario.package,
+              "runner-provider" => provider,
+              "verifier-provider" => provider,
+              "timestamp" => result.completed_at.utc.strftime("%Y-%m-%dT%H:%M:%SZ"),
+              "tcs-passed" => passed,
+              "tcs-failed" => failed,
+              "tcs-total" => total,
+              "score" => score,
+              "verdict" => verdict,
+              "passed" => result.test_cases.select { |tc| tc[:status] == "pass" }.map { |tc| tc[:id] },
+              "failed" => result.test_cases.select { |tc| tc[:status] == "fail" }.map do |tc|
+                {
+                  "tc" => tc[:id],
+                  "category" => tc[:category] || "runner-error",
+                  "evidence" => tc[:notes].to_s
+                }
+              end
+            }
+            frontmatter_yaml = YAML.dump(frontmatter).sub(/\A---\s*\n/, "").sub(/\.\.\.\s*\n\z/, "")
+
+            rows = result.test_cases.map do |tc|
+              "| #{tc[:id]} | #{tc[:status].upcase} | #{tc[:notes]} |"
+            end.join("\n")
+
+            content = <<~REPORT
+              ---
+              #{frontmatter_yaml.rstrip}
+              ---
+
+              # E2E Report: #{scenario.title}
+
+              ## Goal Results
+
+              | Goal | Verdict | Evidence |
+              |------|---------|----------|
+              #{rows}
+
+              ## Summary
+
+              | Metric | Value |
+              |--------|-------|
+              | Passed | #{passed} |
+              | Failed | #{failed} |
+              | Total  | #{total} |
+              | Score  | #{(score * 100).round(1)}% |
+            REPORT
+
+            File.write(path, content)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ace/test/end_to_end_runner/molecules/pipeline_sandbox_builder.rb

@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "open3"
+
+module Ace
+  module Test
+    module EndToEndRunner
+      module Molecules
+        # Builds deterministic sandbox state for standalone execution.
+        class PipelineSandboxBuilder
+          # @param config_root [String] Project root used for provider symlink/bin path
+          def initialize(config_root: Dir.pwd)
+            @config_root = File.expand_path(config_root)
+          end
+
+          # @param scenario [Models::TestScenario]
+          # @param sandbox_path [String]
+          # @param test_cases [Array<String>, nil] Optional TC filter
+          # @return [Hash] Environment variables for subprocess execution
+          def build(scenario:, sandbox_path:, test_cases: nil)
+            sandbox_path = File.expand_path(sandbox_path)
+            FileUtils.mkdir_p(sandbox_path)
+            FileUtils.mkdir_p(File.join(sandbox_path, ".ace-local", "e2e"))
+            FileUtils.mkdir_p(File.join(sandbox_path, "reports"))
+
+            initialize_git_repo(sandbox_path)
+            ensure_package_available(scenario.package, sandbox_path)
+            link_provider_configs(sandbox_path)
+            create_result_directories(scenario, sandbox_path, test_cases: test_cases)
+            verify_tool_access(scenario, sandbox_path)
+
+            {
+              "PROJECT_ROOT_PATH" => sandbox_path
+            }
+          end
+
+          private
+
+          def ensure_package_available(package_name, sandbox_path)
+            package_name = package_name.to_s.strip
+            return if package_name.empty?
+
+            package_source = File.join(@config_root, package_name)
+            package_target = File.join(sandbox_path, package_name)
+
+            return if File.exist?(package_target)
+
+            unless File.directory?(package_source)
+              raise "Scenario package not found: #{package_name} (expected #{package_source})"
+            end
+
+            FileUtils.cp_r(package_source, package_target)
+          end
+
+          def initialize_git_repo(sandbox_path)
+            return if Dir.exist?(File.join(sandbox_path, ".git"))
+
+            _stdout, stderr, status = Open3.capture3("git", "init", "-b", "main", chdir: sandbox_path)
+            return if status.success?
+
+            raise "Sandbox git init failed: #{stderr}".strip
+          end
+
+          def link_provider_configs(sandbox_path)
+            source = File.join(@config_root, ".ace", "llm", "providers")
+            target = File.join(sandbox_path, ".ace", "llm", "providers")
+            FileUtils.mkdir_p(File.dirname(target))
+
+            FileUtils.rm_f(target) if File.symlink?(target)
+            FileUtils.rm_rf(target) if File.directory?(target)
+
+            if File.directory?(source)
+              File.symlink(source, target)
+            else
+              FileUtils.mkdir_p(target)
+            end
+          end
+
+          def create_result_directories(scenario, sandbox_path, test_cases:)
+            result_dirs = resolve_result_dirs(scenario, test_cases: test_cases)
+            result_dirs.each do |relative_dir|
+              FileUtils.mkdir_p(File.join(sandbox_path, relative_dir))
+            end
+          end
+
+          def resolve_result_dirs(scenario, test_cases:)
+            all_cases = scenario.test_cases || []
+            case_positions = all_cases.each_with_index.to_h do |tc, idx|
+              [tc.tc_id.to_s.upcase, idx + 1]
+            end
+
+            selected_positions = if test_cases && !test_cases.empty?
+              test_cases.filter_map { |tc_id| case_positions[tc_id.to_s.upcase] }.uniq.sort
+            else
+              case_positions.values.sort
+            end
+
+            selected_positions = (1..all_cases.size).to_a if selected_positions.empty? && !all_cases.empty?
+
+            layout_keys = (scenario.sandbox_layout || {}).keys
+            if layout_keys.any?
+              selected_from_layout = layout_keys.select do |key|
+                idx = extract_result_dir_index(key)
+                idx.nil? || selected_positions.include?(idx)
+              end
+              return selected_from_layout unless selected_from_layout.empty?
+            end
+
+            selected_positions.map { |idx| "results/tc/#{format("%02d", idx)}" }
+          end
+
+          def extract_result_dir_index(path)
+            match = path.to_s.match(%r{results/tc/(\d{1,3})/?})
+            match ? match[1].to_i : nil
+          end
+
+          def verify_tool_access(scenario, sandbox_path)
+            tool = scenario.tool_under_test.to_s.strip
+            return if tool.empty?
+
+            _stdout, stderr, status = Open3.capture3(tool, "--help", chdir: sandbox_path)
+            return if status.success?
+
+            raise "Sandbox tool check failed for #{tool}: #{stderr}".strip
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ace/test/end_to_end_runner/molecules/progress_display_manager.rb

@@ -0,0 +1,172 @@
+# frozen_string_literal: true
+
+module Ace
+  module Test
+    module EndToEndRunner
+      module Molecules
+        # Animated ANSI table display manager for E2E test output (--progress mode).
+        # Updates test rows in place using cursor movement escape codes.
+        # Modeled on ace-test-runner's DisplayManager for visual consistency.
+        class ProgressDisplayManager
+          # @param scenarios [Array<Models::TestScenario>] tests to run
+          # @param output [IO] output stream
+          # @param parallel [Integer] parallelism level
+          def initialize(scenarios, output:, parallel:)
+            @scenarios = scenarios
+            @output = output
+            @parallel = parallel
+            @use_color = output.respond_to?(:tty?) && output.tty?
+            @start_time = Time.now
+            @last_refresh = Time.at(0)
+            @lines = {}           # scenario.test_id => line number
+            @states = {}          # scenario.test_id => :waiting | :running | :completed
+            @results = {}         # scenario.test_id => Models::TestResult
+            @started_at = {}      # scenario.test_id => Time
+            @title_width = calculate_title_width
+          end
+
+          # Print header and initial table with all tests in waiting state
+          def initialize_display
+            h = Atoms::DisplayHelpers
+            package = @scenarios.first&.package || "unknown"
+
+            # Clear screen (preserves scrollback)
+            @output.print "\033[H\033[J"
+
+            @output.puts h.separator
+            @output.puts "  E2E Tests: #{package} (#{@scenarios.size} tests)"
+            @output.puts h.separator
+            @output.puts
+
+            @scenarios.each_with_index do |scenario, index|
+              line = index + 5  # account for header lines
+              @lines[scenario.test_id] = line
+              @states[scenario.test_id] = :waiting
+              print_row(scenario)
+            end
+
+            @output.puts
+            @output.puts
+            @footer_line = @lines.values.max + 3
+            update_footer
+          end
+
+          # Update row when a test begins
+          # @param scenario [Models::TestScenario]
+          def test_started(scenario)
+            @states[scenario.test_id] = :running
+            @started_at[scenario.test_id] = Time.now
+            print_row(scenario)
+            update_footer
+          end
+
+          # Update row when a test completes
+          # @param scenario [Models::TestScenario]
+          # @param result [Models::TestResult]
+          # @param completed [Integer]
+          # @param total [Integer]
+          def test_completed(scenario, result, completed, total)
+            @states[scenario.test_id] = :completed
+            @results[scenario.test_id] = result
+            print_row(scenario)
+            update_footer
+          end
+
+          # Refresh running test rows to update elapsed timers
+          # Throttled to ~4Hz — redraws are expensive with ANSI cursor movement
+          def refresh
+            now = Time.now
+            return if now - @last_refresh < REFRESH_INTERVAL
+
+            @last_refresh = now
+
+            @states.each do |test_id, state|
+              next unless state == :running
+
+              scenario = @scenarios.find { |s| s.test_id == test_id }
+              print_row(scenario) if scenario
+            end
+            update_footer
+          end
+
+          # Print a single-test result line (for run-single-test mode)
+          # @param result [Models::TestResult]
+          def show_single_result(result)
+            @output.puts Atoms::DisplayHelpers.format_single_result(result, use_color: @use_color)
+          end
+
+          # Print structured summary block
+          # @param results [Array<Models::TestResult>]
+          # @param report_path [String]
+          def show_summary(results, report_path)
+            # Move cursor past the display area
+            move_to_line(@footer_line + 1)
+            @output.puts
+
+            lines = Atoms::DisplayHelpers.format_summary_lines(
+              results, Time.now - @start_time, report_path, use_color: @use_color
+            )
+            lines.each { |line| @output.puts line }
+          end
+
+          private
+
+          def print_row(scenario)
+            h = Atoms::DisplayHelpers
+            line = @lines[scenario.test_id]
+            state = @states[scenario.test_id]
+
+            move_to_line(line)
+            @output.print "\033[K" # clear line
+
+            title = scenario.title.ljust(@title_width)
+
+            case state
+            when :waiting
+              icon = h.color("\u00b7", :gray, use_color: @use_color)
+              elapsed = "  0.0s"
+              status = "waiting"
+              @output.print "#{icon}  #{elapsed}  #{scenario.test_id}  #{title}  #{status}"
+
+            when :running
+              icon = h.color("\u22ef", :cyan, use_color: @use_color)
+              secs = Time.now - (@started_at[scenario.test_id] || Time.now)
+              elapsed = h.format_elapsed(secs)
+              status = "running"
+              @output.print "#{icon}  #{elapsed}  #{scenario.test_id}  #{title}  #{status}"
+
+            when :completed
+              result = @results[scenario.test_id]
+              success = result.success?
+              icon = h.color(h.status_icon(success), success ? :green : :red, use_color: @use_color)
+              elapsed = h.format_elapsed(result.duration)
+              tc = h.tc_count_display(result)
+              status_text = result.status.upcase
+              @output.print "#{icon}  #{elapsed}  #{scenario.test_id}  #{title}  #{status_text}#{tc}"
+            end
+          end
+
+          def update_footer
+            move_to_line(@footer_line)
+            @output.print "\033[K"
+
+            active = @states.count { |_, s| s == :running }
+            completed = @states.count { |_, s| s == :completed }
+            waiting = @states.count { |_, s| s == :waiting }
+
+            @output.print "Active: #{active} | Completed: #{completed} | Waiting: #{waiting}"
+          end
+
+          def move_to_line(line)
+            @output.print "\033[#{line};1H"
+          end
+
+          def calculate_title_width
+            max = @scenarios.map { |s| s.title.length }.max || 0
+            [max, 20].max
+          end
+        end
+      end
+    end
+  end
+end