ace-test-runner-e2e 0.29.0

data/lib/ace/test/end_to_end_runner/atoms/skill_result_parser.rb

@@ -0,0 +1,244 @@
+# frozen_string_literal: true
+
+module Ace
+  module Test
+    module EndToEndRunner
+      module Atoms
+        # Parses structured markdown results from CLI-provider skill/workflow execution
+        #
+        # CLI providers return results in the subagent return contract format:
+        #   - **Test ID**: TS-LINT-001
+        #   - **Status**: pass
+        #   - **Passed**: 8
+        #   - **Failed**: 0
+        #   - **Total**: 8
+        #   - **Report Paths**: 8p5jo2-lint-ts001-reports/*
+        #   - **Issues**: None
+        #
+        # Falls back to ResultParser.parse() for JSON responses.
+        class SkillResultParser
+          # Parse response text from a CLI provider
+          #
+          # @param text [String] Raw response text
+          # @return [Hash] Parsed result with :test_id, :status, :test_cases, :summary, :observations
+          # @raise [ResultParser::ParseError] If neither markdown nor JSON can be parsed
+          def self.parse(text)
+            raise ResultParser::ParseError, "Empty response from CLI provider" if text.nil? || text.strip.empty?
+
+            parsed = parse_markdown(text)
+            return to_normalized(parsed) if parsed
+
+            # Fall back to JSON parsing via ResultParser
+            ResultParser.parse(text)
+          end
+
+          # Parse the markdown return contract format
+          #
+          # @param text [String] Response text
+          # @return [Hash, nil] Parsed fields or nil if format not matched
+          def self.parse_markdown(text)
+            fields = {}
+
+            fields[:test_id] = extract_field(text, "Test ID")
+            fields[:status] = extract_field(text, "Status")
+            fields[:passed] = extract_field(text, "Passed")
+            fields[:failed] = extract_field(text, "Failed")
+            fields[:total] = extract_field(text, "Total")
+            fields[:report_paths] = extract_field(text, "Report Paths")
+            fields[:issues] = extract_field(text, "Issues")
+
+            # Need at least test_id and status for a valid parse
+            return nil unless fields[:test_id] && fields[:status]
+
+            fields
+          end
+
+          # Convert parsed markdown fields to normalized result format
+          #
+          # @param parsed [Hash] Parsed markdown fields
+          # @return [Hash] Normalized result matching ResultParser output format
+          def self.to_normalized(parsed)
+            parsed[:status] = normalize_status(parsed[:status])
+
+            passed = parsed[:passed].to_i
+            failed = parsed[:failed].to_i
+            total = parsed[:total].to_i
+
+            # Build synthetic test cases from counts
+            test_cases = []
+            passed.times { |i| test_cases << {id: "TC-#{format("%03d", i + 1)}", description: "", status: "pass", actual: "", notes: ""} }
+            failed.times { |i| test_cases << {id: "TC-#{format("%03d", passed + i + 1)}", description: "", status: "fail", actual: "", notes: ""} }
+
+            issues = parsed[:issues]
+            observations = (issues && issues.downcase != "none") ? issues : ""
+
+            {
+              test_id: parsed[:test_id],
+              status: parsed[:status],
+              test_cases: test_cases,
+              summary: "#{passed}/#{total} passed",
+              observations: observations
+            }
+          end
+
+          # Extract a field value from markdown bold-key format
+          #
+          # @param text [String] Text to search
+          # @param field_name [String] Field name (e.g., "Test ID")
+          # @return [String, nil] Extracted value or nil
+          def self.extract_field(text, field_name)
+            # Match "- **Field Name**: value" or "**Field Name**: value"
+            match = text.match(/\*\*#{Regexp.escape(field_name)}\*\*:\s*(.+?)$/i)
+            return nil unless match
+
+            value = match[1].strip
+            value.empty? ? nil : value
+          end
+
+          # Parse TC-level response text from a CLI provider
+          #
+          # Handles TC-level markdown with **TC ID** field. Falls back to
+          # parse() if the response has the multi-TC format.
+          #
+          # @param text [String] Raw response text
+          # @return [Hash] Parsed result with single-entry :test_cases array
+          # @raise [ResultParser::ParseError] If neither format can be parsed
+          def self.parse_tc(text)
+            raise ResultParser::ParseError, "Empty response from CLI provider" if text.nil? || text.strip.empty?
+
+            parsed = parse_tc_markdown(text)
+            return to_tc_normalized(parsed) if parsed
+
+            # Fall back to standard parse (handles both markdown and JSON)
+            parse(text)
+          end
+
+          # Parse verifier-mode markdown return contract.
+          #
+          # @param text [String]
+          # @return [Hash] Normalized test result payload
+          def self.parse_verifier(text)
+            raise ResultParser::ParseError, "Empty response from CLI provider" if text.nil? || text.strip.empty?
+
+            fields = {}
+            fields[:test_id] = extract_field(text, "Test ID")
+            fields[:status] = extract_field(text, "Status")
+            fields[:tcs_passed] = extract_field(text, "TCs Passed")
+            fields[:tcs_failed] = extract_field(text, "TCs Failed")
+            fields[:tcs_total] = extract_field(text, "TCs Total")
+            fields[:score] = extract_field(text, "Score")
+            fields[:verdict] = extract_field(text, "Verdict")
+            fields[:failed_tcs] = extract_field(text, "Failed TCs")
+            fields[:issues] = extract_field(text, "Issues")
+
+            return parse(text) unless fields[:test_id] && fields[:status] &&
+              fields[:tcs_passed] && fields[:tcs_failed] && fields[:tcs_total]
+
+            passed = fields[:tcs_passed].to_i
+            failed = fields[:tcs_failed].to_i
+            total = fields[:tcs_total].to_i
+            status = normalize_status(fields[:status])
+
+            failed_entries = parse_failed_tcs(fields[:failed_tcs])
+            failed_ids = failed_entries.map { |e| e[:tc] }.to_set
+            test_cases = []
+            pass_index = 0
+            passed.times do
+              pass_index += 1
+              pass_index += 1 while failed_ids.include?("TC-#{format("%03d", pass_index)}")
+              test_cases << {id: "TC-#{format("%03d", pass_index)}", description: "", status: "pass", actual: "", notes: ""}
+            end
+            if failed_entries.empty?
+              failed.times do |i|
+                test_cases << {id: "TC-#{format("%03d", passed + i + 1)}", description: "", status: "fail", actual: "", notes: ""}
+              end
+            else
+              failed_entries.each do |entry|
+                test_cases << {
+                  id: entry[:tc],
+                  description: "",
+                  status: "fail",
+                  actual: "",
+                  notes: entry[:category],
+                  category: entry[:category]
+                }
+              end
+            end
+
+            summary = if total.positive?
+              "#{passed}/#{total} passed (#{fields[:verdict] || status})"
+            else
+              fields[:verdict] || status
+            end
+
+            {
+              test_id: fields[:test_id],
+              status: status,
+              test_cases: test_cases,
+              summary: summary,
+              observations: (fields[:issues].to_s.strip.casecmp("none").zero? ? "" : fields[:issues].to_s)
+            }
+          end
+
+          # Parse TC-level markdown return contract
+          def self.parse_tc_markdown(text)
+            fields = {}
+
+            fields[:test_id] = extract_field(text, "Test ID")
+            fields[:tc_id] = extract_field(text, "TC ID")
+            fields[:status] = extract_field(text, "Status")
+            fields[:report_paths] = extract_field(text, "Report Paths")
+            fields[:issues] = extract_field(text, "Issues")
+
+            # Need test_id, tc_id, and status for a valid TC parse
+            return nil unless fields[:test_id] && fields[:tc_id] && fields[:status]
+
+            fields
+          end
+
+          # Convert parsed TC markdown to normalized result format
+          def self.to_tc_normalized(parsed)
+            parsed[:status] = normalize_status(parsed[:status])
+
+            issues = parsed[:issues]
+            observations = (issues && issues.downcase != "none") ? issues : ""
+
+            {
+              test_id: parsed[:test_id],
+              status: parsed[:status],
+              test_cases: [{
+                id: parsed[:tc_id],
+                description: "",
+                status: parsed[:status],
+                actual: "",
+                notes: observations
+              }],
+              summary: "#{parsed[:tc_id]} #{parsed[:status]}",
+              observations: observations
+            }
+          end
+
+          # Normalize a status value: take first word, default to "unknown"
+          def self.normalize_status(value)
+            (value.to_s.strip.split(/\s+/).first || "unknown").downcase
+          end
+
+          def self.parse_failed_tcs(value)
+            return [] if value.nil? || value.strip.empty? || value.strip.casecmp("none").zero?
+
+            value.split(",").map(&:strip).filter_map do |entry|
+              tc, category = entry.split(":", 2).map { |part| part.to_s.strip }
+              next if tc.empty?
+
+              {tc: tc.upcase, category: (category.to_s.empty? ? "unknown" : category)}
+            end
+          end
+
+          private_class_method :parse_markdown, :to_normalized, :extract_field,
+            :parse_tc_markdown, :to_tc_normalized, :normalize_status,
+            :parse_failed_tcs
+        end
+      end
+    end
+  end
+end

data/lib/ace/test/end_to_end_runner/atoms/suite_report_prompt_builder.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+module Ace
+  module Test
+    module EndToEndRunner
+      module Atoms
+        # Builds LLM prompts for suite-level final report synthesis
+        #
+        # Pure atom (no I/O). Constructs system and user prompts from
+        # pre-read test result data for LLM-based report generation.
+        class SuiteReportPromptBuilder
+          SYSTEM_PROMPT = <<~PROMPT
+            You are a senior QA engineer writing an E2E test suite report.
+
+            Generate a structured markdown report with YAML frontmatter. The report should provide actionable insights, not just raw data.
+
+            ## Required Sections
+
+            1. **YAML Frontmatter** — suite-id, package, status, tests-run, executed timestamp
+            2. **Summary Table** — Test ID, Title, Status, Passed, Failed, Total columns
+            3. **Overall Line** — "X/Y test cases passed (Z%)"
+            4. **Failed Tests** (if any) — For each failed test: root cause analysis, failed test case details
+            5. **Friction Analysis** — Developer experience issues, tooling pain points, environment problems observed across tests
+            6. **Improvement Suggestions** — Concrete, actionable recommendations based on the failures and friction observed
+            7. **Positive Observations** — What worked well, reliable patterns, strengths
+            8. **Reports Table** — Test ID mapped to report directory names
+
+            ## Formatting Rules
+
+            - Use GitHub-flavored markdown
+            - Frontmatter must be valid YAML between --- fences
+            - Keep root cause analysis concise but specific
+            - Friction analysis should focus on patterns across tests, not individual failures
+            - Suggestions should be actionable (not vague like "improve testing")
+            - If all tests pass, skip Failed Tests section and focus on positive observations and any friction
+          PROMPT
+
+          # Build user prompt from pre-read test result data
+          #
+          # @param results_data [Array<Hash>] Pre-read result data, each with:
+          #   :test_id, :title, :status, :passed, :failed, :total,
+          #   :test_cases, :report_dir_name, :summary_content, :experience_content
+          # @param package [String] Package name
+          # @param timestamp [String] Suite timestamp ID
+          # @param overall_status [String] "pass", "partial", or "fail"
+          # @param executed_at [String] ISO 8601 execution timestamp
+          # @return [String] User prompt for LLM
+          def build(results_data, package:, timestamp:, overall_status:, executed_at:)
+            parts = []
+            parts << "# Suite Report Request"
+            parts << ""
+            parts << "**Package:** #{package}"
+            parts << "**Suite ID:** #{timestamp}"
+            parts << "**Status:** #{overall_status}"
+            parts << "**Executed:** #{executed_at}"
+            parts << "**Tests Run:** #{results_data.size}"
+            parts << ""
+
+            total_passed = results_data.sum { |r| r[:passed] }
+            results_data.sum { |r| r[:failed] }
+            total_tc = results_data.sum { |r| r[:total] }
+            parts << "**Overall:** #{total_passed}/#{total_tc} test cases passed"
+            parts << ""
+
+            parts << "## Test Results"
+            parts << ""
+
+            results_data.each do |r|
+              parts << "### #{r[:test_id]}: #{r[:title]}"
+              parts << "- **Status:** #{r[:status]}"
+              parts << "- **Passed:** #{r[:passed]}/#{r[:total]}"
+              parts << "- **Report Dir:** #{r[:report_dir_name]}" if r[:report_dir_name]
+
+              if r[:test_cases]&.any?
+                parts << ""
+                parts << "**Test Cases:**"
+                r[:test_cases].each do |tc|
+                  parts << "- #{tc[:id]}: #{tc[:description]} — #{tc[:status]}"
+                end
+              end
+
+              if r[:summary_content]
+                parts << ""
+                parts << "**Summary Report:**"
+                parts << r[:summary_content]
+              end
+
+              if r[:experience_content]
+                parts << ""
+                parts << "**Experience Report:**"
+                parts << r[:experience_content]
+              end
+
+              parts << ""
+            end
+
+            parts.join("\n")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ace/test/end_to_end_runner/atoms/tc_fidelity_validator.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+module Ace
+  module Test
+    module EndToEndRunner
+      module Atoms
+        # Validates that agent-reported test cases match the scenario's expected TCs
+        #
+        # Detects when an agent invents its own test cases instead of executing
+        # the defined standalone TC files. Returns an error result when fidelity check fails.
+        class TcFidelityValidator
+          # Validate parsed result against expected test case count
+          #
+          # @param parsed [Hash] Parsed result from SkillResultParser (:test_cases, :status, etc.)
+          # @param scenario [Models::TestScenario] The scenario with expected TCs
+          # @param filtered_tc_ids [Array<String>, nil] TC IDs filter (when subset was requested)
+          # @return [Hash, nil] Error info hash if validation fails, nil if valid
+          def self.validate(parsed, scenario, filtered_tc_ids: nil)
+            expected_ids = filtered_tc_ids || scenario.test_case_ids
+            return nil if expected_ids.empty?
+
+            reported_count = parsed[:test_cases]&.size || 0
+            expected_count = expected_ids.size
+
+            return nil if reported_count == expected_count
+
+            {
+              error: "TC fidelity mismatch: agent reported #{reported_count} test cases " \
+                     "but scenario has #{expected_count} (#{expected_ids.join(", ")})",
+              expected_count: expected_count,
+              reported_count: reported_count,
+              expected_ids: expected_ids
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ace/test/end_to_end_runner/atoms/test_case_parser.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+module Ace
+  module Test
+    module EndToEndRunner
+      module Atoms
+        # Parses and normalizes test case IDs from markdown content
+        #
+        # Provides pure utility methods for:
+        # - Extracting TC-NNN headers from markdown test scenarios
+        # - Normalizing various test case ID formats to TC-NNN
+        # - Filtering test cases by ID list
+        #
+        # Normalization rules (consistent with workflow bash logic):
+        # - "TC-001"  -> "TC-001" (already normalized)
+        # - "tc-001"  -> "TC-001" (uppercased)
+        # - "001"     -> "TC-001" (prefix added)
+        # - "1"       -> "TC-001" (zero-padded and prefixed)
+        # - "TC-1"    -> "TC-001" (zero-padded)
+        class TestCaseParser
+          # Pattern matching TC-NNN headers in markdown
+          # Matches: ### TC-001: Description
+          TC_HEADER_PATTERN = /^###\s+(TC-\d+[a-z]?)[\s:]/i
+
+          # Normalize a single test case identifier to TC-NNN format
+          #
+          # @param id [String] Raw test case ID in any accepted format
+          # @return [String] Normalized TC-NNN format
+          # @raise [ArgumentError] If the ID cannot be normalized
+          def self.normalize_identifier(id)
+            raw = id.to_s.strip
+            raise ArgumentError, "Empty test case ID" if raw.empty?
+
+            # Strip TC- prefix if present (case-insensitive)
+            number_part = raw.sub(/\Atc-/i, "")
+
+            # Extract numeric portion and optional alpha suffix
+            match = number_part.match(/\A(\d+)([a-z]?)\z/i)
+            raise ArgumentError, "Invalid test case ID: '#{id}'" unless match
+
+            numeric = match[1]
+            suffix = match[2].downcase
+
+            # Zero-pad to 3 digits minimum
+            padded = format("%03d", numeric.to_i)
+
+            "TC-#{padded}#{suffix}"
+          end
+
+          # Normalize multiple test case identifiers
+          #
+          # @param ids [Array<String>] Raw test case IDs
+          # @return [Array<String>] Normalized TC-NNN format IDs
+          def self.normalize_identifiers(ids)
+            ids.map { |id| normalize_identifier(id) }
+          end
+
+          # Parse a comma-separated string of test case IDs
+          #
+          # @param input [String] Comma-separated test case IDs (e.g., "tc-001,002,TC-3")
+          # @return [Array<String>] Normalized TC-NNN format IDs
+          # @raise [ArgumentError] If input is empty or contains invalid IDs
+          def self.parse(input)
+            raw = input.to_s.strip
+            raise ArgumentError, "Empty test cases input" if raw.empty?
+
+            ids = raw.split(",").map(&:strip).reject(&:empty?)
+            raise ArgumentError, "No valid test case IDs found in: '#{input}'" if ids.empty?
+
+            normalize_identifiers(ids)
+          end
+
+          # Extract available test case IDs from markdown content
+          #
+          # Scans for ### TC-NNN: headers in the test scenario markdown.
+          #
+          # @param content [String] Markdown content of a test scenario
+          # @return [Array<String>] List of test case IDs found (e.g., ["TC-001", "TC-002"])
+          def self.extract_from_content(content)
+            content.scan(TC_HEADER_PATTERN).map { |match| match[0].upcase }
+          end
+
+          # Filter test case content by ID list
+          #
+          # Given a list of desired test case IDs and the available IDs in content,
+          # validates that all requested IDs exist and returns the validated set.
+          #
+          # @param requested_ids [Array<String>] Normalized test case IDs to filter
+          # @param available_ids [Array<String>] Test case IDs available in the scenario
+          # @return [Array<String>] Validated test case IDs
+          # @raise [ArgumentError] If any requested IDs are not found in the scenario
+          def self.validate_against_available(requested_ids, available_ids)
+            normalized_available = available_ids.map(&:upcase)
+            missing = requested_ids.reject { |id| normalized_available.include?(id.upcase) }
+
+            unless missing.empty?
+              raise ArgumentError,
+                "Test case(s) not found: #{missing.join(", ")}. " \
+                "Available: #{available_ids.join(", ")}"
+            end
+
+            requested_ids
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ace/test/end_to_end_runner/cli/commands/run_suite.rb

@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+require "ace/support/cli"
+require "stringio"
+require "ace/support/cli"
+
+module Ace
+  module Test
+    module EndToEndRunner
+      module CLI
+        module Commands
+          # CLI command for running E2E test suite across all packages
+          #
+          # Discovers all E2E tests in the monorepo and executes them
+          # with optional parallel execution and affected package filtering.
+          class RunSuite < Ace::Support::Cli::Command
+            include Ace::Support::Cli::Base
+
+            desc <<~DESC.strip
+              Run E2E test suite across all packages
+
+              Discovers and executes TS-* test scenarios from all packages
+              in the monorepo. Tests run sequentially by default or in parallel
+              with --parallel flag. Use --affected to only test changed packages.
+              Use --only-failures to re-run only previously failed scenarios.
+              Optionally filter to specific packages with a comma-separated list.
+
+              Output:
+                Exit codes: 0 (all pass), 1 (any fail/error)
+            DESC
+
+            argument :packages, required: false,
+              desc: "Comma-separated package names (e.g., ace-bundle,ace-lint)"
+
+            example [
+              "                              # Run all tests sequentially",
+              "ace-bundle,ace-lint           # Run only specified packages",
+              "--parallel 4                  # Run with 4 parallel workers",
+              "--affected                    # Only test changed packages",
+              "--affected --parallel 8       # Parallel affected tests only",
+              "--only-failures               # Re-run failed scenarios from cache",
+              "--affected --only-failures    # Re-run failed scenarios in affected packages",
+              "--tags smoke,happy-path       # Include scenarios by tag",
+              "--exclude-tags deep           # Exclude scenarios by tag",
+              "--cli-args dangerously-skip-permissions  # Pass args to provider"
+            ]
+
+            option :parallel, type: :string, default: Molecules::ConfigLoader.default_parallel.to_s,
+              desc: "Number of parallel workers (0 = sequential)"
+            option :affected, type: :boolean, desc: "Only test affected packages"
+            option :only_failures, type: :boolean,
+              desc: "Re-run only previously failed scenarios"
+            option :cli_args, type: :string,
+              desc: "Extra args for CLI-based LLM providers"
+            option :provider, type: :string, default: Molecules::ConfigLoader.default_provider,
+              desc: "LLM provider:model (e.g., claude:sonnet, gemini:flash)"
+            option :timeout, type: :string, default: Molecules::ConfigLoader.default_timeout.to_s,
+              desc: "Timeout per test in seconds"
+            option :tags, type: :string, desc: "Comma-separated scenario tags to include"
+            option :exclude_tags, type: :string, desc: "Comma-separated scenario tags to exclude"
+            option :progress, type: :boolean, desc: "Enable live animated display"
+            option :verify, type: :boolean,
+              desc: "Run independent verifier pass for each scenario"
+            option :quiet, type: :boolean, aliases: %w[-q], desc: "Suppress non-essential output"
+            option :verbose, type: :boolean, aliases: %w[-v], desc: "Show verbose output"
+            option :debug, type: :boolean, aliases: %w[-d], desc: "Show debug output"
+
+            def call(packages: nil, **options)
+              options = coerce_types(options, parallel: :integer, timeout: :integer)
+
+              parallel = options[:parallel]
+              affected = options[:affected]
+              only_failures = options[:only_failures]
+              tags = parse_csv_list(options[:tags])
+              exclude_tags = parse_csv_list(options[:exclude_tags])
+
+              output = quiet?(options) ? StringIO.new : $stdout
+              progress = options[:progress] && !quiet?(options)
+
+              orchestrator = Organisms::SuiteOrchestrator.new(
+                max_parallel: [parallel, 1].max,
+                output: output,
+                progress: progress
+              )
+
+              results = orchestrator.run(
+                parallel: parallel > 0,
+                affected: affected,
+                only_failures: only_failures,
+                packages: packages,
+                cli_args: options[:cli_args],
+                provider: options[:provider],
+                timeout: options[:timeout],
+                tags: tags,
+                exclude_tags: exclude_tags,
+                verify: options[:verify]
+              )
+
+              if results[:total].zero?
+                if only_failures
+                  raise Ace::Support::Cli::Error.new(
+                    "No failed test scenarios found in cache"
+                  )
+                else
+                  raise Ace::Support::Cli::Error.new("No tests found to run")
+                end
+              end
+
+              # Exit with error if any test failed
+              if results[:failed] > 0 || results[:errors] > 0
+                failed_count = results[:failed] + results[:errors]
+                raise Ace::Support::Cli::Error.new(
+                  "#{failed_count} test(s) failed or errored"
+                )
+              end
+            end
+
+            private
+
+            def parse_csv_list(raw)
+              return [] if raw.nil? || raw.strip.empty?
+
+              raw.split(",").map(&:strip).reject(&:empty?).map(&:downcase)
+            end
+          end
+        end
+      end
+    end
+  end
+end