ace-test-runner-e2e 0.29.0

data/lib/ace/test/end_to_end_runner/atoms/prompt_builder.rb

@@ -0,0 +1,199 @@
+# frozen_string_literal: true
+
+module Ace
+  module Test
+    module EndToEndRunner
+      module Atoms
+        # Builds LLM prompts for E2E test execution
+        #
+        # Creates a system prompt that instructs the LLM to execute a test scenario
+        # and return structured JSON results, along with the user prompt containing
+        # the test scenario content.
+        class PromptBuilder
+          # System prompt for TC-level (single test case) execution
+          TC_SYSTEM_PROMPT = <<~PROMPT
+            You are an E2E test executor for the ACE (Agentic Coding Environment) toolkit.
+
+            Your task is to execute a single test case in a pre-populated sandbox and return structured results.
+
+            ## Instructions
+
+            1. The test sandbox is pre-populated at the path provided — do NOT create or modify the sandbox setup
+            2. Read the test case steps carefully
+            3. Execute the test case steps in the sandbox
+            4. Record pass/fail status
+            5. Return results as JSON
+
+            ## Output Format
+
+            You MUST return a JSON block wrapped in ```json fences with these fields:
+
+            ```json
+            {
+              "test_id": "TS-XXX-NNN",
+              "tc_id": "TC-NNN",
+              "status": "pass|fail",
+              "actual": "What actually happened",
+              "notes": "Any additional observations",
+              "summary": "Brief result"
+            }
+            ```
+
+            ## Rules
+
+            - Execute ONLY the single test case provided
+            - Execute in the pre-populated sandbox (do not modify setup files)
+            - Record actual output/behavior, not just expected
+            - If the test case cannot be executed (missing tool, permission error), mark as "fail" with explanation
+          PROMPT
+
+          SYSTEM_PROMPT = <<~PROMPT
+            You are an E2E test executor for the ACE (Agentic Coding Environment) toolkit.
+
+            Your task is to execute the provided test scenario step by step and return structured results.
+
+            ## Instructions
+
+            1. Read the test scenario carefully
+            2. Execute the Environment Setup commands
+            3. Create any Test Data as specified
+            4. Execute each Test Case (TC-NNN) in order
+            5. Record pass/fail status for each test case
+            6. Return results as JSON
+
+            ## Output Format
+
+            You MUST return a JSON block wrapped in ```json fences with these fields:
+
+            ```json
+            {
+              "test_id": "TS-XXX-NNN",
+              "status": "pass|fail|partial",
+              "test_cases": [
+                {
+                  "id": "TC-001",
+                  "description": "Brief description",
+                  "status": "pass|fail",
+                  "actual": "What actually happened",
+                  "notes": "Any additional observations"
+                }
+              ],
+              "summary": "Brief execution summary",
+              "observations": "Any friction points or issues discovered"
+            }
+            ```
+
+            ## Rules
+
+            - Execute ALL test cases, even if earlier ones fail
+            - Record actual output/behavior, not just expected
+            - Use "partial" status if some test cases pass and some fail
+            - Include meaningful observations about tool behavior
+            - If a test case cannot be executed (missing tool, permission error), mark as "fail" with explanation
+          PROMPT
+
+          # Build a TC-level user prompt for a single test case
+          #
+          # @param test_case [Models::TestCase] The single test case to execute
+          # @param scenario [Models::TestScenario] The parent scenario for metadata
+          # @param sandbox_path [String] Path to the pre-populated sandbox
+          # @return [String] The TC-level user prompt
+          def build_tc(test_case:, scenario:, sandbox_path:)
+            if test_case.pending?
+              return <<~PROMPT
+                # SKIP Test Case: #{scenario.test_id} / #{test_case.tc_id}
+
+                **Package:** #{scenario.package}
+                **Scenario:** #{scenario.title}
+                **Test Case:** #{test_case.title}
+                **Status:** PENDING — #{test_case.pending}
+
+                This test case is marked as pending and should NOT be executed.
+                Return the following JSON result:
+
+                ```json
+                {
+                  "test_id": "#{scenario.test_id}",
+                  "tc_id": "#{test_case.tc_id}",
+                  "status": "skip",
+                  "actual": "Skipped — pending",
+                  "notes": "#{test_case.pending}",
+                  "summary": "Pending: #{test_case.pending}"
+                }
+                ```
+              PROMPT
+            end
+
+            <<~PROMPT
+              # Execute Test Case: #{scenario.test_id} / #{test_case.tc_id}
+
+              **Package:** #{scenario.package}
+              **Scenario:** #{scenario.title}
+              **Test Case:** #{test_case.title}
+              **Sandbox Path:** #{sandbox_path}
+
+              ## Test Case Content
+
+              #{test_case.content}
+
+              ---
+
+              Execute the test case steps in the sandbox at `#{sandbox_path}` and return JSON results as specified in your instructions.
+            PROMPT
+          end
+
+          # Build the user prompt for a test scenario
+          #
+          # @param scenario [Models::TestScenario] The test scenario to execute
+          # @param test_cases [Array<String>, nil] Optional test case IDs to filter
+          # @return [String] The user prompt containing the test scenario
+          def build(scenario, test_cases: nil)
+            filter_instruction = if test_cases&.any?
+              "\n**IMPORTANT:** Execute ONLY the following test cases: #{test_cases.join(", ")}. Skip all other test cases.\n"
+            else
+              ""
+            end
+
+            pending_instruction = build_pending_tc_skip_instruction(scenario)
+
+            execute_instruction = if test_cases&.any?
+              "Execute only the specified test cases (#{test_cases.join(", ")}) and return the JSON results as specified in your instructions."
+            else
+              "Execute all test cases in this scenario and return the JSON results as specified in your instructions."
+            end
+
+            <<~PROMPT
+              # Execute E2E Test: #{scenario.test_id}
+
+              **Package:** #{scenario.package}
+              **Title:** #{scenario.title}
+              **Priority:** #{scenario.priority}
+              #{filter_instruction}#{pending_instruction}
+              ## Test Scenario
+
+              #{scenario.content}
+
+              ---
+
+              #{execute_instruction}
+            PROMPT
+          end
+
+          private
+
+          # Build instruction for pending test cases if any exist
+          #
+          # @param scenario [Models::TestScenario] The test scenario
+          # @return [String] Pending instruction text or empty string
+          def build_pending_tc_skip_instruction(scenario)
+            pending_tcs = scenario.test_cases.select(&:pending?)
+            return "" unless pending_tcs.any?
+
+            lines = pending_tcs.map { |tc| "- #{tc.tc_id}: #{tc.pending}" }
+            "\n**SKIP these test cases (pending):**\n#{lines.join("\n")}\nFor skipped test cases, report status as \"skip\" in your results.\n"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ace/test/end_to_end_runner/atoms/result_parser.rb

@@ -0,0 +1,166 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Ace
+  module Test
+    module EndToEndRunner
+      module Atoms
+        # Parses structured JSON results from LLM responses
+        #
+        # Extracts JSON from LLM text output, handling various formatting
+        # patterns including fenced code blocks and raw JSON.
+        class ResultParser
+          # Parse LLM response text into a structured result hash
+          #
+          # @param text [String] Raw LLM response text
+          # @return [Hash] Parsed result with :test_id, :status, :test_cases, :summary
+          # @raise [ParseError] If no valid JSON found in response
+          def self.parse(text)
+            json_str = extract_json(text)
+            raise ParseError, "No JSON found in LLM response" if json_str.nil?
+
+            parsed = JSON.parse(json_str, symbolize_names: true)
+            validate_result(parsed)
+            normalize_result(parsed)
+          rescue JSON::ParserError => e
+            raise ParseError, "Invalid JSON in LLM response: #{e.message}"
+          end
+
+          # Extract JSON from text, handling code fences and raw JSON
+          #
+          # @param text [String] Text potentially containing JSON
+          # @return [String, nil] Extracted JSON string or nil
+          def self.extract_json(text)
+            return nil if text.nil? || text.to_s.strip.empty?
+
+            stripped = text.to_s.strip
+
+            # Try to find JSON in code fences first
+            match = stripped.match(/```(?:json)?\s*\n(.*?)\n\s*```/m)
+            return match[1].strip if match
+
+            # Treat unfenced content as JSON only when the whole payload is a JSON object.
+            return stripped if stripped.start_with?("{") && stripped.end_with?("}")
+
+            nil
+          end
+
+          # Validate that parsed result has required fields
+          #
+          # @param result [Hash] Parsed JSON result
+          # @raise [ParseError] If required fields are missing
+          def self.validate_result(result)
+            required = %i[test_id status]
+            missing = required.reject { |field| result.key?(field) }
+            unless missing.empty?
+              raise ParseError, "Missing required fields in result: #{missing.join(", ")}"
+            end
+          end
+
+          # Normalize result to ensure consistent structure
+          #
+          # @param result [Hash] Parsed result
+          # @return [Hash] Normalized result
+          def self.normalize_result(result)
+            {
+              test_id: result[:test_id],
+              status: result[:status].to_s.downcase,
+              test_cases: normalize_test_cases(result[:test_cases] || []),
+              summary: result[:summary] || "",
+              observations: result[:observations] || ""
+            }
+          end
+
+          # Normalize test case entries
+          #
+          # @param test_cases [Array<Hash>] Raw test case data
+          # @return [Array<Hash>] Normalized test cases
+          def self.normalize_test_cases(test_cases)
+            test_cases.map do |tc|
+              {
+                id: tc[:id] || "unknown",
+                description: tc[:description] || "",
+                status: tc[:status] || "fail",
+                actual: tc[:actual] || "",
+                notes: tc[:notes] || "",
+                criteria: normalize_criteria(tc[:criteria] || [])
+              }
+            end
+          end
+
+          # Normalize optional criteria evaluations
+          #
+          # @param criteria [Array<Hash>] Raw criteria results
+          # @return [Array<Hash>] Normalized criteria entries
+          def self.normalize_criteria(criteria)
+            criteria.map do |criterion|
+              {
+                id: criterion[:id] || "",
+                description: criterion[:description] || criterion[:criterion] || "",
+                status: (criterion[:status] || "fail").to_s.downcase,
+                evidence: criterion[:evidence] || ""
+              }
+            end
+          end
+
+          # Parse TC-level LLM response into a structured result hash
+          #
+          # Handles single-TC JSON format with tc_id field. Falls back to
+          # parse() if the response contains multi-TC format.
+          #
+          # @param text [String] Raw LLM response text
+          # @return [Hash] Parsed result with single-entry :test_cases array
+          # @raise [ParseError] If no valid JSON found in response
+          def self.parse_tc(text)
+            json_str = extract_json(text)
+            raise ParseError, "No JSON found in LLM response" if json_str.nil?
+
+            parsed = JSON.parse(json_str, symbolize_names: true)
+
+            # If response has test_cases array, delegate to standard parse
+            return parse(text) if parsed.key?(:test_cases)
+
+            validate_tc_result(parsed)
+            normalize_tc_result(parsed)
+          rescue JSON::ParserError => e
+            raise ParseError, "Invalid JSON in LLM response: #{e.message}"
+          end
+
+          # Validate TC-level result fields
+          def self.validate_tc_result(result)
+            required = %i[test_id tc_id status]
+            missing = required.reject { |field| result.key?(field) }
+            unless missing.empty?
+              raise ParseError, "Missing required fields in TC result: #{missing.join(", ")}"
+            end
+          end
+
+          # Normalize TC-level result to standard format with single-entry test_cases
+          def self.normalize_tc_result(result)
+            {
+              test_id: result[:test_id],
+              status: result[:status],
+              test_cases: [{
+                id: result[:tc_id],
+                description: result[:summary] || "",
+                status: result[:status],
+                actual: result[:actual] || "",
+                notes: result[:notes] || "",
+                criteria: normalize_criteria(result[:criteria] || [])
+              }],
+              summary: result[:summary] || "",
+              observations: result[:notes] || ""
+            }
+          end
+
+          private_class_method :validate_result, :normalize_result, :normalize_test_cases,
+            :normalize_criteria, :validate_tc_result, :normalize_tc_result
+
+          # Error raised when parsing LLM response fails
+          class ParseError < StandardError; end
+        end
+      end
+    end
+  end
+end

data/lib/ace/test/end_to_end_runner/atoms/skill_prompt_builder.rb

@@ -0,0 +1,166 @@
+# frozen_string_literal: true
+
+module Ace
+  module Test
+    module EndToEndRunner
+      module Atoms
+        # Holds CLI-provider detection and CLI-args helpers.
+        #
+        # Standalone scenario execution for CLI providers now runs through the
+        # deterministic runner/verifier pipeline.
+        # Provider lists and CLI args are configurable via config.yml.
+        class CliProviderAdapter
+          # @param config [Hash] Configuration hash (string keys) with providers section
+          def initialize(config = {})
+            @cli_providers = config.dig("providers", "cli") || %w[claude gemini codex codexoss opencode pi]
+          end
+
+          # Check if a provider string refers to a CLI provider
+          #
+          # @param provider_string [String] Provider:model string (e.g., "claude:sonnet")
+          # @return [Boolean]
+          def self.cli_provider?(provider_string)
+            default_instance.cli_provider?(provider_string)
+          end
+
+          # Extract provider name from provider:model string
+          #
+          # @param provider_string [String] e.g., "claude:sonnet"
+          # @return [String] e.g., "claude"
+          def self.provider_name(provider_string)
+            provider_string.to_s.split(":").first.to_s
+          end
+
+          # Instance method: check if a provider string refers to a CLI provider
+          #
+          # @param provider_string [String] Provider:model string
+          # @return [Boolean]
+          def cli_provider?(provider_string)
+            name = self.class.provider_name(provider_string)
+            @cli_providers.include?(name)
+          end
+
+          def build_execution_prompt(command:, tc_mode:)
+            return_contract = if tc_mode
+              "- **Test ID**: ...\n- **TC ID**: ...\n- **Status**: pass | fail\n- **Report Paths**: ...\n- **Issues**: ..."
+            else
+              "- **Test ID**: ...\n- **Status**: pass | fail | partial\n- **Passed**: ...\n- **Failed**: ...\n- **Total**: ...\n- **Report Paths**: ...\n- **Issues**: ..."
+            end
+
+            <<~PROMPT.strip
+              Run this as a slash command in the agent chat interface (not in bash):
+              #{command}
+
+              Execution requirements:
+              - Do not run `/ace-...` inside a shell command.
+              - If slash commands are unavailable, stop and report that limitation in `Issues`.
+              - Write reports under `.ace-local/test-e2e/*-reports/`.
+              - Return only this structured summary:
+              #{return_contract}
+            PROMPT
+          end
+
+          public
+
+          # Build a skill invocation prompt for scenario-level execution
+          #
+          # @param scenario [Models::TestScenario] The test scenario
+          # @param run_id [String, nil] Pre-generated run ID for deterministic report paths
+          # @param test_cases [Array<String>, nil] Optional test case IDs to filter
+          # @param sandbox_path [String, nil] Path to pre-populated sandbox (skips setup steps)
+          # @param env_vars [Hash, nil] Environment variables from setup execution
+          # @param report_dir [String, nil] Explicit report directory path (overrides computed path)
+          # @return [String] Skill invocation prompt
+          def build_skill_prompt(scenario, run_id: nil, test_cases: nil, sandbox_path: nil, env_vars: nil, report_dir: nil)
+            cmd = "/as-e2e-run #{scenario.package} #{scenario.test_id}"
+            cmd += " #{test_cases.join(",")}" if test_cases&.any?
+            cmd += " --run-id #{run_id}" if run_id
+            cmd += " --sandbox #{sandbox_path}" if sandbox_path
+            cmd += " --env #{env_vars.map { |k, v| "#{k}=#{v}" }.join(",")}" if env_vars&.any?
+            cmd += " --report-dir #{report_dir}" if report_dir
+            build_execution_prompt(command: cmd, tc_mode: false)
+          end
+
+          # Build a TC-level skill invocation prompt
+          #
+          # @param test_case [Models::TestCase] The single test case
+          # @param scenario [Models::TestScenario] The parent scenario
+          # @param sandbox_path [String] Path to the pre-populated sandbox
+          # @param run_id [String, nil] Pre-generated run ID
+          # @param env_vars [Hash, nil] Environment variables from setup execution
+          # @return [String] Skill invocation prompt
+          def build_tc_skill_prompt(test_case:, scenario:, sandbox_path:, run_id: nil, env_vars: nil)
+            cmd = "/as-e2e-run #{scenario.package} #{scenario.test_id} #{test_case.tc_id} --tc-mode --sandbox #{sandbox_path}"
+            cmd += " --run-id #{run_id}" if run_id
+            cmd += " --env #{env_vars.map { |k, v| "#{k}=#{v}" }.join(",")}" if env_vars&.any?
+            build_execution_prompt(command: cmd, tc_mode: true)
+          end
+
+          # Build an independent verifier prompt.
+          #
+          # This is intentionally a second invocation to avoid sharing runner context.
+          def build_verifier_prompt(scenario, run_id: nil, sandbox_path: nil, test_cases: nil, report_dir: nil)
+            report_dir ||= if run_id
+              ".ace-local/test-e2e/#{scenario.dir_name(run_id)}-reports"
+            end
+
+            tc_filter = test_cases&.any? ? test_cases.join(", ") : "all discovered test cases"
+            sandbox_info = sandbox_path || "(unknown)"
+            report_info = report_dir || "(unknown)"
+
+            <<~PROMPT.strip
+              You are the independent verifier for an E2E scenario.
+
+              Verify this scenario in a new, isolated agent context:
+              - Package: #{scenario.package}
+              - Test ID: #{scenario.test_id}
+              - Sandbox path: #{sandbox_info}
+              - Report directory: #{report_info}
+              - Scope: #{tc_filter}
+
+              Verification requirements:
+              - Inspect sandbox artifacts and scenario files directly.
+              - Evaluate each test case using `TC-*.verify.md` criteria when present.
+              - Classify each failed test case with one category:
+                `test-spec-error`, `tool-bug`, `runner-error`, or `infrastructure-error`.
+              - Write/update report files under the report directory.
+              - Use TC-first schema in report frontmatter and metadata.
+
+              Return only this structured summary:
+              - **Test ID**: ...
+              - **Status**: pass | fail | partial | error
+              - **TCs Passed**: ...
+              - **TCs Failed**: ...
+              - **TCs Total**: ...
+              - **Score**: ...
+              - **Verdict**: pass | partial | fail
+              - **Failed TCs**: TC-001:tool-bug, TC-002:runner-error (or `None`)
+              - **Issues**: ...
+            PROMPT
+          end
+
+          # Lazily-loaded default instance backed by ConfigLoader
+          # @return [CliProviderAdapter]
+          def self.default_instance
+            @default_instance ||= begin
+              config = if defined?(Molecules::ConfigLoader)
+                Molecules::ConfigLoader.load
+              else
+                {}
+              end
+              new(config)
+            end
+          end
+
+          # Reset the default instance (for testing)
+          def self.reset_default_instance!
+            @default_instance = nil
+          end
+        end
+
+        # Backward-compatible alias while callers migrate off the legacy name.
+        SkillPromptBuilder = CliProviderAdapter
+      end
+    end
+  end
+end