ace-test-runner 0.27.1 → 0.28.1

data/lib/ace/test_runner/molecules/smart_test_executor.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative "test_executor"
+require_relative "selection_resolver"
 require_relative "in_process_runner"
 require_relative "../atoms/test_type_detector"
 
@@ -20,6 +21,7 @@ module Ace
           return empty_result if files.empty?
 
           # Determine execution mode
+          options = options.merge(selection_plan: options[:selection_plan] || SelectionResolver.resolve(files))
           mode = determine_execution_mode(files, options)
 
           # Execute with appropriate runner
@@ -31,6 +33,7 @@ module Ace
         end
 
         def execute_single_file(file, options = {})
+          options = options.merge(selection_plan: options[:selection_plan] || SelectionResolver.resolve([file]))
           mode = determine_execution_mode([file], options)
 
           if mode == :direct
@@ -46,6 +49,7 @@ module Ace
         end
 
         def execute_with_progress(files, options = {}, &block)
+          options = options.merge(selection_plan: options[:selection_plan] || SelectionResolver.resolve(files))
           mode = determine_execution_mode(files, options)
 
           # Add mode information to progress callback

data/lib/ace/test_runner/molecules/test_executor.rb

@@ -3,6 +3,7 @@
 require "open3"
 require "timeout"
 require "ostruct"
+require_relative "selection_resolver"
 
 module Ace
   module TestRunner
@@ -39,15 +40,15 @@ module Ace
           # environment (Process.spawn merges onto the parent by default).
           # MT_NO_AUTORUN is part of the fixture environment.
           env = @launch_env
-          command = command.sub(/^MT_NO_AUTORUN=1\s+/, "")
+          command = command.sub(/^MT_NO_AUTORUN=1\s+/, "") if command.is_a?(String)
 
           begin
             if @timeout
               Timeout.timeout(@timeout) do
-                stdout, stderr, status = Open3.capture3(env, command, unsetenv_others: true)
+                stdout, stderr, status = Open3.capture3(env, *Array(command), unsetenv_others: true)
               end
             else
-              stdout, stderr, status = Open3.capture3(env, command, unsetenv_others: true)
+              stdout, stderr, status = Open3.capture3(env, *Array(command), unsetenv_others: true)
             end
           rescue Timeout::Error
             stderr = "Test execution timed out after #{@timeout} seconds"
@@ -69,6 +70,7 @@ module Ace
         end
 
         def execute_with_progress(files, options = {}, &block)
+          options = options.merge(selection_plan: options[:selection_plan] || SelectionResolver.resolve(files))
           # Fail-fast requires per-file execution to stop on first failure
           # For performance, execute all files together unless explicitly disabled or fail-fast enabled
           if options[:per_file] == true || options[:fail_fast]
@@ -93,12 +95,16 @@ module Ace
         end
 
         def execute_per_file_with_progress(files, options = {}, &block)
+          options = options.merge(selection_plan: options[:selection_plan] || SelectionResolver.resolve(files))
           results = []
+          plan = options.fetch(:selection_plan)
+          files = plan.files if plan.qualified?
 
           files.each do |file|
             yield({type: :start, file: file}) if block_given?
 
-            result = execute_single_file(file, options)
+            file_options = plan.qualified? ? options.merge(selection_plan: plan.for_file(file)) : options
+            result = execute_single_file(file, file_options)
             results << result
 
             if block_given?
@@ -139,6 +145,7 @@ module Ace
             stdout: results.map { |r| r[:stdout] }.join("\n"),
             stderr: results.map { |r| r[:stderr] }.join("\n"),
             commands: results.map { |r| r[:command] },
+            execution_outputs: results.map { |r| r[:stdout] },
             start_time: results.first[:start_time],
             end_time: results.last[:end_time],
             duration: results.sum { |r| r[:duration] },

data/lib/ace/test_runner/organisms/sequential_target_executor.rb

@@ -37,7 +37,7 @@ module Ace
             # Parse result
             parsed = if result[:commands] && result[:commands].is_a?(Array)
               # Multiple commands executed (per-file)
-              aggregate_results(result[:stdout], @result_parser)
+              aggregate_results(result.fetch(:execution_outputs), @result_parser)
             else
               @result_parser.parse_output(result[:stdout])
             end
@@ -81,10 +81,8 @@ module Ace
 
         private
 
-        def aggregate_results(combined_output, parser)
-          # Split output by test file executions
-          individual_outputs = combined_output.split(/^Started with run options/)
-          individual_outputs.shift if individual_outputs.first && individual_outputs.first.empty?
+        def aggregate_results(individual_outputs, parser)
+          # Preserve exact executor boundaries, including headerless partial output.
 
           aggregated = {
             summary: {
@@ -102,7 +100,6 @@ module Ace
           }
 
           individual_outputs.each do |output|
-            output = "Started with run options" + output
             parsed = parser.parse_output(output)
 
             aggregated[:summary][:runs] += parsed[:summary][:runs]
@@ -160,7 +157,8 @@ module Ace
           combined_stderr = all_results.map { |r| r[:stderr] }.join("\n")
 
           # Determine success
-          success = total_summary[:failures] == 0 && total_summary[:errors] == 0
+          success = all_results.all? { |result| result[:success] == true } &&
+            total_summary[:failures] == 0 && total_summary[:errors] == 0
 
           {
             stdout: combined_stdout,
@@ -175,6 +173,7 @@ module Ace
               duration: total_duration,
               test_times: all_test_times
             },
+            files: all_files.uniq,
             stopped_at_target: stopped
           }
         end

data/lib/ace/test_runner/organisms/test_orchestrator.rb

@@ -60,6 +60,7 @@ module Ace
           validate_configuration!
 
           start_time = Time.now
+          @execution_evidence = Molecules::ExecutionEvidence.for_invocation(Dir.pwd)
 
           # Print package context if running in different directory
           if @package_dir
@@ -82,6 +83,7 @@ module Ace
           end
 
           resolve_and_set_report_dir_context(test_files)
+          reserve_execution_report
 
           # Count total available test files (before filtering)
           total_available = count_total_test_files
@@ -104,22 +106,18 @@ module Ace
                 runs: 0,
                 assertions: 0,
                 failures: 0,
-                errors: test_files.size,  # Count all test files as errors
+                errors: 0,  # Execution failure is distinct from completed test errors
                 skips: 0,
                 passed: 0
               },
               failures: [],
-              errors: [{
-                message: execution_result[:stderr].strip,
-                type: "LoadError",
-                files: test_files
-              }],
+              errors: [],
               deprecations: [],
               duration: execution_result[:duration]
             }
           elsif execution_result[:commands] && execution_result[:commands].is_a?(Array)
             # Each file was executed separately, parse and sum them all
-            aggregate_individual_results(execution_result[:stdout])
+            aggregate_individual_results(execution_result.fetch(:execution_outputs))
           else
             # Single command execution (by-target)
             @result_parser.parse_output(execution_result[:stdout])
@@ -127,6 +125,7 @@ module Ace
 
           # Build result object
           @result = build_result(@parsed_result, execution_result, start_time)
+          validate_selected_execution
 
           # Analyze failures and errors
           if @result.has_failures?
@@ -165,6 +164,9 @@ module Ace
             @formatter.report_path = report_path if @formatter.respond_to?(:report_path=)
           end
 
+          # Publish completion only after all optional report files are written.
+          @execution_evidence.publish(@result, files: report.files_tested, report_dir: @execution_report_dir)
+
           # Output to stdout
           @formatter.on_finish(@result)
 
@@ -231,6 +233,7 @@ module Ace
           all_files = targets.flat_map { |t| t[:files] }
 
           resolve_and_set_report_dir_context(all_files)
+          reserve_execution_report
 
           total_available = count_total_test_files
 
@@ -262,11 +265,15 @@ module Ace
             end
           end
 
+          # Only dispatched targets belong in the execution report after an early stop.
+          all_files = execution_result.fetch(:files)
+
           # Use the parsed result from sequential executor
           @parsed_result = execution_result[:parsed_result]
 
           # Build result object
           @result = build_result(@parsed_result, execution_result, start_time)
+          validate_selected_execution
 
           # Analyze failures
           if @result.has_failures?
@@ -301,6 +308,9 @@ module Ace
             @formatter.report_path = report_path if @formatter.respond_to?(:report_path=)
           end
 
+          # Publish completion only after all optional report files are written.
+          @execution_evidence.publish(@result, files: report.files_tested, report_dir: @execution_report_dir)
+
           # Output to stdout
           @formatter.on_finish(@result)
 
@@ -463,6 +473,10 @@ module Ace
 
         def handle_no_tests
           @result = Models::TestResult.new
+          resolve_and_set_report_dir_context([])
+          reserve_execution_report
+          save_reports(@report_generator.generate(@result, [])) if @configuration.save_reports
+          @execution_evidence.publish(@result, files: [], report_dir: @execution_report_dir)
 
           message = if @configuration.filter
             "No test files found matching pattern '#{@configuration.filter}'"
@@ -471,6 +485,7 @@ module Ace
           end
 
           puts message
+          puts(@execution_report_dir ? "Details: #{@execution_report_dir}/" : "No saved report")
           0
         end
 
@@ -507,6 +522,13 @@ module Ace
           end
         end
 
+        def validate_selected_execution
+          return unless @result.success? && @result.total_tests == 0
+
+          @result.execution_success = false
+          @result.stderr = [@result.stderr, "Selected test files produced no executed tests"].reject(&:empty?).join("\n")
+        end
+
         def build_result(parsed_result, execution_result, start_time)
           Models::TestResult.new(
             passed: parsed_result[:summary][:passed],
@@ -519,17 +541,16 @@ module Ace
             end_time: Time.now,
             deprecations: parsed_result[:deprecations],
             raw_output: execution_result[:stdout],
-            stderr: execution_result[:stderr]
+            stderr: execution_result[:stderr],
+            execution_success: execution_result[:success] == true
           )
         end
 
-        def aggregate_individual_results(combined_output)
-          # Split output by test file executions
-          individual_outputs = combined_output.split(/^Started with run options/)
-          individual_outputs.shift if individual_outputs.first && individual_outputs.first.empty?
+        def aggregate_individual_results(individual_outputs)
+          # Process boundaries come from the executor, never text inside stdout.
 
           aggregated = {
-            raw_output: combined_output,
+            raw_output: individual_outputs.join("\n"),
             summary: {
               runs: 0,
               assertions: 0,
@@ -545,7 +566,6 @@ module Ace
           }
 
           individual_outputs.each do |output|
-            output = "Started with run options" + output  # Restore the split text
             parsed = @result_parser.parse_output(output)
 
             # Sum up the counts
@@ -602,19 +622,22 @@ module Ace
           )
         end
 
+        def reserve_execution_report
+          return unless @configuration.save_reports
+
+          @report_storage = Molecules::ReportStorage.new(base_dir: @configuration.report_dir)
+          @execution_report_dir = @report_storage.reserve(@execution_evidence.identity)
+        end
+
         def save_reports(report)
-          timestamp_generator = Atoms::TimestampGenerator.new
-          storage = Molecules::ReportStorage.new(
-            base_dir: @configuration.report_dir,
-            timestamp_generator: timestamp_generator
-          )
+          storage = @report_storage
 
           # Save in appropriate format
           report_path = case @configuration.format
           when "json"
-            storage.save_report(report, format: :json)
+            storage.save_report(report, format: :json, report_dir: @execution_report_dir)
           else
-            storage.save_report(report, format: :all)
+            storage.save_report(report, format: :all, report_dir: @execution_report_dir)
           end
 
           # Always save raw output

data/lib/ace/test_runner/suite/display_manager.rb

@@ -33,9 +33,9 @@ module Ace
 
           # Reserve lines for each package
           @packages.each_with_index do |package, index|
-            @lines[package["name"]] = index + 5  # Account for header lines
-            @package_status[package["name"]] = {status: :waiting}
-            print_package_line(package["name"])
+            @lines[package["entry_id"]] = index + 5  # Account for header lines
+            @package_status[package["entry_id"]] = {status: :waiting}
+            print_package_line(package["entry_id"])
           end
 
           # Print footer space
@@ -45,8 +45,8 @@ module Ace
         end
 
         def update_package(package, status, output = nil)
-          @package_status[package["name"]] = status
-          print_package_line(package["name"])
+          @package_status[package["entry_id"]] = status
+          print_package_line(package["entry_id"])
           update_footer
         end
 
@@ -87,7 +87,7 @@ module Ace
           print "\033[K"  # Clear line
 
           # Format package name (fixed width, no brackets)
-          pkg_name = name.ljust(25)
+          pkg_name = @packages.find { |package| package["entry_id"] == name }.fetch("name").ljust(25)
 
           case status[:status]
           when :waiting
@@ -135,6 +135,7 @@ module Ace
             line_text = "#{icon}  #{elapsed}  #{pkg_name}  #{tests_col}  #{asserts_col}  #{fail_col}"
             line_text += "  #{skipped} skip" if skipped > 0
 
+            line_text += results[:report_dir] ? "  report: #{results[:report_dir]}" : "  no saved report"
             print line_text
           end
         end

data/lib/ace/test_runner/suite/orchestrator.rb

@@ -11,7 +11,9 @@ module Ace
 
         def initialize(config)
           @config = config
-          @packages = config.dig("test_suite", "packages") || []
+          @packages = (config.dig("test_suite", "packages") || []).map do |package|
+            package.merge("entry_id" => SecureRandom.uuid)
+          end
           @results = {}
           @running_processes = {}
           @completed_packages = []
@@ -59,7 +61,7 @@ module Ace
 
               if status[:completed]
                 @completed_packages << pkg
-                @results[pkg["name"]] = status
+                @results[pkg["entry_id"]] = status
               end
             end
           end
@@ -75,7 +77,7 @@ module Ace
           display_manager.show_final_results
 
           # Aggregate results
-          aggregator = ResultAggregator.new(@packages, report_root: report_root, runtime_results: @results)
+          aggregator = ResultAggregator.new(@packages, runtime_results: @results)
           summary = aggregator.aggregate
 
           display_manager.show_summary(summary)

data/lib/ace/test_runner/suite/process_monitor.rb

@@ -30,6 +30,10 @@ module Ace
             return
           end
 
+          package["entry_id"] ||= SecureRandom.uuid
+          evidence = Molecules::ExecutionEvidence.new(package_path: package["path"],
+            package: package["name"], entry_id: package["entry_id"])
+
           # Build command
           cmd = build_command(package, test_options)
 
@@ -45,12 +49,16 @@ module Ace
           fixture_environment.env[Models::EnvironmentPolicy::SUITE_CHANNEL_KEY] =
             @environment_policy.channel_payload
 
+          fixture_environment.env[Molecules::ExecutionEvidence::CHANNEL] = evidence.channel
+
           start_time = now
           stdin, stdout, stderr, thread = Open3.popen3(
             fixture_environment.env, *cmd, chdir: package["path"], pgroup: true, unsetenv_others: true
           )
 
-          @processes[package["name"]] = {
+          @processes[package["entry_id"]] = {
+            evidence: evidence,
+            save_reports: test_options.fetch("save_reports", true),
             package: package,
             thread: thread,
             stdout: stdout,
@@ -61,7 +69,6 @@ module Ace
             fixture_environment: fixture_environment,
             output: +"",
             stderr_output: +"",
-            report_root: test_options["report_dir"],
             test_count: 0,
             tests_run: 0,
             dots: +"",
@@ -114,9 +121,8 @@ module Ace
 
               # Final callback
               if callback
-                # Use results[:success] from summary.json if available, otherwise check exit code
-                # This ensures the package status matches what ace-test actually reported
-                success_status = (!results[:success].nil?) ? results[:success] : (exit_status == 0)
+                # Completion evidence and child outcome must both permit success.
+                success_status = results[:success] == true
 
                 callback.call(package, {
                   status: :completed,
@@ -174,11 +180,14 @@ module Ace
             rescue Timeout::Error, StandardError
               nil
             end
+          end
 
+          # Capture trustworthy partial counts and interruption outcome before cleanup.
+          check_processes
+          @processes.each_value do |process_info|
             close_streams(process_info)
             process_info[:fixture_environment]&.cleanup
           end
-
           @processes.clear
           @completed.clear
         end
@@ -213,7 +222,7 @@ module Ace
           cmd_parts << "--format" << format
 
           # Add other options
-          cmd_parts << "--no-save" unless options["save_reports"]
+          cmd_parts << "--no-save-reports" unless options.fetch("save_reports", true)
           cmd_parts << "--fail-fast" if options["fail_fast"]
           cmd_parts << "--no-color" unless options.fetch("color", true)
           if options["report_dir"]
@@ -304,98 +313,31 @@ module Ace
         end
 
         def build_results(process_info, elapsed, exit_status)
-          return timeout_results(process_info, elapsed) if process_info[:timeout_triggered]
-          return interrupted_results(elapsed) if process_info[:terminated_by] == :interrupt
-          fresh_summary = load_summary_results(process_info, elapsed)
-          return fresh_summary if exit_status == 0 && fresh_summary
-          return failed_process_results(process_info, elapsed, exit_status, fresh_summary) if exit_status != 0
-
-          parse_results(process_info[:output])
-        end
-
-        def timeout_results(process_info, elapsed)
-          {
-            tests: 0,
-            assertions: 0,
-            failures: 0,
-            errors: 1,
-            duration: elapsed,
-            success: false,
-            error: "Timed out after #{process_info[:timeout]} seconds"
-          }
-        end
-
-        def interrupted_results(elapsed)
-          {
-            tests: 0,
-            assertions: 0,
-            failures: 0,
-            errors: 1,
-            duration: elapsed,
-            success: false,
-            error: "Interrupted before completion"
-          }
-        end
-
-        def failed_process_results(process_info, elapsed, exit_status, fresh_summary)
-          return fresh_summary if fresh_summary && fresh_summary[:success] == false
-
-          parsed = parse_results([process_info[:output], process_info[:stderr_output]].join("\n"))
-          parsed[:duration] ||= elapsed
-          parsed[:success] = false
-          parsed[:error] ||= failure_message_for(process_info, exit_status)
-          parsed[:errors] = 1 if parsed[:failures].to_i == 0 && parsed[:errors].to_i == 0
-          parsed
+          snapshot = begin
+            process_info[:evidence].read(pid: process_info[:pid], save_reports: process_info[:save_reports])
+          rescue Ace::TestRunner::Error => e
+            {total: 0, passed: 0, failed: 0, errors: 1, skipped: 0, assertions: 0,
+             duration: elapsed, success: false, error: e.message}
+          end
+          results = snapshot.merge(tests: snapshot[:total], failures: snapshot[:failed],
+            execution_id: process_info[:evidence].identity, completion_path: process_info[:evidence].path)
+          reason = if process_info[:timeout_triggered]
+            "Timed out after #{process_info[:timeout]} seconds"
+          elsif process_info[:terminated_by] == :interrupt
+            "Interrupted before completion"
+          elsif exit_status != 0
+            failure_message_for(process_info, exit_status)
+          end
+          if reason
+            results[:success] = false
+            results[:error] = reason
+          end
+          results.freeze
         end
 
         def failure_message_for(process_info, exit_status)
           stderr = process_info[:stderr_output].to_s.strip
-          stdout = process_info[:output].to_s.strip
-          message = stderr.empty? ? stdout : stderr
-          return message unless message.empty?
-
-          "ace-test exited with status #{exit_status}"
-        end
-
-        def load_summary_results(process_info, elapsed)
-          package = process_info[:package]
-          reports_dir = Atoms::ReportPathResolver.report_directory(
-            package["path"],
-            report_root: process_info[:report_root],
-            package_name: package["name"]
-          )
-          summary_file = reports_dir ? File.join(reports_dir, "summary.json") : nil
-          return nil unless summary_file && File.exist?(summary_file)
-          return nil unless summary_fresh_for_run?(summary_file, process_info[:start_time])
-
-          json_data = JSON.parse(File.read(summary_file))
-          results = {
-            tests: json_data["total"] || 0,
-            assertions: json_data["assertions"] || 0,
-            failures: json_data["failed"] || 0,
-            errors: json_data["errors"] || 0,
-            skipped: json_data["skipped"] || 0,
-            duration: json_data["duration"] || elapsed,
-            success: json_data["success"] || false
-          }
-
-          if results[:assertions] == 0
-            report_file = File.join(reports_dir, "report.json")
-            if File.exist?(report_file)
-              report_data = JSON.parse(File.read(report_file))
-              results[:assertions] = report_data.dig("result", "assertions") || 0
-            end
-          end
-
-          results
-        rescue JSON::ParserError
-          nil
-        end
-
-        def summary_fresh_for_run?(summary_file, start_time)
-          File.mtime(summary_file) >= (start_time - 0.001)
-        rescue StandardError
-          false
+          stderr.empty? ? "ace-test exited with status #{exit_status.inspect}" : stderr
         end
 
         def parse_progress(process_info, chunk)
@@ -411,30 +353,6 @@ module Ace
             process_info[:test_count] = $1.to_i
           end
         end
-
-        def parse_results(output)
-          # Look for summary line in output
-          if output =~ /(\d+) tests?, (\d+) assertions?, (\d+) failures?, (\d+) errors? \(([\d.]+)s\)/
-            {
-              tests: $1.to_i,
-              assertions: $2.to_i,
-              failures: $3.to_i,
-              errors: $4.to_i,
-              duration: $5.to_f,
-              success: $3.to_i == 0 && $4.to_i == 0
-            }
-          else
-            # Fallback to counting dots/F/E/S
-            dots = output.scan(/[.FES]/).join
-            {
-              tests: dots.length,
-              failures: dots.count("F"),
-              errors: dots.count("E"),
-              skipped: dots.count("S"),
-              success: !dots.include?("F") && !dots.include?("E")
-            }
-          end
-        end
       end
     end
   end