ace-test-runner 0.27.1 → 0.28.0

data/lib/ace/test_runner/organisms/test_orchestrator.rb

@@ -60,6 +60,7 @@ module Ace
           validate_configuration!
 
           start_time = Time.now
+          @execution_evidence = Molecules::ExecutionEvidence.for_invocation(Dir.pwd)
 
           # Print package context if running in different directory
           if @package_dir
@@ -82,6 +83,7 @@ module Ace
           end
 
           resolve_and_set_report_dir_context(test_files)
+          reserve_execution_report
 
           # Count total available test files (before filtering)
           total_available = count_total_test_files
@@ -104,22 +106,18 @@ module Ace
                 runs: 0,
                 assertions: 0,
                 failures: 0,
-                errors: test_files.size,  # Count all test files as errors
+                errors: 0,  # Execution failure is distinct from completed test errors
                 skips: 0,
                 passed: 0
               },
               failures: [],
-              errors: [{
-                message: execution_result[:stderr].strip,
-                type: "LoadError",
-                files: test_files
-              }],
+              errors: [],
               deprecations: [],
               duration: execution_result[:duration]
             }
           elsif execution_result[:commands] && execution_result[:commands].is_a?(Array)
             # Each file was executed separately, parse and sum them all
-            aggregate_individual_results(execution_result[:stdout])
+            aggregate_individual_results(execution_result.fetch(:execution_outputs))
           else
             # Single command execution (by-target)
             @result_parser.parse_output(execution_result[:stdout])
@@ -127,6 +125,7 @@ module Ace
 
           # Build result object
           @result = build_result(@parsed_result, execution_result, start_time)
+          validate_selected_execution
 
           # Analyze failures and errors
           if @result.has_failures?
@@ -165,6 +164,9 @@ module Ace
             @formatter.report_path = report_path if @formatter.respond_to?(:report_path=)
           end
 
+          # Publish completion only after all optional report files are written.
+          @execution_evidence.publish(@result, files: report.files_tested, report_dir: @execution_report_dir)
+
           # Output to stdout
           @formatter.on_finish(@result)
 
@@ -231,6 +233,7 @@ module Ace
           all_files = targets.flat_map { |t| t[:files] }
 
           resolve_and_set_report_dir_context(all_files)
+          reserve_execution_report
 
           total_available = count_total_test_files
 
@@ -267,6 +270,7 @@ module Ace
 
           # Build result object
           @result = build_result(@parsed_result, execution_result, start_time)
+          validate_selected_execution
 
           # Analyze failures
           if @result.has_failures?
@@ -301,6 +305,9 @@ module Ace
             @formatter.report_path = report_path if @formatter.respond_to?(:report_path=)
           end
 
+          # Publish completion only after all optional report files are written.
+          @execution_evidence.publish(@result, files: report.files_tested, report_dir: @execution_report_dir)
+
           # Output to stdout
           @formatter.on_finish(@result)
 
@@ -463,6 +470,10 @@ module Ace
 
         def handle_no_tests
           @result = Models::TestResult.new
+          resolve_and_set_report_dir_context([])
+          reserve_execution_report
+          save_reports(@report_generator.generate(@result, [])) if @configuration.save_reports
+          @execution_evidence.publish(@result, files: [], report_dir: @execution_report_dir)
 
           message = if @configuration.filter
             "No test files found matching pattern '#{@configuration.filter}'"
@@ -471,6 +482,7 @@ module Ace
           end
 
           puts message
+          puts(@execution_report_dir ? "Details: #{@execution_report_dir}/" : "No saved report")
           0
         end
 
@@ -507,6 +519,13 @@ module Ace
           end
         end
 
+        def validate_selected_execution
+          return unless @result.success? && @result.total_tests == 0
+
+          @result.execution_success = false
+          @result.stderr = [@result.stderr, "Selected test files produced no executed tests"].reject(&:empty?).join("\n")
+        end
+
         def build_result(parsed_result, execution_result, start_time)
           Models::TestResult.new(
             passed: parsed_result[:summary][:passed],
@@ -519,17 +538,16 @@ module Ace
             end_time: Time.now,
             deprecations: parsed_result[:deprecations],
             raw_output: execution_result[:stdout],
-            stderr: execution_result[:stderr]
+            stderr: execution_result[:stderr],
+            execution_success: execution_result[:success] == true
           )
         end
 
-        def aggregate_individual_results(combined_output)
-          # Split output by test file executions
-          individual_outputs = combined_output.split(/^Started with run options/)
-          individual_outputs.shift if individual_outputs.first && individual_outputs.first.empty?
+        def aggregate_individual_results(individual_outputs)
+          # Process boundaries come from the executor, never text inside stdout.
 
           aggregated = {
-            raw_output: combined_output,
+            raw_output: individual_outputs.join("\n"),
             summary: {
               runs: 0,
               assertions: 0,
@@ -545,7 +563,6 @@ module Ace
           }
 
           individual_outputs.each do |output|
-            output = "Started with run options" + output  # Restore the split text
             parsed = @result_parser.parse_output(output)
 
             # Sum up the counts
@@ -602,19 +619,22 @@ module Ace
           )
         end
 
+        def reserve_execution_report
+          return unless @configuration.save_reports
+
+          @report_storage = Molecules::ReportStorage.new(base_dir: @configuration.report_dir)
+          @execution_report_dir = @report_storage.reserve(@execution_evidence.identity)
+        end
+
         def save_reports(report)
-          timestamp_generator = Atoms::TimestampGenerator.new
-          storage = Molecules::ReportStorage.new(
-            base_dir: @configuration.report_dir,
-            timestamp_generator: timestamp_generator
-          )
+          storage = @report_storage
 
           # Save in appropriate format
           report_path = case @configuration.format
           when "json"
-            storage.save_report(report, format: :json)
+            storage.save_report(report, format: :json, report_dir: @execution_report_dir)
           else
-            storage.save_report(report, format: :all)
+            storage.save_report(report, format: :all, report_dir: @execution_report_dir)
           end
 
           # Always save raw output

data/lib/ace/test_runner/suite/display_manager.rb

@@ -33,9 +33,9 @@ module Ace
 
           # Reserve lines for each package
           @packages.each_with_index do |package, index|
-            @lines[package["name"]] = index + 5  # Account for header lines
-            @package_status[package["name"]] = {status: :waiting}
-            print_package_line(package["name"])
+            @lines[package["entry_id"]] = index + 5  # Account for header lines
+            @package_status[package["entry_id"]] = {status: :waiting}
+            print_package_line(package["entry_id"])
           end
 
           # Print footer space
@@ -45,8 +45,8 @@ module Ace
         end
 
         def update_package(package, status, output = nil)
-          @package_status[package["name"]] = status
-          print_package_line(package["name"])
+          @package_status[package["entry_id"]] = status
+          print_package_line(package["entry_id"])
           update_footer
         end
 
@@ -87,7 +87,7 @@ module Ace
           print "\033[K"  # Clear line
 
           # Format package name (fixed width, no brackets)
-          pkg_name = name.ljust(25)
+          pkg_name = @packages.find { |package| package["entry_id"] == name }.fetch("name").ljust(25)
 
           case status[:status]
           when :waiting
@@ -135,6 +135,7 @@ module Ace
             line_text = "#{icon}  #{elapsed}  #{pkg_name}  #{tests_col}  #{asserts_col}  #{fail_col}"
             line_text += "  #{skipped} skip" if skipped > 0
 
+            line_text += results[:report_dir] ? "  report: #{results[:report_dir]}" : "  no saved report"
             print line_text
           end
         end

data/lib/ace/test_runner/suite/orchestrator.rb

@@ -11,7 +11,9 @@ module Ace
 
         def initialize(config)
           @config = config
-          @packages = config.dig("test_suite", "packages") || []
+          @packages = (config.dig("test_suite", "packages") || []).map do |package|
+            package.merge("entry_id" => SecureRandom.uuid)
+          end
           @results = {}
           @running_processes = {}
           @completed_packages = []
@@ -59,7 +61,7 @@ module Ace
 
               if status[:completed]
                 @completed_packages << pkg
-                @results[pkg["name"]] = status
+                @results[pkg["entry_id"]] = status
               end
             end
           end
@@ -75,7 +77,7 @@ module Ace
           display_manager.show_final_results
 
           # Aggregate results
-          aggregator = ResultAggregator.new(@packages, report_root: report_root, runtime_results: @results)
+          aggregator = ResultAggregator.new(@packages, runtime_results: @results)
           summary = aggregator.aggregate
 
           display_manager.show_summary(summary)

data/lib/ace/test_runner/suite/process_monitor.rb

@@ -30,6 +30,10 @@ module Ace
             return
           end
 
+          package["entry_id"] ||= SecureRandom.uuid
+          evidence = Molecules::ExecutionEvidence.new(package_path: package["path"],
+            package: package["name"], entry_id: package["entry_id"])
+
           # Build command
           cmd = build_command(package, test_options)
 
@@ -45,12 +49,16 @@ module Ace
           fixture_environment.env[Models::EnvironmentPolicy::SUITE_CHANNEL_KEY] =
             @environment_policy.channel_payload
 
+          fixture_environment.env[Molecules::ExecutionEvidence::CHANNEL] = evidence.channel
+
           start_time = now
           stdin, stdout, stderr, thread = Open3.popen3(
             fixture_environment.env, *cmd, chdir: package["path"], pgroup: true, unsetenv_others: true
           )
 
-          @processes[package["name"]] = {
+          @processes[package["entry_id"]] = {
+            evidence: evidence,
+            save_reports: test_options.fetch("save_reports", true),
             package: package,
             thread: thread,
             stdout: stdout,
@@ -61,7 +69,6 @@ module Ace
             fixture_environment: fixture_environment,
             output: +"",
             stderr_output: +"",
-            report_root: test_options["report_dir"],
             test_count: 0,
             tests_run: 0,
             dots: +"",
@@ -114,9 +121,8 @@ module Ace
 
               # Final callback
               if callback
-                # Use results[:success] from summary.json if available, otherwise check exit code
-                # This ensures the package status matches what ace-test actually reported
-                success_status = (!results[:success].nil?) ? results[:success] : (exit_status == 0)
+                # Completion evidence and child outcome must both permit success.
+                success_status = results[:success] == true
 
                 callback.call(package, {
                   status: :completed,
@@ -174,11 +180,14 @@ module Ace
             rescue Timeout::Error, StandardError
               nil
             end
+          end
 
+          # Capture trustworthy partial counts and interruption outcome before cleanup.
+          check_processes
+          @processes.each_value do |process_info|
             close_streams(process_info)
             process_info[:fixture_environment]&.cleanup
           end
-
           @processes.clear
           @completed.clear
         end
@@ -213,7 +222,7 @@ module Ace
           cmd_parts << "--format" << format
 
           # Add other options
-          cmd_parts << "--no-save" unless options["save_reports"]
+          cmd_parts << "--no-save-reports" unless options.fetch("save_reports", true)
           cmd_parts << "--fail-fast" if options["fail_fast"]
           cmd_parts << "--no-color" unless options.fetch("color", true)
           if options["report_dir"]
@@ -304,98 +313,31 @@ module Ace
         end
 
         def build_results(process_info, elapsed, exit_status)
-          return timeout_results(process_info, elapsed) if process_info[:timeout_triggered]
-          return interrupted_results(elapsed) if process_info[:terminated_by] == :interrupt
-          fresh_summary = load_summary_results(process_info, elapsed)
-          return fresh_summary if exit_status == 0 && fresh_summary
-          return failed_process_results(process_info, elapsed, exit_status, fresh_summary) if exit_status != 0
-
-          parse_results(process_info[:output])
-        end
-
-        def timeout_results(process_info, elapsed)
-          {
-            tests: 0,
-            assertions: 0,
-            failures: 0,
-            errors: 1,
-            duration: elapsed,
-            success: false,
-            error: "Timed out after #{process_info[:timeout]} seconds"
-          }
-        end
-
-        def interrupted_results(elapsed)
-          {
-            tests: 0,
-            assertions: 0,
-            failures: 0,
-            errors: 1,
-            duration: elapsed,
-            success: false,
-            error: "Interrupted before completion"
-          }
-        end
-
-        def failed_process_results(process_info, elapsed, exit_status, fresh_summary)
-          return fresh_summary if fresh_summary && fresh_summary[:success] == false
-
-          parsed = parse_results([process_info[:output], process_info[:stderr_output]].join("\n"))
-          parsed[:duration] ||= elapsed
-          parsed[:success] = false
-          parsed[:error] ||= failure_message_for(process_info, exit_status)
-          parsed[:errors] = 1 if parsed[:failures].to_i == 0 && parsed[:errors].to_i == 0
-          parsed
+          snapshot = begin
+            process_info[:evidence].read(pid: process_info[:pid], save_reports: process_info[:save_reports])
+          rescue Ace::TestRunner::Error => e
+            {total: 0, passed: 0, failed: 0, errors: 1, skipped: 0, assertions: 0,
+             duration: elapsed, success: false, error: e.message}
+          end
+          results = snapshot.merge(tests: snapshot[:total], failures: snapshot[:failed],
+            execution_id: process_info[:evidence].identity, completion_path: process_info[:evidence].path)
+          reason = if process_info[:timeout_triggered]
+            "Timed out after #{process_info[:timeout]} seconds"
+          elsif process_info[:terminated_by] == :interrupt
+            "Interrupted before completion"
+          elsif exit_status != 0
+            failure_message_for(process_info, exit_status)
+          end
+          if reason
+            results[:success] = false
+            results[:error] = reason
+          end
+          results.freeze
         end
 
         def failure_message_for(process_info, exit_status)
           stderr = process_info[:stderr_output].to_s.strip
-          stdout = process_info[:output].to_s.strip
-          message = stderr.empty? ? stdout : stderr
-          return message unless message.empty?
-
-          "ace-test exited with status #{exit_status}"
-        end
-
-        def load_summary_results(process_info, elapsed)
-          package = process_info[:package]
-          reports_dir = Atoms::ReportPathResolver.report_directory(
-            package["path"],
-            report_root: process_info[:report_root],
-            package_name: package["name"]
-          )
-          summary_file = reports_dir ? File.join(reports_dir, "summary.json") : nil
-          return nil unless summary_file && File.exist?(summary_file)
-          return nil unless summary_fresh_for_run?(summary_file, process_info[:start_time])
-
-          json_data = JSON.parse(File.read(summary_file))
-          results = {
-            tests: json_data["total"] || 0,
-            assertions: json_data["assertions"] || 0,
-            failures: json_data["failed"] || 0,
-            errors: json_data["errors"] || 0,
-            skipped: json_data["skipped"] || 0,
-            duration: json_data["duration"] || elapsed,
-            success: json_data["success"] || false
-          }
-
-          if results[:assertions] == 0
-            report_file = File.join(reports_dir, "report.json")
-            if File.exist?(report_file)
-              report_data = JSON.parse(File.read(report_file))
-              results[:assertions] = report_data.dig("result", "assertions") || 0
-            end
-          end
-
-          results
-        rescue JSON::ParserError
-          nil
-        end
-
-        def summary_fresh_for_run?(summary_file, start_time)
-          File.mtime(summary_file) >= (start_time - 0.001)
-        rescue StandardError
-          false
+          stderr.empty? ? "ace-test exited with status #{exit_status.inspect}" : stderr
         end
 
         def parse_progress(process_info, chunk)
@@ -411,30 +353,6 @@ module Ace
             process_info[:test_count] = $1.to_i
           end
         end
-
-        def parse_results(output)
-          # Look for summary line in output
-          if output =~ /(\d+) tests?, (\d+) assertions?, (\d+) failures?, (\d+) errors? \(([\d.]+)s\)/
-            {
-              tests: $1.to_i,
-              assertions: $2.to_i,
-              failures: $3.to_i,
-              errors: $4.to_i,
-              duration: $5.to_f,
-              success: $3.to_i == 0 && $4.to_i == 0
-            }
-          else
-            # Fallback to counting dots/F/E/S
-            dots = output.scan(/[.FES]/).join
-            {
-              tests: dots.length,
-              failures: dots.count("F"),
-              errors: dots.count("E"),
-              skipped: dots.count("S"),
-              success: !dots.include?("F") && !dots.include?("E")
-            }
-          end
-        end
       end
     end
   end

data/lib/ace/test_runner/suite/result_aggregator.rb

@@ -1,42 +1,26 @@
 # frozen_string_literal: true
 
-require "json"
-
 module Ace
   module TestRunner
     module Suite
       class ResultAggregator
         attr_reader :packages
 
-        def initialize(packages, report_root: nil, runtime_results: {})
+        def initialize(packages, runtime_results: {})
           @packages = packages
-          @report_root = report_root
           @runtime_results = runtime_results || {}
         end
 
         def aggregate
           results = collect_results
 
-          # Calculate assertion totals
-          total_assertions = 0
-          assertions_failed = 0
-
-          results.each do |r|
-            # Get assertions from either summary or report data
-            if r[:assertions]
-              total_assertions += r[:assertions]
-            elsif r[:report_data]
-              total_assertions += r[:report_data].dig(:result, :assertions) || 0
-            end
-          end
-
           {
             total_tests: results.sum { |r| r[:total] || 0 },
             total_passed: results.sum { |r| r[:passed] || 0 },
             total_failed: results.sum { |r| (r[:failed] || 0) + (r[:errors] || 0) },
             total_skipped: results.sum { |r| r[:skipped] || 0 },
-            total_assertions: total_assertions,
-            assertions_failed: assertions_failed,
+            total_assertions: results.sum { |result| result[:assertions] || 0 },
+            assertions_failed: 0,
             total_duration: results.map { |r| r[:duration] || 0 }.max,
             packages_passed: results.count { |r| r[:success] },
             packages_failed: results.count { |r| !r[:success] },
@@ -47,81 +31,22 @@ module Ace
 
         def collect_results
           @packages.map do |package|
-            runtime = runtime_result(package)
-            next runtime if runtime_result_overrides_summary?(package, runtime)
-
-            reports_dir = Atoms::ReportPathResolver.report_directory(
-              package["path"],
-              report_root: @report_root,
-              package_name: package["name"]
-            )
-            summary_path = reports_dir ? File.join(reports_dir, "summary.json") : nil
-
-            if summary_path && File.exist?(summary_path)
-              begin
-                data = JSON.parse(File.read(summary_path), symbolize_names: true)
-                data[:package] = package["name"]
-                data[:path] = package["path"]
-                data[:report_root] = @report_root
-
-                # Try to get assertions from report.json if not in summary
-                if !data[:assertions] || data[:assertions] == 0
-                  report_path = File.join(reports_dir, "report.json")
-                  if File.exist?(report_path)
-                    begin
-                      report_data = JSON.parse(File.read(report_path), symbolize_names: true)
-                      data[:assertions] = report_data.dig(:result, :assertions) || 0
-                      data[:report_data] = report_data
-                    rescue JSON::ParserError
-                      # Ignore if report.json can't be parsed
-                    end
-                  end
-                end
-
-                data
-              rescue JSON::ParserError => e
-                # If we can't parse the summary, create a failure result
-                {
-                  package: package["name"],
-                  path: package["path"],
-                  report_root: @report_root,
-                  success: false,
-                  error: "Failed to parse summary.json: #{e.message}",
-                  total: 0,
-                  passed: 0,
-                  failed: 0,
-                  errors: 1
-                }
-              end
-            else
-              runtime || {
-                package: package["name"],
-                path: package["path"],
-                report_root: @report_root,
-                success: false,
-                error: "No test results found (summary.json missing)",
-                total: 0,
-                passed: 0,
-                failed: 0,
-                errors: 1
-              }
-            end
+            runtime_result(package) || {
+              package: package["name"], path: package["path"], entry_id: package["entry_id"],
+              report_dir: nil, success: false, error: "No verified execution completion evidence",
+              total: 0, passed: 0, failed: 0, errors: 1, assertions: 0
+            }
           end
         end
 
-        def runtime_result_overrides_summary?(package, runtime)
-          return false unless runtime
-
-          status = @runtime_results[package["name"]] || {}
-          status[:timed_out] || status[:interrupted] || runtime[:success] == false
-        end
-
         def collect_failed_packages(results)
           results.select { |r| !r[:success] }.map do |result|
             {
               name: result[:package],
               path: result[:path],
-              report_root: result[:report_root] || @report_root,
+              report_dir: result[:report_dir],
+              execution_id: result[:execution_id],
+              entry_id: result[:entry_id],
               failures: result[:failed] || 0,
               errors: result[:errors] || 0,
               error_message: result[:error]
@@ -130,7 +55,7 @@ module Ace
         end
 
         def runtime_result(package)
-          status = @runtime_results[package["name"]]
+          status = @runtime_results[package["entry_id"]]
           return nil unless status && status[:completed]
 
           results = status[:results] || {}
@@ -142,11 +67,14 @@ module Ace
           {
             package: package["name"],
             path: package["path"],
-            report_root: @report_root,
+            report_dir: results[:report_dir],
+            completion_path: results[:completion_path],
+            execution_id: results[:execution_id],
+            entry_id: package["entry_id"],
             success: status[:success],
             error: results[:error],
             total: total,
-            passed: total - failures - errors - skipped,
+            passed: results[:passed] || 0,
             failed: failures,
             errors: errors,
             skipped: skipped,

data/lib/ace/test_runner/suite/simple_display_manager.rb

@@ -31,7 +31,7 @@ module Ace
 
         # Called when a package status changes. Prints a line when package completes.
         def update_package(package, status, _output = nil)
-          @package_status[package["name"]] = status
+          @package_status[package["entry_id"]] = status
 
           # Only print when package completes
           return unless status[:completed]
@@ -88,6 +88,7 @@ module Ace
           line += "  #{skipped} skip" if skipped > 0
           line += "  timeout" if status[:timed_out]
 
+          line += results[:report_dir] ? "  report: #{results[:report_dir]}" : "  no saved report"
           puts line
         end
 

data/lib/ace/test_runner/version.rb

@@ -2,6 +2,6 @@
 
 module Ace
   module TestRunner
-    VERSION = "0.27.1"
+    VERSION = "0.28.0"
   end
 end

data/lib/ace/test_runner.rb

@@ -45,6 +45,7 @@ require_relative "test_runner/atoms/environment_sanitizer"
 require_relative "test_runner/molecules/fixture_environment"
 require_relative "test_runner/molecules/test_executor"
 require_relative "test_runner/molecules/failure_analyzer"
+require_relative "test_runner/molecules/execution_evidence"
 require_relative "test_runner/molecules/report_storage"
 require_relative "test_runner/molecules/config_loader"
 require_relative "test_runner/molecules/pattern_resolver"