ace-test-runner-e2e 0.29.8 → 0.38.11

data/lib/ace/test/end_to_end_runner/organisms/suite_orchestrator.rb

@@ -3,6 +3,7 @@
 require "open3"
 require "fileutils"
 require "yaml"
+require "set"
 require "ace/b36ts"
 
 module Ace
@@ -57,6 +58,7 @@ module Ace
           # @option options [Integer] :timeout Timeout per test in seconds
           # @return [Hash] Summary of results
           def run(options = {})
+            pre_run_worktree = git_status_snapshot
             packages = @discoverer.list_packages(base_dir: @base_dir)
 
             if packages.empty?
@@ -135,9 +137,9 @@ module Ace
 
             # Execute tests
             if options[:parallel]
-              run_parallel(package_tests, options)
+              run_parallel(package_tests, options, pre_run_worktree)
             else
-              run_sequential(package_tests, options)
+              run_sequential(package_tests, options, pre_run_worktree)
             end
           end
 
@@ -210,7 +212,7 @@ module Ace
           # @param package_tests [Hash] Package to tests mapping
           # @param options [Hash] Execution options
           # @return [Hash] Summary of results
-          def run_sequential(package_tests, options)
+          def run_sequential(package_tests, options, pre_run_worktree)
             results = {total: 0, passed: 0, failed: 0, errors: 0, total_cases: 0, passed_cases: 0, packages: {}}
             start_time = Time.now
 
@@ -265,7 +267,7 @@ module Ace
             done = true
             refresh_thread&.join
 
-            finalize_run(results, package_tests, start_time)
+            finalize_run(results, package_tests, start_time, pre_run_worktree)
           end
 
           # Run tests in parallel using subprocesses
@@ -273,7 +275,7 @@ module Ace
           # @param package_tests [Hash] Package to tests mapping
           # @param options [Hash] Execution options
           # @return [Hash] Summary of results
-          def run_parallel(package_tests, options)
+          def run_parallel(package_tests, options, pre_run_worktree)
             results = {total: 0, passed: 0, failed: 0, errors: 0, total_cases: 0, passed_cases: 0, packages: {}}
             queue = build_test_queue(package_tests)
             run_ids = generate_run_ids(queue.size)
@@ -297,7 +299,7 @@ module Ace
               check_running_processes(running, results)
             end
 
-            finalize_run(results, package_tests, start_time)
+            finalize_run(results, package_tests, start_time, pre_run_worktree)
           end
 
           # Build a flat queue of test items
@@ -497,6 +499,7 @@ module Ace
           # @return [Hash] Parsed result with :passed_cases and :total_cases
           def parse_subprocess_result(process)
             result = parse_test_output(process[:output], process[:thread].value.exitstatus, extract_test_name(process[:test_file]))
+            result[:report_dir] = normalize_report_dir(result[:report_dir], result[:test_name])
             result[:raw_output] = process[:output]
 
             # For non-pass results, check agent-written metadata as authoritative source
@@ -510,6 +513,34 @@ module Ace
             {status: "error", error: "Failed to parse result: #{e.message}"}
           end
 
+          def normalize_report_dir(report_dir, test_name)
+            return report_dir if report_dir.nil? || report_dir.empty?
+            return report_dir if File.directory?(report_dir)
+            return report_dir unless File.file?(report_dir)
+
+            resolved = resolve_report_dir_from_suite_report(report_dir, canonical_test_id(test_name))
+            resolved || report_dir
+          rescue
+            report_dir
+          end
+
+          def resolve_report_dir_from_suite_report(report_path, test_id)
+            return nil unless report_path.end_with?(".md")
+            return nil if test_id.nil? || test_id.empty?
+
+            content = File.read(report_path)
+            escaped = Regexp.escape(test_id)
+            table_match = content.match(/^\|\s*#{escaped}\s*\|\s*`([^`]+)`\s*\|$/m)
+            return nil unless table_match
+
+            File.expand_path(table_match[1], File.dirname(report_path))
+          end
+
+          def canonical_test_id(test_name)
+            match = test_name.to_s.match(/\A(TS-[A-Z0-9]+-\d+[a-z]*)/i)
+            match ? match[1].upcase : test_name
+          end
+
           # Override result from agent-written metadata.yml when subprocess exit code is misleading
           #
           # @param result [Hash] Parsed result with :report_dir
@@ -576,7 +607,9 @@ module Ace
               error_msg ||= "Test execution returned ERROR status"
               base.merge(status: "error", error: error_msg)
             else
-              summary = output.match(/(\d+)\/(\d+) passed/)&.captures&.join("/") || "Test failed"
+              summary = output.lines.filter_map { |line| line[/^(Preflight failed: .+?)\s*$/, 1] }.last
+              summary ||= output.match(/(\d+)\/(\d+) passed/)&.captures&.join("/")
+              summary ||= "Test failed"
               base.merge(status: "fail", summary: summary)
             end
           rescue => e
@@ -589,8 +622,9 @@ module Ace
           # @param package_tests [Hash] Package to test files mapping
           # @param start_time [Time] When the run started
           # @return [Hash] Results with optional :report_path
-          def finalize_run(results, package_tests, start_time)
+          def finalize_run(results, package_tests, start_time, pre_run_worktree)
             write_failure_stubs(results, package_tests)
+            results[:suite_diagnostics] = build_suite_diagnostics(pre_run_worktree)
 
             @display.show_summary(results, Time.now - start_time)
             warn_on_lingering_claude_processes
@@ -641,6 +675,7 @@ module Ace
                   "status" => result[:status]
                 }
                 File.write(File.join(stub_dir, "metadata.yml"), YAML.dump(stub_data))
+                result[:report_dir] = stub_dir
 
                 if result[:raw_output] && !result[:raw_output].empty?
                   File.write(File.join(stub_dir, "subprocess_output.log"), result[:raw_output])
@@ -709,7 +744,9 @@ module Ace
               all_results, all_scenarios,
               package: "suite",
               timestamp: timestamp,
-              base_dir: @base_dir
+              base_dir: @base_dir,
+              report_kind: :suite,
+              diagnostics: results[:suite_diagnostics]
             )
           rescue => e
             warn "Warning: Suite report generation failed (#{e.class}: #{e.message})"
@@ -726,19 +763,40 @@ module Ace
             total = result_hash[:total_cases] || 0
             failed = [total - passed, 0].max
 
-            test_cases = []
-            passed.times { |i| test_cases << {id: "TC-#{format("%03d", i + 1)}", description: "", status: "pass"} }
-            failed.times { |i| test_cases << {id: "TC-#{format("%03d", passed + i + 1)}", description: "", status: "fail"} }
-
             Models::TestResult.new(
               test_id: result_hash[:test_name] || "unknown",
               status: result_hash[:status] || "error",
-              test_cases: test_cases,
+              test_cases: [],
               summary: result_hash[:summary] || result_hash[:error] || "",
-              report_dir: result_hash[:report_dir]
+              report_dir: result_hash[:report_dir],
+              metadata: {"tcs-passed" => passed, "tcs-total" => total, "tcs-failed" => failed}
             )
           end
 
+          def git_status_snapshot
+            stdout, _stderr, status = Open3.capture3("git", "status", "--short", chdir: @base_dir)
+            return nil unless status.success?
+
+            stdout.lines.map(&:rstrip)
+          rescue
+            nil
+          end
+
+          def build_suite_diagnostics(pre_run_worktree)
+            post_run_worktree = git_status_snapshot
+            return {} unless pre_run_worktree && post_run_worktree
+
+            before = pre_run_worktree.to_set
+            new_entries = post_run_worktree.reject { |line| before.include?(line) }
+            new_tracked_entries = new_entries.reject { |line| line.start_with?("?? ") }
+            return {} if new_tracked_entries.empty?
+
+            {
+              dirty_worktree: true,
+              new_tracked_entries: new_tracked_entries
+            }
+          end
+
           # Load a scenario from file into a Models::TestScenario, with fallback
           #
           # @param package [String] Package name

data/lib/ace/test/end_to_end_runner/organisms/test_orchestrator.rb

@@ -4,6 +4,8 @@ require "fileutils"
 require "date"
 require "yaml"
 require "ace/b36ts"
+require "ace/test_support/sandbox_package_copy"
+require "ace/test/end_to_end_runner/molecules/integration_runner"
 
 module Ace
   module Test
@@ -28,19 +30,29 @@ module Ace
           # @param timestamp_generator [#call] Callable that returns a timestamp string
           # @param executor [#execute] Injectable test executor (for testing)
           # @param progress [Boolean] Enable animated progress display
-          def initialize(provider: nil, timeout: nil, parallel: nil, base_dir: nil, timestamp_generator: nil, executor: nil, progress: false)
+          def initialize(provider: nil, timeout: nil, parallel: nil, base_dir: nil, timestamp_generator: nil,
+            executor: nil, progress: false, discoverer: nil, integration_runner: nil,
+            scenario_loader: nil, report_writer: nil, suite_report_writer: nil,
+            setup_executor_factory: nil, runtime_builder: nil)
             config = Molecules::ConfigLoader.load
-            @provider = provider || config.dig("execution", "provider") || "claude:sonnet"
+            @provider = provider || config.dig("execution", "runner_provider") ||
+              config.dig("execution", "provider") || "claude:sonnet"
             @timeout = timeout || config.dig("execution", "timeout") || 300
             @parallel = parallel || config.dig("execution", "parallel") || 3
             @base_dir = base_dir || Dir.pwd
             @timestamp_generator = timestamp_generator || method(:default_timestamp)
             @progress = progress
-            @discoverer = Molecules::TestDiscoverer.new
-            @loader = Molecules::ScenarioLoader.new
+            @discoverer = discoverer || Molecules::TestDiscoverer.new
+            @integration_runner = integration_runner || Molecules::IntegrationRunner.new(base_dir: @base_dir)
+            @loader = scenario_loader || Molecules::ScenarioLoader.new
             @executor = executor || Molecules::TestExecutor.new(provider: @provider, timeout: @timeout, config: config)
-            @report_writer = Molecules::ReportWriter.new
-            @suite_report_writer = Molecules::SuiteReportWriter.new(config: config)
+            @report_writer = report_writer || Molecules::ReportWriter.new
+            @suite_report_writer = suite_report_writer || Molecules::SuiteReportWriter.new(config: config)
+            @setup_executor_factory = setup_executor_factory || ->(sandbox_backend: nil) { Molecules::SetupExecutor.new(sandbox_backend: sandbox_backend) }
+            @runtime_builder = runtime_builder || Molecules::SandboxRuntimeBuilder.new(
+              source_root: @base_dir,
+              ruby_version: config.dig("sandbox", "ruby_version") || Molecules::ConfigLoader.default_sandbox_ruby_version
+            )
           end
 
           # Run E2E tests for a package, optionally filtering by test ID
@@ -54,6 +66,11 @@ module Ace
           # @return [Array<Models::TestResult>] List of test results
           def run(package:, test_id: nil, test_cases: nil, verify: false, tags: nil,
             cli_args: nil, run_id: nil, report_dir: nil, output: $stdout)
+            integration_files = @discoverer.find_integration_tests(
+              package: package,
+              base_dir: @base_dir
+            )
+
             # Discover tests
             files = @discoverer.find_tests(
               package: package,
@@ -62,7 +79,7 @@ module Ace
               base_dir: @base_dir
             )
 
-            if files.empty?
+            if files.empty? && integration_files.empty?
               output.puts "No E2E tests found in #{package}" +
                 (test_id ? " matching #{test_id}" : "")
               return []
@@ -71,7 +88,7 @@ module Ace
             # Generate timestamp for this run (use external run_id when provided)
             timestamp = run_id || generate_timestamp
 
-            if files.size == 1
+            if files.size == 1 && integration_files.empty?
               run_single_test(
                 files.first,
                 timestamp,
@@ -82,7 +99,16 @@ module Ace
                 report_dir: report_dir
               )
             else
-              run_package_tests(files, package, timestamp, cli_args, output, test_cases: test_cases, verify: verify)
+              run_package_tests(
+                files,
+                package,
+                timestamp,
+                cli_args,
+                output,
+                test_cases: test_cases,
+                verify: verify,
+                integration_files: integration_files
+              )
             end
           end
 
@@ -107,13 +133,43 @@ module Ace
             return [nil, nil, nil] unless cli_provider? && scenario.setup_steps.any?
 
             sandbox_dir = File.join(@base_dir, ".ace-local", "test-e2e", scenario.dir_name(timestamp))
-            setup_executor = Molecules::SetupExecutor.new
+            package_copy = Ace::TestSupport::SandboxPackageCopy.new(source_root: @base_dir)
+            package_source = File.join(@base_dir, scenario.package.to_s)
+            package_copy_result = if File.directory?(package_source)
+              package_copy.prepare(
+                package_name: scenario.package,
+                sandbox_root: sandbox_dir
+              )
+            else
+              {
+                env: {
+                  "PROJECT_ROOT_PATH" => File.expand_path(sandbox_dir),
+                  "ACE_E2E_SOURCE_ROOT" => File.expand_path(@base_dir)
+                }
+              }
+            end
+            Molecules::PipelineSandboxBuilder.new(config_root: @base_dir).sync_protocol_sources_into(sandbox_dir)
+            runtime_result = @runtime_builder.prepare(
+              sandbox_root: sandbox_dir,
+              env: package_copy_result[:env],
+              tool_names: scenario.requires.fetch("tools", [])
+            )
+            sandbox_backend = Molecules::BwrapSandboxBackend.new(
+              sandbox_root: sandbox_dir,
+              source_root: runtime_result.dig(:env, "ACE_E2E_SOURCE_ROOT")
+            )
+            setup_executor = if @setup_executor_factory.arity.zero?
+              @setup_executor_factory.call
+            else
+              @setup_executor_factory.call(sandbox_backend: sandbox_backend)
+            end
             result = setup_executor.execute(
-              setup_steps: scenario.setup_steps,
+              setup_steps: effective_setup_steps_for(scenario),
               sandbox_dir: sandbox_dir,
               fixture_source: scenario.fixture_path,
               scenario_name: scenario.test_id,
-              run_id: timestamp
+              run_id: timestamp,
+              initial_env: runtime_result[:env]
             )
 
             unless result[:success]
@@ -130,6 +186,29 @@ module Ace
             [File.expand_path(sandbox_dir), env, setup_executor]
           end
 
+          def effective_setup_steps_for(scenario)
+            steps = Array(scenario.setup_steps)
+            return steps unless scenario.sandbox_profile == "ace-default"
+
+            has_config_init = setup_contains_command?(steps, "ace-config init")
+            has_handbook_sync = setup_contains_command?(steps, "ace-handbook sync")
+            bootstrap = []
+            bootstrap << {"run" => "ace-config init"} unless has_config_init
+            bootstrap << {"run" => "ace-handbook sync"} unless has_handbook_sync
+            return steps if bootstrap.empty?
+
+            insert_after = steps.index("git-init")
+            return bootstrap + steps unless insert_after
+
+            steps.dup.insert(insert_after + 1, *bootstrap)
+          end
+
+          def setup_contains_command?(steps, fragment)
+            steps.any? do |step|
+              step.is_a?(Hash) && step["run"].to_s.include?(fragment)
+            end
+          end
+
           # Run a single test
           # @param test_cases [Array<String>, nil] Optional test case IDs to filter
           # @param report_dir [String, nil] Explicit report directory path (overrides computed path)
@@ -192,7 +271,23 @@ module Ace
           # Run all tests in a package
           # @param test_cases [Array<String>, nil] Optional test case IDs to filter
           # @return [Array<Models::TestResult>] Results for all tests
-          def run_package_tests(files, package, timestamp, cli_args, output, test_cases: nil, verify: false)
+          def run_package_tests(files, package, timestamp, cli_args, output, test_cases: nil, verify: false,
+            integration_files: [])
+            integration_result = @integration_runner.run(
+              package: package,
+              files: integration_files,
+              timestamp: timestamp,
+              output: output
+            )
+            if integration_result && %w[fail error].include?(integration_result.status)
+              output.puts integration_result.summary
+              return [integration_result]
+            end
+
+            if files.empty?
+              return integration_result ? [integration_result] : []
+            end
+
             # Load scenarios upfront for titles and report generation
             scenarios = files.map { |f| @loader.load(File.dirname(f)) }
 
@@ -291,15 +386,17 @@ module Ace
             done = true
             refresh_thread&.join
 
+            combined_results = integration_result ? [integration_result] + results : results
+
             # Write suite report
             report_path = @suite_report_writer.write(
-              results, scenarios,
-              package: package, timestamp: timestamp, base_dir: @base_dir
+              combined_results, scenarios,
+              package: package, timestamp: timestamp, base_dir: @base_dir, report_kind: :package
             )
 
-            display.show_summary(results, report_path)
+            display.show_summary(combined_results, report_path)
 
-            results
+            combined_results
           end
 
           # Build the appropriate display manager for this run
@@ -332,12 +429,16 @@ module Ace
           # Uses Ace::B36ts library to encode unique IDs with 50ms precision,
           # ensuring distinct timestamps for parallel test runs.
           #
+          # Offset uses 0.1 (100ms) instead of 0.05 to avoid collisions with
+          # the 50ms encoder's approximate bucket size.
+          #
           # @param count [Integer] Number of unique timestamps needed
           # @return [Array<String>] Array of unique timestamp strings
           def generate_timestamps(count)
+            base_time = Time.now.utc
+
             count.times.map do |i|
-              time = Time.now.utc + (i * 0.05) # 50ms offset per ID
-              Ace::B36ts.encode(time, format: :"50ms")
+              Ace::B36ts.encode(base_time + (i * 0.1), format: :"50ms")
             end
           end
 

data/lib/ace/test/end_to_end_runner/version.rb

@@ -3,7 +3,7 @@
 module Ace
   module Test
     module EndToEndRunner
-      VERSION = '0.29.8'
+      VERSION = '0.38.11'
     end
   end
 end

data/lib/ace/test/end_to_end_runner.rb

@@ -20,6 +20,8 @@ require_relative "end_to_end_runner/atoms/display_helpers"
 # Molecules
 require_relative "end_to_end_runner/molecules/fixture_copier"
 require_relative "end_to_end_runner/molecules/scenario_loader"
+require_relative "end_to_end_runner/molecules/bwrap_sandbox_backend"
+require_relative "end_to_end_runner/molecules/sandbox_runtime_builder"
 require_relative "end_to_end_runner/molecules/setup_executor"
 require_relative "end_to_end_runner/molecules/config_loader"
 require_relative "end_to_end_runner/molecules/test_discoverer"

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: ace-test-runner-e2e
 version: !ruby/object:Gem::Version
-  version: 0.29.8
+  version: 0.38.11
 platform: ruby
 authors:
 - Michal Czyz
 bindir: exe
 cert_chain: []
-date: 2026-04-05 00:00:00.000000000 Z
+date: 2026-04-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ace-support-cli
@@ -51,6 +51,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.9'
+- !ruby/object:Gem::Dependency
+  name: ace-support-test-helpers
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.14'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.14'
 - !ruby/object:Gem::Dependency
   name: ace-llm
   requirement: !ruby/object:Gem::Requirement
@@ -172,15 +186,18 @@ files:
 - lib/ace/test/end_to_end_runner/models/test_result.rb
 - lib/ace/test/end_to_end_runner/models/test_scenario.rb
 - lib/ace/test/end_to_end_runner/molecules/affected_detector.rb
+- lib/ace/test/end_to_end_runner/molecules/bwrap_sandbox_backend.rb
 - lib/ace/test/end_to_end_runner/molecules/config_loader.rb
 - lib/ace/test/end_to_end_runner/molecules/failure_finder.rb
 - lib/ace/test/end_to_end_runner/molecules/fixture_copier.rb
+- lib/ace/test/end_to_end_runner/molecules/integration_runner.rb
 - lib/ace/test/end_to_end_runner/molecules/pipeline_executor.rb
 - lib/ace/test/end_to_end_runner/molecules/pipeline_prompt_bundler.rb
 - lib/ace/test/end_to_end_runner/molecules/pipeline_report_generator.rb
 - lib/ace/test/end_to_end_runner/molecules/pipeline_sandbox_builder.rb
 - lib/ace/test/end_to_end_runner/molecules/progress_display_manager.rb
 - lib/ace/test/end_to_end_runner/molecules/report_writer.rb
+- lib/ace/test/end_to_end_runner/molecules/sandbox_runtime_builder.rb
 - lib/ace/test/end_to_end_runner/molecules/scenario_loader.rb
 - lib/ace/test/end_to_end_runner/molecules/setup_executor.rb
 - lib/ace/test/end_to_end_runner/molecules/simple_display_manager.rb