ace-test-runner-e2e 0.29.8 → 0.40.1

data/lib/ace/test/end_to_end_runner/organisms/test_orchestrator.rb

@@ -4,6 +4,8 @@ require "fileutils"
 require "date"
 require "yaml"
 require "ace/b36ts"
+require "ace/test_support/sandbox_package_copy"
+require "ace/test/end_to_end_runner/molecules/integration_runner"
 
 module Ace
   module Test
@@ -28,19 +30,29 @@ module Ace
           # @param timestamp_generator [#call] Callable that returns a timestamp string
           # @param executor [#execute] Injectable test executor (for testing)
           # @param progress [Boolean] Enable animated progress display
-          def initialize(provider: nil, timeout: nil, parallel: nil, base_dir: nil, timestamp_generator: nil, executor: nil, progress: false)
+          def initialize(provider: nil, timeout: nil, parallel: nil, base_dir: nil, timestamp_generator: nil,
+            executor: nil, progress: false, discoverer: nil, integration_runner: nil,
+            scenario_loader: nil, report_writer: nil, suite_report_writer: nil,
+            setup_executor_factory: nil, runtime_builder: nil)
             config = Molecules::ConfigLoader.load
-            @provider = provider || config.dig("execution", "provider") || "claude:sonnet"
+            @provider = provider || config.dig("execution", "runner_provider") ||
+              config.dig("execution", "provider") || "claude:sonnet"
             @timeout = timeout || config.dig("execution", "timeout") || 300
             @parallel = parallel || config.dig("execution", "parallel") || 3
             @base_dir = base_dir || Dir.pwd
             @timestamp_generator = timestamp_generator || method(:default_timestamp)
             @progress = progress
-            @discoverer = Molecules::TestDiscoverer.new
-            @loader = Molecules::ScenarioLoader.new
+            @discoverer = discoverer || Molecules::TestDiscoverer.new
+            @integration_runner = integration_runner || Molecules::IntegrationRunner.new(base_dir: @base_dir)
+            @loader = scenario_loader || Molecules::ScenarioLoader.new
             @executor = executor || Molecules::TestExecutor.new(provider: @provider, timeout: @timeout, config: config)
-            @report_writer = Molecules::ReportWriter.new
-            @suite_report_writer = Molecules::SuiteReportWriter.new(config: config)
+            @report_writer = report_writer || Molecules::ReportWriter.new
+            @suite_report_writer = suite_report_writer || Molecules::SuiteReportWriter.new(config: config)
+            @setup_executor_factory = setup_executor_factory || ->(sandbox_backend: nil) { Molecules::SetupExecutor.new(sandbox_backend: sandbox_backend) }
+            @runtime_builder = runtime_builder || Molecules::SandboxRuntimeBuilder.new(
+              source_root: @base_dir,
+              ruby_version: config.dig("sandbox", "ruby_version") || Molecules::ConfigLoader.default_sandbox_ruby_version
+            )
           end
 
           # Run E2E tests for a package, optionally filtering by test ID
@@ -54,6 +66,11 @@ module Ace
           # @return [Array<Models::TestResult>] List of test results
           def run(package:, test_id: nil, test_cases: nil, verify: false, tags: nil,
             cli_args: nil, run_id: nil, report_dir: nil, output: $stdout)
+            integration_files = @discoverer.find_integration_tests(
+              package: package,
+              base_dir: @base_dir
+            )
+
             # Discover tests
             files = @discoverer.find_tests(
               package: package,
@@ -62,7 +79,7 @@ module Ace
               base_dir: @base_dir
             )
 
-            if files.empty?
+            if files.empty? && integration_files.empty?
               output.puts "No E2E tests found in #{package}" +
                 (test_id ? " matching #{test_id}" : "")
               return []
@@ -71,7 +88,7 @@ module Ace
             # Generate timestamp for this run (use external run_id when provided)
             timestamp = run_id || generate_timestamp
 
-            if files.size == 1
+            if files.size == 1 && integration_files.empty?
               run_single_test(
                 files.first,
                 timestamp,
@@ -82,7 +99,16 @@ module Ace
                 report_dir: report_dir
               )
             else
-              run_package_tests(files, package, timestamp, cli_args, output, test_cases: test_cases, verify: verify)
+              run_package_tests(
+                files,
+                package,
+                timestamp,
+                cli_args,
+                output,
+                test_cases: test_cases,
+                verify: verify,
+                integration_files: integration_files
+              )
             end
           end
 
@@ -107,13 +133,50 @@ module Ace
             return [nil, nil, nil] unless cli_provider? && scenario.setup_steps.any?
 
             sandbox_dir = File.join(@base_dir, ".ace-local", "test-e2e", scenario.dir_name(timestamp))
-            setup_executor = Molecules::SetupExecutor.new
+            setup_steps = effective_setup_steps_for(scenario)
+            package_copy = Ace::TestSupport::SandboxPackageCopy.new(source_root: @base_dir)
+            package_source = File.join(@base_dir, scenario.package.to_s)
+            package_copy_result = if File.directory?(package_source)
+              package_copy.prepare(
+                package_name: scenario.package,
+                sandbox_root: sandbox_dir
+              )
+            else
+              {
+                env: {
+                  "PROJECT_ROOT_PATH" => File.expand_path(sandbox_dir),
+                  "ACE_E2E_SOURCE_ROOT" => File.expand_path(@base_dir)
+                }
+              }
+            end
+            sandbox_builder = Molecules::PipelineSandboxBuilder.new(config_root: @base_dir)
+            protocol_packages = sandbox_builder.sync_protocol_sources_into(sandbox_dir)
+            runtime_result = @runtime_builder.prepare(
+              sandbox_root: sandbox_dir,
+              env: package_copy_result[:env],
+              tool_names: scenario.requires.fetch("tools", [])
+            )
+            sandbox_backend = Molecules::BwrapSandboxBackend.new(
+              sandbox_root: sandbox_dir,
+              source_root: runtime_result.dig(:env, "ACE_E2E_SOURCE_ROOT")
+            )
+            setup_executor = if @setup_executor_factory.arity.zero?
+              @setup_executor_factory.call
+            else
+              @setup_executor_factory.call(sandbox_backend: sandbox_backend)
+            end
             result = setup_executor.execute(
-              setup_steps: scenario.setup_steps,
+              setup_steps: setup_steps,
               sandbox_dir: sandbox_dir,
               fixture_source: scenario.fixture_path,
               scenario_name: scenario.test_id,
-              run_id: timestamp
+              run_id: timestamp,
+              initial_env: runtime_result[:env],
+              git_excludes: sandbox_support_git_excludes(
+                scenario,
+                protocol_packages,
+                setup_steps: setup_steps
+              )
             )
 
             unless result[:success]
@@ -130,6 +193,61 @@ module Ace
             [File.expand_path(sandbox_dir), env, setup_executor]
           end
 
+          def effective_setup_steps_for(scenario)
+            steps = Array(scenario.setup_steps)
+            return steps unless scenario.sandbox_profile == "ace-default"
+
+            has_config_sync = setup_contains_command?(steps, "ace-config sync")
+            has_handbook_sync = setup_contains_command?(steps, "ace-handbook sync")
+            bootstrap = []
+            bootstrap << {"run" => "ace-config sync ace-llm-providers-cli"} unless has_config_sync
+            bootstrap << {"run" => "ace-handbook sync"} unless has_handbook_sync
+            return steps if bootstrap.empty?
+
+            insert_after = steps.index("git-init")
+            return bootstrap + steps unless insert_after
+
+            steps.dup.insert(insert_after + 1, *bootstrap)
+          end
+
+          def setup_contains_command?(steps, fragment)
+            steps.any? do |step|
+              step.is_a?(Hash) && step["run"].to_s.include?(fragment)
+            end
+          end
+
+          def sandbox_support_git_excludes(scenario, protocol_packages, setup_steps:)
+            package_paths = ([scenario.package] + Array(protocol_packages))
+              .map(&:to_s)
+              .map(&:strip)
+              .reject(&:empty?)
+              .uniq
+              .sort
+              .map { |name| "#{name}/" }
+
+            # Most scenarios need the copied package tree to stay visible to tools
+            # like rg/fd. Only fixture-commit setup flows should hide the copied
+            # support trees from git staging.
+            return [] unless setup_contains_initial_commit?(setup_steps)
+
+            support_paths = [".ace-handbook/", ".ace/git/", ".ace/llm/providers/"]
+            (package_paths + support_paths).uniq
+          end
+
+          def setup_contains_initial_commit?(steps)
+            staged_changes = false
+
+            Array(steps).any? do |step|
+              next false unless step.is_a?(Hash)
+
+              command = step["run"].to_s
+              staged_changes ||= command.match?(/\bgit\s+add\b/m)
+
+              command.match?(/\bgit\s+commit\b/m) &&
+                (staged_changes || command.match?(/\bgit\s+commit\b(?=.*(?:\s--all\b|\s-[A-Za-z]*a[A-Za-z]*\b))/m))
+            end
+          end
+
           # Run a single test
           # @param test_cases [Array<String>, nil] Optional test case IDs to filter
           # @param report_dir [String, nil] Explicit report directory path (overrides computed path)
@@ -192,7 +310,23 @@ module Ace
           # Run all tests in a package
           # @param test_cases [Array<String>, nil] Optional test case IDs to filter
           # @return [Array<Models::TestResult>] Results for all tests
-          def run_package_tests(files, package, timestamp, cli_args, output, test_cases: nil, verify: false)
+          def run_package_tests(files, package, timestamp, cli_args, output, test_cases: nil, verify: false,
+            integration_files: [])
+            integration_result = @integration_runner.run(
+              package: package,
+              files: integration_files,
+              timestamp: timestamp,
+              output: output
+            )
+            if integration_result && %w[fail error].include?(integration_result.status)
+              output.puts integration_result.summary
+              return [integration_result]
+            end
+
+            if files.empty?
+              return integration_result ? [integration_result] : []
+            end
+
             # Load scenarios upfront for titles and report generation
             scenarios = files.map { |f| @loader.load(File.dirname(f)) }
 
@@ -291,15 +425,17 @@ module Ace
             done = true
             refresh_thread&.join
 
+            combined_results = integration_result ? [integration_result] + results : results
+
             # Write suite report
             report_path = @suite_report_writer.write(
-              results, scenarios,
-              package: package, timestamp: timestamp, base_dir: @base_dir
+              combined_results, scenarios,
+              package: package, timestamp: timestamp, base_dir: @base_dir, report_kind: :package
             )
 
-            display.show_summary(results, report_path)
+            display.show_summary(combined_results, report_path)
 
-            results
+            combined_results
           end
 
           # Build the appropriate display manager for this run
@@ -332,12 +468,16 @@ module Ace
           # Uses Ace::B36ts library to encode unique IDs with 50ms precision,
           # ensuring distinct timestamps for parallel test runs.
           #
+          # Offset uses 0.1 (100ms) instead of 0.05 to avoid collisions with
+          # the 50ms encoder's approximate bucket size.
+          #
           # @param count [Integer] Number of unique timestamps needed
           # @return [Array<String>] Array of unique timestamp strings
           def generate_timestamps(count)
+            base_time = Time.now.utc
+
             count.times.map do |i|
-              time = Time.now.utc + (i * 0.05) # 50ms offset per ID
-              Ace::B36ts.encode(time, format: :"50ms")
+              Ace::B36ts.encode(base_time + (i * 0.1), format: :"50ms")
             end
           end
 

data/lib/ace/test/end_to_end_runner/version.rb

@@ -3,7 +3,7 @@
 module Ace
   module Test
     module EndToEndRunner
-      VERSION = '0.29.8'
+      VERSION = '0.40.1'
     end
   end
 end

data/lib/ace/test/end_to_end_runner.rb

@@ -13,6 +13,7 @@ require_relative "end_to_end_runner/atoms/result_parser"
 require_relative "end_to_end_runner/atoms/skill_prompt_builder"
 require_relative "end_to_end_runner/atoms/skill_result_parser"
 require_relative "end_to_end_runner/atoms/suite_report_prompt_builder"
+require_relative "end_to_end_runner/atoms/artifact_contract_validator"
 require_relative "end_to_end_runner/atoms/test_case_parser"
 require_relative "end_to_end_runner/atoms/tc_fidelity_validator"
 require_relative "end_to_end_runner/atoms/display_helpers"
@@ -20,6 +21,8 @@ require_relative "end_to_end_runner/atoms/display_helpers"
 # Molecules
 require_relative "end_to_end_runner/molecules/fixture_copier"
 require_relative "end_to_end_runner/molecules/scenario_loader"
+require_relative "end_to_end_runner/molecules/bwrap_sandbox_backend"
+require_relative "end_to_end_runner/molecules/sandbox_runtime_builder"
 require_relative "end_to_end_runner/molecules/setup_executor"
 require_relative "end_to_end_runner/molecules/config_loader"
 require_relative "end_to_end_runner/molecules/test_discoverer"
@@ -30,6 +33,7 @@ require_relative "end_to_end_runner/molecules/pipeline_report_generator"
 require_relative "end_to_end_runner/molecules/pipeline_executor"
 require_relative "end_to_end_runner/molecules/report_writer"
 require_relative "end_to_end_runner/molecules/suite_report_writer"
+require_relative "end_to_end_runner/molecules/artifact_pruner"
 require_relative "end_to_end_runner/molecules/simple_display_manager"
 require_relative "end_to_end_runner/molecules/progress_display_manager"
 require_relative "end_to_end_runner/molecules/suite_simple_display_manager"

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: ace-test-runner-e2e
 version: !ruby/object:Gem::Version
-  version: 0.29.8
+  version: 0.40.1
 platform: ruby
 authors:
 - Michal Czyz
 bindir: exe
 cert_chain: []
-date: 2026-04-05 00:00:00.000000000 Z
+date: 2026-04-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ace-support-cli
@@ -51,6 +51,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.9'
+- !ruby/object:Gem::Dependency
+  name: ace-support-test-helpers
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.14'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.14'
 - !ruby/object:Gem::Dependency
   name: ace-llm
   requirement: !ruby/object:Gem::Requirement
@@ -158,6 +172,7 @@ files:
 - handbook/workflow-instructions/e2e/run.wf.md
 - handbook/workflow-instructions/e2e/setup-sandbox.wf.md
 - lib/ace/test/end_to_end_runner.rb
+- lib/ace/test/end_to_end_runner/atoms/artifact_contract_validator.rb
 - lib/ace/test/end_to_end_runner/atoms/display_helpers.rb
 - lib/ace/test/end_to_end_runner/atoms/prompt_builder.rb
 - lib/ace/test/end_to_end_runner/atoms/result_parser.rb
@@ -172,15 +187,19 @@ files:
 - lib/ace/test/end_to_end_runner/models/test_result.rb
 - lib/ace/test/end_to_end_runner/models/test_scenario.rb
 - lib/ace/test/end_to_end_runner/molecules/affected_detector.rb
+- lib/ace/test/end_to_end_runner/molecules/artifact_pruner.rb
+- lib/ace/test/end_to_end_runner/molecules/bwrap_sandbox_backend.rb
 - lib/ace/test/end_to_end_runner/molecules/config_loader.rb
 - lib/ace/test/end_to_end_runner/molecules/failure_finder.rb
 - lib/ace/test/end_to_end_runner/molecules/fixture_copier.rb
+- lib/ace/test/end_to_end_runner/molecules/integration_runner.rb
 - lib/ace/test/end_to_end_runner/molecules/pipeline_executor.rb
 - lib/ace/test/end_to_end_runner/molecules/pipeline_prompt_bundler.rb
 - lib/ace/test/end_to_end_runner/molecules/pipeline_report_generator.rb
 - lib/ace/test/end_to_end_runner/molecules/pipeline_sandbox_builder.rb
 - lib/ace/test/end_to_end_runner/molecules/progress_display_manager.rb
 - lib/ace/test/end_to_end_runner/molecules/report_writer.rb
+- lib/ace/test/end_to_end_runner/molecules/sandbox_runtime_builder.rb
 - lib/ace/test/end_to_end_runner/molecules/scenario_loader.rb
 - lib/ace/test/end_to_end_runner/molecules/setup_executor.rb
 - lib/ace/test/end_to_end_runner/molecules/simple_display_manager.rb