ace-test-runner-e2e 0.29.8 → 0.40.1

data/lib/ace/test/end_to_end_runner/models/test_scenario.rb

@@ -12,7 +12,7 @@ module Ace
           attr_reader :test_id, :title, :area, :package, :priority, :duration,
             :requires, :file_path, :content, :timeout,
             :setup_steps, :dir_path, :fixture_path, :test_cases,
-            :tags, :tool_under_test, :sandbox_layout
+            :tags, :tool_under_test, :sandbox_layout, :sandbox_profile
 
           # @param test_id [String] Test identifier (e.g., "TS-LINT-001")
           # @param title [String] Test title
@@ -31,11 +31,12 @@ module Ace
           # @param tags [Array<String>] Scenario-level tags for discovery-time filtering
           # @param tool_under_test [String, nil] Primary tool under test
           # @param sandbox_layout [Hash] Declared sandbox artifact layout
+          # @param sandbox_profile [String] Sandbox bootstrap profile
           def initialize(test_id:, title:, area:, package:, file_path:, content:,
             priority: "medium", duration: "~5min", requires: {},
             setup_steps: [], dir_path: nil, fixture_path: nil, test_cases: [],
             timeout: nil, tags: [], tool_under_test: nil,
-            sandbox_layout: {})
+            sandbox_layout: {}, sandbox_profile: "ace-default")
             @test_id = test_id
             @title = title
             @area = area
@@ -53,6 +54,7 @@ module Ace
             @tags = tags
             @tool_under_test = tool_under_test
             @sandbox_layout = sandbox_layout
+            @sandbox_profile = sandbox_profile
           end
 
           # Generate short package name (without ace- prefix)

data/lib/ace/test/end_to_end_runner/molecules/affected_detector.rb

@@ -39,10 +39,15 @@ module Ace
           # @return [Array<String>] Changed file paths
           def get_changed_files(base_dir, ref)
             # Run git diff to get changed files using array-based command for security
-            output, status = Open3.capture2("git", "diff", "--name-only", ref, "--",
+            output, stderr, status = Open3.capture3("git", "diff", "--name-only", ref, "--",
               chdir: base_dir)
 
-            return [] unless status.success?
+            unless status.success?
+              message = stderr.to_s.strip
+              message = "git diff exited with status #{status.exitstatus}" if message.empty?
+              warn "Warning: git detection failed: #{message}"
+              return []
+            end
 
             output.lines.map(&:strip).reject(&:empty?)
           rescue => e

data/lib/ace/test/end_to_end_runner/molecules/artifact_pruner.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require "fileutils"
+
+module Ace
+  module Test
+    module EndToEndRunner
+      module Molecules
+        # Prunes stale E2E run artifacts while preserving suite reports and runtime cache.
+        class ArtifactPruner
+          ROOT_RELATIVE_PATH = File.join(".ace-local", "test-e2e")
+          PRESERVED_DIRECTORY_NAMES = %w[runtime-cache].freeze
+          PRESERVED_FILE_PATTERNS = [
+            /-suite-report\.md\z/,
+            /-suite-final-report\.md\z/
+          ].freeze
+
+          def prune(base_dir: Dir.pwd)
+            root = File.join(File.expand_path(base_dir), ROOT_RELATIVE_PATH)
+            return summary(root, [], []) unless Dir.exist?(root)
+
+            removed_paths = []
+            preserved_paths = []
+
+            Dir.children(root).sort.each do |entry|
+              path = File.join(root, entry)
+              if preserve_entry?(entry, path)
+                preserved_paths << path
+              else
+                FileUtils.rm_rf(path)
+                removed_paths << path
+              end
+            end
+
+            summary(root, removed_paths, preserved_paths)
+          end
+
+          private
+
+          def preserve_entry?(entry, path)
+            return true if File.directory?(path) && PRESERVED_DIRECTORY_NAMES.include?(entry)
+            return false unless File.file?(path)
+
+            PRESERVED_FILE_PATTERNS.any? { |pattern| pattern.match?(entry) }
+          end
+
+          def summary(root, removed_paths, preserved_paths)
+            {
+              root: root,
+              root_display: ROOT_RELATIVE_PATH,
+              removed_paths: removed_paths,
+              preserved_paths: preserved_paths,
+              deleted_count: removed_paths.length,
+              preserved_count: preserved_paths.length
+            }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ace/test/end_to_end_runner/molecules/bwrap_sandbox_backend.rb

@@ -0,0 +1,271 @@
+# frozen_string_literal: true
+
+require "etc"
+require "fileutils"
+require "open3"
+
+module Ace
+  module Test
+    module EndToEndRunner
+      module Molecules
+        # Wraps subprocesses in a lightweight bubblewrap sandbox on Linux.
+        class BwrapSandboxBackend
+          BUNDLER_ENV_PREFIXES = %w[BUNDLE BUNDLER].freeze
+          STRIPPED_ENV_KEYS = %w[RUBYOPT RUBYLIB].freeze
+          PROVIDER_HOME_MOUNTS = [
+            [".claude", ".claude"],
+            [".codex", ".codex"],
+            [".gemini", ".gemini"],
+            [".pi", ".pi"],
+            [".local/share/opencode", ".local/share/opencode"]
+          ].freeze
+          DEFAULT_SYSTEM_MOUNTS = %w[/usr /bin /sbin /lib /lib64 /etc /opt /var/lib/flatpak/exports].freeze
+
+          attr_reader :sandbox_root
+
+          def self.supported?
+            Gem.win_platform? == false && RUBY_PLATFORM.include?("linux")
+          end
+
+          def self.available?(bwrap_path: "bwrap")
+            return false unless supported?
+
+            system("which", bwrap_path, out: File::NULL, err: File::NULL)
+          end
+
+          def initialize(sandbox_root:, source_root: nil, bwrap_path: "bwrap", outer_env: nil)
+            @sandbox_root = File.expand_path(sandbox_root)
+            @source_root = source_root ? File.expand_path(source_root) : infer_source_root
+            @bwrap_path = bwrap_path
+            @outer_env = sanitized_outer_env(outer_env)
+          end
+
+          def ensure_available!
+            return if self.class.available?(bwrap_path: @bwrap_path)
+
+            raise "bubblewrap is required for Linux E2E sandboxing but '#{@bwrap_path}' is not available"
+          end
+
+          def prepared_env(base_env = {})
+            env = stringify_keys(base_env)
+            STRIPPED_ENV_KEYS.each { |key| env.delete(key) }
+            env["PROJECT_ROOT_PATH"] = @sandbox_root
+            env["ACE_E2E_SOURCE_ROOT"] ||= @source_root if @source_root
+            env["HOME"] = sandbox_home
+            env["TMPDIR"] = sandbox_tmp
+            env["XDG_RUNTIME_DIR"] = sandbox_runtime_dir
+            env["TMUX_TMPDIR"] ||= sandbox_runtime_dir
+            env["BUNDLE_GEMFILE"] ||= File.join(@sandbox_root, ".ace-local", "e2e-runtime", "Gemfile")
+            env["ACE_CONFIG_PATH"] ||= File.join(@sandbox_root, ".ace")
+            env["BUNDLE_APP_CONFIG"] ||= bundler_app_config
+            env["BUNDLE_USER_HOME"] ||= bundler_home
+            env["BUNDLE_USER_CACHE"] ||= bundler_cache
+            env["BUNDLE_USER_CONFIG"] ||= bundler_user_config
+            env["PATH"] ||= ENV["PATH"].to_s
+            env
+          end
+
+          def wrap_command(cmd, chdir:, env: {})
+            command_prefix(chdir: chdir, env: env) + Array(cmd)
+          end
+
+          def command_prefix(chdir:, env: {})
+            ensure_available!
+            merged_env = prepared_env(env)
+            ensure_runtime_dirs(merged_env)
+
+            args = [
+              @bwrap_path,
+              "--clearenv",
+              "--die-with-parent",
+              "--new-session",
+              "--proc", "/proc",
+              "--dev-bind", "/dev", "/dev",
+              "--tmpfs", "/tmp",
+              "--tmpfs", home_root,
+              "--dir", merged_env.fetch("HOME"),
+              "--bind", merged_env.fetch("HOME"), merged_env.fetch("HOME")
+            ]
+
+            host_mounts(merged_env.fetch("PATH")).each do |mount|
+              append_bind(args, mount, mount, read_only: true)
+            end
+            append_bind(args, @source_root, @source_root, read_only: true) if @source_root
+            append_bind(args, @sandbox_root, @sandbox_root, read_only: false)
+            append_bind(args, support_root, support_root, read_only: false)
+            ruby_root = merged_env["ACE_E2E_SANDBOX_RUBY_ROOT"].to_s
+            append_bind(args, ruby_root, ruby_root, read_only: true) if !ruby_root.empty? && File.exist?(ruby_root)
+            bind_provider_homes(args, merged_env.fetch("HOME"))
+
+            setenvs = {
+              "HOME" => merged_env.fetch("HOME"),
+              "TMPDIR" => merged_env.fetch("TMPDIR"),
+              "XDG_RUNTIME_DIR" => merged_env.fetch("XDG_RUNTIME_DIR"),
+              "TMUX_TMPDIR" => merged_env.fetch("TMUX_TMPDIR"),
+              "PATH" => merged_env.fetch("PATH"),
+              "PROJECT_ROOT_PATH" => merged_env.fetch("PROJECT_ROOT_PATH")
+            }
+            setenvs["ACE_E2E_SOURCE_ROOT"] = merged_env["ACE_E2E_SOURCE_ROOT"] if merged_env["ACE_E2E_SOURCE_ROOT"]
+            setenvs["ACE_E2E_SANDBOX_RUBY_ROOT"] = ruby_root unless ruby_root.empty?
+            setenvs["ACE_TMUX_SESSION"] = merged_env["ACE_TMUX_SESSION"] if merged_env["ACE_TMUX_SESSION"]
+
+            merged_env.each do |key, value|
+              next if setenvs.key?(key)
+              next if value.nil?
+
+              setenvs[key] = value.to_s
+            end
+
+            setenvs.each do |key, value|
+              args.concat(["--setenv", key, value])
+            end
+
+            args.concat(["--chdir", File.expand_path(chdir), "--"])
+            args
+          end
+
+          def capture3(cmd, chdir:, env: {}, stdin_data: nil)
+            Open3.capture3(
+              @outer_env,
+              *wrap_command(cmd, chdir: chdir, env: env),
+              chdir: "/",
+              stdin_data: stdin_data
+            )
+          end
+
+          def exec(cmd, chdir:, env: {})
+            Kernel.exec(@outer_env, *wrap_command(cmd, chdir: chdir, env: env))
+          end
+
+          private
+
+          def sandbox_home
+            File.join(support_root, "home")
+          end
+
+          def sandbox_tmp
+            File.join(support_root, "tmp")
+          end
+
+          def sandbox_runtime_dir
+            File.join(support_root, "runtime")
+          end
+
+          def bundler_root
+            File.join(support_root, "bundler")
+          end
+
+          def bundler_home
+            File.join(bundler_root, "home")
+          end
+
+          def bundler_cache
+            File.join(bundler_root, "cache")
+          end
+
+          def bundler_user_config
+            File.join(bundler_root, "config")
+          end
+
+          def bundler_app_config
+            File.join(bundler_root, "app-config")
+          end
+
+          def home_root
+            File.dirname(actual_home)
+          end
+
+          def actual_home
+            File.expand_path(ENV.fetch("HOME"))
+          end
+
+          def infer_source_root
+            env_root = ENV["ACE_E2E_SOURCE_ROOT"].to_s.strip
+            return File.expand_path(env_root) unless env_root.empty?
+
+            File.expand_path(Dir.pwd)
+          end
+
+          def support_root
+            "#{@sandbox_root}.support"
+          end
+
+          def host_mounts(path_value)
+            mounts = DEFAULT_SYSTEM_MOUNTS.select { |path| File.exist?(path) }
+            path_value.to_s.split(File::PATH_SEPARATOR).each do |entry|
+              next if entry.to_s.strip.empty?
+
+              expanded = File.expand_path(entry)
+              next unless File.exist?(expanded)
+
+              mounts << expanded
+              mounts << mise_install_root(expanded) if expanded.include?("/.local/share/mise/installs/")
+            end
+
+            mounts.compact.uniq.sort
+          end
+
+          def mise_install_root(path)
+            match = path.match(%r{\A(.*/\.local/share/mise/installs/[^/]+/[^/]+)})
+            match && match[1]
+          end
+
+          def ensure_runtime_dirs(env)
+            [
+              @sandbox_root,
+              support_root,
+              env.fetch("HOME"),
+              env.fetch("TMPDIR"),
+              env.fetch("XDG_RUNTIME_DIR"),
+              env.fetch("BUNDLE_APP_CONFIG"),
+              env.fetch("BUNDLE_USER_HOME"),
+              env.fetch("BUNDLE_USER_CACHE")
+            ].each { |path| FileUtils.mkdir_p(path) }
+            FileUtils.chmod(0o700, env.fetch("XDG_RUNTIME_DIR")) if File.exist?(env.fetch("XDG_RUNTIME_DIR"))
+          end
+
+          def bind_provider_homes(args, sandbox_home_dir)
+            PROVIDER_HOME_MOUNTS.each do |source_suffix, target_suffix|
+              source = File.join(actual_home, source_suffix)
+              next unless File.exist?(source)
+
+              target = File.join(sandbox_home_dir, target_suffix)
+              FileUtils.mkdir_p(File.dirname(target))
+              FileUtils.mkdir_p(target) if File.directory?(source)
+              append_bind(args, source, target, read_only: false)
+            end
+          end
+
+          def append_bind(args, source, target, read_only:)
+            source = File.expand_path(source)
+            target = File.expand_path(target)
+            FileUtils.mkdir_p(File.dirname(target))
+            FileUtils.mkdir_p(target) if File.directory?(source)
+            args.concat([read_only ? "--ro-bind" : "--bind", source, target])
+          end
+
+          def stringify_keys(hash)
+            hash.each_with_object({}) { |(key, value), acc| acc[key.to_s] = value }
+          end
+
+          def sanitized_process_env(base_env)
+            stringify_keys(base_env).each_with_object({}) do |(key, value), env|
+              next if strip_env_key?(key)
+
+              env[key] = value
+            end
+          end
+
+          def sanitized_outer_env(outer_env)
+            path = stringify_keys(outer_env || {"PATH" => ENV["PATH"].to_s})["PATH"]
+            {"PATH" => path.to_s.empty? ? ENV["PATH"].to_s : path.to_s}
+          end
+
+          def strip_env_key?(key)
+            STRIPPED_ENV_KEYS.include?(key) || BUNDLER_ENV_PREFIXES.any? { |prefix| key.start_with?(prefix) }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ace/test/end_to_end_runner/molecules/config_loader.rb

@@ -23,7 +23,22 @@ module Ace
           # @return [String] Default provider from config
           def self.default_provider
             config = load
-            config.dig("execution", "provider") || "claude:sonnet"
+            config.dig("execution", "runner_provider") ||
+              config.dig("execution", "provider") || "claude:sonnet"
+          end
+
+          # @return [String] Default runner provider from config
+          def self.default_runner_provider
+            config = load
+            config.dig("execution", "runner_provider") ||
+              config.dig("execution", "provider") || "claude:sonnet"
+          end
+
+          # @return [String] Default verifier provider from config
+          def self.default_verifier_provider
+            config = load
+            config.dig("execution", "verifier_provider") ||
+              config.dig("execution", "provider") || "claude:sonnet"
           end
 
           # @return [Integer] Default timeout from config
@@ -38,6 +53,18 @@ module Ace
             config.dig("execution", "parallel") || 3
           end
 
+          # @return [String] Default sandbox bootstrap profile
+          def self.default_sandbox_profile
+            config = load
+            config.dig("sandbox", "profile") || "ace-default"
+          end
+
+          # @return [String] Dedicated Ruby version for sandbox runtime
+          def self.default_sandbox_ruby_version
+            config = load
+            config.dig("sandbox", "ruby_version") || "3.4.9"
+          end
+
           # @return [Array<String>] CLI provider names
           def self.cli_providers
             config = load

data/lib/ace/test/end_to_end_runner/molecules/integration_runner.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "open3"
+require "shellwords"
+require "ace/test_support/sandbox_package_copy"
+
+module Ace
+  module Test
+    module EndToEndRunner
+      module Molecules
+        # Runs deterministic preflight tests inside a sandboxed package copy.
+        class IntegrationRunner
+          def initialize(base_dir: Dir.pwd, package_copy: nil)
+            @base_dir = File.expand_path(base_dir)
+            @package_copy = package_copy || Ace::TestSupport::SandboxPackageCopy.new(source_root: @base_dir)
+          end
+
+          def run(package:, files:, timestamp:, output: $stdout)
+            return nil if files.nil? || files.empty?
+
+            started_at = Time.now
+            sandbox_root = File.join(@base_dir, ".ace-local", "test-e2e", "#{timestamp}-#{package}-preflight")
+            FileUtils.mkdir_p(sandbox_root)
+
+            package_copy_result = @package_copy.prepare(package_name: package, sandbox_root: sandbox_root)
+            package_root = resolve_package_root(sandbox_root, package)
+            env = package_copy_result[:env] || {}
+
+            test_cases = files.map do |file|
+              run_file(package_root, file, env, output)
+            end
+
+            status = if test_cases.any? { |tc| tc[:status] == "error" }
+              "error"
+            elsif test_cases.any? { |tc| tc[:status] == "fail" }
+              "fail"
+            else
+              "pass"
+            end
+
+            Models::TestResult.new(
+              test_id: "PREFLIGHT",
+              status: status,
+              test_cases: test_cases,
+              summary: preflight_summary(status, test_cases),
+              started_at: started_at,
+              completed_at: Time.now,
+              metadata: {
+                phase: "preflight",
+                package: package,
+                sandbox_root: sandbox_root
+              }
+            )
+          end
+
+          private
+
+          def run_file(package_root, file, env, output)
+            relative = file.sub(%r{\A#{Regexp.escape(@base_dir)}/?}, "")
+            package_relative = relative.sub(%r{\A[^/]+/}, "")
+
+            stdout, stderr, status = Open3.capture3(
+              env,
+              "ace-test",
+              package_relative,
+              chdir: package_root
+            )
+
+            output.puts "Preflight: #{package_relative} (#{status.success? ? "pass" : "fail"})"
+
+            {
+              id: package_relative,
+              description: package_relative,
+              status: status.success? ? "pass" : "fail",
+              actual: stdout,
+              notes: stderr,
+              metadata: {
+                phase: "preflight",
+                exit_status: status.exitstatus,
+                command: Shellwords.join(["ace-test", package_relative])
+              }
+            }
+          rescue StandardError => e
+            output.puts "Preflight: #{package_relative} (error)"
+
+            {
+              id: package_relative,
+              description: package_relative,
+              status: "error",
+              actual: "",
+              notes: e.message,
+              metadata: {
+                phase: "preflight",
+                command: Shellwords.join(["ace-test", package_relative])
+              }
+            }
+          end
+
+          def resolve_package_root(sandbox_root, package)
+            candidate = File.join(sandbox_root, package)
+            return candidate if Dir.exist?(candidate)
+
+            sandbox_root
+          end
+
+          def preflight_summary(status, test_cases)
+            passed = test_cases.count { |tc| tc[:status] == "pass" }
+            total = test_cases.size
+            prefix =
+              case status
+              when "pass" then "Preflight passed"
+              when "fail" then "Preflight failed"
+              else "Preflight errored"
+              end
+            "#{prefix}: #{passed}/#{total} files passed"
+          end
+        end
+      end
+    end
+  end
+end