ace-test-runner-e2e 0.29.6 → 0.38.11

data/lib/ace/test/end_to_end_runner/molecules/sandbox_runtime_builder.rb

@@ -0,0 +1,282 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "open3"
+
+module Ace
+  module Test
+    module EndToEndRunner
+      module Molecules
+        # Builds a sandbox-local Ruby/Bundler runtime for E2E execution.
+        class SandboxRuntimeBuilder
+          DEFAULT_RUBY_VERSION = "3.4.9"
+          RESERVED_ENV_KEYS = %w[
+            PROJECT_ROOT_PATH
+            ACE_E2E_SOURCE_ROOT
+            ACE_CONFIG_PATH
+            BUNDLE_GEMFILE
+            BUNDLE_APP_CONFIG
+            BUNDLE_USER_HOME
+            BUNDLE_USER_CACHE
+            BUNDLE_USER_CONFIG
+            BUNDLE_PATH
+            BUNDLE_BIN
+            BUNDLE_DISABLE_SHARED_GEMS
+            BUNDLER_VERSION
+            GEM_HOME
+            GEM_PATH
+            RUBYOPT
+            RUBYLIB
+          ].freeze
+
+          def initialize(source_root:, ruby_version: nil, command_runner: nil)
+            @source_root = File.expand_path(source_root)
+            @ruby_version = (ruby_version || DEFAULT_RUBY_VERSION).to_s.strip
+            @command_runner = command_runner || method(:capture3)
+          end
+
+          def prepare(sandbox_root:, env: {}, tool_names: nil)
+            sandbox_root = File.expand_path(sandbox_root)
+            runtime_root = File.join(sandbox_root, ".ace-local", "e2e-runtime")
+            FileUtils.mkdir_p(runtime_root)
+
+            runtime_env = build_runtime_env(sandbox_root, runtime_root, env)
+            ensure_runtime_dirs(runtime_env)
+            write_runtime_gemfile(runtime_root)
+            write_command_shims(runtime_root, tool_names)
+            install_runtime!(runtime_root, runtime_env)
+
+            {
+              runtime_root: runtime_root,
+              env: runtime_env
+            }
+          end
+
+          private
+
+          def capture3(env, *cmd, chdir:)
+            Open3.capture3(env, *cmd, chdir: chdir, unsetenv_others: true)
+          end
+
+          def build_runtime_env(sandbox_root, runtime_root, env)
+            merged = stringify_keys(env).reject { |key, _value| RESERVED_ENV_KEYS.include?(key) }
+            bundler_root = File.join(runtime_root, "bundler")
+            gem_root = File.join(runtime_root, "gems")
+            bin_root = File.join(runtime_root, "bin")
+            path = merged["PATH"].to_s
+            path = ENV["PATH"].to_s if path.empty?
+
+            merged.merge(
+              "PROJECT_ROOT_PATH" => sandbox_root,
+              "ACE_E2E_SOURCE_ROOT" => @source_root,
+              "ACE_CONFIG_PATH" => File.join(sandbox_root, ".ace"),
+              "ACE_E2E_SANDBOX_RUNTIME_ROOT" => runtime_root,
+              "ACE_E2E_SANDBOX_RUBY_VERSION" => @ruby_version,
+              "ACE_E2E_SANDBOX_RUBY_ROOT" => (@ruby_version.empty? ? "" : resolve_ruby_install_path!),
+              "BUNDLE_GEMFILE" => File.join(runtime_root, "Gemfile"),
+              "BUNDLE_APP_CONFIG" => File.join(bundler_root, "app-config"),
+              "BUNDLE_USER_HOME" => File.join(bundler_root, "home"),
+              "BUNDLE_USER_CACHE" => File.join(bundler_root, "cache"),
+              "BUNDLE_USER_CONFIG" => File.join(bundler_root, "config"),
+              "BUNDLE_PATH" => gem_root,
+              "BUNDLE_DISABLE_SHARED_GEMS" => "true",
+              "BUNDLE_WITHOUT" => "",
+              "GEM_HOME" => gem_root,
+              "GEM_PATH" => gem_root,
+              "PATH" => [bin_root, path].reject(&:empty?).join(File::PATH_SEPARATOR)
+            )
+          end
+
+          def ensure_runtime_dirs(env)
+            [
+              env.fetch("ACE_CONFIG_PATH"),
+              env.fetch("BUNDLE_APP_CONFIG"),
+              env.fetch("BUNDLE_USER_HOME"),
+              env.fetch("BUNDLE_USER_CACHE"),
+              File.dirname(env.fetch("BUNDLE_USER_CONFIG")),
+              env.fetch("BUNDLE_PATH"),
+              env.fetch("GEM_HOME"),
+              File.join(env.fetch("ACE_E2E_SANDBOX_RUNTIME_ROOT"), "bin")
+            ].each do |path|
+              FileUtils.mkdir_p(path)
+            end
+          end
+
+          def write_runtime_gemfile(runtime_root)
+            source_gemfile = File.join(@source_root, "Gemfile")
+            content = File.read(source_gemfile)
+            rewritten = content.gsub(/(path:\s*["'])([^"']+)(["'])/) do
+              prefix = Regexp.last_match(1)
+              relative_path = Regexp.last_match(2)
+              suffix = Regexp.last_match(3)
+              absolute_path = File.expand_path(relative_path, @source_root)
+              "#{prefix}#{absolute_path}#{suffix}"
+            end
+            File.write(File.join(runtime_root, "Gemfile"), rewritten)
+          end
+
+          def write_command_shims(runtime_root, tool_names)
+            bin_root = File.join(runtime_root, "bin")
+            requested = normalize_tool_names(tool_names)
+            executable_map = discover_executable_map
+            names = executable_map.keys | requested
+
+            names.each do |tool_name|
+              source_exec = executable_map[tool_name]
+              next unless source_exec
+
+              shim_path = File.join(bin_root, tool_name)
+              shim_body = if @ruby_version.empty?
+                <<~SH
+                  #!/usr/bin/env bash
+                  set -euo pipefail
+                  exec ruby -rbundler/setup "#{source_exec}" "$@"
+                SH
+              else
+                ruby_exec = ruby_executable_path
+                <<~SH
+                  #!/usr/bin/env bash
+                  set -euo pipefail
+                  exec "#{ruby_exec}" -rbundler/setup "#{source_exec}" "$@"
+                SH
+              end
+              File.write(shim_path, shim_body)
+              FileUtils.chmod(0o755, shim_path)
+            end
+          end
+
+          def discover_executable_map
+            @discover_executable_map ||= begin
+              paths = Dir.glob(File.join(@source_root, "bin", "ace-*")).sort
+              Dir.glob(File.join(@source_root, "ace-*", "exe", "ace-*")).sort.each do |path|
+                basename = File.basename(path)
+                next if paths.any? { |candidate| File.basename(candidate) == basename }
+
+                paths << path
+              end
+
+              paths.each_with_object({}) do |path, map|
+                map[File.basename(path)] = path
+              end
+            end
+          end
+
+          def normalize_tool_names(tool_names)
+            Array(tool_names)
+              .flat_map { |entry| entry.to_s.split(",") }
+              .map(&:strip)
+              .reject(&:empty?)
+              .uniq
+          end
+
+          def install_runtime!(runtime_root, env)
+            marker_path = File.join(runtime_root, ".bootstrapped")
+            return if File.exist?(marker_path)
+
+            ensure_ruby_available!
+            ensure_no_global_ace_gems!
+
+            stdout, stderr, status = @command_runner.call(
+              install_env(env),
+              *runtime_command(%w[bundle install]),
+              chdir: runtime_root
+            )
+            return File.write(marker_path, "ok\n") if status.success?
+
+            raise [
+              "Sandbox bundle install failed for Ruby #{@ruby_version}",
+              stdout.to_s.strip,
+              stderr.to_s.strip
+            ].reject(&:empty?).join("\n")
+          end
+
+          def ensure_ruby_available!
+            resolve_ruby_install_path!
+          rescue RuntimeError => e
+            raise e
+
+          end
+
+          def resolve_ruby_install_path!
+            return @ruby_install_path if defined?(@ruby_install_path) && @ruby_install_path
+
+            stdout, stderr, status = @command_runner.call(
+              minimal_host_env,
+              "mise", "where", "ruby@#{@ruby_version}",
+              chdir: @source_root
+            )
+            if status.success?
+              @ruby_install_path = stdout.to_s.strip
+              return @ruby_install_path unless @ruby_install_path.empty?
+            end
+
+            raise [
+              "Dedicated sandbox Ruby #{ruby_label} is not available.",
+              "Install it with: mise install ruby@#{@ruby_version}",
+              stderr.to_s.strip
+            ].reject(&:empty?).join("\n")
+          end
+
+          def ensure_no_global_ace_gems!
+            script = <<~RUBY
+              names = Gem::Specification.map(&:name).grep(/^ace-/)
+              puts names.join("\\n") unless names.empty?
+              exit(names.empty? ? 0 : 42)
+            RUBY
+
+            stdout, stderr, status = @command_runner.call(
+              minimal_host_env,
+              *runtime_command(["ruby", "-e", script]),
+              chdir: @source_root
+            )
+            return if status.success?
+
+            raise [
+              "Dedicated sandbox Ruby #{ruby_label} already exposes ace-* gems globally.",
+              stdout.to_s.strip,
+              stderr.to_s.strip
+            ].reject(&:empty?).join("\n")
+          end
+
+          def runtime_command(cmd)
+            return cmd if @ruby_version.empty?
+            ruby = ruby_executable_path
+            command = Array(cmd)
+            executable = command.shift
+
+            case executable
+            when "ruby"
+              [ruby] + command
+            when "bundle", "gem"
+              [ruby, "-S", executable] + command
+            else
+              [ruby, "-S", executable] + command
+            end
+          end
+
+          def ruby_executable_path
+            File.join(resolve_ruby_install_path!, "bin", "ruby")
+          end
+
+          def minimal_host_env
+            {"HOME" => ENV["HOME"].to_s, "PATH" => ENV["PATH"].to_s}
+          end
+
+          def install_env(env)
+            minimal_host_env.merge(env).merge("PATH" => env.fetch("PATH"))
+          end
+
+          def ruby_label
+            @ruby_version.empty? ? "(default ruby)" : @ruby_version
+          end
+
+          def stringify_keys(hash)
+            hash.each_with_object({}) do |(key, value), acc|
+              acc[key.to_s] = value
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ace/test/end_to_end_runner/molecules/scenario_loader.rb

@@ -47,7 +47,8 @@ module Ace
               test_cases: test_cases,
               tags: parse_tags(frontmatter["tags"]),
               tool_under_test: frontmatter["tool-under-test"],
-              sandbox_layout: frontmatter["sandbox-layout"] || {}
+              sandbox_layout: frontmatter["sandbox-layout"] || {},
+              sandbox_profile: parse_sandbox_profile(frontmatter["sandbox-profile"], yml_path)
             )
           end
 
@@ -154,6 +155,13 @@ module Ace
           def parse_standalone_test_case(tc_id, runner_file, verify_file)
             runner_content = File.read(runner_file)
             verify_content = File.read(verify_file)
+            scenario_dir = File.dirname(runner_file)
+            declared_artifacts, optional_artifacts = declared_artifacts_for(
+              scenario_dir,
+              tc_id,
+              runner_content,
+              verify_content
+            )
 
             Models::TestCase.new(
               tc_id: tc_id,
@@ -161,7 +169,9 @@ module Ace
               content: build_standalone_content(runner_content, verify_content),
               file_path: File.expand_path(runner_file),
               pending: nil,
-              goal_format: "standalone"
+              goal_format: "standalone",
+              declared_artifacts: declared_artifacts,
+              optional_artifacts: optional_artifacts
             )
           end
 
@@ -223,6 +233,18 @@ module Ace
             tags.map(&:to_s).map(&:strip).reject(&:empty?).map(&:downcase)
           end
 
+          def parse_sandbox_profile(raw_profile, source_path)
+            profile = raw_profile.to_s.strip
+            return Molecules::ConfigLoader.default_sandbox_profile if profile.empty?
+
+            allowed = %w[ace-default bundle-only custom]
+            return profile if allowed.include?(profile)
+
+            raise ArgumentError,
+              "Invalid sandbox-profile in #{source_path}: #{profile.inspect}. " \
+              "Allowed values: #{allowed.join(", ")}"
+          end
+
           # Detect fixtures directory if it exists
           #
           # @param scenario_dir [String] Path to the scenario directory
@@ -232,6 +254,66 @@ module Ace
             Dir.exist?(path) ? File.expand_path(path) : nil
           end
 
+          def declared_artifacts_for(scenario_dir, tc_id, runner_content, verify_content)
+            scenario_frontmatter = parse_scenario_yml(File.join(scenario_dir, "scenario.yml"))
+            required_artifacts = []
+            optional_artifacts = []
+
+            required_artifacts.concat(
+              Array((scenario_frontmatter["sandbox-layout"] || {}).keys).select do |path|
+                declared_artifact_matches_tc?(path, tc_id)
+              end
+            )
+
+            [runner_content, verify_content].each do |content|
+              extract_declared_artifacts(content).each do |entry|
+                if entry[:optional]
+                  optional_artifacts << entry[:path]
+                else
+                  required_artifacts << entry[:path]
+                end
+              end
+            end
+
+            required = required_artifacts.map { |path| normalize_declared_artifact(path) }.compact.uniq
+            optional = optional_artifacts.map { |path| normalize_declared_artifact(path) }.compact.uniq
+            optional = optional.reject { |path| required.include?(path) }
+
+            [required.sort, optional.sort]
+          end
+
+          def declared_artifact_matches_tc?(path, tc_id)
+            artifact_index = extract_declared_artifact_index(path)
+            return true if artifact_index.nil?
+
+            tc_index = extract_tc_index(tc_id)
+            tc_index && artifact_index == tc_index
+          end
+
+          def extract_declared_artifact_index(path)
+            match = path.to_s.match(%r{\Aresults/tc/(\d{1,3})(?:/|$)})
+            match ? match[1].to_i : nil
+          end
+
+          def extract_tc_index(tc_id)
+            match = tc_id.to_s.match(/\ATC-(\d+)/i)
+            match ? match[1].to_i : nil
+          end
+
+          def extract_declared_artifacts(markdown)
+            markdown.to_s.scan(%r{(?:`|"|')?(results/tc/\d{2}/[^\s`)"']+|results/tc/\d{2}/)(?:`|"|')?(\s*\(optional\))?}i).map do |match|
+              path, optional = match
+              {path: path, optional: !optional.to_s.empty?}
+            end
+          end
+
+          def normalize_declared_artifact(path)
+            value = path.to_s.strip
+            return nil unless value.start_with?("results/tc/")
+
+            value.sub(%r{/+\z}, "")
+          end
+
           # Infer package name from scenario directory path
           #
           # @param scenario_dir [String] Path to scenario directory
@@ -240,9 +322,7 @@ module Ace
             # Expected path: {package}/test/e2e/TS-{AREA}-{NNN}-{slug}/
             parts = File.expand_path(scenario_dir).split("/")
             parts.each_with_index do |part, idx|
-              next unless part == "test" && idx > 0 && parts[idx + 1] == "e2e"
-
-              return parts[idx - 1]
+              return parts[idx - 1] if part == "test" && idx > 0 && parts[idx + 1] == "e2e"
             end
 
             "unknown"

data/lib/ace/test/end_to_end_runner/molecules/setup_executor.rb

@@ -17,6 +17,20 @@ module Ace
         # Note: This is a Molecule because it performs filesystem I/O and
         # system calls via Open3 and FileUtils.
         class SetupExecutor
+          AMBIENT_TMUX_ENV_VARS = %w[TMUX TMUX_PANE].freeze
+          BUNDLER_ENV_PREFIXES = %w[BUNDLE BUNDLER].freeze
+          STRIPPED_ENV_KEYS = %w[RUBYOPT RUBYLIB].freeze
+          RESERVED_ENV_KEYS = Molecules::SandboxRuntimeBuilder::RESERVED_ENV_KEYS + %w[
+            PATH HOME TMPDIR XDG_RUNTIME_DIR TMUX_TMPDIR ACE_TMUX_SESSION
+          ]
+
+          def initialize(command_runner: nil, system_runner: nil, time_source: nil, sandbox_backend: nil)
+            @command_runner = command_runner || method(:capture3)
+            @system_runner = system_runner || method(:system)
+            @time_source = time_source || -> { Time.now.to_i }
+            @sandbox_backend = sandbox_backend
+          end
+
           # Execute all setup steps in a sandbox directory
           #
           # @param setup_steps [Array] Setup steps from scenario.yml
@@ -25,29 +39,54 @@ module Ace
           # @param scenario_name [String, nil] Test ID for tmux session naming (e.g., "TS-OVERSEER-001")
           # @param run_id [String, nil] Unique run ID for deterministic tmux session naming
           # @return [Hash] Result with :success, :steps_completed, :error, :env, :tmux_session keys
-          def execute(setup_steps:, sandbox_dir:, fixture_source: nil, scenario_name: nil, run_id: nil)
+          def execute(setup_steps:, sandbox_dir:, fixture_source: nil, scenario_name: nil, run_id: nil, initial_env: {})
             FileUtils.mkdir_p(sandbox_dir)
-            env = {}
+            env = if @sandbox_backend
+              @sandbox_backend.prepared_env(initial_env.dup)
+            else
+              initial_env.dup
+            end
             steps_completed = 0
             @tmux_session = nil
             @scenario_name = scenario_name
             @run_id = run_id
+            @teardown_env = nil
 
             setup_steps.each do |step|
               execute_step(step, sandbox_dir, env, fixture_source)
               steps_completed += 1
             end
 
-            {success: true, steps_completed: steps_completed, error: nil, env: env, tmux_session: @tmux_session}
+            {
+              success: true,
+              steps_completed: steps_completed,
+              error: nil,
+              env: merged_environment(env),
+              tmux_session: @tmux_session
+            }
           rescue => e
-            {success: false, steps_completed: steps_completed, error: e.message, env: env, tmux_session: @tmux_session}
+            {
+              success: false,
+              steps_completed: steps_completed,
+              error: e.message,
+              env: merged_environment(env),
+              tmux_session: @tmux_session
+            }
           end
 
           # Clean up resources created during setup (e.g. tmux session)
           def teardown
             return unless @tmux_session
 
-            system("tmux", "kill-session", "-t", @tmux_session, out: File::NULL, err: File::NULL)
+            if @sandbox_backend
+              @sandbox_backend.capture3(
+                ["tmux", "kill-session", "-t", @tmux_session],
+                chdir: @teardown_env&.fetch("PROJECT_ROOT_PATH", Dir.pwd) || Dir.pwd,
+                env: @teardown_env || {}
+              )
+            else
+              @system_runner.call("tmux", "kill-session", "-t", @tmux_session, out: File::NULL, err: File::NULL)
+            end
             @tmux_session = nil
           end
 
@@ -99,13 +138,23 @@ module Ace
             session_name = if name_source == "run-id" && @run_id && !@run_id.to_s.empty?
               @run_id
             else
-              @scenario_name ? "#{@scenario_name}-e2e" : "ace-e2e-#{Time.now.to_i}"
+              @scenario_name ? "#{@scenario_name}-e2e" : "ace-e2e-#{@time_source.call}"
+            end
+            tmux_env = merged_environment(env).merge("TMUX_TMPDIR" => env["TMUX_TMPDIR"].to_s.empty? ? nil : env["TMUX_TMPDIR"])
+            if @sandbox_backend
+              _stdout, stderr, status = @sandbox_backend.capture3(
+                ["tmux", "new-session", "-d", "-s", session_name],
+                chdir: env["PROJECT_ROOT_PATH"] || Dir.pwd,
+                env: tmux_env
+              )
+            else
+              _stdout, stderr, status = @command_runner.call(tmux_env, "tmux", "new-session", "-d", "-s", session_name)
             end
-            _stdout, stderr, status = Open3.capture3("tmux", "new-session", "-d", "-s", session_name)
             raise "Failed to create tmux session '#{session_name}': #{stderr.strip}" unless status.success?
 
             @tmux_session = session_name
             env["ACE_TMUX_SESSION"] = session_name
+            @teardown_env = merged_environment(env)
           end
 
           # Initialize a git repo with test user config
@@ -123,20 +172,25 @@ module Ace
           end
 
           # Execute a shell command in the sandbox
-          # NOTE: Uses shell invocation (bash -lc) intentionally to support
-          # shell operators (&&, |, >) in scenario.yml setup steps. Commands originate from
+          # NOTE: Uses shell invocation intentionally to support shell operators
+          # (&&, |, >) in scenario.yml setup steps. Commands originate from
           # committed scenario.yml files, not user input, so shell injection risk is mitigated.
+          # We explicitly disable profile/rc loading to keep sandbox env authoritative.
           def handle_run(command, sandbox_dir, env)
             full_env = merged_environment(env)
-            # Re-export env vars after profile sourcing to protect against
-            # mise's shell hook clobbering.
+            # Re-export env vars inside the command to keep explicit sandbox
+            # values authoritative across compound shell expressions.
             export_vars = env.dup
             %w[PROJECT_ROOT_PATH].each do |key|
               export_vars[key] ||= ENV[key] if ENV[key]
             end
             exports = export_vars.map { |k, v| "export #{k}=#{Shellwords.shellescape(v.to_s)}" }.join("; ")
             wrapped = exports.empty? ? command : "#{exports}; #{command}"
-            stdout, stderr, status = Open3.capture3(full_env, "bash", "-lc", wrapped, chdir: sandbox_dir)
+            stdout, stderr, status = if @sandbox_backend
+              @sandbox_backend.capture3(["bash", "--noprofile", "--norc", "-c", wrapped], chdir: sandbox_dir, env: full_env)
+            else
+              Open3.capture3(full_env, "bash", "--noprofile", "--norc", "-c", wrapped, chdir: sandbox_dir)
+            end
 
             unless status.success?
               raise "Setup step 'run' failed (exit #{status.exitstatus}): #{command}\n#{stderr}"
@@ -154,7 +208,12 @@ module Ace
 
           # Merge environment variables for subsequent steps
           def handle_env(vars, env)
-            vars.each { |k, v| env[k.to_s] = v.to_s }
+            vars.each do |k, v|
+              key = k.to_s
+              next if RESERVED_ENV_KEYS.include?(key)
+
+              env[key] = v.to_s
+            end
           end
 
           # Merge custom env vars with the process environment
@@ -162,18 +221,41 @@ module Ace
           # @param env [Hash] Custom environment variables
           # @return [Hash] Merged environment
           def merged_environment(env)
-            return ENV.to_h if env.empty?
-            ENV.to_h.merge(env.transform_keys(&:to_s))
+            base_env = sanitized_process_environment
+            return base_env if env.empty?
+
+            base_env.merge(env.transform_keys(&:to_s))
           end
 
           # Run a command and raise on failure
           def run_command(*args, chdir:, env: {})
-            _stdout, stderr, status = Open3.capture3(merged_environment(env), *args, chdir: chdir)
+            merged_env = merged_environment(env)
+            _stdout, stderr, status = if @sandbox_backend
+              @sandbox_backend.capture3(args, chdir: chdir, env: merged_env)
+            else
+              @command_runner.call(merged_env, *args, chdir: chdir)
+            end
 
             unless status.success?
               raise "Command failed (exit #{status.exitstatus}): #{args.join(" ")}\n#{stderr}"
             end
           end
+
+          def capture3(*args, **kwargs)
+            Open3.capture3(*args, **kwargs)
+          end
+
+          def sanitized_process_environment
+            ENV.to_h.each_with_object({}) do |(key, value), env|
+              if AMBIENT_TMUX_ENV_VARS.include?(key) || STRIPPED_ENV_KEYS.include?(key) ||
+                  BUNDLER_ENV_PREFIXES.any? { |prefix| key.start_with?(prefix) }
+                env[key] = nil
+                next
+              end
+
+              env[key] = value
+            end
+          end
         end
       end
     end