ace-test-runner-e2e 0.40.6 → 0.42.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8d69fa4e48a31e4dd0f92cf961dfcc9b1c2fca7b392e61519add56ab806bc492
-  data.tar.gz: d07a32cde8ab19937798129a563d65ef97884caf0158ce0dd5c6f757fdec352c
+  metadata.gz: f8c03365266949f68dd3364885060b9c27b4087d3ff84b0b42ac5c231ab092ed
+  data.tar.gz: ea5b5009323c90d85ec27e417ae012ea09c2af8838b248bf8057caf2fd9174cd
 SHA512:
-  metadata.gz: d6c7df27616d263629bcda75f2a3ff6eb098170115a8ad3a21cab96ff3e860a380f555c9d06603baef3f6922a7616880392c79cace2888ae89da41d40e55a275
-  data.tar.gz: 39158b36f9a7b20d5c1e7ee8c5ae27b488482e4724d63758234a59700ca98ec67c8ac03e6783d5b1369b2fafa027db8013f188ad2c70716a330f2ac4bfeeb12b
+  metadata.gz: e3a65486f96beb2288ca5648d16305da412e1d90f69afc6586818db90f029e8d3dbb3037ca3a780619cf0b067d18d2c96b739dcdf724c7a4dd6ad0a090b00dce
+  data.tar.gz: 34c8755617d888e23307c21c53b21fd0de777195c6c6ac77d7eaf79cc3cc0efe8ba9fd5b8e1741e5ddb29ccaea7d9c181eb10c80b541d026d24d2713edadda9d

data/CHANGELOG.md

@@ -7,6 +7,27 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.42.0] - 2026-10-04
+
+### Added
+- The `tmux-session` setup step is replaced by `runtime-session`: `runtime: tmux` (default) keeps the detached-session behavior and exports `ACE_TMUX_SESSION` + `ACE_RUNTIME=tmux`; `runtime: herdr` inherits the caller's live herdr environment and exports `ACE_RUNTIME=herdr`, failing explicitly when no herdr context is live (no fabricated stand-in).
+
+## [0.41.0] - 2026-09-29
+
+### Changed
+- Pipeline verdict integrity: a passing report requires a completed verifier response covering every selected test case; an incomplete verdict set becomes an explicit ERROR (`incomplete_verdict_set`) instead of an inferred PASS.
+- Pipeline failures record `failure_phase`, `failure_category`, and structured execution evidence in the ERROR report; sessions whose provider process began and then ended without a confirmed completion are marked `uncertain_execution: true` in metadata.yml.
+- The full-suite automatic retry (retry-failures-once) excludes uncertain scenarios — their side effects are never replayed. They stay in the final tally as errors and are listed by `FailureFinder#find_uncertain_scenarios` for explicit reconciliation. Explicit single-run `--only-failures` keeps them runnable (deliberate human reconciliation).
+- `read_agent_result` (single and package runs) and the suite metadata reconcile keep a fresh pipeline ERROR authoritative: retained metadata.yml can no longer upgrade an errored execution to PASS.
+- Verifier failures report `infrastructure-error` (runner/repair failures keep `runner-error`).
+- TS-MONO-001-style scenarios can declare a `release-manifest` setup action: the runner
+  carries only the explicitly allowed `ACE_RELEASE_MANIFEST` host input (absolute path,
+  defaulting to the source root) into deterministic setup, which validates the strict
+  schema-version-1 manifest and copies its exact bytes into the sandbox before any
+  install goal can run. Invalid or missing input fails setup, and a failed deterministic
+  setup now aborts the scenario with an ERROR result instead of degrading to an
+  LLM-driven sandbox.
+
 ## [0.40.6] - 2026-09-29
 
 ### Fixed

data/handbook/guides/scenario-yml-reference.g.md

@@ -112,17 +112,18 @@ Available directives:
 - `copy-fixtures` — Copy fixtures/ directory into sandbox
 - `write-file:` — Write inline content to a file (`path:` + `content:`)
 - `agent-env:` — Environment variables passed to the runner/verifier agent subprocess (not setup commands)
-- `tmux-session` — Create a detached tmux session
-  - String form: `tmux-session` (uses scenario-based naming)
-  - Hash form: `tmux-session: { name-source: run-id }` (uses unique E2E run ID as session name)
-  - Runner teardown removes the created session after test execution
+- `runtime-session` — Establish a live terminal runtime context for the scenario
+  - String form: `runtime-session` (tmux session with scenario-based naming)
+  - Hash form: `runtime-session: { name-source: run-id }` (tmux session named by the E2E run ID)
+  - Hash form: `runtime-session: { runtime: herdr }` (inherits the caller's live HERDR_SESSION/HERDR_PANE; fails explicitly when absent)
+  - tmux sessions export ACE_TMUX_SESSION; both runtimes export ACE_RUNTIME. tmux sessions are removed by runner teardown after test execution
 
 Example:
 
 ```yaml
 setup:
   - git-init
-  - tmux-session:
+  - runtime-session:
       name-source: run-id
   - run: "cp ${ACE_E2E_SOURCE_ROOT:-$PROJECT_ROOT_PATH}/mise.toml mise.toml && mise trust mise.toml"
   - copy-fixtures

data/handbook/templates/scenario.yml.template.yml

@@ -51,14 +51,14 @@ requires:
   ruby: ">= 3.0"
 
 # Setup directives (executed before test cases)
-# Available: git-init, run:, copy-fixtures, write-file:, agent-env:, tmux-session
+# Available: git-init, run:, copy-fixtures, write-file:, agent-env:, runtime-session
 # Rules:
 # - setup is fail-fast (do not use `|| true`)
 # - fixtures/setup own sandbox preparation; runner TCs do not re-implement setup
 setup:
   - git-init
   # Optional detached tmux session for test isolation (uses unique run ID)
-  # - tmux-session:
+  # - runtime-session:
   #     name-source: run-id
   - run: "cp ${ACE_E2E_SOURCE_ROOT:-$PROJECT_ROOT_PATH}/mise.toml mise.toml && mise trust mise.toml"
   # Uncomment as needed:

data/lib/ace/test/end_to_end_runner/cli/commands/run_suite.rb

@@ -116,7 +116,11 @@ module Ace
                 timeout: options[:timeout],
                 tags: tags,
                 exclude_tags: exclude_tags,
-                verify: options[:verify]
+                verify: options[:verify],
+                # An explicit --only-failures run is deliberate human
+                # reconciliation: uncertain scenarios stay selectable there.
+                # The automatic retry pass below always excludes them.
+                include_uncertain_failures: only_failures
               }
 
               results = run_suite_with_retry(
@@ -190,20 +194,35 @@ module Ace
               )
               return annotated unless retry_failures_once && suite_failed?(initial_results)
 
+              # Scenarios whose provider session ended uncertainly are excluded
+              # from replay by the failure finder; account for them explicitly
+              # so they stay visible failures instead of vanishing.
+              uncertain = orchestrator.uncertain_scenarios(packages: run_options[:packages])
+              unless uncertain.empty?
+                output.puts "#{uncertain.length} scenario(s) ended in uncertain execution and will not be retried (side effects are not replayed): #{uncertain.join(", ")}"
+                output.puts "Reconcile those sessions manually before rerunning them."
+              end
+
               output.puts "Retrying failed scenarios once..."
-              retry_results = orchestrator.run(run_options.merge(only_failures: true))
-              if retry_results[:total].zero?
+              retry_results = orchestrator.run(
+                run_options.merge(only_failures: true, include_uncertain_failures: false)
+              )
+              if retry_results[:total].zero? && uncertain.empty?
                 raise Ace::Support::Cli::Error.new(
                   "Retry pass found no failed test scenarios from attempt 1; aborting instead of silently passing"
                 )
               end
 
+              retry_results = with_uncertain_accounting(retry_results, uncertain)
+
               flaky_scenarios = recovered_flaky_scenarios(initial_results, retry_results)
-              remaining_failures = failure_scenarios(retry_results)
+              remaining_failures = (failure_scenarios(retry_results) + uncertain).uniq.sort
               final_report_path = write_retry_summary_report(initial_results, retry_results)
               output.puts "Final Report: #{final_report_path}" if final_report_path
 
-              if remaining_failures.empty?
+              if (remaining_failures - uncertain).empty? && !uncertain.empty?
+                output.puts "All retried scenario(s) recovered; #{uncertain.length} uncertain scenario(s) still require reconciliation"
+              elsif remaining_failures.empty?
                 output.puts "#{flaky_scenarios.length} scenario(s) recovered on retry and were marked flaky"
               else
                 output.puts "#{remaining_failures.length} scenario(s) still failing after retry"
@@ -217,10 +236,23 @@ module Ace
                 remaining_failures: remaining_failures,
                 initial_report_path: initial_results[:report_path],
                 retry_report_path: retry_results[:report_path],
-                report_path: final_report_path || retry_results[:report_path]
+                report_path: final_report_path || retry_results[:report_path],
+                uncertain_scenarios: uncertain
               )
             end
 
+            # Keep uncertain scenarios in the final tally. The retry pass does
+            # not run them (by design), so without this they would disappear
+            # from counts and a suite with uncertain sessions could exit 0.
+            def with_uncertain_accounting(retry_results, uncertain_ids)
+              return retry_results if uncertain_ids.empty?
+
+              tally = retry_results.dup
+              tally[:total] = tally[:total].to_i + uncertain_ids.size
+              tally[:errors] = tally[:errors].to_i + uncertain_ids.size
+              tally
+            end
+
             def write_retry_summary_report(initial_results, retry_results)
               build_retry_report_writer.write_retry_summary(
                 initial_results: initial_results,

data/lib/ace/test/end_to_end_runner/molecules/failure_finder.rb

@@ -6,15 +6,23 @@ module Ace
   module Test
     module EndToEndRunner
       module Molecules
-        # Scans cache for failed test cases from previous E2E test runs
-        #
-        # Reads metadata.yml files from .ace-local/test-e2e/*-reports/ directories
-        # and extracts failed_test_cases arrays. Used by --only-failures CLI flag
-        # to re-run only tests that failed previously.
-        #
-        # Note: This is a Molecule (not an Atom) because it performs filesystem
-        # I/O via Dir.glob and YAML file reading.
-        class FailureFinder
+  # Scans cache for failed test cases from previous E2E test runs
+  #
+  # Reads metadata.yml files from .ace-local/test-e2e/*-reports/ directories
+  # and extracts failed_test_cases arrays. Used by --only-failures CLI flag
+  # to re-run only tests that failed previously.
+  #
+  # Scenarios whose most recent run ended in uncertain execution (the provider
+  # session began and then ended without a confirmed completion) are excluded
+  # from find_failures_by_scenario — the automatic retry surface — because
+  # re-running them would replay side effects. They are listed by
+  # find_uncertain_scenarios instead so callers can surface an actionable
+  # diagnostic. Explicit single-run --only-failures paths keep them included:
+  # invoking it is a deliberate human reconciliation decision.
+  #
+  # Note: This is a Molecule (not an Atom) because it performs filesystem
+  # I/O via Dir.glob and YAML file reading.
+  class FailureFinder
           CACHE_DIR = ".ace-local/test-e2e"
           METADATA_FILE = "metadata.yml"
           REPORTS_SUFFIX = "-reports"
@@ -81,11 +89,17 @@ module Ace
           # Like find_failures_by_package but preserves per-scenario granularity.
           # Callers can use this to re-run full failed scenarios.
           #
+          # By default, scenarios whose most recent run is marked
+          # uncertain_execution are excluded — replaying them would repeat
+          # unverified side effects. The explicit reconciliation path passes
+          # include_uncertain: true to select them deliberately.
+          #
           # @param packages [Array<String>] Package names to scan
           # @param base_dir [String] Base directory to search from (default: current dir)
+          # @param include_uncertain [Boolean] keep uncertain scenarios selectable
           # @return [Hash{String => Hash{String => Array<String>}}]
           #   Package name => { test-id => failed TC IDs }
-          def find_failures_by_scenario(packages:, base_dir: Dir.pwd)
+          def find_failures_by_scenario(packages:, base_dir: Dir.pwd, include_uncertain: false)
             metadata_files = discover_metadata_files(base_dir)
             return {} if metadata_files.empty?
 
@@ -96,6 +110,8 @@ module Ace
 
               scenario_failures = {}
               most_recent.each do |entry|
+                next if !include_uncertain && uncertain_execution?(entry[:data])
+
                 test_id = entry[:data]["test-id"]
                 failed_ids = extract_failed_test_cases(entry[:data])
                 scenario_failures[test_id] = failed_ids unless failed_ids.empty?
@@ -106,6 +122,32 @@ module Ace
             result
           end
 
+          # Find scenarios whose most recent run ended in uncertain execution,
+          # grouped by package. These need manual reconciliation before rerun;
+          # they are excluded from find_failures_by_scenario.
+          #
+          # @param packages [Array<String>] Package names to scan
+          # @param base_dir [String] Base directory to search from (default: current dir)
+          # @return [Hash{String => Array<String>}] Package name => uncertain test-ids
+          def find_uncertain_scenarios(packages:, base_dir: Dir.pwd)
+            metadata_files = discover_metadata_files(base_dir)
+            return {} if metadata_files.empty?
+
+            result = {}
+            packages.each do |package|
+              package_metadata = filter_by_package(metadata_files, package)
+              most_recent = most_recent_per_test(package_metadata)
+
+              uncertain = most_recent
+                .select { |entry| uncertain_execution?(entry[:data]) }
+                .map { |entry| entry[:data]["test-id"] }
+                .sort
+
+              result[package] = uncertain unless uncertain.empty?
+            end
+            result
+          end
+
           private
 
           # Discover all metadata.yml files in the cache directory
@@ -169,6 +211,13 @@ module Ace
             entries.flat_map { |entry| extract_failed_test_cases(entry[:data]) }.uniq
           end
 
+          # True when the latest run of this scenario ended in uncertain
+          # execution (the provider session began and ended without a
+          # confirmed completion; side effects may have happened).
+          def uncertain_execution?(data)
+            data["uncertain_execution"] == true
+          end
+
           # Extract failed test case IDs from a single metadata hash
           #
           # Returns specific TC IDs when available, or ["*"] as a wildcard

data/lib/ace/test/end_to_end_runner/molecules/pipeline_executor.rb

@@ -42,7 +42,12 @@ module Ace
           # @return [Models::TestResult]
           def execute(scenario:, cli_args:, sandbox_path:, report_dir:, env_vars: nil, test_cases: nil)
             started_at = Time.now
+            phase = :setup
             FileUtils.mkdir_p(report_dir)
+            # A retry in the same report directory must start with neither
+            # terminal marker present: only this attempt's outcome may remain.
+            FileUtils.rm_f(File.join(report_dir, ".host-pipeline-complete.json"))
+            FileUtils.rm_f(File.join(report_dir, ".host-pipeline-failed.json"))
             write_command_record(report_dir, "runner", provider: @provider, cli_args: cli_args)
             write_tc_manifests(report_dir, scenario, test_cases: test_cases)
 
@@ -72,6 +77,7 @@ module Ace
               test_cases: test_cases,
               artifact_contract: declared_artifact_contract(scenario, test_cases: test_cases)
             )
+            phase = :runner
             runner_response = run_llm(
               prompt_path: runner[:prompt_path],
               system_path: runner[:system_path],
@@ -101,6 +107,7 @@ module Ace
                 artifact_contract: artifact_contract,
                 repair_mode: true
               )
+              phase = :repair
               repair_response = run_llm(
                 prompt_path: repair_runner[:prompt_path],
                 system_path: repair_runner[:system_path],
@@ -126,6 +133,7 @@ module Ace
               artifact_contract: artifact_contract
             )
             write_command_record(report_dir, "verifier", provider: @verifier_provider, cli_args: cli_args)
+            phase = :verifier
             verifier_response = run_llm(
               prompt_path: verifier[:prompt_path],
               system_path: verifier[:system_path],
@@ -137,13 +145,14 @@ module Ace
               fallback: query_fallback_for(@verifier_provider)
             )
 
-            @report_generator.generate(
+            report = @report_generator.generate(
               scenario: scenario,
               verifier_output: verifier_response[:text],
               report_dir: report_dir,
               provider: @verifier_provider,
               started_at: started_at,
               completed_at: Time.now,
+              expected_test_case_ids: selected_test_case_ids(scenario, test_cases),
               metadata: base_metadata(
                 report_dir,
                 runner_observations: runner_observations,
@@ -151,8 +160,26 @@ module Ace
                 initial_artifact_contract: initial_artifact_contract
               )
             )
+            # Host-side completion record, written after the report: the
+            # runner-writable metadata.yml alone cannot prove the pipeline
+            # finished. A rescue below never writes this file.
+            File.write(File.join(report_dir, ".host-pipeline-complete.json"), JSON.generate({
+              "scenario" => scenario.test_id,
+              "runner_provider" => @provider,
+              "verifier_provider" => @verifier_provider,
+              "completed_at" => Time.now.utc.iso8601
+            }))
+            report
           rescue => e
             begin
+              # A failed pipeline leaves an explicit failure record instead of
+              # a completion record; finalization treats absence as failure.
+              File.write(File.join(report_dir, ".host-pipeline-failed.json"), JSON.generate({
+                "scenario" => scenario.test_id,
+                "failure_phase" => phase.to_s,
+                "error" => "#{e.class}: #{e.message}",
+                "completed_at" => Time.now.utc.iso8601
+              }))
               @report_generator.write_failure_report(
                 scenario: scenario,
                 report_dir: report_dir,
@@ -160,8 +187,8 @@ module Ace
                 started_at: started_at || Time.now,
                 completed_at: Time.now,
                 error_message: "#{e.class}: #{e.message}",
-                failure_category: "runner-error",
-                metadata: base_metadata(report_dir)
+                failure_category: phase == :verifier ? "infrastructure-error" : "runner-error",
+                metadata: base_metadata(report_dir).merge(failure_evidence_metadata(e, phase))
               )
             rescue => write_error
               Models::TestResult.new(
@@ -288,6 +315,25 @@ module Ace
             Array(scenario.test_cases).select { |tc| wanted.include?(tc.tc_id.to_s.upcase) }
           end
 
+          # TC ids the verifier must cover for the verdict set to be complete.
+          def selected_test_case_ids(scenario, test_cases)
+            select_test_cases(scenario, test_cases).map(&:tc_id)
+          end
+
+          # Structured failure metadata for the error report. When the failure
+          # carries execution evidence (the provider session began and ended
+          # without a confirmed completion), the scenario is marked uncertain:
+          # automatic retries must not replay its side effects.
+          def failure_evidence_metadata(error, phase)
+            metadata = {"failure_phase" => phase.to_s}
+            evidence = Ace::LLM::Atoms::ErrorClassifier.execution_evidence_for(error)
+            return metadata unless evidence
+
+            metadata["uncertain_execution"] = true if evidence.uncertain_execution?
+            metadata["execution_evidence"] = evidence.to_h
+            metadata
+          end
+
           def base_metadata(report_dir, runner_observations: nil, artifact_contract: nil, initial_artifact_contract: nil)
             metadata = {
               "runner_provider" => @provider,

data/lib/ace/test/end_to_end_runner/molecules/pipeline_prompt_bundler.rb

@@ -163,9 +163,32 @@ module Ace
             match ? match[1].upcase : nil
           end
 
+          # Bundler machinery (installed gem trees, compact-index caches,
+          # config, shared-gem dirs) is install plumbing, not verifier
+          # evidence: `--full-index` alone pulls a ~97MB compact index, which
+          # previously got inlined into the verifier prompt whole.
+          BUNDLER_MACHINE_SEGMENTS = %w[
+            .bundle .gem bundler-cache bundler-home bundler-config
+            compact_index specifications cache bin
+          ].freeze
+          MAX_ARTIFACT_FILE_BYTES = 256 * 1024
+          MAX_ARTIFACT_CONTENT_BYTES = 2 * 1024 * 1024
+
+          def artifact_evidence?(relative_path)
+            return false if relative_path.start_with?(".bundle/", ".gem/")
+
+            BUNDLER_MACHINE_SEGMENTS.none? do |segment|
+              relative_path.include?("/#{segment}/") || relative_path.start_with?("#{segment}/")
+            end
+          end
+
           def build_artifact_section(sandbox_path)
             sandbox_path = File.expand_path(sandbox_path)
-            files = Dir.glob(File.join(sandbox_path, "results", "**", "*")).select { |f| File.file?(f) }.sort
+            files = Dir.glob(File.join(sandbox_path, "results", "**", "*"))
+              .select { |f| File.file?(f) }
+              .map { |f| File.expand_path(f) }
+              .select { |f| artifact_evidence?(relative_path(f, sandbox_path)) }
+              .sort
             tree_entries = files.map { |f| relative_path(f, sandbox_path) }
 
             parts = []
@@ -186,12 +209,21 @@ module Ace
             parts << "## File contents"
             parts << ""
 
+            content_budget = MAX_ARTIFACT_CONTENT_BYTES
             files.each do |file|
+              size = File.size(file)
+              body = if size <= MAX_ARTIFACT_FILE_BYTES
+                safe_read(file)
+              else
+                "#{safe_read(file, MAX_ARTIFACT_FILE_BYTES)}\n... [truncated, #{size} bytes total]"
+              end
+              content_budget -= body.bytesize
               parts << "### `#{relative_path(file, sandbox_path)}`"
               parts << "```"
-              parts << safe_read(file)
+              parts << body
               parts << "```"
               parts << ""
+              break if content_budget <= 0
             end
 
             parts.join("\n").rstrip
@@ -324,8 +356,9 @@ module Ace
             File.expand_path(path).sub("#{File.expand_path(root)}/", "")
           end
 
-          def safe_read(path)
-            File.binread(path).encode("UTF-8", invalid: :replace, undef: :replace, replace: "?")
+          def safe_read(path, limit = nil)
+            raw = limit ? File.binread(path, limit) : File.binread(path)
+            raw.encode("UTF-8", invalid: :replace, undef: :replace, replace: "?")
           end
         end
       end

data/lib/ace/test/end_to_end_runner/molecules/pipeline_report_generator.rb

@@ -31,9 +31,12 @@ module Ace
           # @param provider [String]
           # @param started_at [Time]
           # @param completed_at [Time]
+          # @param expected_test_case_ids [Array<String>, nil] TC ids the verifier must cover
           # @return [Models::TestResult]
-          def generate(scenario:, verifier_output:, report_dir:, provider:, started_at:, completed_at:, metadata: {})
+          def generate(scenario:, verifier_output:, report_dir:, provider:, started_at:, completed_at:,
+            expected_test_case_ids: nil, metadata: {})
             parsed = parse_verifier_output(verifier_output, scenario)
+            parsed = enforce_verdict_completeness(parsed, expected_test_case_ids)
             merged_metadata = metadata.merge(parsed[:metadata] || {})
 
             result = Models::TestResult.new(
@@ -94,9 +97,30 @@ module Ace
             result.with_report_dir(report_dir)
           end
 
-          private
+        private
+
+        # A passing verdict requires one valid verdict per selected test case.
+        # A verifier response covering only some of them stays an explicit
+        # ERROR — bundle output must never synthesize the missing verdicts.
+        def enforce_verdict_completeness(parsed, expected_test_case_ids)
+          expected = Array(expected_test_case_ids).map { |id| id.to_s.upcase }
+          return parsed if expected.empty? || parsed[:status] == "error"
+
+          verdict_ids = parsed[:test_cases].map { |tc| tc[:id].to_s.upcase }
+          missing = expected - verdict_ids
+          return parsed if missing.empty?
+
+          {
+            status: "error",
+            test_cases: parsed[:test_cases],
+            summary: "Incomplete verifier verdict set (#{verdict_ids.size}/#{expected.size} test cases)",
+            error: "Verifier response is missing verdicts for: #{missing.join(", ")}",
+            observations: parsed[:observations],
+            metadata: (parsed[:metadata] || {}).merge("incomplete_verdict_set" => true)
+          }
+        end
 
-          def parse_verifier_output(text, scenario)
+        def parse_verifier_output(text, scenario)
             goals = parse_goal_sections(text, scenario)
             return build_result_from_goals(goals, text) unless goals.empty?