ace-test-runner-e2e 0.40.6 → 0.41.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8d69fa4e48a31e4dd0f92cf961dfcc9b1c2fca7b392e61519add56ab806bc492
-  data.tar.gz: d07a32cde8ab19937798129a563d65ef97884caf0158ce0dd5c6f757fdec352c
+  metadata.gz: '023288b5afdd9d9303ed0a4b439503062611cb9cd364e29f68aef91dab4a85c6'
+  data.tar.gz: 30026fd012aa74ebe994d0f41b0bcd809d6bf0599f4a842ad087756e88592d8b
 SHA512:
-  metadata.gz: d6c7df27616d263629bcda75f2a3ff6eb098170115a8ad3a21cab96ff3e860a380f555c9d06603baef3f6922a7616880392c79cace2888ae89da41d40e55a275
-  data.tar.gz: 39158b36f9a7b20d5c1e7ee8c5ae27b488482e4724d63758234a59700ca98ec67c8ac03e6783d5b1369b2fafa027db8013f188ad2c70716a330f2ac4bfeeb12b
+  metadata.gz: '062688fc71b5c282c9789c0c60cd1242a4b454436f6f98cfb39475fccc33025da06962c5e18c85728766dd59c3fe49aaff35bc987c4585c75b5218d5027c0dd1'
+  data.tar.gz: c06d63cd12734b3ac962e0699ecab8934e7f8797ae92b8ee0cf4fef2c3992c980e1e2f664b93e7d844b2bf7564b1003513d8b1c0ec51808e7fee6742d9a49645

data/CHANGELOG.md

@@ -7,6 +7,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.41.0] - 2026-09-29
+
+### Changed
+- Pipeline verdict integrity: a passing report requires a completed verifier response covering every selected test case; an incomplete verdict set becomes an explicit ERROR (`incomplete_verdict_set`) instead of an inferred PASS.
+- Pipeline failures record `failure_phase`, `failure_category`, and structured execution evidence in the ERROR report; sessions whose provider process began and then ended without a confirmed completion are marked `uncertain_execution: true` in metadata.yml.
+- The full-suite automatic retry (retry-failures-once) excludes uncertain scenarios — their side effects are never replayed. They stay in the final tally as errors and are listed by `FailureFinder#find_uncertain_scenarios` for explicit reconciliation. Explicit single-run `--only-failures` keeps them runnable (deliberate human reconciliation).
+- `read_agent_result` (single and package runs) and the suite metadata reconcile keep a fresh pipeline ERROR authoritative: retained metadata.yml can no longer upgrade an errored execution to PASS.
+- Verifier failures report `infrastructure-error` (runner/repair failures keep `runner-error`).
+- TS-MONO-001-style scenarios can declare a `release-manifest` setup action: the runner
+  carries only the explicitly allowed `ACE_RELEASE_MANIFEST` host input (absolute path,
+  defaulting to the source root) into deterministic setup, which validates the strict
+  schema-version-1 manifest and copies its exact bytes into the sandbox before any
+  install goal can run. Invalid or missing input fails setup, and a failed deterministic
+  setup now aborts the scenario with an ERROR result instead of degrading to an
+  LLM-driven sandbox.
+
 ## [0.40.6] - 2026-09-29
 
 ### Fixed

data/lib/ace/test/end_to_end_runner/cli/commands/run_suite.rb

@@ -116,7 +116,11 @@ module Ace
                 timeout: options[:timeout],
                 tags: tags,
                 exclude_tags: exclude_tags,
-                verify: options[:verify]
+                verify: options[:verify],
+                # An explicit --only-failures run is deliberate human
+                # reconciliation: uncertain scenarios stay selectable there.
+                # The automatic retry pass below always excludes them.
+                include_uncertain_failures: only_failures
               }
 
               results = run_suite_with_retry(
@@ -190,20 +194,35 @@ module Ace
               )
               return annotated unless retry_failures_once && suite_failed?(initial_results)
 
+              # Scenarios whose provider session ended uncertainly are excluded
+              # from replay by the failure finder; account for them explicitly
+              # so they stay visible failures instead of vanishing.
+              uncertain = orchestrator.uncertain_scenarios(packages: run_options[:packages])
+              unless uncertain.empty?
+                output.puts "#{uncertain.length} scenario(s) ended in uncertain execution and will not be retried (side effects are not replayed): #{uncertain.join(", ")}"
+                output.puts "Reconcile those sessions manually before rerunning them."
+              end
+
               output.puts "Retrying failed scenarios once..."
-              retry_results = orchestrator.run(run_options.merge(only_failures: true))
-              if retry_results[:total].zero?
+              retry_results = orchestrator.run(
+                run_options.merge(only_failures: true, include_uncertain_failures: false)
+              )
+              if retry_results[:total].zero? && uncertain.empty?
                 raise Ace::Support::Cli::Error.new(
                   "Retry pass found no failed test scenarios from attempt 1; aborting instead of silently passing"
                 )
               end
 
+              retry_results = with_uncertain_accounting(retry_results, uncertain)
+
               flaky_scenarios = recovered_flaky_scenarios(initial_results, retry_results)
-              remaining_failures = failure_scenarios(retry_results)
+              remaining_failures = (failure_scenarios(retry_results) + uncertain).uniq.sort
               final_report_path = write_retry_summary_report(initial_results, retry_results)
               output.puts "Final Report: #{final_report_path}" if final_report_path
 
-              if remaining_failures.empty?
+              if (remaining_failures - uncertain).empty? && !uncertain.empty?
+                output.puts "All retried scenario(s) recovered; #{uncertain.length} uncertain scenario(s) still require reconciliation"
+              elsif remaining_failures.empty?
                 output.puts "#{flaky_scenarios.length} scenario(s) recovered on retry and were marked flaky"
               else
                 output.puts "#{remaining_failures.length} scenario(s) still failing after retry"
@@ -217,10 +236,23 @@ module Ace
                 remaining_failures: remaining_failures,
                 initial_report_path: initial_results[:report_path],
                 retry_report_path: retry_results[:report_path],
-                report_path: final_report_path || retry_results[:report_path]
+                report_path: final_report_path || retry_results[:report_path],
+                uncertain_scenarios: uncertain
               )
             end
 
+            # Keep uncertain scenarios in the final tally. The retry pass does
+            # not run them (by design), so without this they would disappear
+            # from counts and a suite with uncertain sessions could exit 0.
+            def with_uncertain_accounting(retry_results, uncertain_ids)
+              return retry_results if uncertain_ids.empty?
+
+              tally = retry_results.dup
+              tally[:total] = tally[:total].to_i + uncertain_ids.size
+              tally[:errors] = tally[:errors].to_i + uncertain_ids.size
+              tally
+            end
+
             def write_retry_summary_report(initial_results, retry_results)
               build_retry_report_writer.write_retry_summary(
                 initial_results: initial_results,

data/lib/ace/test/end_to_end_runner/molecules/failure_finder.rb

@@ -6,15 +6,23 @@ module Ace
   module Test
     module EndToEndRunner
       module Molecules
-        # Scans cache for failed test cases from previous E2E test runs
-        #
-        # Reads metadata.yml files from .ace-local/test-e2e/*-reports/ directories
-        # and extracts failed_test_cases arrays. Used by --only-failures CLI flag
-        # to re-run only tests that failed previously.
-        #
-        # Note: This is a Molecule (not an Atom) because it performs filesystem
-        # I/O via Dir.glob and YAML file reading.
-        class FailureFinder
+  # Scans cache for failed test cases from previous E2E test runs
+  #
+  # Reads metadata.yml files from .ace-local/test-e2e/*-reports/ directories
+  # and extracts failed_test_cases arrays. Used by --only-failures CLI flag
+  # to re-run only tests that failed previously.
+  #
+  # Scenarios whose most recent run ended in uncertain execution (the provider
+  # session began and then ended without a confirmed completion) are excluded
+  # from find_failures_by_scenario — the automatic retry surface — because
+  # re-running them would replay side effects. They are listed by
+  # find_uncertain_scenarios instead so callers can surface an actionable
+  # diagnostic. Explicit single-run --only-failures paths keep them included:
+  # invoking it is a deliberate human reconciliation decision.
+  #
+  # Note: This is a Molecule (not an Atom) because it performs filesystem
+  # I/O via Dir.glob and YAML file reading.
+  class FailureFinder
           CACHE_DIR = ".ace-local/test-e2e"
           METADATA_FILE = "metadata.yml"
           REPORTS_SUFFIX = "-reports"
@@ -81,11 +89,17 @@ module Ace
           # Like find_failures_by_package but preserves per-scenario granularity.
           # Callers can use this to re-run full failed scenarios.
           #
+          # By default, scenarios whose most recent run is marked
+          # uncertain_execution are excluded — replaying them would repeat
+          # unverified side effects. The explicit reconciliation path passes
+          # include_uncertain: true to select them deliberately.
+          #
           # @param packages [Array<String>] Package names to scan
           # @param base_dir [String] Base directory to search from (default: current dir)
+          # @param include_uncertain [Boolean] keep uncertain scenarios selectable
           # @return [Hash{String => Hash{String => Array<String>}}]
           #   Package name => { test-id => failed TC IDs }
-          def find_failures_by_scenario(packages:, base_dir: Dir.pwd)
+          def find_failures_by_scenario(packages:, base_dir: Dir.pwd, include_uncertain: false)
             metadata_files = discover_metadata_files(base_dir)
             return {} if metadata_files.empty?
 
@@ -96,6 +110,8 @@ module Ace
 
               scenario_failures = {}
               most_recent.each do |entry|
+                next if !include_uncertain && uncertain_execution?(entry[:data])
+
                 test_id = entry[:data]["test-id"]
                 failed_ids = extract_failed_test_cases(entry[:data])
                 scenario_failures[test_id] = failed_ids unless failed_ids.empty?
@@ -106,6 +122,32 @@ module Ace
             result
           end
 
+          # Find scenarios whose most recent run ended in uncertain execution,
+          # grouped by package. These need manual reconciliation before rerun;
+          # they are excluded from find_failures_by_scenario.
+          #
+          # @param packages [Array<String>] Package names to scan
+          # @param base_dir [String] Base directory to search from (default: current dir)
+          # @return [Hash{String => Array<String>}] Package name => uncertain test-ids
+          def find_uncertain_scenarios(packages:, base_dir: Dir.pwd)
+            metadata_files = discover_metadata_files(base_dir)
+            return {} if metadata_files.empty?
+
+            result = {}
+            packages.each do |package|
+              package_metadata = filter_by_package(metadata_files, package)
+              most_recent = most_recent_per_test(package_metadata)
+
+              uncertain = most_recent
+                .select { |entry| uncertain_execution?(entry[:data]) }
+                .map { |entry| entry[:data]["test-id"] }
+                .sort
+
+              result[package] = uncertain unless uncertain.empty?
+            end
+            result
+          end
+
           private
 
           # Discover all metadata.yml files in the cache directory
@@ -169,6 +211,13 @@ module Ace
             entries.flat_map { |entry| extract_failed_test_cases(entry[:data]) }.uniq
           end
 
+          # True when the latest run of this scenario ended in uncertain
+          # execution (the provider session began and ended without a
+          # confirmed completion; side effects may have happened).
+          def uncertain_execution?(data)
+            data["uncertain_execution"] == true
+          end
+
           # Extract failed test case IDs from a single metadata hash
           #
           # Returns specific TC IDs when available, or ["*"] as a wildcard

data/lib/ace/test/end_to_end_runner/molecules/pipeline_executor.rb

@@ -42,7 +42,12 @@ module Ace
           # @return [Models::TestResult]
           def execute(scenario:, cli_args:, sandbox_path:, report_dir:, env_vars: nil, test_cases: nil)
             started_at = Time.now
+            phase = :setup
             FileUtils.mkdir_p(report_dir)
+            # A retry in the same report directory must start with neither
+            # terminal marker present: only this attempt's outcome may remain.
+            FileUtils.rm_f(File.join(report_dir, ".host-pipeline-complete.json"))
+            FileUtils.rm_f(File.join(report_dir, ".host-pipeline-failed.json"))
             write_command_record(report_dir, "runner", provider: @provider, cli_args: cli_args)
             write_tc_manifests(report_dir, scenario, test_cases: test_cases)
 
@@ -72,6 +77,7 @@ module Ace
               test_cases: test_cases,
               artifact_contract: declared_artifact_contract(scenario, test_cases: test_cases)
             )
+            phase = :runner
             runner_response = run_llm(
               prompt_path: runner[:prompt_path],
               system_path: runner[:system_path],
@@ -101,6 +107,7 @@ module Ace
                 artifact_contract: artifact_contract,
                 repair_mode: true
               )
+              phase = :repair
               repair_response = run_llm(
                 prompt_path: repair_runner[:prompt_path],
                 system_path: repair_runner[:system_path],
@@ -126,6 +133,7 @@ module Ace
               artifact_contract: artifact_contract
             )
             write_command_record(report_dir, "verifier", provider: @verifier_provider, cli_args: cli_args)
+            phase = :verifier
             verifier_response = run_llm(
               prompt_path: verifier[:prompt_path],
               system_path: verifier[:system_path],
@@ -137,13 +145,14 @@ module Ace
               fallback: query_fallback_for(@verifier_provider)
             )
 
-            @report_generator.generate(
+            report = @report_generator.generate(
               scenario: scenario,
               verifier_output: verifier_response[:text],
               report_dir: report_dir,
               provider: @verifier_provider,
               started_at: started_at,
               completed_at: Time.now,
+              expected_test_case_ids: selected_test_case_ids(scenario, test_cases),
               metadata: base_metadata(
                 report_dir,
                 runner_observations: runner_observations,
@@ -151,8 +160,26 @@ module Ace
                 initial_artifact_contract: initial_artifact_contract
               )
             )
+            # Host-side completion record, written after the report: the
+            # runner-writable metadata.yml alone cannot prove the pipeline
+            # finished. A rescue below never writes this file.
+            File.write(File.join(report_dir, ".host-pipeline-complete.json"), JSON.generate({
+              "scenario" => scenario.test_id,
+              "runner_provider" => @provider,
+              "verifier_provider" => @verifier_provider,
+              "completed_at" => Time.now.utc.iso8601
+            }))
+            report
           rescue => e
             begin
+              # A failed pipeline leaves an explicit failure record instead of
+              # a completion record; finalization treats absence as failure.
+              File.write(File.join(report_dir, ".host-pipeline-failed.json"), JSON.generate({
+                "scenario" => scenario.test_id,
+                "failure_phase" => phase.to_s,
+                "error" => "#{e.class}: #{e.message}",
+                "completed_at" => Time.now.utc.iso8601
+              }))
               @report_generator.write_failure_report(
                 scenario: scenario,
                 report_dir: report_dir,
@@ -160,8 +187,8 @@ module Ace
                 started_at: started_at || Time.now,
                 completed_at: Time.now,
                 error_message: "#{e.class}: #{e.message}",
-                failure_category: "runner-error",
-                metadata: base_metadata(report_dir)
+                failure_category: phase == :verifier ? "infrastructure-error" : "runner-error",
+                metadata: base_metadata(report_dir).merge(failure_evidence_metadata(e, phase))
               )
             rescue => write_error
               Models::TestResult.new(
@@ -288,6 +315,25 @@ module Ace
             Array(scenario.test_cases).select { |tc| wanted.include?(tc.tc_id.to_s.upcase) }
           end
 
+          # TC ids the verifier must cover for the verdict set to be complete.
+          def selected_test_case_ids(scenario, test_cases)
+            select_test_cases(scenario, test_cases).map(&:tc_id)
+          end
+
+          # Structured failure metadata for the error report. When the failure
+          # carries execution evidence (the provider session began and ended
+          # without a confirmed completion), the scenario is marked uncertain:
+          # automatic retries must not replay its side effects.
+          def failure_evidence_metadata(error, phase)
+            metadata = {"failure_phase" => phase.to_s}
+            evidence = Ace::LLM::Atoms::ErrorClassifier.execution_evidence_for(error)
+            return metadata unless evidence
+
+            metadata["uncertain_execution"] = true if evidence.uncertain_execution?
+            metadata["execution_evidence"] = evidence.to_h
+            metadata
+          end
+
           def base_metadata(report_dir, runner_observations: nil, artifact_contract: nil, initial_artifact_contract: nil)
             metadata = {
               "runner_provider" => @provider,

data/lib/ace/test/end_to_end_runner/molecules/pipeline_report_generator.rb

@@ -31,9 +31,12 @@ module Ace
           # @param provider [String]
           # @param started_at [Time]
           # @param completed_at [Time]
+          # @param expected_test_case_ids [Array<String>, nil] TC ids the verifier must cover
           # @return [Models::TestResult]
-          def generate(scenario:, verifier_output:, report_dir:, provider:, started_at:, completed_at:, metadata: {})
+          def generate(scenario:, verifier_output:, report_dir:, provider:, started_at:, completed_at:,
+            expected_test_case_ids: nil, metadata: {})
             parsed = parse_verifier_output(verifier_output, scenario)
+            parsed = enforce_verdict_completeness(parsed, expected_test_case_ids)
             merged_metadata = metadata.merge(parsed[:metadata] || {})
 
             result = Models::TestResult.new(
@@ -94,9 +97,30 @@ module Ace
             result.with_report_dir(report_dir)
           end
 
-          private
+        private
+
+        # A passing verdict requires one valid verdict per selected test case.
+        # A verifier response covering only some of them stays an explicit
+        # ERROR — bundle output must never synthesize the missing verdicts.
+        def enforce_verdict_completeness(parsed, expected_test_case_ids)
+          expected = Array(expected_test_case_ids).map { |id| id.to_s.upcase }
+          return parsed if expected.empty? || parsed[:status] == "error"
+
+          verdict_ids = parsed[:test_cases].map { |tc| tc[:id].to_s.upcase }
+          missing = expected - verdict_ids
+          return parsed if missing.empty?
+
+          {
+            status: "error",
+            test_cases: parsed[:test_cases],
+            summary: "Incomplete verifier verdict set (#{verdict_ids.size}/#{expected.size} test cases)",
+            error: "Verifier response is missing verdicts for: #{missing.join(", ")}",
+            observations: parsed[:observations],
+            metadata: (parsed[:metadata] || {}).merge("incomplete_verdict_set" => true)
+          }
+        end
 
-          def parse_verifier_output(text, scenario)
+        def parse_verifier_output(text, scenario)
             goals = parse_goal_sections(text, scenario)
             return build_result_from_goals(goals, text) unless goals.empty?
 

data/lib/ace/test/end_to_end_runner/molecules/release_manifest.rb

@@ -0,0 +1,212 @@
+# frozen_string_literal: true
+
+require "digest"
+require "fileutils"
+require "json"
+require "rubygems/version"
+
+module Ace
+  module Test
+    module EndToEndRunner
+      module Molecules
+        # Validates the frozen release installation manifest used by release
+        # verification scenarios (TS-MONO-001).
+        #
+        # The manifest is an exact-version contract: schema_version 1, a full
+        # source commit, and a nonempty package array. Every package entry
+        # carries a unique ACE gem name, one exact artifact version (never a
+        # requirement), the full source commit the artifact was built from,
+        # and an optional explicit supersession list of earlier versions of
+        # the same package.
+        #
+        # Validation is strict on purpose: unknown fields, missing fields,
+        # duplicate names, non-exact versions, malformed SHAs, and
+        # contradictory supersession entries all fail before any install
+        # attempt is made.
+        class ReleaseManifest
+          SCHEMA_VERSION = 1
+          GEM_NAME_PATTERN = /\Aace-[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\z/
+          SOURCE_SHA_PATTERN = /\A[0-9a-f]{40}\z/
+
+          # Raised for any manifest content or source-path failure. Setup
+          # converts this into a setup failure before installation runs.
+          class Invalid < StandardError; end
+
+          # Validate the manifest at +path+ and return its parsed content.
+          # @raise [Invalid] when the file is missing, unreadable,
+          #   malformed, or violates the schema.
+          def self.load_validated(path)
+            data = parse_json(path)
+            validate_schema!(data)
+            data
+          end
+
+          # Validate the manifest at +source_path+ and copy the exact bytes
+          # to +target_path+, creating the target directory when needed.
+          # +required_packages+ names the gems the calling scenario's proof
+          # depends on; the manifest must cover every one of them. Entries
+          # may be plain names or {name, supersedes} maps requiring an
+          # explicit supersession declaration. The buffer is read once so
+          # the validated content, the written copy, and the returned digest
+          # are the same bytes even if the source file changes mid-flight.
+          # @raise [Invalid]
+          def self.validate_and_copy(source_path:, target_path:, required_packages: nil)
+            raw = read_source(source_path)
+            data = parse_buffer(raw, source_path)
+            validate_schema!(data)
+            entries = data["packages"].to_h { |package| [package["name"], package] }
+            Array(required_packages).each do |required|
+              name, supersedes = required_packages_entry(required)
+              entry = entries[name]
+              raise Invalid, "release manifest is missing required package: #{name}" unless entry
+
+              missing = Array(supersedes) - Array(entry["supersedes"])
+              next if missing.empty?
+
+              raise Invalid,
+                "release manifest does not declare that #{name} #{entry["artifact_version"]} supersedes #{missing.join(", ")}"
+            end
+            FileUtils.mkdir_p(File.dirname(target_path))
+            File.binwrite(target_path, raw)
+            Digest::SHA256.hexdigest(raw)
+          end
+
+          class << self
+            private
+
+            def read_source(path)
+              raise Invalid, "release manifest is missing: #{path}" unless File.file?(path)
+              raise Invalid, "release manifest is not readable: #{path}" unless File.readable?(path)
+
+              raw = File.binread(path)
+              raise Invalid, "release manifest is empty: #{path}" if raw.strip.empty?
+
+              raw
+            end
+
+            def parse_buffer(raw, path)
+              data = JSON.parse(raw)
+              raise Invalid, "release manifest must be a JSON object: #{path}" unless data.is_a?(Hash)
+
+              data
+            rescue JSON::ParserError => e
+              raise Invalid, "release manifest is not valid JSON: #{path} (#{e.message})"
+            end
+
+            def parse_json(path)
+              parse_buffer(read_source(path), path)
+            end
+
+            def validate_schema!(data)
+              reject_unknown_keys!(data, %w[schema_version source_sha packages], "manifest")
+
+              schema_version = data["schema_version"]
+              unless schema_version.is_a?(Integer) && schema_version == SCHEMA_VERSION
+                raise Invalid,
+                  "unsupported schema_version: #{schema_version.inspect} (expected #{SCHEMA_VERSION})"
+              end
+
+              validate_source_sha!(data["source_sha"], "manifest source_sha")
+
+              packages = data["packages"]
+              raise Invalid, "packages must be a nonempty array" unless packages.is_a?(Array) && !packages.empty?
+
+              seen = {}
+              packages.each_with_index do |entry, index|
+                name = validate_package!(entry, index)
+                raise Invalid, "duplicate package name: #{name}" if seen.key?(name)
+
+                seen[name] = true
+              end
+            end
+
+            def validate_package!(entry, index)
+              label = "packages[#{index}]"
+              raise Invalid, "#{label} must be an object" unless entry.is_a?(Hash)
+
+              reject_unknown_keys!(entry, %w[name artifact_version source_sha supersedes], label)
+
+              name = entry["name"]
+              unless name.is_a?(String) && name.match?(GEM_NAME_PATTERN)
+                raise Invalid, "#{label} has invalid ACE gem name: #{name.inspect}"
+              end
+
+              validate_exact_version!(entry["artifact_version"], "#{label}(#{name}) artifact_version")
+              validate_source_sha!(entry["source_sha"], "#{label}(#{name}) source_sha")
+              validate_supersedes!(entry["supersedes"], entry["artifact_version"], "#{label}(#{name})")
+              name
+            end
+
+            def validate_exact_version!(raw, label)
+              unless raw.is_a?(String) && !raw.strip.empty?
+                raise Invalid, "#{label} must be an exact version string, got #{raw.inspect}"
+              end
+
+              version = begin
+                Gem::Version.new(raw)
+              rescue ArgumentError
+                raise Invalid, "#{label} is not a valid version: #{raw.inspect}"
+              end
+
+              return if version.to_s == raw
+
+              raise Invalid,
+                "#{label} must be an exact version, not a requirement or shorthand: #{raw.inspect}"
+            end
+
+            def validate_source_sha!(raw, label)
+              return if raw.is_a?(String) && raw.match?(SOURCE_SHA_PATTERN)
+
+              raise Invalid, "#{label} must be a full 40-character commit SHA, got #{raw.inspect}"
+            end
+
+            def validate_supersedes!(raw, artifact_version, label)
+              return if raw.nil?
+
+              unless raw.is_a?(Array)
+                raise Invalid, "#{label} supersedes must be an array of version strings"
+              end
+
+              artifact = Gem::Version.new(artifact_version)
+              seen = {}
+              raw.each do |entry|
+                validate_exact_version!(entry, "#{label} supersedes entry")
+                raise Invalid, "#{label} supersedes duplicates version #{entry}" if seen.key?(entry)
+
+                seen[entry] = true
+                next unless Gem::Version.new(entry) >= artifact
+
+                raise Invalid,
+                  "#{label} supersedes must list earlier versions of the same package: #{entry} >= #{artifact_version}"
+              end
+            end
+
+            def reject_unknown_keys!(hash, allowed, label)
+              unknown = hash.keys - allowed
+              return if unknown.empty?
+
+              raise Invalid, "#{label} has unknown fields: #{unknown.sort.join(", ")}"
+            end
+
+            def required_packages_entry(required)
+              case required
+              when String
+                [required, nil]
+              when Hash
+                reject_unknown_keys!(required, %w[name supersedes], "required package entry")
+                name = required["name"]
+                unless name.is_a?(String) && name.match?(GEM_NAME_PATTERN)
+                  raise Invalid, "required package entry has invalid ACE gem name: #{name.inspect}"
+                end
+
+                [name, required["supersedes"]]
+              else
+                raise Invalid, "required package entries must be names or {name, supersedes} maps"
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/ace/test/end_to_end_runner/molecules/report_writer.rb

@@ -222,6 +222,11 @@ module Ace
             metadata["works_for_end_user"] = result.metadata["works_for_end_user"] if result.metadata.key?("works_for_end_user")
             metadata["user_friction"] = result.metadata["user_friction"] if result.metadata.key?("user_friction")
             metadata["user_feedback"] = result.metadata["user_feedback"] if result.metadata.key?("user_feedback")
+            metadata["failure_category"] = result.metadata["failure_category"] if result.metadata.key?("failure_category")
+            metadata["failure_phase"] = result.metadata["failure_phase"] if result.metadata.key?("failure_phase")
+            metadata["uncertain_execution"] = result.metadata["uncertain_execution"] if result.metadata.key?("uncertain_execution")
+            metadata["execution_evidence"] = result.metadata["execution_evidence"] if result.metadata.key?("execution_evidence")
+            metadata["incomplete_verdict_set"] = result.metadata["incomplete_verdict_set"] if result.metadata.key?("incomplete_verdict_set")
 
             if test_case
               metadata["scenario-id"] = scenario.test_id