bitfab 0.51.9 → 0.51.10

data/lib/bitfab/replay_concurrency.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Bitfab
+  class ReplayConcurrency
+    UNSET = Object.new.freeze
+    attr_reader :attempts, :max_concurrency, :primitive, :memory_throttle, :on_item_finish_in_child_process, :child_timeout
+
+    def initialize(attempts: 1, max_concurrency: UNSET, primitive: "async", memory_throttle: UNSET, on_item_finish_in_child_process: nil, child_timeout: 2400)
+      raise ArgumentError, "primitive must be async or process" unless %w[async process].include?(primitive)
+      raise ArgumentError, "attempts must be an integer from 1 to 100" unless attempts.is_a?(Integer) && attempts.between?(1, 100)
+      max_concurrency = (primitive == "process") ? 4 : 10 if max_concurrency.equal?(UNSET)
+      valid_max = max_concurrency.nil? || (max_concurrency.is_a?(Integer) && max_concurrency.positive?)
+      unless valid_max
+        raise ArgumentError, "max_concurrency must be a positive integer or nil"
+      end
+      raise ArgumentError, "process concurrency must be bounded" if primitive == "process" && max_concurrency.nil?
+      memory_throttle = primitive == "process" if memory_throttle.equal?(UNSET)
+      raise ArgumentError, "memory_throttle must be true or false" unless [true, false].include?(memory_throttle)
+      if primitive != "process" && (memory_throttle || on_item_finish_in_child_process)
+        raise ArgumentError, "memory_throttle and on_item_finish_in_child_process require primitive: process"
+      end
+      if on_item_finish_in_child_process && !on_item_finish_in_child_process.respond_to?(:call)
+        raise ArgumentError, "on_item_finish_in_child_process must be callable"
+      end
+      raise ArgumentError, "child_timeout must be a positive finite number" unless child_timeout.is_a?(Numeric) && child_timeout.finite? && child_timeout.positive?
+      @child_timeout = child_timeout
+      @attempts, @max_concurrency, @primitive = attempts, max_concurrency, primitive
+      @memory_throttle, @on_item_finish_in_child_process = memory_throttle, on_item_finish_in_child_process
+      freeze
+    end
+
+    def self.resolve(concurrency, attempts, max_concurrency)
+      if concurrency
+        raise ArgumentError, "concurrency must be a ReplayConcurrency" unless concurrency.is_a?(self)
+        unless attempts.equal?(UNSET) && max_concurrency.equal?(UNSET)
+          raise ArgumentError, "concurrency cannot be combined with attempts or max_concurrency"
+        end
+        concurrency
+      else
+        new(attempts: attempts.equal?(UNSET) ? 1 : attempts, max_concurrency:)
+      end
+    end
+  end
+end

data/lib/bitfab/replay_memory.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+require "open3"
+
+module Bitfab
+  class ReplayMemoryThrottle
+    MIB = 1024 * 1024
+    attr_reader :condition
+
+    def initialize(enabled: true)
+      @enabled = enabled && !%w[0 off false no].include?(ENV.fetch("BITFAB_REPLAY_MEMORY_THROTTLE", "").strip.downcase)
+      @initial_budget = env_bytes("BITFAB_REPLAY_CHILD_MEMORY_MB", 2048 * MIB)
+      @floor = env_bytes("BITFAB_REPLAY_MEMORY_FLOOR_MB", 2048 * MIB)
+      @closed = false
+      @peak = 0
+      @resident = {}
+      @mutex = Mutex.new
+      @condition = ConditionVariable.new
+    end
+
+    def admit(index)
+      @mutex.synchronize do
+        @condition.wait(@mutex, 2) while !@closed && @enabled && !headroom?
+        raise "Replay process launcher closed" if @closed
+        @resident[index] = 0
+      end
+    end
+
+    def observe(index, pid)
+      return unless @enabled
+      value = resident_bytes(pid)
+      return unless value
+      @mutex.synchronize { @resident[index] = [@resident.fetch(index, 0), value].max }
+    end
+
+    def release(index)
+      @mutex.synchronize do
+        @peak = [@peak, @resident.delete(index).to_i].max
+        @condition.broadcast
+      end
+    end
+
+    def close
+      @mutex.synchronize do
+        @closed = true
+        @condition.broadcast
+      end
+    end
+
+    def headroom?
+      return true if @resident.empty?
+      snapshot = memory_snapshot
+      return false if snapshot[:swap_used_ratio] && snapshot[:swap_used_ratio] >= 0.85
+      return true unless snapshot[:available]
+      budget = [@peak.positive? ? @peak : @initial_budget, *@resident.values].max
+      unrealized = @resident.values.sum { |rss| [budget - rss, 0].max }
+      snapshot[:available] - unrealized - budget >= @floor
+    end
+
+    def memory_snapshot
+      if File.file?("/proc/meminfo")
+        fields = File.read("/proc/meminfo").scan(/^(\w+):\s+(\d+)/).to_h.transform_values { |value| value.to_i * 1024 }
+        total = fields["SwapTotal"].to_i
+        {available: fields["MemAvailable"], swap_used_ratio: total.positive? ? 1.0 - fields["SwapFree"].to_f / total : 0.0}
+      elsif RUBY_PLATFORM.include?("darwin")
+        total = Integer(read_command("sysctl", "-n", "hw.memsize").to_s, exception: false)
+        percent = Integer(read_command("sysctl", "-n", "kern.memorystatus_level").to_s, exception: false)
+        swap = read_command("sysctl", "-n", "vm.swapusage").to_s
+        total_swap = swap[/total = ([\d.]+)/, 1].to_f
+        used_swap = swap[/used = ([\d.]+)/, 1].to_f
+        {available: (percent && total&.positive?) ? total * percent.to_i.clamp(0, 100) / 100 : nil, swap_used_ratio: total_swap.positive? ? used_swap / total_swap : 0.0}
+      else
+        {}
+      end
+    rescue
+      {}
+    end
+
+    def resident_bytes(pid)
+      value = if File.file?("/proc/#{pid}/status")
+        File.read("/proc/#{pid}/status")[/^VmRSS:\s+(\d+)/, 1]
+      else
+        read_command("ps", "-o", "rss=", "-p", pid.to_s)
+      end
+      value && value.to_i * 1024
+    rescue
+      nil
+    end
+
+    private
+
+    def env_bytes(key, fallback)
+      value = Integer(ENV.fetch(key, "0"), exception: false)
+      value&.positive? ? value * MIB : fallback
+    end
+
+    def read_command(*argv)
+      Open3.popen3(*argv) do |input, output, _errors, process|
+        input.close
+        unless process.join(5)
+          Process.kill("KILL", process.pid)
+          process.join
+          return nil
+        end
+        process.value.success? ? output.read.strip : nil
+      end
+    rescue
+      nil
+    end
+  end
+end

data/lib/bitfab/replay_processes.rb

@@ -0,0 +1,218 @@
+# frozen_string_literal: true
+
+require "json"
+require "tmpdir"
+require "rbconfig"
+require_relative "replay_concurrency"
+require_relative "replay_memory"
+
+module Bitfab
+  module ReplayProcesses
+    ORIGINAL_ENV = ENV.to_h.freeze
+    LAUNCHER_KEY = :__bitfab_process_launcher
+
+    module_function
+
+    def with_registry(path, argv, stderr:)
+      launcher = Launcher.new(path, argv, stderr:)
+      previous = Thread.current.thread_variable_get(LAUNCHER_KEY)
+      Thread.current.thread_variable_set(LAUNCHER_KEY, launcher)
+      yield
+    ensure
+      launcher&.close
+      Thread.current.thread_variable_set(LAUNCHER_KEY, previous)
+    end
+
+    def current
+      Thread.current.thread_variable_get(LAUNCHER_KEY)
+    end
+
+    def execute_assignment(registry, path, stderr:)
+      assignment = JSON.parse(File.read(path))
+      registration = registry.fetch(assignment.fetch("pipeline"))
+      options = registration.options.dup
+      if registration.options_factory
+        options.merge!(registration.options_factory.call(ReplayRegistryContext.new(params: assignment.fetch("params"))))
+      end
+      client = registration.client
+      receiver, method_name = registration.receiver, registration.method_name
+      unless method_name && Traceable.trace_function_key_for(receiver, method_name)
+        receiver = ManagedCallable.new(client, registration.trace_function_key, method_name ? receiver.method(method_name) : receiver)
+        method_name = :call
+      end
+      http = client.instance_variable_get(:@http_client)
+      overrides = MockOverride.normalize(options[:mock_override]) + (client.instance_variable_get(:@mock_overrides) || [])
+      result = Replay.process_single_item(http, assignment.fetch("item"), receiver, method_name,
+        assignment.fetch("test_run_id"), assignment.fetch("mock"), options[:adapt_inputs],
+        assignment.fetch("include_db_branch_lease"), assignment["db_branch_settings"],
+        mock_overrides: overrides, dry_run: assignment.fetch("dry_run"))
+      unless assignment["dry_run"]
+        begin
+          mapped = Replay.wait_for_replay_persistence(http, assignment.fetch("test_run_id"), [result[:_sdk_trace_id]])
+          result[:trace_id] = mapped[result[:_sdk_trace_id]] || result[:trace_id]
+          if result[:_sdk_trace_id] && result[:trace_id].nil?
+            status = http.get_replay_status(assignment.fetch("test_run_id"), {result[:_sdk_trace_id] => 1})
+            result[:trace_id] = status.fetch("traceIds", {})[result[:_sdk_trace_id]]
+          end
+          raise "Child replay trace persistence could not be confirmed" if result[:error].nil? && result[:trace_id].nil?
+        rescue => error
+          result[:replay_error] = error
+          result[:error] = error.message
+        end
+      end
+      write_result(assignment.fetch("result_path"), result)
+      hook = options[:concurrency]&.on_item_finish_in_child_process
+      if hook && !assignment["dry_run"]
+        begin
+          hook.call({test_run_id: assignment.fetch("test_run_id"), item: Replay.public_replay_items([result]).first})
+        rescue => error
+          stderr.puts "[replay] child grading hook failed for #{result[:original_trace_id]} attempt #{result[:attempt]}: #{error}"
+        end
+      end
+      client.close(timeout: 5)
+      result
+    end
+
+    def write_result(path, result)
+      encoded = begin
+        Marshal.dump(result)
+      rescue TypeError
+        safe = result.to_h do |key, value|
+          serializable = begin
+            Marshal.dump(value)
+            value
+          rescue TypeError
+            value.is_a?(Exception) ? Replay.json_safe(value) : Serialize.serialize_value(value)
+          end
+          [key, serializable]
+        end
+        Marshal.dump(safe)
+      end
+      temporary = "#{path}.#{Process.pid}.tmp"
+      File.binwrite(temporary, encoded)
+      File.rename(temporary, path)
+    end
+
+    class Launcher
+      def initialize(registry_path, argv, stderr:)
+        @registry_path, @argv, @stderr = registry_path, argv, stderr
+        @directory = Dir.mktmpdir("bitfab-replay-process-")
+        @mutex = Mutex.new
+        @children = {}
+        @sequence = 0
+        @closed = false
+      end
+
+      def configure(registry, concurrency)
+        @args = ReplayCli.parse(registry, @argv.dup)
+        @params = ReplayCli.load_parameters(@args[:params], @args.fetch(:param, []))
+        @child_timeout = concurrency.child_timeout
+        @throttle = ReplayMemoryThrottle.new(enabled: concurrency.memory_throttle)
+      end
+
+      def call(item, test_run_id, mock, include_db_branch_lease, db_branch_settings, dry_run: false)
+        index = @mutex.synchronize { @sequence += 1 }
+        @throttle.admit(index)
+        result_path = File.join(@directory, "#{index}.result")
+        assignment_path = File.join(@directory, "#{index}.json")
+        stdout_path, stderr_path = File.join(@directory, "#{index}.out"), File.join(@directory, "#{index}.err")
+        assignment = {pipeline: @args.fetch(:pipeline), params: @params, item:, test_run_id:, mock:, include_db_branch_lease:, db_branch_settings:, dry_run:, result_path:}
+        File.write(assignment_path, JSON.generate(assignment), mode: "w", perm: 0o600)
+        executable = File.expand_path("../../exe/bitfab-replay", __dir__)
+        pid = @mutex.synchronize do
+          raise "Replay process launcher closed" if @closed
+          child = Process.spawn(ORIGINAL_ENV.merge("BITFAB_REPLAY_RESULT_PATH" => nil), RbConfig.ruby,
+            "-I", File.expand_path("..", __dir__), executable, "--registry", @registry_path,
+            "--execute-item", assignment_path, out: stdout_path, err: stderr_path,
+            unsetenv_others: true, **Gem.win_platform? ? {} : {pgroup: true})
+          @children[index] = child
+          child
+        end
+        deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + @child_timeout
+        status = nil
+        timed_out = false
+        terminated = false
+        next_observation = 0
+        loop do
+          now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          if now >= next_observation
+            @throttle.observe(index, pid)
+            next_observation = now + 5
+          end
+          waited = Process.waitpid2(pid, Process::WNOHANG)
+          if waited
+            status = waited[1]
+            break
+          end
+          if Process.clock_gettime(Process::CLOCK_MONOTONIC) >= deadline
+            timed_out = true
+            terminate(pid)
+            terminated = true
+            break
+          end
+          sleep(0.1)
+        end
+        child_errors = read_tail(stderr_path)
+        @mutex.synchronize { @stderr.write(child_errors) } unless child_errors.empty?
+        unless File.file?(result_path)
+          output_tail = read_tail(stdout_path)
+          @mutex.synchronize { @stderr.write(output_tail) } unless output_tail.empty?
+          raise "Replay child timed out after #{@child_timeout} seconds" if timed_out
+          raise "Replay child #{pid} exited #{status.exitstatus || status.termsig} without a complete result"
+        end
+        @stderr.puts "[replay] child exceeded #{@child_timeout} seconds after writing its result" if timed_out
+        Marshal.load(File.binread(result_path))
+      rescue => error
+        {
+          attempt: item["attempt"] || 0, original_trace_id: Replay.original_trace_id_of(item), original_span_id: Replay.original_span_id_of(item),
+          source_trace_id: Replay.original_trace_id_of(item), source_span_id: Replay.original_span_id_of(item),
+          ingestion_type: item["ingestionType"], input: [], result: nil, original_output: nil, trace_id: nil,
+          error: error.message, replay_error: error, trace_error: nil
+        }
+      ensure
+        terminate(pid) if pid && status.nil? && !terminated
+        @mutex.synchronize { @children.delete(index) } if index
+        @throttle.release(index) if index
+      end
+
+      def close
+        children = @mutex.synchronize do
+          @closed = true
+          @children.values.dup
+        end
+        @throttle&.close
+        children.each { |pid| terminate(pid) }
+        FileUtils.remove_entry(@directory) if @directory && File.directory?(@directory)
+      end
+
+      private
+
+      def read_tail(path)
+        File.open(path, "rb") do |file|
+          file.seek(-[file.size, 65_536].min, IO::SEEK_END)
+          file.read.lines.last(20).join
+        end
+      end
+
+      def terminate(pid)
+        Process.kill("TERM", Gem.win_platform? ? pid : -pid)
+        deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + 1
+        loop do
+          return if Process.waitpid(pid, Process::WNOHANG)
+          break if Process.clock_gettime(Process::CLOCK_MONOTONIC) >= deadline
+          sleep(0.05)
+        end
+        Process.kill("KILL", Gem.win_platform? ? pid : -pid)
+        Process.waitpid(pid)
+      rescue Errno::ESRCH, Errno::ECHILD
+        nil
+      ensure
+        begin
+          Process.kill("KILL", Gem.win_platform? ? pid : -pid)
+        rescue Errno::ESRCH
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/bitfab/replay_registry.rb

@@ -19,9 +19,9 @@ module Bitfab
   # Project-owned registry of production replay roots.
   class ReplayRegistry
     OPTION_NAMES = %i[
-      limit trace_ids name max_concurrency code_change_description
+      limit trace_ids name concurrency max_concurrency code_change_description
       code_change_files experiment_group_id dataset_id dataset_ids grader_ids mock
-      mock_override adapt_inputs db_branch
+      mock_override adapt_inputs db_branch attempts only_with_assertions dry_run on_item_finish
     ].freeze
 
     def initialize
@@ -39,14 +39,14 @@ module Bitfab
     # +trace_function_key:+ explicitly. Pass executable per-function replay
     # behavior such as +mock_override:+ and +adapt_inputs:+ as keyword options;
     # the installed command forwards them to +Bitfab::Client#replay+.
-    def register(name, receiver, method_name, client: nil, trace_function_key: nil, options_factory: nil, **options)
+    def register(name, receiver, method_name = nil, client: nil, trace_function_key: nil, options_factory: nil, **options)
       raise ArgumentError, "Replay registry names cannot be empty." if name.to_s.empty?
       raise ArgumentError, "Replay registry already contains '#{name}'." if @entries.key?(name.to_s)
       if options_factory && !options_factory.respond_to?(:call)
         raise ArgumentError, "Replay registry options_factory must be callable."
       end
 
-      key = trace_function_key || Traceable.trace_function_key_for(receiver, method_name)
+      key = trace_function_key || (method_name && Traceable.trace_function_key_for(receiver, method_name))
       unless key
         raise ArgumentError,
           "Replay registry entry uses a plain method. Set trace_function_key: " \
@@ -57,7 +57,7 @@ module Bitfab
       @entries[name.to_s] = ReplayRegistration.new(
         client: client || Bitfab.client,
         receiver:,
-        method_name: method_name.to_sym,
+        method_name: method_name&.to_sym,
         trace_function_key: key,
         options: options.freeze,
         options_factory:
@@ -70,6 +70,12 @@ module Bitfab
     end
 
     def validate_options(options)
+      if options[:concurrency]
+        ReplayConcurrency.resolve(options[:concurrency], options.fetch(:attempts, ReplayConcurrency::UNSET), options.fetch(:max_concurrency, ReplayConcurrency::UNSET))
+      end
+      if options[:on_item_finish] && !options[:on_item_finish].respond_to?(:call)
+        raise ArgumentError, "Replay registry on_item_finish must be callable."
+      end
       unknown = options.keys - OPTION_NAMES
       return if unknown.empty?
 
@@ -85,7 +91,27 @@ module Bitfab
 
     module_function
 
-    def pinned_dataset_members(client, dataset_ids, limit)
+    def bound_trace_ids(client, trace_ids, limit, only_with_assertions)
+      unless limit.is_a?(Integer) && limit.positive?
+        raise ArgumentError, "Replay limit must be a positive integer."
+      end
+      return trace_ids.first(limit) if !only_with_assertions || limit >= trace_ids.length
+
+      selected = []
+      trace_ids.each do |trace_id|
+        result = client.traces.get_assertions(trace_id)
+        if result.fetch("inheritedFrom").nil? && !result.fetch("assertions").empty?
+          selected << trace_id
+          break if selected.length == limit
+        end
+      end
+      if selected.empty?
+        raise ArgumentError, "No traces with active assertions matched this replay selection."
+      end
+      selected
+    end
+
+    def pinned_dataset_members(client, dataset_ids, limit, only_with_assertions = false)
       members = dataset_ids.flat_map { |dataset_id|
         client.datasets.list_traces(dataset_id).fetch("traceIds")
       }.uniq.sort
@@ -98,11 +124,13 @@ module Bitfab
             "every selected dataset in full."
       end
 
-      members.first(limit)
+      bound_trace_ids(client, members, limit, only_with_assertions)
     end
 
     def run(registry, argv: ARGV, stdout: $stdout, stderr: $stderr)
       args = parse(registry, argv.dup)
+      return SeedCli.run(registry, argv:, stdout:, stderr:) if args[:seed] || args[:from_trace]
+
       registration = registry.fetch(args.fetch(:pipeline))
       options = registration.options.dup
       params = load_parameters(args[:params], args.fetch(:param, []))
@@ -132,22 +160,25 @@ module Bitfab
         options.delete(:trace_ids)
       end
 
+      only_with_assertions = args[:only_with_assertions] || options[:only_with_assertions]
       bound = args[:limit] || options[:limit]
       options.delete(:limit)
       bounded_dataset_ids = args[:dataset_ids] || options[:dataset_ids]
 
       if options[:trace_ids]
-        options[:trace_ids] = options[:trace_ids].first(bound) if bound
+        if bound
+          options[:trace_ids] = bound_trace_ids(registration.client, options[:trace_ids], bound, only_with_assertions)
+        end
       elsif bounded_dataset_ids
         if bound
-          pinned = pinned_dataset_members(registration.client, bounded_dataset_ids, bound)
+          pinned = pinned_dataset_members(registration.client, bounded_dataset_ids, bound, only_with_assertions)
           options[:trace_ids] = pinned if pinned
         end
       else
         options[:limit] = bound || 10
       end
 
-      %i[name max_concurrency experiment_group_id dataset_ids grader_ids mock].each do |key|
+      %i[name max_concurrency experiment_group_id dataset_ids grader_ids mock attempts only_with_assertions dry_run].each do |key|
         options[key] = args[key] unless args[key].nil?
       end
 
@@ -169,9 +200,34 @@ module Bitfab
         options[:code_change_files] = nil
       end
 
+      if options[:concurrency] || args[:primitive] || args.key?(:memory_throttle)
+        configured = options[:concurrency]
+        options[:concurrency] = ReplayConcurrency.new(
+          primitive: args[:primitive] || configured&.primitive || "async",
+          attempts: options.delete(:attempts) || configured&.attempts || 1,
+          max_concurrency: if options.key?(:max_concurrency)
+                             options.delete(:max_concurrency)
+                           else
+                             (configured ? configured.max_concurrency : ReplayConcurrency::UNSET)
+                           end,
+          memory_throttle: args.fetch(:memory_throttle, configured ? configured.memory_throttle : ReplayConcurrency::UNSET),
+          on_item_finish_in_child_process: configured&.on_item_finish_in_child_process,
+          child_timeout: configured&.child_timeout || 2400
+        )
+        ReplayProcesses.current&.configure(registry, options[:concurrency]) if options[:concurrency].primitive == "process"
+      end
+
       reporter = Bitfab.method(:report_replay_progress)
       options[:on_item_start] = reporter
-      options[:on_item_finish] = reporter
+      registry_callback = options[:on_item_finish] unless options[:dry_run]
+      options[:on_item_finish] = lambda do |progress|
+        reporter.call(progress)
+        begin
+          registry_callback&.call(progress)
+        rescue => error
+          stderr.puts "[replay] on_item_finish failed for #{progress.dig(:item, :original_trace_id)}: #{error}"
+        end
+      end
 
       stderr.puts "[replay] Replaying #{describe_selection(options)} from \"#{registration.trace_function_key}\"..."
 
@@ -190,7 +246,13 @@ module Bitfab
         raise
       end
 
-      render_summary(args.fetch(:pipeline), result, stderr)
+      raise ArgumentError, "No traces matched this replay selection." if result[:items].empty?
+
+      if options[:dry_run]
+        result[:items].each { |item| stderr.puts "[dry-run] #{item[:original_trace_id]} attempt #{item[:attempt]} inputs=#{JSON.generate(item[:input])}" }
+      else
+        render_summary(args.fetch(:pipeline), result, stderr)
+      end
       stdout.puts Bitfab.serialize_replay_result(result)
       result
     end
@@ -207,6 +269,9 @@ module Bitfab
           args[:limit] = positive_integer("--limit", value)
         end
         options.on("--trace-ids IDS") { |value| args[:trace_ids] = comma_separated("--trace-ids", value) }
+        options.on("--seed PATH", "--cases PATH") { |value| args[:seed] = value }
+        options.on("--run") { args[:run] = true }
+        options.on("--from-trace IDS") { |value| args[:from_trace] = comma_separated("--from-trace", value) }
         options.on("--name NAME") { |value| args[:name] = value }
         options.on("--concurrency N", Integer) do |value|
           args[:max_concurrency] = positive_integer("--concurrency", value)
@@ -218,6 +283,11 @@ module Bitfab
         options.on("--experiment-group-id UUID") { |value| args[:experiment_group_id] = value }
         options.on("--dataset-ids IDS", "--dataset-id IDS") { |value| args[:dataset_ids] = comma_separated("--dataset-ids", value) }
         options.on("--grader-ids IDS") { |value| args[:grader_ids] = comma_separated("--grader-ids", value) }
+        options.on("--attempts N", Integer) { |value| args[:attempts] = positive_integer("--attempts", value) }
+        options.on("--only-with-assertions") { args[:only_with_assertions] = true }
+        options.on("--dry-run") { args[:dry_run] = true }
+        options.on("--primitive TYPE", %w[async process]) { |value| args[:primitive] = value }
+        options.on("--[no-]memory-throttle") { |value| args[:memory_throttle] = value }
         options.on("--mock STRATEGY", %w[none all marked]) { |value| args[:mock] = value }
         options.on("--db-branch") { args[:db_branch] = true }
         options.on("--no-db-branch") { args[:db_branch] = false }
@@ -317,11 +387,19 @@ module Bitfab
     def render_summary(pipeline, result, stderr)
       same = 0
       changed = 0
+      matched = 0
+      missed = 0
       errors = 0
       result[:items].each do |item|
         if item[:error]
           errors += 1
-        elsif item[:result] == item[:original_output]
+        elsif item[:ingestion_type] == "seeded"
+          if Serialize.serialize_value(item[:result]) == Serialize.serialize_value(item[:original_output])
+            matched += 1
+          else
+            missed += 1
+          end
+        elsif Serialize.serialize_value(item[:result]) == Serialize.serialize_value(item[:original_output])
           same += 1
         else
           changed += 1
@@ -331,8 +409,15 @@ module Bitfab
       stderr.puts "\n─── Summary ───"
       stderr.puts "  Pipeline: #{pipeline}"
       stderr.puts "  Replayed: #{result[:items].length}"
-      stderr.puts "  Same:     #{same}"
-      stderr.puts "  Changed:  #{changed}"
+      stderr.puts "  Attempts: #{result[:attempts]}" if result[:attempts].to_i > 1
+      if same.positive? || changed.positive? || matched + missed == 0
+        stderr.puts "  Same:     #{same}"
+        stderr.puts "  Changed:  #{changed}"
+      end
+      if matched.positive? || missed.positive?
+        stderr.puts "  Matched expected: #{matched}"
+        stderr.puts "  Missed expected:  #{missed}"
+      end
       stderr.puts "  Errors:   #{errors}" if errors > 0
       stderr.puts "\n  #{result[:test_run_url]}"
     end