hecks 2.9.0 → 3.0.1

data/lib/hecks/quality_control/cli/qa_sweep/all_mode.rb

@@ -0,0 +1,280 @@
+# frozen_string_literal: true
+
+require "tempfile"
+require_relative "../child"
+
+module Hecks
+  module QualityControlCli
+    class QaSweep
+      # `--all`: one child sweep per waiting target, in parallel, then the later waves and one
+      # consolidated report.
+      #
+      # A child is a fresh process, never a fork: a forked child would share this process's live
+      # Postgres socket.
+      module AllMode
+        # Every child prints one `resolved depth:` line; the child's log is the single source of
+        # truth that the report prints verbatim, so scraping it here cannot disagree.
+        RESOLVED_DEPTH_LINE = /^resolved depth: seeds=(\d+) steps=(\d+) \(clean_streak=(\d+)\)$/
+
+        # Parsed from a child's `resolved modes:` line; wave-2 candidates come from the parent's own
+        # inference, not from this.
+        RESOLVED_MODES_LINE = /^resolved modes: (\S*) \(capabilities=([^;)]*)(?:; deferred=([^)]*))?\)$/
+
+        private
+
+        # Sweeps every waiting target and reports.
+        #
+        # @param modes [Array<Symbol>] the enabled modes each child is told
+        # @param parity_wave [Boolean] whether a `persistence_parity` wave follows a clean wave 1
+        # @param deferred_wave_modes [Array<Symbol>] the modes that get a wave of their own
+        # @return [Integer] 2 when any child found something, else 1 when any errored, else 0
+        def run_all_mode(modes:, parity_wave:, deferred_wave_modes:)
+          now = Time.now.to_i
+          waiting = query("Target.Rotation").map { |row| row[:reference][:value] }
+          stale = stale_holds(now)
+          targets = waiting + stale.map { |row| row[:reference][:value] }
+          if targets.empty?
+            puts "rotation is empty — nothing waiting to sweep, and no stale hold to reclaim. Clean no-op."
+            return EXIT_OK
+          end
+
+          announce_all(targets, stale, modes, now)
+          started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          results = run_pool(targets) do |reference|
+            spawn_sweep_child(reference, @seeds_override, @steps_override, @self_consistency, modes: modes)
+          end
+          results = results.map { |r| r.merge(wave: 1) }
+          results += run_parity_wave(results) if parity_wave
+          # Each enabled deferred mode gets its own wave over wave 1's clean targets.
+          deferred_wave_modes.each do |mode|
+            results += run_deferred_wave(results.select { |r| r[:wave] == 1 }, mode)
+          end
+          elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started_at
+
+          clean = results.select { |r| r[:exit_status] == EXIT_OK }
+          found = results.select { |r| r[:exit_status] == EXIT_FOUND_SOMETHING }
+          errored = results.reject { |r| [EXIT_OK, EXIT_FOUND_SOMETHING].include?(r[:exit_status]) }
+          print_all_mode_report(results, elapsed, clean, found, errored)
+          all_mode_exit_code(found, errored)
+        end
+
+        def announce_all(targets, stale, modes, now)
+          puts "sweeping #{targets.size} target(s), at most #{@max_parallel} at once: #{targets.join(', ')} " \
+               "(adversarial fraction #{@adversarial}, self-consistency #{@self_consistency}, " \
+               "role draw #{@role_draw}, dry-run fraction #{@dry_run}, enabled modes #{modes.join(',')})"
+          stale.each do |row|
+            puts "reclaimed stale hold: #{row[:reference][:value]} (held by #{row[:held_by][:value]}, " \
+                 "#{now - row[:claimed_at][:value]}s ago)"
+          end
+          puts
+        end
+
+        # `Target.Rotation` lists only waiting targets, so stale holds are found here:
+        # `Target.Claim`'s given (claimed_at + window <= now) restated in Ruby, since a `where`
+        # cannot add two fields.
+        def stale_holds(now)
+          query("Target.Held").select { |row| row[:claimed_at][:value] + row[:window][:value] <= now }
+        end
+
+        # Spawns `hecks quality_control ask run target=<target>` as a fresh process, not a fork.
+        # Seeds/steps forward only when given, so each child derives its own depth from its own
+        # streak.
+        def spawn_sweep_child(target_reference, seeds_override, steps_override, self_consistency,
+                              modes:, parity: false)
+          # Unlinked at once: the open fd stays readable, nothing is left on disk, and no fixed path
+          # collides.
+          log = Tempfile.new(["qa_sweep-#{filesystem_safe_component(target_reference)}-", ".log"])
+          log.unlink
+
+          args = [target_reference]
+          args += ["--seeds", seeds_override.to_s] if seeds_override
+          args += ["--steps", steps_override.to_s] if steps_override
+          args += ["--adversarial", @adversarial.to_s, "--self-consistency", self_consistency.to_s]
+          args += ["--role-draw", @role_draw.to_s, "--dry-run", @dry_run.to_s]
+          args += parity ? ["--persistence-parity"] : ["--modes", modes.join(",")]
+
+          pid = Process.spawn(*Child.argv(@root, "qa_sweep", *args), out: log, err: log, chdir: @root)
+          { target: target_reference, pid: pid, log: log }
+        end
+
+        def collect_sweep_child(child, status)
+          child[:log].rewind
+          output = child[:log].read
+          child[:log].close
+
+          { target: child[:target], exit_status: status.exitstatus, termsig: status.termsig, output: output }
+        end
+
+        # Keeps at most `@max_parallel` children alive; results come back in queue order so reports
+        # are stable.
+        def run_pool(targets)
+          pending = targets.dup
+          running = {}
+          results = {}
+
+          until pending.empty? && running.empty?
+            while running.size < @max_parallel && (next_target = pending.shift)
+              child = yield(next_target)
+              running[child[:pid]] = child
+            end
+
+            pid, status = Process.wait2(-1)
+            child = running.delete(pid)
+            next unless child
+
+            results[child[:target]] = collect_sweep_child(child, status)
+          end
+
+          targets.map { |target| results.fetch(target) }
+        end
+
+        # Wave 2: `--persistence-parity` children for wave-1 targets that came back clean and infer
+        # `postgres_era`. Skipped by `--no-parity` or when `--modes` already named the mode.
+        def run_parity_wave(wave1)
+          candidates = wave_candidates(wave1, :persistence_parity)
+          if candidates.empty?
+            puts "parity wave: no clean target binds PostgresEra — nothing to compare Memory against. Skipped."
+            return []
+          end
+
+          puts
+          puts "parity wave: Memory vs real PostgresEra for #{candidates.size} target(s), " \
+               "at most #{@max_parallel} at once: #{candidates.join(', ')}"
+          results = run_pool(candidates) do |reference|
+            spawn_sweep_child(reference, @seeds_override, nil, false, modes: [], parity: true)
+          end
+          # Labelled in the target column so every report section tells a wave-2 row from wave 1's.
+          results.map { |r| r.merge(wave: 2, target: "#{r[:target]} [parity wave]") }
+        end
+
+        # Later wave for deferred modes (era_boundary, concurrency): `--modes <mode>` children over
+        # the clean wave-1 targets whose capabilities make the mode eligible.
+        def run_deferred_wave(wave1, mode)
+          candidates = wave_candidates(wave1, mode)
+          if candidates.empty?
+            puts "#{mode} wave: no clean target qualifies. Skipped."
+            return []
+          end
+
+          puts
+          puts "#{mode} wave: #{candidates.size} target(s), at most #{@max_parallel} at once: #{candidates.join(', ')}"
+          results = run_pool(candidates) do |reference|
+            spawn_sweep_child(reference, @seeds_override, nil, false, modes: [mode])
+          end
+          results.map { |r| r.merge(wave: 2, target: "#{r[:target]} [#{mode} wave]") }
+        end
+
+        # @return [Array<String>] the clean wave-1 targets whose capabilities make `mode` eligible
+        def wave_candidates(wave1, mode)
+          clean_refs = wave1.select { |r| r[:exit_status] == EXIT_OK }.map { |r| r[:target] }
+          rows = query("Target.All").select { |row| clean_refs.include?(row[:reference][:value]) }
+          eligible = rows.select do |row|
+            capabilities = Hecks::Fuzzing::TargetCapabilities.infer(resolve_target_path(row[:path][:value]),
+                                                                    rust_dir: @rust_dir)
+            Hecks::Fuzzing::TargetCapabilities.eligible?(mode, capabilities)
+          end
+          eligible.map { |row| row[:reference][:value] }
+        end
+
+        def resolved_depth_for(output)
+          match = output.match(RESOLVED_DEPTH_LINE)
+          return "depth unknown (target errored before it could claim)" unless match
+
+          "#{match[1]} seeds x #{match[2]} steps (clean streak #{match[3]})"
+        end
+
+        def resolved_modes_for(output)
+          match = output.match(RESOLVED_MODES_LINE)
+          return "modes unknown (target errored before it could resolve them)" unless match
+
+          line = "#{match[1].empty? ? 'none' : match[1]} (capabilities: #{match[2].empty? ? 'none' : match[2]}"
+          line += "; deferred: #{match[3]}" if match[3] && !match[3].empty?
+          "#{line})"
+        end
+
+        def print_resolved_modes(results)
+          return if results.empty?
+
+          puts
+          puts "resolved modes (by target):"
+          results.each { |r| puts "  #{r[:target]}: #{resolved_modes_for(r[:output])}" }
+        end
+
+        def print_resolved_depths(results)
+          return if results.empty?
+
+          puts
+          puts "resolved depth (by target):"
+          results.each { |r| puts "  #{r[:target]}: #{resolved_depth_for(r[:output])}" }
+        end
+
+        def print_operational_errors(errored)
+          return if errored.empty?
+
+          puts
+          puts "OPERATIONAL ERRORS (#{errored.size}) — read each message, fix the actual problem, don't retry blind:"
+          errored.each do |r|
+            label = r[:exit_status] ? "exit #{r[:exit_status]}" : "killed by signal #{r[:termsig]}"
+            puts
+            puts "-- #{r[:target]} (#{label}) --"
+            puts r[:output].strip
+          end
+        end
+
+        def print_found_something(found)
+          return if found.empty?
+
+          puts
+          puts "FOUND SOMETHING (#{found.size}) — full report per target below; act on every one:"
+          found.each do |r|
+            puts
+            puts "#" * 72
+            puts "# #{r[:target]}"
+            puts "#" * 72
+            puts r[:output]
+          end
+        end
+
+        def print_all_mode_report(results, elapsed, clean, found, errored)
+          puts "=" * 72
+          puts "CONSOLIDATED REPORT — #{results.size} target(s) swept, #{elapsed.round(1)}s wall-clock " \
+               "(genuinely parallel — not the sum of every target's own sweep time)"
+          puts "=" * 72
+          puts
+          puts "clean (#{clean.size}): #{clean.empty? ? 'none' : clean.map { |r| r[:target] }.join(', ')}"
+
+          print_resolved_depths(results)
+          print_resolved_modes(results)
+          print_operational_errors(errored)
+          print_found_something(found)
+          print_trace_output(results)
+        end
+
+        # A clean child's trace would otherwise be discarded; shown only under `QA_SWEEP_TRACE=1`.
+        def print_trace_output(results)
+          return unless @trace_enabled
+          return if results.empty?
+
+          puts
+          puts "QA_SWEEP_TRACE output (by target, in sweep order):"
+          results.each do |r|
+            trace_lines = r[:output].lines.grep(/^\[qa_sweep_trace /)
+            next if trace_lines.empty?
+
+            puts "  #{r[:target]}:"
+            trace_lines.each { |line| puts "    #{line.chomp}" }
+          end
+        end
+
+        # A finding outranks operational errors: 2 if any child found something, else 1 if any
+        # errored.
+        def all_mode_exit_code(found, errored)
+          return EXIT_FOUND_SOMETHING unless found.empty?
+          return EXIT_ERROR unless errored.empty?
+
+          EXIT_OK
+        end
+      end
+    end
+  end
+end

data/lib/hecks/quality_control/cli/qa_sweep/checks.rb

@@ -0,0 +1,329 @@
+# frozen_string_literal: true
+
+module Hecks
+  module QualityControlCli
+    class QaSweep
+      # The comparisons one sweep makes per seed, and the checks it logs for each. A sweep logs one
+      # `Check` per mode per seed; the `[mode]` subject prefix tells the ledger which axis held or
+      # surprised, and each expectation is written before looking.
+      module Checks
+        MODE_EXPECTATIONS = {
+          differential:               "Ruby and the compiled Rust conformance binary agree on instances, events, refusals, " \
+                                      "reactions, sagas, queries and dry runs across every generated step",
+          properties_in_differential: "no declared property is violated over the Ruby side of the same " \
+                                      "replayed history",
+          self_consistency:           "each engine agrees with itself on rehydration/idempotency/value-object round trip",
+          ruby_only:                  "no declared property is violated and the interpreter does not crash across every " \
+                                      "generated step",
+          persistence_parity:         "Memory and a real, disposable PostgresEra-backed boot agree on instances, " \
+                                      "events, refusals, reactions, sagas and queries across every generated step",
+          adapter_parity_sqlite:      "Memory and a real, on-disk SQLite-backed boot agree on instances, events, " \
+                                      "refusals, reactions, sagas and queries across every generated step",
+          concurrency:                "two real, forked, cross-process dispatches of the same generated command against " \
+                                      "the same PostgresEra-backed schema settle on the same set of outcomes the identical " \
+                                      "pair settles on when dispatched sequentially with no contention",
+          era_boundary:               "no ancestor PostgresEra era this target's own real lineage holds still carries " \
+                                      "post-cut writes nobody has merged forward (Lineage#diverged_count, every ancestor, " \
+                                      "is zero)"
+        }.freeze
+
+        # Modes whose divergences are a pure function of the step list; a race, the per-sweep checks
+        # and generator crashes are not.
+        SHRINKABLE_MODES = %i[differential properties_in_differential self_consistency ruby_only
+                              adapter_parity_sqlite persistence_parity].freeze
+
+        private
+
+        # Flattens Replay's `:self_consistency` results into the same `{field:, detail:}` shape as
+        # every finding.
+        def self_consistency_divergences(history) = Hecks::Fuzzing::Differential.self_consistency_divergences(history)
+
+        def property_divergences(history) = Hecks::Fuzzing::Differential.property_divergences(history)
+
+        # The Ruby-vs-Rust diff from `spec/rust_conformance_fuzz_spec.rb`, one divergence list per
+        # active mode so `run_one_seed` logs a `Check` per mode.
+        def diff_ruby_vs_rust(differ, domain_path, steps, binary, modes:)
+          Hecks::Fuzzing::Differential.diff(
+            differ, domain_path, steps, binary,
+            modes: modes, adapter_parity_sqlite: -> { adapter_parity_sqlite_divergences(domain_path, steps) }
+          )
+        end
+
+        # Single-runtime fallback (`hecks fuzz`'s `outcome`) for domains with no compiled Rust
+        # binary: a violated property, or an exception escaping `Replay.call`.
+        def ruby_only_outcome(domain_path, steps, modes:)
+          self_consistency = modes.include?(:self_consistency)
+          history = Hecks::Fuzzing::Replay.call(domain_path, steps, self_consistency: self_consistency)
+          outcomes = { ruby_only: property_divergences(history) }
+          outcomes[:self_consistency] = self_consistency_divergences(history) if self_consistency
+          if modes.include?(:adapter_parity_sqlite)
+            outcomes[:adapter_parity_sqlite] = adapter_parity_sqlite_divergences(domain_path, steps)
+          end
+          outcomes
+        rescue StandardError => e
+          # A crash is one mode's finding, not the absence of the others: each other enabled mode
+          # gets an explicit not_checked entry so the ledger still holds one Check per resolved
+          # mode.
+          crashed = { ruby_only: [{ field: "crash", detail: "#{e.class}: #{e.message}" }] }
+          %i[self_consistency adapter_parity_sqlite].each do |mode|
+            next unless modes.include?(mode)
+
+            crashed[mode] = [{ field:  "not_checked",
+                               detail: "the Ruby replay crashed before #{mode} ran — #{e.class}: #{e.message}" }]
+          end
+          crashed
+        end
+
+        def log_check!(sweep, subject:, expectation:)
+          sweep.check!(subject: { value: subject }, expectation: { value: expectation })
+        end
+
+        def check_for(mode, feature, seed, shape, divergences)
+          { mode:        mode,
+            subject:     "[#{mode}] #{feature} fuzz seed #{seed} (#{shape})",
+            expectation: MODE_EXPECTATIONS.fetch(mode),
+            divergences: divergences,
+            clean:       divergences.empty?,
+            observation: divergences.empty? ? "held" : "diverged on: #{divergences.map { |d| d[:field] }.join(', ')}" }
+        end
+
+        # Extra `generate` arguments a guided seed needs in its `reproduce:` line; empty when
+        # unguided.
+        def plan_arguments(plan)
+          return "" unless plan
+
+          parts = []
+          parts << "prefix: #{plan.prefix.inspect}" if plan.prefix
+          parts << "favor: #{plan.favor.inspect}" unless plan.favor.empty?
+          parts.empty? ? "" : ", #{parts.join(', ')}"
+        end
+
+        # Once-per-sweep report of verbs skipped as not generated: held with the full list,
+        # surprised if any skipped verb's construct is outside the documented boundary.
+        def structural_skip_check(differ, binary, boundary)
+          skipped = differ.structural_skips.to_a.sort
+          gaps = Hecks::Fuzzing::RustGapManifest.for_binary(binary)
+          attributed = Hecks::Fuzzing::StructuralSkips.attribute(gaps, skipped)
+          outside = Hecks::Fuzzing::StructuralSkips.outside_boundary(attributed, boundary)
+          listing = attributed.map { |e| "#{e[:verb]} [#{e[:constructs].join(',')}]" }.join("; ")
+          divergences = outside.map do |e|
+            declares = e[:constructs].empty? ? "nothing" : e[:constructs].join(",")
+            { field: "structural_skip", verb: e[:verb], constructs: e[:constructs],
+              detail: "#{e[:verb]} was skipped as not generated (construct: #{declares}), outside " \
+                      "STRUCTURAL_REFUSAL_BOUNDARY — a codegen regression or a new gap, not a documented boundary" }
+          end
+          observation = if divergences.empty?
+                          "all #{skipped.size} inside the documented boundary"
+                        else
+                          "#{divergences.size} outside the boundary"
+                        end
+          { mode: :structural_skip_report,
+            subject: "[structural_skip_report] structurally skipped #{skipped.size} verb(s): [#{listing}]",
+            expectation: "every named query/read model skipped because manifest.json declares it not " \
+                         "generated names a construct inside QualityControlDials::STRUCTURAL_REFUSAL_BOUNDARY",
+            divergences: divergences, clean: divergences.empty?, observation: observation }
+        end
+
+        # `id:` addresses the Sweep; `sequence:` is the Check's own identity. The CLI's
+        # to.aggregate=/to.entity= split is a CLI-layer translation, not what in-process dispatch
+        # takes.
+        def mark_held!(sweep, sequence, observation)
+          @runtime.dispatch_flat("QualityControl::Sweep.Check.Held",
+                                 { id: sweep.id, sequence: { value: sequence }, observation: { value: observation } })
+        end
+
+        # `target:` is what the SuspendOnSurprise policy reads; the target is suspended inside this
+        # dispatch.
+        def mark_surprised!(sweep, sequence, observation, target_reference)
+          @runtime.dispatch_flat("QualityControl::Sweep.Check.Surprised",
+                                 { id: sweep.id, sequence: { value: sequence }, observation: { value: observation },
+                                   target: { value: target_reference } })
+        end
+
+        # One `PersistenceParity.diff` per seed against this sweep's disposable schema. A
+        # `PG::Error` or `WiringError` escaping it is a crash finding, never swallowed.
+        def persistence_parity_outcome(domain_path, steps, database, schema)
+          pair = @adapter_parity_pairs.fetch(:persistence_parity)
+          divergences = Hecks::Fuzzing::PersistenceParity.diff(domain_path, steps, left: pair.fetch(:left),
+                                                                                   right: pair.fetch(:right),
+                                                                                   database: database, schema: schema)
+          [divergences.empty? ? :clean : :divergence, divergences]
+        rescue StandardError => e
+          [:crash, [{ field: "process", detail: "#{e.class}: #{e.message}" }]]
+        end
+
+        # Memory vs SQLite through the same `PersistenceParity.diff`, run inside the primary seat's
+        # loop; an operational failure surfaces as a `process` finding.
+        def adapter_parity_sqlite_divergences(domain_path, steps)
+          pair = @adapter_parity_pairs.fetch(:adapter_parity_sqlite)
+          Hecks::Fuzzing::PersistenceParity.diff(domain_path, steps, left: pair.fetch(:left), right: pair.fetch(:right))
+        rescue StandardError => e
+          [{ field: "process", detail: "#{e.class}: #{e.message}" }]
+        end
+
+        # `ConcurrentDispatch.check` already returns the flat divergence shape and rescues to a
+        # `process` finding; `[]` means the sequence had no command step to race.
+        def concurrency_outcome(domain_path, steps, database, race_schema, reference_schema)
+          Hecks::Fuzzing::ConcurrentDispatch.check(domain_path, steps, database:         database,
+                                                                       race_schema:      race_schema,
+                                                                       reference_schema: reference_schema)
+        end
+
+        # Seedless, once per sweep: audits the target's own persisted lineage. Nothing to audit is a
+        # note, not a finding.
+        def era_boundary_check(domain_path)
+          result = Hecks::Fuzzing::EraBoundary.diverged_ancestor_writes(domain_path)
+          return unchecked_era_boundary(domain_path, result) unless result[:checked]
+
+          divergences =
+            if result[:diverged_total].positive?
+              [{ field: "era_boundary", breakdown: result[:breakdown],
+                 detail: "#{result[:diverged_total]} post-cut write(s) across #{result[:era_count]} era(s) " \
+                         "that nothing has ever merged forward — hecks merge_tail <this target's own domain " \
+                         "path> is the fix; see this script's own header on `era_boundary` for the class of " \
+                         "bug this is" }]
+            else
+              []
+            end
+
+          { mode: :era_boundary, subject: "[era_boundary] #{File.basename(domain_path)} — ancestor-era audit " \
+                                          "(#{result[:era_count]} era(s) on record)",
+            expectation: MODE_EXPECTATIONS.fetch(:era_boundary), divergences: divergences,
+            clean: divergences.empty?,
+            observation: divergences.empty? ? "no ancestor era holds an unmerged write" : "#{result[:diverged_total]} diverged" }
+        end
+
+        # Nothing to audit and could not audit stay apart: `not_applicable` logs no `Check`, while
+        # an error is a finding, so an unreachable database never counts as a held `Check` toward
+        # the streak.
+        def unchecked_era_boundary(domain_path, result)
+          if result[:kind] == :not_applicable
+            return { mode: :era_boundary, skip: true, observation: "not applicable — #{result[:reason]}" }
+          end
+
+          { mode: :era_boundary, subject: "[era_boundary] #{File.basename(domain_path)} — ancestor-era audit",
+            expectation: MODE_EXPECTATIONS.fetch(:era_boundary),
+            divergences: [{ field: "era_boundary_unchecked", detail: result[:reason] }], clean: false,
+            observation: "could not audit — #{result[:reason]}" }
+        end
+
+        # Runs one seed under every active mode: `{ steps:, trace:, checks: }`, one check per mode;
+        # logging is the loop's job. A generator crash is a finding too: a step broke the
+        # interpreter before replay.
+        def run_one_seed(seat, seed)
+          shape = "#{@steps_per_sequence} steps, adversarial #{@adversarial}, role_draw #{@role_draw}, " \
+                  "dry_run #{@dry_run}"
+          plan = @campaign&.plan(seed)
+          shape += ", spliced" if plan&.spliced?
+          begin
+            trace = Hecks::Fuzzing::SequenceGenerator.trace(
+              @domain_path, seed: seed, steps: @steps_per_sequence, adversarial: @adversarial,
+                            role_draw: @role_draw, dry_run: @dry_run, **(plan ? plan.generator_options : {})
+            )
+            steps = trace.steps
+          rescue StandardError => e
+            return { plan: plan, steps: [], checks: [generator_crash_check(seed, shape, e)] }
+          end
+
+          outcomes = seed_outcomes(seat, steps)
+          { plan: plan, steps: steps, trace: trace,
+            checks: outcomes.map { |mode_key, divergences| check_for(mode_key, @feature, seed, shape, divergences) } }
+        end
+
+        def generator_crash_check(seed, shape, error)
+          { mode: :generator, subject: "[generator] #{@feature} fuzz seed #{seed} (#{shape}) — sequence generation",
+            expectation: "SequenceGenerator builds every step without anything but a domain refusal escaping " \
+                         "its own inline dispatch",
+            divergences: [{ field:  "generator_crash",
+                            detail: "#{error.class}: #{error.message}\n#{error.backtrace.first(6).join("\n")}" }],
+            clean: false, observation: "generator crashed: #{error.class}: #{error.message}" }
+        end
+
+        def seed_outcomes(seat, steps)
+          case seat
+          when :differential then diff_ruby_vs_rust(@differ, @domain_path, steps, @binary, modes: @active_modes)
+          when :persistence_parity
+            { persistence_parity: persistence_parity_outcome(@domain_path, steps, @parity_database,
+                                                             @parity_schema).last }
+          when :concurrency
+            { concurrency: concurrency_outcome(@domain_path, steps, @concurrency_database, @race_schema,
+                                               @reference_schema) }
+          else ruby_only_outcome(@domain_path, steps, modes: @active_modes)
+          end
+        end
+
+        # Re-runs only the comparison that surprised; `seat` matters only for `self_consistency`,
+        # which reads the Rust binary's rehydration door under a differential seat.
+        def candidate_divergences(check_mode, seat, steps)
+          case check_mode
+          when :differential
+            diff_ruby_vs_rust(@differ, @domain_path, steps, @binary, modes: [:differential])[:differential]
+          when :properties_in_differential, :ruby_only
+            ruby_only_outcome(@domain_path, steps, modes: [])[:ruby_only]
+          when :self_consistency then self_consistency_candidate(seat, steps)
+          when :adapter_parity_sqlite then adapter_parity_sqlite_divergences(@domain_path, steps)
+          when :persistence_parity
+            persistence_parity_outcome(@domain_path, steps, @parity_database, @parity_schema).last
+          end || []
+        end
+
+        def self_consistency_candidate(seat, steps)
+          if seat == :differential
+            diff_ruby_vs_rust(@differ, @domain_path, steps, @binary, modes: [:self_consistency])[:self_consistency]
+          else
+            ruby_only_outcome(@domain_path, steps, modes: [:self_consistency])[:self_consistency]
+          end
+        end
+
+        # Keeps a candidate only while its `Shrinker.signature` still contains the original's; a
+        # raising candidate is a different finding, so it is rejected.
+        def shrink_check(check, seat, steps)
+          original = Hecks::Fuzzing::Shrinker.signature(check[:divergences])
+          result = Hecks::Fuzzing::Shrinker.call(steps, budget: @shrink_budget) do |candidate|
+            divergences = candidate_divergences(check[:mode], seat, candidate)
+            Hecks::Fuzzing::Shrinker.reproduces?(original, divergences)
+          rescue StandardError
+            false
+          end
+          { mode: check[:mode], steps: result.steps, attempts: result.attempts, exhausted: result.exhausted,
+            original_size: steps.size }
+        end
+
+        # Writes the shrunk steps in `hecks fuzz`'s `{name, note, steps}` shape so `hecks run` and
+        # `hecks check_conformance` replay it unchanged.
+        def write_shrunk!(shrunk, seed)
+          dir = File.join(@root, "tmp/qa-shrunk")
+          FileUtils.mkdir_p(dir)
+          path = File.join(dir, "#{filesystem_safe_component(@sweep_reference)}-#{shrunk[:mode]}.json")
+          File.write(path, JSON.pretty_generate(name:  "#{@feature}-#{shrunk[:mode]}-shrunk",
+                                                note:  "hecks quality_control ask run #{@sweep_reference} seed #{seed}, shrunk " \
+                                                       "from #{shrunk[:original_size]} steps",
+                                                steps: shrunk[:steps]))
+          path
+        end
+
+        # Coverage-guided generation's corpus, one JSON file per (target, mode): `--all`'s waves can
+        # run a target's primary sweep and its persistence-parity/concurrency variants as separate
+        # `hecks quality_control ask run` processes within the same tick, and each explores a
+        # conceptually independent axis of the same domain, so each keeps its own file rather than
+        # sharing one that nothing here locks.
+        def coverage_corpus_path(target_reference, mode)
+          File.join(@coverage_corpus_dir, "#{filesystem_safe_component(target_reference)}-#{mode}.json")
+        end
+
+        # Nil when nothing was ever saved for this (target, mode), or the file cannot be parsed;
+        # either way the campaign starts from an empty corpus.
+        def load_coverage_state(path)
+          JSON.parse(File.read(path))
+        rescue Errno::ENOENT, JSON::ParserError
+          nil
+        end
+
+        def save_coverage_state!(path, campaign)
+          FileUtils.mkdir_p(File.dirname(path))
+          File.write(path, JSON.pretty_generate(campaign.to_h))
+        end
+      end
+    end
+  end
+end