hecks 1.2.0 → 1.4.0

data/lib/hecks/fuzzing/concurrent_dispatch.rb

@@ -0,0 +1,361 @@
+require "fileutils"
+require "tmpdir"
+require "tempfile"
+require "json"
+require_relative "isolated_boot"
+require_relative "../naming"
+
+module Hecks
+  module Fuzzing
+    # **A generated sequence, raced for real** — `spec/adapters/driven/
+    # postgres_era_concurrent_dispatch_spec.rb` proves ADR 0036's own fix
+    # (a real `pg_advisory_xact_lock` serializes a PostgresEra-bound
+    # dispatch across separate OS processes) against one hand-authored
+    # fixture and one hand-picked conflicting pair (two `$6,000` Debits
+    # against a `$10,000` account). This module asks the same question —
+    # does the cross-process write lock actually serialize concurrent
+    # writers? — of an arbitrary generated sequence against a real target
+    # domain, so the practice's own adversarial generator gets to pick the
+    # conflict instead of a human picking it once and never again.
+    #
+    # The mechanism, generalized from that spec rather than REDERIVED: a
+    # command step partway through a generated sequence is chosen as the
+    # race step; every step before it is setup (replayed once, sequentially,
+    # to bring a fresh disposable schema to the state the race step expects
+    # to act against); the race step is then dispatched twice — once from
+    # each of two real, separate, forked OS processes racing against that
+    # same schema, no artificial gating, whichever the scheduler favors.
+    #
+    # The oracle is the same pair, dispatched sequentially, not a
+    # hardcoded expectation — unlike the hand-authored spec (which can
+    # assert `%w[refused succeeded]` because it knows its own fixture's
+    # business rule), this module has no idea whether an arbitrary
+    # generated race step conflicts with itself at all. So it asks the
+    # domain: replay the identical setup on a separate fresh schema, then
+    # dispatch the same race step twice, one after the other, in one
+    # process, with no contention at all — genuinely correct by
+    # construction, since nothing else can touch that schema while it
+    # runs. Two conflicting writes settle as {"succeeded", "refused"};
+    # two independent ones settle as {"succeeded", "succeeded"}; either
+    # way, that is the multiset the concurrent pair must also produce if
+    # the write lock actually serializes them — order does not matter
+    # (which real racer wins a genuine race is never controlled), the set
+    # of outcomes does.
+    #
+    # What a broken lock looks like here: the concurrent pair settling as
+    # {"succeeded", "succeeded"} where the sequential oracle says
+    # {"succeeded", "refused"} — two processes each hydrated the
+    # pre-write state, neither saw the other's write, and the second
+    # commit landed as a silent lost update instead of failing its own
+    # `given`. That is the exact corruption class ADR 0036 fixed for
+    # PostgresEra and `postgres_concurrent_dispatch_spec.rb` still
+    # documents, unfixed, for plain Postgres.
+    #
+    # Not a replacement for the hand-authored spec — that spec proves the
+    # mechanism once, precisely, with controlled gating so the assertion
+    # is deterministic; this module proves the same mechanism holds for
+    # whatever a real domain's own generated sequences throw at it, with
+    # no gating (a genuine, ungated race), on every sweep this mode runs.
+    module ConcurrentDispatch
+      module_function
+
+      COMMAND_STEP = ->(step) { step["verb"] && !step["query"] && !step["dry_run"] }
+
+      # One divergence list, the same shape every other mode in this
+      # practice produces — `[]` when nothing was found (including the
+      # legitimate "this seed's generated sequence has no command step to
+      # race at all" case: a sequence of pure queries/dry-runs has nothing
+      # to concurrently dispatch, and that is not a finding).
+      #
+      # `database:` is the shared, never-dropped scratch database
+      # (`bin/qa_sweep`'s own `persistence_parity_database`, reused here
+      # for the identical reason: a container, not the thing that's
+      # unique per run). `race_schema:`/`reference_schema:` are two
+      # different disposable schema names this one call owns for its own
+      # duration — the caller creates neither ahead of time (both are
+      # wiped fresh by the boots below) and drops both afterward, the
+      # same lifecycle `persistence_parity_schema` already has.
+      def check(domain_path, steps, database:, race_schema:, reference_schema:)
+        normalized = steps.map { |step| step.transform_keys(&:to_s) }
+        lockable, probe_errors = lockable_verbs(domain_path, normalized, database: database, schema: reference_schema)
+        race_index = pick_race_index(normalized, lockable)
+        unless race_index
+          # `[]` here means "nothing to race", and must only ever mean that.
+          # A sequence of pure queries/dry-runs is a legitimate clean
+          # result; a probe that raised for every verb is not — that one
+          # used to produce the identical `[]` and be logged as a clean
+          # concurrency Check (see `lockable_verbs`).
+          return [] if probe_errors.empty?
+
+          return [{ field:  "concurrency_unraceable",
+                    detail: "no command step could be raced because the cross-process-lock probe failed: " \
+                            "#{probe_errors.uniq.join('; ')}" }]
+        end
+
+        setup_steps = normalized[0...race_index]
+        race_step   = normalized[race_index]
+
+        reference  = reference_outcomes(domain_path, setup_steps, race_step, database: database, schema: reference_schema)
+        concurrent = concurrent_outcomes(domain_path, setup_steps, race_step, database: database, schema: race_schema)
+
+        divergences_for(race_step, reference, concurrent)
+      rescue StandardError => e
+        [{ field: "process", detail: "#{e.class}: #{e.message}" }]
+      end
+
+      # The command step CLOSEST to the middle of the sequence — not the
+      # first (racing a bare identity-creation with no setup at all is a
+      # legitimate, useful case, so index 0 is not excluded) and not
+      # chosen for any domain-specific reason: a mid-sequence step has, on
+      # average, the deepest state to act against and the best odds of
+      # actually conflicting with itself. `nil` when the generated
+      # sequence has no command step at all (every step a query or a dry
+      # run) — nothing here for this seed to race.
+      #
+      # `lockable_verbs` — optional, but `check` always passes one: the
+      # set of verbs whose own aggregate is actually bound to an adapter
+      # that declares `:cross_process_lock` in this domain (see
+      # `lockable_verbs` below for why this can't be assumed just because
+      # the domain binds some of its own aggregates to PostgresEra).
+      # Racing anything outside that set is not a legitimate race at
+      # all — nil when nothing eligible is left, same as the "no command
+      # step" case, never a finding of its own.
+      def pick_race_index(steps, lockable_verbs = nil)
+        command_indices = steps.each_index.select { |i| COMMAND_STEP.call(steps[i]) }
+        command_indices = command_indices.select { |i| lockable_verbs.include?(steps[i]["verb"]) } if lockable_verbs
+        return nil if command_indices.empty?
+
+        command_indices[command_indices.size / 2]
+      end
+
+      # Which of this sequence's own command verbs are even candidates to
+      # race — BUG#142 (SW-quality_control-1789768606's own first real
+      # concurrency run, seed 3): a domain that `persisted_by("PostgresEra")`
+      # binds its own aggregates is not thereby binding a framework
+      # member it merely `uses_framework`s — `hecksagon_builder.rb`'s own
+      # `uses_framework` loads only that member's shape, never its
+      # persistence (see `examples/banking/bluebook/banking.hecksagon`'s
+      # own comment: a framework member's aggregates need a sibling
+      # hecksagon, registered under that member's own name, or
+      # `Ports::Persistence::BindingPolicy.default_binding` silently
+      # gives them "Memory" — no hecksagon registered under that name at
+      # all, so `resolve`'s own `missing_binding` refusal (which only
+      # fires when a hecksagon exists for that domain and simply omits
+      # this aggregate) never gets a chance to say so).
+      # `qa/bluebook/quality_control.hecksagon` attaches `Governance` via
+      # `uses_framework` with no such sibling — `Governance::
+      # RoleAssignment`/`RoleTransition` are Memory-backed, process-
+      # local, in the real ledger, `concurrency` mode included. Racing a
+      # Memory-backed aggregate across two real OS processes can never
+      # agree with the single-process sequential oracle — each racer's
+      # own boot gets its own independent, empty store — no matter how
+      # correct any write lock is; that is a guaranteed false positive,
+      # not evidence of a broken lock.
+      #
+      # Boots the same PostgresEra-rebound copy the race itself boots
+      # (`boot_preserving_schema` — structural inspection only, nothing
+      # dispatched, so the schema it's given is left exactly as it found
+      # it) and asks each distinct command verb's own resolved repository
+      # whether it actually declares `:cross_process_lock` — the same
+      # capability `Interpreting#run_dispatch_order_with_isolation`
+      # itself keys off of to decide whether a real advisory lock is
+      # even in play for that aggregate. A verb this boot can't resolve
+      # at all (a malformed adversarial verb, say) is conservatively
+      # excluded, not raced on a guess.
+      # Answers `[lockable, probe_errors]`. The errors half exists because
+      # the probe below rescues to `false`: a verb that cannot be resolved
+      # is indistinguishable, from the outside, from one that resolves fine
+      # and simply declares no `:cross_process_lock`. If resolution broke
+      # for every verb (a renamed capability symbol, a wiring change),
+      # `lockable` came back empty, `check` returned `[]`, and the sweep
+      # logged a clean concurrency Check for a race that never happened.
+      # `check` reports that case now instead of holding it.
+      def lockable_verbs(domain_path, steps, database:, schema:)
+        verbs = steps.select { |step| COMMAND_STEP.call(step) }.map { |step| step["verb"] }.uniq
+        lockable = []
+        probe_errors = []
+        boot_preserving_schema(domain_path, database: database, schema: schema) do |copy|
+          runtime = Hecks.boot(copy)
+          verbs.each do |verb|
+            lockable << verb if verb_cross_process_lockable?(runtime, verb, probe_errors)
+          end
+        end
+        [lockable, probe_errors]
+      end
+
+      def verb_cross_process_lockable?(runtime, verb, probe_errors = [])
+        domain, aggregate_name, = Naming.split_verb(verb)
+        return false unless domain && aggregate_name
+
+        aggregate = runtime.registry.bluebook(domain)&.aggregate(aggregate_name)
+        return false unless aggregate
+
+        repository = runtime.registry.repository(domain, aggregate)
+        repository.capabilities.include?(:cross_process_lock)
+      rescue StandardError => e
+        # Still `false` — a verb this boot cannot resolve is not raced on a
+        # guess — but no longer silent: `check` needs to tell "nothing here
+        # declares a cross-process lock" from "asking broke".
+        probe_errors << "#{verb}: #{e.class}: #{e.message}"
+        false
+      end
+
+      def divergences_for(race_step, reference, concurrent)
+        crashes = (reference + concurrent).select { |outcome| outcome.start_with?("crashed:") }.uniq
+        return crashes.map { |c| { field: "concurrency_crash", verb: race_step["verb"], detail: c } } if crashes.any?
+
+        return [] if reference.sort == concurrent.sort
+
+        [{ field: "concurrency_race", verb: race_step["verb"], reference: reference, concurrent: concurrent,
+           detail: "two concurrent cross-process dispatches of #{race_step['verb']} settled as #{concurrent.sort} " \
+                   "where the identical pair, dispatched sequentially with no contention, settled as " \
+                   "#{reference.sort} — the cross-process write lock did not correctly serialize this write" }]
+      end
+
+      # The oracle — one boot, one process, the setup then the race step
+      # twice in immediate succession. Nothing else ever touches this
+      # schema while this runs, so whatever the domain itself settles on
+      # is correct by construction, not asserted.
+      def reference_outcomes(domain_path, setup_steps, race_step, database:, schema:)
+        outcomes = []
+        IsolatedBoot.call(domain_path, adapter: :postgres_era, database: database, schema: schema) do |copy|
+          runtime = Hecks.boot(copy)
+          dispatch_all!(runtime, setup_steps)
+          outcomes << dispatch_one(runtime, race_step)
+          outcomes << dispatch_one(runtime, race_step)
+        end
+        outcomes
+      end
+
+      # The race itself — setup runs once, sequentially, in this process
+      # (the same `IsolatedBoot.call` wipe-then-boot every other mode
+      # here already uses), and only then do the two real racers run.
+      # Each racer boots its own fresh copy of the domain against the
+      # same now-populated schema — `boot_preserving_schema`, below,
+      # deliberately skips the wipe `IsolatedBoot.call` always does, or
+      # the setup this line just wrote would be gone before either racer
+      # ever dispatched anything.
+      #
+      # Real, separate OS processes, not `Thread.new` — `postgres_era_
+      # concurrent_dispatch_spec.rb`'s own header explains why:
+      # `Runtime::AggregateLock`'s in-process registry would fully (and
+      # misleadingly) serialize two threads sharing one process even with
+      # the cross-process lock fix reverted. Only two genuinely separate
+      # OS processes exercise the gap this check exists to catch.
+      #
+      # `Process.spawn`, not `Process.fork` — `bin/qa_sweep`'s own
+      # top-of-file comment on `--all` names the identical hazard this
+      # sidesteps: by the time a `concurrency` seed runs, this process
+      # already holds the QualityControl ledger's own live PostgresEra
+      # connection (this module's own caller, `bin/qa_sweep`, booted it
+      # long before any seed ran). `Process.fork` duplicates every open
+      # file descriptor, SSL session state included — confirmed live
+      # while wiring this mode up: forking directly from here corrupted
+      # the ledger's own connection the moment either racer child exited,
+      # surfacing on the next unrelated ledger write, nowhere near this
+      # method's own code. `bin/qa_concurrency_racer` is this method's own
+      # worker, one real `ruby` process per racer — read that script's own
+      # header for the rest of this reasoning.
+      def concurrent_outcomes(domain_path, setup_steps, race_step, database:, schema:)
+        IsolatedBoot.call(domain_path, adapter: :postgres_era, database: database, schema: schema) do |copy|
+          dispatch_all!(Hecks.boot(copy), setup_steps)
+        end
+
+        root = File.expand_path("../../..", __dir__)
+        racer = File.join(root, "bin/qa_concurrency_racer")
+        args_json = JSON.generate(race_step["args"] || {})
+        logs = Array.new(2) { Tempfile.new(["qa-concurrency-racer-", ".log"]) }
+        logs.each(&:unlink)
+
+        pids = logs.map do |log|
+          Process.spawn("bundle", "exec", "ruby", racer, domain_path, database, schema, race_step["verb"], args_json,
+                        out: log, err: log, chdir: root)
+        end
+
+        pids.each { |pid| Process.wait(pid) }
+        logs.map do |log|
+          log.rewind
+          output = log.read
+          log.close
+          output.strip.empty? ? "crashed:no output from bin/qa_concurrency_racer" : output.lines.last.chomp
+        end
+      end
+
+      # One step, one outcome — never raises: a declared domain refusal is
+      # "refused" (the expected, ordinary answer a `given`/invariant can
+      # give), anything else escaping is "crashed:<class>: <message>", a
+      # genuine finding this module's own caller surfaces rather than lets
+      # kill a forked racer silently.
+      def dispatch_one(runtime, step)
+        args = (step["args"] || {}).transform_keys(&:to_sym)
+        runtime.dispatch_flat(step["verb"], args)
+        "succeeded"
+      rescue *Hecks::Runtime::DOMAIN_REFUSALS, Hecks::Bluebook::Expression::EvaluationError
+        "refused"
+      rescue StandardError => e
+        "crashed:#{e.class}: #{e.message}"
+      end
+
+      # Setup tolerates an ordinary refusal (a generated sequence's own
+      # earlier step can legitimately refuse — every other mode in this
+      # practice already replays a prefix that way) but never a crash: an
+      # unexpected exception during setup means the schema this race is
+      # about to run against is in an unknown state, which is itself
+      # worth surfacing, not silently racing anyway.
+      def dispatch_all!(runtime, steps)
+        steps.each do |step|
+          outcome = dispatch_one(runtime, step)
+          raise "setup step #{step['verb']} #{outcome}" if outcome.start_with?("crashed:")
+        end
+      end
+
+      # The same copy-and-rebind `IsolatedBoot.call(..., adapter:
+      # :postgres_era, ...)` does, minus the schema wipe — deliberately
+      # not reusing `IsolatedBoot.rebind_to_postgres_era!` itself, which
+      # bundles `ensure_postgres_era_schema!`'s own `DROP SCHEMA` into the
+      # same call with no way to opt out (see that method's own header:
+      # the wipe is the "zero-history guarantee every other adapter mode
+      # already gives," exactly the guarantee this caller must not have —
+      # the whole point of a race is booting against what setup already
+      # wrote). `copy_dereferencing`/`rewrite_bindings!` are the same two
+      # public steps that method itself calls first; only the `.world`
+      # this writes is duplicated from it, not re-derived, because the
+      # shape a `PostgresEra`-bound copy's `.world` needs is exactly that
+      # method's own, one step short.
+      def boot_preserving_schema(domain_path, database:, schema:)
+        Dir.mktmpdir("hecks-concurrency") do |tmp|
+          copy = File.join(tmp, File.basename(domain_path))
+          IsolatedBoot.copy_dereferencing(domain_path, copy)
+          FileUtils.rm_rf(File.join(copy, "data"))
+          IsolatedBoot.rewrite_bindings!(copy, "PostgresEra")
+          write_postgres_era_world!(copy, database: database, schema: schema)
+          yield copy
+        end
+      end
+
+      def write_postgres_era_world!(copy, database:, schema:)
+        Dir.glob(File.join(copy, "**", "*.hecksagon")).each do |hecksagon_path|
+          names = File.read(hecksagon_path).scan(/Hecks\.hecksagon\s+"([^"]+)"/).flatten.uniq
+          next if names.empty?
+
+          world_path = File.join(File.dirname(hecksagon_path), "hecks_fuzz_postgres_era.world")
+          File.write(world_path, names.map do |name|
+            <<~WORLD
+              Hecks.world "#{name}" do
+                persisted_by("PostgresEra") do
+                  database "#{database}"
+                  schema "#{schema}"
+                  allow_superuser true
+                end
+              end
+            WORLD
+          end.join("\n"))
+        end
+
+        Dir.glob(File.join(copy, "**", "*.world")).each do |path|
+          File.delete(path) unless File.basename(path) == "hecks_fuzz_postgres_era.world"
+        end
+      end
+    end
+  end
+end

data/lib/hecks/fuzzing/coverage_campaign.rb

@@ -0,0 +1,118 @@
+module Hecks
+  module Fuzzing
+    # What one sweep has already reached, and where the next seed starts.
+    #
+    # `SequenceGenerator` steers within one sequence (an unexercised verb is
+    # weighted up — picker.rb's `steer`), but every seed of a sweep used to
+    # start from nothing and forget everything the seeds before it reached.
+    # Roughly half of the QA ledger's bugs are refusal-kind splits (BUG#2–4,
+    # 7, 8, 14–16, 19–21, 23, 27, 28, 36–38, 41, 54, 56): a particular verb,
+    # in a particular state, with a particular malformed argument, refused
+    # one way on Ruby and another on Rust. BUG#11 was found by hand because
+    # a 25-step random walk never reached `Annotate` at all. Uniform seeds
+    # reach those corners by luck; this makes the sweep remember.
+    #
+    # The coverage unit is a tuple, not a verb — `verb | step kind |
+    # lifecycle state before | adversarial mutation | outcome` (see
+    # `SequenceGenerator#coverage_tuple`). "Renew refused LifecycleRefused
+    # from free" and "Renew ok from held" are different places a runtime can
+    # be wrong, where a verb-level count calls them the same.
+    #
+    # Two levers, both expressed as plain generator arguments so a seed
+    # stays reproducible from one call:
+    #
+    #   1. Splicing (`prefix:`). A sequence that reached a new tuple joins
+    #      the corpus, cut at its last new tuple. A later seed may start by
+    #      re-generating a random-length prefix of a corpus entry — the same
+    #      seed, favor and nested prefix, so the same steps, dispatched for
+    #      real, reaching the same state — and then carries on with its own
+    #      seed's randomness from there. Deep state gets reached once and
+    #      explored many times, instead of re-rolled per seed.
+    #   2. Favor (`favor:`). The verbs this sweep has hit least often —
+    #      including declared verbs it has never hit at all — are weighted
+    #      up for the next seed.
+    #
+    # Nothing here is random at the campaign's own level beyond
+    # `Random.new(seed)` per plan, and a plan is printed with every finding
+    # (`bin/qa_sweep`'s `reproduce:` line), so no finding depends on
+    # re-running the whole campaign to get back to it.
+    class CoverageCampaign
+      Plan = Struct.new(:prefix, :favor, keyword_init: true) do
+        def generator_options = { prefix: prefix, favor: favor }
+        def spliced? = !prefix.nil?
+      end
+
+      attr_reader :corpus
+
+      def initialize(splice_probability:, favor_count:, corpus_limit: 64, max_prefix_depth: 6)
+        @splice_probability = splice_probability.to_f
+        @favor_count        = favor_count.to_i
+        @corpus_limit       = corpus_limit
+        @max_prefix_depth   = max_prefix_depth
+        @corpus             = []
+        @seen               = Set.new
+        @verb_hits          = Hash.new(0)
+        @declared_verbs     = Set.new
+        @seeds              = 0
+        @spliced            = 0
+        @seeds_with_new     = 0
+      end
+
+      def plan(seed)
+        random = Random.new(seed)
+        prefix = nil
+        if !@corpus.empty? && random.rand < @splice_probability
+          entry  = @corpus[random.rand(@corpus.size)]
+          prefix = entry[:spec].merge("steps" => random.rand(1..entry[:attempts]))
+        end
+        Plan.new(prefix: prefix, favor: rare_verbs)
+      end
+
+      # `trace` is `SequenceGenerator.trace`'s answer: `coverage` is
+      # `[[attempt_index, tuple], ...]`, `verbs` every verb the booted
+      # catalog offered.
+      def record(seed, plan, trace)
+        @seeds   += 1
+        @spliced += 1 if plan.spliced?
+        @declared_verbs.merge(trace.verbs)
+
+        new_at = trace.coverage.filter_map do |index, tuple|
+          @verb_hits[tuple.split(" | ").first] += 1
+          index if @seen.add?(tuple)
+        end
+        return if new_at.empty?
+
+        @seeds_with_new += 1
+        admit(seed, plan, new_at.max + 1)
+      end
+
+      def tuples_seen = @seen.size
+
+      def summary
+        "coverage: #{@seen.size} distinct (verb, kind, state, mutation, outcome) tuple(s) over #{@seeds} seed(s); " \
+          "#{@seeds_with_new} seed(s) reached something new; #{@spliced} spliced from a corpus of #{@corpus.size}"
+      end
+
+      private
+
+      def admit(seed, plan, attempts)
+        return if depth(plan.prefix) + 1 > @max_prefix_depth
+
+        spec = { "seed" => seed, "favor" => plan.favor }
+        spec["prefix"] = plan.prefix if plan.prefix
+        @corpus << { spec: spec, attempts: attempts }
+        @corpus.shift while @corpus.size > @corpus_limit
+      end
+
+      def depth(prefix) = prefix.nil? ? 0 : 1 + depth(prefix["prefix"])
+
+      # Never-hit declared verbs first (hits 0), then the least-hit, ties
+      # broken by name so a plan is a pure function of what was recorded.
+      def rare_verbs
+        return [] unless @favor_count.positive?
+
+        (@declared_verbs | @verb_hits.keys).min_by(@favor_count) { |verb| [@verb_hits[verb], verb] }
+      end
+    end
+  end
+end

data/lib/hecks/fuzzing/differential.rb

@@ -0,0 +1,158 @@
+require "json"
+require "open3"
+require_relative "replay"
+require_relative "properties"
+require_relative "self_consistency"
+require_relative "rust_gap_manifest"
+require_relative "nondeterministic"
+
+module Hecks
+  module Fuzzing
+    # The Ruby-vs-Rust comparison of one generated sequence — moved here,
+    # unchanged, out of `bin/qa_sweep`'s own `diff_ruby_vs_rust` so a second
+    # caller (`bin/qa_generated_domains --rust`, which compares domains the
+    # generator wrote rather than ones on the rotation) runs the same
+    # comparison rather than a re-derived one. `bin/qa_sweep` delegates to
+    # it; its own comments on the known-gap filtering, the wire-precision
+    # reduction and the deep copy taken before `strip_emitted_flags!` apply
+    # here word for word.
+    #
+    # `differ` is anything answering `RustConformanceHelpers`' comparison
+    # helpers (spec/support/rust_conformance_helpers.rb) plus a
+    # `structural_skips` set — duck-typed, so lib never requires spec/.
+    # Which query verbs may diverge is not the differ's call: it is read
+    # off the binary's own manifest.json (`manifest_partition`, below).
+    #
+    # Returns one divergence list per active mode — `{ differential: [...],
+    # self_consistency: [...], properties_in_differential: [...],
+    # adapter_parity_sqlite: [...] }`, keys present only for the modes in
+    # `modes:` (`differential` always). `adapter_parity_sqlite:` is the
+    # caller's own callable, because which two adapters that mode pairs is
+    # the caller's dial, not this module's.
+    module Differential
+      module_function
+
+      # The one place a Ruby/Rust query divergence may be tolerated — and
+      # only for a verb `gaps` (a `RustGapManifest`) declares not generated.
+      # Rust refuses such a verb outright while Ruby answers it for real
+      # (or refuses it for its own business reason), so both sides' rows for
+      # that verb leave the comparison. Everything else stays in: a refusal
+      # the manifest doesn't account for — however its message is worded —
+      # is a real divergence.
+      #
+      # `skipped` is every tolerated verb this history actually reached (for
+      # the sweep's structural_skip_report). `stale` is a divergence per
+      # tolerated verb Rust nonetheless answered with a query row: the
+      # manifest says "not generated" but the binary disagrees, so the
+      # tolerance itself is wrong and must not silently hold.
+      def manifest_partition(gaps, ruby_refusals:, rust_refusals:, ruby_queries:, rust_queries:)
+        verb_of  = ->(row) { row.key?("verb") ? row["verb"] : row["query"] }
+        declared = ->(row) { gaps.not_generated?(verb_of.call(row)) }
+        stale = rust_queries.select(&declared).map do |row|
+          { field: "manifest", verb: row["query"],
+            detail: "#{row['query']} is declared generated: false in manifest.json " \
+                    "(#{gaps.not_generated(row['query']).values_at('gap_class', 'construct').join('/')}), " \
+                    "but the Rust binary answered it — regenerate with bin/project_rust" }
+        end
+        reached = (ruby_refusals + rust_refusals + ruby_queries).select(&declared)
+        { ruby_refusals: ruby_refusals.reject(&declared), rust_refusals: rust_refusals.reject(&declared),
+          ruby_queries: ruby_queries.reject(&declared), rust_queries: rust_queries.reject(&declared),
+          skipped: reached.to_set(&verb_of), stale: stale }
+      end
+
+      def property_divergences(history)
+        Properties.check(history).reject { |_, result| result == true }
+                  .map { |name, message| { field: name.to_s, detail: message } }
+      end
+
+      def self_consistency_divergences(history)
+        return [] unless history[:self_consistency]
+
+        history[:self_consistency].values.flatten(1)
+      end
+
+      # rubocop:disable-next Metrics/AbcSize
+      # rubocop:disable-next Metrics/CyclomaticComplexity
+      # rubocop:disable-next Metrics/PerceivedComplexity
+      # rubocop:disable-next Metrics/MethodLength
+      def diff(differ, domain_path, steps, binary, modes:, adapter_parity_sqlite: nil)
+        self_consistency = modes.include?(:self_consistency)
+        ruby_result    = Replay.call(domain_path, steps, self_consistency: self_consistency)
+        ruby_instances = JSON.parse(JSON.generate(ruby_result[:instances]))
+        ruby_events    = JSON.parse(JSON.generate(ruby_result[:events]))
+        ruby_refusals  = ruby_result[:refusals].map do |r|
+          { "verb" => r[:verb].to_s, "kind" => r[:kind].to_s.split("::").last, "error" => r[:error] }
+        end
+        ruby_queries  = JSON.parse(JSON.generate(ruby_result[:queries].map { |row| Nondeterministic.strip(row, :query_row) }))
+        ruby_sagas    = JSON.parse(JSON.generate(ruby_result[:sagas]))
+        ruby_dry_runs = ruby_result[:dry_runs].map { |d| { "verb" => d[:verb].to_s, "ok" => d[:ok] } }
+
+        outcomes = {}
+        outcomes[:properties_in_differential] = property_divergences(ruby_result) if modes.include?(:properties_in_differential)
+        if modes.include?(:adapter_parity_sqlite) && adapter_parity_sqlite
+          outcomes[:adapter_parity_sqlite] = adapter_parity_sqlite.call
+        end
+
+        stdout, status = Open3.capture2(binary, stdin_data: JSON.generate({ "steps" => steps }))
+        unless status.success?
+          return outcomes.merge(differential: [{ field:  "process",
+                                                 detail: "rust binary exited #{status.exitstatus}: #{stdout}" }])
+        end
+
+        rust_output = JSON.parse(stdout)
+        rust_live_instances = JSON.parse(JSON.generate(rust_output["instances"])) if self_consistency
+        differ.strip_emitted_flags!(rust_output["instances"])
+        differ.strip_emitted_flags!(rust_output["queries"])
+        differ.strip_occurred_at!(rust_output["events"])
+
+        divergences = []
+        divergences << { field: "instances", ruby: ruby_instances, rust: rust_output["instances"] } \
+          unless rust_output["instances"] == ruby_instances
+        divergences << { field: "events", ruby: ruby_events, rust: rust_output["events"] } \
+          unless rust_output["events"] == ruby_events
+
+        kept = manifest_partition(RustGapManifest.for_binary(binary),
+                                  ruby_refusals: ruby_refusals, rust_refusals: rust_output["refusals"],
+                                  ruby_queries: ruby_queries, rust_queries: rust_output["queries"])
+        differ.structural_skips.merge(kept[:skipped])
+        divergences.concat(kept[:stale])
+
+        by_kind = ->(r) { r.slice("verb", "kind") }
+        rust_refusals      = kept[:rust_refusals].map(&by_kind)
+        kept_ruby_refusals = kept[:ruby_refusals].map(&by_kind)
+        divergences << { field: "refusals", ruby: kept_ruby_refusals, rust: rust_refusals } \
+          unless rust_refusals == kept_ruby_refusals
+
+        wordless          = ->(row) { differ.reduce_to_wire_precision(row.except("error", "reference_error")) }
+        rust_queries      = kept[:rust_queries].map(&wordless)
+        kept_ruby_queries = kept[:ruby_queries].map(&wordless)
+        divergences << { field: "queries", ruby: kept_ruby_queries, rust: rust_queries } \
+          unless rust_queries == kept_ruby_queries
+
+        divergences << { field: "sagas", ruby: ruby_sagas, rust: rust_output["sagas"] } \
+          unless rust_output["sagas"] == ruby_sagas
+
+        cross_domain = differ.cross_domain_policy_names(rust_output)
+        kept_ruby_reactions = JSON.parse(JSON.generate(ruby_result[:reactions]))
+                                  .reject { |r| cross_domain.include?(r["policy"]) }
+        rust_reactions = rust_output.fetch("reactions")
+        divergences << { field: "reactions", ruby: kept_ruby_reactions, rust: rust_reactions } \
+          unless rust_reactions == kept_ruby_reactions
+
+        rust_dry_runs = Array(rust_output["dry_runs"]).map { |d| d.slice("verb", "ok") }
+        divergences << { field: "dry_runs", ruby: ruby_dry_runs, rust: rust_dry_runs } \
+          unless rust_dry_runs == ruby_dry_runs
+        outcomes[:differential] = divergences
+
+        if self_consistency
+          outcomes[:self_consistency] =
+            self_consistency_divergences(ruby_result) +
+            SelfConsistency.check_rust_rehydration(binary, differ, rust_live_instances) +
+            SelfConsistency.check_rust_idempotency(binary, differ, rust_live_instances)
+        end
+
+        outcomes
+      end
+    end
+  end
+end