hecks 1.3.0 → 1.5.0

data/lib/hecks/fuzzing/coverage_campaign.rb

@@ -0,0 +1,161 @@
+module Hecks
+  module Fuzzing
+    # What one sweep has already reached, and where the next seed starts.
+    #
+    # `SequenceGenerator` steers within one sequence (an unexercised verb is
+    # weighted up — picker.rb's `steer`), but without this, every seed of a
+    # sweep would start from nothing and forget everything the seeds before
+    # it reached. Roughly half of the QA ledger's bugs are refusal-kind splits (BUG#2–4,
+    # 7, 8, 14–16, 19–21, 23, 27, 28, 36–38, 41, 54, 56): a particular verb,
+    # in a particular state, with a particular malformed argument, refused
+    # one way on Ruby and another on Rust. BUG#11 was found by hand because
+    # a 25-step random walk never reached `Annotate` at all. Uniform seeds
+    # reach those corners by luck; this makes the sweep remember.
+    #
+    # ## The coverage unit
+    #
+    # A tuple, not a verb — `verb | step kind | lifecycle state before |
+    # adversarial mutation | outcome` (see `SequenceGenerator#coverage_tuple`).
+    # "Renew refused LifecycleRefused from free" and "Renew ok from held" are
+    # different places a runtime can be wrong, where a verb-level count
+    # calls them the same.
+    #
+    # ## Two levers
+    #
+    # Both expressed as plain generator arguments so a seed stays
+    # reproducible from one call:
+    #
+    #   1. Splicing (`prefix:`). A sequence that reached a new tuple joins
+    #      the corpus, cut at its last new tuple. A later seed may start by
+    #      re-generating a random-length prefix of a corpus entry — the same
+    #      seed, favor and nested prefix, so the same steps, dispatched for
+    #      real, reaching the same state — and then carries on with its own
+    #      seed's randomness from there. Deep state gets reached once and
+    #      explored many times, instead of re-rolled per seed.
+    #   2. Favor (`favor:`). The verbs this sweep has hit least often —
+    #      including declared verbs it has never hit at all — are weighted
+    #      up for the next seed.
+    #
+    # ## Reproducibility
+    #
+    # Nothing here is random at the campaign's own level beyond
+    # `Random.new(seed)` per plan, and a plan is printed with every finding
+    # (`bin/qa_sweep`'s `reproduce:` line), so no finding depends on
+    # re-running the whole campaign to get back to it.
+    class CoverageCampaign
+      # One seed's generation instructions, printable and reproducible.
+      Plan = Struct.new(:prefix, :favor, keyword_init: true) do
+        # Renders this plan as the keyword arguments a generator call accepts.
+        #
+        # @return [Hash{Symbol => Object}] `{ prefix:, favor: }`, the keyword arguments
+        #   `SequenceGenerator.generate` and `.trace` accept for this plan
+        def generator_options = { prefix: prefix, favor: favor }
+
+        # Answers whether this plan starts from a spliced corpus prefix.
+        #
+        # @return [Boolean] whether this plan starts from a spliced corpus prefix
+        def spliced? = !prefix.nil?
+      end
+
+      attr_reader :corpus
+
+      # @param splice_probability [Float, Integer] chance, per seed with a non-empty
+      #   corpus, of splicing a corpus entry's prefix instead of generating fresh
+      # @param favor_count [Integer, Float] how many rare verbs to favor per plan; 0
+      #   or non-positive disables favoring entirely
+      # @param corpus_limit [Integer] maximum corpus entries kept; the oldest is
+      #   dropped once this is exceeded
+      # @param max_prefix_depth [Integer] maximum nested-prefix depth a new corpus
+      #   entry may reach before splicing stops extending it further
+      def initialize(splice_probability:, favor_count:, corpus_limit: 64, max_prefix_depth: 6)
+        @splice_probability = splice_probability.to_f
+        @favor_count        = favor_count.to_i
+        @corpus_limit       = corpus_limit
+        @max_prefix_depth   = max_prefix_depth
+        @corpus             = []
+        @seen               = Set.new
+        @verb_hits          = Hash.new(0)
+        @declared_verbs     = Set.new
+        @seeds              = 0
+        @spliced            = 0
+        @seeds_with_new     = 0
+      end
+
+      # Builds this seed's generation plan — a possible splice, plus which
+      # verbs to favor.
+      #
+      # @param seed [Integer] the seed this plan is for; drives the splice-or-not
+      #   coin flip and, when splicing, which corpus entry and prefix length are drawn
+      # @return [Fuzzing::CoverageCampaign::Plan] the plan `SequenceGenerator.generate`
+      #   or `.trace` should be run with for this seed
+      def plan(seed)
+        random = Random.new(seed)
+        prefix = nil
+        if !@corpus.empty? && random.rand < @splice_probability
+          entry  = @corpus[random.rand(@corpus.size)]
+          prefix = entry[:spec].merge("steps" => random.rand(1..entry[:attempts]))
+        end
+        Plan.new(prefix: prefix, favor: rare_verbs)
+      end
+
+      # Folds one seed's generation results into the campaign's running
+      # coverage, and admits its corpus entry if it reached anything new.
+      #
+      # @param seed [Integer] the seed this trace was generated from
+      # @param plan [Fuzzing::CoverageCampaign::Plan] the plan that seed was generated with
+      # @param trace [Fuzzing::SequenceGenerator::Trace] `SequenceGenerator.trace`'s answer:
+      #   `coverage` is `[[attempt_index, tuple], ...]`, `verbs` every verb the booted
+      #   catalog offered
+      # @return [void]
+      def record(seed, plan, trace)
+        @seeds   += 1
+        @spliced += 1 if plan.spliced?
+        @declared_verbs.merge(trace.verbs)
+
+        new_at = trace.coverage.filter_map do |index, tuple|
+          @verb_hits[tuple.split(" | ").first] += 1
+          index if @seen.add?(tuple)
+        end
+        return if new_at.empty?
+
+        @seeds_with_new += 1
+        admit(seed, plan, new_at.max + 1)
+      end
+
+      # Counts the distinct coverage tuples this campaign has reached so far.
+      #
+      # @return [Integer] the number of distinct coverage tuples seen
+      def tuples_seen = @seen.size
+
+      # Renders the campaign's running totals as one human-readable line.
+      #
+      # @return [String] a one-line summary of tuples seen, seeds run, seeds that
+      #   reached something new, and how many were spliced from the corpus
+      def summary
+        "coverage: #{@seen.size} distinct (verb, kind, state, mutation, outcome) tuple(s) over #{@seeds} seed(s); " \
+          "#{@seeds_with_new} seed(s) reached something new; #{@spliced} spliced from a corpus of #{@corpus.size}"
+      end
+
+      private
+
+      def admit(seed, plan, attempts)
+        return if depth(plan.prefix) + 1 > @max_prefix_depth
+
+        spec = { "seed" => seed, "favor" => plan.favor }
+        spec["prefix"] = plan.prefix if plan.prefix
+        @corpus << { spec: spec, attempts: attempts }
+        @corpus.shift while @corpus.size > @corpus_limit
+      end
+
+      def depth(prefix) = prefix.nil? ? 0 : 1 + depth(prefix["prefix"])
+
+      # Never-hit declared verbs first (hits 0), then the least-hit, ties
+      # broken by name so a plan is a pure function of what was recorded.
+      def rare_verbs
+        return [] unless @favor_count.positive?
+
+        (@declared_verbs | @verb_hits.keys).min_by(@favor_count) { |verb| [@verb_hits[verb], verb] }
+      end
+    end
+  end
+end

data/lib/hecks/fuzzing/differential.rb

@@ -0,0 +1,192 @@
+require "json"
+require "open3"
+require_relative "replay"
+require_relative "properties"
+require_relative "self_consistency"
+require_relative "rust_gap_manifest"
+require_relative "nondeterministic"
+
+module Hecks
+  module Fuzzing
+    # The Ruby-vs-Rust comparison of one generated sequence — moved here,
+    # unchanged, out of `bin/qa_sweep`'s own `diff_ruby_vs_rust` so a second
+    # caller (`bin/qa_generated_domains --rust`, which compares domains the
+    # generator wrote rather than ones on the rotation) runs the same
+    # comparison rather than a re-derived one. `bin/qa_sweep` delegates to
+    # it; its own comments on the known-gap filtering, the wire-precision
+    # reduction and the deep copy taken before `strip_emitted_flags!` apply
+    # here word for word.
+    #
+    # `differ` is anything answering `RustConformanceHelpers`' comparison
+    # helpers (spec/support/rust_conformance_helpers.rb) plus a
+    # `structural_skips` set — duck-typed, so lib never requires spec/.
+    # Which query verbs may diverge is not the differ's call: it is read
+    # off the binary's own manifest.json (`manifest_partition`, below).
+    #
+    # Returns one divergence list per active mode — `{ differential: [...],
+    # self_consistency: [...], properties_in_differential: [...],
+    # adapter_parity_sqlite: [...] }`, keys present only for the modes in
+    # `modes:` (`differential` always). `adapter_parity_sqlite:` is the
+    # caller's own callable, because which two adapters that mode pairs is
+    # the caller's dial, not this module's.
+    module Differential
+      module_function
+
+      # The one place a Ruby/Rust query divergence may be tolerated — and
+      # only for a verb `gaps` (a `RustGapManifest`) declares not generated.
+      # Rust refuses such a verb outright while Ruby answers it for real
+      # (or refuses it for its own business reason), so both sides' rows for
+      # that verb leave the comparison. Everything else stays in: a refusal
+      # the manifest doesn't account for — however its message is worded —
+      # is a real divergence.
+      #
+      # `skipped` is every tolerated verb this history actually reached (for
+      # the sweep's structural_skip_report). `stale` is a divergence per
+      # tolerated verb Rust nonetheless answered with a query row: the
+      # manifest says "not generated" but the binary disagrees, so the
+      # tolerance itself is wrong and must not silently hold.
+      # @param gaps [Hecks::Fuzzing::RustGapManifest] the compiled binary's manifest
+      # @param ruby_refusals [Array<Hash>] Ruby's own refusal rows for this history
+      # @param rust_refusals [Array<Hash>] the Rust binary's own refusal rows
+      # @param ruby_queries [Array<Hash>] Ruby's own query rows for this history
+      # @param rust_queries [Array<Hash>] the Rust binary's own query rows
+      # @return [Hash] `ruby_refusals:`/`rust_refusals:`/`ruby_queries:`/
+      #   `rust_queries:` (each with tolerated verbs removed), `skipped:` (a
+      #   `Set<String>` of every tolerated verb this history actually reached),
+      #   and `stale:` (an `Array<Hash>` divergence per tolerated verb the Rust
+      #   binary nonetheless answered)
+      def manifest_partition(gaps, ruby_refusals:, rust_refusals:, ruby_queries:, rust_queries:)
+        verb_of  = ->(row) { row.key?("verb") ? row["verb"] : row["query"] }
+        declared = ->(row) { gaps.not_generated?(verb_of.call(row)) }
+        stale = rust_queries.select(&declared).map do |row|
+          { field: "manifest", verb: row["query"],
+            detail: "#{row['query']} is declared generated: false in manifest.json " \
+                    "(#{gaps.not_generated(row['query']).values_at('gap_class', 'construct').join('/')}), " \
+                    "but the Rust binary answered it — regenerate with bin/project_rust" }
+        end
+        reached = (ruby_refusals + rust_refusals + ruby_queries).select(&declared)
+        { ruby_refusals: ruby_refusals.reject(&declared), rust_refusals: rust_refusals.reject(&declared),
+          ruby_queries: ruby_queries.reject(&declared), rust_queries: rust_queries.reject(&declared),
+          skipped: reached.to_set(&verb_of), stale: stale }
+      end
+
+      # Runs the standard property battery and reports every one that failed.
+      #
+      # @param history [Hash] a replayed history as returned by `Replay.call`
+      # @return [Array<Hash>] one `{field:, detail:}` entry per failed property
+      def property_divergences(history)
+        Properties.check(history).reject { |_, result| result == true }
+                  .map { |name, message| { field: name.to_s, detail: message } }
+      end
+
+      # Flattens `history[:self_consistency]`'s own per-check divergence lists.
+      #
+      # @param history [Hash] a replayed history as returned by `Replay.call`,
+      #   optionally carrying `:self_consistency` (`Replay.call`'s own
+      #   `self_consistency: true`)
+      # @return [Array<Hash>] every self-consistency divergence recorded; empty if
+      #   `history` carries no `:self_consistency`
+      def self_consistency_divergences(history)
+        return [] unless history[:self_consistency]
+
+        history[:self_consistency].values.flatten(1)
+      end
+
+      # rubocop:disable-next Metrics/AbcSize
+      # rubocop:disable-next Metrics/CyclomaticComplexity
+      # rubocop:disable-next Metrics/PerceivedComplexity
+      # rubocop:disable-next Metrics/MethodLength
+      # @param differ [Object] duck-typed comparison helper answering
+      #   `RustConformanceHelpers`' interface (adapter-defined)
+      # @param domain_path [String] path to the domain directory to replay
+      # @param steps [Array<Hash>] the step sequence to replay on both engines
+      # @param binary [String] path to the compiled Rust conformance binary to run
+      # @param modes [Array<Symbol>] which comparisons to run — any of
+      #   `:differential` (always run), `:self_consistency`,
+      #   `:properties_in_differential`, `:adapter_parity_sqlite`
+      # @param adapter_parity_sqlite [Proc, nil] callable answering the
+      #   `:adapter_parity_sqlite` mode's own comparison, or `nil` to skip it
+      #   even when named in `modes`
+      # @return [Hash{Symbol => Array<Hash>}] one divergence list per active mode,
+      #   keyed by the mode name; `:differential` is always present
+      def diff(differ, domain_path, steps, binary, modes:, adapter_parity_sqlite: nil)
+        self_consistency = modes.include?(:self_consistency)
+        ruby_result    = Replay.call(domain_path, steps, self_consistency: self_consistency)
+        ruby_instances = JSON.parse(JSON.generate(ruby_result[:instances]))
+        ruby_events    = JSON.parse(JSON.generate(ruby_result[:events]))
+        ruby_refusals  = ruby_result[:refusals].map do |r|
+          { "verb" => r[:verb].to_s, "kind" => r[:kind].to_s.split("::").last, "error" => r[:error] }
+        end
+        ruby_queries  = JSON.parse(JSON.generate(ruby_result[:queries].map { |row| Nondeterministic.strip(row, :query_row) }))
+        ruby_sagas    = JSON.parse(JSON.generate(ruby_result[:sagas]))
+        ruby_dry_runs = ruby_result[:dry_runs].map { |d| { "verb" => d[:verb].to_s, "ok" => d[:ok] } }
+
+        outcomes = {}
+        outcomes[:properties_in_differential] = property_divergences(ruby_result) if modes.include?(:properties_in_differential)
+        if modes.include?(:adapter_parity_sqlite) && adapter_parity_sqlite
+          outcomes[:adapter_parity_sqlite] = adapter_parity_sqlite.call
+        end
+
+        stdout, status = Open3.capture2(binary, stdin_data: JSON.generate({ "steps" => steps }))
+        unless status.success?
+          return outcomes.merge(differential: [{ field:  "process",
+                                                 detail: "rust binary exited #{status.exitstatus}: #{stdout}" }])
+        end
+
+        rust_output = JSON.parse(stdout)
+        rust_live_instances = JSON.parse(JSON.generate(rust_output["instances"])) if self_consistency
+        differ.strip_emitted_flags!(rust_output["instances"])
+        differ.strip_emitted_flags!(rust_output["queries"])
+        differ.strip_occurred_at!(rust_output["events"])
+
+        divergences = []
+        divergences << { field: "instances", ruby: ruby_instances, rust: rust_output["instances"] } \
+          unless rust_output["instances"] == ruby_instances
+        divergences << { field: "events", ruby: ruby_events, rust: rust_output["events"] } \
+          unless rust_output["events"] == ruby_events
+
+        kept = manifest_partition(RustGapManifest.for_binary(binary),
+                                  ruby_refusals: ruby_refusals, rust_refusals: rust_output["refusals"],
+                                  ruby_queries: ruby_queries, rust_queries: rust_output["queries"])
+        differ.structural_skips.merge(kept[:skipped])
+        divergences.concat(kept[:stale])
+
+        by_kind = ->(r) { r.slice("verb", "kind") }
+        rust_refusals      = kept[:rust_refusals].map(&by_kind)
+        kept_ruby_refusals = kept[:ruby_refusals].map(&by_kind)
+        divergences << { field: "refusals", ruby: kept_ruby_refusals, rust: rust_refusals } \
+          unless rust_refusals == kept_ruby_refusals
+
+        wordless          = ->(row) { differ.reduce_to_wire_precision(row.except("error", "reference_error")) }
+        rust_queries      = kept[:rust_queries].map(&wordless)
+        kept_ruby_queries = kept[:ruby_queries].map(&wordless)
+        divergences << { field: "queries", ruby: kept_ruby_queries, rust: rust_queries } \
+          unless rust_queries == kept_ruby_queries
+
+        divergences << { field: "sagas", ruby: ruby_sagas, rust: rust_output["sagas"] } \
+          unless rust_output["sagas"] == ruby_sagas
+
+        cross_domain = differ.cross_domain_policy_names(rust_output)
+        kept_ruby_reactions = JSON.parse(JSON.generate(ruby_result[:reactions]))
+                                  .reject { |r| cross_domain.include?(r["policy"]) }
+        rust_reactions = rust_output.fetch("reactions")
+        divergences << { field: "reactions", ruby: kept_ruby_reactions, rust: rust_reactions } \
+          unless rust_reactions == kept_ruby_reactions
+
+        rust_dry_runs = Array(rust_output["dry_runs"]).map { |d| d.slice("verb", "ok") }
+        divergences << { field: "dry_runs", ruby: ruby_dry_runs, rust: rust_dry_runs } \
+          unless rust_dry_runs == ruby_dry_runs
+        outcomes[:differential] = divergences
+
+        if self_consistency
+          outcomes[:self_consistency] =
+            self_consistency_divergences(ruby_result) +
+            SelfConsistency.check_rust_rehydration(binary, differ, rust_live_instances) +
+            SelfConsistency.check_rust_idempotency(binary, differ, rust_live_instances)
+        end
+
+        outcomes
+      end
+    end
+  end
+end