hecks 3.4.2 → 3.6.0

data/lib/hecks/quality_control/cli/qa_sweep/release_mode.rb

@@ -10,15 +10,37 @@ module Hecks
 
         # `--release`: a person puts a suspended target back, concluding what was open against it.
         def release_suspended_target
+          target = releasable_target
+          open_sweeps = open_sweeps_against(target)
+          surprises, clean = close_open_sweeps(open_sweeps)
+          # No open sweep (a crash between claim and open): nothing was watched, so the streak
+          # resets rather than incrementing.
+          puts "no open sweep against #{@target_reference} — releasing with the streak reset" if open_sweeps.empty?
+
+          capabilities = Hecks::Fuzzing::TargetCapabilities.infer(@domain_path, rust_dir: @rust_dir)
+          was = target.status
+          released = release_target!(target, capabilities, target.clean_streak.value, surprises: surprises, clean: clean)
+          puts released_line(released, was)
+          EXIT_OK
+        end
+
+        def releasable_target
           target = ::QualityControl::Target.find(@target_reference)
           abort "target #{@target_reference.inspect} vanished between being queried and being released" unless target
-          unless %w[suspended held].include?(target.status)
-            abort "target #{@target_reference.inspect} is #{target.status.inspect}, not suspended (or held) — " \
-                  "nothing to release"
-          end
+          return target if %w[suspended held].include?(target.status)
+
+          abort "target #{@target_reference.inspect} is #{target.status.inspect}, not suspended (or held) — " \
+                "nothing to release"
+        end
 
-          open_sweeps = query("Sweep.Sweeping").select { |row| row[:target] == target.id }
-                                               .map { |row| ::QualityControl::Sweep.find(row[:reference][:value]) }
+        def open_sweeps_against(target)
+          query("Sweep.Sweeping").select { |row| row[:target] == target.id }
+                                 .map { |row| ::QualityControl::Sweep.find(row[:reference][:value]) }
+        end
+
+        # @return [Array(Integer, Boolean)] the bugs logged against the sweeps, and whether every
+        #   one of them was clean
+        def close_open_sweeps(open_sweeps)
           surprises = 0
           clean = !open_sweeps.empty?
           open_sweeps.each do |sweep|
@@ -26,27 +48,18 @@ module Hecks
             clean &&= sweep_was_clean?(sweep)
             close_open_sweep(sweep, surprises)
           end
+          [surprises, clean]
+        end
 
-          # No open sweep (a crash between claim and open): nothing was watched, so the streak
-          # resets rather than incrementing.
-          puts "no open sweep against #{@target_reference} — releasing with the streak reset" if open_sweeps.empty?
-
-          capabilities = Hecks::Fuzzing::TargetCapabilities.infer(@domain_path, rust_dir: @rust_dir)
-          was = target.status
-          target = release_target!(target, capabilities, target.clean_streak.value, surprises: surprises, clean: clean)
-          puts "released #{@target_reference} (was #{was}) — status #{target.status}, " \
-               "clean_streak #{target.clean_streak.value}, yield_score #{target.yield_score.value}, " \
-               "capabilities #{target.capabilities.value.inspect}"
-          EXIT_OK
+        def released_line(target, was)
+          "released #{@target_reference} (was #{was}) — status #{target.status}, " \
+            "clean_streak #{target.clean_streak.value}, yield_score #{target.yield_score.value}, " \
+            "capabilities #{target.capabilities.value.inspect}"
         end
 
         def close_open_sweep(sweep, surprises)
           if sweep.made.value.positive? || sweep.waived.value.positive?
-            begin
-              sweep.conclude!(notes: { value: @release_notes })
-            rescue Hecks::Runtime::InvariantViolation => e
-              abort "--notes was refused by Sweep.Conclude — #{e.message}"
-            end
+            conclude_sweep(sweep)
             puts "concluded #{sweep.id} (#{sweep.made.value} check(s), #{surprises} bug(s) logged against it so far)"
           else
             # Never made a check: `Sweep.Conclude` refuses it, so abandon it to leave `Sweeping`.
@@ -54,6 +67,12 @@ module Hecks
             puts "abandoned #{sweep.id} (it never made a check)"
           end
         end
+
+        def conclude_sweep(sweep)
+          sweep.conclude!(notes: { value: @release_notes })
+        rescue Hecks::Runtime::InvariantViolation => e
+          abort "--notes was refused by Sweep.Conclude — #{e.message}"
+        end
       end
     end
   end

data/lib/hecks/quality_control/cli/qa_sweep/scratch.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module Hecks
+  module QualityControlCli
+    class QaSweep
+      # The disposable Postgres schemas the parity and concurrency modes run in: created before the
+      # seeds and dropped once the sweep ends, however it ends.
+      module Scratch
+        private
+
+        # Safe under concurrent callers: several sweeps can reach this at once (the parity wave
+        # runs up to `QualityControlDials::SWEEP_MAX_PARALLEL` targets in parallel), so the
+        # existence check is only a fast-path optimization; the actual safety net is rescuing the
+        # create's own race, since two processes can both see the database missing and both attempt
+        # `CREATE DATABASE`. Postgres reports that race two different ways: a `duplicate_database`
+        # error (42P04) from its own pre-create name check, or, when two backends run that check at
+        # nearly the same instant, a raw `unique_violation` (23505) on `pg_database`'s name index
+        # once both proceed to insert.
+        def ensure_scratch_database!(name)
+          admin = PG.connect(dbname: "postgres")
+          exists = admin.exec_params("SELECT 1 FROM pg_database WHERE datname = $1", [name]).ntuples.positive?
+          admin.exec(%(CREATE DATABASE "#{name}")) unless exists
+        rescue PG::DuplicateDatabase, PG::UniqueViolation
+          nil
+        ensure
+          admin&.close
+        end
+
+        # The scratch database is a fixed name, created if missing and never dropped (a concurrent
+        # sweep may use another schema in it); only this run's uniquely named schemas are dropped,
+        # in `drop_scratch_schemas`. `concurrency` needs two: the real fork race and its sequential
+        # oracle.
+        def prepare_scratch_schemas
+          @parity_database = @concurrency_database = SCRATCH_DATABASE
+          @parity_schema = @race_schema = @reference_schema = nil
+          prepare_parity_schema if @active_modes.include?(:persistence_parity)
+          prepare_concurrency_schemas if @active_modes.include?(:concurrency)
+        end
+
+        def prepare_parity_schema
+          require "pg"
+          @parity_schema = "qa_pp_#{@target_reference}_#{Process.pid}_#{SecureRandom.hex(4)}"
+          ensure_scratch_database!(@parity_database)
+        end
+
+        def prepare_concurrency_schemas
+          require "pg"
+          @race_schema = "qa_cc_race_#{@target_reference}_#{Process.pid}_#{SecureRandom.hex(4)}"
+          @reference_schema = "qa_cc_ref_#{@target_reference}_#{Process.pid}_#{SecureRandom.hex(4)}"
+          ensure_scratch_database!(@concurrency_database)
+        end
+
+        def drop_scratch_schemas
+          drop_schemas(@parity_database, [@parity_schema]) if @parity_schema
+          drop_schemas(@concurrency_database, [@race_schema, @reference_schema]) if @race_schema || @reference_schema
+        end
+
+        def drop_schemas(database, schemas)
+          admin = PG.connect(dbname: database)
+          admin.exec("SET client_min_messages = warning")
+          schemas.compact.each { |schema| admin.exec("DROP SCHEMA IF EXISTS #{admin.quote_ident(schema)} CASCADE") }
+          admin.close
+        end
+      end
+    end
+  end
+end

data/lib/hecks/quality_control/cli/qa_sweep/seat.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+module Hecks
+  module QualityControlCli
+    class QaSweep
+      # The sweep's record in the ledger and its primary seat: which comparison the seeds run
+      # against, and the line that announces it.
+      module Seat
+        private
+
+        # The reference embeds the target: `Sweep` is not scoped to its target, so two sweeps opened
+        # in the same second must not collide.
+        def open_sweep
+          @sweep_reference = "SW-#{@target_reference}-#{Time.now.to_i}"
+          @sweep = ::QualityControl::Sweep.open!(target: @target.id, reference: { value: @sweep_reference },
+                                                 engineer: { value: @engineer })
+          @differ = build_differ
+          @feature = File.basename(@domain_path).downcase
+        end
+
+        # Includes `RustConformanceHelpers` outside RSpec; `structural_skips` collects verbs the
+        # binary's manifest declares not generated.
+        def build_differ
+          require File.join(@root, "spec/support/rust_conformance_helpers")
+          Class.new do
+            include RustConformanceHelpers
+
+            attr_reader :structural_skips
+
+            def initialize
+              @structural_skips = Set.new
+            end
+          end.new
+        end
+
+        # The primary seat is the first of persistence_parity, concurrency, differential, ruby_only
+        # that is active, or none for a seedless-only sweep; differential degrades to ruby_only if
+        # the build fails.
+        def choose_seat
+          @binary = nil
+          return @mode = :seedless if @seeded_modes.empty?
+          return @mode = :persistence_parity if @active_modes.include?(:persistence_parity)
+          return @mode = :concurrency if @active_modes.include?(:concurrency)
+
+          @binary = build_rust_binary if @active_modes.include?(:differential)
+          return @mode = :differential if @binary
+
+          degrade_to_ruby_only if @active_modes.include?(:differential)
+          @mode = :ruby_only
+        end
+
+        def build_rust_binary
+          @differ.build_rust_for(@feature, @rust_dir)
+        rescue RustConformanceHelpers::BuildFailed => e
+          @rust_build_failure = e
+          nil
+        end
+
+        def degrade_to_ruby_only
+          # `build_rust_for` answers nil only for an undeclared feature; a declared one that fails
+          # raises, so its cargo stderr is printed here.
+          reason = @rust_build_failure ? @rust_build_failure.message : "rust/Cargo.toml declares no #{@feature} feature"
+          puts "note: no #{@feature} Rust binary — degrading this sweep to ruby_only. Run `hecks project_rust` for it " \
+               "and re-sweep.\n#{reason}"
+          @active_modes = (@active_modes - %i[differential properties_in_differential structural_skip_report]) |
+                          [:ruby_only]
+        end
+
+        def mode_label
+          case @mode
+          when :differential then "Ruby vs compiled Rust (#{@feature})"
+          when :persistence_parity then "Memory vs real PostgresEra (#{@feature}) — persistence-adapter parity"
+          when :concurrency
+            "real forked cross-process dispatch vs its own sequential oracle (#{@feature}) — write-lock serialization"
+          when :seedless
+            "no primary seat (#{@active_modes.join(",")} — audits this target's own real state, generates nothing)"
+          else "Ruby-only property/exception check (no compiled Rust binary for #{@feature})"
+          end
+        end
+
+        def announce_sweep
+          if @mode == :seedless
+            puts "sweeping #{@target_reference} (#{@target_path}) — #{mode_label}, " \
+                 "active modes #{@active_modes.join(",")}"
+          else
+            puts "sweeping #{@target_reference} (#{@target_path}) — #{mode_label}, #{@seeds} seed(s), " \
+                 "#{@steps_per_sequence} steps each, adversarial fraction #{@adversarial}, role draw #{@role_draw}, " \
+                 "dry-run fraction #{@dry_run}, active modes #{@active_modes.join(",")}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/hecks/quality_control/cli/qa_sweep/seed_loop.rb

@@ -0,0 +1,149 @@
+# frozen_string_literal: true
+
+module Hecks
+  module QualityControlCli
+    class QaSweep
+      # The seeds of a single-target sweep: each one generated, checked and logged, then the checks
+      # made once per sweep, and the shrinking of the first surprise.
+      module SeedLoop
+        private
+
+        # @return [Hash, nil] the first surprise, or nil when every check held
+        def run_seeds
+          # One campaign per sweep, resumed from a prior tick's corpus for this exact (target, mode)
+          # so guided generation's knowledge compounds across ticks instead of rebuilding an
+          # identical corpus from an identical trace every time. Nil when guided generation is off,
+          # or this is a seedless-only sweep that will never record anything into it.
+          @coverage_path = coverage_corpus_path(@target_reference, @mode) if @guided_generation && @seeded_modes.any?
+          @campaign = build_campaign
+          surprise = seed_loop || structural_skip_step || era_boundary_step
+          shrink_surprise(surprise)
+          surprise
+        ensure
+          # Drops this run's disposable schemas whether the sweep ends clean, surprised or crashes;
+          # the databases stay. Persists whatever this run's campaign learned, clean, surprised or
+          # crashed alike: the corpus a surprised run built up to that point is still real coverage
+          # a later tick should not have to rediscover from scratch.
+          drop_scratch_schemas
+          save_coverage_state!(@coverage_path, @campaign) if @campaign
+        end
+
+        def build_campaign
+          return unless @coverage_path
+
+          saved = load_coverage_state(@coverage_path)
+          if saved
+            Hecks::Fuzzing::CoverageCampaign.load(saved, splice_probability: @corpus_splice_probability,
+                                                         favor_count:        @favor_rare_verbs)
+          else
+            Hecks::Fuzzing::CoverageCampaign.new(splice_probability: @corpus_splice_probability,
+                                                 favor_count:        @favor_rare_verbs)
+          end
+        end
+
+        # A seedless-only sweep skips the loop so it never claims to have generated sequences.
+        def seed_loop
+          return unless @seeded_modes.any?
+
+          surprise = nil
+          ((@seed_offset + 1)..(@seed_offset + @seeds)).each do |seed|
+            surprise = run_seed(seed)
+            break if surprise
+          end
+          puts "  #{@campaign.summary}" if @campaign
+          surprise
+        end
+
+        # @return [Hash, nil] the seed's result with its surprised checks, or nil when every
+        #   check held
+        def run_seed(seed)
+          result = run_one_seed(@mode, seed)
+          @campaign.record(seed, result[:plan], result[:trace]) if @campaign && result[:trace]
+
+          surprised = log_seed_checks(result)
+          if surprised.empty?
+            puts "  seed #{seed}: held (#{check_modes(result[:checks])})"
+            return nil
+          end
+
+          puts "  seed #{seed}: SURPRISED (#{check_modes(surprised)})"
+          result.merge(seed: seed, checks: surprised)
+        end
+
+        def check_modes(checks)
+          checks.map { |c| c[:mode] }.join(", ")
+        end
+
+        # Every mode's check is logged before the loop decides, so a seed that held differentially
+        # but surprised on properties records both.
+        #
+        # @return [Array<Hash>] the checks that surprised
+        def log_seed_checks(result)
+          result[:checks].reject do |check|
+            log_and_mark(check)
+            check[:clean]
+          end
+        end
+
+        def log_and_mark(check)
+          @sweep = log_check!(@sweep, subject: check[:subject], expectation: check[:expectation])
+          sequence = @sweep.checks.last[:sequence][:value]
+          if check[:clean]
+            mark_held!(@sweep, sequence, check[:observation])
+          else
+            mark_surprised!(@sweep, sequence, check[:observation], @target_reference)
+          end
+        end
+
+        # Structural-skip check only after a differential loop that ran clean to the end.
+        def structural_skip_step
+          return unless @mode == :differential && @active_modes.include?(:structural_skip_report)
+
+          check = structural_skip_check(@differ, @binary, @structural_boundary)
+          log_and_mark(check)
+          step_result("structural skips", check)
+        end
+
+        # Era-boundary check, once per sweep, independent of the primary seat; skipped once anything
+        # surprised. A target with no lineage logs no Check: holding one would report an unaudited
+        # target as clean.
+        def era_boundary_step
+          return unless @active_modes.include?(:era_boundary)
+
+          check = era_boundary_check(@domain_path)
+          if check[:skip]
+            puts "  era boundary: #{check[:observation]} — no Check logged"
+            return nil
+          end
+
+          log_and_mark(check)
+          step_result("era boundary", check)
+        end
+
+        # @return [Hash, nil] a surprise for a check that surprised, nil for one that held
+        def step_result(label, check)
+          if check[:clean]
+            puts "  #{label}: #{check[:observation]}"
+            nil
+          else
+            puts "  #{label}: SURPRISED — #{check[:observation]}"
+            { seed: nil, steps: [], checks: [check] }
+          end
+        end
+
+        # Shrinks inside the guarded block: the parity schema is dropped afterwards, and candidates
+        # need it.
+        def shrink_surprise(surprise)
+          return unless surprise && surprise[:seed] && @shrink_budget.positive?
+
+          surprise[:shrunk] = surprise[:checks].filter_map do |check|
+            next unless Checks::SHRINKABLE_MODES.include?(check[:mode])
+
+            puts "  shrinking [#{check[:mode]}] (#{surprise[:steps].size} steps, budget #{@shrink_budget})…"
+            shrink_check(check, @mode, surprise[:steps])
+          end
+        end
+      end
+    end
+  end
+end

data/lib/hecks/quality_control/cli/qa_sweep/target_setup.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+module Hecks
+  module QualityControlCli
+    class QaSweep
+      # Everything a single-target sweep settles before the first seed: which target, which modes it
+      # resolves, the claim on it, and how deep the sweep goes.
+      module TargetSetup
+        private
+
+        # Existence-checked: a held or shelved target is invisible to `Rotation`, and
+        # re-`Identify`ing an existing id is refused.
+        def seed_default_targets!
+          existing = query("Target.All").map { |row| row[:reference][:value] }
+          DEFAULT_TARGETS.each do |reference, path|
+            next if existing.include?(reference)
+
+            ::QualityControl::Target.identify!(reference: { value: reference }, path: { value: path })
+          end
+        end
+
+        # The only pick with no name given, where `RotationPriority.pick`'s yield weighting matters
+        # (`--all` sweeps every row regardless of order).
+        def pick_target_row
+          return named_target_row if @target_ref
+
+          Hecks::Fuzzing::RotationPriority.pick(waiting_rotation, now: Time.now.to_i)
+        end
+
+        def named_target_row
+          row = query("Target.All").find { |r| r[:reference][:value] == @target_ref }
+          return row if row
+
+          known = query("Target.All").map { |r| r[:reference][:value] }
+          abort "no such target: #{@target_ref.inspect} — known targets: #{known.inspect}"
+        end
+
+        def waiting_rotation
+          rotation = query("Target.Rotation")
+          if rotation.empty?
+            seed_default_targets!
+            rotation = query("Target.Rotation")
+          end
+          return rotation unless rotation.empty?
+
+          abort "the rotation is empty even after seeding pizzas/banking — nothing waiting to sweep " \
+                "(every known target is currently held or shelved)"
+        end
+
+        def select_target
+          row = pick_target_row
+          @target_reference = row[:reference][:value]
+          @target_path = row[:path][:value]
+          @domain_path = resolve_target_path(@target_path)
+          return if File.directory?(@domain_path)
+
+          abort "target #{@target_reference.inspect} names a path that does not exist on disk: #{@domain_path}"
+        end
+
+        # Before the claim: an ineligible target is an operational error (exit 1), never a finding,
+        # and must not leave the target held. Deferred modes run only when the whole ask is
+        # deferred; mixed with others they wait for `--all`'s later wave.
+        def resolve_target_modes
+          @capabilities = Hecks::Fuzzing::TargetCapabilities.infer(@domain_path, rust_dir: @rust_dir)
+          resolved = Hecks::Fuzzing::TargetCapabilities.resolve(@enabled_modes, @capabilities)
+          trace("capabilities+resolve")
+          @deferred_modes = deferred_modes_for(resolved)
+          @active_modes = resolved - @deferred_modes
+          refuse_unqualified_parity
+          refuse_modeless_target
+          @seeded_modes = @active_modes - SEEDLESS_MODES
+        end
+
+        def deferred_modes_for(resolved)
+          deferred = Hecks::Fuzzing::TargetCapabilities::DEFERRED_MODES
+          explicit_all_deferred = @explicit_modes && !@explicit_modes.empty? && (@explicit_modes - deferred).empty?
+          explicit_all_deferred ? [] : resolved & deferred
+        end
+
+        def refuse_unqualified_parity
+          return unless @persistence_parity_mode && !@active_modes.include?(:persistence_parity)
+
+          abort "target #{@target_reference.inspect} (#{@target_path}) declares no persisted_by(\"PostgresEra\") " \
+                "binding in its own .hecksagon (capabilities: #{@capabilities.join(",")}) — " \
+                "--persistence-parity has nothing to compare Memory against for this domain. " \
+                "examples/directory is the known example that qualifies."
+        end
+
+        # `era_boundary` is seedless and meaningful with no primary seat; `concurrency` is a seat,
+        # so it counts here too.
+        def refuse_modeless_target
+          return if @active_modes.intersect?(%i[differential ruby_only persistence_parity concurrency era_boundary])
+
+          abort "target #{@target_reference.inspect} (#{@target_path}) resolves no comparison mode at all — " \
+                "enabled #{@enabled_modes.join(",")}, capabilities #{@capabilities.join(",")} " \
+                "(deferred: #{@deferred_modes.join(",")}). Nothing here can be swept without at least one of " \
+                "differential/ruby_only/persistence_parity/concurrency/era_boundary."
+        end
+
+        def claim_target
+          @target = ::QualityControl::Target.find(@target_reference)
+          abort "target #{@target_reference.inspect} vanished between being queried and being claimed" unless @target
+
+          begin
+            @target = @target.claim!(held_by: { value: @engineer }, now: { value: Time.now.to_i })
+          rescue Hecks::Runtime::GivenNotMet => e
+            abort "target #{@target_reference.inspect} is already held and its claim has not gone stale yet " \
+                  "(#{e.message})"
+          end
+          trace("claim")
+        end
+      end
+    end
+  end
+end