hecks 2.8.0 → 3.0.0

data/lib/hecks/quality_control/cli/qa_sweep/dials.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module Hecks
+  module QualityControlCli
+    class QaSweep
+      # The dials a sweep reads once the ledger has booted, and the modes they enable. A dial-less
+      # fixture ledger falls back to a sweep with the adversarial layer off and self-consistency on.
+      module Dials
+        private
+
+        def dial(name, fallback)
+          ::QualityControlDials.const_defined?(name, false) ? ::QualityControlDials.const_get(name) : fallback
+        rescue NameError
+          fallback
+        end
+
+        # Dials are read after boot because the bluebook defines them; a dial-less fixture ledger
+        # runs the adversarial layer off and self-consistency on.
+        def read_dials
+          @adversarial ||= dial(:ADVERSARIAL_FRACTION, 0.0).to_f
+          # Remembered before defaulting: the alias below applies only when the flag was typed, so
+          # an explicit `--modes ruby_only` stays exclusive.
+          @self_consistency_explicit = !@self_consistency.nil?
+          @self_consistency = dial(:SELF_CONSISTENCY_CHECKS, true) if @self_consistency.nil?
+          # Same off-without-a-dial default as the adversarial layer.
+          @role_draw ||= dial(:ROLE_DRAW_PROBABILITY, 0.0).to_f
+          @dry_run ||= dial(:DRY_RUN_FRACTION, 0.0).to_f
+          # The loop's identity: `held_by` on claims, `engineer` on sweeps, and the literal
+          # `WaivedBy` refuses.
+          @engineer = dial(:AUTOMATED_ENGINEER, "qa_sweep")
+          @widening_tiers = dial(:WIDENING_TIERS, Hecks::Fuzzing::SweepDepth::DEFAULT_TIERS)
+          @guided_generation = dial(:GUIDED_GENERATION, false)
+          @corpus_splice_probability = dial(:CORPUS_SPLICE_PROBABILITY, 0.5)
+          @favor_rare_verbs = dial(:FAVOR_RARE_VERBS, 3)
+          @max_parallel = dial(:SWEEP_MAX_PARALLEL, 4)
+          @shrink_budget = dial(:SHRINK_BUDGET, 200)
+          @structural_boundary = dial(:STRUCTURAL_REFUSAL_BOUNDARY, [])
+          # The adapter pair each parity mode compares; a new pairing is a data change here, not a
+          # new diff.
+          @adapter_parity_pairs = dial(:ADAPTER_PARITY_PAIRS,
+                                       { persistence_parity:    { left: :memory, right: :postgres_era },
+                                         adapter_parity_sqlite: { left: :memory, right: :sqlite } }.freeze)
+        end
+
+        # `--modes` wins outright; otherwise the dial's enabled modes. `--self-consistency` adjusts
+        # the set only when typed or when `--modes` was not given, so an explicit `--modes a,b`
+        # stays exact.
+        def resolve_enabled_modes
+          @enabled_modes = @explicit_modes || dial_modes
+          if @self_consistency_explicit || @explicit_modes.nil?
+            @enabled_modes = if @self_consistency
+                               @enabled_modes | [:self_consistency]
+                             else
+                               @enabled_modes - [:self_consistency]
+                             end
+          end
+          # `--all --persistence-parity` forces the wave on regardless of the dial.
+          @enabled_modes |= [:persistence_parity] if @force_parity_wave
+
+          # Refuse modes with no implementation before claiming, so `resolved modes:` never
+          # advertises coverage nobody wrote.
+          unrunnable = @enabled_modes - Hecks::Fuzzing::TargetCapabilities::RUNNABLE_MODES
+          return if unrunnable.empty?
+
+          abort "enabled mode(s) #{unrunnable.join(',')} have no implementation in hecks quality_control ask run " \
+                "(Hecks::Fuzzing::TargetCapabilities::RUNNABLE_MODES is what exists). Set them false in " \
+                "qa/settings.yml, or drop them from --modes, until the code lands."
+        end
+
+        def dial_modes
+          return DEFAULT_MODES unless ::QualityControlDials.const_defined?(:MODES, false)
+
+          ::QualityControlDials::MODES.select { |_, on| on }.keys
+        rescue NameError
+          DEFAULT_MODES
+        end
+
+        # `--no-parity` skips every later wave. A mode gets a wave only when the dial enabled it and
+        # `--modes` did not narrow the run.
+        def deferred_waves
+          %i[era_boundary concurrency].select do |mode|
+            !@skip_parity_wave && @explicit_modes.nil? && @enabled_modes.include?(mode)
+          end
+        end
+
+        def parity_wave?
+          !@skip_parity_wave && @explicit_modes.nil? && @enabled_modes.include?(:persistence_parity)
+        end
+      end
+    end
+  end
+end

data/lib/hecks/quality_control/cli/qa_sweep/finding_report.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+module Hecks
+  module QualityControlCli
+    class QaSweep
+      # What a sweep prints when a check surprised: the header, the reproduction, every divergence
+      # and the generated and shrunk steps, so an agent can decide from the report alone.
+      module FindingReport
+        private
+
+        # @param surprise [Hash] the surprising seed's result: `seed`, `steps`, `checks`, `plan` and
+        #   optionally `shrunk`
+        # @return [void]
+        def print_finding(surprise)
+          print_finding_header
+          print_finding_facts(surprise)
+          # One block per surprised check, so a seed that surprised on two axes prints both.
+          surprise[:checks].each { |check| print_surprised_check(check) }
+          print_generated_steps(surprise[:steps])
+          # One shrunk demonstration per shrinkable check, written to a file so the replay line
+          # works as printed.
+          Array(surprise[:shrunk]).each { |shrunk| print_shrunk(shrunk, surprise[:seed]) }
+        end
+
+        def print_finding_header
+          puts
+          puts "=" * 72
+          puts "FOUND SOMETHING — sweep #{@sweep_reference} left OPEN, target #{@target_reference} SUSPENDED."
+          puts "No Bug was logged (that needs a real failing-test demonstration, which"
+          puts "needs judgment this script does not have — hecks quality_control log is the door)"
+          puts "and the sweep was not concluded. The ledger's own SuspendOnSurprise"
+          puts "policy took the target out of the rotation; it stays out until a PERSON"
+          puts "runs: hecks quality_control ask run target=#{@target_reference} " \
+               "arguments=\"--release --notes …\""
+          puts "An agent decides from here: self-contained fix with a regression test"
+          puts "and a PR (hecks quality_control patch.open), or a Bug left open for something bigger."
+          puts "Never waived here, never filed upstream from here."
+          puts "=" * 72
+          puts
+        end
+
+        def print_finding_facts(surprise)
+          puts "domain:      #{@target_path} (#{@domain_path})"
+          puts "target:      #{@target_reference}"
+          puts "sweep:       #{@sweep_reference} (id #{@sweep.id})"
+          puts "mode:        #{@mode} (active modes: #{@active_modes.join(', ')}; " \
+               "capabilities: #{@capabilities.join(', ')})"
+          puts "seed:        #{surprise[:seed] || 'n/a (once-per-sweep check)'}"
+          puts "steps:       #{@steps_per_sequence}"
+          puts "adversarial: #{@adversarial}"
+          puts "role-draw:   #{@role_draw}"
+          puts "dry-run:     #{@dry_run}"
+          puts "streak:      clean_streak was #{@current_streak} going into this pass"
+          if surprise[:seed]
+            puts "reproduce:   Hecks::Fuzzing::SequenceGenerator.generate(#{@domain_path.inspect}, " \
+                 "seed: #{surprise[:seed]}, steps: #{@steps_per_sequence}, adversarial: #{@adversarial}, " \
+                 "role_draw: #{@role_draw}, dry_run: #{@dry_run}#{plan_arguments(surprise[:plan])})"
+          end
+          puts "surprised:   #{surprise[:checks].map { |c| c[:mode] }.join(', ')}"
+          puts
+        end
+
+        def print_surprised_check(check)
+          puts "subject:     #{check[:subject]}"
+          puts "expectation: #{check[:expectation]}"
+          puts "observation: #{check[:observation]}"
+          puts
+
+          # One generic printer for every divergence shape (ruby/rust, memory/postgres_era,
+          # live/rehydrated): print every key except the field and the redundant
+          # domain/aggregate/type.
+          check[:divergences].each do |divergence|
+            label = [divergence[:field], divergence[:domain], divergence[:aggregate], divergence[:type]]
+                    .compact.join(" ")
+            puts "-- #{label} --"
+            divergence.except(:field, :domain, :aggregate, :type).each { |key, value| puts "#{key}: #{value.inspect}" }
+            puts
+          end
+        end
+
+        # Lists every step in order with its `adversarial` metadata, so a divergence's verb matches
+        # back to its step and the Bug can name the mutation.
+        def print_generated_steps(steps)
+          puts "-- generated steps (index: verb, adversarial mutation if any) --"
+          steps.each_with_index do |step, index|
+            label = %w[verb query dry_run].filter_map { |key| step[key] if step.key?(key) }.first
+            mutation = step["adversarial"] ? JSON.generate(step["adversarial"]) : "(not mutated)"
+            puts "#{index}: #{label}  #{mutation}"
+            puts "   args: #{JSON.generate(step['args'])}" if step["adversarial"]
+          end
+          puts
+        end
+
+        def print_shrunk(shrunk, seed)
+          path = write_shrunk!(shrunk, seed)
+          budget = shrunk[:exhausted] ? ", budget exhausted — may shrink further" : ""
+          puts "shrunk:      [#{shrunk[:mode]}] #{shrunk[:original_size]} -> #{shrunk[:steps].size} step(s) " \
+               "(#{shrunk[:attempts]} candidate checks#{budget})"
+          puts "file:        #{path.delete_prefix("#{@root}/")}"
+          replay =
+            if %i[differential self_consistency].include?(shrunk[:mode]) && @binary
+              "hecks check_conformance domain=#{@domain_path} script=#{path} artifact=#{@binary}"
+            else
+              "hecks run #{@domain_path} #{path}"
+            end
+          puts "replay:      #{replay}"
+          puts "-- shrunk steps [#{shrunk[:mode]}] --"
+          shrunk[:steps].each_with_index do |step, index|
+            kind = %w[verb query dry_run].find { |key| step.key?(key) }
+            puts "#{index}: #{"#{kind} " unless kind == 'verb'}#{step[kind]}  args: #{JSON.generate(step['args'])}"
+          end
+          puts
+        end
+      end
+    end
+  end
+end

data/lib/hecks/quality_control/cli/qa_sweep/parsing.rb

@@ -0,0 +1,175 @@
+# frozen_string_literal: true
+
+module Hecks
+  module QualityControlCli
+    class QaSweep
+      # The command line of `QaSweep`: every flag, and the rules for which combinations are refused.
+      #
+      # `seeds_override` and `steps_override` stay nil rather than a default, so an explicit
+      # `--seeds 10` can be told apart from the streak-derived depth. The dials are read after the
+      # ledger boots, so `adversarial`, `self_consistency`, `role_draw` and `dry_run` stay nil
+      # until then.
+      module Parsing
+        # Each flag that reads a value, and the setting it fills.
+        VALUE_SETTINGS = { "--role-draw" => :role_draw, "--dry-run" => :dry_run, "--adversarial" => :adversarial,
+                           "--seeds" => :seeds_override, "--steps" => :steps_override }.freeze
+
+        # Each flag that stands alone, and the setting it turns on.
+        SWITCH_SETTINGS = { "--all" => :all_mode, "--no-parity" => :skip_parity_wave,
+                            "--release" => :release_mode }.freeze
+
+        private
+
+        # Parses the flags into the instance's settings.
+        #
+        # @param argv [Array<String>] the command line's arguments, consumed
+        # @return [Symbol, nil] `:help` once the usage is printed, else nil
+        # @raise [SystemExit] with the usage when a flag or combination is refused
+        def parse_arguments(argv)
+          @target_ref = @seeds_override = @steps_override = nil
+          @all_mode = @persistence_parity_mode = @skip_parity_wave = @release_mode = false
+          @adversarial = @self_consistency = @role_draw = @dry_run = nil
+          # `--modes` (or its alias `--persistence-parity`); nil means the dial decides.
+          @explicit_modes = nil
+          @release_notes = nil
+          until argv.empty?
+            arg = argv.shift
+            if %w[-h --help].include?(arg)
+              puts USAGE
+              return :help
+            end
+            parse_flag(arg, argv)
+          end
+          validate_arguments
+          nil
+        end
+
+        def parse_flag(arg, argv)
+          if SWITCH_SETTINGS.key?(arg)
+            instance_variable_set(:"@#{SWITCH_SETTINGS.fetch(arg)}", true)
+          elsif VALUE_SETTINGS.key?(arg)
+            parse_value_flag(arg, argv.shift)
+          else
+            parse_other_flag(arg, argv)
+          end
+        end
+
+        def parse_value_flag(arg, value)
+          setting = VALUE_SETTINGS.fetch(arg)
+          parsed = %w[--seeds --steps].include?(arg) ? parse_integer(arg, value) : parse_fraction(arg, value)
+          instance_variable_set(:"@#{setting}", parsed)
+        end
+
+        def parse_other_flag(arg, argv)
+          case arg
+          when "--persistence-parity"
+            @persistence_parity_mode = true
+            @explicit_modes = %i[persistence_parity]
+          when "--modes"
+            @explicit_modes = parse_modes(argv.shift)
+            @persistence_parity_mode = @explicit_modes == %i[persistence_parity]
+          when "--notes"
+            @release_notes = argv.shift
+            abort "#{USAGE}\n--notes needs the text a person concluded" unless @release_notes
+          when "--self-consistency" then @self_consistency = parse_boolean(arg, argv.shift)
+          else record_target(arg)
+          end
+        end
+
+        def record_target(arg)
+          abort "#{USAGE}\nunexpected argument: #{arg.inspect} (target already given: #{@target_ref.inspect})" if @target_ref
+
+          @target_ref = arg
+        end
+
+        def parse_modes(value)
+          names = value.to_s.split(",").map(&:strip).reject(&:empty?).map(&:to_sym)
+          abort "#{USAGE}\n--modes needs at least one mode name" if names.empty?
+
+          known = Hecks::Fuzzing::TargetCapabilities::MODE_REQUIREMENTS.keys
+          unknown = names - known
+          abort "#{USAGE}\n--modes names no such mode: #{unknown.join(', ')} (known: #{known.join(', ')})" unless unknown.empty?
+
+          names
+        end
+
+        def parse_fraction(flag, value)
+          abort "#{USAGE}\n#{flag} needs a fraction between 0 and 1" unless value
+
+          fraction = begin
+            Float(value)
+          rescue ArgumentError
+            abort "#{USAGE}\n#{flag} must be a number, got #{value.inspect}"
+          end
+          abort "#{USAGE}\n#{flag} must be between 0 and 1, got #{value}" unless fraction.between?(0, 1)
+          fraction
+        end
+
+        def parse_integer(flag, value)
+          abort "#{USAGE}\n#{flag} needs a number" unless value
+
+          begin
+            Integer(value)
+          rescue ArgumentError
+            abort "#{USAGE}\n#{flag} must be an integer, got #{value.inspect}"
+          end
+        end
+
+        def parse_boolean(flag, value)
+          abort "#{USAGE}\n#{flag} needs true or false" unless value
+
+          case value
+          when "true" then true
+          when "false" then false
+          else abort "#{USAGE}\n#{flag} must be true or false, got #{value.inspect}"
+          end
+        end
+
+        def validate_arguments
+          if @seeds_override && @seeds_override < 1
+            abort "#{USAGE}\n--seeds must be at least 1 — Sweep.Conclude refuses a sweep that checked nothing"
+          end
+          abort "#{USAGE}\n--steps must be at least 1" if @steps_override && @steps_override < 1
+          if @all_mode && @target_ref
+            abort "#{USAGE}\n--all sweeps every waiting target itself — it does not take a target-reference " \
+                  "(got #{@target_ref.inspect})"
+          end
+
+          # `--all --persistence-parity` forces the parity wave on; narrowing every child to that
+          # one mode would abort each ineligible target. A single named target still narrows to that
+          # mode.
+          @force_parity_wave = @all_mode && @persistence_parity_mode
+          if @force_parity_wave
+            @persistence_parity_mode = false
+            @explicit_modes = nil
+          end
+          validate_parity_and_release
+        end
+
+        def validate_parity_and_release
+          if @persistence_parity_mode && !@target_ref
+            abort "#{USAGE}\n--persistence-parity needs an explicit target-reference — it never auto-picks " \
+                  "from the rotation, since only a PostgresEra-bound domain gains anything from this " \
+                  "comparison at all (`resolved modes:` on any single-target run says whether a target qualifies)"
+          end
+          validate_release if @release_mode
+          abort "#{USAGE}\n--notes only means something with --release" if @release_notes && !@release_mode
+        end
+
+        def validate_release
+          unless @target_ref
+            abort "#{USAGE}\n--release needs an explicit target-reference — the suspended target a person is " \
+                  "putting back"
+          end
+          unless @release_notes
+            abort "#{USAGE}\n--release needs --notes — what a person concluded is the one thing this script " \
+                  "cannot supply"
+          end
+          return unless @all_mode || @persistence_parity_mode
+
+          abort "#{USAGE}\n--release is its own mode — it does not combine with --all or --persistence-parity"
+        end
+      end
+    end
+  end
+end

data/lib/hecks/quality_control/cli/qa_sweep/release_mode.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module Hecks
+  module QualityControlCli
+    class QaSweep
+      # `--release`: a person puts a suspended target back. It concludes or abandons what was open
+      # against the target, then releases it with the streak moved by what was found.
+      module ReleaseMode
+        private
+
+        # `--release`: a person puts a suspended target back, concluding what was open against it.
+        def release_suspended_target
+          target = ::QualityControl::Target.find(@target_reference)
+          abort "target #{@target_reference.inspect} vanished between being queried and being released" unless target
+          unless %w[suspended held].include?(target.status)
+            abort "target #{@target_reference.inspect} is #{target.status.inspect}, not suspended (or held) — " \
+                  "nothing to release"
+          end
+
+          open_sweeps = query("Sweep.Sweeping").select { |row| row[:target] == target.id }
+                                               .map { |row| ::QualityControl::Sweep.find(row[:reference][:value]) }
+          surprises = 0
+          clean = !open_sweeps.empty?
+          open_sweeps.each do |sweep|
+            surprises += query("Bug.FoundIn", sweep_id: { value: sweep.id }).size
+            clean &&= sweep_was_clean?(sweep)
+            close_open_sweep(sweep, surprises)
+          end
+
+          # No open sweep (a crash between claim and open): nothing was watched, so the streak
+          # resets rather than incrementing.
+          puts "no open sweep against #{@target_reference} — releasing with the streak reset" if open_sweeps.empty?
+
+          capabilities = Hecks::Fuzzing::TargetCapabilities.infer(@domain_path, rust_dir: @rust_dir)
+          was = target.status
+          target = release_target!(target, capabilities, target.clean_streak.value, surprises: surprises, clean: clean)
+          puts "released #{@target_reference} (was #{was}) — status #{target.status}, " \
+               "clean_streak #{target.clean_streak.value}, yield_score #{target.yield_score.value}, " \
+               "capabilities #{target.capabilities.value.inspect}"
+          EXIT_OK
+        end
+
+        def close_open_sweep(sweep, surprises)
+          if sweep.made.value.positive? || sweep.waived.value.positive?
+            begin
+              sweep.conclude!(notes: { value: @release_notes })
+            rescue Hecks::Runtime::InvariantViolation => e
+              abort "--notes was refused by Sweep.Conclude — #{e.message}"
+            end
+            puts "concluded #{sweep.id} (#{sweep.made.value} check(s), #{surprises} bug(s) logged against it so far)"
+          else
+            # Never made a check: `Sweep.Conclude` refuses it, so abandon it to leave `Sweeping`.
+            sweep.abandon!
+            puts "abandoned #{sweep.id} (it never made a check)"
+          end
+        end
+      end
+    end
+  end
+end