hecks 3.4.2 → 3.6.0

data/lib/hecks/quality_control/cli/qa_generated_domains/evaluation.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+require "open3"
+
+module Hecks
+  module QualityControlCli
+    class QaGeneratedDomains
+      # Checking one candidate domain: written to disk, optionally built against Rust, then run in
+      # its own child process.
+      module Evaluation
+        private
+
+        # `--check DIR`: one domain in this process, one result line.
+        def check_one
+          result = Hecks::Fuzzing::GeneratedDomainCheck.run(
+            @options[:check], seeds: @options[:seeds], steps: @options[:steps], adversarial: @options[:adversarial],
+                              binary: @options[:binary], differ: conformance_differ, match: @options[:match],
+                              shrink_budget: @options[:shrink_budget]
+          )
+          puts "#{RESULT_MARKER}#{JSON.generate(result)}"
+          EXIT_OK
+        end
+
+        def conformance_differ
+          return unless @options[:binary]
+
+          require File.join(@root, "spec/support/rust_conformance_helpers")
+          Class.new do
+            include RustConformanceHelpers
+
+            attr_reader :structural_skips
+
+            def initialize
+              @structural_skips = Set.new
+            end
+          end.new
+        end
+
+        def child_check(dir, binary: nil, match: nil, shrink_budget: 0)
+          args = child_check_args(dir, binary, match, shrink_budget)
+          output, status = Open3.capture2e(*Child.argv(@root, "qa_generated_domains", *args), chdir: @root)
+          line = output.lines.reverse.find { |candidate| candidate.start_with?(RESULT_MARKER) }
+          return JSON.parse(line.delete_prefix(RESULT_MARKER)) if line
+
+          { "status" => "error",
+            "error"  => "child exited #{status.exitstatus} with no result: #{output.lines.last(5).join.strip}" }
+        end
+
+        def child_check_args(dir, binary, match, shrink_budget)
+          args = ["--check", dir, "--seeds", @options[:seeds].to_s, "--steps", @options[:steps].to_s,
+                  "--adversarial", @options[:adversarial].to_s, "--shrink-budget", shrink_budget.to_s]
+          args += ["--binary", binary] if binary
+          args += ["--match", JSON.generate(match)] if match
+          args
+        end
+
+        def build_rust(dir)
+          project = Hecks::RustBuild.capture("project_rust", [File.expand_path(dir, @root)],
+                                             env: { "HECKS_RUST_DIR" => @scratch })
+          return [nil, rust_failure("rust_projection", "#{project.out}#{project.err}".lines.last(12).join)] unless project.ok?
+
+          cargo_build
+        end
+
+        def cargo_build
+          cargo, status = Open3.capture2e("cargo", "build", "--no-default-features", "--features",
+                                          Generator::DIRECTORY, chdir: @scratch)
+          return [nil, rust_failure("rust_build", cargo.lines.grep(/error/).first(12).join)] unless status.success?
+
+          binary = File.join(@scratch, "target/debug/rust-#{Generator::DIRECTORY}")
+          FileUtils.cp(File.join(@scratch, "target/debug/rust"), binary)
+          [binary, nil]
+        end
+
+        def rust_failure(mode, detail) = { "mode" => mode, "detail" => detail }
+
+        def sync_scratch!
+          FileUtils.mkdir_p(@scratch)
+          sources = %w[Cargo.toml Cargo.lock src].map { |entry| File.join(@root, "rust", entry) }
+          system("rsync", "-a", "--delete", "--exclude", "target", *sources, "#{@scratch}/") or
+            abort "rsync into #{@scratch} failed"
+        end
+
+        # Builds one candidate (when --rust) and runs a child check. A build failure is its own
+        # result shape, so a rust_build finding shrinks by "does it still fail to build".
+        def evaluate(blueprint, dir_root, match: nil, shrink_budget: 0)
+          dir = Generator.write(blueprint, dir_root)
+          binary = nil
+          if @options[:rust]
+            binary, failure = build_rust(dir)
+            return rust_failure_result(failure, match, dir) if failure
+            return { "status" => "clean" } if match && match["mode"].start_with?("rust_")
+          end
+          child_check(dir, binary: binary, match: match, shrink_budget: shrink_budget)
+            .merge("binary" => binary, "dir" => dir)
+        end
+
+        def rust_failure_result(failure, match, dir)
+          return { "status" => "clean" } unless match.nil? || match["mode"] == failure["mode"]
+
+          { "status" => "found", "mode" => failure["mode"], "signature" => [failure["mode"]], "dir" => dir,
+            "divergences" => [{ "field" => failure["mode"], "detail" => failure["detail"] }], "steps" => [] }
+        end
+      end
+    end
+  end
+end

data/lib/hecks/quality_control/cli/qa_generated_domains/promotion.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+module Hecks
+  module QualityControlCli
+    class QaGeneratedDomains
+      # `--promote`: turns a finding's minimal domain into a stress domain under
+      # `qa/stress_domains`, with notes that carry the blueprint and the finding.
+      module Promotion
+        private
+
+        def camelize(name) = name.split("_").map(&:capitalize).join
+
+        def promote
+          name = promotion_name
+          source = promotion_source
+          target = File.join(@root, "qa/stress_domains", name)
+          abort "#{target} already exists — pick another --name" if File.exist?(target)
+
+          write_promotion(name, source, target)
+          print_promotion(name)
+          EXIT_OK
+        end
+
+        def promotion_name
+          name = @options[:name] || abort("#{USAGE}\n--promote needs --name")
+          return name if name.match?(/\A[a-z][a-z0-9_]*\z/)
+
+          abort "--name must be a lowercase identifier (it becomes a Rust module and Cargo feature)"
+        end
+
+        def promotion_source
+          source = File.join(@options[:promote], Generator::DIRECTORY, "bluebook", "#{Generator::DIRECTORY}.bluebook")
+          abort "no generated domain at #{source}" unless File.exist?(source)
+
+          source
+        end
+
+        def write_promotion(name, source, target)
+          FileUtils.mkdir_p(File.join(target, "bluebook"))
+          File.write(File.join(target, "bluebook", "#{name}.bluebook"),
+                     File.read(source).gsub(Generator::DOMAIN_NAME, camelize(name)))
+          File.write(File.join(target, "NOTES.md"), notes_for(name))
+        end
+
+        def print_promotion(name)
+          puts "promoted: qa/stress_domains/#{name}"
+          puts "next:"
+          puts "  hecks quality_control judge_novelty qa/stress_domains/#{name}"
+          puts "  hecks project_rust qa/stress_domains/#{name}  # when the finding needs the Rust comparison"
+          # `hecks quality_control target.seed` derives targets from
+          # `Hecks::Corpus.rotation_targets`, so a hand-typed `target.identify` line is not printed:
+          # nobody ran it and the domain never
+          # entered the rotation.
+          puts "  hecks quality_control target.seed   # idempotent; picks this domain up from the corpus"
+        end
+
+        def notes_for(name)
+          recorded = recorded_blueprint
+          notes_header(name, recorded) + notes_sections(recorded)
+        end
+
+        def notes_header(name, recorded)
+          hypothesis = recorded&.dig("hypothesis")
+          <<~NOTES
+            # #{name}
+
+            Promoted from a `hecks quality_control check_generated_domains` finding — #{origin_phrase(recorded)}, not
+            hand-written. The bluebook is the minimal form of the domain that
+            surprised; `QaGenerated` was renamed to `#{camelize(name)}` and nothing
+            else changed.
+            #{"\n## Hypothesis\n\n#{hypothesis.strip}\n" if hypothesis}
+          NOTES
+        end
+
+        def notes_sections(recorded)
+          <<~NOTES
+            ## Blueprint
+
+            ```json
+            #{recorded ? JSON.pretty_generate(recorded.except("source", "hypothesis")) : "(not recorded)"}
+            ```
+
+            ## Finding
+
+            ```json
+            #{recorded_finding}
+            ```
+          NOTES
+        end
+
+        def recorded_blueprint
+          blueprint = File.join(@options[:promote], "blueprint.json")
+          File.exist?(blueprint) ? JSON.parse(File.read(blueprint)) : nil
+        end
+
+        def recorded_finding
+          finding = File.join(@options[:promote], "finding.json")
+          File.exist?(finding) ? File.read(finding).strip : "(not recorded)"
+        end
+
+        def origin_phrase(recorded)
+          recorded&.key?("source") ? "written by `hecks quality_control mine_combinations`' agent" : "generated"
+        end
+      end
+    end
+  end
+end

data/lib/hecks/quality_control/cli/qa_generated_domains/reporting.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+module Hecks
+  module QualityControlCli
+    class QaGeneratedDomains
+      # What `target.check_generated_domains` prints: a line per domain, and a block per finding.
+      module Reporting
+        private
+
+        def relative(path) = path.delete_prefix("#{@root}/")
+
+        def announce_run(start, run_dir)
+          puts "generated domains: #{@options[:domains]} starting at seed #{start}; #{@options[:seeds]} seed(s) x " \
+               "#{@options[:steps]} steps each, adversarial #{@options[:adversarial]}" \
+               "#{", against Rust" if @options[:rust]} — #{relative(run_dir)}"
+        end
+
+        def print_finding_header(report)
+          puts
+          puts "=" * 72
+          puts "GENERATED DOMAIN FOUND SOMETHING — domain seed #{report[:seed]}, forms #{report[:forms].join(" + ")}"
+          puts "=" * 72
+          print_finding_locations(report)
+        end
+
+        def print_finding_locations(report)
+          puts "domain:      #{relative(report[:dir])}"
+          puts "             shrunk from #{report[:size_before]} to #{report[:size_after]} removable element(s) " \
+               "in #{report[:domain_attempts]} candidate check(s)"
+          puts "blueprint:   #{relative(File.join(report[:root], "blueprint.json"))}"
+        end
+
+        def print_finding(report)
+          found = report[:final]
+          print_finding_header(report)
+          print_finding_facts(found, report[:options])
+          print_shrunk(report, found["steps"], found["shrunk_steps"])
+          puts "promote:     hecks quality_control check_generated_domains --promote " \
+               "#{relative(report[:root])} --name <stress_domain_name>"
+          puts
+          found["divergences"].each { |divergence| print_divergence(divergence) }
+        end
+
+        def print_finding_facts(found, options)
+          puts "mode:        #{found["mode"]}"
+          puts "signature:   #{found["signature"].join(", ")}"
+          return unless found["seed"]
+
+          puts "sequence:    seed #{found["seed"]} of --seeds #{options[:seeds]}, #{options[:steps]} steps, " \
+               "adversarial #{options[:adversarial]}"
+        end
+
+        def print_divergence(divergence)
+          puts "-- #{divergence["field"]} --"
+          divergence.except("field").each do |key, value|
+            puts "#{key}: #{value.is_a?(String) ? value : JSON.generate(value)}"
+          end
+          puts
+        end
+
+        def print_shrunk(report, steps, shrunk)
+          return unless shrunk
+
+          puts "shrunk:      #{steps.size} -> #{shrunk.size} step(s) — #{relative(report[:steps_file])}"
+          puts "replay:      #{replay_line(report)}"
+          shrunk.each_with_index do |step, index|
+            kind = %w[verb query dry_run].find { |key| step.key?(key) }
+            puts "  #{index}: #{"#{kind} " unless kind == "verb"}#{step[kind]}  args: #{JSON.generate(step["args"])}"
+          end
+        end
+
+        def replay_line(report)
+          if report[:binary]
+            "hecks check_conformance #{report[:dir]} script=#{report[:steps_file]} " \
+              "artifact=#{report[:binary]}"
+          else
+            "hecks run #{report[:dir]} #{report[:steps_file]}"
+          end
+        end
+
+        def report(counts, findings)
+          puts
+          puts "generated domains: #{counts[:clean]} clean, #{counts[:found]} found something, " \
+               "#{counts[:invalid]} invalid, #{counts[:error]} errored"
+          findings.each { |finding| print_finding(finding) }
+          exit_status(counts, findings)
+        end
+
+        def exit_status(counts, findings)
+          return EXIT_FOUND if findings.any?
+
+          counts[:clean].zero? ? EXIT_ERROR : EXIT_OK
+        end
+      end
+    end
+  end
+end

data/lib/hecks/quality_control/cli/qa_generated_domains/shrinking.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Hecks
+  module QualityControlCli
+    class QaGeneratedDomains
+      # What happens to a domain that found something: shrunk to its minimal form, re-checked, and
+      # written beside its blueprint and shrunk steps.
+      module Shrinking
+        private
+
+        # @return [Array(Hash, Integer)] the smallest blueprint that still finds the same thing,
+        #   and the candidate checks it took
+        def shrink_domain(blueprint, found, root)
+          match = { "mode" => found["mode"], "signature" => found["signature"] }
+          current = blueprint
+          attempts = 0
+          loop do
+            candidate, attempts = next_shrink(current, attempts, root, match)
+            break unless candidate
+
+            current = candidate
+            break unless attempts < @options[:domain_shrink_budget]
+          end
+          [current, attempts]
+        end
+
+        # @return [Array(Hash, Integer)] the first smaller candidate that still finds the same thing
+        #   (nil when none or the budget ran out), and the attempts so far
+        def next_shrink(current, attempts, root, match)
+          Generator.shrink_candidates(current).each do |candidate|
+            return [nil, attempts] if attempts >= @options[:domain_shrink_budget]
+
+            attempts += 1
+            return [candidate, attempts] if evaluate(candidate, File.join(root, "candidate"), match: match)["status"] == "found"
+          end
+          [nil, attempts]
+        end
+
+        def record_finding(blueprint, result, root)
+          minimal, attempts = shrink_domain(blueprint, result, root)
+          final_root = File.join(root, "minimal")
+          minimal, final = final_evaluation(blueprint, minimal, result, final_root)
+          steps_file = write_finding(final, final_root)
+          { forms: blueprint["forms"], root: final_root, dir: final["dir"], binary: final["binary"], final: final,
+            options: @options, steps_file: steps_file, domain_attempts: attempts,
+            size_before: Generator.removals(blueprint).size, size_after: Generator.removals(minimal).size }
+        end
+
+        # @return [Array(Hash, Hash)] the blueprint the finding is recorded for, and its final check
+        def final_evaluation(blueprint, minimal, result, final_root)
+          match = { "mode" => result["mode"], "signature" => result["signature"] }
+          final = evaluate(minimal, final_root, match: match, shrink_budget: @options[:shrink_budget])
+          return [minimal, final] if final["status"] == "found"
+
+          [blueprint, evaluate(blueprint, final_root, shrink_budget: @options[:shrink_budget])]
+        end
+
+        # @return [String] the path of the shrunk steps file
+        def write_finding(final, final_root)
+          File.write(File.join(final_root, "finding.json"), JSON.pretty_generate(final.except("dir", "binary")))
+          steps_file = File.join(final_root, "shrunk_steps.json")
+          note = "hecks quality_control check_generated_domains finding"
+          File.write(steps_file, JSON.pretty_generate(name: "qa_generated-shrunk", note: note,
+                                                      steps: final.fetch("shrunk_steps") { final.fetch("steps", []) }))
+          steps_file
+        end
+      end
+    end
+  end
+end