ibex 0.2.0 → 0.4.0

data/lib/ibex/cli/fuzz_regressions.rb

@@ -0,0 +1,145 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+require "digest"
+require "fileutils"
+require "json"
+
+module Ibex
+  # @rbs!
+  #   type fuzz_external_description = {
+  #     command: String,
+  #     target_runtime: String,
+  #     timeout_seconds: Integer,
+  #     max_output_bytes: Integer,
+  #     host_ruby_engine: String,
+  #     host_ruby_version: String,
+  #     host_platform: String
+  #   }
+  #   type fuzz_external_target = [
+  #     ^(Array[String]) -> Symbol,
+  #     fuzz_external_description
+  #   ]
+
+  # Automatic bounded minimization and persistence for fuzz differences.
+  module CLIFuzzRegressions
+    # @rbs!
+    #   private def atomic_write_ir: (String path, String source) -> void
+
+    private
+
+    # @rbs (Fuzz fuzzer, Fuzz::Mismatch mismatch, String grammar_path,
+    #   fuzz_external_target? external) -> Integer
+    def write_minimized_fuzz_mismatch(fuzzer, mismatch, grammar_path, external)
+      details = minimized_fuzz_mismatch(fuzzer, mismatch, grammar_path, external)
+      report = {
+        ibex_report: "fuzz", schema_version: 1, result: "difference", mismatch: details
+      } #: Hash[Symbol, untyped]
+      report[:external] = external[1] if external
+      write_fuzz_report(report)
+      1
+    end
+
+    # @rbs (Hash[Symbol, untyped] report) -> void
+    def write_fuzz_report(report)
+      if @options.fetch(:fuzz_format, "json") == "json"
+        @stdout.puts JSON.pretty_generate(report)
+        return
+      end
+
+      @stdout.puts("result=#{report.fetch(:result)}")
+      case report.fetch(:result)
+      when "no_difference_within_bounds"
+        @stdout.puts("seed=#{report.fetch(:seed)} bounds=#{report.fetch(:bounds).inspect}")
+        @stdout.puts("no difference found within the declared bounds; this is not a proof of equivalence")
+      when "difference"
+        mismatch = report.fetch(:mismatch) #: Hash[Symbol, untyped]
+        @stdout.puts("kind=#{mismatch.fetch(:kind)} tokens=#{mismatch.fetch(:tokens).inspect}")
+        @stdout.puts("minimized=#{mismatch.fetch(:minimized_tokens).inspect} " \
+                     "reduction_complete=#{mismatch.dig(:reduction, :complete)}")
+        @stdout.puts("regression=#{mismatch.dig(:regression, :path)}") if mismatch[:regression]
+      when "budget_exhausted"
+        @stdout.puts("search incomplete: #{report.dig(:budget, :message)}")
+      end
+      external = report[:external] #: fuzz_external_description?
+      return unless external
+
+      @stdout.puts("target_runtime=#{external.fetch(:target_runtime)}")
+      @stdout.puts("target_command=#{external.fetch(:command)}")
+    end
+
+    # @rbs (Fuzz::BudgetExceeded error,
+    #   fuzz_external_target? external, phase: String) -> Integer
+    def write_fuzz_budget_report(error, external, phase:)
+      report = {
+        ibex_report: "fuzz", schema_version: 1, result: "budget_exhausted",
+        budget: error.details.merge(phase: phase)
+      } #: Hash[Symbol, untyped]
+      report[:external] = external[1] if external
+      write_fuzz_report(report)
+      2
+    end
+
+    # @rbs (Fuzz fuzzer, Fuzz::Mismatch mismatch, String grammar_path,
+    #   fuzz_external_target? external) -> Hash[Symbol, untyped]
+    def minimized_fuzz_mismatch(fuzzer, mismatch, grammar_path, external)
+      result = fuzzer.minimize(
+        mismatch, max_trials: @options.fetch(:fuzz_max_reduction_trials, 1_000)
+      )
+      details = mismatch.details.dup #: Hash[Symbol, untyped]
+      details[:minimized_tokens] = result.items
+      details[:reduction] = {
+        original_size: result.original_size, minimized_size: result.items.length,
+        trials: result.trials, complete: result.complete
+      }
+      regression = persist_fuzz_regression(
+        mismatch.details, result, grammar_path, external&.[](1)
+      )
+      details[:regression] = regression if regression
+      details
+    end
+
+    # @rbs (fuzz_mismatch mismatch, DeltaReducer::Result result, String grammar_path,
+    #   fuzz_external_description? external) -> Hash[Symbol, String]?
+    def persist_fuzz_regression(mismatch, result, grammar_path, external)
+      return if @options.fetch(:fuzz_save_regression, true) == false
+
+      directory = @options.fetch(:fuzz_regression_dir, "test/fuzz/regressions")
+      FileUtils.mkdir_p(directory)
+      document = fuzz_regression_document(mismatch, result, grammar_path, external)
+      source = "#{JSON.pretty_generate(document)}\n"
+      digest = Digest::SHA256.hexdigest(source)
+      seed = @options.fetch(:fuzz_seed, 0)
+      path = File.join(directory, "fuzz-seed-#{seed}-#{digest.slice(0, 16)}.json")
+      validate_fuzz_regression_target!(path)
+      atomic_write_ir(path, source)
+      { path: path, sha256: digest }
+    end
+
+    # @rbs (String path) -> void
+    def validate_fuzz_regression_target!(path)
+      return unless File.symlink?(path) || (File.exist?(path) && File.stat(path).nlink > 1)
+
+      raise Ibex::Error, "(fuzz):1:1: regression output refuses symlinks and files with multiple hard links"
+    end
+
+    # @rbs (fuzz_mismatch mismatch, DeltaReducer::Result result, String grammar_path,
+    #   fuzz_external_description? external) -> Hash[Symbol, Object?]
+    def fuzz_regression_document(mismatch, result, grammar_path, external)
+      document = {
+        ibex_report: "fuzz-regression", schema_version: 1, grammar: grammar_path,
+        seed: @options.fetch(:fuzz_seed, 0), kind: mismatch[:kind],
+        sentence: mismatch[:sentence],
+        algorithms: mismatch[:outcomes].keys.reject { |name| name == :external }.map(&:to_s),
+        original_tokens: mismatch[:tokens], minimized_tokens: result.items,
+        bounds: mismatch[:bounds],
+        reduction: {
+          original_size: result.original_size, minimized_size: result.items.length,
+          trials: result.trials, complete: result.complete
+        }
+      } #: Hash[Symbol, Object?]
+      document[:external] = external if external
+      document
+    end
+  end
+end

data/lib/ibex/cli/generation_artifacts.rb

@@ -12,6 +12,8 @@ module Ibex
     #   private def report_status: (String) -> void
     #   private def verify_file: (String, String, String) -> void
     #   private def default_output_path: (String, String) -> String
+    #   private def configuration_value: (String) -> Object?
+    #   private def effective_configuration: () -> Configuration::Resolver
 
     private
 
@@ -21,6 +23,7 @@ module Ibex
       @generation_statuses = [] #: Array[String]
       @generation_sources = [] #: Array[String]
       @generation_inputs = [] #: Array[GenerationInput]
+      @generation_automaton = nil
     end
 
     # @rbs (Symbol kind, String path, String content, ?mode: Integer?, ?status: bool) -> Artifact
@@ -120,19 +123,67 @@ module Ibex
       default_output_path(parser.path, ".ibex.json")
     end
 
-    # @rbs () -> Hash[String, untyped]
+    # @rbs () -> Hash[String, Object?]
     def generation_manifest_options
       keys = %i[
         action_source algorithm counterexample_max_configurations counterexample_max_tokens debug
         embedded emit entry_isolation executable frozen line_convert line_convert_all messages mode omit_actions rbs
         superclass table
       ]
-      options = {} #: Hash[String, untyped]
+      options = {} #: Hash[String, Object?]
       keys.each do |key|
         value = @options[key]
         options[key.to_s] = value unless value.nil?
       end
+      options["cst_trivia"] = effective_cst_trivia.to_s
+      append_ir_manifest_options(options)
       options
     end
+
+    # @rbs (Hash[String, Object?] options) -> void
+    def append_ir_manifest_options(options)
+      automaton = @generation_automaton
+      return unless automaton
+
+      grammar = automaton.grammar
+      contract = grammar.parser_contract
+      options["grammar_ir"] = {
+        "schema_version" => grammar.schema_version,
+        "digest" => automaton.grammar_digest,
+        "parser_contract" => stringify_contract(contract)
+      }
+      options["automaton_ir"] = {
+        "schema_version" => automaton.schema_version,
+        "algorithm" => automaton.algorithm,
+        "entry_construction" => automaton.entry_construction,
+        "construction_authority" => construction_authority
+      }
+      options["effective_configuration"] = manifest_effective_configuration
+    end
+
+    # @rbs () -> String
+    def construction_authority
+      @options[:from] == "automaton-ir" ? "embedded_automaton" : "grammar_contract"
+    end
+
+    # @rbs () -> Array[Hash[String, Object?]]
+    def manifest_effective_configuration
+      values = effective_configuration.to_h.fetch("configuration")
+      return values unless @options[:from] == "automaton-ir"
+
+      values.reject { |entry| %w[parser.algorithm parser.entries].include?(entry.fetch("key")) }
+    end
+
+    # @rbs (IR::ParserContract contract) -> Hash[String, Object?]
+    def stringify_contract(contract)
+      contract.to_h.to_h do |name, entry|
+        [name.to_s, entry.to_h { |key, value| [key.to_s, value] }]
+      end
+    end
+
+    # @rbs () -> Symbol
+    def effective_cst_trivia
+      configuration_value("cst.trivia") #: Symbol
+    end
   end
 end

data/lib/ibex/cli/generation_error_messages.rb

@@ -16,7 +16,7 @@ module Ibex
       end
     end
 
-    # @rbs (IR::Automaton automaton) -> Hash[Integer, untyped]
+    # @rbs (IR::Automaton automaton) -> Hash[Integer, String | { id: String, message: String }]
     def configured_error_messages(automaton)
       path = @options[:messages]
       return {} unless path

data/lib/ibex/cli/grammar_tests.rb

@@ -7,26 +7,19 @@ module Ibex
   # `%test` command parsing and human-readable result reporting.
   module CLIGrammarTests
     # @rbs!
-    #   type grammar_test_settings = {
-    #     mode: Symbol,
-    #     algorithm: Symbol,
-    #     entry_isolation: bool,
-    #     timeout: Integer,
-    #     coverage: Integer?,
-    #     ?help: bool
-    #   }
+    #   type grammar_test_settings = Hash[Symbol, untyped]
     #   private def print_help: (OptionParser) -> Integer
     #   private def resolve_grammar_path: (String) -> Frontend::Resolution
     #   private def handle_grammar_warnings: (IR::Grammar, String) -> void
     #   private def build_automaton: (IR::Grammar, String) -> IR::Automaton
+    #   private def configuration_value: (String) -> untyped
+    #   private def mark_configuration_option: (Symbol) -> void
 
     private
 
     # @rbs (Array[String] arguments) -> Integer
     def run_grammar_tests_command(arguments)
-      settings = {
-        mode: :default, algorithm: :lalr, entry_isolation: false, timeout: GrammarTests::DEFAULT_TIMEOUT, coverage: nil
-      } #: grammar_test_settings
+      settings = default_grammar_test_settings
       options = grammar_test_option_parser(settings)
       remaining = options.parse(arguments)
       return print_help(options) if settings[:help]
@@ -39,7 +32,7 @@ module Ibex
       @options[:algorithm] = settings[:algorithm]
       @options[:entry_isolation] = settings[:entry_isolation]
       resolution = resolve_grammar_path(path)
-      grammar = Normalizer.new(resolution, mode: settings[:mode]).normalize
+      grammar = normalize_analysis_grammar(resolution)
       handle_grammar_warnings(grammar, path)
       automaton = build_automaton(grammar, path)
       runner = GrammarTests::Runner.new(automaton, timeout: settings[:timeout])
@@ -51,16 +44,31 @@ module Ibex
       render_grammar_test_coverage(runner.production_coverage(results), coverage, failures)
     end
 
-    # @rbs (grammar_test_settings settings) -> OptionParser
+    # @rbs () -> Hash[Symbol, untyped]
+    def default_grammar_test_settings
+      {
+        mode: Configuration::Registry.fetch("grammar.mode").default,
+        algorithm: Configuration::Registry.fetch("parser.algorithm").default,
+        entry_isolation: Configuration::Registry.fetch("parser.entries").default == :isolated,
+        timeout: GrammarTests::DEFAULT_TIMEOUT, coverage: nil
+      }
+    end
+
+    # @rbs (Hash[Symbol, untyped] settings) -> OptionParser
     def grammar_test_option_parser(settings)
       OptionParser.new do |options|
         options.banner = "Usage: ibex test [options] grammarfile"
-        options.on("--mode=MODE", %w[default extended], "grammar mode") { |value| settings[:mode] = value.to_sym }
+        options.on("--mode=MODE", %w[default extended], "grammar mode") do |value|
+          settings[:mode] = value.to_sym
+          mark_configuration_option(:mode)
+        end
         options.on("--algorithm=NAME", %w[slr lalr ielr lr1], "parser construction algorithm") do |value|
           settings[:algorithm] = value.to_sym
+          mark_configuration_option(:algorithm)
         end
         options.on("--entry-isolation", "build independent state sets for each start symbol") do
           settings[:entry_isolation] = true
+          mark_configuration_option(:entry_isolation)
         end
         options.on("--timeout=SECONDS", Integer, "whole-suite timeout (default: 10)") do |value|
           raise OptionParser::InvalidArgument, "timeout must be positive" unless value.positive?
@@ -83,6 +91,16 @@ module Ibex
       raise OptionParser::InvalidArgument, "test accepts exactly one grammarfile"
     end
 
+    # @rbs (Frontend::Resolution resolution) -> IR::Grammar
+    def normalize_analysis_grammar(resolution)
+      grammar = Normalizer.new(resolution, mode: configuration_value("grammar.mode")).normalize
+      activate_analysis_grammar(grammar)
+    end
+
+    # @rbs!
+    #   private def activate_analysis_grammar: (IR::Grammar, ?options: Hash[Symbol, Object?],
+    #     ?explicit_keys: Array[Symbol]) -> IR::Grammar
+
     # @rbs (Array[GrammarTests::Result] results) -> Integer
     def render_grammar_test_results(results)
       failures = 0