ibex 0.2.0 → 0.4.0

data/lib/ibex/frontend/generated_parser_base.rb

@@ -3,6 +3,7 @@
 require_relative "generated_parser_metadata"
 require_relative "generated_parser_includes"
 require_relative "generated_parser_parameters"
+require_relative "parser_configuration_support"
 
 module Ibex
   module Frontend
@@ -12,6 +13,7 @@ module Ibex
       include GeneratedParserMetadata
       include GeneratedParserIncludes
       include GeneratedParserParameters
+      include ParserConfigurationSupport
 
       attr_reader :diagnostic_token #: Token?
 
@@ -29,7 +31,7 @@ module Ibex
 
       # @rbs () -> (AST::Root | AST::Fragment)
       def parse
-        do_parse
+        do_parse #: AST::Root | AST::Fragment
       end
 
       # @rbs () -> ([external_token, Token] | false)
@@ -42,7 +44,7 @@ module Ibex
         raise
       end
 
-      # @rbs (Integer? _token_id, Token? value, Array[untyped] _value_stack) -> bot
+      # @rbs (Integer? _token_id, Token? value, Array[Object?] _value_stack) -> bot
       def on_error(_token_id, value, _value_stack)
         token = value || @adapter.eof_token
         @diagnostic_token = token
@@ -104,10 +106,12 @@ module Ibex
       # @rbs (Token class_token, Array[String] class_parts, Array[String]? superclass,
       #   Array[AST::declaration] declarations, Array[AST::Rule] rules, AST::user_code user_code) -> AST::Root
       def build_root(class_token, class_parts, superclass, declarations, rules, user_code)
+        validate_root_parser_configuration(declarations)
         AST::Root.new(class_name: class_parts.join("::"), superclass: superclass&.join("::"),
                       declarations: declarations, rules: rules, user_code: user_code, loc: class_token.location,
                       extended: @mode == :extended || @adapter.extended_pragma? || @adapter.cst_pragma?,
-                      cst: @adapter.cst_pragma?)
+                      cst: @adapter.cst_pragma?,
+                      extended_loc: @adapter.extended_pragma_location || @adapter.cst_pragma_location)
       end
 
       # @rbs (Token keyword, Array[[String, String?]] entries) -> AST::Tokens

data/lib/ibex/frontend/generated_parser_includes.rb

@@ -44,17 +44,7 @@ module Ibex
 
       # @rbs (AST::declaration declaration) -> String?
       def fragment_root_declaration_name(declaration)
-        return "options" if declaration.is_a?(AST::Options)
-        return "expect" if declaration.is_a?(AST::Expect)
-        return "%expect-rr" if declaration.is_a?(AST::ExpectRR)
-        return "%param" if declaration.is_a?(AST::Parameter)
-        return "start" if declaration.is_a?(AST::Start)
-        return "%recover" if declaration.is_a?(AST::Recovery)
-        return "%on_error_reduce" if declaration.is_a?(AST::OnErrorReduce)
-        return "%test" if declaration.is_a?(AST::GrammarTest)
-        return "lexer" if declaration.is_a?(AST::Lexer)
-
-        nil
+        AST::ROOT_ONLY_DECLARATION_NAMES.find { |node_class, _name| declaration.is_a?(node_class) }&.last
       end
     end
   end

data/lib/ibex/frontend/generation.rb

@@ -5,7 +5,7 @@
 # the DSL, and every other frontend facility.
 require_relative "../error"
 require_relative "../tables"
-require "ibex/runtime"
+require "ibex/runtime/parser"
 require_relative "source_span"
 require_relative "source_cursor"
 require_relative "source_document"

data/lib/ibex/frontend/parser/declarations.rb

@@ -8,7 +8,7 @@ module Ibex
     module BootstrapParserDeclarations
       DECLARATIONS = %w[
         include import token prechigh preclow options expect expect_rr start recover on_error_reduce
-        test lexer convert display type param printer pragma rule
+        test lexer convert display type param printer parser pragma rule
       ].freeze #: Array[String]
       ASSOCIATIVITIES = %w[left right nonassoc precedence].freeze #: Array[String]
 
@@ -33,7 +33,7 @@ module Ibex
           fail_at(value.location, "unknown pragma #{name}") unless %w[extended cst].include?(name)
           fail_at(keyword.location, "duplicate pragma #{name}") if @pragmas[name]
 
-          @pragmas[name] = true
+          @pragmas[name] = keyword.location
           @mode = :extended
         end
       end
@@ -58,11 +58,26 @@ module Ibex
         when "type" then parse_symbol_metadata(AST::SemanticType, "type")
         when "param" then parse_parameter
         when "printer" then parse_printer
+        when "parser" then parse_parser_configuration
         when "pragma" then fail_at(current.location, "expected rule, got pragma")
         else fail_expected("a declaration or rule")
         end
       end
 
+      # @rbs () -> AST::ParserConfiguration
+      def parse_parser_configuration
+        # @type self: BootstrapParser
+        keyword = advance
+        settings = [] #: Array[AST::ParserSetting]
+        until keyword?("end") || current.type == :eof
+          key = expect(:identifier)
+          value = expect(:identifier)
+          settings << build_parser_setting(key, value)
+        end
+        expect_keyword("end")
+        build_parser_configuration(keyword, settings)
+      end
+
       # @rbs () -> AST::Include
       def parse_import
         # @type self: BootstrapParser

data/lib/ibex/frontend/parser_configuration_support.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module Ibex
+  module Frontend
+    # Shares the closed parser-declaration semantics between bootstrap and generated frontends.
+    module ParserConfigurationSupport
+      # @rbs!
+      #   private def token_string: (Token) -> String
+      #   private def fail_at: (Location, String) -> bot
+      #   private def extended_only!: (Location, String) -> void
+
+      PARSER_SETTING_VALUES = {
+        "algorithm" => %w[slr lalr ielr lr1].freeze,
+        "entries" => %w[shared isolated].freeze,
+        "cst_trivia" => %w[leading balanced drop].freeze
+      }.freeze #: Hash[String, Array[String]]
+
+      private
+
+      # @rbs (Token key, Token value) -> AST::ParserSetting
+      def build_parser_setting(key, value)
+        name = token_string(key)
+        admitted = PARSER_SETTING_VALUES[name]
+        fail_at(key.location, "unknown parser setting #{name}") unless admitted
+
+        selected = token_string(value)
+        unless admitted.include?(selected)
+          fail_at(value.location, "parser.#{name} must be one of #{admitted.join(', ')}; got #{selected}")
+        end
+
+        AST::ParserSetting.new(key: name.to_sym, value: selected.to_sym, loc: key.location)
+      end
+
+      # @rbs (Token keyword, Array[AST::ParserSetting] settings) -> AST::ParserConfiguration
+      def build_parser_configuration(keyword, settings)
+        extended_only!(keyword.location, "parser declarations")
+        seen = {} #: Hash[Symbol, bool]
+        settings.each do |setting|
+          fail_at(setting.loc, "duplicate parser setting #{setting.key}") if seen[setting.key]
+
+          seen[setting.key] = true
+        end
+        fail_at(keyword.location, "parser declaration requires at least one setting") if settings.empty?
+
+        AST::ParserConfiguration.new(settings: settings, loc: keyword.location)
+      end
+
+      # @rbs (Array[AST::declaration] declarations) -> void
+      def validate_root_parser_configuration(declarations)
+        configurations = declarations.grep(AST::ParserConfiguration)
+        duplicate = configurations.fetch(1, nil)
+        fail_at(duplicate.loc, "duplicate parser declaration") if duplicate
+      end
+    end
+  end
+end

data/lib/ibex/frontend/regenerator.rb

@@ -8,6 +8,7 @@ require_relative "source_document"
 require_relative "action_scanner"
 require_relative "lexer"
 require_relative "ast"
+require_relative "parser_configuration_support"
 require_relative "parser/declarations"
 require_relative "parser/rules"
 require_relative "parser/parameters"

data/lib/ibex/frontend/resolution.rb

@@ -36,7 +36,8 @@ module Ibex
 
       private
 
-      # @rbs (untyped value) -> untyped
+      # AST payloads are recursively heterogeneous and are only traversed to freeze them.
+      # @rbs (Object? value) -> Object?
       def deep_freeze_ast(value)
         case value
         when Struct

data/lib/ibex/frontend/resolver.rb

@@ -44,7 +44,7 @@ module Ibex
         root = AST::Root.new(
           class_name: parsed_root.class_name, superclass: parsed_root.superclass,
           declarations: declarations, rules: rules, user_code: parsed_root.user_code, loc: parsed_root.loc,
-          extended: parsed_root.extended, cst: parsed_root.cst
+          extended: parsed_root.extended, cst: parsed_root.cst, extended_loc: parsed_root.extended_loc
         )
         @loaded[@root_path] = true
         @visiting.pop

data/lib/ibex/frontend/rule_documentation.rb

@@ -26,7 +26,8 @@ module Ibex
         if node.is_a?(AST::Root)
           AST::Root.new(
             class_name: node.class_name, superclass: node.superclass, declarations: node.declarations,
-            rules: rules, user_code: node.user_code, loc: node.loc, extended: node.extended, cst: node.cst
+            rules: rules, user_code: node.user_code, loc: node.loc, extended: node.extended, cst: node.cst,
+            extended_loc: node.extended_loc
           )
         else
           AST::Fragment.new(declarations: node.declarations, rules: rules, loc: node.loc)

data/lib/ibex/frontend/source_cursor.rb

@@ -32,7 +32,7 @@ module Ibex
     )
 
     class Token
-      # @rbs () -> Hash[Symbol, untyped]
+      # @rbs () -> Hash[Symbol, Object?]
       def to_h
         { type: type, value: value, location: location.to_h }
       end
@@ -50,7 +50,7 @@ module Ibex
       # @rbs (String source, String file) -> void
       def initialize(source, file)
         @source = SourceEncoding.validated_utf8(source, file)
-        @file = file.dup.freeze
+        @file = SourceEncoding.validated_file(file)
         @ascii_only = @source.ascii_only?
         @character_length = @ascii_only ? @source.bytesize : @source.length
         @index = 0

data/lib/ibex/frontend/source_span.rb

@@ -11,6 +11,14 @@ module Ibex
 
         validated.freeze
       end
+
+      # @rbs (String file) -> String
+      def self.validated_file(file)
+        validated = file.dup.force_encoding(Encoding::UTF_8)
+        raise Ibex::Error, "#{file}: file path must be valid UTF-8" unless validated.valid_encoding?
+
+        validated.freeze
+      end
     end
 
     # An immutable position in grammar source.
@@ -72,7 +80,7 @@ module Ibex
         length.zero?
       end
 
-      # @rbs () -> Hash[Symbol, untyped]
+      # @rbs () -> Hash[Symbol, Object?]
       def to_h
         { file: file, start: start.to_h, end: finish.to_h }
       end

data/lib/ibex/frontend/token_adapter/declaration_state.rb

@@ -26,6 +26,7 @@ module Ibex
           "display" => %i[DISPLAY display_symbol], "type" => %i[TYPE type_symbol],
           "param" => %i[PARAM param_name],
           "printer" => %i[PRINTER printer_symbol],
+          "parser" => %i[PARSER parser_key],
           "rule" => %i[RULE rules]
         }.freeze #: Hash[String, [external_token, Symbol]]
         ASSOCIATIONS = {
@@ -49,7 +50,8 @@ module Ibex
           test_source: "a double-quoted string",
           lexer_entries: "a lexer rule, state, or end", lexer_state_name: "a state name",
           lexer_state_do: "do", lexer_pattern: "a regular expression or quoted literal",
-          lexer_action_or_entry: "an action, lexer rule, state, or end"
+          lexer_action_or_entry: "an action, lexer rule, state, or end",
+          parser_key: "a parser setting or end", parser_value: "a parser setting value"
         }.freeze #: Hash[Symbol, String]
 
         attr_reader :conversion_name #: Token?
@@ -59,13 +61,13 @@ module Ibex
 
         # @rbs @extended_mode: bool
         # @rbs @fragment: bool?
-        # @rbs @pragmas: Hash[String, bool]
+        # @rbs @pragmas: Hash[String, Location]
         # @rbs @pragma_location: Location?
 
         # @rbs (?extended: bool) -> void
         def initialize(extended: false)
           @extended_mode = extended
-          @pragmas = {} #: Hash[String, bool]
+          @pragmas = {} #: Hash[String, Location]
           @state = :class_keyword
         end
 
@@ -84,12 +86,22 @@ module Ibex
 
         # @rbs () -> bool
         def extended_pragma?
-          @pragmas["extended"] == true
+          @pragmas.key?("extended")
         end
 
         # @rbs () -> bool
         def cst_pragma?
-          @pragmas["cst"] == true
+          @pragmas.key?("cst")
+        end
+
+        # @rbs () -> Location?
+        def extended_pragma_location
+          @pragmas["extended"]
+        end
+
+        # @rbs () -> Location?
+        def cst_pragma_location
+          @pragmas["cst"]
         end
 
         # @rbs (Token? token) -> String?
@@ -134,10 +146,30 @@ module Ibex
           when :test_expectation then begin_test_source(token)
           when :pragma_value then finish_pragma(token)
           when :convert_name then begin_conversion(token, :IDENTIFIER, remaining)
+          when :parser_key then begin_parser_setting(token)
+          when :parser_value then finish_parser_setting
           else :IDENTIFIER
           end
         end
 
+        # @rbs (Token token) -> external_token
+        def begin_parser_setting(token)
+          if string_value(token) == "end"
+            @state = :declaration
+            @declaration = nil
+            return :END
+          end
+
+          @state = :parser_value
+          :IDENTIFIER
+        end
+
+        # @rbs () -> external_token
+        def finish_parser_setting
+          @state = :parser_key
+          :IDENTIFIER
+        end
+
         # @rbs (Array[Token] remaining) -> external_token
         def constant_name(remaining)
           following = remaining.first
@@ -179,7 +211,7 @@ module Ibex
           location = @pragma_location || token.location
           raise Ibex::Error, "#{location}: duplicate pragma #{value}" if @pragmas[value]
 
-          @pragmas[value] = true
+          @pragmas[value] = location
           @pragma_location = nil
           @state = :declaration
           @declaration = nil

data/lib/ibex/frontend/token_adapter.rb

@@ -65,6 +65,16 @@ module Ibex
         @declarations.cst_pragma?
       end
 
+      # @rbs () -> Location?
+      def extended_pragma_location
+        @declarations.extended_pragma_location
+      end
+
+      # @rbs () -> Location?
+      def cst_pragma_location
+        @declarations.cst_pragma_location
+      end
+
       # @rbs () -> parser_section
       def section
         rules = @rules

data/lib/ibex/frontend.rb

@@ -11,6 +11,7 @@ require_relative "frontend/action_scanner"
 require_relative "frontend/lexer"
 require_relative "frontend/lexer_recovery"
 require_relative "frontend/ast"
+require_relative "frontend/parser_configuration_support"
 require_relative "frontend/rule_documentation"
 require_relative "frontend/token_adapter/declaration_document_state"
 require_relative "frontend/token_adapter/declaration_state"

data/lib/ibex/fuzz.rb

@@ -0,0 +1,219 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+require_relative "samples"
+require_relative "table_simulation"
+require_relative "delta_reducer"
+
+module Ibex
+  # @rbs!
+  #   type fuzz_outcome = Symbol | [Symbol, String, String]
+  #   type fuzz_mismatch = {
+  #     tokens: Array[String],
+  #     kind: Symbol,
+  #     sentence: Integer,
+  #     outcomes: Hash[Symbol, fuzz_outcome],
+  #     bounds: Hash[Symbol, Integer]
+  #   }
+  #   type fuzz_budget = {
+  #     message: String,
+  #     bounds: Hash[Symbol, Integer],
+  #     ?tokens: Array[String],
+  #     ?phase: String
+  #   }
+
+  # Bounded grammar-derived differential fuzzing without semantic execution.
+  class Fuzz
+    ALGORITHMS = %i[slr lalr ielr lr1].freeze #: Array[Symbol]
+    DEFAULT_MAX_ACTIONS = 100_000 #: Integer
+    DEFAULT_MAX_STACK = 10_000 #: Integer
+
+    class Mismatch < Ibex::Error
+      attr_reader :details #: fuzz_mismatch
+
+      # @rbs (fuzz_mismatch details) -> void
+      def initialize(details)
+        @details = IR.deep_freeze(details)
+        super("(fuzz):1:1: differential mismatch for #{details[:tokens].inspect}")
+      end
+    end
+
+    class BudgetExceeded < Ibex::Error
+      attr_reader :details #: fuzz_budget
+
+      # @rbs (fuzz_budget details) -> void
+      def initialize(details)
+        @details = IR.deep_freeze(details)
+        super("(fuzz):1:1: configured budget was exhausted: #{details[:message]}")
+      end
+    end
+
+    # @rbs (IR::Grammar grammar, ?seed: Integer, ?count: Integer, ?max_tokens: Integer,
+    #   ?max_depth: Integer, ?max_expansions: Integer, ?max_actions: Integer, ?max_stack: Integer,
+    #   ?coverage_guided: bool, ?path_length: Integer, ?algorithms: Array[Symbol],
+    #   ?ielr_strategy: Symbol, ?automata: Hash[Symbol, IR::Automaton]?, ?against: (^(Array[String]) -> Symbol)?,
+    #   ?against_description: Hash[Symbol, Object?]?) -> void
+    # rubocop:disable Metrics/ParameterLists
+    def initialize(grammar, seed: 0, count: 100, max_tokens: 32, max_depth: 16,
+                   max_expansions: Samples::DEFAULT_MAX_EXPANSIONS, max_actions: DEFAULT_MAX_ACTIONS,
+                   max_stack: DEFAULT_MAX_STACK, coverage_guided: false, path_length: 2,
+                   algorithms: ALGORITHMS, ielr_strategy: :partition, automata: nil,
+                   against: nil, against_description: nil)
+      raise ArgumentError, "count must be positive" unless count.positive?
+      raise ArgumentError, "algorithms must not be empty" if algorithms.empty?
+      unless LALR::Builder::IELR_STRATEGIES.include?(ielr_strategy.to_sym)
+        raise ArgumentError, "unknown IELR construction strategy #{ielr_strategy.inspect}"
+      end
+
+      @grammar = grammar
+      @seed = seed
+      @count = count
+      @random = Random.new(seed ^ 0x1BE)
+      @max_tokens = max_tokens
+      @max_depth = max_depth
+      @max_expansions = max_expansions
+      @max_actions = max_actions
+      @max_stack = max_stack
+      @coverage_guided = coverage_guided
+      @path_length = path_length
+      @algorithms = algorithms.map(&:to_sym).freeze
+      @ielr_strategy = ielr_strategy.to_sym
+      @automata = automata || build_automata
+      @against = against
+      @against_description = against_description
+    end
+    # rubocop:enable Metrics/ParameterLists
+
+    # @rbs () -> Hash[Symbol, Object?]
+    def run
+      sentences = generate_sentences
+      mutation_count = 0
+      sentences.each_with_index do |tokens, index|
+        compare!(tokens, kind: :generated, sentence: index)
+        mutations(tokens).each do |mutation|
+          compare!(mutation, kind: :mutation, sentence: index)
+          mutation_count += 1
+        end
+      end
+      successful_report(sentences.length, mutation_count)
+    rescue Ibex::Error => e
+      raise if e.is_a?(Mismatch) || e.is_a?(BudgetExceeded) || !budget_error?(e)
+
+      raise BudgetExceeded.new(
+        message: e.message,
+        bounds: { max_expansions: @max_expansions, max_actions: @max_actions, max_stack: @max_stack }
+      )
+    end
+
+    # Minimize one observed mismatch without changing its kind or outcomes.
+    # @rbs (Mismatch mismatch, ?max_trials: Integer) -> DeltaReducer::Result
+    def minimize(mismatch, max_trials: 1_000)
+      details = mismatch.details
+      original = details[:tokens]
+      kind = details[:kind].to_sym
+      sentence = details[:sentence]
+      outcomes = details[:outcomes]
+      DeltaReducer.new(max_trials: max_trials).minimize(original) do |candidate|
+        mismatch_reproduced?(candidate, kind: kind, sentence: sentence, outcomes: outcomes)
+      end
+    end
+
+    private
+
+    # @rbs () -> Array[Array[String]]
+    def generate_sentences
+      Samples.new(
+        @grammar, seed: @seed, max_tokens: @max_tokens, max_depth: @max_depth,
+                  max_expansions: @max_expansions, strategy: @coverage_guided ? :coverage : :random,
+                  path_length: @path_length
+      ).generate(count: @count)
+    end
+
+    # @rbs (Integer sentence_count, Integer mutation_count) -> untyped
+    def successful_report(sentence_count, mutation_count)
+      report = {
+        ibex_report: "fuzz", schema_version: 1, seed: @seed,
+        bounds: {
+          sentences: @count, max_tokens: @max_tokens, max_depth: @max_depth,
+          max_expansions: @max_expansions, max_actions: @max_actions, max_stack: @max_stack
+        },
+        strategy: @coverage_guided ? "coverage" : "random",
+        path_length: @path_length,
+        ielr_strategy: @ielr_strategy.to_s,
+        algorithms: @algorithms.map(&:to_s),
+        generated_sentences: sentence_count,
+        mutated_sentences: mutation_count,
+        result: "no_difference_within_bounds"
+      }
+      report[:external] = @against_description if @against_description
+      IR.deep_freeze(report)
+    end
+
+    # @rbs () -> Hash[Symbol, IR::Automaton]
+    def build_automata
+      @algorithms.to_h do |algorithm|
+        builder = LALR::Builder.new(
+          @grammar,
+          algorithm: algorithm,
+          ielr_strategy: @ielr_strategy
+        )
+        [algorithm, builder.build]
+      end
+    end
+
+    # @rbs (Array[String] tokens, kind: Symbol, sentence: Integer) -> void
+    def compare!(tokens, kind:, sentence:)
+      outcomes = @algorithms.to_h do |algorithm|
+        automaton = @automata.fetch(algorithm)
+        simulator = TableSimulation::Simulator.new(
+          automaton, max_steps: @max_actions, max_stack: @max_stack
+        )
+        [algorithm, simulator.simulate(tokens).status]
+      rescue Ibex::Error => e
+        if budget_error?(e)
+          raise BudgetExceeded.new(
+            message: e.message, tokens: tokens,
+            bounds: { max_actions: @max_actions, max_stack: @max_stack }
+          )
+        end
+        [algorithm, [:failure, e.class.name, e.message]]
+      end
+      outcomes[:external] = @against.call(tokens) if @against
+      values = outcomes.values.uniq
+      return if values.length == 1 && (kind != :generated || values.first == :accepted)
+
+      raise Mismatch.new(tokens: tokens, kind: kind, sentence: sentence, outcomes: outcomes,
+                         bounds: { max_actions: @max_actions, max_stack: @max_stack })
+    end
+
+    # @rbs (Array[String] tokens) -> Array[Array[String]]
+    def mutations(tokens)
+      terminals = @grammar.terminals.reject(&:reserved).map(&:name)
+      replacement = terminals.fetch(@random.rand(terminals.length))
+      index = tokens.empty? ? 0 : @random.rand(tokens.length)
+      inserted = tokens.dup.insert(index, replacement)
+      return [inserted] if tokens.empty?
+
+      deleted = tokens.dup.tap { |items| items.delete_at(index) }
+      replaced = tokens.dup.tap { |items| items[index] = replacement }
+      [inserted, deleted, replaced].uniq
+    end
+
+    # @rbs (Array[String] tokens, kind: Symbol, sentence: Integer,
+    #   outcomes: Hash[Symbol, fuzz_outcome]) -> bool
+    def mismatch_reproduced?(tokens, kind:, sentence:, outcomes:)
+      compare!(tokens, kind: kind, sentence: sentence)
+      false
+    rescue Mismatch => e
+      e.details[:kind].to_sym == kind &&
+        e.details[:outcomes] == outcomes
+    end
+
+    # @rbs (Ibex::Error error) -> bool
+    def budget_error?(error)
+      error.message.match?(
+        /simulation exceeded|(?:limit|maximum|budget).*(?:exceed|exhaust)|exceed.*(?:limit|maximum|budget)/i
+      )
+    end
+  end
+end

data/lib/ibex/generation_input.rb

@@ -31,7 +31,7 @@ module Ibex
       @access_paths << expanded unless @access_paths.include?(expanded)
     end
 
-    # @rbs () -> Hash[String, untyped]
+    # @rbs () -> Hash[String, String | Integer]
     def to_h
       { "path" => @path, "sha256" => @sha256, "bytesize" => @bytesize }
     end