ibex 0.1.0 → 0.3.0

data/lib/ibex/frontend/ast.rb

@@ -2,23 +2,65 @@
 
 module Ibex
   module Frontend
+    # Keeps the public Rule constructor compatible with ASTs built before parameters and inline rules existed.
+    module ASTRuleDefaults
+      # @rbs (*untyped arguments, **untyped keywords) -> void
+      def initialize(*arguments, **keywords)
+        if arguments.empty?
+          keywords = keywords.merge(parameters: keywords[:parameters] || [], inline: keywords[:inline] || false)
+        elsif arguments.one? && arguments.first.is_a?(Hash)
+          attributes = arguments.first
+          arguments = [
+            attributes.merge(parameters: attributes[:parameters] || [], inline: attributes[:inline] || false)
+          ]
+        end
+        super(*arguments, **keywords) # rubocop:disable Style/SuperArguments
+      end
+    end
+
     # Grammar frontend node types.
+    # rubocop:disable Metrics/ModuleLength -- the public node vocabulary is intentionally kept together.
     module AST
       # @rbs!
-      #   type declaration = Tokens | Precedence | Options | Expect | Start | Convert
-      #   type item = SymbolReference | InlineAction | Optional | Star | Plus | Group | SeparatedList
+      #   type symbol_metadata = DisplayName | SemanticType
+      #   type declaration = Include | Tokens | Precedence | Options | Expect | ExpectRR | Start | Recovery |
+      #     OnErrorReduce | GrammarTest | Lexer | Convert | Parameter | Printer | ParserConfiguration | symbol_metadata
+      #   type lexer_entry = LexerRule | LexerState
+      #   type item = SymbolReference | ParameterizedReference | InlineAction | Optional | Star | Plus | Group |
+      #     SeparatedList | Empty
       #   type user_code = Hash[String, Array[UserCode]]
+      #   class Rule < Struct[String | Array[String] | Array[Alternative] | Location | String? | bool]
+      #     attr_accessor lhs: String
+      #     attr_accessor parameters: Array[String]
+      #     attr_accessor alternatives: Array[Alternative]
+      #     attr_accessor loc: Location
+      #     attr_accessor documentation: String?
+      #     attr_accessor inline: bool
+      #     def self.new: (?lhs: String, ?parameters: Array[String]?, ?alternatives: Array[Alternative],
+      #       ?loc: Location, ?documentation: String?, ?inline: bool?) -> instance
+      #       | ({ ?lhs: String, ?parameters: Array[String]?, ?alternatives: Array[Alternative],
+      #         ?loc: Location, ?documentation: String?, ?inline: bool? }) -> instance
+      #   end
 
       # Adds deterministic, recursively serializable hashes to Struct nodes.
-      # @rbs module-self Struct[untyped]
+      # @rbs module-self Struct[Object?]
       module Node
+        # @rbs () -> Hash[Symbol, untyped]
         def to_h
           fields = each_pair.to_h { |name, value| [name, serialize(value)] }
+          fields.delete(:aliases) if self.class.name.end_with?("::Tokens") && !fields[:aliases]
+          if self.class.name.end_with?("::Root")
+            fields.delete(:extended) unless fields[:extended]
+            fields.delete(:extended_loc) unless fields[:extended_loc]
+            fields.delete(:cst) unless fields[:cst]
+          end
+          fields.delete(:node_annotation) if self.class.name.end_with?("::Alternative") && !fields[:node_annotation]
           { node: self.class.name.split("::").last }.merge(fields)
         end
 
         private
 
+        # @rbs (untyped value) -> untyped
         def serialize(value)
           return value if value.nil?
 
@@ -37,12 +79,25 @@ module Ibex
         :rules, #: Array[Rule]
         :user_code, #: user_code
         :loc, #: Location
+        :extended, #: bool?
+        :cst, #: bool?
+        :extended_loc, #: Location?
         keyword_init: true
-      ) do
-        include Node
-      end
+      ) { include Node }
+      Fragment = Struct.new(
+        :declarations, #: Array[declaration]
+        :rules, #: Array[Rule]
+        :loc, #: Location
+        keyword_init: true
+      ) { include Node }
+      Include = Struct.new(
+        :path, #: String
+        :loc, #: Location
+        keyword_init: true
+      ) { include Node }
       Tokens = Struct.new(
         :names, #: Array[String]
+        :aliases, #: Hash[String, String]?
         :loc, #: Location
         keyword_init: true
       ) { include Node }
@@ -68,8 +123,49 @@ module Ibex
         :loc, #: Location
         keyword_init: true
       ) { include Node }
+      ExpectRR = Struct.new(
+        :conflicts, #: Integer
+        :loc, #: Location
+        keyword_init: true
+      ) { include Node }
       Start = Struct.new(
+        :names, #: Array[String]
+        :loc, #: Location
+        keyword_init: true
+      ) { include Node }
+      Recovery = Struct.new(
+        :sync_tokens, #: Array[String]
+        :loc, #: Location
+        keyword_init: true
+      ) { include Node }
+      OnErrorReduce = Struct.new(
+        :names, #: Array[String]
+        :loc, #: Location
+        keyword_init: true
+      ) { include Node }
+      GrammarTest = Struct.new(
+        :expectation, #: Symbol
+        :source, #: String
+        :loc, #: Location
+        keyword_init: true
+      ) { include Node }
+      Lexer = Struct.new(
+        :definitions, #: Array[lexer_entry]
+        :loc, #: Location
+        keyword_init: true
+      ) { include Node }
+      LexerRule = Struct.new(
+        :kind, #: Symbol
+        :token, #: String?
+        :pattern, #: String
+        :pattern_kind, #: Symbol
+        :action, #: String?
+        :loc, #: Location
+        keyword_init: true
+      ) { include Node }
+      LexerState = Struct.new(
         :name, #: String
+        :definitions, #: Array[lexer_entry]
         :loc, #: Location
         keyword_init: true
       ) { include Node }
@@ -84,16 +180,67 @@ module Ibex
         :loc, #: Location
         keyword_init: true
       ) { include Node }
+      DisplayName = Struct.new(
+        :name, #: String
+        :value, #: String
+        :loc, #: Location
+        keyword_init: true
+      ) { include Node }
+      SemanticType = Struct.new(
+        :name, #: String
+        :value, #: String
+        :loc, #: Location
+        keyword_init: true
+      ) { include Node }
+      Parameter = Struct.new(
+        :name, #: String
+        :semantic_type, #: String?
+        :loc, #: Location
+        keyword_init: true
+      ) { include Node }
+      Printer = Struct.new(
+        :name, #: String
+        :code, #: String
+        :loc, #: Location
+        keyword_init: true
+      ) { include Node }
+      ParserConfiguration = Struct.new(
+        :settings, #: Array[ParserSetting]
+        :loc, #: Location
+        keyword_init: true
+      ) { include Node }
+      ParserSetting = Struct.new(
+        :key, #: Symbol
+        :value, #: Symbol
+        :loc, #: Location
+        keyword_init: true
+      ) { include Node }
+      ROOT_ONLY_DECLARATION_NAMES = {
+        Options => "options", Expect => "expect", ExpectRR => "%expect-rr", Parameter => "%param",
+        Start => "start", Recovery => "%recover", OnErrorReduce => "%on_error_reduce", GrammarTest => "%test",
+        Lexer => "lexer", ParserConfiguration => "parser"
+      }.freeze #: Hash[Class, String]
+      # @rbs skip
       Rule = Struct.new(
         :lhs, #: String
+        :parameters, #: Array[String]
         :alternatives, #: Array[Alternative]
         :loc, #: Location
+        :documentation, #: String?
+        :inline, #: bool
         keyword_init: true
-      ) { include Node }
+      ) { include Node, ASTRuleDefaults }
       Alternative = Struct.new(
         :items, #: Array[item]
         :action, #: InlineAction?
         :precedence, #: String?
+        :node_annotation, #: NodeAnnotation?
+        :loc, #: Location
+        keyword_init: true
+      ) { include Node }
+      NodeAnnotation = Struct.new(
+        :name, #: String
+        :fields, #: Array[String]
         :loc, #: Location
         keyword_init: true
       ) { include Node }
@@ -103,11 +250,22 @@ module Ibex
         :loc, #: Location
         keyword_init: true
       ) { include Node }
+      ParameterizedReference = Struct.new(
+        :name, #: String
+        :arguments, #: Array[item]
+        :named_reference, #: String?
+        :loc, #: Location
+        keyword_init: true
+      ) { include Node }
       InlineAction = Struct.new(
         :code, #: String
         :loc, #: Location
         keyword_init: true
       ) { include Node }
+      Empty = Struct.new(
+        :loc, #: Location
+        keyword_init: true
+      ) { include Node }
       Optional = Struct.new(
         :item, #: item
         :loc, #: Location
@@ -142,5 +300,6 @@ module Ibex
         keyword_init: true
       ) { include Node }
     end
+    # rubocop:enable Metrics/ModuleLength
   end
 end

data/lib/ibex/frontend/bootstrap_parser.rb

@@ -6,18 +6,21 @@ module Ibex
     class BootstrapParser
       include BootstrapParserDeclarations
       include BootstrapParserRules
+      include BootstrapParserParameters
+      include ParserConfigurationSupport
 
       # @rbs @tokens: Array[Token]
       # @rbs @index: Integer
       # @rbs @mode: Symbol
 
       # @rbs (String | Array[Token] source, ?file: String, ?mode: Symbol) -> void
-      def initialize(source, file: "(grammar)", mode: :racc)
-        raise ArgumentError, "mode must be :racc or :extended" unless %i[racc extended].include?(mode)
+      def initialize(source, file: "(grammar)", mode: :default)
+        raise ArgumentError, "mode must be :default or :extended" unless %i[default extended].include?(mode)
 
         @tokens = source.is_a?(Array) ? source : Lexer.new(source, file: file).tokenize
         @index = 0
         @mode = mode
+        @pragmas = {} #: Hash[String, Location]
       end
 
       # @rbs () -> AST::Root
@@ -27,17 +30,50 @@ module Ibex
         superclass = accept(:<) ? parse_constant_path : nil
         parse_pragmas
         declarations = parse_declarations
+        validate_root_parser_configuration(declarations)
         expect_keyword("rule")
         rules = parse_rules
-        expect_keyword("end")
+        expect_keyword("end") unless current.type == :user_code
         user_code = parse_user_code
         expect(:eof)
         AST::Root.new(class_name: class_name, superclass: superclass, declarations: declarations,
-                      rules: rules, user_code: user_code, loc: location)
+                      rules: rules, user_code: user_code, loc: location,
+                      extended: @mode == :extended || @pragmas.key?("extended") || @pragmas.key?("cst"),
+                      cst: @pragmas.key?("cst"),
+                      extended_loc: @pragmas["extended"] || @pragmas["cst"])
+      end
+
+      # @rbs () -> AST::Fragment
+      def parse_fragment
+        location = expect_keyword("fragment").location
+        extended_only!(location, "fragments")
+        declarations = parse_declarations
+        reject_fragment_root_declarations(declarations)
+        expect_keyword("rule")
+        rules = keyword?("end") ? Array.new(0) : parse_rules #: Array[AST::Rule]
+        expect_keyword("end")
+        user_code = parse_user_code
+        fail_at(user_code.values.flatten.first.loc, "user code is not allowed in fragments") unless user_code.empty?
+        expect(:eof)
+        AST::Fragment.new(declarations: declarations, rules: rules, loc: location)
       end
 
       private
 
+      # @rbs (Array[AST::declaration] declarations) -> void
+      def reject_fragment_root_declarations(declarations)
+        root_only = declarations.find { |declaration| fragment_root_declaration_name(declaration) }
+        return unless root_only
+
+        name = fragment_root_declaration_name(root_only)
+        fail_at(root_only.loc, "#{name} declarations are not allowed in fragments")
+      end
+
+      # @rbs (AST::declaration declaration) -> String?
+      def fragment_root_declaration_name(declaration)
+        AST::ROOT_ONLY_DECLARATION_NAMES.find { |node_class, _name| declaration.is_a?(node_class) }&.last
+      end
+
       # @rbs () -> String
       def parse_constant_path
         parts = [token_string(expect(:identifier))]

data/lib/ibex/frontend/diagnostic.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+module Ibex
+  module Frontend
+    # An immutable machine-readable frontend error.
+    class Diagnostic
+      attr_reader :code #: String
+      attr_reader :phase #: Symbol
+      attr_reader :message #: String
+      attr_reader :location #: Location
+      attr_reader :span #: SourceSpan?
+      attr_reader :expected #: Array[String]
+      attr_reader :received #: String?
+
+      # @rbs (code: String, phase: Symbol, message: String, location: Location, ?span: SourceSpan?,
+      #   ?expected: Array[String], ?received: String?, ?rendered: String?) -> void
+      def initialize(code:, phase:, message:, location:, span: nil, expected: [], received: nil, rendered: nil)
+        @code = code.dup.freeze
+        @phase = phase
+        @message = message.dup.freeze
+        @location = Location.new(
+          file: location.file.dup.freeze, line: location.line, column: location.column
+        ).freeze
+        @span = span
+        @expected = expected.map { |value| value.dup.freeze }.freeze
+        @received = received&.dup&.freeze
+        @rendered = (rendered || "#{location}: #{message}").dup.freeze
+        freeze
+      end
+
+      # @rbs () -> String
+      def severity
+        "error"
+      end
+
+      # @rbs () -> String
+      def to_s
+        @rendered
+      end
+
+      # @rbs () -> Hash[Symbol, Object?]
+      def to_h
+        {
+          code: code,
+          severity: severity,
+          phase: phase.to_s,
+          message: message,
+          location: location.to_h,
+          span: span&.to_h,
+          expected: expected,
+          received: received
+        }
+      end
+    end
+
+    # Immutable output from a diagnostic parse.
+    class ParseResult
+      attr_reader :diagnostics #: Array[Diagnostic]
+      attr_reader :ast #: AST::Root?
+      attr_reader :document #: SourceDocument?
+
+      # @rbs (diagnostics: Array[Diagnostic], ast: AST::Root?, document: SourceDocument?) -> void
+      def initialize(diagnostics:, ast:, document:)
+        @diagnostics = diagnostics.dup.freeze
+        @ast = ast
+        @document = document
+        freeze
+      end
+
+      # @rbs () -> bool
+      def success?
+        diagnostics.empty? && !ast.nil?
+      end
+
+      # @rbs () -> bool
+      def partial?
+        diagnostics.any? && !ast.nil?
+      end
+    end
+  end
+end

data/lib/ibex/frontend/diagnostic_recovery.rb

@@ -0,0 +1,268 @@
+# frozen_string_literal: true
+
+module Ibex
+  module Frontend
+    # Re-runs the generated parser after suppressing only conservative source regions.
+    class DiagnosticRecovery
+      DECLARATION_STARTS = %w[
+        pragma include import token prechigh preclow options expect expect_rr start recover on_error_reduce test lexer
+        convert display type param printer
+      ].freeze #: Array[String]
+
+      # @rbs @tokens: Array[Token]
+      # @rbs @mode: Symbol
+      # @rbs @max_diagnostics: Integer
+      # @rbs @suppressed: Hash[Integer, bool]
+
+      # @rbs (Array[Token] tokens, mode: Symbol, max_diagnostics: Integer) -> void
+      def initialize(tokens, mode:, max_diagnostics:)
+        @tokens = tokens
+        @mode = mode
+        @max_diagnostics = max_diagnostics
+        @diagnostics = [] #: Array[Diagnostic]
+        @suppressed = {} #: Hash[Integer, bool]
+      end
+
+      # @rbs () -> [AST::Root | AST::Fragment | nil, Array[Diagnostic]]
+      def parse
+        ast = parse_until_stable
+        [ast, sorted_diagnostics]
+      end
+
+      private
+
+      # @rbs () -> (AST::Root | AST::Fragment | nil)
+      def parse_until_stable
+        attempts = 0
+        loop do
+          parser = GeneratedParser.new(working_tokens, mode: @mode)
+          begin
+            return parser.parse
+          rescue Ibex::Error => e
+            return if @diagnostics.length >= @max_diagnostics
+
+            diagnostic = diagnostic_from(e, parser.diagnostic_token, phase: :syntax)
+            append_diagnostic(diagnostic)
+            range = suppression_range(parser.diagnostic_token, e)
+            return unless range && suppress(range)
+
+            attempts += 1
+            return if attempts >= @tokens.length
+          end
+        end
+      end
+
+      # @rbs () -> Array[Token]
+      def working_tokens
+        @tokens.each_with_index.filter_map { |token, index| token unless @suppressed[index] }
+      end
+
+      # @rbs (Diagnostic diagnostic) -> void
+      def append_diagnostic(diagnostic)
+        key = diagnostic_key(diagnostic)
+        return if @diagnostics.any? { |existing| diagnostic_key(existing) == key }
+
+        @diagnostics << diagnostic
+      end
+
+      # @rbs (Diagnostic diagnostic) -> [String, String, Integer, Integer, String]
+      def diagnostic_key(diagnostic)
+        location = diagnostic.location
+        [diagnostic.phase.to_s, location.file, location.line, location.column, diagnostic.message]
+      end
+
+      # @rbs () -> Array[Diagnostic]
+      def sorted_diagnostics
+        @diagnostics.uniq { |diagnostic| diagnostic_key(diagnostic) }
+                    .sort_by { |diagnostic| diagnostic_sort_key(diagnostic) }
+                    .first(@max_diagnostics)
+                    .freeze
+      end
+
+      # @rbs (Diagnostic diagnostic) -> [String, Integer, Integer, String, String]
+      def diagnostic_sort_key(diagnostic)
+        location = diagnostic.location
+        [location.file, location.line, location.column, diagnostic.phase.to_s, diagnostic.code]
+      end
+
+      # @rbs (Token? token, Exception error) -> Range[Integer]?
+      def suppression_range(token, error)
+        index = token_index(token) || token_index_for_message(error.message)
+        return unless index
+
+        rule_marker = rule_marker_index
+        return declaration_range(index, rule_marker) if rule_marker && index < rule_marker
+        return rule_range(index, rule_marker) if rule_marker
+
+        nil
+      end
+
+      # @rbs (Token? token) -> Integer?
+      def token_index(token)
+        return unless token
+
+        @tokens.index { |candidate| candidate.equal?(token) }
+      end
+
+      # @rbs (String message) -> Integer?
+      def token_index_for_message(message)
+        @tokens.index { |token| message.start_with?("#{token.location}:") }
+      end
+
+      # @rbs () -> Integer?
+      def rule_marker_index
+        @tokens.index { |token| token.type == :identifier && token.value == "rule" }
+      end
+
+      # @rbs (Integer failure, Integer rule_marker) -> Range[Integer]?
+      def declaration_range(failure, rule_marker)
+        starts = (0...rule_marker).select { |index| declaration_start?(index) }
+        start = starts.reverse.find { |index| index <= failure }
+        return unless start
+
+        finish = declaration_finish(start, rule_marker, starts)
+        start...finish
+      end
+
+      # @rbs (Integer index) -> bool
+      def declaration_start?(index)
+        token = @tokens[index]
+        token&.type == :identifier && DECLARATION_STARTS.include?(token.value)
+      end
+
+      # @rbs (Integer start, Integer rule_marker, Array[Integer] starts) -> Integer
+      def declaration_finish(start, rule_marker, starts)
+        value = @tokens.fetch(start).value
+        return matching_declaration_end(start, rule_marker, value == "prechigh" ? "preclow" : "prechigh") if
+          %w[prechigh preclow].include?(value)
+        return matching_declaration_end(start, rule_marker, "end") if %w[convert lexer].include?(value)
+
+        starts.find { |index| index > start } || rule_marker
+      end
+
+      # @rbs (Integer start, Integer fallback, String closer) -> Integer
+      def matching_declaration_end(start, fallback, closer)
+        index = ((start + 1)...fallback).find { |candidate| @tokens[candidate]&.value == closer }
+        index ? index + 1 : fallback
+      end
+
+      # @rbs (Integer failure, Integer rule_marker) -> Range[Integer]?
+      def rule_range(failure, rule_marker)
+        grammar_end = grammar_end_index(rule_marker)
+        definitions = rule_definition_starts(rule_marker, grammar_end)
+        start = definitions.reverse.find { |index| index <= failure }
+        return unless start
+
+        finish = definitions.find { |index| index > start } || grammar_end
+        alternative_range(failure, start, finish) || (start...finish)
+      end
+
+      # @rbs (Integer rule_marker) -> Integer
+      def grammar_end_index(rule_marker)
+        depth = 0
+        ((rule_marker + 1)...@tokens.length).each do |index|
+          token = @tokens.fetch(index)
+          depth += 1 if token.type == :"("
+          depth -= 1 if token.type == :")" && depth.positive?
+          return index if depth.zero? && token.type == :identifier && token.value == "end"
+        end
+        @tokens.length - 1
+      end
+
+      # @rbs (Integer rule_marker, Integer grammar_end) -> Array[Integer]
+      def rule_definition_starts(rule_marker, grammar_end)
+        starts = [] #: Array[Integer]
+        depth = 0
+        lhs_column = nil #: Integer?
+        ((rule_marker + 1)...grammar_end).each do |index|
+          token = @tokens.fetch(index)
+          depth += 1 if token.type == :"("
+          depth -= 1 if token.type == :")" && depth.positive?
+          next unless depth.zero? && lhs_candidate?(index, lhs_column)
+
+          lhs_column ||= token.location.column
+          starts << index
+        end
+        starts
+      end
+
+      # @rbs (Integer index, Integer? lhs_column) -> bool
+      def lhs_candidate?(index, lhs_column)
+        token = @tokens[index]
+        following = @tokens[index + 1]
+        return false unless token&.type == :identifier && following&.type == :":"
+
+        lhs_column.nil? || token.location.column <= lhs_column
+      end
+
+      # @rbs (Integer failure, Integer start, Integer finish) -> Range[Integer]?
+      def alternative_range(failure, start, finish)
+        colon = ((start + 1)...finish).find { |index| @tokens[index]&.type == :":" }
+        return unless colon
+
+        pipes = top_level_pipes(colon + 1, finish)
+        return if pipes.empty?
+
+        previous = pipes.reverse.find { |index| index < failure }
+        following = pipes.find { |index| index >= failure }
+        alternative_slice(previous, following, colon, finish)
+      end
+
+      # @rbs (Integer? previous, Integer? following, Integer colon, Integer finish) -> Range[Integer]?
+      def alternative_slice(previous, following, colon, finish)
+        if previous
+          following ? (previous...following) : (previous...finish)
+        elsif following
+          (colon + 1)..following
+        end
+      end
+
+      # @rbs (Integer start, Integer finish) -> Array[Integer]
+      def top_level_pipes(start, finish)
+        depth = 0
+        (start...finish).filter_map do |index|
+          token = @tokens.fetch(index)
+          depth += 1 if token.type == :"("
+          depth -= 1 if token.type == :")" && depth.positive?
+          index if depth.zero? && token.type == :|
+        end
+      end
+
+      # @rbs (Range[Integer] range) -> bool
+      def suppress(range)
+        changed = false
+        range.each do |index|
+          next if index.negative? || index >= @tokens.length || @tokens[index]&.type == :eof
+          next if @suppressed[index]
+
+          @suppressed[index] = true
+          changed = true
+        end
+        changed
+      end
+
+      # @rbs (Exception error, Token? token, phase: Symbol) -> Diagnostic
+      def diagnostic_from(error, token, phase:)
+        matching = @tokens.find { |candidate| error.message.start_with?("#{candidate.location}:") }
+        token = matching || token || @tokens.last
+        location = token&.location || Location.new(file: "(grammar)", line: 1, column: 1)
+        prefix = "#{location}: "
+        message = error.message.delete_prefix(prefix)
+        expected, received = expected_and_received(message)
+        Diagnostic.new(code: "frontend.#{phase}_error", phase: phase, message: message,
+                       location: location, span: token&.span, expected: expected,
+                       received: received, rendered: error.message)
+      end
+
+      # @rbs (String message) -> [Array[String], String?]
+      def expected_and_received(message)
+        match = message.match(/\Aexpected (.+), got (.+)\z/)
+        return [[], nil] unless match
+
+        expected = match[1]
+        received = match[2]
+        [expected ? [expected] : [], received]
+      end
+    end
+  end
+end