ibex 0.1.0 → 0.3.0

data/lib/ibex/frontend/parser/declarations.rb

@@ -3,9 +3,14 @@
 module Ibex
   module Frontend
     # Parses the declaration section of a grammar.
+    # rubocop:disable Metrics/ModuleLength, Metrics/CyclomaticComplexity
+    # Bootstrap declarations intentionally mirror the generated frontend's complete declaration vocabulary.
     module BootstrapParserDeclarations
-      DECLARATIONS = %w[token prechigh preclow options expect start convert pragma rule].freeze #: Array[String]
-      ASSOCIATIVITIES = %w[left right nonassoc].freeze #: Array[String]
+      DECLARATIONS = %w[
+        include import token prechigh preclow options expect expect_rr start recover on_error_reduce
+        test lexer convert display type param printer parser pragma rule
+      ].freeze #: Array[String]
+      ASSOCIATIVITIES = %w[left right nonassoc precedence].freeze #: Array[String]
 
       private
 
@@ -20,18 +25,16 @@ module Ibex
       # @rbs () -> void
       def parse_pragmas
         # @type self: BootstrapParser
-        seen = false
         while keyword?("pragma")
           keyword = current
-          fail_at(keyword.location, "duplicate pragma extended") if seen
-
           advance
           value = expect(:identifier)
           name = token_string(value)
-          fail_at(value.location, "unknown pragma #{name}") unless name == "extended"
+          fail_at(value.location, "unknown pragma #{name}") unless %w[extended cst].include?(name)
+          fail_at(keyword.location, "duplicate pragma #{name}") if @pragmas[name]
 
+          @pragmas[name] = keyword.location
           @mode = :extended
-          seen = true
         end
       end
 
@@ -39,24 +42,77 @@ module Ibex
       def parse_declaration
         # @type self: BootstrapParser
         case current.value
+        when "include", "import" then parse_import
         when "token" then parse_tokens
         when "prechigh", "preclow" then parse_precedence
         when "options" then parse_options
         when "expect" then parse_expect
+        when "expect_rr" then parse_expect_rr
         when "start" then parse_start
+        when "recover" then parse_recovery
+        when "on_error_reduce" then parse_on_error_reduce
+        when "test" then parse_grammar_test
+        when "lexer" then parse_lexer
         when "convert" then parse_convert
+        when "display" then parse_symbol_metadata(AST::DisplayName, "display")
+        when "type" then parse_symbol_metadata(AST::SemanticType, "type")
+        when "param" then parse_parameter
+        when "printer" then parse_printer
+        when "parser" then parse_parser_configuration
         when "pragma" then fail_at(current.location, "expected rule, got pragma")
         else fail_expected("a declaration or rule")
         end
       end
 
+      # @rbs () -> AST::ParserConfiguration
+      def parse_parser_configuration
+        # @type self: BootstrapParser
+        keyword = advance
+        settings = [] #: Array[AST::ParserSetting]
+        until keyword?("end") || current.type == :eof
+          key = expect(:identifier)
+          value = expect(:identifier)
+          settings << build_parser_setting(key, value)
+        end
+        expect_keyword("end")
+        build_parser_configuration(keyword, settings)
+      end
+
+      # @rbs () -> AST::Include
+      def parse_import
+        # @type self: BootstrapParser
+        keyword = advance
+        extended_only!(keyword.location, "#{keyword.value}s")
+        path = expect(:literal)
+        value = token_string(path)
+        unless value.start_with?('"') && value.end_with?('"')
+          fail_at(path.location, "#{keyword.value} path must use a double-quoted string")
+        end
+
+        decoded = begin
+          value.undump
+        rescue RuntimeError => e
+          fail_at(path.location, "invalid #{keyword.value} path: #{e.message}")
+        end
+        AST::Include.new(path: decoded, loc: keyword.location)
+      end
+
       # @rbs () -> AST::Tokens
       def parse_tokens
         # @type self: BootstrapParser
         location = advance.location
         names = [] #: Array[String]
-        names << parse_symbol_name until declaration_start?
-        AST::Tokens.new(names: names, loc: location)
+        aliases = {} #: Hash[String, String]
+        until declaration_start?
+          name_token = expect_symbol
+          name = token_string(name_token)
+          names << name
+          next unless @mode == :extended && name_token.type == :identifier &&
+                      current.type == :literal && current.location.line == name_token.location.line
+
+          aliases[name] = decode_quoted_literal(advance, "token alias")
+        end
+        AST::Tokens.new(names: names, aliases: aliases.empty? ? nil : aliases, loc: location)
       end
 
       # @rbs () -> AST::Precedence
@@ -77,6 +133,7 @@ module Ibex
         levels = [] #: Array[AST::PrecedenceLevel]
         until keyword?(closing) || current.type == :eof
           association = expect_one_of(ASSOCIATIVITIES)
+          extended_only!(association.location, "%precedence") if token_string(association) == "precedence"
           symbols = [] #: Array[String]
           symbols << parse_symbol_name until association_start? || keyword?(closing) || current.type == :eof
           fail_at(association.location, "expected at least one precedence symbol") if symbols.empty?
@@ -102,11 +159,152 @@ module Ibex
         AST::Expect.new(conflicts: token_integer(expect(:integer)), loc: location)
       end
 
+      # @rbs () -> AST::ExpectRR
+      def parse_expect_rr
+        # @type self: BootstrapParser
+        location = advance.location
+        extended_only!(location, "expect-rr declarations")
+        AST::ExpectRR.new(conflicts: token_integer(expect(:integer)), loc: location)
+      end
+
+      # @rbs () -> AST::Parameter
+      def parse_parameter
+        # @type self: BootstrapParser
+        keyword = advance
+        extended_only!(keyword.location, "%param")
+        name = expect(:identifier)
+        type = current.type == :literal ? advance : nil
+        if type && (keyword.location.line != name.location.line || name.location.line != type.location.line)
+          fail_at(keyword.location, "%param declaration must be written on one line")
+        end
+
+        semantic_type = type && decode_quoted_literal(type, "%param")
+        AST::Parameter.new(name: token_string(name), semantic_type: semantic_type, loc: keyword.location)
+      end
+
+      # @rbs () -> AST::Printer
+      def parse_printer
+        # @type self: BootstrapParser
+        keyword = advance
+        extended_only!(keyword.location, "%printer")
+        name = parse_symbol_name
+        action = expect(:action)
+        AST::Printer.new(name: name, code: token_string(action), loc: keyword.location)
+      end
+
       # @rbs () -> AST::Start
       def parse_start
         # @type self: BootstrapParser
         location = advance.location
-        AST::Start.new(name: parse_symbol_name, loc: location)
+        names = [parse_symbol_name]
+        names << parse_symbol_name until declaration_start?
+        extended_only!(location, "multiple start symbols") if names.length > 1
+        AST::Start.new(names: names, loc: location)
+      end
+
+      # @rbs () -> AST::Recovery
+      def parse_recovery
+        # @type self: BootstrapParser
+        keyword = advance
+        extended_only!(keyword.location, "%recover")
+        kind = expect(:identifier)
+        fail_at(kind.location, "expected sync, got #{kind.value}") unless token_string(kind) == "sync"
+        expect(:":")
+        tokens = [parse_symbol_name]
+        tokens << parse_symbol_name until declaration_start?
+        AST::Recovery.new(sync_tokens: tokens, loc: keyword.location)
+      end
+
+      # @rbs () -> AST::OnErrorReduce
+      def parse_on_error_reduce
+        # @type self: BootstrapParser
+        keyword = advance
+        extended_only!(keyword.location, "%on_error_reduce")
+        names = [parse_symbol_name]
+        names << parse_symbol_name until declaration_start?
+        AST::OnErrorReduce.new(names: names, loc: keyword.location)
+      end
+
+      # @rbs () -> AST::GrammarTest
+      def parse_grammar_test
+        # @type self: BootstrapParser
+        keyword = advance
+        extended_only!(keyword.location, "%test")
+        expectation = token_string(expect(:identifier))
+        fail_at(keyword.location, "expected accept or reject, got #{expectation}") unless
+          %w[accept reject].include?(expectation)
+        source = expect(:literal)
+        literal = token_string(source)
+        fail_at(source.location, "%test source must use a double-quoted string") unless literal.start_with?('"')
+        decoded = begin
+          literal.undump
+        rescue RuntimeError => e
+          fail_at(source.location, "invalid %test source: #{e.message}")
+        end
+
+        AST::GrammarTest.new(expectation: expectation.to_sym, source: decoded, loc: keyword.location)
+      end
+
+      # @rbs () -> AST::Lexer
+      def parse_lexer
+        # @type self: BootstrapParser
+        keyword = advance
+        extended_only!(keyword.location, "lexer declarations")
+        entries = parse_lexer_entries
+        expect_keyword("end")
+        AST::Lexer.new(definitions: entries, loc: keyword.location)
+      end
+
+      # @rbs () -> Array[AST::lexer_entry]
+      def parse_lexer_entries
+        # @type self: BootstrapParser
+        entries = [] #: Array[AST::lexer_entry]
+        entries << parse_lexer_entry until keyword?("end") || current.type == :eof
+        entries
+      end
+
+      # @rbs () -> AST::lexer_entry
+      def parse_lexer_entry
+        # @type self: BootstrapParser
+        return parse_lexer_state if keyword?("state")
+
+        marker = expect(:identifier)
+        kind = case token_string(marker)
+               when "skip" then :skip
+               when "on" then :on
+               else :token
+               end
+        pattern = expect_lexer_pattern
+        action = current.type == :action ? advance : nil
+        fail_at(marker.location, "on lexer rules require an action") if kind == :on && !action
+        build_bootstrap_lexer_rule(kind, marker, pattern, action)
+      end
+
+      # @rbs () -> AST::LexerState
+      def parse_lexer_state
+        # @type self: BootstrapParser
+        marker = advance
+        name = expect(:identifier)
+        expect_keyword("do")
+        entries = parse_lexer_entries
+        expect_keyword("end")
+        AST::LexerState.new(name: token_string(name), definitions: entries, loc: marker.location)
+      end
+
+      # @rbs () -> Token
+      def expect_lexer_pattern
+        # @type self: BootstrapParser
+        return advance if %i[regexp literal].include?(current.type)
+
+        fail_expected("a regular expression or quoted literal")
+      end
+
+      # @rbs (Symbol kind, Token marker, Token pattern, Token? action) -> AST::LexerRule
+      def build_bootstrap_lexer_rule(kind, marker, pattern, action)
+        # @type self: BootstrapParser
+        token = kind == :token ? token_string(marker) : nil
+        AST::LexerRule.new(kind: kind, token: token, pattern: token_string(pattern),
+                           pattern_kind: pattern.type, action: action && token_string(action), loc: marker.location)
       end
 
       # @rbs () -> AST::Convert
@@ -124,6 +322,27 @@ module Ibex
         AST::Convert.new(pairs: pairs, loc: location)
       end
 
+      # @rbs (singleton(AST::DisplayName) | singleton(AST::SemanticType) node_class, String feature) ->
+      #   (AST::DisplayName | AST::SemanticType)
+      def parse_symbol_metadata(node_class, feature)
+        # @type self: BootstrapParser
+        keyword = advance
+        extended_only!(keyword.location, "#{feature} declarations")
+        name = expect_symbol
+        value = expect_metadata_value
+        validate_metadata_line(keyword, name, value, feature)
+        decoded = decode_quoted_literal(value, feature)
+        node_class.new(name: token_string(name), value: decoded, loc: keyword.location)
+      end
+
+      # @rbs () -> Token
+      def expect_metadata_value
+        # @type self: BootstrapParser
+        return advance if current.type == :literal
+
+        fail_expected("a quoted string")
+      end
+
       # @rbs (Integer line) -> Array[Token]
       def tokens_on_line(line)
         # @type self: BootstrapParser
@@ -147,10 +366,41 @@ module Ibex
         fail_at(location, "invalid conversion expression: #{e.message}")
       end
 
+      # @rbs (Token keyword, Token name, Token value, String feature) -> void
+      def validate_metadata_line(keyword, name, value, feature)
+        # @type self: BootstrapParser
+        return if keyword.location.line == name.location.line && name.location.line == value.location.line
+
+        fail_at(keyword.location, "#{feature} declaration must be written on one line")
+      end
+
+      # @rbs (Token token, String feature) -> String
+      def decode_quoted_literal(token, feature)
+        # @type self: BootstrapParser
+        literal = token_string(token)
+        decoded = if literal.start_with?('"')
+                    literal.undump
+                  else
+                    (literal[1...-1] || "").gsub("\\'", "'").gsub("\\\\", "\\")
+                  end
+        fail_at(token.location, "#{feature} value must not be empty") if decoded.strip.empty?
+        fail_at(token.location, "#{feature} value must be a single line") if decoded.match?(/[\r\n]/)
+        fail_at(token.location, "#{feature} value must not contain control characters") if
+          decoded.match?(/[[:cntrl:]]/)
+
+        decoded
+      rescue RuntimeError => e
+        fail_at(token.location, "invalid #{feature} value: #{e.message}")
+      end
+
       # @rbs () -> bool
       def declaration_start?
         # @type self: BootstrapParser
-        current.type == :eof || (current.type == :identifier && DECLARATIONS.include?(current.value))
+        return true if current.type == :eof
+        return false unless current.type == :identifier && DECLARATIONS.include?(current.value)
+        return false if %w[display type param printer lexer].include?(current.value) && @mode != :extended
+
+        true
       end
 
       # @rbs () -> bool
@@ -159,5 +409,6 @@ module Ibex
         current.type == :identifier && ASSOCIATIVITIES.include?(current.value)
       end
     end
+    # rubocop:enable Metrics/ModuleLength, Metrics/CyclomaticComplexity
   end
 end

data/lib/ibex/frontend/parser/parameters.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+module Ibex
+  module Frontend
+    # Parses extended-mode parameter lists and parameterized references during bootstrap.
+    module BootstrapParserParameters
+      private
+
+      # @rbs (Token lhs) -> Array[String]
+      def parse_rule_parameters(lhs)
+        # @type self: BootstrapParser
+        return [] unless current.type == :"(" && adjacent_tokens?(lhs, current)
+
+        opening = advance
+        extended_only!(opening.location, "parameterized rules")
+        parameters = [token_string(expect(:identifier))]
+        parameters << token_string(expect(:identifier)) while accept(:",")
+        expect(:")")
+        parameters
+      end
+
+      # @rbs () -> AST::item
+      def parse_parameterized_reference
+        # @type self: BootstrapParser
+        callee = advance
+        opening = expect(:"(")
+        extended_only!(opening.location, "parameterized rules")
+        arguments = [] #: Array[AST::item]
+        unless current.type == :")"
+          loop do
+            fail_at(current.location, "actions are not valid parameter arguments") if current.type == :action
+
+            arguments << parse_item
+            break unless accept(:",")
+          end
+        end
+        expect(:")")
+        item = AST::ParameterizedReference.new(
+          name: token_string(callee), arguments: arguments, named_reference: parse_named_reference, loc: callee.location
+        )
+        parse_suffix(item)
+      end
+
+      # @rbs () -> bool
+      def parameterized_call?
+        # @type self: BootstrapParser
+        current.type == :identifier && lookahead.type == :"(" && adjacent_tokens?(current, lookahead)
+      end
+
+      # @rbs () -> bool
+      def parameterized_rule_start?
+        # @type self: BootstrapParser
+        opening = lookahead
+        return false unless opening.type == :"(" && adjacent_tokens?(current, opening)
+
+        parameter_definition_tail?(@tokens.drop(@index + 2))
+      end
+
+      # @rbs (Array[Token] tail) -> bool
+      def parameter_definition_tail?(tail)
+        # @type self: BootstrapParser
+        closing = tail.index { |token| token.type == :")" }
+        return false unless closing && tail[closing + 1]&.type == :":"
+
+        formals = tail.take(closing)
+        !formals.empty? && formals.length.odd? &&
+          formals.each_with_index.all? { |token, index| token.type == (index.even? ? :identifier : :",") }
+      end
+
+      # @rbs (Token left, Token right) -> bool
+      def adjacent_tokens?(left, right)
+        # @type self: BootstrapParser
+        left_span = left.span
+        right_span = right.span
+        return left_span.end_byte == right_span.start_byte if left_span && right_span
+
+        left.location.line == right.location.line &&
+          left.location.column + token_string(left).length == right.location.column
+      end
+    end
+  end
+end

data/lib/ibex/frontend/parser/rules.rb

@@ -14,7 +14,7 @@ module Ibex
       def parse_rules
         # @type self: BootstrapParser
         rules = [] #: Array[AST::Rule]
-        rules << parse_rule until keyword?("end") || current.type == :eof
+        rules << parse_rule until keyword?("end") || %i[eof user_code].include?(current.type)
         fail_expected("at least one rule") if rules.empty?
         rules
       end
@@ -22,7 +22,10 @@ module Ibex
       # @rbs () -> AST::Rule
       def parse_rule
         # @type self: BootstrapParser
+        inline = accept(:inline)
+        extended_only!(inline.location, "inline rules") if inline
         lhs = expect(:identifier)
+        parameters = parse_rule_parameters(lhs)
         expect(:":")
         alternatives = [] #: Array[AST::Alternative]
         loop do
@@ -32,7 +35,10 @@ module Ibex
           accept(:";")
           break
         end
-        AST::Rule.new(lhs: token_string(lhs), alternatives: alternatives, loc: lhs.location)
+        AST::Rule.new(
+          lhs: token_string(lhs), parameters: parameters, alternatives: alternatives, loc: lhs.location,
+          inline: !!inline
+        )
       end
 
       # @rbs (Token lhs) -> AST::Alternative
@@ -54,14 +60,42 @@ module Ibex
           items.pop
           action = last_item
         end
-        AST::Alternative.new(items: items, action: action, precedence: precedence, loc: location)
+        node_annotation = parse_node_annotation
+        AST::Alternative.new(
+          items: items, action: action, precedence: precedence, node_annotation: node_annotation, loc: location
+        )
+      end
+
+      # @rbs () -> AST::NodeAnnotation?
+      def parse_node_annotation
+        # @type self: BootstrapParser
+        marker = accept(:node)
+        return unless marker
+
+        extended_only!(marker.location, "@node")
+        name = expect(:identifier)
+        expect(:"(")
+        fields = [] #: Array[String]
+        unless current.type == :")"
+          fields << token_string(expect(:identifier))
+          fields << token_string(expect(:identifier)) while accept(:",")
+        end
+        expect(:")")
+        AST::NodeAnnotation.new(name: token_string(name), fields: fields, loc: marker.location)
       end
 
       # @rbs () -> AST::item
       def parse_item
         # @type self: BootstrapParser
         return parse_action if current.type == :action
+
+        if current.type == :empty
+          token = advance
+          extended_only!(token.location, "%empty")
+          return AST::Empty.new(loc: token.location)
+        end
         return parse_separated_list if separated_list?
+        return parse_parameterized_reference if parameterized_call?
         return parse_group if current.type == :"("
 
         token = expect_symbol
@@ -105,7 +139,7 @@ module Ibex
         extended_only!(opening.location, "EBNF groups")
         alternatives = [[]] #: Array[Array[AST::item]]
         until current.type == :")"
-          fail_at(opening.location, "unterminated EBNF group") if current.type == :eof || keyword?("end")
+          fail_at(opening.location, "unterminated EBNF group") if current.type == :eof
           if accept(:|)
             alternatives << []
             next
@@ -135,13 +169,16 @@ module Ibex
       # @rbs (Token lhs) -> bool
       def alternative_end?(lhs)
         # @type self: BootstrapParser
-        %i[| ; eof].include?(current.type) || keyword?("end") || rule_start?(lhs)
+        %i[| ; inline node eof user_code].include?(current.type) || keyword?("end") || rule_start?(lhs)
       end
 
       # @rbs (Token lhs) -> bool
       def rule_start?(lhs)
         # @type self: BootstrapParser
-        current.type == :identifier && lookahead.type == :":" && current.location.column <= lhs.location.column
+        return false unless current.type == :identifier && current.location.column <= lhs.location.column
+        return true if lookahead.type == :":"
+
+        parameterized_rule_start?
       end
 
       # @rbs () -> bool