ibex 0.1.0 → 0.2.0

data/lib/ibex/frontend/token_adapter/declaration_lexer_state.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module Ibex
+  module Frontend
+    class TokenAdapter
+      # Context transitions for the extended root-only lexer declaration.
+      module DeclarationLexerState
+        private
+
+        # @rbs () -> bool
+        def lexer_declaration_state?
+          @declaration == :lexer
+        end
+
+        # @rbs (Token token) -> external_token
+        def classify_lexer_identifier(token)
+          # @type self: DeclarationState
+          case @state
+          when :lexer_entries, :lexer_action_or_entry then begin_lexer_entry(token)
+          when :lexer_state_name
+            @state = :lexer_state_do
+            :IDENTIFIER
+          when :lexer_state_do
+            value = string_value(token)
+            raise Ibex::Error, "#{token.location}: expected do, got #{value}" unless value == "do"
+
+            @lexer_state_depth = (@lexer_state_depth || 0) + 1
+            @state = :lexer_entries
+            :DO
+          else
+            :IDENTIFIER
+          end
+        end
+
+        # @rbs (Token token) -> external_token
+        def begin_lexer_entry(token)
+          # @type self: DeclarationState
+          value = string_value(token)
+          case value
+          when "end" then finish_lexer_scope
+          when "state"
+            @state = :lexer_state_name
+            :STATE
+          when "skip"
+            @state = :lexer_pattern
+            :SKIP
+          when "on"
+            @state = :lexer_pattern
+            :ON
+          else
+            @state = :lexer_pattern
+            :IDENTIFIER
+          end
+        end
+
+        # @rbs () -> external_token
+        def finish_lexer_scope
+          depth = @lexer_state_depth || 0
+          if depth.positive?
+            @lexer_state_depth = depth - 1
+            @state = :lexer_entries
+          else
+            @state = :declaration
+            @declaration = nil
+          end
+          :END
+        end
+
+        # @rbs (external_token type) -> external_token?
+        def classify_lexer_scalar(type)
+          if @state == :lexer_pattern && %i[REGEXP LITERAL].include?(type)
+            @state = :lexer_action_or_entry
+            return type
+          end
+          return unless @state == :lexer_action_or_entry && type == :ACTION
+
+          @state = :lexer_entries
+          type
+        end
+      end
+    end
+  end
+end

data/lib/ibex/frontend/token_adapter/declaration_state.rb

@@ -1,25 +1,55 @@
 # frozen_string_literal: true
 
+require_relative "declaration_lexer_state"
+
 module Ibex
   module Frontend
     class TokenAdapter
       # Classifies tokens through the class header and declaration section.
+      # rubocop:disable Metrics/ClassLength, Metrics/CyclomaticComplexity
+      # One state machine owns declaration-boundary classification.
       class DeclarationState
+        include DeclarationDocumentState
+        include DeclarationLexerState
+
         DECLARATIONS = {
+          "include" => %i[INCLUDE include_path],
+          "import" => %i[IMPORT include_path],
           "token" => %i[TOKEN token_symbols], "options" => %i[OPTIONS options_identifiers],
-          "expect" => %i[EXPECT expect_integer], "start" => %i[START start_symbol],
+          "expect" => %i[EXPECT expect_integer], "start" => %i[START start_first_symbol],
+          "expect_rr" => %i[EXPECT_RR expect_rr_integer],
+          "recover" => %i[RECOVER recover_kind],
+          "on_error_reduce" => %i[ON_ERROR_REDUCE on_error_reduce_first_symbol],
+          "test" => %i[TEST test_expectation],
+          "lexer" => %i[LEXER lexer_entries],
           "convert" => %i[CONVERT convert_name], "pragma" => %i[PRAGMA pragma_value],
+          "display" => %i[DISPLAY display_symbol], "type" => %i[TYPE type_symbol],
+          "param" => %i[PARAM param_name],
+          "printer" => %i[PRINTER printer_symbol],
           "rule" => %i[RULE rules]
         }.freeze #: Hash[String, [external_token, Symbol]]
         ASSOCIATIONS = {
-          "left" => :LEFT, "right" => :RIGHT, "nonassoc" => :NONASSOC
+          "left" => :LEFT, "right" => :RIGHT, "nonassoc" => :NONASSOC, "precedence" => :PRECEDENCE
         }.freeze #: Hash[String, external_token]
         SCALAR_TYPES = {
-          literal: :LITERAL, integer: :INTEGER, action: :ACTION, user_code: :USER_CODE
+          literal: :LITERAL, regexp: :REGEXP, integer: :INTEGER, action: :ACTION, user_code: :USER_CODE
         }.freeze #: Hash[Symbol, external_token]
         EXPECTATIONS = {
           class_keyword: "class", class_name: "identifier", superclass_name: "identifier",
-          expect_integer: "integer", start_symbol: "a grammar symbol"
+          expect_integer: "integer", expect_rr_integer: "integer",
+          start_first_symbol: "a grammar symbol", start_symbols: "a grammar symbol",
+          include_path: "a double-quoted relative path",
+          display_symbol: "a grammar symbol", type_symbol: "a grammar symbol",
+          display_value: "a quoted string", type_value: "a quoted string",
+          param_name: "an identifier", param_type: "a quoted type or declaration",
+          printer_symbol: "a grammar symbol", printer_action: "an action",
+          recover_kind: "sync", recovery_colon: ":", recovery_first_symbol: "a grammar symbol",
+          recovery_symbols: "a grammar symbol", on_error_reduce_first_symbol: "a grammar symbol",
+          on_error_reduce_symbols: "a grammar symbol", test_expectation: "accept or reject",
+          test_source: "a double-quoted string",
+          lexer_entries: "a lexer rule, state, or end", lexer_state_name: "a state name",
+          lexer_state_do: "do", lexer_pattern: "a regular expression or quoted literal",
+          lexer_action_or_entry: "an action, lexer rule, state, or end"
         }.freeze #: Hash[Symbol, String]
 
         attr_reader :conversion_name #: Token?
@@ -27,8 +57,15 @@ module Ibex
         attr_reader :precedence_closer #: String?
         attr_reader :state #: Symbol
 
-        # @rbs () -> void
-        def initialize
+        # @rbs @extended_mode: bool
+        # @rbs @fragment: bool?
+        # @rbs @pragmas: Hash[String, bool]
+        # @rbs @pragma_location: Location?
+
+        # @rbs (?extended: bool) -> void
+        def initialize(extended: false)
+          @extended_mode = extended
+          @pragmas = {} #: Hash[String, bool]
           @state = :class_keyword
         end
 
@@ -47,7 +84,12 @@ module Ibex
 
         # @rbs () -> bool
         def extended_pragma?
-          @extended_pragma == true
+          @pragmas["extended"] == true
+        end
+
+        # @rbs () -> bool
+        def cst_pragma?
+          @pragmas["cst"] == true
         end
 
         # @rbs (Token? token) -> String?
@@ -70,27 +112,32 @@ module Ibex
 
         # @rbs (Token token, Array[Token] remaining) -> external_token
         def classify_identifier(token, remaining)
+          return classify_lexer_identifier(token) if lexer_declaration_state?
+
           case @state
           when :class_keyword then class_keyword(token)
           when :class_name, :superclass_name then constant_name(remaining)
-          when :declaration then begin_declaration(token)
-          when :token_symbols, :options_identifiers then declaration_symbol(token)
+          when :declaration, :param_type then begin_declaration(token)
+          when :token_symbols
+            @token_alias_candidate = token
+            declaration_symbol(token)
+          when :options_identifiers, :start_symbols, :recovery_symbols, :on_error_reduce_symbols
+            declaration_symbol(token)
           when :precedence_association, :precedence_symbols then precedence_identifier(token)
-          when :start_symbol then finish_single_symbol(:IDENTIFIER)
+          when :start_first_symbol then continue_start_symbols(:IDENTIFIER)
+          when :display_symbol, :type_symbol then begin_metadata_value(:IDENTIFIER)
+          when :param_name then begin_param_type
+          when :printer_symbol then begin_printer_action(:IDENTIFIER)
+          when :recover_kind then begin_recovery_colon(token)
+          when :recovery_first_symbol then continue_recovery_symbols(:IDENTIFIER)
+          when :on_error_reduce_first_symbol then continue_on_error_reduce_symbols(:IDENTIFIER)
+          when :test_expectation then begin_test_source(token)
           when :pragma_value then finish_pragma(token)
           when :convert_name then begin_conversion(token, :IDENTIFIER, remaining)
           else :IDENTIFIER
           end
         end
 
-        # @rbs (Token token) -> external_token
-        def class_keyword(token)
-          return :IDENTIFIER unless string_value(token) == "class"
-
-          @state = :class_name
-          :CLASS
-        end
-
         # @rbs (Array[Token] remaining) -> external_token
         def constant_name(remaining)
           following = remaining.first
@@ -111,11 +158,13 @@ module Ibex
           value = string_value(token)
           return begin_precedence(token) if %w[prechigh preclow].include?(value)
 
-          raise Ibex::Error, "#{token.location}: duplicate pragma extended" if value == "pragma" && @extended_pragma
+          reject_fragment_pragma(token, value)
 
           terminal, next_state = DECLARATIONS[value]
           return :IDENTIFIER unless terminal
 
+          @pragma_location = token.location if terminal == :PRAGMA
+          @token_alias_candidate = nil
           @state = next_state
           @declaration = value.to_sym unless terminal == :RULE
           @declaration = nil if terminal == :RULE
@@ -125,9 +174,13 @@ module Ibex
         # @rbs (Token token) -> external_token
         def finish_pragma(token)
           value = string_value(token)
-          raise Ibex::Error, "#{token.location}: unknown pragma #{value}" unless value == "extended"
+          raise Ibex::Error, "#{token.location}: unknown pragma #{value}" unless %w[extended cst].include?(value)
 
-          @extended_pragma = true
+          location = @pragma_location || token.location
+          raise Ibex::Error, "#{location}: duplicate pragma #{value}" if @pragmas[value]
+
+          @pragmas[value] = true
+          @pragma_location = nil
           @state = :declaration
           @declaration = nil
           :IDENTIFIER
@@ -151,9 +204,16 @@ module Ibex
 
         # @rbs (String value) -> bool
         def declaration_boundary?(value)
+          return false if %w[display type param printer].include?(value) && !extended_features?
+
           DECLARATIONS.key?(value) || %w[prechigh preclow].include?(value)
         end
 
+        # @rbs () -> bool
+        def extended_features?
+          @extended_mode || extended_pragma? || cst_pragma?
+        end
+
         # @rbs (Token token) -> external_token
         def precedence_identifier(token)
           value = string_value(token)
@@ -189,14 +249,80 @@ module Ibex
         # @rbs (Token token, Array[Token] remaining) -> external_token
         def classify_scalar(token, remaining)
           type = SCALAR_TYPES.fetch(token.type)
-          return finish_single_symbol(type) if @state == :expect_integer && type == :INTEGER
-          return finish_single_symbol(type) if @state == :start_symbol && type == :LITERAL
-          return begin_conversion(token, type, remaining) if @state == :convert_name && type == :LITERAL
-          return finish_conversion(type) if @state == :convert_expression && type == :LITERAL
+          classified = classify_lexer_scalar(type) ||
+                       classify_token_alias(token, type) || classify_include(type) || classify_single_symbol(type) ||
+                       classify_metadata(type) || classify_printer(type) || classify_grammar_test(type) ||
+                       classify_conversion(token, type, remaining)
+
+          classified || type
+        end
+
+        # @rbs (external_token type) -> external_token?
+        def classify_single_symbol(type)
+          return finish_single_symbol(type) if %i[expect_integer expect_rr_integer].include?(@state) && type == :INTEGER
+
+          first = continue_first_symbol(type)
+          return first if first
+
+          type if @state == :start_symbols && type == :LITERAL
+        end
 
+        # @rbs (external_token type) -> external_token?
+        def continue_first_symbol(type)
+          return unless type == :LITERAL
+          return continue_start_symbols(type) if @state == :start_first_symbol
+          return continue_recovery_symbols(type) if @state == :recovery_first_symbol
+
+          continue_on_error_reduce_symbols(type) if @state == :on_error_reduce_first_symbol
+        end
+
+        # @rbs (Token token, external_token type) -> external_token?
+        def classify_token_alias(token, type)
+          candidate = @token_alias_candidate
+          return unless extended_features? && @state == :token_symbols &&
+                        type == :LITERAL && candidate
+          return unless candidate.location.line == token.location.line
+
+          @token_alias_candidate = nil
+          :TOKEN_ALIAS
+        end
+
+        # @rbs (external_token type) -> external_token?
+        def classify_metadata(type)
+          return unless type == :LITERAL
+          return begin_metadata_value(type) if %i[display_symbol type_symbol].include?(@state)
+          return finish_param_type(type) if @state == :param_type
+
+          finish_metadata_value(type) if %i[display_value type_value].include?(@state)
+        end
+
+        # @rbs (external_token type) -> external_token?
+        def classify_printer(type)
+          return begin_printer_action(type) if @state == :printer_symbol && type == :LITERAL
+          return unless @state == :printer_action && type == :ACTION
+
+          @state = :declaration
+          @declaration = nil
+          type
+        end
+
+        # @rbs (external_token type) -> external_token?
+        def classify_grammar_test(type)
+          return unless @state == :test_source && type == :LITERAL
+
+          @state = :declaration
+          @declaration = nil
           type
         end
 
+        # @rbs (Token token, external_token type, Array[Token] remaining) -> external_token?
+        def classify_conversion(token, type, remaining)
+          return unless type == :LITERAL
+          return begin_conversion(token, type, remaining) if @state == :convert_name
+
+          finish_conversion(type) if @state == :convert_expression
+        end
+
         # @rbs (external_token type) -> external_token
         def finish_single_symbol(type)
           @state = :declaration
@@ -204,6 +330,68 @@ module Ibex
           type
         end
 
+        # @rbs (external_token type) -> external_token
+        def continue_start_symbols(type)
+          @state = :start_symbols
+          type
+        end
+
+        # @rbs (external_token type) -> external_token
+        def continue_recovery_symbols(type)
+          @state = :recovery_symbols
+          type
+        end
+
+        # @rbs (external_token type) -> external_token
+        def continue_on_error_reduce_symbols(type)
+          @state = :on_error_reduce_symbols
+          type
+        end
+
+        # @rbs (external_token type) -> external_token
+        def begin_metadata_value(type)
+          @state = @state == :display_symbol ? :display_value : :type_value
+          type
+        end
+
+        # @rbs () -> external_token
+        def begin_param_type
+          @state = :param_type
+          :IDENTIFIER
+        end
+
+        # @rbs (external_token type) -> external_token
+        def begin_printer_action(type)
+          @state = :printer_action
+          type
+        end
+
+        # @rbs (Token token) -> external_token
+        def begin_recovery_colon(_token)
+          @state = :recovery_colon
+          :IDENTIFIER
+        end
+
+        # @rbs (Token token) -> external_token
+        def begin_test_source(_token)
+          @state = :test_source
+          :IDENTIFIER
+        end
+
+        # @rbs (external_token type) -> external_token
+        def finish_param_type(type)
+          @state = :declaration
+          @declaration = nil
+          type
+        end
+
+        # @rbs (external_token type) -> external_token
+        def finish_metadata_value(type)
+          @state = :declaration
+          @declaration = nil
+          type
+        end
+
         # @rbs (external_token type) -> external_token
         def finish_conversion(type)
           @state = :convert_name
@@ -214,6 +402,7 @@ module Ibex
         # @rbs (Token token) -> external_token
         def classify_punctuation(token)
           @state = :superclass_name if @state == :superclass_marker && token.type == :<
+          @state = :recovery_first_symbol if @state == :recovery_colon && token.type == :":"
           string_value(token)
         end
 
@@ -232,7 +421,7 @@ module Ibex
         def precedence_expectation(token)
           return @precedence_closer if token&.type == :eof
 
-          "left or right or nonassoc" if @state == :precedence_association
+          "left or right or nonassoc or precedence" if @state == :precedence_association
         end
 
         # @rbs (Token token) -> String
@@ -243,6 +432,7 @@ module Ibex
           raise Ibex::Error, "#{token.location}: expected text token"
         end
       end
+      # rubocop:enable Metrics/ClassLength, Metrics/CyclomaticComplexity
     end
   end
 end

data/lib/ibex/frontend/token_adapter/delimiter_tracker.rb

@@ -23,8 +23,14 @@ module Ibex
         # @rbs (Token token, external_token external, external_token? previous_external) -> void
         def observe(token, external, previous_external)
           if external == "("
-            separated = %i[SEPARATED_LIST SEPARATED_NONEMPTY_LIST].include?(previous_external)
-            @stack << [separated ? :separated : :group, token] #: [delimiter_kind, Token]
+            kind = if %i[SEPARATED_LIST SEPARATED_NONEMPTY_LIST].include?(previous_external)
+                     :separated
+                   elsif %i[LHS PARAMETERIZED_REFERENCE].include?(previous_external)
+                     :parameter
+                   else
+                     :group
+                   end #: delimiter_kind
+            @stack << [kind, token] #: [delimiter_kind, Token]
           elsif external == ")"
             @stack.pop
           end

data/lib/ibex/frontend/token_adapter/rule_state.rb

@@ -15,6 +15,7 @@ module Ibex
         # @rbs @delimiters: DelimiterTracker
         # @rbs @lhs_column: Integer?
         # @rbs @rule_seen: bool?
+        # @rbs @node_annotation: bool?
 
         # @rbs () -> void
         def initialize
@@ -28,12 +29,20 @@ module Ibex
         def classify(token, remaining, last_external:, previous_external:)
           external = if token.type == :identifier
                        classify_identifier(token, remaining, last_external, previous_external)
+                     elsif token.type == :node
+                       @node_annotation = true
+                       :NODE
+                     elsif token.type == :inline
+                       classify_inline
+                     elsif token.type == :empty
+                       :EMPTY
                      elsif SCALAR_TYPES.key?(token.type)
                        SCALAR_TYPES.fetch(token.type)
                      else
                        classify_punctuation(token)
                      end
           @delimiters.observe(token, external, last_external)
+          @node_annotation = false if token.type == :")" && @node_annotation
           external
         end
 
@@ -60,21 +69,35 @@ module Ibex
 
         private
 
+        # @rbs () -> external_token
+        def classify_inline
+          reset_rule_boundary if @state == :rule_rhs && @delimiters.empty?
+          :INLINE
+        end
+
         # @rbs (Token token, Array[Token] remaining, external_token? last_external,
         #   external_token? previous_external) -> external_token
         def classify_identifier(token, remaining, last_external, previous_external)
+          return :IDENTIFIER if @node_annotation
           return rule_lhs(token) if @state == :rules_lhs
           return :IDENTIFIER unless @state == :rule_rhs
           return :IDENTIFIER if named_reference_name?(last_external, previous_external)
 
+          classify_rhs_identifier(token, remaining)
+        end
+
+        # @rbs (Token token, Array[Token] remaining) -> external_token
+        def classify_rhs_identifier(token, remaining)
           value = string_value(token)
-          return :IDENTIFIER if value == "end" && @delimiters.open_kind == :separated
+          return :IDENTIFIER if value == "end" && @delimiters.open_kind
           return finish_rules if value == "end"
 
           following = remaining.first
           return :IDENTIFIER unless following
-          return start_rule(token) if rule_boundary?(token, following)
+          return start_rule(token) if rule_boundary?(token, following) ||
+                                      parameterized_rule_boundary?(token, remaining)
           return separated_terminal(token) if separated_list_call?(token, following)
+          return :PARAMETERIZED_REFERENCE if parameterized_call?(token, following)
 
           :IDENTIFIER
         end
@@ -105,6 +128,26 @@ module Ibex
           @delimiters.empty? && following.type == :":" && token.location.column <= lhs_column
         end
 
+        # @rbs (Token token, Array[Token] remaining) -> bool
+        def parameterized_rule_boundary?(token, remaining)
+          lhs_column = @lhs_column
+          opening = remaining.first
+          return false unless lhs_column && @delimiters.empty? && token.location.column <= lhs_column
+          return false unless opening&.type == :"(" && adjacent?(token, opening)
+
+          parameter_definition_tail?(remaining.drop(1))
+        end
+
+        # @rbs (Array[Token] tail) -> bool
+        def parameter_definition_tail?(tail)
+          closing = tail.index { |entry| entry.type == :")" }
+          return false unless closing && tail[closing + 1]&.type == :":"
+
+          formals = tail.take(closing)
+          !formals.empty? && formals.length.odd? &&
+            formals.each_with_index.all? { |entry, index| entry.type == (index.even? ? :identifier : :",") }
+        end
+
         # @rbs (Token token) -> external_token
         def start_rule(token)
           @rule_seen = true
@@ -129,6 +172,21 @@ module Ibex
           string_value(token) == "separated_list" ? :SEPARATED_LIST : :SEPARATED_NONEMPTY_LIST
         end
 
+        # @rbs (Token token, Token following) -> bool
+        def parameterized_call?(token, following)
+          following.type == :"(" && adjacent?(token, following)
+        end
+
+        # @rbs (Token left, Token right) -> bool
+        def adjacent?(left, right)
+          left_span = left.span
+          right_span = right.span
+          return left_span.end_byte == right_span.start_byte if left_span && right_span
+
+          left.location.line == right.location.line &&
+            left.location.column + string_value(left).length == right.location.column
+        end
+
         # @rbs (Token token) -> external_token
         def classify_punctuation(token)
           @state = :rule_rhs if @state == :rule_colon && token.type == :":"

data/lib/ibex/frontend/token_adapter.rb

@@ -15,11 +15,11 @@ module Ibex
       # @rbs @rules: RuleState?
       # @rbs @last_external: external_token?
 
-      # @rbs (Array[Token] tokens) -> void
-      def initialize(tokens)
+      # @rbs (Array[Token] tokens, ?extended: bool) -> void
+      def initialize(tokens, extended: false)
         @tokens = tokens
         @index = 0
-        @declarations = DeclarationState.new
+        @declarations = DeclarationState.new(extended: extended)
         @classifier = @declarations
       end
 
@@ -60,6 +60,11 @@ module Ibex
         @declarations.extended_pragma?
       end
 
+      # @rbs () -> bool
+      def cst_pragma?
+        @declarations.cst_pragma?
+      end
+
       # @rbs () -> parser_section
       def section
         rules = @rules

data/lib/ibex/frontend.rb

@@ -2,18 +2,29 @@
 
 require_relative "error"
 require_relative "tables"
-require_relative "runtime"
+require "ibex/runtime"
+require_relative "frontend/source_span"
 require_relative "frontend/source_cursor"
+require_relative "frontend/source_document"
+require_relative "frontend/diagnostic"
 require_relative "frontend/action_scanner"
 require_relative "frontend/lexer"
+require_relative "frontend/lexer_recovery"
 require_relative "frontend/ast"
+require_relative "frontend/rule_documentation"
+require_relative "frontend/token_adapter/declaration_document_state"
 require_relative "frontend/token_adapter/declaration_state"
 require_relative "frontend/token_adapter/delimiter_tracker"
 require_relative "frontend/token_adapter/rule_state"
 require_relative "frontend/token_adapter"
 require_relative "frontend/generated_parser_base"
 require_relative "frontend/generated_parser"
+require_relative "frontend/diagnostic_recovery"
 require_relative "frontend/parser"
+require_relative "frontend/formatter"
+require_relative "frontend/resolution"
+require_relative "frontend/source_loader"
+require_relative "frontend/resolver"
 require_relative "frontend/dsl"
 
 module Ibex
@@ -23,6 +34,6 @@ module Ibex
     #   type token_value = String | Integer | user_code_token | nil
     #   type external_token = Symbol | String
     #   type parser_section = :declarations | :rules | :user_code
-    #   type delimiter_kind = :group | :separated
+    #   type delimiter_kind = :group | :separated | :parameter
   end
 end