ibex 0.1.0 → 0.3.0

data/lib/ibex/frontend/source_span.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+module Ibex
+  module Frontend
+    # Normalizes grammar source without changing its bytes.
+    module SourceEncoding
+      # @rbs (String source, String file) -> String
+      def self.validated_utf8(source, file)
+        validated = source.dup.force_encoding(Encoding::UTF_8)
+        raise Ibex::Error, "#{file}: input must be valid UTF-8" unless validated.valid_encoding?
+
+        validated.freeze
+      end
+
+      # @rbs (String file) -> String
+      def self.validated_file(file)
+        validated = file.dup.force_encoding(Encoding::UTF_8)
+        raise Ibex::Error, "#{file}: file path must be valid UTF-8" unless validated.valid_encoding?
+
+        validated.freeze
+      end
+    end
+
+    # An immutable position in grammar source.
+    class SourcePosition
+      attr_reader :byte_offset #: Integer
+      attr_reader :line #: Integer
+      attr_reader :column #: Integer
+
+      # @rbs (byte_offset: Integer, line: Integer, column: Integer) -> void
+      def initialize(byte_offset:, line:, column:)
+        raise ArgumentError, "byte_offset must be non-negative" if byte_offset.negative?
+        raise ArgumentError, "line and column must be positive" unless line.positive? && column.positive?
+
+        @byte_offset = byte_offset
+        @line = line
+        @column = column
+        freeze
+      end
+
+      # @rbs () -> Hash[Symbol, Integer]
+      def to_h
+        { byte_offset: byte_offset, line: line, column: column }
+      end
+    end
+
+    # An immutable half-open byte span in one grammar source.
+    class SourceSpan
+      attr_reader :file #: String
+      attr_reader :start #: SourcePosition
+      attr_reader :finish #: SourcePosition
+
+      # @rbs (file: String, start: SourcePosition, finish: SourcePosition) -> void
+      def initialize(file:, start:, finish:)
+        raise ArgumentError, "span end precedes its start" if finish.byte_offset < start.byte_offset
+
+        @file = file.dup.freeze
+        @start = start
+        @finish = finish
+        freeze
+      end
+
+      # @rbs () -> Integer
+      def start_byte
+        start.byte_offset
+      end
+
+      # @rbs () -> Integer
+      def end_byte
+        finish.byte_offset
+      end
+
+      # @rbs () -> Integer
+      def length
+        end_byte - start_byte
+      end
+
+      # @rbs () -> bool
+      def empty?
+        length.zero?
+      end
+
+      # @rbs () -> Hash[Symbol, Object?]
+      def to_h
+        { file: file, start: start.to_h, end: finish.to_h }
+      end
+    end
+  end
+end

data/lib/ibex/frontend/token_adapter/declaration_document_state.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Ibex
+  module Frontend
+    class TokenAdapter
+      # Root/fragment header and include-path transitions for declaration classification.
+      module DeclarationDocumentState
+        private
+
+        # @rbs (Token token) -> external_token
+        def class_keyword(token)
+          # @type self: DeclarationState
+          value = string_value(token)
+          if value == "fragment"
+            raise Ibex::Error, "#{token.location}: fragments require extended mode" unless @extended_mode
+
+            @fragment = true
+            @state = :declaration
+            return :FRAGMENT
+          end
+          return :IDENTIFIER unless value == "class"
+
+          @state = :class_name
+          :CLASS
+        end
+
+        # @rbs (external_token type) -> external_token?
+        def classify_include(type)
+          # @type self: DeclarationState
+          return unless @state == :include_path && type == :LITERAL
+
+          @state = :declaration
+          @declaration = nil
+          type
+        end
+
+        # @rbs (Token token, String value) -> void
+        def reject_fragment_pragma(token, value)
+          # @type self: DeclarationState
+          return unless @fragment && value == "pragma"
+
+          raise Ibex::Error, "#{token.location}: pragma declarations are not allowed in fragments"
+        end
+      end
+    end
+  end
+end

data/lib/ibex/frontend/token_adapter/declaration_lexer_state.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module Ibex
+  module Frontend
+    class TokenAdapter
+      # Context transitions for the extended root-only lexer declaration.
+      module DeclarationLexerState
+        private
+
+        # @rbs () -> bool
+        def lexer_declaration_state?
+          @declaration == :lexer
+        end
+
+        # @rbs (Token token) -> external_token
+        def classify_lexer_identifier(token)
+          # @type self: DeclarationState
+          case @state
+          when :lexer_entries, :lexer_action_or_entry then begin_lexer_entry(token)
+          when :lexer_state_name
+            @state = :lexer_state_do
+            :IDENTIFIER
+          when :lexer_state_do
+            value = string_value(token)
+            raise Ibex::Error, "#{token.location}: expected do, got #{value}" unless value == "do"
+
+            @lexer_state_depth = (@lexer_state_depth || 0) + 1
+            @state = :lexer_entries
+            :DO
+          else
+            :IDENTIFIER
+          end
+        end
+
+        # @rbs (Token token) -> external_token
+        def begin_lexer_entry(token)
+          # @type self: DeclarationState
+          value = string_value(token)
+          case value
+          when "end" then finish_lexer_scope
+          when "state"
+            @state = :lexer_state_name
+            :STATE
+          when "skip"
+            @state = :lexer_pattern
+            :SKIP
+          when "on"
+            @state = :lexer_pattern
+            :ON
+          else
+            @state = :lexer_pattern
+            :IDENTIFIER
+          end
+        end
+
+        # @rbs () -> external_token
+        def finish_lexer_scope
+          depth = @lexer_state_depth || 0
+          if depth.positive?
+            @lexer_state_depth = depth - 1
+            @state = :lexer_entries
+          else
+            @state = :declaration
+            @declaration = nil
+          end
+          :END
+        end
+
+        # @rbs (external_token type) -> external_token?
+        def classify_lexer_scalar(type)
+          if @state == :lexer_pattern && %i[REGEXP LITERAL].include?(type)
+            @state = :lexer_action_or_entry
+            return type
+          end
+          return unless @state == :lexer_action_or_entry && type == :ACTION
+
+          @state = :lexer_entries
+          type
+        end
+      end
+    end
+  end
+end

data/lib/ibex/frontend/token_adapter/declaration_state.rb

@@ -1,25 +1,57 @@
 # frozen_string_literal: true
 
+require_relative "declaration_lexer_state"
+
 module Ibex
   module Frontend
     class TokenAdapter
       # Classifies tokens through the class header and declaration section.
+      # rubocop:disable Metrics/ClassLength, Metrics/CyclomaticComplexity
+      # One state machine owns declaration-boundary classification.
       class DeclarationState
+        include DeclarationDocumentState
+        include DeclarationLexerState
+
         DECLARATIONS = {
+          "include" => %i[INCLUDE include_path],
+          "import" => %i[IMPORT include_path],
           "token" => %i[TOKEN token_symbols], "options" => %i[OPTIONS options_identifiers],
-          "expect" => %i[EXPECT expect_integer], "start" => %i[START start_symbol],
+          "expect" => %i[EXPECT expect_integer], "start" => %i[START start_first_symbol],
+          "expect_rr" => %i[EXPECT_RR expect_rr_integer],
+          "recover" => %i[RECOVER recover_kind],
+          "on_error_reduce" => %i[ON_ERROR_REDUCE on_error_reduce_first_symbol],
+          "test" => %i[TEST test_expectation],
+          "lexer" => %i[LEXER lexer_entries],
           "convert" => %i[CONVERT convert_name], "pragma" => %i[PRAGMA pragma_value],
+          "display" => %i[DISPLAY display_symbol], "type" => %i[TYPE type_symbol],
+          "param" => %i[PARAM param_name],
+          "printer" => %i[PRINTER printer_symbol],
+          "parser" => %i[PARSER parser_key],
           "rule" => %i[RULE rules]
         }.freeze #: Hash[String, [external_token, Symbol]]
         ASSOCIATIONS = {
-          "left" => :LEFT, "right" => :RIGHT, "nonassoc" => :NONASSOC
+          "left" => :LEFT, "right" => :RIGHT, "nonassoc" => :NONASSOC, "precedence" => :PRECEDENCE
         }.freeze #: Hash[String, external_token]
         SCALAR_TYPES = {
-          literal: :LITERAL, integer: :INTEGER, action: :ACTION, user_code: :USER_CODE
+          literal: :LITERAL, regexp: :REGEXP, integer: :INTEGER, action: :ACTION, user_code: :USER_CODE
         }.freeze #: Hash[Symbol, external_token]
         EXPECTATIONS = {
           class_keyword: "class", class_name: "identifier", superclass_name: "identifier",
-          expect_integer: "integer", start_symbol: "a grammar symbol"
+          expect_integer: "integer", expect_rr_integer: "integer",
+          start_first_symbol: "a grammar symbol", start_symbols: "a grammar symbol",
+          include_path: "a double-quoted relative path",
+          display_symbol: "a grammar symbol", type_symbol: "a grammar symbol",
+          display_value: "a quoted string", type_value: "a quoted string",
+          param_name: "an identifier", param_type: "a quoted type or declaration",
+          printer_symbol: "a grammar symbol", printer_action: "an action",
+          recover_kind: "sync", recovery_colon: ":", recovery_first_symbol: "a grammar symbol",
+          recovery_symbols: "a grammar symbol", on_error_reduce_first_symbol: "a grammar symbol",
+          on_error_reduce_symbols: "a grammar symbol", test_expectation: "accept or reject",
+          test_source: "a double-quoted string",
+          lexer_entries: "a lexer rule, state, or end", lexer_state_name: "a state name",
+          lexer_state_do: "do", lexer_pattern: "a regular expression or quoted literal",
+          lexer_action_or_entry: "an action, lexer rule, state, or end",
+          parser_key: "a parser setting or end", parser_value: "a parser setting value"
         }.freeze #: Hash[Symbol, String]
 
         attr_reader :conversion_name #: Token?
@@ -27,8 +59,15 @@ module Ibex
         attr_reader :precedence_closer #: String?
         attr_reader :state #: Symbol
 
-        # @rbs () -> void
-        def initialize
+        # @rbs @extended_mode: bool
+        # @rbs @fragment: bool?
+        # @rbs @pragmas: Hash[String, Location]
+        # @rbs @pragma_location: Location?
+
+        # @rbs (?extended: bool) -> void
+        def initialize(extended: false)
+          @extended_mode = extended
+          @pragmas = {} #: Hash[String, Location]
           @state = :class_keyword
         end
 
@@ -47,7 +86,22 @@ module Ibex
 
         # @rbs () -> bool
         def extended_pragma?
-          @extended_pragma == true
+          @pragmas.key?("extended")
+        end
+
+        # @rbs () -> bool
+        def cst_pragma?
+          @pragmas.key?("cst")
+        end
+
+        # @rbs () -> Location?
+        def extended_pragma_location
+          @pragmas["extended"]
+        end
+
+        # @rbs () -> Location?
+        def cst_pragma_location
+          @pragmas["cst"]
         end
 
         # @rbs (Token? token) -> String?
@@ -70,25 +124,50 @@ module Ibex
 
         # @rbs (Token token, Array[Token] remaining) -> external_token
         def classify_identifier(token, remaining)
+          return classify_lexer_identifier(token) if lexer_declaration_state?
+
           case @state
           when :class_keyword then class_keyword(token)
           when :class_name, :superclass_name then constant_name(remaining)
-          when :declaration then begin_declaration(token)
-          when :token_symbols, :options_identifiers then declaration_symbol(token)
+          when :declaration, :param_type then begin_declaration(token)
+          when :token_symbols
+            @token_alias_candidate = token
+            declaration_symbol(token)
+          when :options_identifiers, :start_symbols, :recovery_symbols, :on_error_reduce_symbols
+            declaration_symbol(token)
           when :precedence_association, :precedence_symbols then precedence_identifier(token)
-          when :start_symbol then finish_single_symbol(:IDENTIFIER)
+          when :start_first_symbol then continue_start_symbols(:IDENTIFIER)
+          when :display_symbol, :type_symbol then begin_metadata_value(:IDENTIFIER)
+          when :param_name then begin_param_type
+          when :printer_symbol then begin_printer_action(:IDENTIFIER)
+          when :recover_kind then begin_recovery_colon(token)
+          when :recovery_first_symbol then continue_recovery_symbols(:IDENTIFIER)
+          when :on_error_reduce_first_symbol then continue_on_error_reduce_symbols(:IDENTIFIER)
+          when :test_expectation then begin_test_source(token)
           when :pragma_value then finish_pragma(token)
           when :convert_name then begin_conversion(token, :IDENTIFIER, remaining)
+          when :parser_key then begin_parser_setting(token)
+          when :parser_value then finish_parser_setting
           else :IDENTIFIER
           end
         end
 
         # @rbs (Token token) -> external_token
-        def class_keyword(token)
-          return :IDENTIFIER unless string_value(token) == "class"
+        def begin_parser_setting(token)
+          if string_value(token) == "end"
+            @state = :declaration
+            @declaration = nil
+            return :END
+          end
+
+          @state = :parser_value
+          :IDENTIFIER
+        end
 
-          @state = :class_name
-          :CLASS
+        # @rbs () -> external_token
+        def finish_parser_setting
+          @state = :parser_key
+          :IDENTIFIER
         end
 
         # @rbs (Array[Token] remaining) -> external_token
@@ -111,11 +190,13 @@ module Ibex
           value = string_value(token)
           return begin_precedence(token) if %w[prechigh preclow].include?(value)
 
-          raise Ibex::Error, "#{token.location}: duplicate pragma extended" if value == "pragma" && @extended_pragma
+          reject_fragment_pragma(token, value)
 
           terminal, next_state = DECLARATIONS[value]
           return :IDENTIFIER unless terminal
 
+          @pragma_location = token.location if terminal == :PRAGMA
+          @token_alias_candidate = nil
           @state = next_state
           @declaration = value.to_sym unless terminal == :RULE
           @declaration = nil if terminal == :RULE
@@ -125,9 +206,13 @@ module Ibex
         # @rbs (Token token) -> external_token
         def finish_pragma(token)
           value = string_value(token)
-          raise Ibex::Error, "#{token.location}: unknown pragma #{value}" unless value == "extended"
+          raise Ibex::Error, "#{token.location}: unknown pragma #{value}" unless %w[extended cst].include?(value)
+
+          location = @pragma_location || token.location
+          raise Ibex::Error, "#{location}: duplicate pragma #{value}" if @pragmas[value]
 
-          @extended_pragma = true
+          @pragmas[value] = location
+          @pragma_location = nil
           @state = :declaration
           @declaration = nil
           :IDENTIFIER
@@ -151,9 +236,16 @@ module Ibex
 
         # @rbs (String value) -> bool
         def declaration_boundary?(value)
+          return false if %w[display type param printer].include?(value) && !extended_features?
+
           DECLARATIONS.key?(value) || %w[prechigh preclow].include?(value)
         end
 
+        # @rbs () -> bool
+        def extended_features?
+          @extended_mode || extended_pragma? || cst_pragma?
+        end
+
         # @rbs (Token token) -> external_token
         def precedence_identifier(token)
           value = string_value(token)
@@ -189,14 +281,80 @@ module Ibex
         # @rbs (Token token, Array[Token] remaining) -> external_token
         def classify_scalar(token, remaining)
           type = SCALAR_TYPES.fetch(token.type)
-          return finish_single_symbol(type) if @state == :expect_integer && type == :INTEGER
-          return finish_single_symbol(type) if @state == :start_symbol && type == :LITERAL
-          return begin_conversion(token, type, remaining) if @state == :convert_name && type == :LITERAL
-          return finish_conversion(type) if @state == :convert_expression && type == :LITERAL
+          classified = classify_lexer_scalar(type) ||
+                       classify_token_alias(token, type) || classify_include(type) || classify_single_symbol(type) ||
+                       classify_metadata(type) || classify_printer(type) || classify_grammar_test(type) ||
+                       classify_conversion(token, type, remaining)
 
+          classified || type
+        end
+
+        # @rbs (external_token type) -> external_token?
+        def classify_single_symbol(type)
+          return finish_single_symbol(type) if %i[expect_integer expect_rr_integer].include?(@state) && type == :INTEGER
+
+          first = continue_first_symbol(type)
+          return first if first
+
+          type if @state == :start_symbols && type == :LITERAL
+        end
+
+        # @rbs (external_token type) -> external_token?
+        def continue_first_symbol(type)
+          return unless type == :LITERAL
+          return continue_start_symbols(type) if @state == :start_first_symbol
+          return continue_recovery_symbols(type) if @state == :recovery_first_symbol
+
+          continue_on_error_reduce_symbols(type) if @state == :on_error_reduce_first_symbol
+        end
+
+        # @rbs (Token token, external_token type) -> external_token?
+        def classify_token_alias(token, type)
+          candidate = @token_alias_candidate
+          return unless extended_features? && @state == :token_symbols &&
+                        type == :LITERAL && candidate
+          return unless candidate.location.line == token.location.line
+
+          @token_alias_candidate = nil
+          :TOKEN_ALIAS
+        end
+
+        # @rbs (external_token type) -> external_token?
+        def classify_metadata(type)
+          return unless type == :LITERAL
+          return begin_metadata_value(type) if %i[display_symbol type_symbol].include?(@state)
+          return finish_param_type(type) if @state == :param_type
+
+          finish_metadata_value(type) if %i[display_value type_value].include?(@state)
+        end
+
+        # @rbs (external_token type) -> external_token?
+        def classify_printer(type)
+          return begin_printer_action(type) if @state == :printer_symbol && type == :LITERAL
+          return unless @state == :printer_action && type == :ACTION
+
+          @state = :declaration
+          @declaration = nil
+          type
+        end
+
+        # @rbs (external_token type) -> external_token?
+        def classify_grammar_test(type)
+          return unless @state == :test_source && type == :LITERAL
+
+          @state = :declaration
+          @declaration = nil
           type
         end
 
+        # @rbs (Token token, external_token type, Array[Token] remaining) -> external_token?
+        def classify_conversion(token, type, remaining)
+          return unless type == :LITERAL
+          return begin_conversion(token, type, remaining) if @state == :convert_name
+
+          finish_conversion(type) if @state == :convert_expression
+        end
+
         # @rbs (external_token type) -> external_token
         def finish_single_symbol(type)
           @state = :declaration
@@ -204,6 +362,68 @@ module Ibex
           type
         end
 
+        # @rbs (external_token type) -> external_token
+        def continue_start_symbols(type)
+          @state = :start_symbols
+          type
+        end
+
+        # @rbs (external_token type) -> external_token
+        def continue_recovery_symbols(type)
+          @state = :recovery_symbols
+          type
+        end
+
+        # @rbs (external_token type) -> external_token
+        def continue_on_error_reduce_symbols(type)
+          @state = :on_error_reduce_symbols
+          type
+        end
+
+        # @rbs (external_token type) -> external_token
+        def begin_metadata_value(type)
+          @state = @state == :display_symbol ? :display_value : :type_value
+          type
+        end
+
+        # @rbs () -> external_token
+        def begin_param_type
+          @state = :param_type
+          :IDENTIFIER
+        end
+
+        # @rbs (external_token type) -> external_token
+        def begin_printer_action(type)
+          @state = :printer_action
+          type
+        end
+
+        # @rbs (Token token) -> external_token
+        def begin_recovery_colon(_token)
+          @state = :recovery_colon
+          :IDENTIFIER
+        end
+
+        # @rbs (Token token) -> external_token
+        def begin_test_source(_token)
+          @state = :test_source
+          :IDENTIFIER
+        end
+
+        # @rbs (external_token type) -> external_token
+        def finish_param_type(type)
+          @state = :declaration
+          @declaration = nil
+          type
+        end
+
+        # @rbs (external_token type) -> external_token
+        def finish_metadata_value(type)
+          @state = :declaration
+          @declaration = nil
+          type
+        end
+
         # @rbs (external_token type) -> external_token
         def finish_conversion(type)
           @state = :convert_name
@@ -214,6 +434,7 @@ module Ibex
         # @rbs (Token token) -> external_token
         def classify_punctuation(token)
           @state = :superclass_name if @state == :superclass_marker && token.type == :<
+          @state = :recovery_first_symbol if @state == :recovery_colon && token.type == :":"
           string_value(token)
         end
 
@@ -232,7 +453,7 @@ module Ibex
         def precedence_expectation(token)
           return @precedence_closer if token&.type == :eof
 
-          "left or right or nonassoc" if @state == :precedence_association
+          "left or right or nonassoc or precedence" if @state == :precedence_association
         end
 
         # @rbs (Token token) -> String
@@ -243,6 +464,7 @@ module Ibex
           raise Ibex::Error, "#{token.location}: expected text token"
         end
       end
+      # rubocop:enable Metrics/ClassLength, Metrics/CyclomaticComplexity
     end
   end
 end

data/lib/ibex/frontend/token_adapter/delimiter_tracker.rb

@@ -23,8 +23,14 @@ module Ibex
         # @rbs (Token token, external_token external, external_token? previous_external) -> void
         def observe(token, external, previous_external)
           if external == "("
-            separated = %i[SEPARATED_LIST SEPARATED_NONEMPTY_LIST].include?(previous_external)
-            @stack << [separated ? :separated : :group, token] #: [delimiter_kind, Token]
+            kind = if %i[SEPARATED_LIST SEPARATED_NONEMPTY_LIST].include?(previous_external)
+                     :separated
+                   elsif %i[LHS PARAMETERIZED_REFERENCE].include?(previous_external)
+                     :parameter
+                   else
+                     :group
+                   end #: delimiter_kind
+            @stack << [kind, token] #: [delimiter_kind, Token]
           elsif external == ")"
             @stack.pop
           end