ibex 0.1.0

data/lib/ibex/frontend/token_adapter/declaration_state.rb

@@ -0,0 +1,248 @@
+# frozen_string_literal: true
+
+module Ibex
+  module Frontend
+    class TokenAdapter
+      # Classifies tokens through the class header and declaration section.
+      class DeclarationState
+        DECLARATIONS = {
+          "token" => %i[TOKEN token_symbols], "options" => %i[OPTIONS options_identifiers],
+          "expect" => %i[EXPECT expect_integer], "start" => %i[START start_symbol],
+          "convert" => %i[CONVERT convert_name], "pragma" => %i[PRAGMA pragma_value],
+          "rule" => %i[RULE rules]
+        }.freeze #: Hash[String, [external_token, Symbol]]
+        ASSOCIATIONS = {
+          "left" => :LEFT, "right" => :RIGHT, "nonassoc" => :NONASSOC
+        }.freeze #: Hash[String, external_token]
+        SCALAR_TYPES = {
+          literal: :LITERAL, integer: :INTEGER, action: :ACTION, user_code: :USER_CODE
+        }.freeze #: Hash[Symbol, external_token]
+        EXPECTATIONS = {
+          class_keyword: "class", class_name: "identifier", superclass_name: "identifier",
+          expect_integer: "integer", start_symbol: "a grammar symbol"
+        }.freeze #: Hash[Symbol, String]
+
+        attr_reader :conversion_name #: Token?
+        attr_reader :declaration #: Symbol?
+        attr_reader :precedence_closer #: String?
+        attr_reader :state #: Symbol
+
+        # @rbs () -> void
+        def initialize
+          @state = :class_keyword
+        end
+
+        # @rbs (Token token, Array[Token] remaining) -> external_token
+        def classify(token, remaining)
+          return classify_identifier(token, remaining) if token.type == :identifier
+          return classify_scalar(token, remaining) if SCALAR_TYPES.key?(token.type)
+
+          classify_punctuation(token)
+        end
+
+        # @rbs () -> bool
+        def rules?
+          @state == :rules
+        end
+
+        # @rbs () -> bool
+        def extended_pragma?
+          @extended_pragma == true
+        end
+
+        # @rbs (Token? token) -> String?
+        def expectation(token)
+          return "identifier" if @state == :pragma_value && token&.type != :identifier
+
+          expected = EXPECTATIONS[@state]
+          return expected if expected
+
+          if @declaration == :precedence
+            precedence_expectation(token)
+          elsif @declaration == :convert
+            "end"
+          elsif @state == :declaration
+            token&.type == :eof ? "rule" : "a declaration or rule"
+          end
+        end
+
+        private
+
+        # @rbs (Token token, Array[Token] remaining) -> external_token
+        def classify_identifier(token, remaining)
+          case @state
+          when :class_keyword then class_keyword(token)
+          when :class_name, :superclass_name then constant_name(remaining)
+          when :declaration then begin_declaration(token)
+          when :token_symbols, :options_identifiers then declaration_symbol(token)
+          when :precedence_association, :precedence_symbols then precedence_identifier(token)
+          when :start_symbol then finish_single_symbol(:IDENTIFIER)
+          when :pragma_value then finish_pragma(token)
+          when :convert_name then begin_conversion(token, :IDENTIFIER, remaining)
+          else :IDENTIFIER
+          end
+        end
+
+        # @rbs (Token token) -> external_token
+        def class_keyword(token)
+          return :IDENTIFIER unless string_value(token) == "class"
+
+          @state = :class_name
+          :CLASS
+        end
+
+        # @rbs (Array[Token] remaining) -> external_token
+        def constant_name(remaining)
+          following = remaining.first
+          raise Ibex::Error, "unexpected end of token stream" unless following
+
+          @state = if following.type == :scope
+                     @state
+                   elsif following.type == :<
+                     :superclass_marker
+                   else
+                     :declaration
+                   end
+          :IDENTIFIER
+        end
+
+        # @rbs (Token token) -> external_token
+        def begin_declaration(token)
+          value = string_value(token)
+          return begin_precedence(token) if %w[prechigh preclow].include?(value)
+
+          raise Ibex::Error, "#{token.location}: duplicate pragma extended" if value == "pragma" && @extended_pragma
+
+          terminal, next_state = DECLARATIONS[value]
+          return :IDENTIFIER unless terminal
+
+          @state = next_state
+          @declaration = value.to_sym unless terminal == :RULE
+          @declaration = nil if terminal == :RULE
+          terminal
+        end
+
+        # @rbs (Token token) -> external_token
+        def finish_pragma(token)
+          value = string_value(token)
+          raise Ibex::Error, "#{token.location}: unknown pragma #{value}" unless value == "extended"
+
+          @extended_pragma = true
+          @state = :declaration
+          @declaration = nil
+          :IDENTIFIER
+        end
+
+        # @rbs (Token token) -> external_token
+        def begin_precedence(token)
+          high_to_low = string_value(token) == "prechigh"
+          @precedence_closer = high_to_low ? "preclow" : "prechigh"
+          @declaration = :precedence
+          @state = :precedence_association
+          high_to_low ? :PRECHIGH : :PRECLOW
+        end
+
+        # @rbs (Token token) -> external_token
+        def declaration_symbol(token)
+          return begin_declaration(token) if declaration_boundary?(string_value(token))
+
+          :IDENTIFIER
+        end
+
+        # @rbs (String value) -> bool
+        def declaration_boundary?(value)
+          DECLARATIONS.key?(value) || %w[prechigh preclow].include?(value)
+        end
+
+        # @rbs (Token token) -> external_token
+        def precedence_identifier(token)
+          value = string_value(token)
+          return finish_precedence(token) if value == @precedence_closer
+
+          association = ASSOCIATIONS[value]
+          @state = :precedence_symbols if association
+          association || :IDENTIFIER
+        end
+
+        # @rbs (Token token) -> external_token
+        def finish_precedence(token)
+          @state = :declaration
+          @declaration = nil
+          @precedence_closer = nil
+          string_value(token) == "prechigh" ? :PRECHIGH : :PRECLOW
+        end
+
+        # @rbs (Token token, external_token type, Array[Token] remaining) -> external_token
+        def begin_conversion(token, type, remaining)
+          if token.type == :identifier && token.value == "end"
+            @state = :declaration
+            @declaration = nil
+            return :END
+          end
+
+          validate_conversion_line(token, remaining)
+          @conversion_name = token
+          @state = :convert_expression
+          type
+        end
+
+        # @rbs (Token token, Array[Token] remaining) -> external_token
+        def classify_scalar(token, remaining)
+          type = SCALAR_TYPES.fetch(token.type)
+          return finish_single_symbol(type) if @state == :expect_integer && type == :INTEGER
+          return finish_single_symbol(type) if @state == :start_symbol && type == :LITERAL
+          return begin_conversion(token, type, remaining) if @state == :convert_name && type == :LITERAL
+          return finish_conversion(type) if @state == :convert_expression && type == :LITERAL
+
+          type
+        end
+
+        # @rbs (external_token type) -> external_token
+        def finish_single_symbol(type)
+          @state = :declaration
+          @declaration = nil
+          type
+        end
+
+        # @rbs (external_token type) -> external_token
+        def finish_conversion(type)
+          @state = :convert_name
+          @conversion_name = nil
+          type
+        end
+
+        # @rbs (Token token) -> external_token
+        def classify_punctuation(token)
+          @state = :superclass_name if @state == :superclass_marker && token.type == :<
+          string_value(token)
+        end
+
+        # @rbs (Token name, Array[Token] remaining) -> void
+        def validate_conversion_line(name, remaining)
+          line = name.location.line
+          rest = remaining.take_while do |token|
+            token.type != :eof && token.location.line == line && !(token.type == :identifier && token.value == "end")
+          end
+          return if rest.length == 1 && rest.first.type == :literal
+
+          raise Ibex::Error, "#{name.location}: expected a quoted Ruby conversion expression"
+        end
+
+        # @rbs (Token? token) -> String?
+        def precedence_expectation(token)
+          return @precedence_closer if token&.type == :eof
+
+          "left or right or nonassoc" if @state == :precedence_association
+        end
+
+        # @rbs (Token token) -> String
+        def string_value(token)
+          value = token.value
+          return value if value.is_a?(String)
+
+          raise Ibex::Error, "#{token.location}: expected text token"
+        end
+      end
+    end
+  end
+end

data/lib/ibex/frontend/token_adapter/delimiter_tracker.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Ibex
+  module Frontend
+    class TokenAdapter
+      # Tracks ordinary EBNF groups separately from separated-list calls.
+      class DelimiterTracker
+        attr_reader :group_opening #: Token?
+        attr_reader :open_kind #: delimiter_kind?
+
+        # @rbs @stack: Array[[delimiter_kind, Token]]
+
+        # @rbs () -> void
+        def initialize
+          @stack = [] #: Array[[delimiter_kind, Token]]
+        end
+
+        # @rbs () -> bool
+        def empty?
+          @stack.empty?
+        end
+
+        # @rbs (Token token, external_token external, external_token? previous_external) -> void
+        def observe(token, external, previous_external)
+          if external == "("
+            separated = %i[SEPARATED_LIST SEPARATED_NONEMPTY_LIST].include?(previous_external)
+            @stack << [separated ? :separated : :group, token] #: [delimiter_kind, Token]
+          elsif external == ")"
+            @stack.pop
+          end
+          @open_kind = @stack.last&.first
+          @group_opening = @stack.reverse.find { |kind, _opening| kind == :group }&.last
+        end
+      end
+    end
+  end
+end

data/lib/ibex/frontend/token_adapter/rule_state.rb

@@ -0,0 +1,162 @@
+# frozen_string_literal: true
+
+module Ibex
+  module Frontend
+    class TokenAdapter
+      # Classifies context-dependent tokens in the grammar rule section.
+      class RuleState
+        SCALAR_TYPES = {
+          literal: :LITERAL, integer: :INTEGER, action: :ACTION, user_code: :USER_CODE
+        }.freeze #: Hash[Symbol, external_token]
+
+        attr_reader :section #: parser_section
+        attr_reader :state #: Symbol
+
+        # @rbs @delimiters: DelimiterTracker
+        # @rbs @lhs_column: Integer?
+        # @rbs @rule_seen: bool?
+
+        # @rbs () -> void
+        def initialize
+          @state = :rules_lhs
+          @section = :rules
+          @delimiters = DelimiterTracker.new
+        end
+
+        # @rbs (Token token, Array[Token] remaining, last_external: external_token?,
+        #   previous_external: external_token?) -> external_token
+        def classify(token, remaining, last_external:, previous_external:)
+          external = if token.type == :identifier
+                       classify_identifier(token, remaining, last_external, previous_external)
+                     elsif SCALAR_TYPES.key?(token.type)
+                       SCALAR_TYPES.fetch(token.type)
+                     else
+                       classify_punctuation(token)
+                     end
+          @delimiters.observe(token, external, last_external)
+          external
+        end
+
+        # @rbs () -> Token?
+        def group_opening
+          @delimiters.group_opening
+        end
+
+        # @rbs () -> delimiter_kind?
+        def open_delimiter_kind
+          @delimiters.open_kind
+        end
+
+        # @rbs (Token? token) -> String?
+        def expectation(token)
+          if @state == :rules_lhs && @section == :rules
+            rules_lhs_expectation(token)
+          elsif @state == :rule_colon
+            ":"
+          elsif @state == :rule_rhs && %w[) ,].include?(token&.value)
+            "a grammar symbol"
+          end
+        end
+
+        private
+
+        # @rbs (Token token, Array[Token] remaining, external_token? last_external,
+        #   external_token? previous_external) -> external_token
+        def classify_identifier(token, remaining, last_external, previous_external)
+          return rule_lhs(token) if @state == :rules_lhs
+          return :IDENTIFIER unless @state == :rule_rhs
+          return :IDENTIFIER if named_reference_name?(last_external, previous_external)
+
+          value = string_value(token)
+          return :IDENTIFIER if value == "end" && @delimiters.open_kind == :separated
+          return finish_rules if value == "end"
+
+          following = remaining.first
+          return :IDENTIFIER unless following
+          return start_rule(token) if rule_boundary?(token, following)
+          return separated_terminal(token) if separated_list_call?(token, following)
+
+          :IDENTIFIER
+        end
+
+        # @rbs (Token token) -> external_token
+        def rule_lhs(token)
+          if string_value(token) == "end"
+            @section = :user_code
+            return :END
+          end
+
+          @rule_seen = true
+          @lhs_column = token.location.column
+          @state = :rule_colon
+          :LHS
+        end
+
+        # @rbs (external_token? last_external, external_token? previous_external) -> bool
+        def named_reference_name?(last_external, previous_external)
+          last_external == ":" && %i[IDENTIFIER LITERAL].include?(previous_external)
+        end
+
+        # @rbs (Token token, Token following) -> bool
+        def rule_boundary?(token, following)
+          lhs_column = @lhs_column
+          return false unless lhs_column
+
+          @delimiters.empty? && following.type == :":" && token.location.column <= lhs_column
+        end
+
+        # @rbs (Token token) -> external_token
+        def start_rule(token)
+          @rule_seen = true
+          @lhs_column = token.location.column
+          @state = :rule_colon
+          :LHS
+        end
+
+        # @rbs () -> external_token
+        def finish_rules
+          @section = :user_code
+          :END
+        end
+
+        # @rbs (Token token, Token following) -> bool
+        def separated_list_call?(token, following)
+          %w[separated_list separated_nonempty_list].include?(string_value(token)) && following.type == :"("
+        end
+
+        # @rbs (Token token) -> external_token
+        def separated_terminal(token)
+          string_value(token) == "separated_list" ? :SEPARATED_LIST : :SEPARATED_NONEMPTY_LIST
+        end
+
+        # @rbs (Token token) -> external_token
+        def classify_punctuation(token)
+          @state = :rule_rhs if @state == :rule_colon && token.type == :":"
+          reset_rule_boundary if @state == :rule_rhs && token.type == :";" && @delimiters.empty?
+          string_value(token)
+        end
+
+        # @rbs () -> void
+        def reset_rule_boundary
+          @lhs_column = nil
+          @state = :rules_lhs
+        end
+
+        # @rbs (Token? token) -> String
+        def rules_lhs_expectation(token)
+          return "identifier" unless token&.type == :eof
+
+          @rule_seen ? "end" : "at least one rule"
+        end
+
+        # @rbs (Token token) -> String
+        def string_value(token)
+          value = token.value
+          return value if value.is_a?(String)
+
+          raise Ibex::Error, "#{token.location}: expected text token"
+        end
+      end
+    end
+  end
+end

data/lib/ibex/frontend/token_adapter.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+module Ibex
+  module Frontend
+    # Delivers lexer tokens through declaration and rule context classifiers.
+    class TokenAdapter
+      attr_reader :eof_token #: Token?
+      attr_reader :last_token #: Token?
+      attr_reader :previous_external #: external_token?
+
+      # @rbs @tokens: Array[Token]
+      # @rbs @index: Integer
+      # @rbs @declarations: DeclarationState
+      # @rbs @classifier: DeclarationState | RuleState
+      # @rbs @rules: RuleState?
+      # @rbs @last_external: external_token?
+
+      # @rbs (Array[Token] tokens) -> void
+      def initialize(tokens)
+        @tokens = tokens
+        @index = 0
+        @declarations = DeclarationState.new
+        @classifier = @declarations
+      end
+
+      # @rbs () -> ([external_token, Token] | false)
+      def next_token
+        token = @tokens.fetch(@index)
+        @index += 1
+        @last_token = token
+        if token.type == :eof
+          @eof_token = token
+          return false
+        end
+
+        external = classify(token)
+        enter_rules if @classifier == @declarations && @declarations.rules?
+        @previous_external = @last_external
+        @last_external = external
+        [external, token]
+      end
+
+      # @rbs () -> Token?
+      def conversion_name
+        @declarations.conversion_name
+      end
+
+      # @rbs () -> Symbol?
+      def declaration
+        @declarations.declaration
+      end
+
+      # @rbs () -> String?
+      def precedence_closer
+        @declarations.precedence_closer
+      end
+
+      # @rbs () -> bool
+      def extended_pragma?
+        @declarations.extended_pragma?
+      end
+
+      # @rbs () -> parser_section
+      def section
+        rules = @rules
+        rules ? rules.section : :declarations
+      end
+
+      # @rbs () -> Symbol
+      def state
+        @classifier.state
+      end
+
+      # @rbs () -> Token?
+      def group_opening
+        @rules&.group_opening
+      end
+
+      # @rbs () -> delimiter_kind?
+      def open_delimiter_kind
+        @rules&.open_delimiter_kind
+      end
+
+      # @rbs (Token? token) -> String?
+      def expectation(token)
+        @classifier.expectation(token)
+      end
+
+      private
+
+      # @rbs (Token token) -> external_token
+      def classify(token)
+        remaining = @tokens.drop(@index)
+        rules = @rules
+        return @declarations.classify(token, remaining) unless rules
+
+        rules.classify(token, remaining, last_external: @last_external,
+                                         previous_external: @previous_external)
+      end
+
+      # @rbs () -> void
+      def enter_rules
+        @rules = RuleState.new
+        @classifier = @rules
+      end
+    end
+  end
+end

data/lib/ibex/frontend.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require_relative "error"
+require_relative "tables"
+require_relative "runtime"
+require_relative "frontend/source_cursor"
+require_relative "frontend/action_scanner"
+require_relative "frontend/lexer"
+require_relative "frontend/ast"
+require_relative "frontend/token_adapter/declaration_state"
+require_relative "frontend/token_adapter/delimiter_tracker"
+require_relative "frontend/token_adapter/rule_state"
+require_relative "frontend/token_adapter"
+require_relative "frontend/generated_parser_base"
+require_relative "frontend/generated_parser"
+require_relative "frontend/parser"
+require_relative "frontend/dsl"
+
+module Ibex
+  module Frontend
+    # @rbs!
+    #   type user_code_token = { name: String, code: String }
+    #   type token_value = String | Integer | user_code_token | nil
+    #   type external_token = Symbol | String
+    #   type parser_section = :declarations | :rules | :user_code
+    #   type delimiter_kind = :group | :separated
+  end
+end

data/lib/ibex/ir/automaton_ir.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+module Ibex
+  module IR
+    # A merged LALR item with its lookahead token ids.
+    class AutomatonItem
+      attr_reader :production #: Integer
+      attr_reader :dot #: Integer
+      attr_reader :lookaheads #: Array[Integer]
+
+      # @rbs (production: Integer, dot: Integer, lookaheads: Array[Integer]) -> void
+      def initialize(production:, dot:, lookaheads:)
+        @production = production
+        @dot = dot
+        @lookaheads = lookaheads.sort.freeze
+        freeze
+      end
+
+      # @rbs (Grammar grammar) -> Hash[Symbol, untyped]
+      def to_h(grammar)
+        { production: @production, dot: @dot,
+          lookaheads: @lookaheads.map do |id|
+            symbol = grammar.symbol_by_id(id) || raise(Ibex::Error, "missing grammar symbol id #{id}")
+            symbol.name
+          end }
+      end
+    end
+
+    # A deterministic LALR automaton state.
+    class AutomatonState
+      attr_reader :id #: Integer
+      attr_reader :items #: Array[AutomatonItem]
+      attr_reader :transitions #: Hash[Integer, Integer]
+      attr_reader :actions #: Hash[Integer, parser_action]
+      attr_reader :gotos #: Hash[Integer, Integer]
+      attr_reader :default_action #: parser_action?
+      attr_reader :conflicts #: Array[conflict]
+
+      # @rbs (id: Integer, items: Array[AutomatonItem], transitions: Hash[Integer, Integer],
+      #   actions: Hash[Integer, parser_action], gotos: Hash[Integer, Integer], ?default_action: parser_action?,
+      #   ?conflicts: Array[conflict]) -> void
+      def initialize(id:, items:, transitions:, actions:, gotos:, default_action: nil, conflicts: [])
+        @id = id
+        @items = items.freeze
+        @transitions = IR.deep_freeze(transitions)
+        @actions = IR.deep_freeze(actions)
+        @gotos = IR.deep_freeze(gotos)
+        @default_action = IR.deep_freeze(default_action)
+        @conflicts = IR.deep_freeze(conflicts)
+        freeze
+      end
+
+      # @rbs (Grammar grammar) -> Hash[Symbol, untyped]
+      def to_h(grammar)
+        { id: @id, items: @items.map { |item| item.to_h(grammar) },
+          transitions: named_keys(@transitions, grammar), actions: named_keys(@actions, grammar),
+          gotos: named_keys(@gotos, grammar), default_action: @default_action, conflicts: @conflicts }
+      end
+
+      private
+
+      # @rbs (Hash[Integer, untyped] values, Grammar grammar) -> Hash[String, untyped]
+      def named_keys(values, grammar)
+        values.to_h do |symbol_id, value|
+          symbol = grammar.symbol_by_id(symbol_id) || raise(Ibex::Error, "missing grammar symbol id #{symbol_id}")
+          [symbol.name, value]
+        end
+      end
+    end
+
+    # Immutable LALR automaton and its source grammar.
+    class Automaton
+      attr_reader :algorithm #: String
+      attr_reader :grammar_digest #: String
+      attr_reader :grammar #: Grammar
+      attr_reader :states #: Array[AutomatonState]
+      attr_reader :conflict_summary #: conflict_summary
+      attr_reader :schema_version #: Integer
+
+      # @rbs (grammar: Grammar, states: Array[AutomatonState], conflict_summary: conflict_summary,
+      #   ?algorithm: String, ?grammar_digest: String?, ?schema_version: Integer) -> void
+      def initialize(grammar:, states:, conflict_summary:, algorithm: "lalr1", grammar_digest: nil,
+                     schema_version: SCHEMA_VERSION)
+        @algorithm = algorithm.freeze
+        @grammar = grammar
+        @grammar_digest = (grammar_digest || digest_for(grammar)).freeze
+        @states = states.freeze
+        @conflict_summary = IR.deep_freeze(conflict_summary)
+        @schema_version = schema_version
+        freeze
+      end
+
+      # @rbs () -> Hash[Symbol, untyped]
+      def to_h
+        { ibex_ir: "automaton", schema_version: @schema_version, algorithm: @algorithm,
+          grammar_digest: @grammar_digest, grammar: @grammar.to_h,
+          states: @states.map { |state| state.to_h(@grammar) }, conflict_summary: @conflict_summary }
+      end
+
+      private
+
+      # @rbs (Grammar grammar) -> String
+      def digest_for(grammar)
+        require "digest"
+        "sha256:#{Digest::SHA256.hexdigest(IR::Serialize.dump(grammar))}"
+      end
+    end
+  end
+end