ibex 0.1.0

data/lib/ibex/frontend/ast.rb

@@ -0,0 +1,146 @@
+# frozen_string_literal: true
+
+module Ibex
+  module Frontend
+    # Grammar frontend node types.
+    module AST
+      # @rbs!
+      #   type declaration = Tokens | Precedence | Options | Expect | Start | Convert
+      #   type item = SymbolReference | InlineAction | Optional | Star | Plus | Group | SeparatedList
+      #   type user_code = Hash[String, Array[UserCode]]
+
+      # Adds deterministic, recursively serializable hashes to Struct nodes.
+      # @rbs module-self Struct[untyped]
+      module Node
+        def to_h
+          fields = each_pair.to_h { |name, value| [name, serialize(value)] }
+          { node: self.class.name.split("::").last }.merge(fields)
+        end
+
+        private
+
+        def serialize(value)
+          return value if value.nil?
+
+          case value
+          when Array then value.map { |item| serialize(item) }
+          when Hash then value.to_h { |key, item| [key, serialize(item)] }
+          else value.respond_to?(:to_h) ? value.to_h : value
+          end
+        end
+      end
+
+      Root = Struct.new(
+        :class_name, #: String
+        :superclass, #: String?
+        :declarations, #: Array[declaration]
+        :rules, #: Array[Rule]
+        :user_code, #: user_code
+        :loc, #: Location
+        keyword_init: true
+      ) do
+        include Node
+      end
+      Tokens = Struct.new(
+        :names, #: Array[String]
+        :loc, #: Location
+        keyword_init: true
+      ) { include Node }
+      Precedence = Struct.new(
+        :direction, #: Symbol
+        :levels, #: Array[PrecedenceLevel]
+        :loc, #: Location
+        keyword_init: true
+      ) { include Node }
+      PrecedenceLevel = Struct.new(
+        :associativity, #: Symbol
+        :symbols, #: Array[String]
+        :loc, #: Location
+        keyword_init: true
+      ) { include Node }
+      Options = Struct.new(
+        :names, #: Array[String]
+        :loc, #: Location
+        keyword_init: true
+      ) { include Node }
+      Expect = Struct.new(
+        :conflicts, #: Integer
+        :loc, #: Location
+        keyword_init: true
+      ) { include Node }
+      Start = Struct.new(
+        :name, #: String
+        :loc, #: Location
+        keyword_init: true
+      ) { include Node }
+      Convert = Struct.new(
+        :pairs, #: Array[Conversion]
+        :loc, #: Location
+        keyword_init: true
+      ) { include Node }
+      Conversion = Struct.new(
+        :name, #: String
+        :expression, #: String
+        :loc, #: Location
+        keyword_init: true
+      ) { include Node }
+      Rule = Struct.new(
+        :lhs, #: String
+        :alternatives, #: Array[Alternative]
+        :loc, #: Location
+        keyword_init: true
+      ) { include Node }
+      Alternative = Struct.new(
+        :items, #: Array[item]
+        :action, #: InlineAction?
+        :precedence, #: String?
+        :loc, #: Location
+        keyword_init: true
+      ) { include Node }
+      SymbolReference = Struct.new(
+        :name, #: String
+        :named_reference, #: String?
+        :loc, #: Location
+        keyword_init: true
+      ) { include Node }
+      InlineAction = Struct.new(
+        :code, #: String
+        :loc, #: Location
+        keyword_init: true
+      ) { include Node }
+      Optional = Struct.new(
+        :item, #: item
+        :loc, #: Location
+        keyword_init: true
+      ) { include Node }
+      Star = Struct.new(
+        :item, #: item
+        :loc, #: Location
+        keyword_init: true
+      ) { include Node }
+      Plus = Struct.new(
+        :item, #: item
+        :loc, #: Location
+        keyword_init: true
+      ) { include Node }
+      Group = Struct.new(
+        :alternatives, #: Array[Array[item]]
+        :loc, #: Location
+        keyword_init: true
+      ) { include Node }
+      SeparatedList = Struct.new(
+        :item, #: item
+        :separator, #: item
+        :nonempty, #: bool
+        :loc, #: Location
+        keyword_init: true
+      ) { include Node }
+      UserCode = Struct.new(
+        :name, #: String
+        :code, #: String
+        :loc, #: Location
+        keyword_init: true
+      ) { include Node }
+    end
+  end
+end

data/lib/ibex/frontend/bootstrap_parser.rb

@@ -0,0 +1,163 @@
+# frozen_string_literal: true
+
+module Ibex
+  module Frontend
+    # Handwritten parser used only to regenerate the self-hosted frontend.
+    class BootstrapParser
+      include BootstrapParserDeclarations
+      include BootstrapParserRules
+
+      # @rbs @tokens: Array[Token]
+      # @rbs @index: Integer
+      # @rbs @mode: Symbol
+
+      # @rbs (String | Array[Token] source, ?file: String, ?mode: Symbol) -> void
+      def initialize(source, file: "(grammar)", mode: :racc)
+        raise ArgumentError, "mode must be :racc or :extended" unless %i[racc extended].include?(mode)
+
+        @tokens = source.is_a?(Array) ? source : Lexer.new(source, file: file).tokenize
+        @index = 0
+        @mode = mode
+      end
+
+      # @rbs () -> AST::Root
+      def parse
+        location = expect_keyword("class").location
+        class_name = parse_constant_path
+        superclass = accept(:<) ? parse_constant_path : nil
+        parse_pragmas
+        declarations = parse_declarations
+        expect_keyword("rule")
+        rules = parse_rules
+        expect_keyword("end")
+        user_code = parse_user_code
+        expect(:eof)
+        AST::Root.new(class_name: class_name, superclass: superclass, declarations: declarations,
+                      rules: rules, user_code: user_code, loc: location)
+      end
+
+      private
+
+      # @rbs () -> String
+      def parse_constant_path
+        parts = [token_string(expect(:identifier))]
+        parts << token_string(expect(:identifier)) while accept(:scope)
+        parts.join("::")
+      end
+
+      # @rbs () -> String
+      def parse_symbol_name
+        token_string(expect_symbol)
+      end
+
+      # @rbs () -> Token
+      def expect_symbol
+        return advance if %i[identifier literal].include?(current.type)
+
+        fail_expected("a grammar symbol")
+      end
+
+      # @rbs (Array[String] values) -> Token
+      def expect_one_of(values)
+        return advance if current.type == :identifier && values.include?(current.value)
+
+        fail_expected(values.join(" or "))
+      end
+
+      # @rbs (String value) -> bool
+      def keyword?(value)
+        current.type == :identifier && current.value == value
+      end
+
+      # @rbs (String value) -> Token
+      def expect_keyword(value)
+        return advance if keyword?(value)
+
+        fail_expected(value)
+      end
+
+      # @rbs (Symbol type) -> (Token | false)
+      def accept(type)
+        return false unless current.type == type
+
+        advance
+      end
+
+      # @rbs (Symbol type) -> Token
+      def expect(type)
+        return advance if current.type == type
+
+        fail_expected(type)
+      end
+
+      # @rbs () -> Token
+      def current
+        @tokens.fetch(@index)
+      end
+
+      # @rbs () -> Token
+      def lookahead
+        @tokens.fetch(@index + 1) { @tokens.fetch(-1) }
+      end
+
+      # @rbs () -> Token
+      def advance
+        token = current
+        @index += 1
+        token
+      end
+
+      # @rbs () -> AST::user_code
+      def parse_user_code
+        blocks = Hash.new { |hash, key| hash[key] = Array.new(0) } #: AST::user_code
+        while current.type == :user_code
+          token = advance
+          value = token_user_code(token)
+          blocks[value[:name]] << AST::UserCode.new(name: value[:name], code: value[:code], loc: token.location)
+        end
+        blocks
+      end
+
+      # @rbs (Location location, String feature) -> void
+      def extended_only!(location, feature)
+        return if @mode == :extended
+
+        fail_at(location, "#{feature} require extended mode")
+      end
+
+      # @rbs (String | Symbol expected) -> bot
+      def fail_expected(expected)
+        fail_at(current.location, "expected #{expected}, got #{current.value || current.type}")
+      end
+
+      # @rbs (Location location, String message) -> bot
+      def fail_at(location, message)
+        raise Ibex::Error, "#{location}: #{message}"
+      end
+
+      # @rbs (Token token) -> String
+      def token_string(token)
+        value = token.value
+        return value if value.is_a?(String)
+
+        fail_at(token.location, "expected text token")
+      end
+
+      # @rbs (Token token) -> Integer
+      def token_integer(token)
+        value = token.value
+        return value if value.is_a?(Integer)
+
+        fail_at(token.location, "expected integer token")
+      end
+
+      # @rbs (Token token) -> user_code_token
+      def token_user_code(token)
+        value = token.value
+        return value if value.is_a?(Hash)
+
+        fail_at(token.location, "expected user-code token")
+      end
+    end
+  end
+end

data/lib/ibex/frontend/dsl.rb

@@ -0,0 +1,196 @@
+# frozen_string_literal: true
+
+module Ibex
+  module Frontend
+    # Builds the same Grammar AST as the text frontend through a Ruby API.
+    module DSL
+      # @rbs (class_name: Object, ?superclass: Object?, ?file: String) { (Builder) -> void } -> AST::Root
+      def grammar(class_name:, superclass: nil, file: "(dsl)")
+        builder = Builder.new(class_name: class_name, superclass: superclass, file: file)
+        yield builder
+        builder.to_ast
+      end
+      module_function :grammar
+
+      # Mutable definition context whose output is an immutable-shape AST.
+      class Builder
+        # @rbs @class_name: String
+        # @rbs @superclass: String?
+        # @rbs @file: String
+        # @rbs @line: Integer
+        # @rbs @declarations: Array[AST::declaration]
+        # @rbs @rules: Array[AST::Rule]
+        # @rbs @user_code: AST::user_code
+
+        # @rbs (class_name: Object, superclass: Object?, file: String) -> void
+        def initialize(class_name:, superclass:, file:)
+          @class_name = class_name.to_s
+          @superclass = superclass&.to_s
+          @file = file
+          @line = 1
+          @declarations = [] #: Array[AST::declaration]
+          @rules = [] #: Array[AST::Rule]
+          @user_code = Hash.new { |hash, key| hash[key] = Array.new(0) } #: AST::user_code
+        end
+
+        # @rbs (*Object names) -> void
+        def token(*names)
+          @declarations << AST::Tokens.new(names: names.map(&:to_s), loc: next_location)
+        end
+
+        # @rbs (*Object names) -> void
+        def options(*names)
+          @declarations << AST::Options.new(names: names.map(&:to_s), loc: next_location)
+        end
+
+        # @rbs (Integer conflicts) -> void
+        def expect(conflicts)
+          @declarations << AST::Expect.new(conflicts: conflicts, loc: next_location)
+        end
+
+        # @rbs (Object name) -> void
+        def start(name)
+          @declarations << AST::Start.new(name: name.to_s, loc: next_location)
+        end
+
+        # @rbs (Object name, Object expression) -> void
+        def convert(name, expression)
+          location = next_location
+          pair = AST::Conversion.new(name: name.to_s, expression: expression.to_s, loc: location)
+          @declarations << AST::Convert.new(pairs: [pair], loc: location)
+        end
+
+        # @rbs (?direction: Symbol) { (PrecedenceBuilder) -> void } -> void
+        def precedence(direction: :low_to_high)
+          location = next_location
+          builder = PrecedenceBuilder.new(self)
+          yield builder
+          @declarations << AST::Precedence.new(direction: direction, levels: builder.levels, loc: location)
+        end
+
+        # @rbs (Object lhs) { (RuleBuilder) -> void } -> void
+        def rule(lhs)
+          location = next_location
+          builder = RuleBuilder.new(self, location)
+          yield builder
+          @rules << AST::Rule.new(lhs: lhs.to_s, alternatives: builder.alternatives, loc: location)
+        end
+
+        # @rbs (Object name, Object code) -> void
+        def user_code(name, code)
+          key = name.to_s
+          @user_code[key] << AST::UserCode.new(name: key, code: code.to_s, loc: next_location)
+        end
+
+        # @rbs (Object name, ?as: Object?) -> AST::SymbolReference
+        def ref(name, as: nil)
+          AST::SymbolReference.new(name: name.to_s, named_reference: as&.to_s, loc: next_location)
+        end
+
+        # @rbs (Object item) -> AST::Optional
+        def optional(item) = AST::Optional.new(item: normalize_item(item), loc: next_location)
+        # @rbs (Object item) -> AST::Star
+        def star(item) = AST::Star.new(item: normalize_item(item), loc: next_location)
+        # @rbs (Object item) -> AST::Plus
+        def plus(item) = AST::Plus.new(item: normalize_item(item), loc: next_location)
+
+        # @rbs (*Object alternatives) -> AST::Group
+        def group(*alternatives)
+          normalized = alternatives.map { |alternative| Array(alternative).map { |item| normalize_item(item) } }
+          AST::Group.new(alternatives: normalized, loc: next_location)
+        end
+
+        # @rbs (Object item, Object separator, ?nonempty: bool) -> AST::SeparatedList
+        def separated_list(item, separator, nonempty: false)
+          AST::SeparatedList.new(item: normalize_item(item), separator: normalize_item(separator),
+                                 nonempty: nonempty, loc: next_location)
+        end
+
+        # @rbs (Object code) -> AST::InlineAction
+        def inline(code)
+          AST::InlineAction.new(code: code.to_s, loc: next_location)
+        end
+
+        # @rbs () -> AST::Root
+        def to_ast
+          location = Location.new(file: @file, line: 1, column: 1)
+          AST::Root.new(class_name: @class_name, superclass: @superclass, declarations: @declarations,
+                        rules: @rules, user_code: @user_code, loc: location)
+        end
+
+        # @rbs (Object item) -> AST::item
+        def normalize_item(item)
+          if item.respond_to?(:loc)
+            located_item = item #: AST::item
+            return located_item
+          end
+
+          ref(item)
+        end
+
+        # @rbs () -> Location
+        def next_location
+          location = Location.new(file: @file, line: @line, column: 1)
+          @line += 1
+          location
+        end
+      end
+
+      # Collects ordered associativity levels.
+      class PrecedenceBuilder
+        attr_reader :levels #: Array[AST::PrecedenceLevel]
+
+        # @rbs @grammar: Builder
+
+        # @rbs (Builder grammar) -> void
+        def initialize(grammar)
+          @grammar = grammar
+          @levels = [] #: Array[AST::PrecedenceLevel]
+        end
+
+        # @rbs (*Object symbols) -> void
+        def left(*symbols) = add_level(:left, symbols)
+
+        # @rbs (*Object symbols) -> void
+        def right(*symbols) = add_level(:right, symbols)
+
+        # @rbs (*Object symbols) -> void
+        def nonassoc(*symbols) = add_level(:nonassoc, symbols)
+
+        private
+
+        # @rbs (Symbol associativity, Array[Object] symbols) -> void
+        def add_level(associativity, symbols)
+          @levels << AST::PrecedenceLevel.new(associativity: associativity, symbols: symbols.map(&:to_s),
+                                              loc: @grammar.next_location)
+        end
+      end
+
+      # Collects alternatives for one nonterminal.
+      class RuleBuilder
+        attr_reader :alternatives #: Array[AST::Alternative]
+
+        # @rbs @grammar: Builder
+        # @rbs @default_location: Location
+
+        # @rbs (Builder grammar, Location default_location) -> void
+        def initialize(grammar, default_location)
+          @grammar = grammar
+          @default_location = default_location
+          @alternatives = [] #: Array[AST::Alternative]
+        end
+
+        # @rbs (*Object items, ?action: Object?, ?precedence: Object?) -> void
+        def alt(*items, action: nil, precedence: nil)
+          location = @grammar.next_location || @default_location
+          normalized = items.map { |item| @grammar.normalize_item(item) }
+          action_node = action && AST::InlineAction.new(code: action.to_s, loc: location)
+          alternative = AST::Alternative.new(
+            items: normalized, action: action_node, precedence: precedence&.to_s, loc: location
+          )
+          @alternatives << alternative
+        end
+      end
+    end
+  end
+end