ibex 0.1.0

data/lib/ibex/frontend/parser/declarations.rb

@@ -0,0 +1,163 @@
+# frozen_string_literal: true
+
+module Ibex
+  module Frontend
+    # Parses the declaration section of a grammar.
+    module BootstrapParserDeclarations
+      DECLARATIONS = %w[token prechigh preclow options expect start convert pragma rule].freeze #: Array[String]
+      ASSOCIATIVITIES = %w[left right nonassoc].freeze #: Array[String]
+
+      private
+
+      # @rbs () -> Array[AST::declaration]
+      def parse_declarations
+        # @type self: BootstrapParser
+        declarations = [] #: Array[AST::declaration]
+        declarations << parse_declaration until keyword?("rule") || current.type == :eof
+        declarations
+      end
+
+      # @rbs () -> void
+      def parse_pragmas
+        # @type self: BootstrapParser
+        seen = false
+        while keyword?("pragma")
+          keyword = current
+          fail_at(keyword.location, "duplicate pragma extended") if seen
+
+          advance
+          value = expect(:identifier)
+          name = token_string(value)
+          fail_at(value.location, "unknown pragma #{name}") unless name == "extended"
+
+          @mode = :extended
+          seen = true
+        end
+      end
+
+      # @rbs () -> AST::declaration
+      def parse_declaration
+        # @type self: BootstrapParser
+        case current.value
+        when "token" then parse_tokens
+        when "prechigh", "preclow" then parse_precedence
+        when "options" then parse_options
+        when "expect" then parse_expect
+        when "start" then parse_start
+        when "convert" then parse_convert
+        when "pragma" then fail_at(current.location, "expected rule, got pragma")
+        else fail_expected("a declaration or rule")
+        end
+      end
+
+      # @rbs () -> AST::Tokens
+      def parse_tokens
+        # @type self: BootstrapParser
+        location = advance.location
+        names = [] #: Array[String]
+        names << parse_symbol_name until declaration_start?
+        AST::Tokens.new(names: names, loc: location)
+      end
+
+      # @rbs () -> AST::Precedence
+      def parse_precedence
+        # @type self: BootstrapParser
+        opening = advance
+        opening_name = token_string(opening)
+        closing = opening_name == "prechigh" ? "preclow" : "prechigh"
+        levels = precedence_levels(closing)
+        expect_keyword(closing)
+        AST::Precedence.new(direction: opening_name == "prechigh" ? :high_to_low : :low_to_high,
+                            levels: levels, loc: opening.location)
+      end
+
+      # @rbs (String closing) -> Array[AST::PrecedenceLevel]
+      def precedence_levels(closing)
+        # @type self: BootstrapParser
+        levels = [] #: Array[AST::PrecedenceLevel]
+        until keyword?(closing) || current.type == :eof
+          association = expect_one_of(ASSOCIATIVITIES)
+          symbols = [] #: Array[String]
+          symbols << parse_symbol_name until association_start? || keyword?(closing) || current.type == :eof
+          fail_at(association.location, "expected at least one precedence symbol") if symbols.empty?
+          levels << AST::PrecedenceLevel.new(associativity: token_string(association).to_sym, symbols: symbols,
+                                             loc: association.location)
+        end
+        levels
+      end
+
+      # @rbs () -> AST::Options
+      def parse_options
+        # @type self: BootstrapParser
+        location = advance.location
+        names = [] #: Array[String]
+        names << token_string(expect(:identifier)) until declaration_start?
+        AST::Options.new(names: names, loc: location)
+      end
+
+      # @rbs () -> AST::Expect
+      def parse_expect
+        # @type self: BootstrapParser
+        location = advance.location
+        AST::Expect.new(conflicts: token_integer(expect(:integer)), loc: location)
+      end
+
+      # @rbs () -> AST::Start
+      def parse_start
+        # @type self: BootstrapParser
+        location = advance.location
+        AST::Start.new(name: parse_symbol_name, loc: location)
+      end
+
+      # @rbs () -> AST::Convert
+      def parse_convert
+        # @type self: BootstrapParser
+        location = advance.location
+        pairs = [] #: Array[AST::Conversion]
+        until keyword?("end") || current.type == :eof
+          name_token = current
+          name = parse_symbol_name
+          expression = decode_conversion(tokens_on_line(name_token.location.line), name_token.location)
+          pairs << AST::Conversion.new(name: name, expression: expression, loc: name_token.location)
+        end
+        expect_keyword("end")
+        AST::Convert.new(pairs: pairs, loc: location)
+      end
+
+      # @rbs (Integer line) -> Array[Token]
+      def tokens_on_line(line)
+        # @type self: BootstrapParser
+        tokens = [] #: Array[Token]
+        tokens << advance while current.type != :eof && current.location.line == line && !keyword?("end")
+        tokens
+      end
+
+      # @rbs (Array[Token] tokens, Location location) -> String
+      def decode_conversion(tokens, location)
+        # @type self: BootstrapParser
+        unless tokens.length == 1 && tokens.first.type == :literal
+          fail_at(location, "expected a quoted Ruby conversion expression")
+        end
+
+        literal = token_string(tokens.first)
+        return literal.undump if literal.start_with?('"')
+
+        (literal[1...-1] || "").gsub("\\'", "'").gsub("\\\\", "\\")
+      rescue RuntimeError => e
+        fail_at(location, "invalid conversion expression: #{e.message}")
+      end
+
+      # @rbs () -> bool
+      def declaration_start?
+        # @type self: BootstrapParser
+        current.type == :eof || (current.type == :identifier && DECLARATIONS.include?(current.value))
+      end
+
+      # @rbs () -> bool
+      def association_start?
+        # @type self: BootstrapParser
+        current.type == :identifier && ASSOCIATIVITIES.include?(current.value)
+      end
+    end
+  end
+end

data/lib/ibex/frontend/parser/rules.rb

@@ -0,0 +1,154 @@
+# frozen_string_literal: true
+
+module Ibex
+  module Frontend
+    # Parses productions and extended RHS items.
+    module BootstrapParserRules
+      EXTENDED_SUFFIXES = {
+        "?": AST::Optional, "*": AST::Star, "+": AST::Plus
+      }.freeze #: Hash[Symbol, singleton(AST::Optional) | singleton(AST::Star) | singleton(AST::Plus)]
+
+      private
+
+      # @rbs () -> Array[AST::Rule]
+      def parse_rules
+        # @type self: BootstrapParser
+        rules = [] #: Array[AST::Rule]
+        rules << parse_rule until keyword?("end") || current.type == :eof
+        fail_expected("at least one rule") if rules.empty?
+        rules
+      end
+
+      # @rbs () -> AST::Rule
+      def parse_rule
+        # @type self: BootstrapParser
+        lhs = expect(:identifier)
+        expect(:":")
+        alternatives = [] #: Array[AST::Alternative]
+        loop do
+          alternatives << parse_alternative(lhs)
+          next if accept(:|)
+
+          accept(:";")
+          break
+        end
+        AST::Rule.new(lhs: token_string(lhs), alternatives: alternatives, loc: lhs.location)
+      end
+
+      # @rbs (Token lhs) -> AST::Alternative
+      def parse_alternative(lhs)
+        # @type self: BootstrapParser
+        location = current.location
+        items = [] #: Array[AST::item]
+        precedence = nil #: String?
+        until alternative_end?(lhs)
+          if accept(:"=")
+            precedence = parse_symbol_name
+            break
+          end
+          items << parse_item
+        end
+        action = nil #: AST::InlineAction?
+        last_item = items.last
+        if last_item.is_a?(AST::InlineAction)
+          items.pop
+          action = last_item
+        end
+        AST::Alternative.new(items: items, action: action, precedence: precedence, loc: location)
+      end
+
+      # @rbs () -> AST::item
+      def parse_item
+        # @type self: BootstrapParser
+        return parse_action if current.type == :action
+        return parse_separated_list if separated_list?
+        return parse_group if current.type == :"("
+
+        token = expect_symbol
+        named_reference = parse_named_reference
+        item = AST::SymbolReference.new(name: token_string(token), named_reference: named_reference,
+                                        loc: token.location)
+        parse_suffix(item)
+      end
+
+      # @rbs () -> AST::InlineAction
+      def parse_action
+        # @type self: BootstrapParser
+        token = advance
+        AST::InlineAction.new(code: token_string(token), loc: token.location)
+      end
+
+      # @rbs () -> String?
+      def parse_named_reference
+        # @type self: BootstrapParser
+        return nil unless current.type == :":"
+
+        extended_only!(current.location, "named references")
+        advance
+        token_string(expect(:identifier))
+      end
+
+      # @rbs (AST::item item) -> AST::item
+      def parse_suffix(item)
+        # @type self: BootstrapParser
+        while (wrapper = EXTENDED_SUFFIXES[current.type])
+          extended_only!(current.location, "EBNF suffixes")
+          item = wrapper.new(item: item, loc: advance.location)
+        end
+        item
+      end
+
+      # @rbs () -> AST::item
+      def parse_group
+        # @type self: BootstrapParser
+        opening = advance
+        extended_only!(opening.location, "EBNF groups")
+        alternatives = [[]] #: Array[Array[AST::item]]
+        until current.type == :")"
+          fail_at(opening.location, "unterminated EBNF group") if current.type == :eof || keyword?("end")
+          if accept(:|)
+            alternatives << []
+            next
+          end
+          fail_at(current.location, "actions inside EBNF groups are not supported") if current.type == :action
+
+          alternatives.last << parse_item
+        end
+        expect(:")")
+        parse_suffix(AST::Group.new(alternatives: alternatives, loc: opening.location))
+      end
+
+      # @rbs () -> AST::SeparatedList
+      def parse_separated_list
+        # @type self: BootstrapParser
+        function = advance
+        extended_only!(function.location, "separated lists")
+        expect(:"(")
+        item = parse_item
+        expect(:",")
+        separator = parse_item
+        expect(:")")
+        AST::SeparatedList.new(item: item, separator: separator,
+                               nonempty: function.value == "separated_nonempty_list", loc: function.location)
+      end
+
+      # @rbs (Token lhs) -> bool
+      def alternative_end?(lhs)
+        # @type self: BootstrapParser
+        %i[| ; eof].include?(current.type) || keyword?("end") || rule_start?(lhs)
+      end
+
+      # @rbs (Token lhs) -> bool
+      def rule_start?(lhs)
+        # @type self: BootstrapParser
+        current.type == :identifier && lookahead.type == :":" && current.location.column <= lhs.location.column
+      end
+
+      # @rbs () -> bool
+      def separated_list?
+        # @type self: BootstrapParser
+        %w[separated_list separated_nonempty_list].include?(current.value) && lookahead.type == :"("
+      end
+    end
+  end
+end

data/lib/ibex/frontend/parser.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Ibex
+  module Frontend
+    # Public grammar parser backed by Ibex's generated LR frontend.
+    class Parser
+      attr_reader :implementation #: GeneratedParser
+
+      # @rbs (String | Array[Token] source, ?file: String, ?mode: Symbol) -> void
+      def initialize(source, file: "(grammar)", mode: :racc)
+        raise ArgumentError, "mode must be :racc or :extended" unless %i[racc extended].include?(mode)
+
+        tokens = source.is_a?(Array) ? source : Lexer.new(source, file: file).tokenize
+        @implementation = GeneratedParser.new(tokens, mode: mode)
+      end
+
+      # @rbs () -> AST::Root
+      def parse
+        @implementation.parse
+      end
+    end
+  end
+end

data/lib/ibex/frontend/regenerator.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require_relative "../error"
+require_relative "../tables"
+require_relative "source_cursor"
+require_relative "action_scanner"
+require_relative "lexer"
+require_relative "ast"
+require_relative "parser/declarations"
+require_relative "parser/rules"
+require_relative "bootstrap_parser"
+require_relative "../ir"
+require_relative "../normalize"
+require_relative "../analysis"
+require_relative "../lalr"
+require_relative "../codegen/ruby"
+
+module Ibex
+  module Frontend
+    # Builds the committed frontend parser from its canonical Ibex grammar.
+    module Regenerator
+      GRAMMAR_PATH = File.expand_path("grammar.y", File.dirname(__FILE__)) #: String
+
+      module_function
+
+      # @rbs () -> String
+      def generate
+        source = File.read(GRAMMAR_PATH)
+        ast = BootstrapParser.new(source, file: relative_grammar_path).parse
+        grammar = Normalizer.new(ast).normalize
+        automaton = LALR::Builder.new(grammar).build
+        Codegen::Ruby.new(automaton, table: :compact, line_convert: false).generate
+      end
+
+      # @rbs () -> String
+      def relative_grammar_path
+        "lib/ibex/frontend/grammar.y"
+      end
+    end
+  end
+end

data/lib/ibex/frontend/source_cursor.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+module Ibex
+  module Frontend
+    # A source coordinate retained by frontend and IR objects.
+    Location = Struct.new(
+      :file, #: String
+      :line, #: Integer
+      :column, #: Integer
+      keyword_init: true
+    )
+
+    class Location
+      # @rbs () -> IR::location
+      def to_h
+        { file: file, line: line, column: column }
+      end
+
+      # @rbs () -> String
+      def to_s
+        "#{file}:#{line}:#{column}"
+      end
+    end
+
+    # A grammar token and its source coordinate.
+    Token = Struct.new(
+      :type, #: Symbol
+      :value, #: token_value
+      :location, #: Location
+      keyword_init: true
+    )
+
+    class Token
+      # @rbs () -> Hash[Symbol, untyped]
+      def to_h
+        { type: type, value: value, location: location.to_h }
+      end
+    end
+
+    # Advances through source text while maintaining one-based coordinates.
+    class SourceCursor
+      attr_reader :source #: String
+      attr_reader :file #: String
+      attr_reader :index #: Integer
+      attr_reader :line #: Integer
+      attr_reader :column #: Integer
+
+      # @rbs (String source, String file) -> void
+      def initialize(source, file)
+        @source = source
+        @file = file
+        @index = 0
+        @line = 1
+        @column = 1
+      end
+
+      # @rbs () -> bool
+      def eof?
+        @index >= @source.length
+      end
+
+      # @rbs (?Integer offset) -> String?
+      def peek(offset = 0)
+        @source[@index + offset]
+      end
+
+      # @rbs () -> String
+      def rest
+        @source[@index..] || ""
+      end
+
+      # @rbs () -> Location
+      def location
+        Location.new(file: @file, line: @line, column: @column)
+      end
+
+      # @rbs (?Integer count) -> void
+      def advance(count = 1)
+        count.times do
+          character = @source[@index]
+          break unless character
+
+          @index += 1
+          if character == "\n"
+            @line += 1
+            @column = 1
+          else
+            @column += 1
+          end
+        end
+      end
+    end
+  end
+end