ibex 0.1.0

data/lib/ibex/frontend/generated_parser_base.rb

@@ -0,0 +1,273 @@
+# frozen_string_literal: true
+
+module Ibex
+  module Frontend
+    # Semantic support and token delivery for the generated grammar parser.
+    class GeneratedParserBase < Runtime::Parser
+      # @rbs @adapter: TokenAdapter
+      # @rbs @mode: Symbol
+
+      # @rbs (Array[Token] tokens, ?mode: Symbol) -> void
+      def initialize(tokens, mode: :racc)
+        raise ArgumentError, "mode must be :racc or :extended" unless %i[racc extended].include?(mode)
+
+        super()
+        @adapter = TokenAdapter.new(tokens)
+        @mode = mode
+      end
+
+      # @rbs () -> AST::Root
+      def parse
+        do_parse
+      end
+
+      # @rbs () -> ([external_token, Token] | false)
+      def next_token
+        @adapter.next_token
+      end
+
+      # @rbs (Integer? _token_id, Token? value, Array[untyped] _value_stack) -> bot
+      def on_error(_token_id, value, _value_stack)
+        token = value || @adapter.eof_token
+        raise_contextual_error(token)
+
+        received = token&.value || token&.type || :eof
+        location = token&.location || Location.new(file: "(grammar)", line: 1, column: 1)
+        expected = expected_description(token)
+        raise Ibex::Error, "#{location}: expected #{expected}, got #{received}"
+      end
+
+      private
+
+      # @rbs (Token? token) -> void
+      def raise_contextual_error(token)
+        raise_group_contextual_error(token)
+        raise_conversion_contextual_error
+      end
+
+      # @rbs (Token? token) -> void
+      def raise_group_contextual_error(token)
+        group_opening = @adapter.group_opening
+        if group_opening && token && token.type == :action
+          fail_at(token.location, "actions inside EBNF groups are not supported")
+        end
+        group_unterminated = @adapter.open_delimiter_kind == :group &&
+                             (token.nil? || token.type == :eof || token.value == "end")
+        fail_at(group_opening.location, "unterminated EBNF group") if group_unterminated && group_opening
+      end
+
+      # @rbs () -> void
+      def raise_conversion_contextual_error
+        conversion_name = @adapter.conversion_name
+        return unless @adapter.declaration == :convert
+        return unless conversion_name
+
+        fail_at(conversion_name.location, "expected a quoted Ruby conversion expression")
+      end
+
+      # @rbs (Token? token) -> String
+      def expected_description(token)
+        expectation = @adapter.expectation(token)
+        return expectation if expectation
+        return ")" if @adapter.open_delimiter_kind == :separated
+
+        structural_expectation(token)
+      end
+
+      # @rbs (Token? token) -> String
+      def structural_expectation(token)
+        return "rule" if @adapter.section == :declarations
+        return "at least one rule" if @adapter.section == :user_code && token&.value == "end"
+        return "eof" if @adapter.section == :user_code
+        return "end" if @adapter.section == :rules && @adapter.eof_token
+
+        "grammar syntax"
+      end
+
+      # @rbs (Token class_token, Array[String] class_parts, Array[String]? superclass,
+      #   Array[AST::declaration] declarations, Array[AST::Rule] rules, AST::user_code user_code) -> AST::Root
+      def build_root(class_token, class_parts, superclass, declarations, rules, user_code)
+        AST::Root.new(class_name: class_parts.join("::"), superclass: superclass&.join("::"),
+                      declarations: declarations, rules: rules, user_code: user_code, loc: class_token.location)
+      end
+
+      # @rbs (Token keyword, Array[String] names) -> AST::Tokens
+      def build_tokens(keyword, names)
+        AST::Tokens.new(names: names, loc: keyword.location)
+      end
+
+      # @rbs (Token keyword, Symbol direction, Array[AST::PrecedenceLevel] levels) -> AST::Precedence
+      def build_precedence(keyword, direction, levels)
+        AST::Precedence.new(direction: direction, levels: levels, loc: keyword.location)
+      end
+
+      # @rbs (Token association, Array[String] symbols) -> AST::PrecedenceLevel
+      def build_precedence_level(association, symbols)
+        fail_at(association.location, "expected at least one precedence symbol") if symbols.empty?
+
+        AST::PrecedenceLevel.new(associativity: token_string(association).to_sym, symbols: symbols,
+                                 loc: association.location)
+      end
+
+      # @rbs (Token keyword, Array[String] names) -> AST::Options
+      def build_options(keyword, names)
+        AST::Options.new(names: names, loc: keyword.location)
+      end
+
+      # @rbs (Token keyword, Token integer) -> AST::Expect
+      def build_expect(keyword, integer)
+        AST::Expect.new(conflicts: token_integer(integer), loc: keyword.location)
+      end
+
+      # @rbs (Token keyword, String name) -> AST::Start
+      def build_start(keyword, name)
+        AST::Start.new(name: name, loc: keyword.location)
+      end
+
+      # @rbs (Token keyword, Array[AST::Conversion] pairs) -> AST::Convert
+      def build_convert(keyword, pairs)
+        AST::Convert.new(pairs: pairs, loc: keyword.location)
+      end
+
+      # @rbs (Token name_token, Token literal_token) -> AST::Conversion
+      def build_conversion(name_token, literal_token)
+        unless name_token.location.line == literal_token.location.line
+          fail_at(name_token.location, "expected a quoted Ruby conversion expression")
+        end
+
+        literal = token_string(literal_token)
+        expression = if literal.start_with?('"')
+                       literal.undump
+                     else
+                       (literal[1...-1] || "").gsub("\\'", "'").gsub("\\\\", "\\")
+                     end
+        AST::Conversion.new(name: token_string(name_token), expression: expression, loc: name_token.location)
+      rescue RuntimeError => e
+        fail_at(name_token.location, "invalid conversion expression: #{e.message}")
+      end
+
+      # @rbs (Token lhs, Array[AST::Alternative] alternatives) -> AST::Rule
+      def build_rule(lhs, alternatives)
+        AST::Rule.new(lhs: token_string(lhs), alternatives: alternatives, loc: lhs.location)
+      end
+
+      # @rbs (Array[AST::item] items, Token? precedence) -> AST::Alternative
+      def build_alternative(items, precedence)
+        last_token = @adapter.last_token
+        location = item_start_location(items.first) || precedence&.location || last_token&.location
+        raise Ibex::Error, "missing alternative location" unless location
+
+        action = nil #: AST::InlineAction?
+        last_item = items.last
+        if last_item.is_a?(AST::InlineAction)
+          items.pop
+          action = last_item
+        end
+        AST::Alternative.new(items: items, action: action,
+                             precedence: precedence && token_string(precedence), loc: location)
+      end
+
+      # @rbs (AST::item? item) -> Location?
+      def item_start_location(item)
+        return unless item
+
+        if item.is_a?(AST::Optional) || item.is_a?(AST::Star) || item.is_a?(AST::Plus)
+          return item_start_location(item.item)
+        end
+
+        item.loc
+      end
+
+      # @rbs (Token token, [Token, Token]? named_reference, Array[Token] suffixes) -> AST::item
+      def build_symbol_reference(token, named_reference, suffixes)
+        if named_reference
+          colon, name = named_reference
+          extended_only!(colon.location, "named references")
+          named_reference = token_string(name)
+        end
+        item = AST::SymbolReference.new(name: token_string(token), named_reference: named_reference,
+                                        loc: token.location)
+        apply_suffixes(item, suffixes)
+      end
+
+      # @rbs (Token token) -> AST::InlineAction
+      def build_action(token)
+        AST::InlineAction.new(code: token_string(token), loc: token.location)
+      end
+
+      # @rbs (Token opening, Array[Array[AST::item]] alternatives, Array[Token] suffixes) -> AST::item
+      def build_group(opening, alternatives, suffixes)
+        extended_only!(opening.location, "EBNF groups")
+        apply_suffixes(AST::Group.new(alternatives: alternatives, loc: opening.location), suffixes)
+      end
+
+      # @rbs (Token function, AST::item item, AST::item separator) -> AST::SeparatedList
+      def build_separated_list(function, item, separator)
+        extended_only!(function.location, "separated lists")
+        AST::SeparatedList.new(item: item, separator: separator,
+                               nonempty: function.value == "separated_nonempty_list", loc: function.location)
+      end
+
+      # @rbs (AST::item item, Array[Token] suffixes) -> AST::item
+      def apply_suffixes(item, suffixes)
+        suffixes.reduce(item) do |wrapped, suffix|
+          extended_only!(suffix.location, "EBNF suffixes")
+          case token_string(suffix)
+          when "?" then AST::Optional.new(item: wrapped, loc: suffix.location)
+          when "*" then AST::Star.new(item: wrapped, loc: suffix.location)
+          when "+" then AST::Plus.new(item: wrapped, loc: suffix.location)
+          else raise Ibex::Error, "#{suffix.location}: unknown EBNF suffix #{suffix.value.inspect}"
+          end
+        end
+      end
+
+      # @rbs (AST::user_code blocks, Token token) -> AST::user_code
+      def append_user_code(blocks, token)
+        value = token_user_code(token)
+        blocks[value[:name]] << AST::UserCode.new(name: value[:name], code: value[:code], loc: token.location)
+        blocks
+      end
+
+      # @rbs () -> AST::user_code
+      def empty_user_code
+        Hash.new { |hash, key| hash[key] = [] } #: AST::user_code
+      end
+
+      # @rbs (Location location, String feature) -> void
+      def extended_only!(location, feature)
+        return if @mode == :extended || @adapter.extended_pragma?
+
+        fail_at(location, "#{feature} require extended mode")
+      end
+
+      # @rbs (Location location, String message) -> bot
+      def fail_at(location, message)
+        raise Ibex::Error, "#{location}: #{message}"
+      end
+
+      # @rbs (Token token) -> String
+      def token_string(token)
+        value = token.value
+        return value if value.is_a?(String)
+
+        fail_at(token.location, "expected text token")
+      end
+
+      # @rbs (Token token) -> Integer
+      def token_integer(token)
+        value = token.value
+        return value if value.is_a?(Integer)
+
+        fail_at(token.location, "expected integer token")
+      end
+
+      # @rbs (Token token) -> user_code_token
+      def token_user_code(token)
+        value = token.value
+        return value if value.is_a?(Hash)
+
+        fail_at(token.location, "expected user-code token")
+      end
+    end
+  end
+end

data/lib/ibex/frontend/grammar.y

@@ -0,0 +1,156 @@
+class Ibex::Frontend::GeneratedParser < Ibex::Frontend::GeneratedParserBase
+token CLASS TOKEN PRECHIGH PRECLOW OPTIONS EXPECT START CONVERT PRAGMA RULE END
+token LEFT RIGHT NONASSOC IDENTIFIER LITERAL INTEGER ACTION USER_CODE LHS
+token SEPARATED_LIST SEPARATED_NONEMPTY_LIST
+rule
+  grammar
+    : CLASS constant_path superclass pragmas declarations RULE rules END user_code
+      { result = build_root(val[0], val[1], val[2], val[4], val[6], val[8]) }
+
+  constant_path
+    : IDENTIFIER                         { result = [val[0].value] }
+    | constant_path '::' IDENTIFIER      { result = val[0] + [val[2].value] }
+
+  superclass
+    :                                    { result = nil }
+    | '<' constant_path                  { result = val[1] }
+
+  pragmas
+    :
+    | pragmas PRAGMA IDENTIFIER
+
+  declarations
+    :                                    { result = Array.new(0) }
+    | declarations declaration           { result = val[0] + [val[1]] }
+
+  declaration
+    : token_declaration                  { result = val[0] }
+    | precedence_declaration             { result = val[0] }
+    | options_declaration                { result = val[0] }
+    | expect_declaration                 { result = val[0] }
+    | start_declaration                  { result = val[0] }
+    | convert_declaration                { result = val[0] }
+
+  token_declaration
+    : TOKEN symbols                      { result = build_tokens(val[0], val[1]) }
+
+  precedence_declaration
+    : PRECHIGH precedence_levels PRECLOW { result = build_precedence(val[0], :high_to_low, val[1]) }
+    | PRECLOW precedence_levels PRECHIGH { result = build_precedence(val[0], :low_to_high, val[1]) }
+
+  precedence_levels
+    :                                    { result = Array.new(0) }
+    | precedence_levels precedence_level { result = val[0] + [val[1]] }
+
+  precedence_level
+    : association symbols                { result = build_precedence_level(val[0], val[1]) }
+
+  association
+    : LEFT                               { result = val[0] }
+    | RIGHT                              { result = val[0] }
+    | NONASSOC                           { result = val[0] }
+
+  options_declaration
+    : OPTIONS identifiers                { result = build_options(val[0], val[1]) }
+
+  expect_declaration
+    : EXPECT INTEGER                     { result = build_expect(val[0], val[1]) }
+
+  start_declaration
+    : START grammar_symbol               { result = build_start(val[0], val[1].value) }
+
+  convert_declaration
+    : CONVERT conversions END            { result = build_convert(val[0], val[1]) }
+
+  conversions
+    :                                    { result = Array.new(0) }
+    | conversions conversion             { result = val[0] + [val[1]] }
+
+  conversion
+    : grammar_symbol LITERAL              { result = build_conversion(val[0], val[1]) }
+
+  identifiers
+    :                                    { result = Array.new(0) }
+    | identifiers IDENTIFIER              { result = val[0] + [val[1].value] }
+
+  symbols
+    :                                    { result = Array.new(0) }
+    | symbols grammar_symbol              { result = val[0] + [val[1].value] }
+
+  grammar_symbol
+    : IDENTIFIER                         { result = val[0] }
+    | LITERAL                            { result = val[0] }
+
+  rules
+    : rule_definition                    { result = [val[0]] }
+    | rules rule_definition              { result = val[0] + [val[1]] }
+
+  rule_definition
+    : LHS ':' alternatives semicolon     { result = build_rule(val[0], val[2]) }
+
+  semicolon
+    :
+    | ';'
+
+  alternatives
+    : alternative                        { result = [val[0]] }
+    | alternatives '|' alternative       { result = val[0] + [val[2]] }
+
+  alternative
+    : items precedence_override          { result = build_alternative(val[0], val[1]) }
+
+  precedence_override
+    :                                    { result = nil }
+    | '=' grammar_symbol                 { result = val[1] }
+
+  items
+    :                                    { result = Array.new(0) }
+    | items item                         { result = val[0] + [val[1]] }
+
+  item
+    : symbol_item                        { result = val[0] }
+    | ACTION                             { result = build_action(val[0]) }
+    | group_item                         { result = val[0] }
+    | separated_item                     { result = val[0] }
+
+  symbol_item
+    : grammar_symbol named_reference suffixes
+      { result = build_symbol_reference(val[0], val[1], val[2]) }
+
+  named_reference
+    :                                    { result = nil }
+    | ':' IDENTIFIER                     { result = [val[0], val[1]] }
+
+  suffixes
+    :                                    { result = Array.new(0) }
+    | suffixes '?'                       { result = val[0] + [val[1]] }
+    | suffixes '*'                       { result = val[0] + [val[1]] }
+    | suffixes '+'                       { result = val[0] + [val[1]] }
+
+  group_item
+    : '(' group_alternatives ')' suffixes
+      { result = build_group(val[0], val[1], val[3]) }
+
+  group_alternatives
+    : group_items                        { result = [val[0]] }
+    | group_alternatives '|' group_items { result = val[0] + [val[2]] }
+
+  group_items
+    :                                    { result = Array.new(0) }
+    | group_items group_item_element      { result = val[0] + [val[1]] }
+
+  group_item_element
+    : symbol_item                        { result = val[0] }
+    | group_item                         { result = val[0] }
+    | separated_item                     { result = val[0] }
+
+  separated_item
+    : SEPARATED_LIST '(' item ',' item ')'
+      { result = build_separated_list(val[0], val[2], val[4]) }
+    | SEPARATED_NONEMPTY_LIST '(' item ',' item ')'
+      { result = build_separated_list(val[0], val[2], val[4]) }
+
+  user_code
+    :                                    { result = empty_user_code }
+    | user_code USER_CODE                { result = append_user_code(val[0], val[1]) }
+end

data/lib/ibex/frontend/lexer.rb

@@ -0,0 +1,165 @@
+# frozen_string_literal: true
+
+module Ibex
+  module Frontend
+    # Tokenizes a racc-compatible grammar while preserving source locations.
+    class Lexer
+      PUNCTUATION = %w[: | ; = < > ? * + , ( )].to_h do |character|
+        [character, character.to_sym]
+      end.freeze #: Hash[String, Symbol]
+      USER_CODE = /\A----[ \t]+(header|inner|footer)[ \t]*(?:\r?\n|\z)/ #: Regexp
+
+      # @rbs @cursor: SourceCursor
+
+      # @rbs (String source, ?file: String) -> void
+      def initialize(source, file: "(grammar)")
+        @cursor = SourceCursor.new(source, file)
+      end
+
+      # @rbs () -> Array[Token]
+      def tokenize
+        tokens = [] #: Array[Token]
+        loop do
+          token = next_token
+          tokens << token
+          return tokens if token.type == :eof
+        end
+      end
+
+      # @rbs () -> Token
+      def next_token
+        skip_ignored
+        return token(:eof, nil) if @cursor.eof?
+
+        character = @cursor.peek || ""
+
+        return scan_user_code if line_start? && @cursor.rest.start_with?("----")
+        return ActionScanner.new(@cursor).scan if character == "{"
+        return scan_scope if @cursor.rest.start_with?("::")
+
+        scan_regular_token(character)
+      end
+
+      private
+
+      # @rbs (String character) -> Token
+      def scan_regular_token(character)
+        return scan_identifier if character.match?(/[A-Za-z_]/)
+        return scan_integer if character.match?(/\d/)
+        return scan_literal if ["'", '"'].include?(character)
+        return scan_punctuation if PUNCTUATION.key?(character)
+
+        raise Ibex::Error, "#{@cursor.location}: unexpected character #{character.inspect}"
+      end
+
+      # @rbs () -> void
+      def skip_ignored
+        loop do
+          @cursor.advance while @cursor.peek&.match?(/\s/)
+          if @cursor.peek == "#"
+            @cursor.advance until @cursor.eof? || @cursor.peek == "\n"
+          elsif @cursor.rest.start_with?("/*")
+            skip_block_comment
+          else
+            return
+          end
+        end
+      end
+
+      # @rbs () -> void
+      def skip_block_comment
+        location = @cursor.location
+        finish = @cursor.source.index("*/", @cursor.index + 2)
+        raise Ibex::Error, "#{location}: unterminated block comment" unless finish
+
+        @cursor.advance(finish + 2 - @cursor.index)
+      end
+
+      # @rbs () -> bool
+      def line_start?
+        @cursor.column == 1
+      end
+
+      # @rbs () -> Token
+      def scan_user_code
+        location = @cursor.location
+        match = @cursor.rest.match(USER_CODE)
+        raise Ibex::Error, "#{location}: expected ---- header, inner, or footer" unless match
+
+        name = match[1]
+        marker = match[0]
+        raise Ibex::Error, "#{location}: expected ---- header, inner, or footer" unless name && marker
+
+        @cursor.advance(marker.length)
+        start = @cursor.index
+        finish = @cursor.source.index(/^----/, start) || @cursor.source.length
+        @cursor.advance(finish - start)
+        code = @cursor.source[start...finish] || ""
+        token(:user_code, { name: name, code: code }, location)
+      end
+
+      # @rbs () -> Token
+      def scan_scope
+        location = @cursor.location
+        @cursor.advance(2)
+        token(:scope, "::", location)
+      end
+
+      # @rbs () -> Token
+      def scan_identifier
+        scan_match(:identifier, /\A[A-Za-z_][A-Za-z0-9_]*/)
+      end
+
+      # @rbs () -> Token
+      def scan_integer
+        scan_match(:integer, /\A\d+/) { |value| Integer(value, 10) }
+      end
+
+      # @rbs () -> Token
+      def scan_literal
+        location = @cursor.location
+        quote = @cursor.peek
+        start = @cursor.index
+        @cursor.advance
+        until @cursor.eof?
+          if @cursor.peek == "\\"
+            @cursor.advance(2)
+          elsif @cursor.peek == quote
+            @cursor.advance
+            value = @cursor.source[start...@cursor.index] || ""
+            return token(:literal, value, location)
+          else
+            @cursor.advance
+          end
+        end
+        raise Ibex::Error, "#{location}: unterminated quoted token"
+      end
+
+      # @rbs () -> Token
+      def scan_punctuation
+        location = @cursor.location
+        character = @cursor.peek
+        raise Ibex::Error, "#{location}: expected punctuation" unless character
+
+        @cursor.advance
+        token(PUNCTUATION.fetch(character), character, location)
+      end
+
+      # @rbs (Symbol type, Regexp pattern) ?{ (String) -> token_value } -> Token
+      def scan_match(type, pattern)
+        location = @cursor.location
+        value = @cursor.rest.match(pattern)&.[](0)
+        raise Ibex::Error, "#{location}: invalid #{type} token" unless value
+
+        @cursor.advance(value.length)
+        value = yield(value) if block_given?
+        token(type, value, location)
+      end
+
+      # @rbs (Symbol type, token_value value, ?Location location) -> Token
+      def token(type, value, location = @cursor.location)
+        Token.new(type: type, value: value, location: location)
+      end
+    end
+  end
+end