ibex 0.1.0 → 0.2.0

data/lib/ibex/frontend/generated_parser_base.rb

@@ -1,34 +1,51 @@
 # frozen_string_literal: true
 
+require_relative "generated_parser_metadata"
+require_relative "generated_parser_includes"
+require_relative "generated_parser_parameters"
+
 module Ibex
   module Frontend
     # Semantic support and token delivery for the generated grammar parser.
+    # rubocop:disable Metrics/ClassLength -- generated grammar semantics share one parser support surface.
     class GeneratedParserBase < Runtime::Parser
+      include GeneratedParserMetadata
+      include GeneratedParserIncludes
+      include GeneratedParserParameters
+
+      attr_reader :diagnostic_token #: Token?
+
       # @rbs @adapter: TokenAdapter
       # @rbs @mode: Symbol
 
       # @rbs (Array[Token] tokens, ?mode: Symbol) -> void
-      def initialize(tokens, mode: :racc)
-        raise ArgumentError, "mode must be :racc or :extended" unless %i[racc extended].include?(mode)
+      def initialize(tokens, mode: :default)
+        raise ArgumentError, "mode must be :default or :extended" unless %i[default extended].include?(mode)
 
         super()
-        @adapter = TokenAdapter.new(tokens)
+        @adapter = TokenAdapter.new(tokens, extended: mode == :extended)
         @mode = mode
       end
 
-      # @rbs () -> AST::Root
+      # @rbs () -> (AST::Root | AST::Fragment)
       def parse
         do_parse
       end
 
       # @rbs () -> ([external_token, Token] | false)
       def next_token
-        @adapter.next_token
+        delivered = @adapter.next_token
+        @diagnostic_token = delivered ? delivered.fetch(1) : @adapter.eof_token
+        delivered
+      rescue Ibex::Error
+        @diagnostic_token = @adapter.last_token || @adapter.eof_token
+        raise
       end
 
       # @rbs (Integer? _token_id, Token? value, Array[untyped] _value_stack) -> bot
       def on_error(_token_id, value, _value_stack)
         token = value || @adapter.eof_token
+        @diagnostic_token = token
         raise_contextual_error(token)
 
         received = token&.value || token&.type || :eof
@@ -69,7 +86,7 @@ module Ibex
       def expected_description(token)
         expectation = @adapter.expectation(token)
         return expectation if expectation
-        return ")" if @adapter.open_delimiter_kind == :separated
+        return ")" if %i[separated parameter].include?(@adapter.open_delimiter_kind)
 
         structural_expectation(token)
       end
@@ -88,12 +105,30 @@ module Ibex
       #   Array[AST::declaration] declarations, Array[AST::Rule] rules, AST::user_code user_code) -> AST::Root
       def build_root(class_token, class_parts, superclass, declarations, rules, user_code)
         AST::Root.new(class_name: class_parts.join("::"), superclass: superclass&.join("::"),
-                      declarations: declarations, rules: rules, user_code: user_code, loc: class_token.location)
+                      declarations: declarations, rules: rules, user_code: user_code, loc: class_token.location,
+                      extended: @mode == :extended || @adapter.extended_pragma? || @adapter.cst_pragma?,
+                      cst: @adapter.cst_pragma?)
       end
 
-      # @rbs (Token keyword, Array[String] names) -> AST::Tokens
-      def build_tokens(keyword, names)
-        AST::Tokens.new(names: names, loc: keyword.location)
+      # @rbs (Token keyword, Array[[String, String?]] entries) -> AST::Tokens
+      def build_tokens(keyword, entries)
+        aliases = entries.filter_map { |name, value| [name, value] if value }.to_h
+        AST::Tokens.new(names: entries.map(&:first), aliases: aliases.empty? ? nil : aliases, loc: keyword.location)
+      end
+
+      # @rbs (Token token) -> [String, nil]
+      def build_token_entry(token)
+        [token_string(token), nil]
+      end
+
+      # @rbs (Token name, Token value) -> [String, String]
+      def build_token_alias(name, value)
+        extended_only!(name.location, "token aliases")
+        unless name.location.line == value.location.line
+          fail_at(name.location, "token alias must be written on one line")
+        end
+
+        [token_string(name), decode_metadata_value(value, "token alias")]
       end
 
       # @rbs (Token keyword, Symbol direction, Array[AST::PrecedenceLevel] levels) -> AST::Precedence
@@ -104,6 +139,7 @@ module Ibex
       # @rbs (Token association, Array[String] symbols) -> AST::PrecedenceLevel
       def build_precedence_level(association, symbols)
         fail_at(association.location, "expected at least one precedence symbol") if symbols.empty?
+        extended_only!(association.location, "%precedence") if token_string(association) == "precedence"
 
         AST::PrecedenceLevel.new(associativity: token_string(association).to_sym, symbols: symbols,
                                  loc: association.location)
@@ -119,9 +155,83 @@ module Ibex
         AST::Expect.new(conflicts: token_integer(integer), loc: keyword.location)
       end
 
-      # @rbs (Token keyword, String name) -> AST::Start
-      def build_start(keyword, name)
-        AST::Start.new(name: name, loc: keyword.location)
+      # @rbs (Token keyword, Token integer) -> AST::ExpectRR
+      def build_expect_rr(keyword, integer)
+        extended_only!(keyword.location, "expect-rr declarations")
+        AST::ExpectRR.new(conflicts: token_integer(integer), loc: keyword.location)
+      end
+
+      # @rbs (Token keyword, Token name, Token? type) -> AST::Parameter
+      def build_parameter(keyword, name, type)
+        extended_only!(keyword.location, "%param")
+        if type && (keyword.location.line != name.location.line || name.location.line != type.location.line)
+          fail_at(keyword.location, "%param declaration must be written on one line")
+        end
+
+        semantic_type = type && decode_metadata_value(type, "%param")
+        AST::Parameter.new(name: token_string(name), semantic_type: semantic_type, loc: keyword.location)
+      end
+
+      # @rbs (Token keyword, Token symbol, Token action) -> AST::Printer
+      def build_printer(keyword, symbol, action)
+        extended_only!(keyword.location, "%printer")
+        AST::Printer.new(name: token_string(symbol), code: token_string(action), loc: keyword.location)
+      end
+
+      # @rbs (Token keyword, Array[String] names) -> AST::Start
+      def build_start(keyword, names)
+        fail_at(keyword.location, "start declaration requires at least one symbol") if names.empty?
+        extended_only!(keyword.location, "multiple start symbols") if names.length > 1
+        AST::Start.new(names: names, loc: keyword.location)
+      end
+
+      # @rbs (Token keyword, Token kind, Array[String] sync_tokens) -> AST::Recovery
+      def build_recovery(keyword, kind, sync_tokens)
+        extended_only!(keyword.location, "%recover")
+        fail_at(kind.location, "expected sync, got #{kind.value}") unless token_string(kind) == "sync"
+        fail_at(keyword.location, "%recover sync requires at least one token") if sync_tokens.empty?
+        AST::Recovery.new(sync_tokens: sync_tokens, loc: keyword.location)
+      end
+
+      # @rbs (Token keyword, Array[String] names) -> AST::OnErrorReduce
+      def build_on_error_reduce(keyword, names)
+        extended_only!(keyword.location, "%on_error_reduce")
+        fail_at(keyword.location, "%on_error_reduce requires at least one nonterminal") if names.empty?
+        AST::OnErrorReduce.new(names: names, loc: keyword.location)
+      end
+
+      # @rbs (Token keyword, Token expectation, Token source) -> AST::GrammarTest
+      def build_grammar_test(keyword, expectation, source)
+        extended_only!(keyword.location, "%test")
+        expected = token_string(expectation)
+        fail_at(expectation.location, "expected accept or reject, got #{expected}") unless
+          %w[accept reject].include?(expected)
+        literal = token_string(source)
+        fail_at(source.location, "%test source must use a double-quoted string") unless literal.start_with?('"')
+
+        AST::GrammarTest.new(expectation: expected.to_sym, source: literal.undump, loc: keyword.location)
+      rescue RuntimeError => e
+        fail_at(source.location, "invalid %test source: #{e.message}")
+      end
+
+      # @rbs (Token keyword, Array[AST::lexer_entry] entries) -> AST::Lexer
+      def build_lexer(keyword, entries)
+        extended_only!(keyword.location, "lexer declarations")
+        AST::Lexer.new(definitions: entries, loc: keyword.location)
+      end
+
+      # @rbs (Symbol kind, Token? name, Token pattern, Token? action, Token marker) -> AST::LexerRule
+      def build_lexer_rule(kind, name, pattern, action, marker)
+        fail_at(marker.location, "on lexer rules require an action") if kind == :on && !action
+        AST::LexerRule.new(
+          kind: kind, token: name && token_string(name), pattern: token_string(pattern),
+          pattern_kind: pattern.type, action: action && token_string(action), loc: marker.location
+        )
+      end
+
+      # @rbs (Token marker, Token name, Array[AST::lexer_entry] entries) -> AST::LexerState
+      def build_lexer_state(marker, name, entries)
+        AST::LexerState.new(name: token_string(name), definitions: entries, loc: marker.location)
       end
 
       # @rbs (Token keyword, Array[AST::Conversion] pairs) -> AST::Convert
@@ -146,13 +256,8 @@ module Ibex
         fail_at(name_token.location, "invalid conversion expression: #{e.message}")
       end
 
-      # @rbs (Token lhs, Array[AST::Alternative] alternatives) -> AST::Rule
-      def build_rule(lhs, alternatives)
-        AST::Rule.new(lhs: token_string(lhs), alternatives: alternatives, loc: lhs.location)
-      end
-
-      # @rbs (Array[AST::item] items, Token? precedence) -> AST::Alternative
-      def build_alternative(items, precedence)
+      # @rbs (Array[AST::item] items, Token? precedence, AST::NodeAnnotation? node_annotation) -> AST::Alternative
+      def build_alternative(items, precedence, node_annotation)
         last_token = @adapter.last_token
         location = item_start_location(items.first) || precedence&.location || last_token&.location
         raise Ibex::Error, "missing alternative location" unless location
@@ -163,8 +268,24 @@ module Ibex
           items.pop
           action = last_item
         end
-        AST::Alternative.new(items: items, action: action,
-                             precedence: precedence && token_string(precedence), loc: location)
+        AST::Alternative.new(
+          items: items, action: action, precedence: precedence && token_string(precedence),
+          node_annotation: node_annotation, loc: location
+        )
+      end
+
+      # @rbs (Token marker, Token name, Array[Token] fields) -> AST::NodeAnnotation
+      def build_node_annotation(marker, name, fields)
+        extended_only!(marker.location, "@node")
+        AST::NodeAnnotation.new(
+          name: token_string(name), fields: fields.map { |field| token_string(field) }, loc: marker.location
+        )
+      end
+
+      # @rbs (Token token) -> AST::Empty
+      def build_empty(token)
+        extended_only!(token.location, "%empty")
+        AST::Empty.new(loc: token.location)
       end
 
       # @rbs (AST::item? item) -> Location?
@@ -180,16 +301,21 @@ module Ibex
 
       # @rbs (Token token, [Token, Token]? named_reference, Array[Token] suffixes) -> AST::item
       def build_symbol_reference(token, named_reference, suffixes)
-        if named_reference
-          colon, name = named_reference
-          extended_only!(colon.location, "named references")
-          named_reference = token_string(name)
-        end
-        item = AST::SymbolReference.new(name: token_string(token), named_reference: named_reference,
-                                        loc: token.location)
+        item = AST::SymbolReference.new(
+          name: token_string(token), named_reference: named_reference_value(named_reference), loc: token.location
+        )
         apply_suffixes(item, suffixes)
       end
 
+      # @rbs ([Token, Token]? named_reference) -> String?
+      def named_reference_value(named_reference)
+        return unless named_reference
+
+        colon, name = named_reference
+        extended_only!(colon.location, "named references")
+        token_string(name)
+      end
+
       # @rbs (Token token) -> AST::InlineAction
       def build_action(token)
         AST::InlineAction.new(code: token_string(token), loc: token.location)
@@ -228,6 +354,13 @@ module Ibex
         blocks
       end
 
+      # @rbs (Token token, AST::user_code blocks) -> AST::user_code
+      def prepend_user_code(token, blocks)
+        result = append_user_code(empty_user_code, token)
+        blocks.each { |name, chunks| result[name].concat(chunks) }
+        result
+      end
+
       # @rbs () -> AST::user_code
       def empty_user_code
         Hash.new { |hash, key| hash[key] = [] } #: AST::user_code
@@ -235,7 +368,7 @@ module Ibex
 
       # @rbs (Location location, String feature) -> void
       def extended_only!(location, feature)
-        return if @mode == :extended || @adapter.extended_pragma?
+        return if @mode == :extended || @adapter.extended_pragma? || @adapter.cst_pragma?
 
         fail_at(location, "#{feature} require extended mode")
       end
@@ -269,5 +402,6 @@ module Ibex
         fail_at(token.location, "expected user-code token")
       end
     end
+    # rubocop:enable Metrics/ClassLength
   end
 end

data/lib/ibex/frontend/generated_parser_includes.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module Ibex
+  module Frontend
+    # Semantic builders for fragment and include frontend nodes.
+    module GeneratedParserIncludes
+      private
+
+      # @rbs (Token keyword, Array[AST::declaration] declarations, Array[AST::Rule] rules,
+      #   AST::user_code user_code) -> AST::Fragment
+      def build_fragment(keyword, declarations, rules, user_code)
+        # @type self: GeneratedParserBase
+        extended_only!(keyword.location, "fragments")
+        reject_fragment_root_declarations(declarations)
+        user_block = user_code.values.flatten.first
+        fail_at(user_block.loc, "user code is not allowed in fragments") if user_block
+        AST::Fragment.new(declarations: declarations, rules: rules, loc: keyword.location)
+      end
+
+      # @rbs (Token keyword, Token path) -> AST::Include
+      def build_include(keyword, path)
+        # @type self: GeneratedParserBase
+        label = token_string(keyword)
+        extended_only!(keyword.location, "#{label}s")
+        value = token_string(path)
+        unless value.start_with?('"') && value.end_with?('"')
+          fail_at(path.location, "#{label} path must use a double-quoted string")
+        end
+
+        AST::Include.new(path: value.undump, loc: keyword.location)
+      rescue RuntimeError => e
+        fail_at(path.location, "invalid #{label} path: #{e.message}")
+      end
+
+      # @rbs (Array[AST::declaration] declarations) -> void
+      def reject_fragment_root_declarations(declarations)
+        # @type self: GeneratedParserBase
+        root_only = declarations.find { |declaration| fragment_root_declaration_name(declaration) }
+        return unless root_only
+
+        name = fragment_root_declaration_name(root_only)
+        fail_at(root_only.loc, "#{name} declarations are not allowed in fragments")
+      end
+
+      # @rbs (AST::declaration declaration) -> String?
+      def fragment_root_declaration_name(declaration)
+        return "options" if declaration.is_a?(AST::Options)
+        return "expect" if declaration.is_a?(AST::Expect)
+        return "%expect-rr" if declaration.is_a?(AST::ExpectRR)
+        return "%param" if declaration.is_a?(AST::Parameter)
+        return "start" if declaration.is_a?(AST::Start)
+        return "%recover" if declaration.is_a?(AST::Recovery)
+        return "%on_error_reduce" if declaration.is_a?(AST::OnErrorReduce)
+        return "%test" if declaration.is_a?(AST::GrammarTest)
+        return "lexer" if declaration.is_a?(AST::Lexer)
+
+        nil
+      end
+    end
+  end
+end

data/lib/ibex/frontend/generated_parser_metadata.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module Ibex
+  module Frontend
+    # Builds extended symbol metadata declarations for the generated grammar parser.
+    module GeneratedParserMetadata
+      private
+
+      # @rbs (Token keyword, Token name, Token value) -> AST::DisplayName
+      def build_display_name(keyword, name, value)
+        # @type self: GeneratedParserBase
+        extended_only!(keyword.location, "display declarations")
+        result = build_symbol_metadata(AST::DisplayName, keyword, name, value, "display")
+        return result if result.is_a?(AST::DisplayName)
+
+        raise Ibex::Error, "display metadata builder returned the wrong node"
+      end
+
+      # @rbs (Token keyword, Token name, Token value) -> AST::SemanticType
+      def build_semantic_type(keyword, name, value)
+        # @type self: GeneratedParserBase
+        extended_only!(keyword.location, "type declarations")
+        result = build_symbol_metadata(AST::SemanticType, keyword, name, value, "type")
+        return result if result.is_a?(AST::SemanticType)
+
+        raise Ibex::Error, "type metadata builder returned the wrong node"
+      end
+
+      # @rbs (singleton(AST::DisplayName) | singleton(AST::SemanticType) node_class,
+      #   Token keyword, Token name, Token value, String feature) -> (AST::DisplayName | AST::SemanticType)
+      def build_symbol_metadata(node_class, keyword, name, value, feature)
+        # @type self: GeneratedParserBase
+        unless keyword.location.line == name.location.line && name.location.line == value.location.line
+          fail_at(keyword.location, "#{feature} declaration must be written on one line")
+        end
+
+        decoded = decode_metadata_value(value, feature)
+        node_class.new(name: token_string(name), value: decoded, loc: keyword.location)
+      end
+
+      # @rbs (Token token, String feature) -> String
+      def decode_metadata_value(token, feature)
+        # @type self: GeneratedParserBase
+        literal = token_string(token)
+        decoded = if literal.start_with?('"')
+                    literal.undump
+                  else
+                    (literal[1...-1] || "").gsub("\\'", "'").gsub("\\\\", "\\")
+                  end
+        fail_at(token.location, "#{feature} value must not be empty") if decoded.strip.empty?
+        fail_at(token.location, "#{feature} value must be a single line") if decoded.match?(/[\r\n]/)
+        fail_at(token.location, "#{feature} value must not contain control characters") if
+          decoded.match?(/[[:cntrl:]]/)
+
+        decoded
+      rescue RuntimeError => e
+        fail_at(token.location, "invalid #{feature} value: #{e.message}")
+      end
+    end
+  end
+end

data/lib/ibex/frontend/generated_parser_parameters.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module Ibex
+  module Frontend
+    # Parameter-list and parameterized-reference builders for the generated frontend.
+    module GeneratedParserParameters
+      private
+
+      # @rbs (Token opening, Array[Token] parameters) -> [Token, Array[String]]
+      def build_rule_parameters(opening, parameters)
+        # @type self: GeneratedParserBase
+        extended_only!(opening.location, "parameterized rules")
+        [opening, parameters.map { |parameter| token_string(parameter) }]
+      end
+
+      # @rbs (Token? inline_marker, Token lhs, [Token, Array[String]]? parameter_clause,
+      #   Array[AST::Alternative] alternatives) -> AST::Rule
+      def build_rule(inline_marker, lhs, parameter_clause, alternatives)
+        # @type self: GeneratedParserBase
+        extended_only!(inline_marker.location, "inline rules") if inline_marker
+        parameters = [] #: Array[String]
+        if parameter_clause
+          opening, parameters = parameter_clause
+          require_adjacency!(lhs, opening, "parameter list must immediately follow rule name")
+        end
+        AST::Rule.new(
+          lhs: token_string(lhs), parameters: parameters, alternatives: alternatives, loc: lhs.location,
+          inline: !!inline_marker
+        )
+      end
+
+      # @rbs (Token callee, Token opening, Array[AST::item] arguments,
+      #   [Token, Token]? named_reference, Array[Token] suffixes) -> AST::item
+      def build_parameterized_reference(callee, opening, arguments, named_reference, suffixes)
+        # @type self: GeneratedParserBase
+        extended_only!(opening.location, "parameterized rules")
+        require_adjacency!(callee, opening, "parameter list must immediately follow rule name")
+        item = AST::ParameterizedReference.new(
+          name: token_string(callee), arguments: arguments, named_reference: named_reference_value(named_reference),
+          loc: callee.location
+        )
+        apply_suffixes(item, suffixes)
+      end
+
+      # @rbs (Token left, Token right, String message) -> void
+      def require_adjacency!(left, right, message)
+        # @type self: GeneratedParserBase
+        left_span = left.span
+        right_span = right.span
+        adjacent = if left_span && right_span
+                     left_span.end_byte == right_span.start_byte
+                   else
+                     left.location.line == right.location.line &&
+                       left.location.column + token_string(left).length == right.location.column
+                   end
+        fail_at(right.location, message) unless adjacent
+      end
+    end
+  end
+end

data/lib/ibex/frontend/generation.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+# Internal frontend surface needed by the ordinary grammar-to-Ruby pipeline.
+# Public `require "ibex/frontend"` continues to load formatting, diagnostics,
+# the DSL, and every other frontend facility.
+require_relative "../error"
+require_relative "../tables"
+require "ibex/runtime"
+require_relative "source_span"
+require_relative "source_cursor"
+require_relative "source_document"
+require_relative "diagnostic"
+require_relative "action_scanner"
+require_relative "lexer"
+require_relative "lexer_recovery"
+require_relative "ast"
+require_relative "rule_documentation"
+require_relative "token_adapter/declaration_document_state"
+require_relative "token_adapter/declaration_state"
+require_relative "token_adapter/delimiter_tracker"
+require_relative "token_adapter/rule_state"
+require_relative "token_adapter"
+require_relative "generated_parser_base"
+require_relative "generated_parser"
+require_relative "parser"
+require_relative "resolution"
+require_relative "source_loader"
+require_relative "resolver"
+
+module Ibex
+  module Frontend
+  end
+end