ibex 0.1.0 → 0.3.0

data/lib/ibex/frontend/lexer.rb

@@ -3,17 +3,33 @@
 module Ibex
   module Frontend
     # Tokenizes a racc-compatible grammar while preserving source locations.
+    # rubocop:disable Metrics/ClassLength -- token scanners stay together to preserve cursor invariants.
     class Lexer
       PUNCTUATION = %w[: | ; = < > ? * + , ( )].to_h do |character|
         [character, character.to_sym]
       end.freeze #: Hash[String, Symbol]
       USER_CODE = /\A----[ \t]+(header|inner|footer)[ \t]*(?:\r?\n|\z)/ #: Regexp
+      NODE_DIRECTIVE = /\A@node(?=\s|\z)/ #: Regexp
+      PERCENT_DIRECTIVES = {
+        "%expect-rr" => [:identifier, "expect_rr"],
+        "%precedence" => [:identifier, "precedence"],
+        "%param" => [:identifier, "param"], "%printer" => [:identifier, "printer"],
+        "%recover" => [:identifier, "recover"], "%on_error_reduce" => [:identifier, "on_error_reduce"],
+        "%test" => [:identifier, "test"], "%inline" => [:inline, "%inline"],
+        "%empty" => [:empty, "%empty"]
+      }.freeze #: Hash[String, [Symbol, String]]
 
       # @rbs @cursor: SourceCursor
+      # @rbs @segments: Array[Segment]
+      # @rbs @emitted_tokens: Array[Token]
+      # @rbs @eof_token: Token?
+      # @rbs @active_token_start: SourcePosition?
 
       # @rbs (String source, ?file: String) -> void
       def initialize(source, file: "(grammar)")
         @cursor = SourceCursor.new(source, file)
+        @segments = [] #: Array[Segment]
+        @emitted_tokens = [] #: Array[Token]
       end
 
       # @rbs () -> Array[Token]
@@ -26,27 +42,55 @@ module Ibex
         end
       end
 
+      # @rbs () -> SourceDocument
+      def tokenize_document
+        tokenize unless @eof_token
+        cst = CST::Document.new(@segments)
+        SourceDocument.new(source: @cursor.source, file: @cursor.file, tokens: @emitted_tokens, cst: cst)
+      end
+
       # @rbs () -> Token
       def next_token
+        eof_token = @eof_token
+        return eof_token if eof_token
+
+        @active_token_start = nil
         skip_ignored
-        return token(:eof, nil) if @cursor.eof?
+        start = @cursor.position
+        @active_token_start = start
+        scanned = if @cursor.eof?
+                    token(:eof, nil)
+                  else
+                    scan_from_cursor
+                  end
+        emit_token(scanned, start) unless scanned.span
+        @emitted_tokens << scanned
+        @eof_token = scanned if scanned.type == :eof
+        @active_token_start = nil
+        scanned
+      end
 
+      private
+
+      # @rbs () -> Token
+      def scan_from_cursor
         character = @cursor.peek || ""
 
         return scan_user_code if line_start? && @cursor.rest.start_with?("----")
         return ActionScanner.new(@cursor).scan if character == "{"
         return scan_scope if @cursor.rest.start_with?("::")
+        return scan_percent_directive if @cursor.peek == "%"
+        return scan_node_directive if @cursor.rest.match?(NODE_DIRECTIVE)
 
         scan_regular_token(character)
       end
 
-      private
-
       # @rbs (String character) -> Token
       def scan_regular_token(character)
         return scan_identifier if character.match?(/[A-Za-z_]/)
         return scan_integer if character.match?(/\d/)
         return scan_literal if ["'", '"'].include?(character)
+        return scan_regexp if character == "/"
         return scan_punctuation if PUNCTUATION.key?(character)
 
         raise Ibex::Error, "#{@cursor.location}: unexpected character #{character.inspect}"
@@ -55,9 +99,12 @@ module Ibex
       # @rbs () -> void
       def skip_ignored
         loop do
-          @cursor.advance while @cursor.peek&.match?(/\s/)
-          if @cursor.peek == "#"
-            @cursor.advance until @cursor.eof? || @cursor.peek == "\n"
+          if newline?
+            scan_newline
+          elsif @cursor.peek&.match?(/\s/)
+            scan_whitespace
+          elsif @cursor.peek == "#"
+            scan_line_comment
           elsif @cursor.rest.start_with?("/*")
             skip_block_comment
           else
@@ -66,13 +113,41 @@ module Ibex
         end
       end
 
+      # @rbs () -> void
+      def scan_whitespace
+        start = @cursor.position
+        @cursor.advance while @cursor.peek&.match?(/\s/) && !newline?
+        emit_segment(:whitespace, start)
+      end
+
+      # @rbs () -> bool
+      def newline?
+        @cursor.peek == "\n" || @cursor.rest.start_with?("\r\n")
+      end
+
+      # @rbs () -> void
+      def scan_newline
+        start = @cursor.position
+        @cursor.advance(@cursor.rest.start_with?("\r\n") ? 2 : 1)
+        emit_segment(:newline, start)
+      end
+
+      # @rbs () -> void
+      def scan_line_comment
+        start = @cursor.position
+        @cursor.advance until @cursor.eof? || newline?
+        emit_segment(:line_comment, start)
+      end
+
       # @rbs () -> void
       def skip_block_comment
+        start = @cursor.position
         location = @cursor.location
         finish = @cursor.source.index("*/", @cursor.index + 2)
         raise Ibex::Error, "#{location}: unterminated block comment" unless finish
 
         @cursor.advance(finish + 2 - @cursor.index)
+        emit_segment(:block_comment, start)
       end
 
       # @rbs () -> bool
@@ -82,6 +157,7 @@ module Ibex
 
       # @rbs () -> Token
       def scan_user_code
+        start_position = @cursor.position
         location = @cursor.location
         match = @cursor.rest.match(USER_CODE)
         raise Ibex::Error, "#{location}: expected ---- header, inner, or footer" unless match
@@ -91,11 +167,16 @@ module Ibex
         raise Ibex::Error, "#{location}: expected ---- header, inner, or footer" unless name && marker
 
         @cursor.advance(marker.length)
+        emit_segment(
+          :user_code_marker, start_position, token_type: :user_code, token_index: @emitted_tokens.length
+        )
+        body_position = @cursor.position
         start = @cursor.index
         finish = @cursor.source.index(/^----/, start) || @cursor.source.length
         @cursor.advance(finish - start)
         code = @cursor.source[start...finish] || ""
-        token(:user_code, { name: name, code: code }, location)
+        emit_segment(:user_code_body, body_position)
+        token(:user_code, { name: name, code: code }, location, @cursor.span_from(start_position))
       end
 
       # @rbs () -> Token
@@ -105,6 +186,26 @@ module Ibex
         token(:scope, "::", location)
       end
 
+      # @rbs () -> Token
+      def scan_percent_directive
+        location = @cursor.location
+        directive = PERCENT_DIRECTIVES.each_key.find do |candidate|
+          @cursor.rest.match?(/\A#{Regexp.escape(candidate)}(?=\s|\z)/)
+        end
+        raise Ibex::Error, "#{location}: unexpected character \"%\"" unless directive
+
+        type, value = PERCENT_DIRECTIVES.fetch(directive)
+        @cursor.advance(directive.length)
+        token(type, value, location)
+      end
+
+      # @rbs () -> Token
+      def scan_node_directive
+        location = @cursor.location
+        @cursor.advance("@node".length)
+        token(:node, "@node", location)
+      end
+
       # @rbs () -> Token
       def scan_identifier
         scan_match(:identifier, /\A[A-Za-z_][A-Za-z0-9_]*/)
@@ -135,6 +236,37 @@ module Ibex
         raise Ibex::Error, "#{location}: unterminated quoted token"
       end
 
+      # @rbs () -> Token
+      # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+      def scan_regexp
+        location = @cursor.location
+        start = @cursor.index
+        @cursor.advance
+        in_class = false
+        until @cursor.eof?
+          character = @cursor.peek
+          if character == "\\"
+            @cursor.advance(2)
+          elsif character == "["
+            in_class = true
+            @cursor.advance
+          elsif character == "]" && in_class
+            in_class = false
+            @cursor.advance
+          elsif character == "/" && !in_class
+            @cursor.advance
+            @cursor.advance while @cursor.peek&.match?(/[imx]/)
+            return token(:regexp, @cursor.source[start...@cursor.index] || "", location)
+          elsif newline?
+            raise Ibex::Error, "#{location}: unterminated regular expression"
+          else
+            @cursor.advance
+          end
+        end
+        raise Ibex::Error, "#{location}: unterminated regular expression"
+      end
+      # rubocop:enable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+
       # @rbs () -> Token
       def scan_punctuation
         location = @cursor.location
@@ -156,10 +288,30 @@ module Ibex
         token(type, value, location)
       end
 
-      # @rbs (Symbol type, token_value value, ?Location location) -> Token
-      def token(type, value, location = @cursor.location)
-        Token.new(type: type, value: value, location: location)
+      # @rbs (Symbol type, token_value value, ?Location location, ?SourceSpan? span) -> Token
+      def token(type, value, location = @cursor.location, span = nil)
+        Token.new(type: type, value: value, location: location, span: span)
+      end
+
+      # @rbs (Token token, SourcePosition start) -> void
+      def emit_token(token, start)
+        token.span = @cursor.span_from(start)
+        kind = case token.type
+               when :action then :action
+               when :eof then :eof
+               else :token
+               end
+        emit_segment(kind, start, token_type: token.type, token_index: @emitted_tokens.length)
+      end
+
+      # @rbs (Symbol kind, SourcePosition start, ?token_type: Symbol?, ?token_index: Integer?) -> void
+      def emit_segment(kind, start, token_type: nil, token_index: nil)
+        span = @cursor.span_from(start)
+        text = @cursor.source.byteslice(span.start_byte, span.length) || ""
+        @segments << Segment.new(kind: kind, span: span, text: text,
+                                 token_type: token_type, token_index: token_index)
       end
     end
+    # rubocop:enable Metrics/ClassLength
   end
 end

data/lib/ibex/frontend/lexer_recovery.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+module Ibex
+  module Frontend
+    class Lexer
+      # Recover lexical failures while retaining every source byte.
+      # @rbs (?max_diagnostics: Integer) -> [SourceDocument, Array[Diagnostic]]
+      def tokenize_document_recovering(max_diagnostics: 20)
+        unless max_diagnostics.is_a?(Integer) && max_diagnostics.positive?
+          raise ArgumentError, "max_diagnostics must be a positive integer"
+        end
+
+        diagnostics = [] #: Array[Diagnostic]
+        until @eof_token
+          begin
+            next_token
+          rescue Ibex::Error => e
+            diagnostic = recover_lexical_error(e, capture: diagnostics.length < max_diagnostics)
+            diagnostics << diagnostic if diagnostic
+          end
+        end
+        cst = CST::Document.new(@segments)
+        document = SourceDocument.new(source: @cursor.source, file: @cursor.file,
+                                      tokens: @emitted_tokens, cst: cst)
+        [document, diagnostics.freeze]
+      end
+
+      private
+
+      # @rbs (Exception error, capture: bool) -> Diagnostic?
+      def recover_lexical_error(error, capture:)
+        start = @active_token_start || @cursor.position
+        advance_after_lexical_error(error, start)
+        emit_invalid_segment(start)
+        return unless capture
+
+        location, message = lexical_error_details(error, start)
+        Diagnostic.new(code: "frontend.lexical_error", phase: :lexical,
+                       message: message, location: location,
+                       span: @cursor.span_from(start), rendered: error.message)
+      ensure
+        @active_token_start = nil
+      end
+
+      # @rbs (Exception error, SourcePosition fallback) -> [Location, String]
+      def lexical_error_details(error, fallback)
+        pattern = /\A#{Regexp.escape(@cursor.file)}:(\d+):(\d+): (.*)\z/m
+        match = error.message.match(pattern)
+        if match
+          return [
+            Location.new(file: @cursor.file, line: Integer(match[1]), column: Integer(match[2])),
+            match[3] || ""
+          ]
+        end
+
+        location = Location.new(file: @cursor.file, line: fallback.line, column: fallback.column)
+        [location, error.message.delete_prefix("#{location}: ")]
+      end
+
+      # @rbs (SourcePosition start) -> void
+      def emit_invalid_segment(start)
+        span = @cursor.span_from(start)
+        previous = @segments.last
+        unless previous&.kind == :invalid && previous.span.end_byte == span.start_byte
+          emit_segment(:invalid, start)
+          return
+        end
+
+        combined_span = SourceSpan.new(file: @cursor.file, start: previous.span.start, finish: span.finish)
+        text = @cursor.source.byteslice(combined_span.start_byte, combined_span.length) || ""
+        @segments[-1] = Segment.new(kind: :invalid, span: combined_span, text: text)
+      end
+
+      # @rbs (Exception error, SourcePosition start) -> void
+      def advance_after_lexical_error(error, start)
+        return unless @cursor.position.byte_offset == start.byte_offset
+
+        remaining = @cursor.source.length - @cursor.index
+        count = error.message.include?("unterminated block comment") ? remaining : 1
+        @cursor.advance(count)
+      end
+    end
+  end
+end