kumi-parser 0.0.32 → 0.1.0

data/lib/kumi/parser/lexer.rb

@@ -0,0 +1,232 @@
+# frozen_string_literal: true
+
+require 'strscan'
+
+module Kumi
+  module Parser
+    # Turns source text into a flat array of Tokens in a single StringScanner
+    # pass. Whitespace (except newlines, which are significant statement
+    # separators) is skipped; comments and newlines are emitted so the parser
+    # can ignore them uniformly. Every token carries its start offset, so all
+    # location and error-frame work is deferred to Source.
+    #
+    # The lexer is deliberately context-free: it does not track whether it is
+    # inside `input do … end`. Disambiguation that used to live in the old
+    # tokenizer's context stack is the parser's job now.
+    class Lexer
+      def initialize(source)
+        @source = source
+        @ss = StringScanner.new(source.text)
+        @tokens = []
+      end
+
+      def tokenize
+        until @ss.eos?
+          skip_inline_whitespace
+          break if @ss.eos?
+
+          offset = @ss.pos
+          scan_token(offset)
+        end
+        push(:eof, nil, @ss.pos)
+        @tokens
+      end
+
+      private
+
+      def scan_token(offset)
+        if @ss.scan(/\n/)
+          push(:newline, "\n", offset)
+        elsif @ss.scan(/#[^\n]*/)
+          push(:comment, @ss.matched[1..], offset)
+        elsif scan_negative_number(offset) || scan_number(offset) ||
+              scan_string(offset) || scan_word(offset) ||
+              scan_symbol_or_colon(offset) || scan_operator(offset)
+          nil
+        else
+          raise_lex_error("unexpected character #{@ss.peek(1).inspect}", offset)
+        end
+      end
+
+      def skip_inline_whitespace
+        @ss.skip(/[ \t\r]+/)
+      end
+
+      # ---- numbers -------------------------------------------------------
+
+      # `-5` is a single negative-number literal, but only in operand position:
+      # `x - 5` is subtraction, `shift(a, -1)` and `x - -5` carry a negative
+      # literal. We decide by the previous significant token — a `-` right after
+      # a value (number, name, `)`, `]`, …) is the binary operator and is left
+      # for scan_operator; anywhere else, `-<digit>` is a negative literal.
+      def scan_negative_number(offset)
+        return false unless @ss.check(/-\d/)
+        return false if value_ending_previous?
+
+        @ss.pos += 1
+        emit_number("-#{@ss.scan(/\d[\d_]*(\.\d[\d_]*)?/)}", offset)
+        true
+      end
+
+      # Tokens after which a `-` is the binary subtraction operator rather than
+      # the sign of a literal.
+      VALUE_ENDING_KINDS = %i[integer float string boolean symbol identifier constant rparen rbracket rbrace].freeze
+
+      def value_ending_previous?
+        VALUE_ENDING_KINDS.include?(@tokens.last&.kind)
+      end
+
+      # A number is digits with optional underscores and an optional single
+      # decimal point followed by digits (so `1..5` and `a.b` aren't swallowed).
+      def scan_number(offset)
+        return false unless @ss.check(/\d/)
+
+        emit_number(@ss.scan(/\d[\d_]*(\.\d[\d_]*)?/), offset)
+        true
+      end
+
+      def emit_number(matched, offset)
+        is_float = matched.include?('.')
+        clean = matched.delete('_')
+        push(is_float ? :float : :integer, is_float ? clean.to_f : clean.to_i, offset)
+      end
+
+      # ---- strings -------------------------------------------------------
+
+      def scan_string(offset)
+        quote = @ss.peek(1)
+        return false unless ['"', "'"].include?(quote)
+
+        @ss.pos += 1
+        buffer = +''
+        until @ss.eos?
+          ch = @ss.getch
+          case ch
+          when quote
+            push(:string, buffer, offset)
+            return true
+          when '\\'
+            buffer << read_escape
+          when "\n"
+            raise_lex_error('unterminated string literal', offset)
+          else
+            buffer << ch
+          end
+        end
+        raise_lex_error('unterminated string literal', offset)
+      end
+
+      ESCAPES = { 'n' => "\n", 't' => "\t", 'r' => "\r", '\\' => '\\', '"' => '"', "'" => "'" }.freeze
+
+      def read_escape
+        ch = @ss.getch
+        return '' if ch.nil?
+
+        ESCAPES.fetch(ch, ch)
+      end
+
+      # ---- words: keywords, types, sugar, identifiers, labels, constants -
+
+      def scan_word(offset)
+        return false unless @ss.check(/[A-Za-z_]/)
+
+        word = @ss.scan(/[A-Za-z_][A-Za-z0-9_]*/)
+        return true if scan_constant_path(word, offset)
+        return true if scan_label(word, offset)
+
+        emit_word(word, offset)
+        true
+      end
+
+      # Constant path: Foo::Bar::Baz (any word followed by one or more `::name`).
+      def scan_constant_path(word, offset)
+        return false unless @ss.check(/::/)
+
+        path = word.dup
+        while @ss.scan(/::/)
+          part = @ss.scan(/[A-Za-z_][A-Za-z0-9_]*/)
+          raise_lex_error("expected a name after '::'", @ss.pos) unless part
+          path << '::' << part
+        end
+        push(:constant, path, offset)
+        true
+      end
+
+      # Label: `name:` (but not `name::`, handled by the constant path above).
+      def scan_label(word, offset)
+        return false unless @ss.check(/:(?!:)/)
+
+        @ss.pos += 1
+        push(:label, word, offset)
+        true
+      end
+
+      def emit_word(word, offset)
+        if Grammar.boolean?(word)
+          push(:boolean, Grammar.boolean(word), offset)
+        elsif (kw = Grammar.keyword(word))
+          push(kw, word, offset)
+        elsif (type = Grammar.type_keyword(word))
+          push(:type_keyword, type, offset)
+        elsif (fn = Grammar.function_sugar(word))
+          push(:function_sugar, fn, offset)
+        else
+          push(:identifier, word, offset)
+        end
+      end
+
+      # ---- symbols and the bare colon ------------------------------------
+
+      def scan_symbol_or_colon(offset)
+        if @ss.scan(/:[A-Za-z_][A-Za-z0-9_]*/)
+          push(:symbol, @ss.matched[1..].to_sym, offset)
+          true
+        elsif @ss.scan(/:/)
+          push(:colon, ':', offset)
+          true
+        else
+          false
+        end
+      end
+
+      # ---- operators and punctuation -------------------------------------
+
+      # Ordered longest-first so multi-char operators win over their prefixes.
+      OPERATORS = [
+        ['...', :dot_dot_dot], ['..', :dot_dot],
+        ['**', :power], ['==', :eq], ['!=', :ne], ['>=', :gte], ['<=', :lte], ['=>', :arrow],
+        ['+', :add], ['-', :subtract], ['*', :multiply], ['/', :divide], ['%', :modulo],
+        ['>', :gt], ['<', :lt], ['&', :and], ['|', :or],
+        ['.', :dot], [',', :comma],
+        ['(', :lparen], [')', :rparen], ['[', :lbracket], [']', :rbracket], ['{', :lbrace], ['}', :rbrace]
+      ].freeze
+
+      def scan_operator(offset)
+        OPERATORS.each do |text, kind|
+          next unless @ss.scan(literal_regexp(text))
+
+          push(kind, text, offset)
+          return true
+        end
+        # A stray '=' is the classic typo for '=='; name it specifically.
+        raise_lex_error("unexpected '=' (did you mean '=='?)", offset) if @ss.peek(1) == '='
+        false
+      end
+
+      def literal_regexp(text)
+        @literal_regexps ||= {}
+        @literal_regexps[text] ||= /#{Regexp.escape(text)}/
+      end
+
+      # ---- helpers -------------------------------------------------------
+
+      def push(kind, value, offset)
+        @tokens << Token.new(kind, value, offset)
+      end
+
+      def raise_lex_error(message, offset)
+        raise ParseError.new(message, source: @source, offset: offset)
+      end
+    end
+  end
+end

data/lib/kumi/parser/parse_error.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Kumi
+  module Parser
+    # A syntax error detected during the parse phase — lexing or AST
+    # construction — and nothing past it. Parse errors are about *shape*: an
+    # unexpected character, a missing `end`, a hash key that isn't a symbol or
+    # string. Anything that needs to know what a name *means* (types, axes,
+    # whether a referenced declaration exists) is a semantic concern and belongs
+    # to kumi-core's analyzer, not here.
+    #
+    # The error knows the byte offset where parsing got stuck and a plain-English
+    # description of what was expected versus what was found. The Source renders
+    # the caret frame; we keep a reference so callers that want a fully formatted
+    # message (file:line:col + frame) can get one without re-deriving locations.
+    class ParseError < StandardError
+      attr_reader :source, :offset
+
+      def initialize(message, source:, offset:)
+        @source = source
+        @offset = offset
+        @short_message = message
+        super(build_message(message))
+      end
+
+      def location
+        source.location(offset)
+      end
+
+      def line
+        location.line
+      end
+
+      def column
+        location.column
+      end
+
+      # The bare "what/why" without location or frame — used where a caller
+      # wants to compose its own message.
+      attr_reader :short_message
+
+      private
+
+      def build_message(message)
+        line, col = source.line_col(offset)
+        frame = source.code_frame(offset)
+        header = "#{source.file}:#{line}:#{col}: #{message}"
+        frame.empty? ? header : "#{header}\n#{frame}"
+      end
+    end
+  end
+end