fusion-lang 0.0.1.alpha1 → 0.0.1

data/lib/fusion/lexer.rb

@@ -0,0 +1,249 @@
+# frozen_string_literal: true
+
+# === Transformation ===
+#
+# Input: String (source code)
+# Output: Array<Token>
+
+require_relative "token"
+require_relative "null"
+
+module Fusion
+  class Lexer
+    PUNCT = {
+      "(" => :lparen, ")" => :rparen,
+      "[" => :lbracket, "]" => :rbracket,
+      "{" => :lbrace, "}" => :rbrace,
+      "," => :comma, ":" => :colon,
+      "?" => :question, "." => :dot,
+      "@" => :at, "/" => :slash,
+      "=" => :equals,
+      "+" => :plus, "-" => :minus, "*" => :star,
+      "%" => :percent, "~" => :tilde,
+    }.freeze
+
+    def initialize(src)
+      @src = src
+      @i = 0
+      @n = src.length
+    end
+
+    def tokens
+      out = []
+      loop do
+        t = next_token
+        out << t
+        # A file-reference path is one tight token, lexed only right after `@`/`@@`.
+        if t.type == :at || t.type == :atat
+          p = try_lex_path
+          out << p unless p.nil?
+        end
+        break if t.type == :eof
+      end
+      out
+    end
+
+    private
+
+    def peek(o = 0) = @i + o < @n ? @src[@i + o] : nil
+
+    def next_token
+      skip_trivia
+      start = @i
+      c = peek
+      return Token.new(type: :eof, value: nil, pos: start) if c.nil?
+
+      # "=>" and "..." handled specially ("#" line comments handled in skip_trivia)
+      if c == "=" && peek(1) == ">"
+        @i += 2
+        return Token.new(type: :arrow, value: "=>", pos: start)
+      end
+      if c == "." && peek(1) == "." && peek(2) == "."
+        @i += 3
+        return Token.new(type: :spread, value: "...", pos: start)
+      end
+      if c == "@" && peek(1) == "@"
+        @i += 2
+        return Token.new(type: :atat, value: "@@", pos: start)
+      end
+      # Two-character operators, matched before their single-char prefixes.
+      if c == "=" && peek(1) == "="
+        @i += 2
+        return Token.new(type: :eqeq, value: "==", pos: start)
+      end
+      if c == "/" && peek(1) == "/"
+        @i += 2
+        return Token.new(type: :slashslash, value: "//", pos: start)
+      end
+      if c == "?" && peek(1) == "?"
+        @i += 2
+        return Token.new(type: :qq, value: "??", pos: start)
+      end
+      if c == "&" && peek(1) == "&"
+        @i += 2
+        return Token.new(type: :andand, value: "&&", pos: start)
+      end
+      if c == "|"
+        case peek(1)
+        when "|" then @i += 2; return Token.new(type: :oror, value: "||", pos: start)
+        when ":" then @i += 2; return Token.new(type: :pipemap, value: "|:", pos: start)
+        when "?" then @i += 2; return Token.new(type: :pipefilter, value: "|?", pos: start)
+        when "+" then @i += 2; return Token.new(type: :pipereduce, value: "|+", pos: start)
+        else @i += 1; return Token.new(type: :pipe, value: "|", pos: start)
+        end
+      end
+      if c == "!"
+        @i += 1
+        return Token.new(type: :bang, value: "!", pos: start)
+      end
+      if c == '"'
+        return lex_string(start)
+      end
+      if digit?(c)
+        return lex_number(start)
+      end
+      if ident_start?(c)
+        return lex_word(start)
+      end
+      if (type = PUNCT[c])
+        @i += 1
+        return Token.new(type: type, value: c, pos: start)
+      end
+      raise ParseError, "Unexpected character #{c.inspect} at #{start}"
+    end
+
+    def skip_trivia
+      loop do
+        c = peek
+        if c == " " || c == "\t" || c == "\n" || c == "\r"
+          @i += 1
+        elsif c == "#" && at_line_start?
+          # A line is a comment iff its first non-whitespace char is "#".
+          # This also covers shebang lines (#!) for free.
+          @i += 1 until peek.nil? || peek == "\n"
+        else
+          break
+        end
+      end
+    end
+
+    # True when only whitespace precedes @i on the current physical line.
+    def at_line_start?
+      j = @i - 1
+      j -= 1 while j >= 0 && (@src[j] == " " || @src[j] == "\t")
+      j < 0 || @src[j] == "\n" || @src[j] == "\r"
+    end
+
+    def lex_string(start)
+      @i += 1 # opening quote
+      buf = +""
+      while (c = peek)
+        if c == '"'
+          @i += 1
+          return Token.new(type: :string, value: buf, pos: start)
+        elsif c == "\\"
+          @i += 1
+          e = peek
+          buf << case e
+                 when '"' then '"'
+                 when "\\" then "\\"
+                 when "/" then "/"
+                 when "n" then "\n"
+                 when "t" then "\t"
+                 when "r" then "\r"
+                 when "b" then "\b"
+                 when "f" then "\f"
+                 when "u"
+                   hex = @src[@i + 1, 4]
+                   @i += 4
+                   code_point = hex.to_i(16)
+                   # Reject surrogates and out-of-range code points up front:
+                   # pack("U") would otherwise build an invalid-encoding string
+                   # whose malformed-UTF-8 error surfaces far downstream.
+                   if code_point.between?(0xD800, 0xDFFF) || code_point > 0x10FFFF
+                     raise ParseError, "Invalid unicode escape \\u#{hex}"
+                   end
+                   [code_point].pack("U")
+                 else
+                   raise ParseError, "Bad escape \\#{e}"
+                 end
+          @i += 1
+        elsif c == "\n" || c == "\r"
+          raise ParseError, "Raw newline in string starting at #{start}; use \\n"
+        else
+          buf << c
+          @i += 1
+        end
+      end
+      raise ParseError, "Unterminated string starting at #{start}"
+    end
+
+    def lex_number(start)
+      j = @i
+      j += 1 while j < @n && digit?(@src[j])
+      is_float = false
+      if @src[j] == "." && digit?(@src[j + 1])
+        is_float = true
+        j += 1
+        j += 1 while j < @n && digit?(@src[j])
+      end
+      if (@src[j] == "e" || @src[j] == "E")
+        is_float = true
+        j += 1
+        j += 1 if (@src[j] == "+" || @src[j] == "-")
+        j += 1 while j < @n && digit?(@src[j])
+      end
+      text = @src[@i...j]
+      @i = j
+      val = is_float ? text.to_f : text.to_i
+      Token.new(type: :number, value: val, pos: start)
+    end
+
+    def lex_word(start)
+      j = @i
+      j += 1 while j < @n && ident_part?(@src[j])
+      text = @src[@i...j]
+      @i = j
+      case text
+      when "true"  then Token.new(type: :true_kw, value: true, pos: start)
+      when "false" then Token.new(type: :false_kw, value: false, pos: start)
+      when "null"  then Token.new(type: :null_kw, value: NULL, pos: start)
+      else Token.new(type: :ident, value: text, pos: start)
+      end
+    end
+
+    # Path lexing — see `tokens`. Only called immediately after `@`/`@@`, with no
+    # whitespace skipped, so the path must be tight. Returns nil (position restored)
+    # when nothing path-like follows, i.e. a bare `@`/`@@`.
+    def try_lex_path
+      start = @i
+      buf = +""
+      while peek == "." && peek(1) == "." && peek(2) == "/"
+        buf << "../"
+        @i += 3
+      end
+      unless ident_start?(peek)
+        @i = start
+        return nil
+      end
+      buf << lex_ident_text
+      while peek == "/" && ident_start?(peek(1))
+        @i += 1
+        buf << "/" << lex_ident_text
+      end
+      Token.new(type: :path, value: buf, pos: start)
+    end
+
+    def lex_ident_text
+      j = @i
+      j += 1 while j < @n && ident_part?(@src[j])
+      text = @src[@i...j]
+      @i = j
+      text
+    end
+
+    def digit?(c) = c && c >= "0" && c <= "9"
+    def ident_start?(c) = c && (c =~ /[A-Za-z_]/)
+    def ident_part?(c) = c && (c =~ /[A-Za-z0-9_]/)
+  end
+end

data/lib/fusion/null.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+# === Value ===
+#
+# Runtime null value.
+
+module Fusion
+  NULL = :null
+end