fusion-lang 0.0.1.alpha2 → 0.0.2

data/lib/fusion/lexer.rb

@@ -15,9 +15,12 @@ module Fusion
       "[" => :lbracket, "]" => :rbracket,
       "{" => :lbrace, "}" => :rbrace,
       "," => :comma, ":" => :colon,
-      "|" => :pipe, "?" => :question, "." => :dot,
+      "?" => :question, "." => :dot,
       "@" => :at, "/" => :slash,
       "=" => :equals,
+      "+" => :plus, "-" => :minus, "*" => :star,
+      "%" => :percent, "~" => :tilde,
+      "<" => :lt, ">" => :gt,
     }.freeze
 
     def initialize(src)
@@ -31,6 +34,11 @@ module Fusion
       loop do
         t = next_token
         out << t
+        # A file-reference path is one tight token, lexed only right after `@`/`@@`.
+        if [:at, :atat].include?(t.type)
+          p = try_lex_path
+          out << p unless p.nil?
+        end
         break if t.type == :eof
       end
       out
@@ -55,6 +63,54 @@ module Fusion
         @i += 3
         return Token.new(type: :spread, value: "...", pos: start)
       end
+      if c == "@" && peek(1) == "@"
+        @i += 2
+        return Token.new(type: :atat, value: "@@", pos: start)
+      end
+      # Two-character operators, matched before their single-char prefixes.
+      if c == "=" && peek(1) == "="
+        @i += 2
+        return Token.new(type: :eqeq, value: "==", pos: start)
+      end
+      if c == "/" && peek(1) == "/"
+        @i += 2
+        return Token.new(type: :slashslash, value: "//", pos: start)
+      end
+      if c == "?" && peek(1) == "?"
+        @i += 2
+        return Token.new(type: :qq, value: "??", pos: start)
+      end
+      if c == "<" && peek(1) == "="
+        @i += 2
+        return Token.new(type: :lte, value: "<=", pos: start)
+      end
+      if c == ">" && peek(1) == "="
+        @i += 2
+        return Token.new(type: :gte, value: ">=", pos: start)
+      end
+      if c == "&" && peek(1) == "&"
+        @i += 2
+        return Token.new(type: :andand, value: "&&", pos: start)
+      end
+      if c == "|"
+        case peek(1)
+        when "|"
+          @i += 2
+          return Token.new(type: :oror, value: "||", pos: start)
+        when ":"
+          @i += 2
+          return Token.new(type: :pipemap, value: "|:", pos: start)
+        when "?"
+          @i += 2
+          return Token.new(type: :pipefilter, value: "|?", pos: start)
+        when "+"
+          @i += 2
+          return Token.new(type: :pipereduce, value: "|+", pos: start)
+        else
+          @i += 1
+          return Token.new(type: :pipe, value: "|", pos: start)
+        end
+      end
       if c == "!"
         @i += 1
         return Token.new(type: :bang, value: "!", pos: start)
@@ -62,12 +118,13 @@ module Fusion
       if c == '"'
         return lex_string(start)
       end
-      if digit?(c) || (c == "-" && digit?(peek(1)))
+      if digit?(c)
         return lex_number(start)
       end
       if ident_start?(c)
         return lex_word(start)
       end
+
       if (type = PUNCT[c])
         @i += 1
         return Token.new(type: type, value: c, pos: start)
@@ -77,13 +134,13 @@ module Fusion
 
     def skip_trivia
       loop do
-        c = peek
-        if c == " " || c == "\t" || c == "\n" || c == "\r"
+        case peek
+        when " ", "\t", "\n", "\r"
           @i += 1
-        elsif c == "#" && at_line_start?
-          # A line is a comment iff its first non-whitespace char is "#".
-          # This also covers shebang lines (#!) for free.
-          @i += 1 until peek.nil? || peek == "\n"
+        when "#"
+          break if !at_line_start?
+
+          @i += 1 until [nil, "\n"].include?(peek)
         else
           break
         end
@@ -93,7 +150,7 @@ module Fusion
     # True when only whitespace precedes @i on the current physical line.
     def at_line_start?
       j = @i - 1
-      j -= 1 while j >= 0 && (@src[j] == " " || @src[j] == "\t")
+      j -= 1 while j >= 0 && [" ", "\t"].include?(@src[j])
       j < 0 || @src[j] == "\n" || @src[j] == "\r"
     end
 
@@ -101,37 +158,39 @@ module Fusion
       @i += 1 # opening quote
       buf = +""
       while (c = peek)
-        if c == '"'
+        case c
+        when '"'
           @i += 1
           return Token.new(type: :string, value: buf, pos: start)
-        elsif c == "\\"
+        when "\\"
           @i += 1
           e = peek
           buf << case e
-                 when '"' then '"'
-                 when "\\" then "\\"
-                 when "/" then "/"
-                 when "n" then "\n"
-                 when "t" then "\t"
-                 when "r" then "\r"
-                 when "b" then "\b"
-                 when "f" then "\f"
-                 when "u"
-                   hex = @src[@i + 1, 4]
-                   @i += 4
-                   code_point = hex.to_i(16)
-                   # Reject surrogates and out-of-range code points up front:
-                   # pack("U") would otherwise build an invalid-encoding string
-                   # whose malformed-UTF-8 error surfaces far downstream.
-                   if code_point.between?(0xD800, 0xDFFF) || code_point > 0x10FFFF
-                     raise ParseError, "Invalid unicode escape \\u#{hex}"
-                   end
-                   [code_point].pack("U")
-                 else
-                   raise ParseError, "Bad escape \\#{e}"
-                 end
+          when '"' then '"'
+          when "\\" then "\\"
+          when "/" then "/"
+          when "n" then "\n"
+          when "t" then "\t"
+          when "r" then "\r"
+          when "b" then "\b"
+          when "f" then "\f"
+          when "u"
+            hex = @src[@i + 1, 4]
+            @i += 4
+            code_point = hex.to_i(16)
+            # Reject surrogates and out-of-range code points up front:
+            # pack("U") would otherwise build an invalid-encoding string
+            # whose malformed-UTF-8 error surfaces far downstream.
+            if code_point.between?(0xD800, 0xDFFF) || code_point > 0x10FFFF
+              raise ParseError, "Invalid unicode escape \\u#{hex}"
+            end
+
+            [code_point].pack("U")
+          else
+            raise ParseError, "Bad escape \\#{e}"
+          end
           @i += 1
-        elsif c == "\n" || c == "\r"
+        when "\n", "\r"
           raise ParseError, "Raw newline in string starting at #{start}; use \\n"
         else
           buf << c
@@ -143,7 +202,6 @@ module Fusion
 
     def lex_number(start)
       j = @i
-      j += 1 if @src[j] == "-"
       j += 1 while j < @n && digit?(@src[j])
       is_float = false
       if @src[j] == "." && digit?(@src[j + 1])
@@ -151,10 +209,10 @@ module Fusion
         j += 1
         j += 1 while j < @n && digit?(@src[j])
       end
-      if (@src[j] == "e" || @src[j] == "E")
+      if ["e", "E"].include?(@src[j])
         is_float = true
         j += 1
-        j += 1 if (@src[j] == "+" || @src[j] == "-")
+        j += 1 if ["+", "-"].include?(@src[j])
         j += 1 while j < @n && digit?(@src[j])
       end
       text = @src[@i...j]
@@ -176,6 +234,36 @@ module Fusion
       end
     end
 
+    # Path lexing — see `tokens`. Only called immediately after `@`/`@@`, with no
+    # whitespace skipped, so the path must be tight. Returns nil (position restored)
+    # when nothing path-like follows, i.e. a bare `@`/`@@`.
+    def try_lex_path
+      start = @i
+      buf = +""
+      while peek == "." && peek(1) == "." && peek(2) == "/"
+        buf << "../"
+        @i += 3
+      end
+      unless ident_start?(peek)
+        @i = start
+        return nil
+      end
+      buf << lex_ident_text
+      while peek == "/" && ident_start?(peek(1))
+        @i += 1
+        buf << "/" << lex_ident_text
+      end
+      Token.new(type: :path, value: buf, pos: start)
+    end
+
+    def lex_ident_text
+      j = @i
+      j += 1 while j < @n && ident_part?(@src[j])
+      text = @src[@i...j]
+      @i = j
+      text
+    end
+
     def digit?(c) = c && c >= "0" && c <= "9"
     def ident_start?(c) = c && (c =~ /[A-Za-z_]/)
     def ident_part?(c) = c && (c =~ /[A-Za-z0-9_]/)