liquid2 0.1.1 → 0.3.0

data/lib/liquid2/parser.rb

@@ -7,6 +7,7 @@ require_relative "node"
 require_relative "nodes/comment"
 require_relative "nodes/output"
 require_relative "expressions/arguments"
+require_relative "expressions/arithmetic"
 require_relative "expressions/array"
 require_relative "expressions/blank"
 require_relative "expressions/boolean"
@@ -233,7 +234,6 @@ module Liquid2
       left = parse_primary
       left = parse_array_literal(left) if current_kind == :token_comma
       filters = parse_filters if current_kind == :token_pipe
-      filters ||= [] # : Array[Filter]
       expr = FilteredExpression.new(token, left, filters)
 
       if current_kind == :token_if
@@ -337,14 +337,26 @@ module Liquid2
 
       left = case kind
              when :token_true
-               self.next
-               looks_like_a_path ? parse_path : true
+               if looks_like_a_path
+                 parse_path
+               else
+                 self.next
+                 true
+               end
              when :token_false
-               self.next
-               looks_like_a_path ? parse_path : false
+               if looks_like_a_path
+                 parse_path
+               else
+                 self.next
+                 false
+               end
              when :token_nil
-               self.next
-               looks_like_a_path ? parse_path : nil
+               if looks_like_a_path
+                 parse_path
+               else
+                 self.next
+                 nil
+               end
              when :token_int
                Liquid2.to_liquid_int(self.next[1])
              when :token_float
@@ -359,7 +371,7 @@ module Liquid2
                parse_path
              when :token_lparen
                parse_range_lambda_or_grouped_expression
-             when :token_not
+             when :token_not, :token_plus, :token_minus
                parse_prefix_expression
              else
                unless looks_like_a_path && RESERVED_WORDS.include?(kind)
@@ -396,7 +408,7 @@ module Liquid2
     # @raises [LiquidTypeError].
     def parse_string
       node = parse_primary
-      raise LiquidTypeError, "expected a string" unless node.is_a?(String)
+      raise LiquidTypeError, "expected a string literal" unless node.is_a?(String)
 
       node
     end
@@ -411,6 +423,23 @@ module Liquid2
       Identifier.new(token)
     end
 
+    # Parse a string literals or unquoted word.
+    def parse_name
+      case current_kind
+      when :token_word
+        parse_identifier.name
+      when :token_single_quote_string, :token_double_quote_string
+        node = parse_string_literal
+        unless node.is_a?(String)
+          raise LiquidSyntaxError.new("names can't be template strings", node.token)
+        end
+
+        node
+      else
+        raise LiquidSyntaxError.new("expected a string literal or unquoted word", current)
+      end
+    end
+
     # Parse comma separated expression.
     # Leading commas should be consumed by the caller.
     # @return [Array<Expression>]
@@ -427,7 +456,7 @@ module Liquid2
       args
     end
 
-    # Parse comma name/value pairs.
+    # Parse comma separated name/value pairs.
     # Leading commas should be consumed by the caller, if allowed.
     # @return [Array<KeywordArgument>]
     def parse_keyword_arguments
@@ -438,8 +467,7 @@ module Liquid2
 
         word = eat(:token_word)
         eat_one_of(:token_assign, :token_colon)
-        val = parse_primary
-        args << KeywordArgument.new(word, word[1] || raise, val)
+        args << KeywordArgument.new(word, word[1] || raise, parse_primary)
 
         break unless current_kind == :token_comma
 
@@ -449,6 +477,68 @@ module Liquid2
       args
     end
 
+    # Parse comma separated parameter names with optional default expressions.
+    # Leading commas should be consumed by the caller, if allowed.
+    # @return [Hash[String, Parameter]]
+    def parse_parameters
+      args = {} # : Hash[String, Parameter]
+
+      loop do
+        break if TERMINATE_EXPRESSION.member?(current_kind)
+
+        word = eat(:token_word)
+        name = word[1] || raise
+
+        case current_kind
+        when :token_assign, :token_colon
+          @pos += 1
+          args[name] = Parameter.new(word, name, parse_primary)
+          @pos += 1 if current_kind == :token_comma
+        when :comma
+          args[name] = Parameter.new(word, name, :undefined)
+          @pos += 1
+        else
+          args[name] = Parameter.new(word, name, :undefined)
+          break
+        end
+      end
+
+      args
+    end
+
+    # Parse mixed positional and keyword arguments.
+    # Leading commas should be consumed by the caller, if allowed.
+    # @return [[Array[untyped], Array[KeywordArgument]]]
+    def parse_arguments
+      args = [] # : Array[untyped]
+      kwargs = [] # : Array[KeywordArgument]
+
+      loop do
+        break if TERMINATE_EXPRESSION.member?(current_kind)
+
+        case current_kind
+        when :token_word
+          if KEYWORD_ARGUMENT_DELIMITERS.include?(peek_kind)
+            token = self.next
+            @pos += 1 # = or :
+            kwargs << KeywordArgument.new(token, token[1] || raise, parse_primary)
+          else
+            # A positional argument
+            args << parse_primary
+          end
+        else
+          # A positional argument
+          args << parse_primary
+        end
+
+        break unless current_kind == :token_comma
+
+        @pos += 1
+      end
+
+      [args, kwargs]
+    end
+
     protected
 
     class Precedence
@@ -458,7 +548,10 @@ module Liquid2
       LOGICAL_AND = 4
       RELATIONAL = 5
       MEMBERSHIP = 6
-      PREFIX = 7
+      ADD_SUB = 8
+      MUL_DIV = 9
+      POW = 10
+      PREFIX = 11
     end
 
     PRECEDENCES = {
@@ -474,7 +567,14 @@ module Liquid2
       token_ne: Precedence::RELATIONAL,
       token_lg: Precedence::RELATIONAL,
       token_le: Precedence::RELATIONAL,
-      token_ge: Precedence::RELATIONAL
+      token_ge: Precedence::RELATIONAL,
+      token_plus: Precedence::ADD_SUB,
+      token_minus: Precedence::ADD_SUB,
+      token_times: Precedence::MUL_DIV,
+      token_divide: Precedence::MUL_DIV,
+      token_floor_div: Precedence::MUL_DIV,
+      token_mod: Precedence::MUL_DIV,
+      token_pow: Precedence::POW
     }.freeze
 
     BINARY_OPERATORS = Set[
@@ -488,7 +588,14 @@ module Liquid2
       :token_contains,
       :token_in,
       :token_and,
-      :token_or
+      :token_or,
+      :token_plus,
+      :token_minus,
+      :token_times,
+      :token_divide,
+      :token_floor_div,
+      :token_mod,
+      :token_pow
     ]
 
     TERMINATE_EXPRESSION = Set[
@@ -726,14 +833,35 @@ module Liquid2
       end
 
       eat(:token_rparen)
-      GroupedExpression.new(token, expr)
+      expr
     end
 
     # @return [Node]
     def parse_prefix_expression
-      token = eat(:token_not)
-      expr = parse_primary
-      LogicalNot.new(token, expr)
+      case current_kind
+      when :token_not
+        token = self.next
+        expr = parse_primary(precedence: Precedence::PREFIX)
+        LogicalNot.new(token, expr)
+      when :token_plus
+        token = self.next
+        unless @env.arithmetic_operators
+          raise LiquidSyntaxError.new("unexpected prefix operator +",
+                                      token)
+        end
+
+        Positive.new(token, parse_primary(precedence: Precedence::PREFIX))
+      when :token_minus
+        token = self.next
+        unless @env.arithmetic_operators
+          raise LiquidSyntaxError.new("unexpected prefix operator -",
+                                      token)
+        end
+
+        Negative.new(token, parse_primary(precedence: Precedence::PREFIX))
+      else
+        raise LiquidSyntaxError.new("unexpected prefix operator #{current[1]}", current)
+      end
     end
 
     # @param left [Expression]
@@ -765,7 +893,27 @@ module Liquid2
       when :token_or
         LogicalOr.new(op_token, left, right)
       else
-        raise LiquidSyntaxError.new("unexpected infix operator, #{op_token[1]}", op_token)
+        unless @env.arithmetic_operators
+          raise LiquidSyntaxError.new("unexpected infix operator, #{op_token[1]}",
+                                      op_token)
+        end
+
+        case op_token.first
+        when :token_plus
+          Plus.new(op_token, left, right)
+        when :token_minus
+          Minus.new(op_token, left, right)
+        when :token_times
+          Times.new(op_token, left, right)
+        when :token_divide
+          Divide.new(op_token, left, right)
+        when :token_mod
+          Modulo.new(op_token, left, right)
+        when :token_pow
+          Pow.new(op_token, left, right)
+        else
+          raise LiquidSyntaxError.new("unexpected infix operator, #{op_token[1]}", op_token)
+        end
       end
     end
 
@@ -783,7 +931,7 @@ module Liquid2
 
       unless current_kind == :token_colon || !TERMINATE_FILTER.member?(current_kind)
         # No arguments
-        return Filter.new(name, name[1] || raise, []) # TODO: optimize
+        return Filter.new(name, name[1] || raise, nil)
       end
 
       @pos += 1 # token_colon
@@ -804,7 +952,7 @@ module Liquid2
             args << parse_arrow_function
           else
             # A positional argument that is a path.
-            args << parse_path
+            args << parse_primary
           end
         when :token_lparen
           # A grouped expression or range or arrow function

data/lib/liquid2/scanner.rb

@@ -12,13 +12,11 @@ module Liquid2
   class Scanner
     attr_reader :tokens
 
-    RE_MARKUP_START = /\{[\{%#]/
-    RE_WHITESPACE = /[ \n\r\t]+/
     RE_LINE_SPACE = /[ \t]+/
     RE_WORD = /[\u0080-\uFFFFa-zA-Z_][\u0080-\uFFFFa-zA-Z0-9_-]*/
     RE_INT  = /-?\d+(?:[eE]\+?\d+)?/
     RE_FLOAT = /((?:-?\d+\.\d+(?:[eE][+-]?\d+)?)|(-?\d+[eE]-\d+))/
-    RE_PUNCTUATION = /\?|\[|\]|\|{1,2}|\.{1,2}|,|:|\(|\)|[<>=!]+/
+    RE_PUNCTUATION = %r{\?|\[|\]|\|{1,2}|\.{1,2}|,|:|\(|\)|[<>=!]+|[+\-%*/]+(?![\}%])}
     RE_SINGLE_QUOTE_STRING_SPECIAL = /[\\'\$]/
     RE_DOUBLE_QUOTE_STRING_SPECIAL = /[\\"\$]/
 
@@ -60,7 +58,14 @@ module Liquid2
       ">=" => :token_ge,
       "==" => :token_eq,
       "!=" => :token_ne,
-      "=>" => :token_arrow
+      "=>" => :token_arrow,
+      "+" => :token_plus,
+      "-" => :token_minus,
+      "%" => :token_mod,
+      "*" => :token_times,
+      "/" => :token_divide,
+      "//" => :token_floor_div,
+      "**" => :token_pow
     }.freeze
 
     def self.tokenize(source, scanner)
@@ -94,31 +99,19 @@ module Liquid2
     # @param value [String?]
     # @return void
     def emit(kind, value)
-      # TODO: For debugging. Comment this out when benchmarking.
-      raise "empty span (#{kind}, #{value})" if @scanner.pos == @start
-
       @tokens << [kind, value, @start]
       @start = @scanner.pos
     end
 
     def skip_trivia
-      # TODO: For debugging. Comment this out when benchmarking.
-      raise "must emit before skipping trivia" if @scanner.pos != @start
-
-      @start = @scanner.pos if @scanner.skip(RE_WHITESPACE)
+      @start = @scanner.pos if @scanner.skip(/[ \n\r\t]+/)
     end
 
     def skip_line_trivia
-      # TODO: For debugging. Comment this out when benchmarking.
-      raise "must emit before skipping line trivia" if @scanner.pos != @start
-
       @start = @scanner.pos if @scanner.skip(RE_LINE_SPACE)
     end
 
     def accept_whitespace_control
-      # TODO: For debugging. Comment this out when benchmarking.
-      raise "must emit before accepting whitespace control" if @scanner.pos != @start
-
       ch = @scanner.peek(1)
 
       case ch
@@ -133,7 +126,7 @@ module Liquid2
     end
 
     def lex_markup
-      case @scanner.scan(RE_MARKUP_START)
+      case @scanner.scan(/\{[\{%#]/)
       when "{#"
         :lex_comment
       when "{{"
@@ -197,37 +190,30 @@ module Liquid2
     end
 
     def lex_expression
-      # TODO: For debugging. Comment this out when benchmarking.
-      raise "must emit before accepting an expression token" if @scanner.pos != @start
-
       loop do
         skip_trivia
-
-        case @scanner.get_byte
-        when "'"
+        if (value = @scanner.scan(RE_FLOAT))
+          @tokens << [:token_float, value, @start]
           @start = @scanner.pos
-          scan_string("'", :token_single_quote_string, RE_SINGLE_QUOTE_STRING_SPECIAL)
-        when "\""
+        elsif (value = @scanner.scan(RE_INT))
+          @tokens << [:token_int, value, @start]
+          @start = @scanner.pos
+        elsif (value = @scanner.scan(RE_PUNCTUATION))
+          @tokens << [TOKEN_MAP[value] || :token_unknown, value, @start]
+          @start = @scanner.pos
+        elsif (value = @scanner.scan(RE_WORD))
+          @tokens << [TOKEN_MAP[value] || :token_word, value, @start]
           @start = @scanner.pos
-          scan_string("\"", :token_double_quote_string, RE_DOUBLE_QUOTE_STRING_SPECIAL)
-        when nil
-          # End of scanner. Unclosed expression or string literal.
-          break
         else
-          @scanner.pos -= 1
-          if (value = @scanner.scan(RE_FLOAT))
-            @tokens << [:token_float, value, @start]
+          case @scanner.get_byte
+          when "'"
             @start = @scanner.pos
-          elsif (value = @scanner.scan(RE_INT))
-            @tokens << [:token_int, value, @start]
-            @start = @scanner.pos
-          elsif (value = @scanner.scan(RE_PUNCTUATION))
-            @tokens << [TOKEN_MAP[value] || :token_unknown, value, @start]
-            @start = @scanner.pos
-          elsif (value = @scanner.scan(RE_WORD))
-            @tokens << [TOKEN_MAP[value] || :token_word, value, @start]
+            scan_string("'", :token_single_quote_string, RE_SINGLE_QUOTE_STRING_SPECIAL)
+          when "\""
             @start = @scanner.pos
+            scan_string("\"", :token_double_quote_string, RE_DOUBLE_QUOTE_STRING_SPECIAL)
           else
+            @scanner.pos -= 1
             break
           end
         end
@@ -413,9 +399,6 @@ module Liquid2
     end
 
     def lex_line_statements
-      # TODO: For debugging. Comment this out when benchmarking.
-      raise "must emit before accepting an expression token" if @scanner.pos != @start
-
       skip_trivia # Leading newlines are OK
 
       if (tag_name = @scanner.scan(/(?:[a-z][a-z_0-9]*|#)/))

data/lib/liquid2/static_analysis.rb

@@ -231,7 +231,7 @@ module Liquid2
     def self.extract_filters(expression, template_name)
       filters = [] # : Array[[String, Span]]
 
-      if expression.is_a?(Liquid2::FilteredExpression)
+      if expression.is_a?(Liquid2::FilteredExpression) && !expression.filters.nil?
         expression.filters.each do |filter|
           filters << [filter.name, Span.new(template_name, filter.token.last)]
         end

data/lib/liquid2/template.rb

@@ -43,7 +43,6 @@ module Liquid2
     end
 
     def render_with_context(context, buffer, partial: false, block_scope: false, namespace: nil)
-      # TODO: don't extend if namespace is nil
       context.extend(namespace || {}) do
         index = 0
         while (node = @ast[index])
@@ -59,6 +58,8 @@ module Liquid2
 
           next unless (interrupt = context.interrupts.pop)
 
+          break if interrupt == :stop_render
+
           if !partial || block_scope
             raise LiquidSyntaxError.new("unexpected #{interrupt}",
                                         node.token) # steep:ignore
@@ -69,15 +70,14 @@ module Liquid2
         end
       end
     rescue LiquidError => e
-      e.source = @source
-      e.template_name = @name unless @name.empty?
+      e.source = context.template.source unless e.source
+      e.template_name = @name unless e.template_name || @name.empty?
       raise
     end
 
     # Merge template globals with another namespace.
     def make_globals(namespace)
-      # TODO: optimize
-      @globals.merge(@overlay || {}, namespace || {})
+      @globals.merge(@overlay, namespace || {})
     end
 
     # Return `false` if this template is stale and needs to be loaded again.
@@ -123,6 +123,53 @@ module Liquid2
       nodes
     end
 
+    # Return an array of `{% doc %}` nodes found in this template.
+    #
+    # Each instance of `Liquid2::DocTag` has a `token` and `text` attribute. Use
+    # `Template#docs.map(&:text)` to get an array of doc strings.
+    #
+    # @return [Array[DocTag]]
+    def docs
+      context = RenderContext.new(self)
+      nodes = [] # : Array[DocTag]
+
+      # @type var visit: ^(Node) -> void
+      visit = lambda do |node|
+        nodes << node if node.is_a?(DocTag)
+
+        node.children(context, include_partials: false).each do |child|
+          visit.call(child) if child.is_a?(Node)
+        end
+      end
+
+      @ast.each { |node| visit.call(node) if node.is_a?(Node) }
+
+      nodes
+    end
+
+    # Return arrays of `{% macro %}` and `{% call %}` tags found in this template.
+    # @param include_partials [bool]
+    # @return [Array[MacroTag], Array[CallTag]]
+    def macros(include_partials: false)
+      context = RenderContext.new(self)
+      macro_nodes = [] # : Array[MacroTag]
+      call_nodes = [] # : Array[CallTag]
+
+      # @type var visit: ^(Node) -> void
+      visit = lambda do |node|
+        macro_nodes << node if node.is_a?(MacroTag)
+        call_nodes << node if node.is_a?(CallTag)
+
+        node.children(context, include_partials: include_partials).each do |child|
+          visit.call(child) if child.is_a?(Node)
+        end
+      end
+
+      @ast.each { |node| visit.call(node) if node.is_a?(Node) }
+
+      [macro_nodes, call_nodes]
+    end
+
     # Return an array of variables used in this template, without path segments.
     # @param include_partials [bool]
     # @return [Array[String]]

data/lib/liquid2/undefined.rb

@@ -5,6 +5,8 @@ require_relative "errors"
 module Liquid2
   # The default undefined type. Can be iterated over an indexed without error.
   class Undefined
+    include Enumerable
+
     attr_reader :force_default
 
     def initialize(name, node: nil)
@@ -26,6 +28,8 @@ module Liquid2
     def to_s = ""
     def to_i = 0
     def to_f = 0.0
+    def -@ = self
+    def +@ = self
     def each(...) = Enumerator.new {} # rubocop:disable Lint/EmptyBlock
     def each_with_index(...) = Enumerator.new {} # rubocop:disable Lint/EmptyBlock
     def join(...) = ""
@@ -37,87 +41,95 @@ module Liquid2
   class StrictUndefined < Undefined
     def initialize(name, node: nil)
       super
-      @message = "#{name.inspect} is undefined"
+      @message = "#{@node.is_a?(Path) ? @node : name.inspect} is undefined"
     end
 
     def respond_to_missing? = true
 
     def method_missing(...)
-      raise UndefinedError.new(@message, @node)
+      raise UndefinedError.new(@message, @node.token)
     end
 
     def [](...)
-      raise UndefinedError.new(@message, @node)
+      raise UndefinedError.new(@message, @node.token)
     end
 
     def key?(...)
-      raise UndefinedError.new(@message, @node)
+      raise UndefinedError.new(@message, @node.token)
     end
 
     def include?(...)
-      raise UndefinedError.new(@message, @node)
+      raise UndefinedError.new(@message, @node.token)
     end
 
     def member?(...)
-      raise UndefinedError.new(@message, @node)
+      raise UndefinedError.new(@message, @node.token)
     end
 
     def fetch(...)
-      raise UndefinedError.new(@message, @node)
+      raise UndefinedError.new(@message, @node.token)
     end
 
     def !
-      raise UndefinedError.new(@message, @node)
+      raise UndefinedError.new(@message, @node.token)
     end
 
     def ==(_other)
-      raise UndefinedError.new(@message, @node)
+      raise UndefinedError.new(@message, @node.token)
     end
 
     def !=(_other)
-      raise UndefinedError.new(@message, @node)
+      raise UndefinedError.new(@message, @node.token)
     end
 
     alias eql? ==
 
     def size
-      raise UndefinedError.new(@message, @node)
+      raise UndefinedError.new(@message, @node.token)
     end
 
     def length
-      raise UndefinedError.new(@message, @node)
+      raise UndefinedError.new(@message, @node.token)
     end
 
     def to_s
-      raise UndefinedError.new(@message, @node)
+      raise UndefinedError.new(@message, @node.token)
     end
 
     def to_i
-      raise UndefinedError.new(@message, @node)
+      raise UndefinedError.new(@message, @node.token)
     end
 
     def to_f
-      raise UndefinedError.new(@message, @node)
+      raise UndefinedError.new(@message, @node.token)
+    end
+
+    def +@
+      self
+    end
+
+    def -@
+      self
     end
 
     def each(...)
-      raise UndefinedError.new(@message, @node)
+      raise UndefinedError.new(@message, @node.token)
     end
 
     def each_with_index(...)
-      raise UndefinedError.new(@message, @node)
+      raise UndefinedError.new(@message, @node.token)
     end
 
     def join(...)
-      raise UndefinedError.new(@message, @node)
+      raise UndefinedError.new(@message, @node.token)
     end
 
     def to_liquid(_context)
-      raise UndefinedError.new(@message, @node)
+      self
     end
 
     def poke
-      raise UndefinedError.new(@message, @node)
+      raise UndefinedError.new(@message, @node.token)
     end
   end
 

data/lib/liquid2/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Liquid2
-  VERSION = "0.1.1"
+  VERSION = "0.3.0"
 end

data/lib/liquid2.rb

@@ -73,6 +73,8 @@ module Liquid2
   # @param obj [Object]
   # @return [bool]
   def self.truthy?(context, obj)
+    return false if context.env.falsy_undefined && undefined?(obj)
+
     obj = obj.to_liquid(context) if obj.respond_to?(:to_liquid)
     !!obj
   end