liquid 5.4.0 → 5.6.4

data/lib/liquid/expression.rb

@@ -3,41 +3,120 @@
 module Liquid
   class Expression
     LITERALS = {
-      nil => nil, 'nil' => nil, 'null' => nil, '' => nil,
+      nil => nil,
+      'nil' => nil,
+      'null' => nil,
+      '' => nil,
       'true' => true,
       'false' => false,
       'blank' => '',
-      'empty' => ''
+      'empty' => '',
+      # in lax mode, minus sign can be a VariableLookup
+      # For simplicity and performace, we treat it like a literal
+      '-' => VariableLookup.parse("-", nil).freeze,
     }.freeze
 
-    INTEGERS_REGEX       = /\A(-?\d+)\z/
-    FLOATS_REGEX         = /\A(-?\d[\d\.]+)\z/
+    DOT = ".".ord
+    ZERO = "0".ord
+    NINE = "9".ord
+    DASH = "-".ord
 
     # Use an atomic group (?>...) to avoid pathological backtracing from
     # malicious input as described in https://github.com/Shopify/liquid/issues/1357
-    RANGES_REGEX         = /\A\(\s*(?>(\S+)\s*\.\.)\s*(\S+)\s*\)\z/
+    RANGES_REGEX = /\A\(\s*(?>(\S+)\s*\.\.)\s*(\S+)\s*\)\z/
+    INTEGER_REGEX = /\A(-?\d+)\z/
+    FLOAT_REGEX = /\A(-?\d+)\.\d+\z/
 
-    def self.parse(markup)
-      return nil unless markup
+    class << self
+      def parse(markup, ss = StringScanner.new(""), cache = nil)
+        return unless markup
 
-      markup = markup.strip
-      if (markup.start_with?('"') && markup.end_with?('"')) ||
-         (markup.start_with?("'") && markup.end_with?("'"))
-        return markup[1..-2]
+        markup = markup.strip # markup can be a frozen string
+
+        if (markup.start_with?('"') && markup.end_with?('"')) ||
+          (markup.start_with?("'") && markup.end_with?("'"))
+          return markup[1..-2]
+        elsif LITERALS.key?(markup)
+          return LITERALS[markup]
+        end
+
+        # Cache only exists during parsing
+        if cache
+          return cache[markup] if cache.key?(markup)
+
+          cache[markup] = inner_parse(markup, ss, cache).freeze
+        else
+          inner_parse(markup, ss, nil).freeze
+        end
       end
 
-      case markup
-      when INTEGERS_REGEX
-        Regexp.last_match(1).to_i
-      when RANGES_REGEX
-        RangeLookup.parse(Regexp.last_match(1), Regexp.last_match(2))
-      when FLOATS_REGEX
-        Regexp.last_match(1).to_f
-      else
-        if LITERALS.key?(markup)
-          LITERALS[markup]
+      def inner_parse(markup, ss, cache)
+        if (markup.start_with?("(") && markup.end_with?(")")) && markup =~ RANGES_REGEX
+          return RangeLookup.parse(
+            Regexp.last_match(1),
+            Regexp.last_match(2),
+            ss,
+            cache,
+          )
+        end
+
+        if (num = parse_number(markup, ss))
+          num
+        else
+          VariableLookup.parse(markup, ss, cache)
+        end
+      end
+
+      def parse_number(markup, ss)
+        # check if the markup is simple integer or float
+        case markup
+        when INTEGER_REGEX
+          return Integer(markup, 10)
+        when FLOAT_REGEX
+          return markup.to_f
+        end
+
+        ss.string = markup
+        # the first byte must be a digit or  a dash
+        byte = ss.scan_byte
+
+        return false if byte != DASH && (byte < ZERO || byte > NINE)
+
+        if byte == DASH
+          peek_byte = ss.peek_byte
+
+          # if it starts with a dash, the next byte must be a digit
+          return false if peek_byte.nil? || !(peek_byte >= ZERO && peek_byte <= NINE)
+        end
+
+        # The markup could be a float with multiple dots
+        first_dot_pos = nil
+        num_end_pos = nil
+
+        while (byte = ss.scan_byte)
+          return false if byte != DOT && (byte < ZERO || byte > NINE)
+
+          # we found our number and now we are just scanning the rest of the string
+          next if num_end_pos
+
+          if byte == DOT
+            if first_dot_pos.nil?
+              first_dot_pos = ss.pos
+            else
+              # we found another dot, so we know that the number ends here
+              num_end_pos = ss.pos - 1
+            end
+          end
+        end
+
+        num_end_pos = markup.length if ss.eos?
+
+        if num_end_pos
+          # number ends with a number "123.123"
+          markup.byteslice(0, num_end_pos).to_f
         else
-          VariableLookup.parse(markup)
+          # number ends with a dot "123."
+          markup.byteslice(0, first_dot_pos).to_f
         end
       end
     end

data/lib/liquid/forloop_drop.rb

@@ -5,7 +5,7 @@ module Liquid
   # @liquid_type object
   # @liquid_name forloop
   # @liquid_summary
-  #   Information about a parent [`for` loop](/api/liquid/tags#for).
+  #   Information about a parent [`for` loop](/docs/api/liquid/tags/for).
   class ForloopDrop < Drop
     def initialize(name, length, parentloop)
       @name       = name
@@ -30,10 +30,7 @@ module Liquid
     # @liquid_return [forloop]
     attr_reader :parentloop
 
-    def name
-      Usage.increment('forloop_drop_name')
-      @name
-    end
+    attr_reader :name
 
     # @liquid_public_docs
     # @liquid_summary

data/lib/liquid/lexer.rb

@@ -1,62 +1,173 @@
 # frozen_string_literal: true
 
-require "strscan"
 module Liquid
   class Lexer
-    SPECIALS = {
-      '|' => :pipe,
-      '.' => :dot,
-      ':' => :colon,
-      ',' => :comma,
-      '[' => :open_square,
-      ']' => :close_square,
-      '(' => :open_round,
-      ')' => :close_round,
-      '?' => :question,
-      '-' => :dash,
-    }.freeze
-    IDENTIFIER            = /[a-zA-Z_][\w-]*\??/
-    SINGLE_STRING_LITERAL = /'[^\']*'/
+    CLOSE_ROUND = [:close_round, ")"].freeze
+    CLOSE_SQUARE = [:close_square, "]"].freeze
+    COLON = [:colon, ":"].freeze
+    COMMA = [:comma, ","].freeze
+    COMPARISION_NOT_EQUAL = [:comparison, "!="].freeze
+    COMPARISON_CONTAINS = [:comparison, "contains"].freeze
+    COMPARISON_EQUAL = [:comparison, "=="].freeze
+    COMPARISON_GREATER_THAN = [:comparison, ">"].freeze
+    COMPARISON_GREATER_THAN_OR_EQUAL = [:comparison, ">="].freeze
+    COMPARISON_LESS_THAN = [:comparison, "<"].freeze
+    COMPARISON_LESS_THAN_OR_EQUAL = [:comparison, "<="].freeze
+    COMPARISON_NOT_EQUAL_ALT = [:comparison, "<>"].freeze
+    DASH = [:dash, "-"].freeze
+    DOT = [:dot, "."].freeze
+    DOTDOT = [:dotdot, ".."].freeze
+    DOT_ORD = ".".ord
     DOUBLE_STRING_LITERAL = /"[^\"]*"/
+    EOS = [:end_of_string].freeze
+    IDENTIFIER            = /[a-zA-Z_][\w-]*\??/
     NUMBER_LITERAL        = /-?\d+(\.\d+)?/
-    DOTDOT                = /\.\./
-    COMPARISON_OPERATOR   = /==|!=|<>|<=?|>=?|contains(?=\s)/
+    OPEN_ROUND = [:open_round, "("].freeze
+    OPEN_SQUARE = [:open_square, "["].freeze
+    PIPE = [:pipe, "|"].freeze
+    QUESTION = [:question, "?"].freeze
+    RUBY_WHITESPACE = [" ", "\t", "\r", "\n", "\f"].freeze
+    SINGLE_STRING_LITERAL = /'[^\']*'/
     WHITESPACE_OR_NOTHING = /\s*/
 
-    def initialize(input)
-      @ss = StringScanner.new(input)
+    SINGLE_COMPARISON_TOKENS = [].tap do |table|
+      table["<".ord] = COMPARISON_LESS_THAN
+      table[">".ord] = COMPARISON_GREATER_THAN
+      table.freeze
+    end
+
+    TWO_CHARS_COMPARISON_JUMP_TABLE = [].tap do |table|
+      table["=".ord] = [].tap do |sub_table|
+        sub_table["=".ord] = COMPARISON_EQUAL
+        sub_table.freeze
+      end
+      table["!".ord] = [].tap do |sub_table|
+        sub_table["=".ord] = COMPARISION_NOT_EQUAL
+        sub_table.freeze
+      end
+      table.freeze
     end
 
-    def tokenize
-      @output = []
-
-      until @ss.eos?
-        @ss.skip(WHITESPACE_OR_NOTHING)
-        break if @ss.eos?
-        tok      = if (t = @ss.scan(COMPARISON_OPERATOR))
-          [:comparison, t]
-        elsif (t = @ss.scan(SINGLE_STRING_LITERAL))
-          [:string, t]
-        elsif (t = @ss.scan(DOUBLE_STRING_LITERAL))
-          [:string, t]
-        elsif (t = @ss.scan(NUMBER_LITERAL))
-          [:number, t]
-        elsif (t = @ss.scan(IDENTIFIER))
-          [:id, t]
-        elsif (t = @ss.scan(DOTDOT))
-          [:dotdot, t]
-        else
-          c     = @ss.getch
-          if (s = SPECIALS[c])
-            [s, c]
+    COMPARISON_JUMP_TABLE = [].tap do |table|
+      table["<".ord] = [].tap do |sub_table|
+        sub_table["=".ord] = COMPARISON_LESS_THAN_OR_EQUAL
+        sub_table[">".ord] = COMPARISON_NOT_EQUAL_ALT
+        sub_table.freeze
+      end
+      table[">".ord] = [].tap do |sub_table|
+        sub_table["=".ord] = COMPARISON_GREATER_THAN_OR_EQUAL
+        sub_table.freeze
+      end
+      table.freeze
+    end
+
+    NEXT_MATCHER_JUMP_TABLE = [].tap do |table|
+      "a".upto("z") do |c|
+        table[c.ord] = [:id, IDENTIFIER].freeze
+        table[c.upcase.ord] = [:id, IDENTIFIER].freeze
+      end
+      table["_".ord] = [:id, IDENTIFIER].freeze
+
+      "0".upto("9") do |c|
+        table[c.ord] = [:number, NUMBER_LITERAL].freeze
+      end
+      table["-".ord] = [:number, NUMBER_LITERAL].freeze
+
+      table["'".ord] = [:string, SINGLE_STRING_LITERAL].freeze
+      table["\"".ord] = [:string, DOUBLE_STRING_LITERAL].freeze
+      table.freeze
+    end
+
+    SPECIAL_TABLE = [].tap do |table|
+      table["|".ord] = PIPE
+      table[".".ord] = DOT
+      table[":".ord] = COLON
+      table[",".ord] = COMMA
+      table["[".ord] = OPEN_SQUARE
+      table["]".ord] = CLOSE_SQUARE
+      table["(".ord] = OPEN_ROUND
+      table[")".ord] = CLOSE_ROUND
+      table["?".ord] = QUESTION
+      table["-".ord] = DASH
+    end
+
+    NUMBER_TABLE = [].tap do |table|
+      "0".upto("9") do |c|
+        table[c.ord] = true
+      end
+      table.freeze
+    end
+
+    # rubocop:disable Metrics/BlockNesting
+    class << self
+      def tokenize(ss)
+        output = []
+
+        until ss.eos?
+          ss.skip(WHITESPACE_OR_NOTHING)
+
+          break if ss.eos?
+
+          start_pos = ss.pos
+          peeked = ss.peek_byte
+
+          if (special = SPECIAL_TABLE[peeked])
+            ss.scan_byte
+            # Special case for ".."
+            if special == DOT && ss.peek_byte == DOT_ORD
+              ss.scan_byte
+              output << DOTDOT
+            elsif special == DASH
+              # Special case for negative numbers
+              if (peeked_byte = ss.peek_byte) && NUMBER_TABLE[peeked_byte]
+                ss.pos -= 1
+                output << [:number, ss.scan(NUMBER_LITERAL)]
+              else
+                output << special
+              end
+            else
+              output << special
+            end
+          elsif (sub_table = TWO_CHARS_COMPARISON_JUMP_TABLE[peeked])
+            ss.scan_byte
+            if (peeked_byte = ss.peek_byte) && (found = sub_table[peeked_byte])
+              output << found
+              ss.scan_byte
+            else
+              raise_syntax_error(start_pos, ss)
+            end
+          elsif (sub_table = COMPARISON_JUMP_TABLE[peeked])
+            ss.scan_byte
+            if (peeked_byte = ss.peek_byte) && (found = sub_table[peeked_byte])
+              output << found
+              ss.scan_byte
+            else
+              output << SINGLE_COMPARISON_TOKENS[peeked]
+            end
           else
-            raise SyntaxError, "Unexpected character #{c}"
+            type, pattern = NEXT_MATCHER_JUMP_TABLE[peeked]
+
+            if type && (t = ss.scan(pattern))
+              # Special case for "contains"
+              output << if type == :id && t == "contains" && output.last&.first != :dot
+                COMPARISON_CONTAINS
+              else
+                [type, t]
+              end
+            else
+              raise_syntax_error(start_pos, ss)
+            end
           end
         end
-        @output << tok
+        # rubocop:enable Metrics/BlockNesting
+        output << EOS
       end
 
-      @output << [:end_of_string]
+      def raise_syntax_error(start_pos, ss)
+        ss.pos = start_pos
+        # the character could be a UTF-8 character, use getch to get all the bytes
+        raise SyntaxError, "Unexpected character #{ss.getch}"
+      end
     end
   end
 end

data/lib/liquid/locales/en.yml

@@ -15,6 +15,7 @@
       include: "Error in tag 'include' - Valid syntax: include '[template]' (with|for) [object|collection]"
       inline_comment_invalid: "Syntax error in tag '#' - Each line of comments must be prefixed by the '#' character"
       invalid_delimiter: "'%{tag}' is not a valid delimiter for %{block_name} tags. use %{block_delimiter}"
+      invalid_template_encoding: "Invalid template encoding"
       render: "Syntax error in tag 'render' - Template name must be a quoted string"
       table_row: "Syntax Error in 'table_row loop' - Valid syntax: table_row [item] in [collection] cols=3"
       tag_never_closed: "'%{block_name}' tag was never closed"

data/lib/liquid/parse_context.rb

@@ -3,14 +3,27 @@
 module Liquid
   class ParseContext
     attr_accessor :locale, :line_number, :trim_whitespace, :depth
-    attr_reader :partial, :warnings, :error_mode
+    attr_reader :partial, :warnings, :error_mode, :environment
 
-    def initialize(options = {})
+    def initialize(options = Const::EMPTY_HASH)
+      @environment = options.fetch(:environment, Environment.default)
       @template_options = options ? options.dup : {}
 
       @locale   = @template_options[:locale] ||= I18n.new
       @warnings = []
 
+      # constructing new StringScanner in Lexer, Tokenizer, etc is expensive
+      # This StringScanner will be shared by all of them
+      @string_scanner = StringScanner.new("")
+
+      @expression_cache = if options[:expression_cache].nil?
+        {}
+      elsif options[:expression_cache].respond_to?(:[]) && options[:expression_cache].respond_to?(:[]=)
+        options[:expression_cache]
+      elsif options[:expression_cache]
+        {}
+      end
+
       self.depth   = 0
       self.partial = false
     end
@@ -23,19 +36,29 @@ module Liquid
       Liquid::BlockBody.new
     end
 
-    def new_tokenizer(markup, start_line_number: nil, for_liquid_tag: false)
-      Tokenizer.new(markup, line_number: start_line_number, for_liquid_tag: for_liquid_tag)
+    def new_parser(input)
+      @string_scanner.string = input
+      Parser.new(@string_scanner)
+    end
+
+    def new_tokenizer(source, start_line_number: nil, for_liquid_tag: false)
+      Tokenizer.new(
+        source: source,
+        string_scanner: @string_scanner,
+        line_number: start_line_number,
+        for_liquid_tag: for_liquid_tag,
+      )
     end
 
     def parse_expression(markup)
-      Expression.parse(markup)
+      Expression.parse(markup, @string_scanner, @expression_cache)
     end
 
     def partial=(value)
       @partial = value
       @options = value ? partial_options : @template_options
 
-      @error_mode = @options[:error_mode] || Template.error_mode
+      @error_mode = @options[:error_mode] || @environment.error_mode
     end
 
     def partial_options

data/lib/liquid/parse_tree_visitor.rb

@@ -36,7 +36,7 @@ module Liquid
     protected
 
     def children
-      @node.respond_to?(:nodelist) ? Array(@node.nodelist) : []
+      @node.respond_to?(:nodelist) ? Array(@node.nodelist) : Const::EMPTY_ARRAY
     end
   end
 end

data/lib/liquid/parser.rb

@@ -3,8 +3,8 @@
 module Liquid
   class Parser
     def initialize(input)
-      l       = Lexer.new(input)
-      @tokens = l.tokenize
+      ss = input.is_a?(StringScanner) ? input : StringScanner.new(input)
+      @tokens = Lexer.tokenize(ss)
       @p      = 0 # pointer to current location
     end
 
@@ -53,7 +53,7 @@ module Liquid
         str = consume
         str << variable_lookups
       when :open_square
-        str = consume
+        str = consume.dup
         str << expression
         str << consume(:close_square)
         str << variable_lookups

data/lib/liquid/partial_cache.rb

@@ -4,7 +4,8 @@ module Liquid
   class PartialCache
     def self.load(template_name, context:, parse_context:)
       cached_partials = context.registers[:cached_partials]
-      cached = cached_partials[template_name]
+      cache_key = "#{template_name}:#{parse_context.error_mode}"
+      cached = cached_partials[cache_key]
       return cached if cached
 
       file_system = context.registers[:file_system]
@@ -15,8 +16,16 @@ module Liquid
       template_factory = context.registers[:template_factory]
       template = template_factory.for(template_name)
 
-      partial = template.parse(source, parse_context)
-      cached_partials[template_name] = partial
+      begin
+        partial = template.parse(source, parse_context)
+      rescue Liquid::Error => e
+        e.template_name = template&.name || template_name
+        raise e
+      end
+
+      partial.name ||= template_name
+
+      cached_partials[cache_key] = partial
     ensure
       parse_context.partial = false
     end

data/lib/liquid/range_lookup.rb

@@ -2,13 +2,23 @@
 
 module Liquid
   class RangeLookup
-    def self.parse(start_markup, end_markup)
-      start_obj = Expression.parse(start_markup)
-      end_obj   = Expression.parse(end_markup)
+    def self.parse(start_markup, end_markup, string_scanner, cache = nil)
+      start_obj = Expression.parse(start_markup, string_scanner, cache)
+      end_obj   = Expression.parse(end_markup, string_scanner, cache)
       if start_obj.respond_to?(:evaluate) || end_obj.respond_to?(:evaluate)
         new(start_obj, end_obj)
       else
-        start_obj.to_i..end_obj.to_i
+        begin
+          start_obj.to_i..end_obj.to_i
+        rescue NoMethodError
+          invalid_expr = start_markup unless start_obj.respond_to?(:to_i)
+          invalid_expr ||= end_markup unless end_obj.respond_to?(:to_i)
+          if invalid_expr
+            raise Liquid::SyntaxError, "Invalid expression type '#{invalid_expr}' in range expression"
+          end
+
+          raise
+        end
       end
     end