liquid 4.0.0 → 5.10.0

data/lib/liquid/tokenizer.rb

@@ -1,31 +1,161 @@
+# frozen_string_literal: true
+
+require "strscan"
+
 module Liquid
   class Tokenizer
-    attr_reader :line_number
+    attr_reader :line_number, :for_liquid_tag
+
+    TAG_END = /%\}/
+    TAG_OR_VARIABLE_START = /\{[\{\%]/
+    NEWLINE = /\n/
+
+    OPEN_CURLEY = "{".ord
+    CLOSE_CURLEY = "}".ord
+    PERCENTAGE = "%".ord
+
+    def initialize(
+      source:,
+      string_scanner:,
+      line_numbers: false,
+      line_number: nil,
+      for_liquid_tag: false
+    )
+      @line_number = line_number || (line_numbers ? 1 : nil)
+      @for_liquid_tag = for_liquid_tag
+      @source = source.to_s.to_str
+      @offset = 0
+      @tokens = []
 
-    def initialize(source, line_numbers = false)
-      @source = source
-      @line_number = line_numbers ? 1 : nil
-      @tokens = tokenize
+      if @source
+        @ss = string_scanner
+        @ss.string = @source
+        tokenize
+      end
     end
 
     def shift
-      token = @tokens.shift
-      @line_number += token.count("\n") if @line_number && token
+      token = @tokens[@offset]
+
+      return unless token
+
+      @offset += 1
+
+      if @line_number
+        @line_number += @for_liquid_tag ? 1 : token.count("\n")
+      end
+
       token
     end
 
     private
 
     def tokenize
-      @source = @source.source if @source.respond_to?(:source)
-      return [] if @source.to_s.empty?
+      if @for_liquid_tag
+        @tokens = @source.split("\n")
+      else
+        @tokens << shift_normal until @ss.eos?
+      end
+
+      @source = nil
+      @ss = nil
+    end
+
+    def shift_normal
+      token = next_token
+
+      return unless token
+
+      token
+    end
+
+    def next_token
+      # possible states: :text, :tag, :variable
+      byte_a = @ss.peek_byte
+
+      if byte_a == OPEN_CURLEY
+        @ss.scan_byte
+
+        byte_b = @ss.peek_byte
 
-      tokens = @source.split(TemplateParser)
+        if byte_b == PERCENTAGE
+          @ss.scan_byte
+          return next_tag_token
+        elsif byte_b == OPEN_CURLEY
+          @ss.scan_byte
+          return next_variable_token
+        end
 
-      # removes the rogue empty element at the beginning of the array
-      tokens.shift if tokens[0] && tokens[0].empty?
+        @ss.pos -= 1
+      end
+
+      next_text_token
+    end
+
+    def next_text_token
+      start = @ss.pos
+
+      unless @ss.skip_until(TAG_OR_VARIABLE_START)
+        token = @ss.rest
+        @ss.terminate
+        return token
+      end
+
+      pos = @ss.pos -= 2
+      @source.byteslice(start, pos - start)
+    rescue ::ArgumentError => e
+      if e.message == "invalid byte sequence in #{@ss.string.encoding}"
+        raise SyntaxError, "Invalid byte sequence in #{@ss.string.encoding}"
+      else
+        raise
+      end
+    end
+
+    def next_variable_token
+      start = @ss.pos - 2
+
+      byte_a = byte_b = @ss.scan_byte
+
+      while byte_b
+        byte_a = @ss.scan_byte while byte_a && (byte_a != CLOSE_CURLEY && byte_a != OPEN_CURLEY)
+
+        break unless byte_a
+
+        if @ss.eos?
+          return byte_a == CLOSE_CURLEY ? @source.byteslice(start, @ss.pos - start) : "{{"
+        end
+
+        byte_b = @ss.scan_byte
+
+        if byte_a == CLOSE_CURLEY
+          if byte_b == CLOSE_CURLEY
+            return @source.byteslice(start, @ss.pos - start)
+          elsif byte_b != CLOSE_CURLEY
+            @ss.pos -= 1
+            return @source.byteslice(start, @ss.pos - start)
+          end
+        elsif byte_a == OPEN_CURLEY && byte_b == PERCENTAGE
+          return next_tag_token_with_start(start)
+        end
+
+        byte_a = byte_b
+      end
+
+      "{{"
+    end
+
+    def next_tag_token
+      start = @ss.pos - 2
+      if (len = @ss.skip_until(TAG_END))
+        @source.byteslice(start, len + 2)
+      else
+        "{%"
+      end
+    end
 
-      tokens
+    def next_tag_token_with_start(start)
+      @ss.skip_until(TAG_END)
+      @source.byteslice(start, @ss.pos - start)
     end
   end
 end

data/lib/liquid/usage.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Liquid
+  module Usage
+    def self.increment(name)
+    end
+  end
+end

data/lib/liquid/utils.rb

@@ -1,5 +1,10 @@
+# frozen_string_literal: true
+
 module Liquid
   module Utils
+    DECIMAL_REGEX = /\A-?\d+\.\d+\z/
+    UNIX_TIMESTAMP_REGEX = /\A\d+\z/
+
     def self.slice_collection(collection, from, to)
       if (from != 0 || !to.nil?) && collection.respond_to?(:load_slice)
         collection.load_slice(from, to)
@@ -10,7 +15,7 @@ module Liquid
 
     def self.slice_collection_using_each(collection, from, to)
       segments = []
-      index = 0
+      index    = 0
 
       # Maintains Ruby 1.8.7 String#each behaviour on 1.9
       if collection.is_a?(String)
@@ -46,11 +51,11 @@ module Liquid
     def self.to_number(obj)
       case obj
       when Float
-        BigDecimal.new(obj.to_s)
+        BigDecimal(obj.to_s)
       when Numeric
         obj
       when String
-        (obj.strip =~ /\A-?\d+\.\d+\z/) ? BigDecimal.new(obj) : obj.to_i
+        DECIMAL_REGEX.match?(obj.strip) ? BigDecimal(obj) : obj.to_i
       else
         if obj.respond_to?(:to_number)
           obj.to_number
@@ -69,9 +74,9 @@ module Liquid
       end
 
       case obj
-      when 'now'.freeze, 'today'.freeze
+      when 'now', 'today'
         Time.now
-      when /\A\d+\z/, Integer
+      when UNIX_TIMESTAMP_REGEX, Integer
         Time.at(obj.to_i)
       when String
         Time.parse(obj)
@@ -79,5 +84,109 @@ module Liquid
     rescue ::ArgumentError
       nil
     end
+
+    def self.to_liquid_value(obj)
+      # Enable "obj" to represent itself as a primitive value like integer, string, or boolean
+      return obj.to_liquid_value if obj.respond_to?(:to_liquid_value)
+
+      # Otherwise return the object itself
+      obj
+    end
+
+    def self.to_s(obj, seen = {})
+      case obj
+      when Hash
+        # If the custom hash implementation overrides `#to_s`, use their
+        # custom implementation. Otherwise we use Liquid's default
+        # implementation.
+        if obj.class.instance_method(:to_s) == HASH_TO_S_METHOD
+          hash_inspect(obj, seen)
+        else
+          obj.to_s
+        end
+      when Array
+        array_inspect(obj, seen)
+      else
+        obj.to_s
+      end
+    end
+
+    def self.inspect(obj, seen = {})
+      case obj
+      when Hash
+        # If the custom hash implementation overrides `#inspect`, use their
+        # custom implementation. Otherwise we use Liquid's default
+        # implementation.
+        if obj.class.instance_method(:inspect) == HASH_INSPECT_METHOD
+          hash_inspect(obj, seen)
+        else
+          obj.inspect
+        end
+      when Array
+        array_inspect(obj, seen)
+      else
+        obj.inspect
+      end
+    end
+
+    def self.array_inspect(arr, seen = {})
+      if seen[arr.object_id]
+        return "[...]"
+      end
+
+      seen[arr.object_id] = true
+      str = +"["
+      cursor = 0
+      len = arr.length
+
+      while cursor < len
+        if cursor > 0
+          str << ", "
+        end
+
+        item_str = inspect(arr[cursor], seen)
+        str << item_str
+        cursor += 1
+      end
+
+      str << "]"
+      str
+    ensure
+      seen.delete(arr.object_id)
+    end
+
+    def self.hash_inspect(hash, seen = {})
+      if seen[hash.object_id]
+        return "{...}"
+      end
+      seen[hash.object_id] = true
+
+      str = +"{"
+      first = true
+      hash.each do |key, value|
+        if first
+          first = false
+        else
+          str << ", "
+        end
+
+        key_str = inspect(key, seen)
+        str << key_str
+        str << "=>"
+
+        value_str = inspect(value, seen)
+        str << value_str
+      end
+      str << "}"
+      str
+    ensure
+      seen.delete(hash.object_id)
+    end
+
+    HASH_TO_S_METHOD = Hash.instance_method(:to_s)
+    private_constant :HASH_TO_S_METHOD
+
+    HASH_INSPECT_METHOD = Hash.instance_method(:inspect)
+    private_constant :HASH_INSPECT_METHOD
   end
 end

data/lib/liquid/variable.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Liquid
   # Holds variables. Variables are only loaded "just in time"
   # and are not evaluated as part of the render stage
@@ -10,19 +12,25 @@ module Liquid
   #   {{ user | link }}
   #
   class Variable
-    FilterParser = /(?:\s+|#{QuotedFragment}|#{ArgumentSeparator})+/o
+    FilterMarkupRegex        = /#{FilterSeparator}\s*(.*)/om
+    FilterParser             = /(?:\s+|#{QuotedFragment}|#{ArgumentSeparator})+/o
+    FilterArgsRegex          = /(?:#{FilterArgumentSeparator}|#{ArgumentSeparator})\s*((?:\w+\s*\:\s*)?#{QuotedFragment})/o
+    JustTagAttributes        = /\A#{TagAttributes}\z/o
+    MarkupWithQuotedFragment = /(#{QuotedFragment})(.*)/om
+
     attr_accessor :filters, :name, :line_number
     attr_reader :parse_context
     alias_method :options, :parse_context
+
     include ParserSwitching
 
     def initialize(markup, parse_context)
-      @markup  = markup
-      @name    = nil
+      @markup        = markup
+      @name          = nil
       @parse_context = parse_context
-      @line_number = parse_context.line_number
+      @line_number   = parse_context.line_number
 
-      parse_with_selected_parser(markup)
+      strict_parse_with_error_mode_fallback(markup)
     end
 
     def raw
@@ -35,35 +43,48 @@ module Liquid
 
     def lax_parse(markup)
       @filters = []
-      return unless markup =~ /(#{QuotedFragment})(.*)/om
+      return unless markup =~ MarkupWithQuotedFragment
 
-      name_markup = $1
-      filter_markup = $2
-      @name = Expression.parse(name_markup)
-      if filter_markup =~ /#{FilterSeparator}\s*(.*)/om
-        filters = $1.scan(FilterParser)
+      name_markup   = Regexp.last_match(1)
+      filter_markup = Regexp.last_match(2)
+      @name         = parse_context.parse_expression(name_markup)
+      if filter_markup =~ FilterMarkupRegex
+        filters = Regexp.last_match(1).scan(FilterParser)
         filters.each do |f|
           next unless f =~ /\w+/
           filtername = Regexp.last_match(0)
-          filterargs = f.scan(/(?:#{FilterArgumentSeparator}|#{ArgumentSeparator})\s*((?:\w+\s*\:\s*)?#{QuotedFragment})/o).flatten
-          @filters << parse_filter_expressions(filtername, filterargs)
+          filterargs = f.scan(FilterArgsRegex).flatten
+          @filters << lax_parse_filter_expressions(filtername, filterargs)
         end
       end
     end
 
     def strict_parse(markup)
       @filters = []
-      p = Parser.new(markup)
+      p = @parse_context.new_parser(markup)
 
-      @name = Expression.parse(p.expression)
+      return if p.look(:end_of_string)
+
+      @name = parse_context.safe_parse_expression(p)
       while p.consume?(:pipe)
         filtername = p.consume(:id)
-        filterargs = p.consume?(:colon) ? parse_filterargs(p) : []
-        @filters << parse_filter_expressions(filtername, filterargs)
+        filterargs = p.consume?(:colon) ? parse_filterargs(p) : Const::EMPTY_ARRAY
+        @filters << lax_parse_filter_expressions(filtername, filterargs)
       end
       p.consume(:end_of_string)
     end
 
+    def rigid_parse(markup)
+      @filters = []
+      p = @parse_context.new_parser(markup)
+
+      return if p.look(:end_of_string)
+
+      @name = parse_context.safe_parse_expression(p)
+      @filters << rigid_parse_filter_expressions(p) while p.consume?(:pipe)
+      p.consume(:end_of_string)
+    end
+
     def parse_filterargs(p)
       # first argument
       filterargs = [p.argument]
@@ -73,37 +94,103 @@ module Liquid
     end
 
     def render(context)
-      obj = @filters.inject(context.evaluate(@name)) do |output, (filter_name, filter_args, filter_kwargs)|
+      obj = context.evaluate(@name)
+
+      @filters.each do |filter_name, filter_args, filter_kwargs|
         filter_args = evaluate_filter_expressions(context, filter_args, filter_kwargs)
-        context.invoke(filter_name, output, *filter_args)
+        obj = context.invoke(filter_name, obj, *filter_args)
       end
 
-      obj = context.apply_global_filter(obj)
+      context.apply_global_filter(obj)
+    end
 
-      taint_check(context, obj)
+    def render_to_output_buffer(context, output)
+      obj = render(context)
+      render_obj_to_output(obj, output)
+      output
+    end
+
+    def render_obj_to_output(obj, output)
+      case obj
+      when NilClass
+        # Do nothing
+      when Array
+        obj.each do |o|
+          render_obj_to_output(o, output)
+        end
+      else
+        output << Liquid::Utils.to_s(obj)
+      end
+    end
+
+    def disabled?(_context)
+      false
+    end
 
-      obj
+    def disabled_tags
+      []
     end
 
     private
 
-    def parse_filter_expressions(filter_name, unparsed_args)
-      filter_args = []
-      keyword_args = {}
+    def lax_parse_filter_expressions(filter_name, unparsed_args)
+      filter_args  = []
+      keyword_args = nil
       unparsed_args.each do |a|
-        if matches = a.match(/\A#{TagAttributes}\z/o)
-          keyword_args[matches[1]] = Expression.parse(matches[2])
+        if (matches = a.match(JustTagAttributes))
+          keyword_args           ||= {}
+          keyword_args[matches[1]] = parse_context.parse_expression(matches[2])
         else
-          filter_args << Expression.parse(a)
+          filter_args << parse_context.parse_expression(a)
         end
       end
       result = [filter_name, filter_args]
+      result << keyword_args if keyword_args
+      result
+    end
+
+    # Surprisingly, positional and keyword arguments can be mixed.
+    #
+    # filter = filtername [":" filterargs?]
+    # filterargs = argument ("," argument)*
+    # argument = (positional_argument | keyword_argument)
+    # positional_argument = expression
+    # keyword_argument = id ":" expression
+    def rigid_parse_filter_expressions(p)
+      filtername = p.consume(:id)
+      filter_args = []
+      keyword_args = {}
+
+      if p.consume?(:colon)
+        # Parse first argument (no leading comma)
+        argument(p, filter_args, keyword_args) unless end_of_arguments?(p)
+
+        # Parse remaining arguments (with leading commas) and optional trailing comma
+        argument(p, filter_args, keyword_args) while p.consume?(:comma) && !end_of_arguments?(p)
+      end
+
+      result = [filtername, filter_args]
       result << keyword_args unless keyword_args.empty?
       result
     end
 
+    def argument(p, positional_arguments, keyword_arguments)
+      if p.look(:id) && p.look(:colon, 1)
+        key = p.consume(:id)
+        p.consume(:colon)
+        value = parse_context.safe_parse_expression(p)
+        keyword_arguments[key] = value
+      else
+        positional_arguments << parse_context.safe_parse_expression(p)
+      end
+    end
+
+    def end_of_arguments?(p)
+      p.look(:pipe) || p.look(:end_of_string)
+    end
+
     def evaluate_filter_expressions(context, filter_args, filter_kwargs)
-      parsed_args = filter_args.map{ |expr| context.evaluate(expr) }
+      parsed_args = filter_args.map { |expr| context.evaluate(expr) }
       if filter_kwargs
         parsed_kwargs = {}
         filter_kwargs.each do |key, expr|
@@ -114,22 +201,9 @@ module Liquid
       parsed_args
     end
 
-    def taint_check(context, obj)
-      return unless obj.tainted?
-      return if Template.taint_mode == :lax
-
-      @markup =~ QuotedFragment
-      name = Regexp.last_match(0)
-
-      error = TaintedError.new("variable '#{name}' is tainted and was not escaped")
-      error.line_number = line_number
-      error.template_name = context.template_name
-
-      case Template.taint_mode
-      when :warn
-        context.warnings << error
-      when :error
-        raise error
+    class ParseTreeVisitor < Liquid::ParseTreeVisitor
+      def children
+        [@node.name] + @node.filters.flatten
       end
     end
   end

data/lib/liquid/variable_lookup.rb

@@ -1,43 +1,59 @@
+# frozen_string_literal: true
+
 module Liquid
   class VariableLookup
-    SQUARE_BRACKETED = /\A\[(.*)\]\z/m
-    COMMAND_METHODS = ['size'.freeze, 'first'.freeze, 'last'.freeze]
+    COMMAND_METHODS = ['size', 'first', 'last'].freeze
 
     attr_reader :name, :lookups
 
-    def self.parse(markup)
-      new(markup)
+    def self.parse(markup, string_scanner = StringScanner.new(""), cache = nil)
+      new(markup, string_scanner, cache)
     end
 
-    def initialize(markup)
+    def initialize(markup, string_scanner = StringScanner.new(""), cache = nil)
       lookups = markup.scan(VariableParser)
 
       name = lookups.shift
-      if name =~ SQUARE_BRACKETED
-        name = Expression.parse($1)
+      if name&.start_with?('[') && name&.end_with?(']')
+        name = Expression.parse(
+          name[1..-2],
+          string_scanner,
+          cache,
+        )
       end
       @name = name
 
-      @lookups = lookups
+      @lookups       = lookups
       @command_flags = 0
 
       @lookups.each_index do |i|
         lookup = lookups[i]
-        if lookup =~ SQUARE_BRACKETED
-          lookups[i] = Expression.parse($1)
+        if lookup&.start_with?('[') && lookup&.end_with?(']')
+          lookups[i] = Expression.parse(
+            lookup[1..-2],
+            string_scanner,
+            cache,
+          )
         elsif COMMAND_METHODS.include?(lookup)
           @command_flags |= 1 << i
         end
       end
     end
 
+    def lookup_command?(lookup_index)
+      @command_flags & (1 << lookup_index) != 0
+    end
+
     def evaluate(context)
-      name = context.evaluate(@name)
+      name   = context.evaluate(@name)
       object = context.find_variable(name)
 
       @lookups.each_index do |i|
         key = context.evaluate(@lookups[i])
 
+        # Cast "key" to its liquid value to enable it to act as a primitive value
+        key = Liquid::Utils.to_liquid_value(key)
+
         # If object is a hash- or array-like object we look for the
         # presence of the key and if its available we return it
         if object.respond_to?(:[]) &&
@@ -45,13 +61,13 @@ module Liquid
              (object.respond_to?(:fetch) && key.is_a?(Integer)))
 
           # if its a proc we will replace the entry with the proc
-          res = context.lookup_and_evaluate(object, key)
+          res    = context.lookup_and_evaluate(object, key)
           object = res.to_liquid
 
           # Some special cases. If the part wasn't in square brackets and
           # no key with the same name was found we interpret following calls
           # as commands and call them on the current object
-        elsif @command_flags & (1 << i) != 0 && object.respond_to?(key)
+        elsif lookup_command?(i) && object.respond_to?(key)
           object = object.send(key).to_liquid
 
           # No key was present with the desired value and it wasn't one of the directly supported
@@ -78,5 +94,11 @@ module Liquid
     def state
       [@name, @lookups, @command_flags]
     end
+
+    class ParseTreeVisitor < Liquid::ParseTreeVisitor
+      def children
+        @node.lookups
+      end
+    end
   end
 end

data/lib/liquid/version.rb

@@ -1,4 +1,6 @@
 # encoding: utf-8
+# frozen_string_literal: true
+
 module Liquid
-  VERSION = "4.0.0"
+  VERSION = "5.10.0"
 end