liquid 4.0.0 → 5.10.0

data/lib/liquid/block_body.rb

@@ -1,52 +1,171 @@
+# frozen_string_literal: true
+
+require 'English'
+
 module Liquid
   class BlockBody
-    FullToken = /\A#{TagStart}#{WhitespaceControl}?\s*(\w+)\s*(.*?)#{WhitespaceControl}?#{TagEnd}\z/om
-    ContentOfVariable = /\A#{VariableStart}#{WhitespaceControl}?(.*?)#{WhitespaceControl}?#{VariableEnd}\z/om
-    TAGSTART = "{%".freeze
-    VARSTART = "{{".freeze
+    LiquidTagToken      = /\A\s*(#{TagName})\s*(.*?)\z/o
+    FullToken           = /\A#{TagStart}#{WhitespaceControl}?(\s*)(#{TagName})(\s*)(.*?)#{WhitespaceControl}?#{TagEnd}\z/om
+    FullTokenPossiblyInvalid = /\A(.*)#{TagStart}#{WhitespaceControl}?\s*(\w+)\s*(.*)?#{WhitespaceControl}?#{TagEnd}\z/om
+    ContentOfVariable   = /\A#{VariableStart}#{WhitespaceControl}?(.*?)#{WhitespaceControl}?#{VariableEnd}\z/om
+    WhitespaceOrNothing = /\A\s*\z/
+    TAGSTART            = "{%"
+    VARSTART            = "{{"
 
     attr_reader :nodelist
 
     def initialize
       @nodelist = []
-      @blank = true
+      @blank    = true
     end
 
-    def parse(tokenizer, parse_context)
+    def parse(tokenizer, parse_context, &block)
+      raise FrozenError, "can't modify frozen Liquid::BlockBody" if frozen?
+
       parse_context.line_number = tokenizer.line_number
-      while token = tokenizer.shift
-        unless token.empty?
-          case
-          when token.start_with?(TAGSTART)
-            whitespace_handler(token, parse_context)
-            if token =~ FullToken
-              tag_name = $1
-              markup = $2
-              # fetch the tag from registered blocks
-              if tag = registered_tags[tag_name]
-                new_tag = tag.parse(tag_name, markup, tokenizer, parse_context)
-                @blank &&= new_tag.blank?
-                @nodelist << new_tag
-              else
-                # end parsing if we reach an unknown tag and let the caller decide
-                # determine how to proceed
-                return yield tag_name, markup
-              end
-            else
-              raise_missing_tag_terminator(token, parse_context)
-            end
-          when token.start_with?(VARSTART)
-            whitespace_handler(token, parse_context)
-            @nodelist << create_variable(token, parse_context)
-            @blank = false
-          else
-            if parse_context.trim_whitespace
-              token.lstrip!
-            end
-            parse_context.trim_whitespace = false
-            @nodelist << token
-            @blank &&= !!(token =~ /\A\s*\z/)
+
+      if tokenizer.for_liquid_tag
+        parse_for_liquid_tag(tokenizer, parse_context, &block)
+      else
+        parse_for_document(tokenizer, parse_context, &block)
+      end
+    end
+
+    def freeze
+      @nodelist.freeze
+      super
+    end
+
+    private def parse_for_liquid_tag(tokenizer, parse_context)
+      while (token = tokenizer.shift)
+        unless token.empty? || token.match?(WhitespaceOrNothing)
+          unless token =~ LiquidTagToken
+            # line isn't empty but didn't match tag syntax, yield and let the
+            # caller raise a syntax error
+            return yield token, token
+          end
+          tag_name = Regexp.last_match(1)
+          markup   = Regexp.last_match(2)
+
+          if tag_name == 'liquid'
+            parse_context.line_number -= 1
+            next parse_liquid_tag(markup, parse_context)
+          end
+
+          unless (tag = parse_context.environment.tag_for_name(tag_name))
+            # end parsing if we reach an unknown tag and let the caller decide
+            # determine how to proceed
+            return yield tag_name, markup
           end
+          new_tag = tag.parse(tag_name, markup, tokenizer, parse_context)
+          @blank &&= new_tag.blank?
+          @nodelist << new_tag
+        end
+        parse_context.line_number = tokenizer.line_number
+      end
+
+      yield nil, nil
+    end
+
+    # @api private
+    def self.unknown_tag_in_liquid_tag(tag, parse_context)
+      Block.raise_unknown_tag(tag, 'liquid', '%}', parse_context)
+    end
+
+    # @api private
+    def self.raise_missing_tag_terminator(token, parse_context)
+      raise SyntaxError, parse_context.locale.t("errors.syntax.tag_termination", token: token, tag_end: TagEnd.inspect)
+    end
+
+    # @api private
+    def self.raise_missing_variable_terminator(token, parse_context)
+      raise SyntaxError, parse_context.locale.t("errors.syntax.variable_termination", token: token, tag_end: VariableEnd.inspect)
+    end
+
+    # @api private
+    def self.render_node(context, output, node)
+      node.render_to_output_buffer(context, output)
+    rescue => exc
+      blank_tag = !node.instance_of?(Variable) && node.blank?
+      rescue_render_node(context, output, node.line_number, exc, blank_tag)
+    end
+
+    # @api private
+    def self.rescue_render_node(context, output, line_number, exc, blank_tag)
+      case exc
+      when MemoryError
+        raise
+      when UndefinedVariable, UndefinedDropMethod, UndefinedFilter
+        context.handle_error(exc, line_number)
+      else
+        error_message = context.handle_error(exc, line_number)
+        unless blank_tag # conditional for backwards compatibility
+          output << error_message
+        end
+      end
+    end
+
+    private def parse_liquid_tag(markup, parse_context)
+      liquid_tag_tokenizer = parse_context.new_tokenizer(
+        markup, start_line_number: parse_context.line_number, for_liquid_tag: true
+      )
+      parse_for_liquid_tag(liquid_tag_tokenizer, parse_context) do |end_tag_name, _end_tag_markup|
+        if end_tag_name
+          BlockBody.unknown_tag_in_liquid_tag(end_tag_name, parse_context)
+        end
+      end
+    end
+
+    private def handle_invalid_tag_token(token, parse_context)
+      if token.end_with?('%}')
+        yield token, token
+      else
+        BlockBody.raise_missing_tag_terminator(token, parse_context)
+      end
+    end
+
+    private def parse_for_document(tokenizer, parse_context, &block)
+      while (token = tokenizer.shift)
+        next if token.empty?
+        case
+        when token.start_with?(TAGSTART)
+          whitespace_handler(token, parse_context)
+          unless token =~ FullToken
+            return handle_invalid_tag_token(token, parse_context, &block)
+          end
+          tag_name = Regexp.last_match(2)
+          markup   = Regexp.last_match(4)
+
+          if parse_context.line_number
+            # newlines inside the tag should increase the line number,
+            # particularly important for multiline {% liquid %} tags
+            parse_context.line_number += Regexp.last_match(1).count("\n") + Regexp.last_match(3).count("\n")
+          end
+
+          if tag_name == 'liquid'
+            parse_liquid_tag(markup, parse_context)
+            next
+          end
+
+          unless (tag = parse_context.environment.tag_for_name(tag_name))
+            # end parsing if we reach an unknown tag and let the caller decide
+            # determine how to proceed
+            return yield tag_name, markup
+          end
+          new_tag = tag.parse(tag_name, markup, tokenizer, parse_context)
+          @blank &&= new_tag.blank?
+          @nodelist << new_tag
+        when token.start_with?(VARSTART)
+          whitespace_handler(token, parse_context)
+          @nodelist << create_variable(token, parse_context)
+          @blank = false
+        else
+          if parse_context.trim_whitespace
+            token.lstrip!
+          end
+          parse_context.trim_whitespace = false
+          @nodelist << token
+          @blank &&= token.match?(WhitespaceOrNothing)
         end
         parse_context.line_number = tokenizer.line_number
       end
@@ -57,8 +176,12 @@ module Liquid
     def whitespace_handler(token, parse_context)
       if token[2] == WhitespaceControl
         previous_token = @nodelist.last
-        if previous_token.is_a? String
+        if previous_token.is_a?(String)
+          first_byte = previous_token.getbyte(0)
           previous_token.rstrip!
+          if previous_token.empty? && parse_context[:bug_compatible_whitespace_trimming] && first_byte
+            previous_token << first_byte
+          end
         end
       end
       parse_context.trim_whitespace = (token[-3] == WhitespaceControl)
@@ -68,72 +191,83 @@ module Liquid
       @blank
     end
 
+    # Remove blank strings in the block body for a control flow tag (e.g. `if`, `for`, `case`, `unless`)
+    # with a blank body.
+    #
+    # For example, in a conditional assignment like the following
+    #
+    # ```
+    # {% if size > max_size %}
+    #   {% assign size = max_size %}
+    # {% endif %}
+    # ```
+    #
+    # we assume the intention wasn't to output the blank spaces in the `if` tag's block body, so this method
+    # will remove them to reduce the render output size.
+    #
+    # Note that it is now preferred to use the `liquid` tag for this use case.
+    def remove_blank_strings
+      raise "remove_blank_strings only support being called on a blank block body" unless @blank
+      @nodelist.reject! { |node| node.instance_of?(String) }
+    end
+
     def render(context)
-      output = []
-      context.resource_limits.render_score += @nodelist.length
+      render_to_output_buffer(context, +'')
+    end
+
+    def render_to_output_buffer(context, output)
+      freeze unless frozen?
 
-      @nodelist.each do |token|
-        # Break out if we have any unhanded interrupts.
-        break if context.interrupt?
+      context.resource_limits.increment_render_score(@nodelist.length)
 
-        begin
+      idx = 0
+      while (node = @nodelist[idx])
+        if node.instance_of?(String)
+          output << node
+        else
+          render_node(context, output, node)
           # If we get an Interrupt that means the block must stop processing. An
           # Interrupt is any command that stops block execution such as {% break %}
-          # or {% continue %}
-          if token.is_a?(Continue) || token.is_a?(Break)
-            context.push_interrupt(token.interrupt)
-            break
-          end
-
-          node_output = render_node(token, context)
-
-          unless token.is_a?(Block) && token.blank?
-            output << node_output
-          end
-        rescue MemoryError => e
-          raise e
-        rescue UndefinedVariable, UndefinedDropMethod, UndefinedFilter => e
-          context.handle_error(e, token.line_number, token.raw)
-          output << nil
-        rescue ::StandardError => e
-          output << context.handle_error(e, token.line_number, token.raw)
+          # or {% continue %}. These tags may also occur through Block or Include tags.
+          break if context.interrupt? # might have happened in a for-block
         end
+        idx += 1
+
+        context.resource_limits.increment_write_score(output)
       end
 
-      output.join
+      output
     end
 
     private
 
-    def render_node(node, context)
-      node_output = (node.respond_to?(:render) ? node.render(context) : node)
-      node_output = node_output.is_a?(Array) ? node_output.join : node_output.to_s
-
-      context.resource_limits.render_length += node_output.length
-      if context.resource_limits.reached?
-        raise MemoryError.new("Memory limits exceeded".freeze)
-      end
-      node_output
+    def render_node(context, output, node)
+      BlockBody.render_node(context, output, node)
     end
 
     def create_variable(token, parse_context)
-      token.scan(ContentOfVariable) do |content|
-        markup = content.first
+      if token.end_with?("}}")
+        i = 2
+        i = 3 if token[i] == "-"
+        parse_end = token.length - 3
+        parse_end -= 1 if token[parse_end] == "-"
+        markup_end = parse_end - i + 1
+        markup = markup_end <= 0 ? "" : token.slice(i, markup_end)
+
         return Variable.new(markup, parse_context)
       end
-      raise_missing_variable_terminator(token, parse_context)
+
+      BlockBody.raise_missing_variable_terminator(token, parse_context)
     end
 
+    # @deprecated Use {.raise_missing_tag_terminator} instead
     def raise_missing_tag_terminator(token, parse_context)
-      raise SyntaxError.new(parse_context.locale.t("errors.syntax.tag_termination".freeze, token: token, tag_end: TagEnd.inspect))
+      BlockBody.raise_missing_tag_terminator(token, parse_context)
     end
 
+    # @deprecated Use {.raise_missing_variable_terminator} instead
     def raise_missing_variable_terminator(token, parse_context)
-      raise SyntaxError.new(parse_context.locale.t("errors.syntax.variable_termination".freeze, token: token, tag_end: VariableEnd.inspect))
-    end
-
-    def registered_tags
-      Template.tags
+      BlockBody.raise_missing_variable_terminator(token, parse_context)
     end
   end
 end

data/lib/liquid/condition.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Liquid
   # Container for liquid nodes which conveniently wraps decision making logic
   #
@@ -6,60 +8,88 @@ module Liquid
   #   c = Condition.new(1, '==', 1)
   #   c.evaluate #=> true
   #
-  class Condition #:nodoc:
+  class Condition # :nodoc:
     @@operators = {
-      '=='.freeze => ->(cond, left, right) {  cond.send(:equal_variables, left, right) },
-      '!='.freeze => ->(cond, left, right) { !cond.send(:equal_variables, left, right) },
-      '<>'.freeze => ->(cond, left, right) { !cond.send(:equal_variables, left, right) },
-      '<'.freeze  => :<,
-      '>'.freeze  => :>,
-      '>='.freeze => :>=,
-      '<='.freeze => :<=,
-      'contains'.freeze => lambda do |cond, left, right|
+      '==' => ->(cond, left, right) {  cond.send(:equal_variables, left, right) },
+      '!=' => ->(cond, left, right) { !cond.send(:equal_variables, left, right) },
+      '<>' => ->(cond, left, right) { !cond.send(:equal_variables, left, right) },
+      '<' => :<,
+      '>' => :>,
+      '>=' => :>=,
+      '<=' => :<=,
+      'contains' => lambda do |_cond, left, right|
         if left && right && left.respond_to?(:include?)
           right = right.to_s if left.is_a?(String)
           left.include?(right)
         else
           false
         end
+      rescue Encoding::CompatibilityError
+        # "✅".b.include?("✅") raises Encoding::CompatibilityError despite being materially equal
+        left.b.include?(right.b)
+      end,
+    }
+
+    class MethodLiteral
+      attr_reader :method_name, :to_s
+
+      def initialize(method_name, to_s)
+        @method_name = method_name
+        @to_s = to_s
       end
+    end
+
+    @@method_literals = {
+      'blank' => MethodLiteral.new(:blank?, '').freeze,
+      'empty' => MethodLiteral.new(:empty?, '').freeze,
     }
 
     def self.operators
       @@operators
     end
 
-    attr_reader :attachment
+    def self.parse_expression(parse_context, markup, safe: false)
+      @@method_literals[markup] || parse_context.parse_expression(markup, safe: safe)
+    end
+
+    attr_reader :attachment, :child_condition
     attr_accessor :left, :operator, :right
 
     def initialize(left = nil, operator = nil, right = nil)
-      @left = left
+      @left     = left
       @operator = operator
-      @right = right
+      @right    = right
+
       @child_relation  = nil
       @child_condition = nil
     end
 
-    def evaluate(context = Context.new)
-      result = interpret_condition(left, right, operator, context)
-
-      case @child_relation
-      when :or
-        result || @child_condition.evaluate(context)
-      when :and
-        result && @child_condition.evaluate(context)
-      else
-        result
+    def evaluate(context = deprecated_default_context)
+      condition = self
+      result = nil
+      loop do
+        result = interpret_condition(condition.left, condition.right, condition.operator, context)
+
+        case condition.child_relation
+        when :or
+          break if Liquid::Utils.to_liquid_value(result)
+        when :and
+          break unless Liquid::Utils.to_liquid_value(result)
+        else
+          break
+        end
+        condition = condition.child_condition
       end
+      result
     end
 
     def or(condition)
-      @child_relation = :or
+      @child_relation  = :or
       @child_condition = condition
     end
 
     def and(condition)
-      @child_relation = :and
+      @child_relation  = :and
       @child_condition = condition
     end
 
@@ -72,13 +102,17 @@ module Liquid
     end
 
     def inspect
-      "#<Condition #{[@left, @operator, @right].compact.join(' '.freeze)}>"
+      "#<Condition #{[@left, @operator, @right].compact.join(' ')}>"
     end
 
+    protected
+
+    attr_reader :child_relation
+
     private
 
     def equal_variables(left, right)
-      if left.is_a?(Liquid::Expression::MethodLiteral)
+      if left.is_a?(MethodLiteral)
         if right.respond_to?(left.method_name)
           return right.send(left.method_name)
         else
@@ -86,7 +120,7 @@ module Liquid
         end
       end
 
-      if right.is_a?(Liquid::Expression::MethodLiteral)
+      if right.is_a?(MethodLiteral)
         if left.respond_to?(right.method_name)
           return left.send(right.method_name)
         else
@@ -103,21 +137,38 @@ module Liquid
       # return this as the result.
       return context.evaluate(left) if op.nil?
 
-      left = context.evaluate(left)
-      right = context.evaluate(right)
+      left  = Liquid::Utils.to_liquid_value(context.evaluate(left))
+      right = Liquid::Utils.to_liquid_value(context.evaluate(right))
 
-      operation = self.class.operators[op] || raise(Liquid::ArgumentError.new("Unknown operator #{op}"))
+      operation = self.class.operators[op] || raise(Liquid::ArgumentError, "Unknown operator #{op}")
 
       if operation.respond_to?(:call)
         operation.call(self, left, right)
-      elsif left.respond_to?(operation) && right.respond_to?(operation)
+      elsif left.respond_to?(operation) && right.respond_to?(operation) && !left.is_a?(Hash) && !right.is_a?(Hash)
         begin
           left.send(operation, right)
         rescue ::ArgumentError => e
-          raise Liquid::ArgumentError.new(e.message)
+          raise Liquid::ArgumentError, e.message
         end
       end
     end
+
+    def deprecated_default_context
+      warn("DEPRECATION WARNING: Condition#evaluate without a context argument is deprecated" \
+        " and will be removed from Liquid 6.0.0.")
+      Context.new
+    end
+
+    class ParseTreeVisitor < Liquid::ParseTreeVisitor
+      def children
+        [
+          @node.left,
+          @node.right,
+          @node.child_condition,
+          @node.attachment
+        ].compact
+      end
+    end
   end
 
   class ElseCondition < Condition

data/lib/liquid/const.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Liquid
+  module Const
+    EMPTY_HASH = {}.freeze
+    EMPTY_ARRAY = [].freeze
+  end
+end