liquid 4.0.2 → 5.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e3124f91e43cab43cf6782b093c0387d9189bd951b911ab1ade84024e5b84bf3
-  data.tar.gz: 40c04d46b3a1a51a775832cabf161fc870bde43b72d2857a8a8cf736e1909a43
+  metadata.gz: d033978c659ebc86ff6ae3a36adaa1a90b8f69580e0cd033eb9c2c669ff6d66d
+  data.tar.gz: 5c440b450df6ac2c673b0a23091d97e497ccb8207a443b0b6011f2b1a9b6529b
 SHA512:
-  metadata.gz: 71deb80c6d970684e424fb8e4ca0a817399dc0b6564a7308093158a94ace0e24cac5d8e08a8b015327937b8dd45be25532dd8d9e02de0c3677f08297a08638dd
-  data.tar.gz: 79490a2f0db69914e01c69fdcf135cc3541609cfca29017154d3c33325c57f682c3d90bbfa0bca1a7f3d82e4cbe4af5259a77cf42861c142921491e21949bddf
+  metadata.gz: 114697a0375800b4f67d7c216f9c56f823bb4468203632d45bda92bc597e0677acd0c4151c60f92fc121c2cd1b8d0d1c18d2482f1265acb62dbf190f12773d5c
+  data.tar.gz: 3c013aef59a531829e202b73c4ce33d63ad5f58d763b7f2c610a3ff96fdab13fe2fdcf7e9401b63e09bb136fb1518ad69300b41114ebc1f076565c7d2953de78

data/History.md

@@ -1,5 +1,64 @@
 # Liquid Change Log
 
+## 5.1.0 / 2021-09-09
+
+### Features
+* Add `base64_encode`, `base64_decode`, `base64_url_safe_encode`, and `base64_url_safe_decode` filters (#1450) [Daniel Insley]
+* Introduce `to_liquid_value` in `Liquid::Drop` (#1441) [Michael Go]
+
+### Fixes
+* Fix support for using a String subclass for the liquid source (#1421) [Dylan Thacker-Smith]
+* Add `ParseTreeVisitor` to `RangeLookup` (#1470) [CP Clermont]
+* Translate `RangeError` to `Liquid::Error` for `truncatewords` with large int (#1431) [Dylan Thacker-Smith]
+
+## 5.0.1 / 2021-03-24
+
+### Fixes
+* Add ParseTreeVisitor to Echo tag (#1414) [CP Clermont]
+* Test with ruby 3.0 as the latest ruby version (#1398) [Dylan Thacker-Smith]
+* Handle carriage return in newlines_to_br (#1391) [Unending]
+
+### Performance Improvements
+* Use split limit in truncatewords (#1361) [Dylan Thacker-Smith]
+
+## 5.0.0 / 2021-01-06
+
+### Features
+* Add new `{% render %}` tag (#1122) [Samuel Doiron]
+* Add support for `as` in `{% render %}` and `{% include %}` (#1181) [Mike Angell]
+* Add `{% liquid %}` and `{% echo %}` tags (#1086) [Justin Li]
+* Add [usage tracking](README.md#usage-tracking) [Mike Angell]
+* Add `Tag.disable_tags` for disabling tags that prepend `Tag::Disableable` at render time (#1162, #1274, #1275) [Mike Angell]
+* Support using a profiler for multiple renders (#1365, #1366) [Dylan Thacker-Smith]
+
+### Fixes
+* Fix catastrophic backtracking in `RANGES_REGEX` regular expression (#1357) [Dylan Thacker-Smith]
+* Make sure the for tag's limit and offset are integers (#1094) [David Cornu]
+* Invokable methods for enumerable reject include (#1151) [Thierry Joyal]
+* Allow `default` filter to handle `false` as value (#1144) [Mike Angell]
+* Fix render length resource limit so it doesn't multiply nested output (#1285) [Dylan Thacker-Smith]
+* Fix duplication of text in raw tags (#1304) [Peter Zhu]
+* Fix strict parsing of find variable with a name expression (#1317) [Dylan Thacker-Smith]
+* Use monotonic time to measure durations in Liquid::Profiler (#1362) [Dylan Thacker-Smith]
+
+### Breaking Changes
+* Require Ruby >= 2.5 (#1131, #1310) [Mike Angell, Dylan Thacker-Smith]
+* Remove support for taint checking (#1268) [Dylan Thacker-Smith]
+* Split Strainer class into StrainerFactory and StrainerTemplate (#1208) [Thierry Joyal]
+* Remove handling of a nil context in the Strainer class (#1218) [Thierry Joyal]
+* Handle `BlockBody#blank?` at parse time (#1287) [Dylan Thacker-Smith]
+* Pass the tag markup and tokenizer to `Document#unknown_tag` (#1290) [Dylan Thacker-Smith]
+* And several internal changes
+
+### Performance Improvements
+* Reduce allocations (#1073, #1091, #1115, #1099, #1117, #1141, #1322, #1341) [Richard Monette, Florian Weingarten, Ashwin Maroli]
+* Improve resources limits performance (#1093, #1323) [Florian Weingarten, Dylan Thacker-Smith]
+
+## 4.0.3 / 2019-03-12
+
+### Fixed
+* Fix break and continue tags inside included templates in loops (#1072) [Justin Li]
+
 ## 4.0.2 / 2019-03-08
 
 ### Changed

data/README.md

@@ -106,3 +106,9 @@ template = Liquid::Template.parse("{{x}} {{y}}")
 template.render!({ 'x' => 1}, { strict_variables: true })
 #=> Liquid::UndefinedVariable: Liquid error: undefined variable y
 ```
+
+### Usage tracking
+
+To help track usages of a feature or code path in production, we have released opt-in usage tracking. To enable this, we provide an empty `Liquid:: Usage.increment` method which you can customize to your needs. The feature is well suited to https://github.com/Shopify/statsd-instrument. However, the choice of implementation is up to you.
+
+Once you have enabled usage tracking, we recommend reporting any events through Github Issues that your system may be logging. It is highly likely this event has been added to consider deprecating or improving code specific to this event, so please raise any concerns.

data/lib/liquid/block.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Liquid
   class Block < Tag
     MAX_DEPTH = 100
@@ -8,11 +10,13 @@ module Liquid
     end
 
     def parse(tokens)
-      @body = BlockBody.new
+      @body = new_body
       while parse_body(@body, tokens)
       end
+      @body.freeze
     end
 
+    # For backwards compatibility
     def render(context)
       @body.render(context)
     end
@@ -25,20 +29,29 @@ module Liquid
       @body.nodelist
     end
 
-    def unknown_tag(tag, _params, _tokens)
-      if tag == 'else'.freeze
-        raise SyntaxError.new(parse_context.locale.t("errors.syntax.unexpected_else".freeze,
-          block_name: block_name))
-      elsif tag.start_with?('end'.freeze)
-        raise SyntaxError.new(parse_context.locale.t("errors.syntax.invalid_delimiter".freeze,
+    def unknown_tag(tag_name, _markup, _tokenizer)
+      Block.raise_unknown_tag(tag_name, block_name, block_delimiter, parse_context)
+    end
+
+    # @api private
+    def self.raise_unknown_tag(tag, block_name, block_delimiter, parse_context)
+      if tag == 'else'
+        raise SyntaxError, parse_context.locale.t("errors.syntax.unexpected_else",
+          block_name: block_name)
+      elsif tag.start_with?('end')
+        raise SyntaxError, parse_context.locale.t("errors.syntax.invalid_delimiter",
           tag: tag,
           block_name: block_name,
-          block_delimiter: block_delimiter))
+          block_delimiter: block_delimiter)
       else
-        raise SyntaxError.new(parse_context.locale.t("errors.syntax.unknown_tag".freeze, tag: tag))
+        raise SyntaxError, parse_context.locale.t("errors.syntax.unknown_tag", tag: tag)
       end
     end
 
+    def raise_tag_never_closed(block_name)
+      raise SyntaxError, parse_context.locale.t("errors.syntax.tag_never_closed", block_name: block_name)
+    end
+
     def block_name
       @tag_name
     end
@@ -47,11 +60,17 @@ module Liquid
       @block_delimiter ||= "end#{block_name}"
     end
 
-    protected
+    private
+
+    # @api public
+    def new_body
+      parse_context.new_block_body
+    end
 
+    # @api public
     def parse_body(body, tokens)
       if parse_context.depth >= MAX_DEPTH
-        raise StackLevelError, "Nesting too deep".freeze
+        raise StackLevelError, "Nesting too deep"
       end
       parse_context.depth += 1
       begin
@@ -59,9 +78,7 @@ module Liquid
           @blank &&= body.blank?
 
           return false if end_tag_name == block_delimiter
-          unless end_tag_name
-            raise SyntaxError.new(parse_context.locale.t("errors.syntax.tag_never_closed".freeze, block_name: block_name))
-          end
+          raise_tag_never_closed(block_name) unless end_tag_name
 
           # this tag is not registered with the system
           # pass it to the current block for special handling or error reporting

data/lib/liquid/block_body.rb

@@ -1,32 +1,138 @@
+# frozen_string_literal: true
+
+require 'English'
+
 module Liquid
   class BlockBody
-    FullToken = /\A#{TagStart}#{WhitespaceControl}?\s*(\w+)\s*(.*?)#{WhitespaceControl}?#{TagEnd}\z/om
-    ContentOfVariable = /\A#{VariableStart}#{WhitespaceControl}?(.*?)#{WhitespaceControl}?#{VariableEnd}\z/om
+    LiquidTagToken      = /\A\s*(\w+)\s*(.*?)\z/o
+    FullToken           = /\A#{TagStart}#{WhitespaceControl}?(\s*)(\w+)(\s*)(.*?)#{WhitespaceControl}?#{TagEnd}\z/om
+    ContentOfVariable   = /\A#{VariableStart}#{WhitespaceControl}?(.*?)#{WhitespaceControl}?#{VariableEnd}\z/om
     WhitespaceOrNothing = /\A\s*\z/
-    TAGSTART = "{%".freeze
-    VARSTART = "{{".freeze
+    TAGSTART            = "{%"
+    VARSTART            = "{{"
 
     attr_reader :nodelist
 
     def initialize
       @nodelist = []
-      @blank = true
+      @blank    = true
     end
 
-    def parse(tokenizer, parse_context)
+    def parse(tokenizer, parse_context, &block)
+      raise FrozenError, "can't modify frozen Liquid::BlockBody" if frozen?
+
       parse_context.line_number = tokenizer.line_number
-      while token = tokenizer.shift
+
+      if tokenizer.for_liquid_tag
+        parse_for_liquid_tag(tokenizer, parse_context, &block)
+      else
+        parse_for_document(tokenizer, parse_context, &block)
+      end
+    end
+
+    def freeze
+      @nodelist.freeze
+      super
+    end
+
+    private def parse_for_liquid_tag(tokenizer, parse_context)
+      while (token = tokenizer.shift)
+        unless token.empty? || token =~ WhitespaceOrNothing
+          unless token =~ LiquidTagToken
+            # line isn't empty but didn't match tag syntax, yield and let the
+            # caller raise a syntax error
+            return yield token, token
+          end
+          tag_name = Regexp.last_match(1)
+          markup   = Regexp.last_match(2)
+          unless (tag = registered_tags[tag_name])
+            # end parsing if we reach an unknown tag and let the caller decide
+            # determine how to proceed
+            return yield tag_name, markup
+          end
+          new_tag = tag.parse(tag_name, markup, tokenizer, parse_context)
+          @blank &&= new_tag.blank?
+          @nodelist << new_tag
+        end
+        parse_context.line_number = tokenizer.line_number
+      end
+
+      yield nil, nil
+    end
+
+    # @api private
+    def self.unknown_tag_in_liquid_tag(tag, parse_context)
+      Block.raise_unknown_tag(tag, 'liquid', '%}', parse_context)
+    end
+
+    # @api private
+    def self.raise_missing_tag_terminator(token, parse_context)
+      raise SyntaxError, parse_context.locale.t("errors.syntax.tag_termination", token: token, tag_end: TagEnd.inspect)
+    end
+
+    # @api private
+    def self.raise_missing_variable_terminator(token, parse_context)
+      raise SyntaxError, parse_context.locale.t("errors.syntax.variable_termination", token: token, tag_end: VariableEnd.inspect)
+    end
+
+    # @api private
+    def self.render_node(context, output, node)
+      node.render_to_output_buffer(context, output)
+    rescue => exc
+      blank_tag = !node.instance_of?(Variable) && node.blank?
+      rescue_render_node(context, output, node.line_number, exc, blank_tag)
+    end
+
+    # @api private
+    def self.rescue_render_node(context, output, line_number, exc, blank_tag)
+      case exc
+      when MemoryError
+        raise
+      when UndefinedVariable, UndefinedDropMethod, UndefinedFilter
+        context.handle_error(exc, line_number)
+      else
+        error_message = context.handle_error(exc, line_number)
+        unless blank_tag # conditional for backwards compatibility
+          output << error_message
+        end
+      end
+    end
+
+    private def parse_liquid_tag(markup, parse_context)
+      liquid_tag_tokenizer = parse_context.new_tokenizer(
+        markup, start_line_number: parse_context.line_number, for_liquid_tag: true
+      )
+      parse_for_liquid_tag(liquid_tag_tokenizer, parse_context) do |end_tag_name, _end_tag_markup|
+        if end_tag_name
+          BlockBody.unknown_tag_in_liquid_tag(end_tag_name, parse_context)
+        end
+      end
+    end
+
+    private def parse_for_document(tokenizer, parse_context)
+      while (token = tokenizer.shift)
         next if token.empty?
         case
         when token.start_with?(TAGSTART)
           whitespace_handler(token, parse_context)
           unless token =~ FullToken
-            raise_missing_tag_terminator(token, parse_context)
+            BlockBody.raise_missing_tag_terminator(token, parse_context)
+          end
+          tag_name = Regexp.last_match(2)
+          markup   = Regexp.last_match(4)
+
+          if parse_context.line_number
+            # newlines inside the tag should increase the line number,
+            # particularly important for multiline {% liquid %} tags
+            parse_context.line_number += Regexp.last_match(1).count("\n") + Regexp.last_match(3).count("\n")
+          end
+
+          if tag_name == 'liquid'
+            parse_liquid_tag(markup, parse_context)
+            next
           end
-          tag_name = $1
-          markup = $2
-          # fetch the tag from registered blocks
-          unless tag = registered_tags[tag_name]
+
+          unless (tag = registered_tags[tag_name])
             # end parsing if we reach an unknown tag and let the caller decide
             # determine how to proceed
             return yield tag_name, markup
@@ -55,8 +161,12 @@ module Liquid
     def whitespace_handler(token, parse_context)
       if token[2] == WhitespaceControl
         previous_token = @nodelist.last
-        if previous_token.is_a? String
+        if previous_token.is_a?(String)
+          first_byte = previous_token.getbyte(0)
           previous_token.rstrip!
+          if previous_token.empty? && parse_context[:bug_compatible_whitespace_trimming] && first_byte
+            previous_token << first_byte
+          end
         end
       end
       parse_context.trim_whitespace = (token[-3] == WhitespaceControl)
@@ -66,57 +176,58 @@ module Liquid
       @blank
     end
 
+    # Remove blank strings in the block body for a control flow tag (e.g. `if`, `for`, `case`, `unless`)
+    # with a blank body.
+    #
+    # For example, in a conditional assignment like the following
+    #
+    # ```
+    # {% if size > max_size %}
+    #   {% assign size = max_size %}
+    # {% endif %}
+    # ```
+    #
+    # we assume the intention wasn't to output the blank spaces in the `if` tag's block body, so this method
+    # will remove them to reduce the render output size.
+    #
+    # Note that it is now preferred to use the `liquid` tag for this use case.
+    def remove_blank_strings
+      raise "remove_blank_strings only support being called on a blank block body" unless @blank
+      @nodelist.reject! { |node| node.instance_of?(String) }
+    end
+
     def render(context)
-      output = []
-      context.resource_limits.render_score += @nodelist.length
+      render_to_output_buffer(context, +'')
+    end
+
+    def render_to_output_buffer(context, output)
+      freeze unless frozen?
+
+      context.resource_limits.increment_render_score(@nodelist.length)
 
       idx = 0
-      while node = @nodelist[idx]
-        case node
-        when String
-          check_resources(context, node)
+      while (node = @nodelist[idx])
+        if node.instance_of?(String)
           output << node
-        when Variable
-          render_node_to_output(node, output, context)
-        when Block
-          render_node_to_output(node, output, context, node.blank?)
-          break if context.interrupt? # might have happened in a for-block
-        when Continue, Break
+        else
+          render_node(context, output, node)
           # If we get an Interrupt that means the block must stop processing. An
           # Interrupt is any command that stops block execution such as {% break %}
-          # or {% continue %}
-          context.push_interrupt(node.interrupt)
-          break
-        else # Other non-Block tags
-          render_node_to_output(node, output, context)
+          # or {% continue %}. These tags may also occur through Block or Include tags.
+          break if context.interrupt? # might have happened in a for-block
         end
         idx += 1
+
+        context.resource_limits.increment_write_score(output)
       end
 
-      output.join
+      output
     end
 
     private
 
-    def render_node_to_output(node, output, context, skip_output = false)
-      node_output = node.render(context)
-      node_output = node_output.is_a?(Array) ? node_output.join : node_output.to_s
-      check_resources(context, node_output)
-      output << node_output unless skip_output
-    rescue MemoryError => e
-      raise e
-    rescue UndefinedVariable, UndefinedDropMethod, UndefinedFilter => e
-      context.handle_error(e, node.line_number)
-      output << nil
-    rescue ::StandardError => e
-      line_number = node.is_a?(String) ? nil : node.line_number
-      output << context.handle_error(e, line_number)
-    end
-
-    def check_resources(context, node_output)
-      context.resource_limits.render_length += node_output.length
-      return unless context.resource_limits.reached?
-      raise MemoryError.new("Memory limits exceeded".freeze)
+    def render_node(context, output, node)
+      BlockBody.render_node(context, output, node)
     end
 
     def create_variable(token, parse_context)
@@ -124,15 +235,17 @@ module Liquid
         markup = content.first
         return Variable.new(markup, parse_context)
       end
-      raise_missing_variable_terminator(token, parse_context)
+      BlockBody.raise_missing_variable_terminator(token, parse_context)
     end
 
+    # @deprecated Use {.raise_missing_tag_terminator} instead
     def raise_missing_tag_terminator(token, parse_context)
-      raise SyntaxError.new(parse_context.locale.t("errors.syntax.tag_termination".freeze, token: token, tag_end: TagEnd.inspect))
+      BlockBody.raise_missing_tag_terminator(token, parse_context)
     end
 
+    # @deprecated Use {.raise_missing_variable_terminator} instead
     def raise_missing_variable_terminator(token, parse_context)
-      raise SyntaxError.new(parse_context.locale.t("errors.syntax.variable_termination".freeze, token: token, tag_end: VariableEnd.inspect))
+      BlockBody.raise_missing_variable_terminator(token, parse_context)
     end
 
     def registered_tags