liquid-c 4.0.1 → 4.1.0

data/test/unit/block_test.rb

@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+class BlockTest < MiniTest::Test
+  def test_no_allocation_of_trimmed_strings
+    template = Liquid::Template.parse("{{ a -}}     {{- b }}")
+    assert_equal(2, template.root.nodelist.size)
+
+    template = Liquid::Template.parse("{{ a -}} foo {{- b }}")
+    assert_equal(3, template.root.nodelist.size)
+  end
+
+  def test_raise_on_output_with_non_utf8_encoding
+    output = String.new(encoding: Encoding::ASCII)
+    template = Liquid::Template.parse("ascii text")
+    exc = assert_raises(Encoding::CompatibilityError) do
+      template.render!({}, output: output)
+    end
+    assert_equal("non-UTF8 encoded output (US-ASCII) not supported", exc.message)
+  end
+
+  def test_write_unicode_characters
+    output = String.new(encoding: Encoding::UTF_8)
+    template = Liquid::Template.parse("ü{{ unicode_char }}")
+    assert_equal("üñ", template.render!({ "unicode_char" => "ñ" }, output: output))
+  end
+
+  def test_op_write_raw_w
+    source = "a" * 2**8
+    template = Liquid::Template.parse(source)
+    assert_equal(source, template.render!)
+  end
+
+  def test_raise_for_non_c_parse_context
+    parse_context = Liquid::ParseContext.new
+    assert_raises(RuntimeError) do
+      Liquid::C::BlockBody.new(parse_context)
+    end
+  end
+
+  # Test for bug: https://github.com/Shopify/liquid-c/pull/120
+  def test_bug_120_instrument
+    calls = []
+    Liquid::Usage.stub(:increment, ->(name) { calls << name }) do
+      Liquid::Template.parse("{{ -.1 }}")
+    end
+    assert_equal(["liquid_c_negative_float_without_integer"], calls)
+
+    calls = []
+    Liquid::Usage.stub(:increment, ->(name) { calls << name }) do
+      Liquid::Template.parse("{{ .1 }}")
+    end
+    assert_equal([], calls)
+  end
+
+  def test_disassemble_raw_w
+    source = "a" * 2**8
+    template = Liquid::Template.parse(source)
+    block_body = template.root.body
+    assert_equal(<<~ASM, block_body.disassemble)
+      0x0000: write_raw_w("#{source}")
+      0x0104: leave
+    ASM
+  end
+
+  def test_disassemble
+    source = <<~LIQUID
+      raw
+      {{- var | default: "none", allow_false: true -}}
+      {%- increment counter -%}
+    LIQUID
+    template = Liquid::Template.parse(source, line_numbers: true)
+    block_body = template.root.body
+    increment_node = block_body.nodelist[2]
+    assert_instance_of(Liquid::Increment, increment_node)
+    assert_equal(<<~ASM, block_body.disassemble)
+      0x0000: write_raw("raw")
+      0x0005: render_variable_rescue(line_number: 2)
+      0x0009: find_static_var("var")
+      0x000c: push_const(\"none\")
+      0x000f: push_const(\"allow_false\")
+      0x0012: push_true
+      0x0013: hash_new(1)
+      0x0015: builtin_filter(name: :default, num_args: 3)
+      0x0018: pop_write
+      0x0019: write_node(#{increment_node.inspect})
+      0x001c: leave
+    ASM
+  end
+
+  def test_exception_renderer_exception
+    original_error = Liquid::Error.new("original")
+    handler_error = RuntimeError.new("exception handler error")
+    context = Liquid::Context.new("raise_error" => ->(_ctx) { raise(original_error) })
+    context.exception_renderer = lambda do |exc|
+      if exc == original_error
+        raise(handler_error)
+      end
+      exc
+    end
+    template = Liquid::Template.parse("{% assign x = raise_error %}")
+    exc = assert_raises(RuntimeError) do
+      template.render(context)
+    end
+    assert_equal(handler_error, exc)
+  end
+
+  StubFileSystem = Struct.new(:partials) do
+    def read_template_file(template_path)
+      partials.fetch(template_path)
+    end
+  end
+
+  def test_include_partial_with_syntax_error
+    old_file_system = Liquid::Template.file_system
+    begin
+      Liquid::Template.file_system = StubFileSystem.new({
+        "invalid" => "{% foo %}",
+        "valid" => '{% include "nested" %}',
+        "nested" => "valid",
+      })
+
+      template = Liquid::Template.parse("{% include 'invalid' %},{% include 'valid' %}")
+      assert_equal("Liquid syntax error: Unknown tag 'foo',valid", template.render)
+    ensure
+      Liquid::Template.file_system = old_file_system
+    end
+  end
+end

data/test/unit/context_test.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+require "test_helper"
+require "bigdecimal"
+
+class ContextTest < Minitest::Test
+  def test_evaluate_works_with_normal_values
+    context = Liquid::Context.new
+
+    ["abc", 123, false, 1.21, BigDecimal(42)].each do |value|
+      assert_equal(value, context.evaluate(value))
+    end
+
+    assert_nil(context.evaluate(nil))
+  end
+
+  def test_evaluate_works_with_classes_that_have_an_evaluate_method
+    class_with_evaluate = Class.new do
+      def evaluate(_context)
+        42
+      end
+    end
+
+    assert_equal(42, Liquid::Context.new.evaluate(class_with_evaluate.new))
+  end
+
+  def test_evaluate_works_with_variable_lookup
+    assert_equal(42, Liquid::Context.new({ "var" => 42 }).evaluate(Liquid::C::Expression.strict_parse("var")))
+  end
+
+  def test_evaluating_a_variable_entirely_within_c
+    context = Liquid::Context.new({ "var" => 42 })
+    lookup = Liquid::C::Expression.strict_parse("var")
+    context.evaluate(lookup) # memoize vm_internal_new calls
+
+    called_ruby_method_count = 0
+    called_c_method_count = 0
+
+    test_thread = Thread.current
+    begin
+      call_trace = TracePoint.trace(:call) do |t|
+        unless t.self == TracePoint || t.self.is_a?(TracePoint) || Thread.current != test_thread
+          called_ruby_method_count += 1
+        end
+      end
+
+      c_call_trace = TracePoint.trace(:c_call) do |t|
+        unless t.self == TracePoint || t.self.is_a?(TracePoint) || Thread.current != test_thread
+          called_c_method_count += 1
+        end
+      end
+
+      context.evaluate(lookup)
+    ensure
+      call_trace&.disable
+      c_call_trace&.disable
+    end
+
+    assert_equal(0, called_ruby_method_count)
+    assert_equal(1, called_c_method_count) # context.evaluate call
+  end
+
+  class TestDrop < Liquid::Drop
+    def is_filtering # rubocop:disable Naming/PredicateName
+      @context.send(:c_filtering?)
+    end
+  end
+
+  def test_c_filtering_predicate
+    context = Liquid::Context.new({ "test" => [TestDrop.new] })
+    template = Liquid::Template.parse('{{ test[0].is_filtering }},{{ test | map: "is_filtering" }}')
+
+    assert_equal("false,true", template.render!(context))
+    assert_equal(false, context.send(:c_filtering?))
+  end
+
+  def test_strict_variables=
+    context = Liquid::Context.new
+    assert_equal(false, context.strict_variables)
+    context.strict_variables = true
+    assert_equal(true, context.strict_variables)
+  end
+end

data/test/unit/expression_test.rb

@@ -0,0 +1,186 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+class ExpressionTest < MiniTest::Test
+  def test_constant_literals
+    assert_equal(true, Liquid::C::Expression.strict_parse("true"))
+    assert_equal(false, Liquid::C::Expression.strict_parse("false"))
+    assert_nil(Liquid::C::Expression.strict_parse("nil"))
+    assert_nil(Liquid::C::Expression.strict_parse("null"))
+
+    empty = Liquid::C::Expression.strict_parse("empty")
+    assert_equal("", empty)
+    assert_same(empty, Liquid::C::Expression.strict_parse("blank"))
+  end
+
+  def test_push_literals
+    assert_nil(compile_and_eval("nil"))
+    assert_equal(true, compile_and_eval("true"))
+    assert_equal(false, compile_and_eval("false"))
+  end
+
+  def test_constant_integer
+    assert_equal(42, Liquid::C::Expression.strict_parse("42"))
+  end
+
+  def test_push_int8
+    assert_equal(127, compile_and_eval("127"))
+    assert_equal(-128, compile_and_eval("-128"))
+  end
+
+  def test_push_int16
+    assert_equal(128, compile_and_eval("128"))
+    assert_equal(-129, compile_and_eval("-129"))
+    assert_equal(32767, compile_and_eval("32767"))
+    assert_equal(-32768, compile_and_eval("-32768"))
+  end
+
+  def test_push_large_fixnum
+    assert_equal(32768, compile_and_eval("32768"))
+    assert_equal(-2147483648, compile_and_eval("-2147483648"))
+    assert_equal(2147483648, compile_and_eval("2147483648"))
+    assert_equal(4611686018427387903, compile_and_eval("4611686018427387903"))
+  end
+
+  def test_push_big_int
+    num = 1 << 128
+    assert_equal(num, compile_and_eval(num.to_s))
+  end
+
+  def test_float
+    assert_equal(123.4, Liquid::C::Expression.strict_parse("123.4"))
+    assert_equal(-1.5, compile_and_eval("-1.5"))
+  end
+
+  def test_string
+    assert_equal("hello", Liquid::C::Expression.strict_parse('"hello"'))
+    assert_equal("world", compile_and_eval("'world'"))
+  end
+
+  def test_find_static_variable
+    context = Liquid::Context.new({ "x" => 123 })
+    expr = Liquid::C::Expression.strict_parse("x")
+
+    assert_instance_of(Liquid::C::Expression, expr)
+    assert_equal(123, context.evaluate(expr))
+  end
+
+  def test_find_dynamic_variable
+    context = Liquid::Context.new({ "x" => "y", "y" => 42 })
+    expr = Liquid::C::Expression.strict_parse("[x]")
+    assert_equal(42, context.evaluate(expr))
+  end
+
+  def test_find_missing_variable
+    context = Liquid::Context.new({})
+    expr = Liquid::C::Expression.strict_parse("missing")
+
+    assert_nil(context.evaluate(expr))
+
+    context.strict_variables = true
+
+    assert_raises(Liquid::UndefinedVariable) do
+      context.evaluate(expr)
+    end
+  end
+
+  def test_lookup_const_key
+    context = Liquid::Context.new({ "obj" => { "prop" => "some value" } })
+
+    expr = Liquid::C::Expression.strict_parse("obj.prop")
+    assert_equal("some value", context.evaluate(expr))
+
+    expr = Liquid::C::Expression.strict_parse('obj["prop"]')
+    assert_equal("some value", context.evaluate(expr))
+  end
+
+  def test_lookup_variable_key
+    context = Liquid::Context.new({ "field_name" => "prop", "obj" => { "prop" => "another value" } })
+    expr = Liquid::C::Expression.strict_parse("obj[field_name]")
+    assert_equal("another value", context.evaluate(expr))
+  end
+
+  def test_lookup_command
+    context = Liquid::Context.new({ "ary" => ["a", "b", "c"] })
+    assert_equal(3, context.evaluate(Liquid::C::Expression.strict_parse("ary.size")))
+    assert_equal("a", context.evaluate(Liquid::C::Expression.strict_parse("ary.first")))
+    assert_equal("c", context.evaluate(Liquid::C::Expression.strict_parse("ary.last")))
+  end
+
+  def test_lookup_missing_key
+    context = Liquid::Context.new({ "obj" => {} })
+    expr = Liquid::C::Expression.strict_parse("obj.missing")
+
+    assert_nil(context.evaluate(expr))
+
+    context.strict_variables = true
+
+    assert_raises(Liquid::UndefinedVariable) do
+      context.evaluate(expr)
+    end
+  end
+
+  def test_lookup_on_var_with_literal_name
+    context = Liquid::Context.new({ "blank" => { "x" => "result" } })
+
+    assert_equal("result", context.evaluate(Liquid::C::Expression.strict_parse("blank.x")))
+    assert_equal("result", context.evaluate(Liquid::C::Expression.strict_parse('blank["x"]')))
+  end
+
+  def test_const_range
+    assert_equal((1..2), Liquid::C::Expression.strict_parse("(1..2)"))
+  end
+
+  def test_dynamic_range
+    context = Liquid::Context.new({ "var" => 42 })
+    expr = Liquid::C::Expression.strict_parse("(1..var)")
+    assert_instance_of(Liquid::C::Expression, expr)
+    assert_equal((1..42), context.evaluate(expr))
+  end
+
+  def test_disassemble
+    expression = Liquid::C::Expression.strict_parse("foo.bar[123]")
+    assert_equal(<<~ASM, expression.disassemble)
+      0x0000: find_static_var("foo")
+      0x0003: lookup_const_key("bar")
+      0x0006: push_int8(123)
+      0x0008: lookup_key
+      0x0009: leave
+    ASM
+  end
+
+  def test_disassemble_int16
+    assert_equal(<<~ASM, Liquid::C::Expression.strict_parse("[12345]").disassemble)
+      0x0000: push_int16(12345)
+      0x0003: find_var
+      0x0004: leave
+    ASM
+  end
+
+  def test_disable_c_nodes
+    context = Liquid::Context.new({ "x" => 123 })
+
+    expr = Liquid::ParseContext.new.parse_expression("x")
+    assert_instance_of(Liquid::C::Expression, expr)
+    assert_equal(123, context.evaluate(expr))
+
+    expr = Liquid::ParseContext.new(disable_liquid_c_nodes: true).parse_expression("x")
+    assert_instance_of(Liquid::VariableLookup, expr)
+    assert_equal(123, context.evaluate(expr))
+  end
+
+  private
+
+  class ReturnKeyDrop < Liquid::Drop
+    def liquid_method_missing(key)
+      key
+    end
+  end
+
+  def compile_and_eval(source)
+    context = Liquid::Context.new({ "ret_key" => ReturnKeyDrop.new })
+    expr = Liquid::C::Expression.strict_parse("ret_key[#{source}]")
+    context.evaluate(expr)
+  end
+end

data/test/unit/gc_stress_test.rb

@@ -0,0 +1,28 @@
+# encoding: utf-8
+# frozen_string_literal: true
+
+require "test_helper"
+
+# Help catch bugs from objects not being marked at all
+# GC opportunities.
+class GCStressTest < Minitest::Test
+  def test_compile_and_render
+    source = "{% assign x = 1 %}{% if x -%} x: {{ x | plus: 2 }}{% endif %}"
+    result = gc_stress do
+      Liquid::Template.parse(source).render!
+    end
+    assert_equal("x: 3", result)
+  end
+
+  private
+
+  def gc_stress
+    old_value = GC.stress
+    GC.stress = true
+    begin
+      yield
+    ensure
+      GC.stress = old_value
+    end
+  end
+end

data/test/unit/raw_test.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+class RawTest < Minitest::Test
+  class RawWrapper < Liquid::Raw
+    def render_to_output_buffer(_context, output)
+      output << "<"
+      super
+      output << ">"
+    end
+  end
+  Liquid::Template.register_tag("raw_wrapper", RawWrapper)
+
+  def test_derived_class
+    output = Liquid::Template.parse("{% raw_wrapper %}body{% endraw_wrapper %}").render!
+    assert_equal("<body>", output)
+  end
+end

data/test/unit/resource_limits_test.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+class ResourceLimitsTest < Minitest::Test
+  def test_increment_render_score
+    resource_limits = Liquid::ResourceLimits.new(render_score_limit: 5)
+    resource_limits.increment_render_score(4)
+    assert_raises(Liquid::MemoryError) do
+      resource_limits.increment_render_score(2)
+    end
+    assert_equal(6, resource_limits.render_score)
+  end
+
+  def test_increment_assign_score
+    resource_limits = Liquid::ResourceLimits.new(assign_score_limit: 5)
+    resource_limits.increment_assign_score(5)
+    assert_raises(Liquid::MemoryError) do
+      resource_limits.increment_assign_score(1)
+    end
+    assert_equal(6, resource_limits.assign_score)
+  end
+
+  def test_increment_write_score
+    resource_limits = Liquid::ResourceLimits.new(render_length_limit: 5)
+    output = "a" * 10
+    assert_raises(Liquid::MemoryError) do
+      resource_limits.increment_write_score(output)
+    end
+  end
+
+  def test_raise_limits_reached
+    resource_limits = Liquid::ResourceLimits.new({})
+    assert_raises(Liquid::MemoryError) do
+      resource_limits.raise_limits_reached
+    end
+    assert(resource_limits.reached?)
+  end
+
+  def test_with_capture
+    resource_limits = Liquid::ResourceLimits.new(assign_score_limit: 5)
+    output = "foo"
+
+    resource_limits.with_capture do
+      resource_limits.increment_write_score(output)
+    end
+
+    assert_equal(3, resource_limits.assign_score)
+  end
+end

data/test/unit/tokenizer_test.rb

@@ -1,41 +1,111 @@
 # encoding: utf-8
-require 'test_helper'
+# frozen_string_literal: true
+
+require "test_helper"
+
+class TokenizerTest < Minitest::Test
+  def test_tokenizer_nil
+    tokenizer = new_tokenizer(nil)
+    assert_nil(tokenizer.send(:shift))
+  end
 
-class TokenizerTest < MiniTest::Unit::TestCase
   def test_tokenize_strings
-    assert_equal [' '], tokenize(' ')
-    assert_equal ['hello world'], tokenize('hello world')
+    assert_equal([" "], tokenize(" "))
+    assert_equal(["hello world"], tokenize("hello world"))
   end
 
   def test_tokenize_variables
-    assert_equal ['{{funk}}'], tokenize('{{funk}}')
-    assert_equal [' ', '{{funk}}', ' '], tokenize(' {{funk}} ')
-    assert_equal [' ', '{{funk}}', ' ', '{{so}}', ' ', '{{brother}}', ' '], tokenize(' {{funk}} {{so}} {{brother}} ')
-    assert_equal [' ', '{{  funk  }}', ' '], tokenize(' {{  funk  }} ')
+    assert_equal(["{{funk}}"], tokenize("{{funk}}"))
+    assert_equal([" ", "{{funk}}", " "], tokenize(" {{funk}} "))
+    assert_equal([" ", "{{funk}}", " ", "{{so}}", " ", "{{brother}}", " "], tokenize(" {{funk}} {{so}} {{brother}} "))
+    assert_equal([" ", "{{  funk  }}", " "], tokenize(" {{  funk  }} "))
+
+    # Doesn't strip whitespace
+    assert_equal([" ", "  funk  ", " "], tokenize(" {{  funk  }} ", trimmed: true))
   end
 
   def test_tokenize_blocks
-    assert_equal ['{%comment%}'], tokenize('{%comment%}')
-    assert_equal [' ', '{%comment%}', ' '], tokenize(' {%comment%} ')
+    assert_equal(["{%comment%}"], tokenize("{%comment%}"))
+    assert_equal([" ", "{%comment%}", " "], tokenize(" {%comment%} "))
+
+    assert_equal([" ", "{%comment%}", " ", "{%endcomment%}", " "], tokenize(" {%comment%} {%endcomment%} "))
+    assert_equal(["  ", "{% comment %}", " ", "{% endcomment %}", " "], tokenize("  {% comment %} {% endcomment %} "))
+
+    # Doesn't strip whitespace
+    assert_equal([" ", "  comment  ", " "], tokenize(" {%  comment  %} ", trimmed: true))
+  end
+
+  def test_tokenize_for_liquid_tag
+    source = "\nfunk\n\n  so | brother   \n"
+
+    assert_equal(["", "funk", "", "  so | brother   "], tokenize(source, for_liquid_tag: true))
 
-    assert_equal [' ', '{%comment%}', ' ', '{%endcomment%}', ' '], tokenize(' {%comment%} {%endcomment%} ')
-    assert_equal ['  ', '{% comment %}', ' ', '{% endcomment %}', ' '], tokenize("  {% comment %} {% endcomment %} ")
+    # Strips whitespace
+    assert_equal(["", "funk", "", "so | brother"], tokenize(source, for_liquid_tag: true, trimmed: true))
   end
 
-  def test_utf8_encoded_template
-    source = 'auswählen'
-    assert_equal Encoding::UTF_8, source.encoding
+  def test_invalid_tags
+    assert_equal([""], tokenize("{%-%}", trimmed: true))
+    assert_equal([""], tokenize("{{-}}", trimmed: true))
+  end
+
+  def test_utf8_encoded_source
+    source = "auswählen"
+    assert_equal(Encoding::UTF_8, source.encoding)
     output = tokenize(source)
-    assert_equal [Encoding::UTF_8], output.map(&:encoding)
-    assert_equal [source], output
+    assert_equal([Encoding::UTF_8], output.map(&:encoding))
+    assert_equal([source], output)
+  end
+
+  def test_utf8_compatible_source
+    source = String.new("ascii", encoding: Encoding::ASCII)
+    tokenizer = new_tokenizer(source)
+    output = tokenizer.send(:shift)
+    assert_equal(Encoding::UTF_8, output.encoding)
+    assert_equal(source, output)
+    assert_nil(tokenizer.send(:shift))
+  end
+
+  def test_non_utf8_compatible_source
+    source = "üñicode".dup.force_encoding(Encoding::BINARY) # rubocop:disable Performance/UnfreezeString
+    exc = assert_raises(Encoding::CompatibilityError) do
+      Liquid::C::Tokenizer.new(source, 1, false)
+    end
+    assert_equal("non-UTF8 encoded source (ASCII-8BIT) not supported", exc.message)
+  end
+
+  def test_source_too_large
+    too_large_source = "a" * 2**24
+    max_length_source = too_large_source.chop
+
+    # C safety check
+    err = assert_raises(ArgumentError) do
+      Liquid::C::Tokenizer.new(too_large_source, 1, false)
+    end
+    assert_match(/Source too large, max \d+ bytes/, err.message)
+
+    # ruby patch fallback
+    parse_context = Liquid::ParseContext.new
+    liquid_c_tokenizer = parse_context.new_tokenizer(max_length_source)
+    assert_instance_of(Liquid::C::Tokenizer, liquid_c_tokenizer)
+    refute(parse_context.liquid_c_nodes_disabled?)
+
+    parse_context = Liquid::ParseContext.new
+    fallback_tokenizer = parse_context.new_tokenizer(too_large_source)
+    assert_instance_of(Liquid::Tokenizer, fallback_tokenizer)
+    assert_equal(true, parse_context.liquid_c_nodes_disabled?)
   end
 
   private
 
-  def tokenize(source)
-    tokenizer = Liquid::C::Tokenizer.new(source, false)
+  def new_tokenizer(source, parse_context: Liquid::ParseContext.new)
+    parse_context.new_tokenizer(source)
+  end
+
+  def tokenize(source, for_liquid_tag: false, trimmed: false)
+    tokenizer = Liquid::C::Tokenizer.new(source, 1, for_liquid_tag)
     tokens = []
-    while t = tokenizer.shift
+    while (t = trimmed ? tokenizer.send(:shift_trimmed) : tokenizer.send(:shift))
       tokens << t
     end
     tokens