liquid-c 4.0.1 → 4.2.0

data/test/unit/block_test.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+class BlockTest < Minitest::Test
+  def test_no_allocation_of_trimmed_strings
+    template = Liquid::Template.parse("{{ a -}}     {{- b }}")
+    assert_equal(2, template.root.nodelist.size)
+
+    template = Liquid::Template.parse("{{ a -}} foo {{- b }}")
+    assert_equal(3, template.root.nodelist.size)
+  end
+
+  def test_raise_on_output_with_non_utf8_encoding
+    output = String.new(encoding: Encoding::ASCII)
+    template = Liquid::Template.parse("ascii text")
+    exc = assert_raises(Encoding::CompatibilityError) do
+      template.render!({}, output: output)
+    end
+    assert_equal("non-UTF8 encoded output (US-ASCII) not supported", exc.message)
+  end
+
+  def test_write_unicode_characters
+    output = String.new(encoding: Encoding::UTF_8)
+    template = Liquid::Template.parse("ü{{ unicode_char }}")
+    assert_equal("üñ", template.render!({ "unicode_char" => "ñ" }, output: output))
+  end
+
+  def test_op_write_raw_w
+    source = "a" * 2**8
+    template = Liquid::Template.parse(source)
+    assert_equal(source, template.render!)
+  end
+
+  def test_raise_for_non_c_parse_context
+    parse_context = Liquid::ParseContext.new
+    assert_raises(RuntimeError) do
+      Liquid::C::BlockBody.new(parse_context)
+    end
+  end
+
+  # Test for bug: https://github.com/Shopify/liquid-c/pull/120
+  def test_bug_120_instrument
+    calls = []
+    Liquid::Usage.stub(:increment, ->(name) { calls << name }) do
+      Liquid::Template.parse("{{ -.1 }}")
+    end
+    assert_equal(["liquid_c_negative_float_without_integer"], calls)
+
+    calls = []
+    Liquid::Usage.stub(:increment, ->(name) { calls << name }) do
+      Liquid::Template.parse("{{ .1 }}")
+    end
+    assert_equal([], calls)
+  end
+
+  def test_disassemble_raw_w
+    source = "a" * 2**8
+    template = Liquid::Template.parse(source)
+    block_body = template.root.body
+    assert_equal(<<~ASM, block_body.disassemble)
+      0x0000: write_raw_w("#{source}")
+      0x0104: leave
+    ASM
+  end
+
+  def test_disassemble
+    source = <<~LIQUID
+      raw
+      {{- var | default: "none", allow_false: true -}}
+      {%- increment counter -%}
+    LIQUID
+    template = Liquid::Template.parse(source, line_numbers: true)
+    block_body = template.root.body
+    increment_node = block_body.nodelist[2]
+    assert_instance_of(Liquid::Increment, increment_node)
+    assert_equal(<<~ASM, block_body.disassemble)
+      0x0000: write_raw("raw")
+      0x0005: render_variable_rescue(line_number: 2)
+      0x0009: find_static_var("var")
+      0x000c: push_const(\"none\")
+      0x000f: push_const(\"allow_false\")
+      0x0012: push_true
+      0x0013: hash_new(1)
+      0x0015: builtin_filter(name: :default, num_args: 3)
+      0x0018: pop_write
+      0x0019: write_node(#{increment_node.inspect})
+      0x001c: leave
+    ASM
+  end
+
+  def test_exception_renderer_exception
+    original_error = Liquid::Error.new("original")
+    handler_error = RuntimeError.new("exception handler error")
+    context = Liquid::Context.new("raise_error" => ->(_ctx) { raise(original_error) })
+    context.exception_renderer = lambda do |exc|
+      if exc == original_error
+        raise(handler_error)
+      end
+      exc
+    end
+    template = Liquid::Template.parse("{% assign x = raise_error %}")
+    exc = assert_raises(RuntimeError) do
+      template.render(context)
+    end
+    assert_equal(handler_error, exc)
+  end
+
+  StubFileSystem = Struct.new(:partials) do
+    def read_template_file(template_path)
+      partials.fetch(template_path)
+    end
+  end
+
+  def test_include_partial_with_syntax_error
+    old_file_system = Liquid::Template.file_system
+    begin
+      Liquid::Template.file_system = StubFileSystem.new({
+        "invalid" => "{% foo %}",
+        "valid" => '{% include "nested" %}',
+        "nested" => "valid",
+      })
+
+      template = Liquid::Template.parse("{% include 'invalid' %},{% include 'valid' %}")
+      assert_equal("Liquid syntax error: Unknown tag 'foo',valid", template.render)
+    ensure
+      Liquid::Template.file_system = old_file_system
+    end
+  end
+
+  def test_assign_filter_argument_exception
+    source = "{% assign v = 'IN' | truncate: 123, liquid_error %}{{ v | default: 'err swallowed' }}"
+    template = Liquid::Template.parse(source)
+    output = template.render({ "liquid_error" => -> { raise Liquid::Error, "var lookup error" } })
+    assert_equal("err swallowed", output)
+  end
+end

data/test/unit/context_test.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+require "test_helper"
+require "bigdecimal"
+
+class ContextTest < Minitest::Test
+  def test_evaluate_works_with_normal_values
+    context = Liquid::Context.new
+
+    ["abc", 123, false, 1.21, BigDecimal(42)].each do |value|
+      assert_equal(value, context.evaluate(value))
+    end
+
+    assert_nil(context.evaluate(nil))
+  end
+
+  def test_evaluate_works_with_classes_that_have_an_evaluate_method
+    class_with_evaluate = Class.new do
+      def evaluate(_context)
+        42
+      end
+    end
+
+    assert_equal(42, Liquid::Context.new.evaluate(class_with_evaluate.new))
+  end
+
+  def test_evaluate_works_with_variable_lookup
+    assert_equal(42, Liquid::Context.new({ "var" => 42 }).evaluate(Liquid::C::Expression.strict_parse("var")))
+  end
+
+  def test_evaluating_a_variable_entirely_within_c
+    skip("TracePoint :call not yet supported") if RUBY_ENGINE == "truffleruby"
+
+    context = Liquid::Context.new({ "var" => 42 })
+    lookup = Liquid::C::Expression.strict_parse("var")
+    context.evaluate(lookup) # memoize vm_internal_new calls
+
+    called_ruby_method_count = 0
+    called_c_method_count = 0
+
+    test_thread = Thread.current
+    begin
+      call_trace = TracePoint.trace(:call) do |t|
+        unless t.self == TracePoint || t.self.is_a?(TracePoint) || Thread.current != test_thread
+          called_ruby_method_count += 1
+        end
+      end
+
+      c_call_trace = TracePoint.trace(:c_call) do |t|
+        unless t.self == TracePoint || t.self.is_a?(TracePoint) || Thread.current != test_thread
+          called_c_method_count += 1
+        end
+      end
+
+      context.evaluate(lookup)
+    ensure
+      call_trace&.disable
+      c_call_trace&.disable
+    end
+
+    assert_equal(0, called_ruby_method_count)
+    assert_equal(1, called_c_method_count) # context.evaluate call
+  end
+
+  class TestDrop < Liquid::Drop
+    def is_filtering # rubocop:disable Naming/PredicateName
+      @context.send(:c_filtering?)
+    end
+  end
+
+  def test_c_filtering_predicate
+    context = Liquid::Context.new({ "test" => [TestDrop.new] })
+    template = Liquid::Template.parse('{{ test[0].is_filtering }},{{ test | map: "is_filtering" }}')
+
+    assert_equal("false,true", template.render!(context))
+    assert_equal(false, context.send(:c_filtering?))
+  end
+
+  def test_strict_variables=
+    context = Liquid::Context.new
+    assert_equal(false, context.strict_variables)
+    context.strict_variables = true
+    assert_equal(true, context.strict_variables)
+  end
+end

data/test/unit/expression_test.rb

@@ -0,0 +1,191 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+class ExpressionTest < Minitest::Test
+  def test_constant_literals
+    assert_equal(true, Liquid::C::Expression.strict_parse("true"))
+    assert_equal(false, Liquid::C::Expression.strict_parse("false"))
+    assert_nil(Liquid::C::Expression.strict_parse("nil"))
+    assert_nil(Liquid::C::Expression.strict_parse("null"))
+
+    empty = Liquid::C::Expression.strict_parse("empty")
+    assert_equal("", empty)
+    assert_same(empty, Liquid::C::Expression.strict_parse("blank"))
+  end
+
+  def test_push_literals
+    assert_nil(compile_and_eval("nil"))
+    assert_equal(true, compile_and_eval("true"))
+    assert_equal(false, compile_and_eval("false"))
+  end
+
+  def test_constant_integer
+    assert_equal(42, Liquid::C::Expression.strict_parse("42"))
+  end
+
+  def test_push_int8
+    assert_equal(127, compile_and_eval("127"))
+    assert_equal(-128, compile_and_eval("-128"))
+  end
+
+  def test_push_int16
+    assert_equal(128, compile_and_eval("128"))
+    assert_equal(-129, compile_and_eval("-129"))
+    assert_equal(32767, compile_and_eval("32767"))
+    assert_equal(-32768, compile_and_eval("-32768"))
+  end
+
+  def test_push_large_fixnum
+    assert_equal(32768, compile_and_eval("32768"))
+    assert_equal(-2147483648, compile_and_eval("-2147483648"))
+    assert_equal(2147483648, compile_and_eval("2147483648"))
+    assert_equal(4611686018427387903, compile_and_eval("4611686018427387903"))
+  end
+
+  def test_push_big_int
+    num = 1 << 128
+    assert_equal(num, compile_and_eval(num.to_s))
+  end
+
+  def test_float
+    assert_equal(123.4, Liquid::C::Expression.strict_parse("123.4"))
+    assert_equal(-1.5, compile_and_eval("-1.5"))
+  end
+
+  def test_string
+    assert_equal("hello", Liquid::C::Expression.strict_parse('"hello"'))
+    assert_equal("world", compile_and_eval("'world'"))
+  end
+
+  def test_find_static_variable
+    context = Liquid::Context.new({ "x" => 123 })
+    expr = Liquid::C::Expression.strict_parse("x")
+
+    assert_instance_of(Liquid::C::Expression, expr)
+    assert_equal(123, context.evaluate(expr))
+  end
+
+  def test_find_dynamic_variable
+    context = Liquid::Context.new({ "x" => "y", "y" => 42 })
+    expr = Liquid::C::Expression.strict_parse("[x]")
+    assert_equal(42, context.evaluate(expr))
+  end
+
+  def test_find_missing_variable
+    context = Liquid::Context.new({})
+    expr = Liquid::C::Expression.strict_parse("missing")
+
+    assert_nil(context.evaluate(expr))
+
+    context.strict_variables = true
+
+    assert_raises(Liquid::UndefinedVariable) do
+      context.evaluate(expr)
+    end
+  end
+
+  def test_lookup_const_key
+    context = Liquid::Context.new({ "obj" => { "prop" => "some value" } })
+
+    expr = Liquid::C::Expression.strict_parse("obj.prop")
+    assert_equal("some value", context.evaluate(expr))
+
+    expr = Liquid::C::Expression.strict_parse('obj["prop"]')
+    assert_equal("some value", context.evaluate(expr))
+  end
+
+  def test_lookup_variable_key
+    context = Liquid::Context.new({ "field_name" => "prop", "obj" => { "prop" => "another value" } })
+    expr = Liquid::C::Expression.strict_parse("obj[field_name]")
+    assert_equal("another value", context.evaluate(expr))
+  end
+
+  def test_lookup_command
+    context = Liquid::Context.new({ "ary" => ["a", "b", "c"] })
+    assert_equal(3, context.evaluate(Liquid::C::Expression.strict_parse("ary.size")))
+    assert_equal("a", context.evaluate(Liquid::C::Expression.strict_parse("ary.first")))
+    assert_equal("c", context.evaluate(Liquid::C::Expression.strict_parse("ary.last")))
+  end
+
+  def test_lookup_missing_key
+    context = Liquid::Context.new({ "obj" => {} })
+    expr = Liquid::C::Expression.strict_parse("obj.missing")
+
+    assert_nil(context.evaluate(expr))
+
+    context.strict_variables = true
+
+    assert_raises(Liquid::UndefinedVariable) do
+      context.evaluate(expr)
+    end
+  end
+
+  def test_lookup_on_var_with_literal_name
+    context = Liquid::Context.new({ "blank" => { "x" => "result" } })
+
+    assert_equal("result", context.evaluate(Liquid::C::Expression.strict_parse("blank.x")))
+    assert_equal("result", context.evaluate(Liquid::C::Expression.strict_parse('blank["x"]')))
+  end
+
+  def test_const_range
+    assert_equal((1..2), Liquid::C::Expression.strict_parse("(1..2)"))
+  end
+
+  def test_dynamic_range
+    context = Liquid::Context.new({ "var" => 42 })
+    expr = Liquid::C::Expression.strict_parse("(1..var)")
+    assert_instance_of(Liquid::C::Expression, expr)
+    assert_equal((1..42), context.evaluate(expr))
+  end
+
+  def test_disassemble
+    expression = Liquid::C::Expression.strict_parse("foo.bar[123]")
+    assert_equal(<<~ASM, expression.disassemble)
+      0x0000: find_static_var("foo")
+      0x0003: lookup_const_key("bar")
+      0x0006: push_int8(123)
+      0x0008: lookup_key
+      0x0009: leave
+    ASM
+  end
+
+  def test_disassemble_int16
+    assert_equal(<<~ASM, Liquid::C::Expression.strict_parse("[12345]").disassemble)
+      0x0000: push_int16(12345)
+      0x0003: find_var
+      0x0004: leave
+    ASM
+  end
+
+  def test_disable_c_nodes
+    context = Liquid::Context.new({ "x" => 123, "y" => { 123 => 42 } })
+
+    expr = Liquid::ParseContext.new.parse_expression("x")
+    assert_instance_of(Liquid::C::Expression, expr)
+    assert_equal(123, context.evaluate(expr))
+
+    expr = Liquid::ParseContext.new(disable_liquid_c_nodes: true).parse_expression("x")
+    assert_instance_of(Liquid::VariableLookup, expr)
+    assert_equal(123, context.evaluate(expr))
+
+    expr = Liquid::ParseContext.new(disable_liquid_c_nodes: true).parse_expression("y[x]")
+    assert_instance_of(Liquid::VariableLookup, expr)
+    assert_instance_of(Liquid::VariableLookup, expr.lookups.first)
+    assert_equal(42, context.evaluate(expr))
+  end
+
+  private
+
+  class ReturnKeyDrop < Liquid::Drop
+    def liquid_method_missing(key)
+      key
+    end
+  end
+
+  def compile_and_eval(source)
+    context = Liquid::Context.new({ "ret_key" => ReturnKeyDrop.new })
+    expr = Liquid::C::Expression.strict_parse("ret_key[#{source}]")
+    context.evaluate(expr)
+  end
+end

data/test/unit/gc_stress_test.rb

@@ -0,0 +1,28 @@
+# encoding: utf-8
+# frozen_string_literal: true
+
+require "test_helper"
+
+# Help catch bugs from objects not being marked at all
+# GC opportunities.
+class GCStressTest < Minitest::Test
+  def test_compile_and_render
+    source = "{% assign x = 1 %}{% if x -%} x: {{ x | plus: 2 }}{% endif %}"
+    result = gc_stress do
+      Liquid::Template.parse(source).render!
+    end
+    assert_equal("x: 3", result)
+  end
+
+  private
+
+  def gc_stress
+    old_value = GC.stress
+    GC.stress = true
+    begin
+      yield
+    ensure
+      GC.stress = old_value
+    end
+  end
+end

data/test/unit/raw_test.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+class RawTest < Minitest::Test
+  class RawWrapper < Liquid::Raw
+    def render_to_output_buffer(_context, output)
+      output << "<"
+      super
+      output << ">"
+    end
+  end
+  Liquid::Template.register_tag("raw_wrapper", RawWrapper)
+
+  def test_derived_class
+    [
+      "{% raw_wrapper %}body{% endraw_wrapper %}",
+      "{% raw_wrapper %}body{%endraw_wrapper%}",
+      "{% raw_wrapper %}body{%- endraw_wrapper -%}",
+      "{% raw_wrapper %}body{%- endraw_wrapper %}",
+      "{% raw_wrapper %}body{% endraw_wrapper -%}",
+    ].each do |template|
+      output = Liquid::Template.parse(template).render!
+
+      assert_equal(
+        "<body>",
+        output,
+        "Template: #{template}"
+      )
+    end
+  end
+
+  def test_allows_extra_string_after_tag_delimiter
+    output = Liquid::Template.parse("{% raw %}message{% endraw this_is_allowed %}").render
+    assert_equal("message", output)
+
+    output = Liquid::Template.parse("{% raw %}message{%   endraw r%}").render
+    assert_equal("message", output)
+  end
+
+  def test_ignores_incomplete_tag_delimter
+    output = Liquid::Template.parse("{% raw %}{% endraw {% endraw %}").render
+    assert_equal("{% endraw ", output)
+
+    output = Liquid::Template.parse("{% raw %}{%endraw{% endraw %}").render
+    assert_equal("{%endraw", output)
+
+    output = Liquid::Template.parse("{% raw %}{%- endraw {% endraw %}").render
+    assert_equal("{%- endraw ", output)
+  end
+
+  def test_does_not_allow_nbsp_in_tag_delimiter
+    # these are valid
+    Liquid::Template.parse("{% raw %}body{%endraw%}")
+    Liquid::Template.parse("{% raw %}body{% endraw-%}")
+    Liquid::Template.parse("{% raw %}body{% endraw -%}")
+    Liquid::Template.parse("{% raw %}body{%-endraw %}")
+    Liquid::Template.parse("{% raw %}body{%- endraw %}")
+    Liquid::Template.parse("{% raw %}body{%-endraw-%}")
+    Liquid::Template.parse("{% raw %}body{%- endraw -%}")
+    Liquid::Template.parse("{% raw %}body{% endraw\u00A0%}")
+    Liquid::Template.parse("{% raw %}body{% endraw \u00A0%}")
+    Liquid::Template.parse("{% raw %}body{% endraw\u00A0 %}")
+    Liquid::Template.parse("{% raw %}body{% endraw \u00A0 %}")
+    Liquid::Template.parse("{% raw %}body{% endraw \u00A0 endraw %}")
+    Liquid::Template.parse("{% raw %}body{% endraw\u00A0endraw %}")
+
+    [
+      "{%\u00A0endraw%}",
+      "{%\u00A0 endraw%}",
+      "{% \u00A0endraw%}",
+      "{% \u00A0 endraw%}",
+      "{%\u00A0endraw\u00A0%}",
+      "{% - endraw %}",
+      "{% endnot endraw %}",
+    ].each do |bad_delimiter|
+      exception = assert_raises(
+        Liquid::SyntaxError,
+        "#{bad_delimiter.inspect} did not raise Liquid::SyntaxError"
+      ) do
+        Liquid::Template.parse(
+          "{% raw %}body#{bad_delimiter}"
+        )
+      end
+
+      assert_equal(
+        exception.message,
+        "Liquid syntax error: 'raw' tag was never closed",
+        "#{bad_delimiter.inspect} raised the wrong exception message",
+      )
+    end
+  end
+end

data/test/unit/resource_limits_test.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+require "test_helper"
+
+class ResourceLimitsTest < Minitest::Test
+  def test_increment_render_score
+    resource_limits = Liquid::ResourceLimits.new(render_score_limit: 5)
+    resource_limits.increment_render_score(4)
+    assert_raises(Liquid::MemoryError) do
+      resource_limits.increment_render_score(2)
+    end
+    assert_equal(6, resource_limits.render_score)
+  end
+
+  def test_increment_assign_score
+    resource_limits = Liquid::ResourceLimits.new(assign_score_limit: 5)
+    resource_limits.increment_assign_score(5)
+    assert_raises(Liquid::MemoryError) do
+      resource_limits.increment_assign_score(1)
+    end
+    assert_equal(6, resource_limits.assign_score)
+  end
+
+  def test_increment_write_score
+    resource_limits = Liquid::ResourceLimits.new(render_length_limit: 5)
+    output = "a" * 10
+    assert_raises(Liquid::MemoryError) do
+      resource_limits.increment_write_score(output)
+    end
+  end
+
+  def test_raise_limits_reached
+    resource_limits = Liquid::ResourceLimits.new({})
+    assert_raises(Liquid::MemoryError) do
+      resource_limits.raise_limits_reached
+    end
+    assert(resource_limits.reached?)
+  end
+
+  def test_with_capture
+    resource_limits = Liquid::ResourceLimits.new(assign_score_limit: 5)
+    output = "foo"
+
+    resource_limits.with_capture do
+      resource_limits.increment_write_score(output)
+    end
+
+    assert_equal(3, resource_limits.assign_score)
+  end
+end