liquid-c 4.0.1 → 4.2.0

data/test/unit/tokenizer_test.rb

@@ -1,41 +1,111 @@
 # encoding: utf-8
-require 'test_helper'
+# frozen_string_literal: true
+
+require "test_helper"
+
+class TokenizerTest < Minitest::Test
+  def test_tokenizer_nil
+    tokenizer = new_tokenizer(nil)
+    assert_nil(tokenizer.shift)
+  end
 
-class TokenizerTest < MiniTest::Unit::TestCase
   def test_tokenize_strings
-    assert_equal [' '], tokenize(' ')
-    assert_equal ['hello world'], tokenize('hello world')
+    assert_equal([" "], tokenize(" "))
+    assert_equal(["hello world"], tokenize("hello world"))
   end
 
   def test_tokenize_variables
-    assert_equal ['{{funk}}'], tokenize('{{funk}}')
-    assert_equal [' ', '{{funk}}', ' '], tokenize(' {{funk}} ')
-    assert_equal [' ', '{{funk}}', ' ', '{{so}}', ' ', '{{brother}}', ' '], tokenize(' {{funk}} {{so}} {{brother}} ')
-    assert_equal [' ', '{{  funk  }}', ' '], tokenize(' {{  funk  }} ')
+    assert_equal(["{{funk}}"], tokenize("{{funk}}"))
+    assert_equal([" ", "{{funk}}", " "], tokenize(" {{funk}} "))
+    assert_equal([" ", "{{funk}}", " ", "{{so}}", " ", "{{brother}}", " "], tokenize(" {{funk}} {{so}} {{brother}} "))
+    assert_equal([" ", "{{  funk  }}", " "], tokenize(" {{  funk  }} "))
+
+    # Doesn't strip whitespace
+    assert_equal([" ", "  funk  ", " "], tokenize(" {{  funk  }} ", trimmed: true))
   end
 
   def test_tokenize_blocks
-    assert_equal ['{%comment%}'], tokenize('{%comment%}')
-    assert_equal [' ', '{%comment%}', ' '], tokenize(' {%comment%} ')
+    assert_equal(["{%comment%}"], tokenize("{%comment%}"))
+    assert_equal([" ", "{%comment%}", " "], tokenize(" {%comment%} "))
+
+    assert_equal([" ", "{%comment%}", " ", "{%endcomment%}", " "], tokenize(" {%comment%} {%endcomment%} "))
+    assert_equal(["  ", "{% comment %}", " ", "{% endcomment %}", " "], tokenize("  {% comment %} {% endcomment %} "))
+
+    # Doesn't strip whitespace
+    assert_equal([" ", "  comment  ", " "], tokenize(" {%  comment  %} ", trimmed: true))
+  end
+
+  def test_tokenize_for_liquid_tag
+    source = "\nfunk\n\n  so | brother   \n"
+
+    assert_equal(["", "funk", "", "  so | brother   "], tokenize(source, for_liquid_tag: true))
 
-    assert_equal [' ', '{%comment%}', ' ', '{%endcomment%}', ' '], tokenize(' {%comment%} {%endcomment%} ')
-    assert_equal ['  ', '{% comment %}', ' ', '{% endcomment %}', ' '], tokenize("  {% comment %} {% endcomment %} ")
+    # Strips whitespace
+    assert_equal(["", "funk", "", "so | brother"], tokenize(source, for_liquid_tag: true, trimmed: true))
   end
 
-  def test_utf8_encoded_template
-    source = 'auswählen'
-    assert_equal Encoding::UTF_8, source.encoding
+  def test_invalid_tags
+    assert_equal([""], tokenize("{%-%}", trimmed: true))
+    assert_equal([""], tokenize("{{-}}", trimmed: true))
+  end
+
+  def test_utf8_encoded_source
+    source = "auswählen"
+    assert_equal(Encoding::UTF_8, source.encoding)
     output = tokenize(source)
-    assert_equal [Encoding::UTF_8], output.map(&:encoding)
-    assert_equal [source], output
+    assert_equal([Encoding::UTF_8], output.map(&:encoding))
+    assert_equal([source], output)
+  end
+
+  def test_utf8_compatible_source
+    source = String.new("ascii", encoding: Encoding::ASCII)
+    tokenizer = new_tokenizer(source)
+    output = tokenizer.shift
+    assert_equal(Encoding::UTF_8, output.encoding)
+    assert_equal(source, output)
+    assert_nil(tokenizer.shift)
+  end
+
+  def test_non_utf8_compatible_source
+    source = "üñicode".dup.force_encoding(Encoding::BINARY) # rubocop:disable Performance/UnfreezeString
+    exc = assert_raises(Encoding::CompatibilityError) do
+      Liquid::C::Tokenizer.new(source, 1, false)
+    end
+    assert_equal("non-UTF8 encoded source (ASCII-8BIT) not supported", exc.message)
+  end
+
+  def test_source_too_large
+    too_large_source = "a" * 2**24
+    max_length_source = too_large_source.chop
+
+    # C safety check
+    err = assert_raises(ArgumentError) do
+      Liquid::C::Tokenizer.new(too_large_source, 1, false)
+    end
+    assert_match(/Source too large, max \d+ bytes/, err.message)
+
+    # ruby patch fallback
+    parse_context = Liquid::ParseContext.new
+    liquid_c_tokenizer = parse_context.new_tokenizer(max_length_source)
+    assert_instance_of(Liquid::C::Tokenizer, liquid_c_tokenizer)
+    refute(parse_context.liquid_c_nodes_disabled?)
+
+    parse_context = Liquid::ParseContext.new
+    fallback_tokenizer = parse_context.new_tokenizer(too_large_source)
+    assert_instance_of(Liquid::Tokenizer, fallback_tokenizer)
+    assert_equal(true, parse_context.liquid_c_nodes_disabled?)
   end
 
   private
 
-  def tokenize(source)
-    tokenizer = Liquid::C::Tokenizer.new(source, false)
+  def new_tokenizer(source, parse_context: Liquid::ParseContext.new)
+    parse_context.new_tokenizer(source)
+  end
+
+  def tokenize(source, for_liquid_tag: false, trimmed: false)
+    tokenizer = Liquid::C::Tokenizer.new(source, 1, for_liquid_tag)
     tokens = []
-    while t = tokenizer.shift
+    while (t = trimmed ? tokenizer.send(:shift_trimmed) : tokenizer.shift)
       tokens << t
     end
     tokens

data/test/unit/variable_test.rb

@@ -1,110 +1,329 @@
 # encoding: utf-8
-require 'test_helper'
+# frozen_string_literal: true
 
-class VariableTest < MiniTest::Unit::TestCase
+require "test_helper"
+
+class VariableTest < Minitest::Test
   def test_variable_parse
-    assert_equal [lookup('hello'), []], variable_parse('hello')
-    assert_equal ['world', []], variable_parse(' "world" ')
-    assert_equal [lookup('hello["world"]'), []], variable_parse(' hello["world"] ')
-    assert_equal [nil, []], variable_parse('')
-    assert_equal [lookup('question?'), []], variable_parse('question?')
-    assert_equal [lookup('[meta]'), []], variable_parse('[meta]')
-    assert_equal [lookup('a-b'), []], variable_parse('a-b')
-    assert_equal [lookup('a-2'), []], variable_parse('a-2')
+    assert_equal("world", variable_strict_parse("hello").render!({ "hello" => "world" }))
+    assert_equal("world", variable_strict_parse('"world"').render!)
+    assert_equal("answer", variable_strict_parse('hello["world"]').render!({ "hello" => { "world" => "answer" } }))
+    assert_equal("answer", variable_strict_parse("question?").render!({ "question?" => "answer" }))
+    assert_equal("value", variable_strict_parse("[meta]").render!({ "meta" => "key", "key" => "value" }))
+    assert_equal("result", variable_strict_parse("a-b").render!({ "a-b" => "result" }))
+    assert_equal("result", variable_strict_parse("a-2").render!({ "a-2" => "result" }))
   end
 
   def test_strictness
-    assert_raises(Liquid::SyntaxError) { variable_parse(' hello["world\']" ') }
-    assert_raises(Liquid::SyntaxError) { variable_parse('-..') }
-    assert_raises(Liquid::SyntaxError) { variable_parse('question?mark') }
-    assert_raises(Liquid::SyntaxError) { variable_parse('123.foo') }
-    assert_raises(Liquid::SyntaxError) { variable_parse(' | nothing') }
-
-    ['a .b', 'a. b', 'a . b'].each do |var|
-      assert_raises(Liquid::SyntaxError) { variable_parse(var) }
-    end
+    assert_raises(Liquid::SyntaxError) { variable_strict_parse(' hello["world\']" ') }
+    assert_raises(Liquid::SyntaxError) { variable_strict_parse(" -..") }
+    assert_raises(Liquid::SyntaxError) { variable_strict_parse("question?mark") }
+    assert_raises(Liquid::SyntaxError) { variable_strict_parse("123.foo") }
+    assert_raises(Liquid::SyntaxError) { variable_strict_parse(" | nothing") }
 
-    ['a -b', 'a- b', 'a - b'].each do |var|
-      assert_raises(Liquid::SyntaxError) { variable_parse(var) }
+    ["a -b", "a- b", "a - b"].each do |var|
+      assert_raises(Liquid::SyntaxError) { variable_strict_parse(var) }
     end
   end
 
   def test_literals
-    assert_equal [true, []], variable_parse('true')
-    assert_equal [nil, []], variable_parse('nil')
-    assert_equal [123.4, []], variable_parse('123.4')
+    assert_equal("", variable_strict_parse("").render!)
+    assert_equal("true", variable_strict_parse("true").render!)
+    assert_equal("", variable_strict_parse("nil").render!)
+    assert_equal("123.4", variable_strict_parse("123.4").render!)
+
+    assert_equal("blank_value", variable_strict_parse("[blank]").render!({ "" => "blank_value" }))
+    assert_equal("result", variable_strict_parse("[true][blank]").render!({ true => { "" => "result" } }))
+    assert_equal("result", variable_strict_parse('x["size"]').render!({ "x" => { "size" => "result" } }))
+    assert_equal("result", variable_strict_parse("blank.x").render!({ "blank" => { "x" => "result" } }))
+    assert_equal("result", variable_strict_parse('blank["x"]').render!({ "blank" => { "x" => "result" } }))
+  end
+
+  module InspectCallFilters
+    def filter1(input, *args)
+      inspect_call(__method__, input, args)
+    end
+
+    def filter2(input, *args)
+      inspect_call(__method__, input, args)
+    end
 
-    assert_equal [lookup('[blank]'), []], variable_parse('[blank]')
-    assert_equal [lookup(false, true, [Liquid::Expression::LITERALS['blank']], 0), []], variable_parse('[true][blank]')
-    assert_equal [lookup('[true][blank]'), []], variable_parse('[true][blank]')
-    assert_equal [lookup('x["size"]'), []], variable_parse('x["size"]')
+    private
+
+    def inspect_call(filter_name, input, args)
+      "{ filter: #{filter_name.inspect}, input: #{input.inspect}, args: #{args.inspect} }"
+    end
   end
 
   def test_variable_filter
-    name = lookup('name')
-    assert_equal [name, [['filter', []]]], variable_parse(' name | filter ')
-    assert_equal [name, [['filter1', []], ['filter2', []]]], variable_parse(' name | filter1 | filter2 ')
+    context = { "name" => "Bob" }
+
+    filter1_output = variable_strict_parse("name | filter1").render!(context, filters: [InspectCallFilters])
+    assert_equal('{ filter: :filter1, input: "Bob", args: [] }', filter1_output)
+
+    filter2_output = variable_strict_parse("name | filter1 | filter2").render!(context, filters: [InspectCallFilters])
+    assert_equal("{ filter: :filter2, input: #{filter1_output.inspect}, args: [] }", filter2_output)
   end
 
   def test_variable_filter_args
-    name = lookup('name')
-    abc = lookup('abc')
+    context = { "name" => "Bob", "abc" => "xyz" }
+    render_opts = { filters: [InspectCallFilters] }
+
+    filter1_output = variable_strict_parse("name | filter1: abc").render!(context, render_opts)
+    assert_equal('{ filter: :filter1, input: "Bob", args: ["xyz"] }', filter1_output)
 
-    assert_equal [name, [['filter', [abc]]]], variable_parse(' name | filter: abc ')
+    filter2_output = variable_strict_parse("name | filter1: abc | filter2: abc").render!(context, render_opts)
+    assert_equal("{ filter: :filter2, input: #{filter1_output.inspect}, args: [\"xyz\"] }", filter2_output)
 
-    assert_equal [name, [['filter1', [abc]], ['filter2', [abc]]]],
-      variable_parse(' name | filter1: abc | filter2: abc ')
+    context = { "name" => "Bob", "a" => 1, "c" => 3, "e" => 5 }
 
-    assert_equal [name, [['filter', [lookup('a')], {'b' => lookup('c'), 'd' => lookup('e')}]]],
-      variable_parse('name | filter : a , b : c , d : e')
+    output = variable_strict_parse("name | filter1 : a , b : c , d : e").render!(context, render_opts)
+    assert_equal('{ filter: :filter1, input: "Bob", args: [1, {"b"=>3, "d"=>5}] }', output)
 
-    assert_raises Liquid::SyntaxError do
-      variable_parse('name | filter : a : b : c : d : e')
+    assert_raises(Liquid::SyntaxError) do
+      variable_strict_parse("name | filter : a : b : c : d : e")
     end
   end
 
   def test_unicode_strings
-    assert_equal ['å߀êùｉｄｈｔлｓԁѵ߀ｒáƙìｓｔɦｅƅêｓｔｐｃｍáѕｔｅｒｒãｃêｃհèｒｒ', []],
-      variable_parse('"å߀êùｉｄｈｔлｓԁѵ߀ｒáƙìｓｔɦｅƅêｓｔｐｃｍáѕｔｅｒｒãｃêｃհèｒｒ"')
+    string_content = "å߀êùｉｄｈｔлｓԁѵ߀ｒáƙìｓｔɦｅƅêｓｔｐｃｍáѕｔｅｒｒãｃêｃհèｒｒ"
+    assert_equal(string_content, variable_strict_parse("\"#{string_content}\"").render!)
   end
 
   def test_broken_unicode_errors
     err = assert_raises(Liquid::SyntaxError) do
       Liquid::Template.parse("test {{ \xC2\xA0 test }}", error_mode: :strict)
     end
-    assert err.message
+    assert(err.message)
   end
 
   def test_callbacks
-    variable_parses = 0
     variable_fallbacks = 0
 
     callbacks = {
-      variable_parse: lambda { variable_parses += 1 },
-      variable_fallback: lambda { variable_fallbacks += 1 }
+      variable_fallback: lambda { variable_fallbacks += 1 },
     }
 
-    create_variable('abc', error_mode: :lax, stats_callbacks: callbacks)
-    assert_equal 1, variable_parses
-    assert_equal 0, variable_fallbacks
+    Liquid::Template.parse("{{abc}}", error_mode: :lax, stats_callbacks: callbacks)
+    assert_equal(0, variable_fallbacks)
 
-    create_variable('@!#', error_mode: :lax, stats_callbacks: callbacks)
-    assert_equal 2, variable_parses
-    assert_equal 1, variable_fallbacks
+    Liquid::Template.parse("{{@!#}}", error_mode: :lax, stats_callbacks: callbacks)
+    assert_equal(1, variable_fallbacks)
   end
 
-  private
+  def test_write_string
+    output = Liquid::Template.parse("{{ str }}").render({ "str" => "foo" })
+    assert_equal("foo", output)
+  end
+
+  def test_write_fixnum
+    output = Liquid::Template.parse("{{ num }}").render({ "num" => 123456 })
+    assert_equal("123456", output)
+  end
 
-  def create_variable(markup, options={})
-    Liquid::Variable.new(markup, Liquid::ParseContext.new(options))
+  def test_write_array
+    output = Liquid::Template.parse("{{ ary }}").render({ "ary" => ["foo", 123, ["nested", "ary"], nil, 0.5] })
+    assert_equal("foo123nestedary0.5", output)
   end
 
-  def variable_parse(markup)
-    name = Liquid::Variable.c_strict_parse(markup, filters = [])
-    [name, filters]
+  def test_write_nil
+    output = Liquid::Template.parse("{{ obj }}").render({ "obj" => nil })
+    assert_equal("", output)
   end
 
-  def lookup(*args)
-    Liquid::VariableLookup.new(*args)
+  class StringConvertible
+    def initialize(as_string)
+      @as_string = as_string
+    end
+
+    def to_s
+      @as_string
+    end
+
+    def to_liquid
+      self
+    end
+  end
+
+  def test_write_to_s_convertible_object
+    output = Liquid::Template.parse("{{ obj }}").render!({ "obj" => StringConvertible.new("foo") })
+    assert_equal("foo", output)
+  end
+
+  def test_write_object_with_broken_to_s
+    template = Liquid::Template.parse("{{ obj }}")
+    exc = assert_raises(TypeError) do
+      template.render!({ "obj" => StringConvertible.new(123) })
+    end
+    assert_equal(
+      "VariableTest::StringConvertible#to_s returned a non-String convertible value of type Integer",
+      exc.message
+    )
+  end
+
+  class DerivedString < String
+    def to_s
+      self
+    end
+  end
+
+  def test_write_derived_string
+    output = Liquid::Template.parse("{{ obj }}").render!({ "obj" => DerivedString.new("bar") })
+    assert_equal("bar", output)
+  end
+
+  def test_filter_without_args
+    output = Liquid::Template.parse("{{ var | upcase }}").render({ "var" => "Hello" })
+    assert_equal("HELLO", output)
+  end
+
+  def test_filter_with_const_arg
+    output = Liquid::Template.parse("{{ x | plus: 2 }}").render({ "x" => 3 })
+    assert_equal("5", output)
+  end
+
+  def test_filter_with_variable_arg
+    output = Liquid::Template.parse("{{ x | plus: y }}").render({ "x" => 10, "y" => 123 })
+    assert_equal("133", output)
+  end
+
+  def test_filter_with_variable_arg_after_const_arg
+    output = Liquid::Template.parse("{{ ary | slice: 1, 2 }}").render({ "ary" => [1, 2, 3, 4] })
+    assert_equal("23", output)
+  end
+
+  def test_filter_with_const_keyword_arg
+    output = Liquid::Template.parse("{{ value | default: 'None' }}").render({ "value" => false })
+    assert_equal("None", output)
+
+    output = Liquid::Template.parse("{{ value | default: 'None', allow_false: true }}").render({ "value" => false })
+    assert_equal("false", output)
+  end
+
+  def test_filter_with_variable_keyword_arg
+    template = Liquid::Template.parse("{{ value | default: 'None', allow_false: false_allowed }}")
+
+    assert_equal("None", template.render({ "value" => false, "false_allowed" => false }))
+    assert_equal("false", template.render({ "value" => false, "false_allowed" => true }))
+  end
+
+  def test_filter_error
+    output = Liquid::Template.parse("before ({{ ary | concat: 2 }}) after").render({ "ary" => [1] })
+    assert_equal("before (Liquid error: concat filter requires an array argument) after", output)
+  end
+
+  def test_render_variable_object
+    variable = Liquid::Variable.new("ary | concat: ary2", Liquid::ParseContext.new)
+    assert_instance_of(Liquid::C::VariableExpression, variable.name)
+
+    context = Liquid::Context.new("ary" => [1], "ary2" => [2])
+    assert_equal([1, 2], variable.render(context))
+
+    context["ary2"] = 2
+    exc = assert_raises(Liquid::ArgumentError) do
+      variable.render(context)
+    end
+    assert_equal("Liquid error: concat filter requires an array argument", exc.message)
+  end
+
+  def test_filter_argument_error_translation
+    variable = Liquid::Variable.new("'some words' | split", Liquid::ParseContext.new)
+    context = Liquid::Context.new
+    exc = assert_raises(Liquid::ArgumentError) { variable.render(context) }
+    assert_equal("Liquid error: wrong number of arguments (given 1, expected 2)", exc.message)
+  end
+
+  class IntegerDrop < Liquid::Drop
+    def initialize(value)
+      super()
+      @value = value.to_i
+    end
+
+    def to_liquid_value
+      @value
+    end
+  end
+
+  def test_to_liquid_value_on_variable_lookup
+    context = {
+      "number" => IntegerDrop.new("1"),
+      "list" => [1, 2, 3, 4, 5],
+    }
+
+    output = variable_strict_parse("list[number]").render!(context)
+    assert_equal("2", output)
+  end
+
+  def test_encoding_error_message_with_multi_byte_characters
+    # 2 byte character
+    exc = assert_raises(Liquid::SyntaxError) do
+      variable_strict_parse("\u00A0")
+    end
+    assert_equal(
+      "Liquid syntax error: Unexpected character \u00A0 in \"{{\u00a0}}\"",
+      exc.message
+    )
+
+    # 3 byte character
+    exc = assert_raises(Liquid::SyntaxError) do
+      variable_strict_parse("\u3042")
+    end
+    assert_equal(
+      "Liquid syntax error: Unexpected character \u3042 in \"{{\u3042}}\"",
+      exc.message
+    )
+
+    # 4 byte character
+    exc = assert_raises(Liquid::SyntaxError) do
+      variable_strict_parse("\u{1F600}")
+    end
+    assert_equal(
+      "Liquid syntax error: Unexpected character \u{1F600} in \"{{\u{1F600}}}\"",
+      exc.message
+    )
+  end
+
+  def test_invalid_utf8_sequence
+    # 2 byte character with 1 byte missing
+    exc = assert_raises(ArgumentError) do
+      variable_strict_parse("\xC0")
+    end
+    assert_equal("invalid byte sequence in UTF-8", exc.message)
+
+    # 3 byte character with 1 byte missing
+    exc = assert_raises(ArgumentError) do
+      variable_strict_parse("\xE0\x01")
+    end
+    assert_equal("invalid byte sequence in UTF-8", exc.message)
+
+    # 3 byte character with 2 byte missing
+    exc = assert_raises(ArgumentError) do
+      variable_strict_parse("\xE0")
+    end
+    assert_equal("invalid byte sequence in UTF-8", exc.message)
+
+    # 4 byte character with 1 byte missing
+    exc = assert_raises(ArgumentError) do
+      variable_strict_parse("\xF0\x01\x01")
+    end
+    assert_equal("invalid byte sequence in UTF-8", exc.message)
+
+    # 4 byte character with 2 byte missing
+    exc = assert_raises(ArgumentError) do
+      variable_strict_parse("\xF0\x01")
+    end
+    assert_equal("invalid byte sequence in UTF-8", exc.message)
+
+    # 4 byte character with 3 byte missing
+    exc = assert_raises(ArgumentError) do
+      variable_strict_parse("\xF0")
+    end
+    assert_equal("invalid byte sequence in UTF-8", exc.message)
+  end
+
+  private
+
+  def variable_strict_parse(markup)
+    Liquid::Template.parse("{{#{markup}}}", error_mode: :strict)
   end
 end