liquid 4.0.3 → 5.1.0

data/test/unit/regexp_unit_test.rb

@@ -1,44 +1,46 @@
+# frozen_string_literal: true
+
 require 'test_helper'
 
 class RegexpUnitTest < Minitest::Test
   include Liquid
 
   def test_empty
-    assert_equal [], ''.scan(QuotedFragment)
+    assert_equal([], ''.scan(QuotedFragment))
   end
 
   def test_quote
-    assert_equal ['"arg 1"'], '"arg 1"'.scan(QuotedFragment)
+    assert_equal(['"arg 1"'], '"arg 1"'.scan(QuotedFragment))
   end
 
   def test_words
-    assert_equal ['arg1', 'arg2'], 'arg1 arg2'.scan(QuotedFragment)
+    assert_equal(['arg1', 'arg2'], 'arg1 arg2'.scan(QuotedFragment))
   end
 
   def test_tags
-    assert_equal ['<tr>', '</tr>'], '<tr> </tr>'.scan(QuotedFragment)
-    assert_equal ['<tr></tr>'], '<tr></tr>'.scan(QuotedFragment)
-    assert_equal ['<style', 'class="hello">', '</style>'], %(<style class="hello">' </style>).scan(QuotedFragment)
+    assert_equal(['<tr>', '</tr>'], '<tr> </tr>'.scan(QuotedFragment))
+    assert_equal(['<tr></tr>'], '<tr></tr>'.scan(QuotedFragment))
+    assert_equal(['<style', 'class="hello">', '</style>'], %(<style class="hello">' </style>).scan(QuotedFragment))
   end
 
   def test_double_quoted_words
-    assert_equal ['arg1', 'arg2', '"arg 3"'], 'arg1 arg2 "arg 3"'.scan(QuotedFragment)
+    assert_equal(['arg1', 'arg2', '"arg 3"'], 'arg1 arg2 "arg 3"'.scan(QuotedFragment))
   end
 
   def test_single_quoted_words
-    assert_equal ['arg1', 'arg2', "'arg 3'"], 'arg1 arg2 \'arg 3\''.scan(QuotedFragment)
+    assert_equal(['arg1', 'arg2', "'arg 3'"], 'arg1 arg2 \'arg 3\''.scan(QuotedFragment))
   end
 
   def test_quoted_words_in_the_middle
-    assert_equal ['arg1', 'arg2', '"arg 3"', 'arg4'], 'arg1 arg2 "arg 3" arg4   '.scan(QuotedFragment)
+    assert_equal(['arg1', 'arg2', '"arg 3"', 'arg4'], 'arg1 arg2 "arg 3" arg4   '.scan(QuotedFragment))
   end
 
   def test_variable_parser
-    assert_equal ['var'],                               'var'.scan(VariableParser)
-    assert_equal ['var', 'method'],                     'var.method'.scan(VariableParser)
-    assert_equal ['var', '[method]'],                   'var[method]'.scan(VariableParser)
-    assert_equal ['var', '[method]', '[0]'],            'var[method][0]'.scan(VariableParser)
-    assert_equal ['var', '["method"]', '[0]'],          'var["method"][0]'.scan(VariableParser)
-    assert_equal ['var', '[method]', '[0]', 'method'],  'var[method][0].method'.scan(VariableParser)
+    assert_equal(['var'],                               'var'.scan(VariableParser))
+    assert_equal(['var', 'method'],                     'var.method'.scan(VariableParser))
+    assert_equal(['var', '[method]'],                   'var[method]'.scan(VariableParser))
+    assert_equal(['var', '[method]', '[0]'],            'var[method][0]'.scan(VariableParser))
+    assert_equal(['var', '["method"]', '[0]'],          'var["method"][0]'.scan(VariableParser))
+    assert_equal(['var', '[method]', '[0]', 'method'],  'var[method][0].method'.scan(VariableParser))
   end
 end # RegexpTest

data/test/unit/static_registers_unit_test.rb

@@ -0,0 +1,156 @@
+# frozen_string_literal: true
+
+require 'test_helper'
+
+class StaticRegistersUnitTest < Minitest::Test
+  include Liquid
+
+  def test_set
+    static_register = StaticRegisters.new(a: 1, b: 2)
+    static_register[:b] = 22
+    static_register[:c] = 33
+
+    assert_equal(1, static_register[:a])
+    assert_equal(22, static_register[:b])
+    assert_equal(33, static_register[:c])
+  end
+
+  def test_get_missing_key
+    static_register = StaticRegisters.new
+
+    assert_nil(static_register[:missing])
+  end
+
+  def test_delete
+    static_register = StaticRegisters.new(a: 1, b: 2)
+    static_register[:b] = 22
+    static_register[:c] = 33
+
+    assert_nil(static_register.delete(:a))
+
+    assert_equal(22, static_register.delete(:b))
+
+    assert_equal(33, static_register.delete(:c))
+    assert_nil(static_register[:c])
+
+    assert_nil(static_register.delete(:d))
+  end
+
+  def test_fetch
+    static_register = StaticRegisters.new(a: 1, b: 2)
+    static_register[:b] = 22
+    static_register[:c] = 33
+
+    assert_equal(1, static_register.fetch(:a))
+    assert_equal(1, static_register.fetch(:a, "default"))
+    assert_equal(22, static_register.fetch(:b))
+    assert_equal(22, static_register.fetch(:b, "default"))
+    assert_equal(33, static_register.fetch(:c))
+    assert_equal(33, static_register.fetch(:c, "default"))
+
+    assert_raises(KeyError) do
+      static_register.fetch(:d)
+    end
+    assert_equal("default", static_register.fetch(:d, "default"))
+
+    result = static_register.fetch(:d) { "default" }
+    assert_equal("default", result)
+
+    result = static_register.fetch(:d, "default 1") { "default 2" }
+    assert_equal("default 2", result)
+  end
+
+  def test_key
+    static_register = StaticRegisters.new(a: 1, b: 2)
+    static_register[:b] = 22
+    static_register[:c] = 33
+
+    assert_equal(true, static_register.key?(:a))
+    assert_equal(true, static_register.key?(:b))
+    assert_equal(true, static_register.key?(:c))
+    assert_equal(false, static_register.key?(:d))
+  end
+
+  def test_static_register_can_be_frozen
+    static_register = StaticRegisters.new(a: 1)
+
+    static_register.static.freeze
+
+    assert_raises(RuntimeError) do
+      static_register.static[:a] = "foo"
+    end
+
+    assert_raises(RuntimeError) do
+      static_register.static[:b] = "foo"
+    end
+
+    assert_raises(RuntimeError) do
+      static_register.static.delete(:a)
+    end
+
+    assert_raises(RuntimeError) do
+      static_register.static.delete(:c)
+    end
+  end
+
+  def test_new_static_retains_static
+    static_register = StaticRegisters.new(a: 1, b: 2)
+    static_register[:b] = 22
+    static_register[:c] = 33
+
+    new_static_register = StaticRegisters.new(static_register)
+    new_static_register[:b] = 222
+
+    newest_static_register = StaticRegisters.new(new_static_register)
+    newest_static_register[:c] = 333
+
+    assert_equal(1, static_register[:a])
+    assert_equal(22, static_register[:b])
+    assert_equal(33, static_register[:c])
+
+    assert_equal(1, new_static_register[:a])
+    assert_equal(222, new_static_register[:b])
+    assert_nil(new_static_register[:c])
+
+    assert_equal(1, newest_static_register[:a])
+    assert_equal(2, newest_static_register[:b])
+    assert_equal(333, newest_static_register[:c])
+  end
+
+  def test_multiple_instances_are_unique
+    static_register_1 = StaticRegisters.new(a: 1, b: 2)
+    static_register_1[:b] = 22
+    static_register_1[:c] = 33
+
+    static_register_2 = StaticRegisters.new(a: 10, b: 20)
+    static_register_2[:b] = 220
+    static_register_2[:c] = 330
+
+    assert_equal({ a: 1, b: 2 }, static_register_1.static)
+    assert_equal(1, static_register_1[:a])
+    assert_equal(22, static_register_1[:b])
+    assert_equal(33, static_register_1[:c])
+
+    assert_equal({ a: 10, b: 20 }, static_register_2.static)
+    assert_equal(10, static_register_2[:a])
+    assert_equal(220, static_register_2[:b])
+    assert_equal(330, static_register_2[:c])
+  end
+
+  def test_initialization_reused_static_same_memory_object
+    static_register_1 = StaticRegisters.new(a: 1, b: 2)
+    static_register_1[:b] = 22
+    static_register_1[:c] = 33
+
+    static_register_2 = StaticRegisters.new(static_register_1)
+
+    assert_equal(1, static_register_2[:a])
+    assert_equal(2, static_register_2[:b])
+    assert_nil(static_register_2[:c])
+
+    static_register_1.static[:b] = 222
+    static_register_1.static[:c] = 333
+
+    assert_same(static_register_1.static, static_register_2.static)
+  end
+end

data/test/unit/strainer_factory_unit_test.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+require 'test_helper'
+
+class StrainerFactoryUnitTest < Minitest::Test
+  include Liquid
+
+  module AccessScopeFilters
+    def public_filter
+      "public"
+    end
+
+    def private_filter
+      "private"
+    end
+    private :private_filter
+  end
+
+  StrainerFactory.add_global_filter(AccessScopeFilters)
+
+  module LateAddedFilter
+    def late_added_filter(_input)
+      "filtered"
+    end
+  end
+
+  def setup
+    @context = Context.build
+  end
+
+  def test_strainer
+    strainer = StrainerFactory.create(@context)
+    assert_equal(5, strainer.invoke('size', 'input'))
+    assert_equal("public", strainer.invoke("public_filter"))
+  end
+
+  def test_stainer_raises_argument_error
+    strainer = StrainerFactory.create(@context)
+    assert_raises(Liquid::ArgumentError) do
+      strainer.invoke("public_filter", 1)
+    end
+  end
+
+  def test_stainer_argument_error_contains_backtrace
+    strainer = StrainerFactory.create(@context)
+
+    exception = assert_raises(Liquid::ArgumentError) do
+      strainer.invoke("public_filter", 1)
+    end
+
+    assert_match(
+      /\ALiquid error: wrong number of arguments \((1 for 0|given 1, expected 0)\)\z/,
+      exception.message
+    )
+    assert_equal(exception.backtrace[0].split(':')[0], __FILE__)
+  end
+
+  def test_strainer_only_invokes_public_filter_methods
+    strainer = StrainerFactory.create(@context)
+    assert_equal(false, strainer.class.invokable?('__test__'))
+    assert_equal(false, strainer.class.invokable?('test'))
+    assert_equal(false, strainer.class.invokable?('instance_eval'))
+    assert_equal(false, strainer.class.invokable?('__send__'))
+    assert_equal(true, strainer.class.invokable?('size')) # from the standard lib
+  end
+
+  def test_strainer_returns_nil_if_no_filter_method_found
+    strainer = StrainerFactory.create(@context)
+    assert_nil(strainer.invoke("private_filter"))
+    assert_nil(strainer.invoke("undef_the_filter"))
+  end
+
+  def test_strainer_returns_first_argument_if_no_method_and_arguments_given
+    strainer = StrainerFactory.create(@context)
+    assert_equal("password", strainer.invoke("undef_the_method", "password"))
+  end
+
+  def test_strainer_only_allows_methods_defined_in_filters
+    strainer = StrainerFactory.create(@context)
+    assert_equal("1 + 1", strainer.invoke("instance_eval", "1 + 1"))
+    assert_equal("puts",  strainer.invoke("__send__", "puts", "Hi Mom"))
+    assert_equal("has_method?", strainer.invoke("invoke", "has_method?", "invoke"))
+  end
+
+  def test_strainer_uses_a_class_cache_to_avoid_method_cache_invalidation
+    a = Module.new
+    b = Module.new
+    strainer = StrainerFactory.create(@context, [a, b])
+    assert_kind_of(StrainerTemplate, strainer)
+    assert_kind_of(a, strainer)
+    assert_kind_of(b, strainer)
+    assert_kind_of(Liquid::StandardFilters, strainer)
+  end
+
+  def test_add_global_filter_clears_cache
+    assert_equal('input', StrainerFactory.create(@context).invoke('late_added_filter', 'input'))
+    StrainerFactory.add_global_filter(LateAddedFilter)
+    assert_equal('filtered', StrainerFactory.create(nil).invoke('late_added_filter', 'input'))
+  end
+end

data/test/unit/strainer_template_unit_test.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+require 'test_helper'
+
+class StrainerTemplateUnitTest < Minitest::Test
+  include Liquid
+
+  def test_add_filter_when_wrong_filter_class
+    c = Context.new
+    s = c.strainer
+    wrong_filter = ->(v) { v.reverse }
+
+    exception = assert_raises(TypeError) do
+      s.class.add_filter(wrong_filter)
+    end
+    assert_equal(exception.message, "wrong argument type Proc (expected Module)")
+  end
+
+  module PrivateMethodOverrideFilter
+    private
+
+    def public_filter
+      "overriden as private"
+    end
+  end
+
+  def test_add_filter_raises_when_module_privately_overrides_registered_public_methods
+    strainer = Context.new.strainer
+
+    error = assert_raises(Liquid::MethodOverrideError) do
+      strainer.class.add_filter(PrivateMethodOverrideFilter)
+    end
+    assert_equal('Liquid error: Filter overrides registered public methods as non public: public_filter', error.message)
+  end
+
+  module ProtectedMethodOverrideFilter
+    protected
+
+    def public_filter
+      "overriden as protected"
+    end
+  end
+
+  def test_add_filter_raises_when_module_overrides_registered_public_method_as_protected
+    strainer = Context.new.strainer
+
+    error = assert_raises(Liquid::MethodOverrideError) do
+      strainer.class.add_filter(ProtectedMethodOverrideFilter)
+    end
+    assert_equal('Liquid error: Filter overrides registered public methods as non public: public_filter', error.message)
+  end
+
+  module PublicMethodOverrideFilter
+    def public_filter
+      "public"
+    end
+  end
+
+  def test_add_filter_does_not_raise_when_module_overrides_previously_registered_method
+    strainer = Context.new.strainer
+    with_global_filter do
+      strainer.class.add_filter(PublicMethodOverrideFilter)
+      assert(strainer.class.send(:filter_methods).include?('public_filter'))
+    end
+  end
+
+  def test_add_filter_does_not_include_already_included_module
+    mod = Module.new do
+      class << self
+        attr_accessor :include_count
+        def included(_mod)
+          self.include_count += 1
+        end
+      end
+      self.include_count = 0
+    end
+    strainer = Context.new.strainer
+    strainer.class.add_filter(mod)
+    strainer.class.add_filter(mod)
+    assert_equal(1, mod.include_count)
+  end
+end

data/test/unit/tag_unit_test.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'test_helper'
 
 class TagUnitTest < Minitest::Test
@@ -5,8 +7,8 @@ class TagUnitTest < Minitest::Test
 
   def test_tag
     tag = Tag.parse('tag', "", Tokenizer.new(""), ParseContext.new)
-    assert_equal 'liquid::tag', tag.name
-    assert_equal '', tag.render(Context.new)
+    assert_equal('liquid::tag', tag.name)
+    assert_equal('', tag.render(Context.new))
   end
 
   def test_return_raw_text_of_tag
@@ -16,6 +18,6 @@ class TagUnitTest < Minitest::Test
 
   def test_tag_name_should_return_name_of_the_tag
     tag = Tag.parse("some_tag", "", Tokenizer.new(""), ParseContext.new)
-    assert_equal 'some_tag', tag.tag_name
+    assert_equal('some_tag', tag.tag_name)
   end
 end

data/test/unit/tags/case_tag_unit_test.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'test_helper'
 
 class CaseTagUnitTest < Minitest::Test
@@ -5,6 +7,6 @@ class CaseTagUnitTest < Minitest::Test
 
   def test_case_nodelist
     template = Liquid::Template.parse('{% case var %}{% when true %}WHEN{% else %}ELSE{% endcase %}')
-    assert_equal ['WHEN', 'ELSE'], template.root.nodelist[0].nodelist.map(&:nodelist).flatten
+    assert_equal(['WHEN', 'ELSE'], template.root.nodelist[0].nodelist.map(&:nodelist).flatten)
   end
 end

data/test/unit/tags/for_tag_unit_test.rb

@@ -1,13 +1,15 @@
+# frozen_string_literal: true
+
 require 'test_helper'
 
 class ForTagUnitTest < Minitest::Test
   def test_for_nodelist
     template = Liquid::Template.parse('{% for item in items %}FOR{% endfor %}')
-    assert_equal ['FOR'], template.root.nodelist[0].nodelist.map(&:nodelist).flatten
+    assert_equal(['FOR'], template.root.nodelist[0].nodelist.map(&:nodelist).flatten)
   end
 
   def test_for_else_nodelist
     template = Liquid::Template.parse('{% for item in items %}FOR{% else %}ELSE{% endfor %}')
-    assert_equal ['FOR', 'ELSE'], template.root.nodelist[0].nodelist.map(&:nodelist).flatten
+    assert_equal(['FOR', 'ELSE'], template.root.nodelist[0].nodelist.map(&:nodelist).flatten)
   end
 end

data/test/unit/tags/if_tag_unit_test.rb

@@ -1,8 +1,10 @@
+# frozen_string_literal: true
+
 require 'test_helper'
 
 class IfTagUnitTest < Minitest::Test
   def test_if_nodelist
     template = Liquid::Template.parse('{% if true %}IF{% else %}ELSE{% endif %}')
-    assert_equal ['IF', 'ELSE'], template.root.nodelist[0].nodelist.map(&:nodelist).flatten
+    assert_equal(['IF', 'ELSE'], template.root.nodelist[0].nodelist.map(&:nodelist).flatten)
   end
 end

data/test/unit/template_factory_unit_test.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+require 'test_helper'
+
+class TemplateFactoryUnitTest < Minitest::Test
+  include Liquid
+
+  def test_for_returns_liquid_template_instance
+    template = TemplateFactory.new.for("anything")
+    assert_instance_of(Liquid::Template, template)
+  end
+end

data/test/unit/template_unit_test.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'test_helper'
 
 class TemplateUnitTest < Minitest::Test
@@ -6,7 +8,7 @@ class TemplateUnitTest < Minitest::Test
   def test_sets_default_localization_in_document
     t = Template.new
     t.parse('{%comment%}{%endcomment%}')
-    assert_instance_of I18n, t.root.nodelist[0].options[:locale]
+    assert_instance_of(I18n, t.root.nodelist[0].options[:locale])
   end
 
   def test_sets_default_localization_in_context_with_quick_initialization
@@ -14,13 +16,13 @@ class TemplateUnitTest < Minitest::Test
     t.parse('{%comment%}{%endcomment%}', locale: I18n.new(fixture("en_locale.yml")))
 
     locale = t.root.nodelist[0].options[:locale]
-    assert_instance_of I18n, locale
-    assert_equal fixture("en_locale.yml"), locale.path
+    assert_instance_of(I18n, locale)
+    assert_equal(fixture("en_locale.yml"), locale.path)
   end
 
   def test_with_cache_classes_tags_returns_the_same_class
     original_cache_setting = Liquid.cache_classes
-    Liquid.cache_classes = true
+    Liquid.cache_classes   = true
 
     original_klass = Class.new
     Object.send(:const_set, :CustomTag, original_klass)
@@ -31,7 +33,7 @@ class TemplateUnitTest < Minitest::Test
     new_klass = Class.new
     Object.send(:const_set, :CustomTag, new_klass)
 
-    assert Template.tags['custom'].equal?(original_klass)
+    assert(Template.tags['custom'].equal?(original_klass))
   ensure
     Object.send(:remove_const, :CustomTag)
     Template.tags.delete('custom')
@@ -40,7 +42,7 @@ class TemplateUnitTest < Minitest::Test
 
   def test_without_cache_classes_tags_reloads_the_class
     original_cache_setting = Liquid.cache_classes
-    Liquid.cache_classes = false
+    Liquid.cache_classes   = false
 
     original_klass = Class.new
     Object.send(:const_set, :CustomTag, original_klass)
@@ -51,7 +53,7 @@ class TemplateUnitTest < Minitest::Test
     new_klass = Class.new
     Object.send(:const_set, :CustomTag, new_klass)
 
-    assert Template.tags['custom'].equal?(new_klass)
+    assert(Template.tags['custom'].equal?(new_klass))
   ensure
     Object.send(:remove_const, :CustomTag)
     Template.tags.delete('custom')
@@ -62,17 +64,24 @@ class TemplateUnitTest < Minitest::Test
 
   def test_tags_delete
     Template.register_tag('fake', FakeTag)
-    assert_equal FakeTag, Template.tags['fake']
+    assert_equal(FakeTag, Template.tags['fake'])
 
     Template.tags.delete('fake')
-    assert_nil Template.tags['fake']
+    assert_nil(Template.tags['fake'])
   end
 
   def test_tags_can_be_looped_over
     Template.register_tag('fake', FakeTag)
     result = Template.tags.map { |name, klass| [name, klass] }
-    assert result.include?(["fake", "TemplateUnitTest::FakeTag"])
+    assert(result.include?(["fake", "TemplateUnitTest::FakeTag"]))
   ensure
     Template.tags.delete('fake')
   end
+
+  class TemplateSubclass < Liquid::Template
+  end
+
+  def test_template_inheritance
+    assert_equal("foo", TemplateSubclass.parse("foo").render)
+  end
 end

data/test/unit/tokenizer_unit_test.rb

@@ -1,50 +1,57 @@
+# frozen_string_literal: true
+
 require 'test_helper'
 
 class TokenizerTest < Minitest::Test
   def test_tokenize_strings
-    assert_equal [' '], tokenize(' ')
-    assert_equal ['hello world'], tokenize('hello world')
+    assert_equal([' '], tokenize(' '))
+    assert_equal(['hello world'], tokenize('hello world'))
   end
 
   def test_tokenize_variables
-    assert_equal ['{{funk}}'], tokenize('{{funk}}')
-    assert_equal [' ', '{{funk}}', ' '], tokenize(' {{funk}} ')
-    assert_equal [' ', '{{funk}}', ' ', '{{so}}', ' ', '{{brother}}', ' '], tokenize(' {{funk}} {{so}} {{brother}} ')
-    assert_equal [' ', '{{  funk  }}', ' '], tokenize(' {{  funk  }} ')
+    assert_equal(['{{funk}}'], tokenize('{{funk}}'))
+    assert_equal([' ', '{{funk}}', ' '], tokenize(' {{funk}} '))
+    assert_equal([' ', '{{funk}}', ' ', '{{so}}', ' ', '{{brother}}', ' '], tokenize(' {{funk}} {{so}} {{brother}} '))
+    assert_equal([' ', '{{  funk  }}', ' '], tokenize(' {{  funk  }} '))
   end
 
   def test_tokenize_blocks
-    assert_equal ['{%comment%}'], tokenize('{%comment%}')
-    assert_equal [' ', '{%comment%}', ' '], tokenize(' {%comment%} ')
+    assert_equal(['{%comment%}'], tokenize('{%comment%}'))
+    assert_equal([' ', '{%comment%}', ' '], tokenize(' {%comment%} '))
 
-    assert_equal [' ', '{%comment%}', ' ', '{%endcomment%}', ' '], tokenize(' {%comment%} {%endcomment%} ')
-    assert_equal ['  ', '{% comment %}', ' ', '{% endcomment %}', ' '], tokenize("  {% comment %} {% endcomment %} ")
+    assert_equal([' ', '{%comment%}', ' ', '{%endcomment%}', ' '], tokenize(' {%comment%} {%endcomment%} '))
+    assert_equal(['  ', '{% comment %}', ' ', '{% endcomment %}', ' '], tokenize("  {% comment %} {% endcomment %} "))
   end
 
   def test_calculate_line_numbers_per_token_with_profiling
-    assert_equal [1],       tokenize_line_numbers("{{funk}}")
-    assert_equal [1, 1, 1], tokenize_line_numbers(" {{funk}} ")
-    assert_equal [1, 2, 2], tokenize_line_numbers("\n{{funk}}\n")
-    assert_equal [1, 1, 3], tokenize_line_numbers(" {{\n funk \n}} ")
+    assert_equal([1],       tokenize_line_numbers("{{funk}}"))
+    assert_equal([1, 1, 1], tokenize_line_numbers(" {{funk}} "))
+    assert_equal([1, 2, 2], tokenize_line_numbers("\n{{funk}}\n"))
+    assert_equal([1, 1, 3], tokenize_line_numbers(" {{\n funk \n}} "))
   end
 
   private
 
+  def new_tokenizer(source, parse_context: Liquid::ParseContext.new, start_line_number: nil)
+    parse_context.new_tokenizer(source, start_line_number: start_line_number)
+  end
+
   def tokenize(source)
-    tokenizer = Liquid::Tokenizer.new(source)
-    tokens = []
-    while t = tokenizer.shift
+    tokenizer = new_tokenizer(source)
+    tokens    = []
+    # shift is private in Liquid::C::Tokenizer, since it is only for unit testing
+    while (t = tokenizer.send(:shift))
       tokens << t
     end
     tokens
   end
 
   def tokenize_line_numbers(source)
-    tokenizer = Liquid::Tokenizer.new(source, true)
+    tokenizer    = new_tokenizer(source, start_line_number: 1)
     line_numbers = []
     loop do
       line_number = tokenizer.line_number
-      if tokenizer.shift
+      if tokenizer.send(:shift)
         line_numbers << line_number
       else
         break