liquid-c 4.0.1 → 4.1.0

data/lib/liquid/c.rb

@@ -1,77 +1,213 @@
-require 'liquid/c/version'
-require 'liquid'
-require 'liquid_c'
+# frozen_string_literal: true
+
+require "liquid/c/version"
+require "liquid"
+require "liquid_c"
+require "liquid/c/compile_ext"
+
+Liquid::C::BlockBody.class_eval do
+  def render(context)
+    render_to_output_buffer(context, +"")
+  end
+end
+
+module Liquid
+  BlockBody.class_eval do
+    def self.c_rescue_render_node(context, output, line_number, exc, blank_tag)
+      # There seems to be a MRI ruby bug with how the rb_rescue C function,
+      # where $! gets set for its rescue callback, but it doesn't stay set
+      # after a nested exception is raised and handled as is the case in
+      # Liquid::Context#internal_error. This isn't a problem for plain ruby code,
+      # so use a ruby rescue block to have setup $! properly.
+      raise(exc)
+    rescue => exc
+      rescue_render_node(context, output, line_number, exc, blank_tag)
+    end
+  end
+end
 
 module Liquid
   module C
-    @enabled = true
+    # Placeholder for variables in the Liquid::C::BlockBody#nodelist.
+    class VariablePlaceholder
+      class << self
+        private :new
+      end
+    end
 
-    class << self
-      attr_accessor :enabled
+    class Tokenizer
+      MAX_SOURCE_BYTE_SIZE = (1 << 24) - 1
     end
   end
 end
 
-Liquid::Tokenizer.class_eval do
-  def self.new(source, line_numbers = false)
-    if Liquid::C.enabled
-      Liquid::C::Tokenizer.new(source.to_s, line_numbers)
+Liquid::Raw.class_eval do
+  alias_method :ruby_parse, :parse
+
+  def parse(tokenizer)
+    if parse_context.liquid_c_nodes_disabled?
+      ruby_parse(tokenizer)
     else
-      super
+      c_parse(tokenizer)
     end
   end
 end
 
-Liquid::BlockBody.class_eval do
-  alias_method :ruby_parse, :parse
+Liquid::ParseContext.class_eval do
+  class << self
+    attr_accessor :liquid_c_nodes_disabled
+  end
+  self.liquid_c_nodes_disabled = false
+
+  alias_method :ruby_new_block_body, :new_block_body
+
+  def new_block_body
+    if liquid_c_nodes_disabled?
+      ruby_new_block_body
+    else
+      Liquid::C::BlockBody.new(self)
+    end
+  end
+
+  alias_method :ruby_new_tokenizer, :new_tokenizer
+  def new_tokenizer(source, start_line_number: nil, for_liquid_tag: false)
+    unless liquid_c_nodes_disabled?
+      source = source.to_s.to_str
+      if source.bytesize <= Liquid::C::Tokenizer::MAX_SOURCE_BYTE_SIZE
+        source = source.encode(Encoding::UTF_8)
+        return Liquid::C::Tokenizer.new(source, start_line_number || 0, for_liquid_tag)
+      else
+        @liquid_c_nodes_disabled = true
+      end
+    end
 
-  def parse(tokens, options)
-    if Liquid::C.enabled && !options[:profile]
-      c_parse(tokens, options) { |t, m| yield t, m }
+    ruby_new_tokenizer(source, start_line_number: start_line_number, for_liquid_tag: for_liquid_tag)
+  end
+
+  def parse_expression(markup)
+    if liquid_c_nodes_disabled?
+      Liquid::Expression.ruby_parse(markup)
     else
-      ruby_parse(tokens, options) { |t, m| yield t, m }
+      Liquid::C::Expression.lax_parse(markup)
     end
   end
+
+  # @api private
+  def liquid_c_nodes_disabled?
+    # Liquid::Profiler exposes the internal parse tree that we don't want to build when
+    # parsing with liquid-c, so consider liquid-c to be disabled when using it.
+    # Also, some templates are parsed before the profiler is running, on which case we
+    # provide the `disable_liquid_c_nodes` option to enable the Ruby AST to be produced
+    # so the profiler can use it on future runs.
+    return @liquid_c_nodes_disabled if defined?(@liquid_c_nodes_disabled)
+    @liquid_c_nodes_disabled = !Liquid::C.enabled || @template_options[:profile] ||
+      @template_options[:disable_liquid_c_nodes] || self.class.liquid_c_nodes_disabled
+  end
+end
+
+module Liquid
+  module C
+    module DocumentClassPatch
+      def parse(tokenizer, parse_context)
+        if tokenizer.is_a?(Liquid::C::Tokenizer)
+          if parse_context[:bug_compatible_whitespace_trimming]
+            # Temporary to test rollout of the fix for this bug
+            tokenizer.bug_compatible_whitespace_trimming!
+          end
+
+          begin
+            parse_context.start_liquid_c_parsing
+            super
+          ensure
+            parse_context.cleanup_liquid_c_parsing
+          end
+        else
+          super
+        end
+      end
+    end
+    Liquid::Document.singleton_class.prepend(DocumentClassPatch)
+  end
 end
 
 Liquid::Variable.class_eval do
-  alias_method :ruby_lax_parse, :lax_parse
-  alias_method :ruby_strict_parse, :strict_parse
+  class << self
+    # @api private
+    def call_variable_fallback_stats_callback(parse_context)
+      callbacks = parse_context[:stats_callbacks]
+      callbacks[:variable_fallback]&.call if callbacks
+    end
 
-  def lax_parse(markup)
-    stats = options[:stats_callbacks]
-    stats[:variable_parse].call if stats
+    private
 
-    if Liquid::C.enabled
+    # helper method for C code
+    def rescue_strict_parse_syntax_error(error, markup, parse_context)
+      error.line_number = parse_context.line_number
+      error.markup_context = "in \"{{#{markup}}}\""
+      case parse_context.error_mode
+      when :strict
+        raise
+      when :warn
+        parse_context.warnings << error
+      end
+      call_variable_fallback_stats_callback(parse_context)
+      lax_parse(markup, parse_context) # Avoid redundant strict parse
+    end
+
+    def lax_parse(markup, parse_context)
+      old_error_mode = parse_context.error_mode
       begin
-        return strict_parse(markup)
-      rescue Liquid::SyntaxError
-        stats[:variable_fallback].call if stats
+        set_error_mode(parse_context, :lax)
+        new(markup, parse_context)
+      ensure
+        set_error_mode(parse_context, old_error_mode)
       end
     end
 
-    ruby_lax_parse(markup)
+    def set_error_mode(parse_context, mode)
+      parse_context.instance_variable_set(:@error_mode, mode)
+    end
   end
 
+  alias_method :ruby_strict_parse, :strict_parse
+
   def strict_parse(markup)
-    if Liquid::C.enabled
-      @name = Liquid::Variable.c_strict_parse(markup, @filters = [])
-    else
+    if parse_context.liquid_c_nodes_disabled?
       ruby_strict_parse(markup)
+    else
+      c_strict_parse(markup)
     end
   end
 end
 
-Liquid::VariableLookup.class_eval do
-  alias_method :ruby_initialize, :initialize
+Liquid::StrainerTemplate.class_eval do
+  class << self
+    private
 
-  def initialize(markup, name = nil, lookups = nil, command_flags = nil)
-    if Liquid::C.enabled && markup == false
-      @name = name
-      @lookups = lookups
-      @command_flags = command_flags
-    else
-      ruby_initialize(markup)
+    def filter_methods_hash
+      @filter_methods_hash ||= {}.tap do |hash|
+        filter_methods.each do |method_name|
+          hash[method_name.to_sym] = true
+        end
+      end
+    end
+
+    # Convert wrong number of argument error into a liquid exception to
+    # treat it as an error in the template, not an internal error.
+    def arg_exc_to_liquid_exc(argument_error)
+      raise Liquid::ArgumentError, argument_error.message, argument_error.backtrace
+    rescue Liquid::ArgumentError => liquid_exc
+      liquid_exc
+    end
+  end
+end
+
+Liquid::C::Expression.class_eval do
+  class << self
+    def lax_parse(markup)
+      strict_parse(markup)
+    rescue Liquid::SyntaxError
+      Liquid::Expression.ruby_parse(markup)
     end
   end
 end
@@ -80,16 +216,59 @@ Liquid::Expression.class_eval do
   class << self
     alias_method :ruby_parse, :parse
 
-    def parse(markup)
-      return nil unless markup
+    def c_parse(markup)
+      Liquid::C::Expression.lax_parse(markup)
+    end
+  end
+end
+
+Liquid::Context.class_eval do
+  alias_method :ruby_evaluate, :evaluate
+  alias_method :ruby_find_variable, :find_variable
+  alias_method :ruby_strict_variables=, :strict_variables=
+
+  # This isn't entered often by Ruby (most calls stay in C via VariableLookup#evaluate)
+  # so the wrapper method isn't costly.
+  def c_find_variable_kwarg(key, raise_on_not_found: true)
+    c_find_variable(key, raise_on_not_found)
+  end
+end
+
+Liquid::ResourceLimits.class_eval do
+  def self.new(limits)
+    if Liquid::C.enabled
+      Liquid::C::ResourceLimits.new(
+        limits[:render_length_limit],
+        limits[:render_score_limit],
+        limits[:assign_score_limit]
+      )
+    else
+      super
+    end
+  end
+end
+
+module Liquid
+  module C
+    class << self
+      attr_reader :enabled
 
-      if Liquid::C.enabled
-        begin
-          return c_parse(markup)
-        rescue Liquid::SyntaxError
+      def enabled=(value)
+        @enabled = value
+        if value
+          Liquid::Context.send(:alias_method, :evaluate, :c_evaluate)
+          Liquid::Context.send(:alias_method, :find_variable, :c_find_variable_kwarg)
+          Liquid::Context.send(:alias_method, :strict_variables=, :c_strict_variables=)
+          Liquid::Expression.singleton_class.send(:alias_method, :parse, :c_parse)
+        else
+          Liquid::Context.send(:alias_method, :evaluate, :ruby_evaluate)
+          Liquid::Context.send(:alias_method, :find_variable, :ruby_find_variable)
+          Liquid::Context.send(:alias_method, :strict_variables=, :ruby_strict_variables=)
+          Liquid::Expression.singleton_class.send(:alias_method, :parse, :ruby_parse)
         end
       end
-      ruby_parse(markup)
     end
+
+    self.enabled = true
   end
 end

data/liquid-c.gemspec

@@ -1,7 +1,11 @@
 # coding: utf-8
-lib = File.expand_path('../lib', __FILE__)
+# frozen_string_literal: true
+
+# rubocop:disable Gemspec/RubyVersionGlobalsUsage
+
+lib = File.expand_path("../lib", __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require 'liquid/c/version'
+require "liquid/c/version"
 
 Gem::Specification.new do |spec|
   spec.name          = "liquid-c"
@@ -12,19 +16,21 @@ Gem::Specification.new do |spec|
   spec.homepage      = "https://github.com/shopify/liquid-c"
   spec.license       = "MIT"
 
-  spec.extensions    = ['ext/liquid_c/extconf.rb']
-  spec.files         = `git ls-files -z`.split("\x0")
+  spec.extensions    = ["ext/liquid_c/extconf.rb"]
+  spec.files         = %x(git ls-files -z).split("\x0")
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
+  spec.required_ruby_version = ">= 2.5.0"
+
   spec.metadata["allowed_push_host"] = "https://rubygems.org"
 
-  spec.add_dependency 'liquid', '>= 3.0.0'
+  spec.add_dependency("liquid", ">= 5.0.1")
 
-  spec.add_development_dependency "bundler", ">= 1.5"
-  spec.add_development_dependency "rake"
-  spec.add_development_dependency 'rake-compiler'
-  spec.add_development_dependency 'minitest'
-  spec.add_development_dependency 'stackprof' if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new("2.1.0")
+  spec.add_development_dependency("bundler", ">= 1.5") # has bugfix for segfaulting deploys
+  spec.add_development_dependency("minitest")
+  spec.add_development_dependency("rake")
+  spec.add_development_dependency("rake-compiler")
+  spec.add_development_dependency("stackprof") if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new("2.1.0")
 end

data/performance/c_profile.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require "liquid"
+require "liquid/c"
+liquid_lib_dir = $LOAD_PATH.detect { |p| File.exist?(File.join(p, "liquid.rb")) }
+require File.join(File.dirname(liquid_lib_dir), "performance/theme_runner")
+
+TASK_NAMES = ["run", "compile", "render"]
+task_name = ARGV.first || "run"
+unless TASK_NAMES.include?(task_name)
+  raise "Unsupported task '#{task_name}' (must be one of #{TASK_NAMES})"
+end
+task = ThemeRunner.new.method(task_name)
+
+runner_id = fork do
+  end_time = Time.now + 5.0
+  task.call until Time.now >= end_time
+end
+
+profiler_pid = spawn("instruments -t 'Time Profiler' -p #{runner_id}")
+
+Process.waitpid(runner_id)
+Process.waitpid(profiler_pid)

data/performance.rb

@@ -1,6 +1,8 @@
-require 'liquid'
-require 'liquid/c' if ARGV.shift == "c"
-liquid_lib_dir = $LOAD_PATH.detect{ |p| File.exists?(File.join(p, 'liquid.rb')) }
+# frozen_string_literal: true
 
-script = ARGV.shift or abort("unspecified performance script")
+require "liquid"
+require "liquid/c" if ARGV.shift == "c"
+liquid_lib_dir = $LOAD_PATH.detect { |p| File.exist?(File.join(p, "liquid.rb")) }
+
+(script = ARGV.shift) || abort("unspecified performance script")
 require File.join(File.dirname(liquid_lib_dir), "performance/#{script}")

data/rakelib/compile.rake

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+ext_task = Rake::ExtensionTask.new("liquid_c")
+
+# For MacOS, generate debug information that ruby can read
+dsymutil = RbConfig::CONFIG["dsymutil"]
+unless dsymutil.to_s.empty?
+  ext_lib_path = "lib/#{ext_task.binary}"
+  dsym_path = "#{ext_lib_path}.dSYM"
+
+  file dsym_path => [ext_lib_path] do
+    sh dsymutil, ext_lib_path
+  end
+  Rake::Task["compile"].enhance([dsym_path])
+end

data/rakelib/integration_test.rake

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+namespace :test do
+  integration_test_config = lambda do |t|
+    t.libs << "lib"
+    t.test_files = ["test/integration_test.rb"]
+  end
+
+  desc "run test suite with default parser"
+  Rake::TestTask.new(integration: :compile, &integration_test_config)
+
+  namespace :integration do
+    define_env_integration_tests = lambda do |task_name|
+      rake_task = Rake::Task[task_name]
+
+      [
+        [:lax, { "LIQUID_PARSER_MODE" => "lax" }],
+        [:strict, { "LIQUID_PARSER_MODE" => "strict" }],
+        [:without_vm, { "LIQUID_C_DISABLE_VM" => "true" }],
+      ].each do |name, env_vars|
+        task(name) do
+          old_env_values = ENV.to_hash.slice(*env_vars.keys)
+          begin
+            env_vars.each { |key, value| ENV[key] = value }
+            rake_task.invoke
+          ensure
+            old_env_values.each { |key, value| ENV[key] = value }
+            rake_task.reenable
+          end
+        end
+      end
+
+      task(all: [:lax, :strict, :without_vm])
+    end
+
+    define_env_integration_tests.call("test:integration")
+
+    RubyMemcheck::TestTask.new(valgrind: :compile, &integration_test_config)
+    namespace :valgrind do
+      define_env_integration_tests.call("test:integration:valgrind")
+    end
+  end
+end

data/rakelib/performance.rake

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+namespace :benchmark do
+  desc "Run the liquid benchmark with lax parsing"
+  task :run do
+    ruby "./performance.rb c benchmark lax"
+  end
+
+  desc "Run the liquid benchmark with strict parsing"
+  task :strict do
+    ruby "./performance.rb c benchmark strict"
+  end
+end
+
+namespace :c_profile do
+  [:run, :compile, :render].each do |task_name|
+    task(task_name) do
+      ruby "./performance/c_profile.rb #{task_name}"
+    end
+  end
+end
+
+namespace :profile do
+  desc "Run the liquid profile/performance coverage"
+  task :run do
+    ruby "./performance.rb c profile lax"
+  end
+
+  desc "Run the liquid profile/performance coverage with strict parsing"
+  task :strict do
+    ruby "./performance.rb c profile strict"
+  end
+end
+
+namespace :compare do
+  ["lax", "warn", "strict"].each do |type|
+    desc "Compare Liquid to Liquid-C in #{type} mode"
+    task type.to_sym do
+      ruby "./performance.rb bare benchmark #{type}"
+      ruby "./performance.rb c benchmark #{type}"
+    end
+  end
+end

data/rakelib/rubocop.rake

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+task :rubocop do
+  require "rubocop/rake_task"
+  RuboCop::RakeTask.new
+end

data/rakelib/unit_test.rake

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+namespace :test do
+  unit_test_config = lambda do |t|
+    t.libs << "lib" << "test"
+    t.test_files = FileList["test/unit/**/*_test.rb"]
+  end
+
+  Rake::TestTask.new(unit: :compile, &unit_test_config)
+
+  namespace :unit do
+    RubyMemcheck::TestTask.new(valgrind: :compile, &unit_test_config)
+  end
+end

data/test/integration_test.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+at_exit { GC.start }
+
+require_relative "liquid_test_helper"
+
+test_files = Dir[File.join(LIQUID_TEST_DIR, "integration/**/*_test.rb")]
+test_files << File.join(LIQUID_TEST_DIR, "unit/tokenizer_unit_test.rb")
+test_files.each do |test_file|
+  require test_file
+end

data/test/liquid_test_helper.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+# This can be used to setup for running tests from the liquid test suite.
+# For example, you could run a single liquid test as follows:
+# $ ruby -r./test/liquid_test_helper `bundle info liquid --path`/test/integration/template_test.rb
+
+require "bundler/setup"
+
+liquid_lib_dir = $LOAD_PATH.detect { |p| File.exist?(File.join(p, "liquid.rb")) }
+LIQUID_TEST_DIR = File.join(File.dirname(liquid_lib_dir), "test")
+$LOAD_PATH << LIQUID_TEST_DIR << File.expand_path("../lib", __dir__)
+
+require "test_helper"
+require "liquid/c"
+
+if ENV["LIQUID_C_DISABLE_VM"]
+  puts "-- Liquid-C VM Disabled"
+  Liquid::ParseContext.liquid_c_nodes_disabled = true
+end
+
+GC.stress = true if ENV["GC_STRESS"]

data/test/test_helper.rb

@@ -1,2 +1,14 @@
-require 'minitest/autorun'
-require 'liquid/c'
+# frozen_string_literal: true
+
+at_exit { GC.start }
+
+require "minitest/autorun"
+require "liquid/c"
+
+if GC.respond_to?(:verify_compaction_references)
+  # This method was added in Ruby 3.0.0. Calling it this way asks the GC to
+  # move objects around, helping to find object movement bugs.
+  GC.verify_compaction_references(double_heap: true, toward: :empty)
+end
+
+GC.stress = true if ENV["GC_STRESS"]