liquid-c 4.1.0 → 4.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f0b1d438c8d9e73c713f3c66e14cc41a0b5351f33153047d46728a897c1bc576
-  data.tar.gz: 5275881737af08a486e1ebeb672333ffe534e0de20e8d018246013419950f382
+  metadata.gz: 8b5a8a5b1b5f7409b378626e889898eeac6996ef2127894d25d2f342183c633c
+  data.tar.gz: 165117bbbcf583f3c9c0223c3232de5e5c42b67a8ed90b3d268864e4732c685d
 SHA512:
-  metadata.gz: c21caebe56294b8ef4cce7712261c6f7a50ddb3fd72d08ad2463b7b561a8c13b2d0db1c9b3941ee3a8d4f9a816c85764db182b181a1eacf86032b6d27878e974
-  data.tar.gz: 736ebd21fd1cf147ed1f88cc3828a0209027d22cbec79f72f71fde2599523ec40e974b3ad1aa47acfd5724f4c0b1e3508ba74a13efee84e9e9771c39b012ef65
+  metadata.gz: a7c7ed2cd71c5470c4b53671611279e71279cba16f0a6fcd5623935b80373ed941f6b3360264399ec7f8ab1efe39728f0bba1a6fecdd3e704675e7464ab660aa
+  data.tar.gz: 006af34fd3a151cae0387a0d70f18e5f64f98dba95f1e2bf6ea36fe239db3153f301405c349ff25a852977ad38b8873a6d8b2554753f29f37665ef20f7a30fda

data/.github/workflows/cla.yml

@@ -0,0 +1,23 @@
+# .github/workflows/cla.yml
+name: Contributor License Agreement (CLA)
+
+on:
+  pull_request_target:
+    types: [opened, synchronize]
+  issue_comment:
+    types: [created]
+
+jobs:
+  cla:
+    runs-on: ubuntu-latest
+    if: |
+      (github.event.issue.pull_request
+        && !github.event.issue.pull_request.merged_at
+        && contains(github.event.comment.body, 'signed')
+      )
+      || (github.event.pull_request && !github.event.pull_request.merged)
+    steps:
+      - uses: Shopify/shopify-cla-action@v1
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          cla-token: ${{ secrets.CLA_TOKEN }}

data/.github/workflows/liquid.yml

@@ -5,28 +5,36 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        entry:
-          - { ruby: '2.5', allowed-failure: false }
+        include:
           - { ruby: '2.7', allowed-failure: false }
           - { ruby: '3.0', allowed-failure: false }
+          - { ruby: '3.1', allowed-failure: false }
+          - { ruby: '3.2', allowed-failure: false }
+          - { ruby: '3.3', allowed-failure: false }
           - { ruby: ruby-head, allowed-failure: true }
-    name: test (${{ matrix.entry.ruby }})
+          - { ruby: truffleruby-head, allowed-failure: true }
+    name: test (${{ matrix.ruby }})
     steps:
       - uses: actions/checkout@v2
       - uses: ruby/setup-ruby@v1
         with:
-          ruby-version: ${{ matrix.entry.ruby }}
-      - uses: actions/cache@v1
-        with:
-          path: vendor/bundle
-          key: ${{ runner.os }}-gems-${{ hashFiles('Gemfile') }}
-          restore-keys: ${{ runner.os }}-gems-
-      - run: bundle install --jobs=3 --retry=3 --path=vendor/bundle
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
+
       - run: bundle exec rake
-        continue-on-error: ${{ matrix.entry.allowed-failure }}
+        continue-on-error: ${{ matrix.allowed-failure }}
         env:
           LIQUID_C_PEDANTIC: 'true'
+        if: matrix.ruby != 'truffleruby-head'
+
+      - run: bundle exec rake test:unit
+        continue-on-error: ${{ matrix.allowed-failure }}
+        env:
+          LIQUID_C_PEDANTIC: 'true'
+        if: matrix.ruby == 'truffleruby-head'
+
       - run: bundle exec rubocop
+        if: matrix.ruby != 'truffleruby-head'
 
   valgrind:
     runs-on: ubuntu-latest
@@ -34,12 +42,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: ruby/setup-ruby@v1
         with:
-          ruby-version: '3.0'
+          ruby-version: 3.3
+          bundler-cache: true
       - run: sudo apt-get install -y valgrind
-      - uses: actions/cache@v1
-        with:
-          path: vendor/bundle
-          key: ${{ runner.os }}-gems-${{ hashFiles('Gemfile') }}
-          restore-keys: ${{ runner.os }}-gems-
-      - run: bundle install --jobs=3 --retry=3 --path=vendor/bundle
       - run: bundle exec rake test:valgrind

data/Gemfile

@@ -10,6 +10,7 @@ gemspec
 gem "liquid", github: "Shopify/liquid", ref: "master"
 
 group :test do
+  gem "base64", require: false # for older rubocop on Ruby 3.4
   gem "rubocop", "~> 1.24.1", require: false
   gem "rubocop-performance", "~> 1.13.2", require: false
   gem "rubocop-shopify", "~> 2.4.0", require: false

data/README.md

@@ -1,5 +1,5 @@
 # Liquid::C
-[![Build Status](https://travis-ci.org/Shopify/liquid-c.svg?branch=master)](https://travis-ci.org/Shopify/liquid-c)
+[![Build Status](https://travis-ci.org/Shopify/liquid-c.svg?branch=main)](https://travis-ci.org/Shopify/liquid-c)
 
 Partial native implementation of the liquid ruby gem in C.
 
@@ -7,8 +7,8 @@ Partial native implementation of the liquid ruby gem in C.
 
 Add these lines to your application's Gemfile:
 
-    gem 'liquid', github: 'Shopify/liquid', branch: 'master'
-    gem 'liquid-c', github: 'Shopify/liquid-c', branch: 'master'
+    gem 'liquid', github: 'Shopify/liquid', branch: 'main'
+    gem 'liquid-c', github: 'Shopify/liquid-c', branch: 'main'
 
 And then execute:
 

data/Rakefile

@@ -9,8 +9,6 @@ require "ruby_memcheck"
 
 ENV["DEBUG"] ||= "true"
 
-RubyMemcheck.config(binary_name: "liquid_c")
-
 task default: [:test, :rubocop]
 
 task test: ["test:unit", "test:integration:all"]

data/ext/liquid_c/block.c

@@ -3,7 +3,7 @@
 #include "intutil.h"
 #include "tokenizer.h"
 #include "stringutil.h"
-#include "vm.h"
+#include "liquid_vm.h"
 #include "variable.h"
 #include "context.h"
 #include "parse_context.h"
@@ -49,7 +49,7 @@ static void block_body_mark(void *ptr)
     } else {
         rb_gc_mark(body->as.intermediate.parse_context);
         if (body->as.intermediate.vm_assembler_pool)
-            vm_assembler_pool_gc_mark(body->as.intermediate.vm_assembler_pool);
+            rb_gc_mark(body->as.intermediate.vm_assembler_pool->self);
         if (body->as.intermediate.code)
             vm_assembler_gc_mark(body->as.intermediate.code);
     }
@@ -199,9 +199,14 @@ static tag_markup_t internal_block_body_parse(block_body_t *body, parse_context_
                 long name_len = name_end - name_start;
 
                 if (name_len == 0) {
-                    VALUE str = rb_enc_str_new(token.str_trimmed, token.len_trimmed, utf8_encoding);
-                    unknown_tag = (tag_markup_t) { str, str };
-                    goto loop_break;
+                    if (name_start < end && *name_start == '#') { // inline comment
+                        name_end++;
+                        name_len++;
+                    } else {
+                        VALUE str = rb_enc_str_new(token.str_trimmed, token.len_trimmed, utf8_encoding);
+                        unknown_tag = (tag_markup_t) { str, str };
+                        goto loop_break;
+                    }
                 }
 
                 if (name_len == 6 && strncmp(name_start, "liquid", 6) == 0) {

data/ext/liquid_c/context.c

@@ -2,7 +2,7 @@
 #include "context.h"
 #include "variable_lookup.h"
 #include "variable.h"
-#include "vm.h"
+#include "liquid_vm.h"
 #include "expression.h"
 #include "document_body.h"
 

data/ext/liquid_c/context.h

@@ -38,7 +38,7 @@ inline static VALUE value_to_liquid_and_set_context(VALUE value, VALUE context_t
     if (RB_SPECIAL_CONST_P(value))
         return value;
 
-    VALUE klass = RBASIC(value)->klass;
+    VALUE klass = RBASIC_CLASS(value);
 
     // More basic types having #to_liquid of self and no #context=
     if (klass == rb_cString || klass == rb_cArray || klass == rb_cHash)

data/ext/liquid_c/document_body.c

@@ -9,6 +9,14 @@ static VALUE cLiquidCDocumentBody;
 static void document_body_mark(void *ptr)
 {
     document_body_t *body = ptr;
+    /* When Liquid::C::BlockBody#freeze is called, it calls
+     * document_body_write_block_body which sets the document_body_entry but
+     * does not yet set the compiled flag to true. During this time, the only
+     * reference to this Liquid::C::DocumentBody object is in the instance
+     * variables of the parse_context which is marked movable by Ruby. This
+     * causes the self reference here to be moved by compaction causing it to
+     * point to an incorrect object. */
+    rb_gc_mark(body->self);
     rb_gc_mark(body->constants);
 }
 

data/ext/liquid_c/document_body.h

@@ -46,7 +46,7 @@ static inline block_body_header_t *document_body_get_block_body_header_ptr(const
 static inline const VALUE *document_body_get_constants_ptr(const document_body_entry_t *entry)
 {
     block_body_header_t *header = document_body_get_block_body_header_ptr(entry);
-    return RARRAY_PTR(entry->body->constants) + header->constants_offset;
+    return RARRAY_CONST_PTR(entry->body->constants) + header->constants_offset;
 }
 
 static inline void document_body_ensure_compile_finished(document_body_t *body)

data/ext/liquid_c/expression.c

@@ -1,7 +1,7 @@
 #include "liquid.h"
 #include "vm_assembler.h"
 #include "parser.h"
-#include "vm.h"
+#include "liquid_vm.h"
 #include "expression.h"
 
 VALUE cLiquidCExpression;

data/ext/liquid_c/extconf.rb

@@ -12,13 +12,15 @@ end
 if ENV["DEBUG"] == "true"
   append_cflags("-fbounds-check")
   CONFIG["optflags"] = " -O0"
+  # Hack to enable assertions since ruby/assert.h disables assertions unless
+  # Ruby was compiled with -DRUBY_DEBUG.
+  # https://github.com/ruby/ruby/blob/9e678cdbd054f78576a8f21b3f97cccc395ade22/include/ruby/assert.h#L36-L41
+  $CFLAGS << " -DRUBY_DEBUG"
 else
   $CFLAGS << " -DNDEBUG"
 end
 
-if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new("2.7.0") # added in 2.7
-  $CFLAGS << " -DHAVE_RB_HASH_BULK_INSERT"
-end
+have_func "rb_hash_bulk_insert"
 
 $warnflags&.gsub!(/-Wdeclaration-after-statement/, "")
 create_makefile("liquid_c")

data/ext/liquid_c/lexer.c

@@ -144,7 +144,39 @@ const char *lex_one(const char *start, const char *end, lexer_token_t *token)
 
     if (is_special(c)) RETURN_TOKEN(c, 1);
 
-    rb_enc_raise(utf8_encoding, cLiquidSyntaxError, "Unexpected character %c", c);
+    long remaining_str_len = end - str;
+    int char_len = 0;
+
+    // read multibyte UTF-8 character
+    if ((c & 0x80) == 0) {
+        // 1-byte character
+        char_len = 1;
+    } else if ((c & 0xE0) == 0xC0) {
+        // 2-byte character
+        if (remaining_str_len >= 2) {
+            char_len = 2;
+        }
+    } else if ((c & 0xF0) == 0xE0) {
+        // 3-byte character
+        if (remaining_str_len >= 3) {
+            char_len = 3;
+        }
+    } else if ((c & 0xF8) == 0xF0) {
+        // 4-byte character
+        if (remaining_str_len >= 4) {
+            char_len = 4;
+        }
+    } else {
+        // this should never happen
+        rb_enc_raise(utf8_encoding, cLiquidSyntaxError, "Unexpected character %c", c);
+    }
+
+    if (char_len > 0) {
+        rb_enc_raise(utf8_encoding, cLiquidSyntaxError, "Unexpected character %.*s", char_len, str);
+    } else {
+        rb_raise(rb_eArgError, "invalid byte sequence in UTF-8");
+    }
+
     return NULL;
 }
 

data/ext/liquid_c/liquid.c

@@ -12,7 +12,7 @@
 #include "parse_context.h"
 #include "variable_lookup.h"
 #include "vm_assembler_pool.h"
-#include "vm.h"
+#include "liquid_vm.h"
 #include "usage.h"
 
 ID id_evaluate;

data/ext/liquid_c/{vm.c → liquid_vm.c}

@@ -2,7 +2,7 @@
 #include <assert.h>
 
 #include "liquid.h"
-#include "vm.h"
+#include "liquid_vm.h"
 #include "variable_lookup.h"
 #include "intutil.h"
 #include "document_body.h"
@@ -141,31 +141,40 @@ static inline void vm_stack_push(vm_t *vm, VALUE value)
     vm->stack.data_end = (uint8_t *)stack_ptr;
 }
 
-static inline VALUE vm_stack_pop(vm_t *vm)
+static inline VALUE *vm_stack_peek_n(vm_t *vm, size_t n)
 {
     VALUE *stack_ptr = (VALUE *)vm->stack.data_end;
-    stack_ptr--;
+    stack_ptr -= n;
     assert((VALUE *)vm->stack.data <= stack_ptr);
-    vm->stack.data_end = (uint8_t *)stack_ptr;
-    return *stack_ptr;
+    return stack_ptr;
 }
 
-static inline VALUE *vm_stack_pop_n_use_in_place(vm_t *vm, size_t n)
+static inline VALUE *vm_stack_pop_n(vm_t *vm, size_t n)
 {
-    VALUE *stack_ptr = (VALUE *)vm->stack.data_end;
-    stack_ptr -= n;
-    assert((VALUE *)vm->stack.data <= stack_ptr);
+    VALUE *stack_ptr = vm_stack_peek_n(vm, n);
     vm->stack.data_end = (uint8_t *)stack_ptr;
     return stack_ptr;
 }
 
+static inline VALUE vm_stack_pop(vm_t *vm)
+{
+    return *vm_stack_pop_n(vm, 1);
+}
+
 static inline void vm_stack_reserve_for_write(vm_t *vm, size_t num_values)
 {
     c_buffer_reserve_for_write(&vm->stack, num_values * sizeof(VALUE));
 }
 
-static VALUE vm_invoke_filter(vm_t *vm, VALUE filter_name, int num_args, VALUE *args)
+static VALUE vm_invoke_filter(vm_t *vm, VALUE filter_name, int num_args)
 {
+    VALUE *popped_args = vm_stack_pop_n(vm, num_args);
+    /* We have to copy popped_args_ptr to the stack because the VM
+     * no longer holds onto these objects, so they have to exist on
+     * the stack to ensure they don't get garbage collected. */
+    VALUE *args = alloca(sizeof(VALUE *) * num_args);
+    memcpy(args, popped_args, sizeof(VALUE *) * num_args);
+
     bool not_invokable = rb_hash_lookup(vm->context.filter_methods, filter_name) != Qtrue;
     if (RB_UNLIKELY(not_invokable)) {
         if (vm->context.strict_filters) {
@@ -313,8 +322,11 @@ static VALUE vm_render_until_error(VALUE uncast_args)
                 size_t hash_size = *ip++;
                 size_t num_keys_and_values = hash_size * 2;
                 VALUE hash = rb_hash_new();
-                VALUE *args_ptr = vm_stack_pop_n_use_in_place(vm, num_keys_and_values);
+
+                VALUE *args_ptr = vm_stack_peek_n(vm, num_keys_and_values);
                 hash_bulk_insert(num_keys_and_values, args_ptr, hash);
+                vm_stack_pop_n(vm, num_keys_and_values);
+
                 vm_stack_push(vm, hash);
                 break;
             }
@@ -336,8 +348,7 @@ static VALUE vm_render_until_error(VALUE uncast_args)
                     num_args = *ip++; // includes input argument
                 }
 
-                VALUE *args_ptr = vm_stack_pop_n_use_in_place(vm, num_args);
-                VALUE result = vm_invoke_filter(vm, filter_name, num_args, args_ptr);
+                VALUE result = vm_invoke_filter(vm, filter_name, num_args);
                 vm_stack_push(vm, result);
                 break;
             }
@@ -424,24 +435,43 @@ static VALUE vm_render_until_error(VALUE uncast_args)
     }
 }
 
+typedef struct vm_evaluate_rescue_args {
+    vm_render_until_error_args_t *render_args;
+    size_t old_stack_byte_size;
+} vm_evaluate_rescue_args_t;
+
+static VALUE vm_evaluate_rescue(VALUE uncast_args, VALUE exception)
+{
+    vm_evaluate_rescue_args_t *args = (void *)uncast_args;
+    vm_render_until_error_args_t *render_args = args->render_args;
+    vm_t *vm = render_args->vm;
+
+    vm->stack.data_end = vm->stack.data + args->old_stack_byte_size;
+
+    rb_exc_raise(exception);
+    return Qnil;
+}
+
 // Evaluate instructions that avoid using rendering instructions and leave with the result on
 // the top of the stack
 VALUE liquid_vm_evaluate(VALUE context, vm_assembler_t *code)
 {
     vm_t *vm = vm_from_context(context);
     vm_stack_reserve_for_write(vm, code->max_stack_size);
-#ifndef NDEBUG
-    size_t old_stack_byte_size = c_buffer_size(&vm->stack);
-#endif
 
     vm_render_until_error_args_t args = {
         .vm = vm,
         .const_ptr = (const size_t *)code->constants.data,
         .ip = code->instructions.data
     };
-    vm_render_until_error((VALUE)&args);
+    vm_evaluate_rescue_args_t rescue_args = {
+        .render_args = &args,
+        .old_stack_byte_size = c_buffer_size(&vm->stack),
+    };
+    rb_rescue(vm_render_until_error, (VALUE)&args, vm_evaluate_rescue, (VALUE)&rescue_args);
+
     VALUE ret = vm_stack_pop(vm);
-    assert(old_stack_byte_size == c_buffer_size(&vm->stack));
+    assert(rescue_args.old_stack_byte_size == c_buffer_size(&vm->stack));
     return ret;
 }
 

data/ext/liquid_c/parser.c

@@ -67,28 +67,40 @@ static VALUE parse_number(parser_t *p)
     return out;
 }
 
+__attribute__((noreturn)) static void raise_invalid_expression_type(const char *expr, int expr_len)
+{
+    rb_enc_raise(utf8_encoding, cLiquidSyntaxError, "Invalid expression type '%.*s' in range expression", expr_len, expr);
+}
+
 static VALUE try_parse_constant_range(parser_t *p)
 {
     parser_t saved_state = *p;
 
     parser_must_consume(p, TOKEN_OPEN_ROUND);
 
+    const char *begin_str = p->cur.val;
     VALUE begin = try_parse_constant_expression(p);
+    const char *begin_str_end = p->cur.val;
     if (begin == Qundef) {
         *p = saved_state;
         return Qundef;
     }
     parser_must_consume(p, TOKEN_DOTDOT);
 
+    const char *end_str = p->cur.val;
     VALUE end = try_parse_constant_expression(p);
+    const char *end_str_end = p->cur.val;
     if (end == Qundef) {
         *p = saved_state;
         return Qundef;
     }
     parser_must_consume(p, TOKEN_CLOSE_ROUND);
 
-    begin = rb_funcall(begin, id_to_i, 0);
-    end = rb_funcall(end, id_to_i, 0);
+    begin = rb_check_funcall(begin, id_to_i, 0, NULL);
+    if (begin == Qundef) raise_invalid_expression_type(begin_str, (int)(begin_str_end - begin_str));
+
+    end = rb_check_funcall(end, id_to_i, 0, NULL);
+    if (end == Qundef) raise_invalid_expression_type(end_str, (int)(end_str_end - end_str));
 
     bool exclude_end = false;
     return rb_range_new(begin, end, exclude_end);

data/ext/liquid_c/raw.c

@@ -27,25 +27,51 @@ static bool match_full_token_possibly_invalid(token_t *token, struct full_token_
     const char *curr_delimiter_start;
     long curr_delimiter_len = 0;
 
-    for (long i = len - 3; i >= 0; i--) {
+    bool is_last_char_whitespace = true;
+
+    // Search from the end of the string.
+    // The token could have a part of the body like this:
+    // {% endraw {% endraw %}
+    // In this case, we need to return body_len to 10 to preserve the body content.
+    for (long i = len - 3; i > 1; i--) {
         char c = str[i];
 
+        // match \s
+        bool is_whitespace = rb_isspace(c);
+
         if (is_word_char(c)) {
             curr_delimiter_start = str + i;
-            curr_delimiter_len++;
-        } else {
-            if (curr_delimiter_len > 0) {
-                match->delimiter_start = curr_delimiter_start;
-                match->delimiter_len = curr_delimiter_len;
+
+            if (is_last_char_whitespace) {
+                // start a new delimiter match
+                curr_delimiter_len = 1;
+            } else {
+                curr_delimiter_len++;
             }
+        } else if (!is_word_char(c) && !is_whitespace) {
             curr_delimiter_start = NULL;
             curr_delimiter_len = 0;
         }
 
-        if (c == '%' && match->delimiter_len > 0 &&
-                i - 1 >= 0 && str[i - 1] == '{') {
-            match->body_len = i - 1;
-            return true;
+        is_last_char_whitespace = is_whitespace;
+
+        if (curr_delimiter_len > 0) {
+            // match start of a tag which is {% or {%-
+            if (
+                (str[i - 1] == '%' && str[i - 2] == '{') ||
+                (i - 3 >= 0 && str[i - 1] == '-' && str[i - 2] == '%' && str[i - 3] == '{')
+            ) {
+                match->delimiter_start = curr_delimiter_start;
+                match->delimiter_len = curr_delimiter_len;
+
+                if (str[i - 1] == '-') {
+                    match->body_len = i - 3;
+                } else {
+                    match->body_len = i - 2;
+                }
+
+                return true;
+            }
         }
     }
 

data/ext/liquid_c/tokenizer.c

@@ -290,9 +290,9 @@ void liquid_define_tokenizer(void)
     rb_define_method(cLiquidTokenizer, "line_number", tokenizer_line_number_method, 0);
     rb_define_method(cLiquidTokenizer, "for_liquid_tag", tokenizer_for_liquid_tag_method, 0);
     rb_define_method(cLiquidTokenizer, "bug_compatible_whitespace_trimming!", tokenizer_bug_compatible_whitespace_trimming, 0);
+    rb_define_method(cLiquidTokenizer, "shift", tokenizer_shift_method, 0);
 
     // For testing the internal token representation.
-    rb_define_private_method(cLiquidTokenizer, "shift", tokenizer_shift_method, 0);
     rb_define_private_method(cLiquidTokenizer, "shift_trimmed", tokenizer_shift_trimmed_method, 0);
 }
 

data/ext/liquid_c/variable.c

@@ -2,7 +2,7 @@
 #include "variable.h"
 #include "parser.h"
 #include "expression.h"
-#include "vm.h"
+#include "liquid_vm.h"
 
 #include <stdio.h>
 

data/ext/liquid_c/vm_assembler.c

@@ -1,7 +1,7 @@
 #include "liquid.h"
 #include "vm_assembler.h"
 #include "expression.h"
-#include "vm.h"
+#include "liquid_vm.h"
 
 #define ARRAY_LENGTH(array) (sizeof(array) / sizeof(array[0]))
 

data/ext/liquid_c/vm_assembler_pool.c

@@ -3,8 +3,10 @@
 
 static VALUE cLiquidCVMAssemblerPool;
 
-void vm_assembler_pool_gc_mark(vm_assembler_pool_t *pool)
+static void vm_assembler_pool_mark(void *ptr)
 {
+    vm_assembler_pool_t *pool = ptr;
+
     rb_gc_mark(pool->self);
 }
 
@@ -39,7 +41,7 @@ static size_t vm_assembler_pool_memsize(const void *ptr)
 
 const rb_data_type_t vm_assembler_pool_data_type = {
     "liquid_vm_assembler_pool",
-    { NULL, vm_assembler_pool_free, vm_assembler_pool_memsize, },
+    { vm_assembler_pool_mark, vm_assembler_pool_free, vm_assembler_pool_memsize, },
     NULL, NULL, RUBY_TYPED_FREE_IMMEDIATELY
 };
 

data/ext/liquid_c/vm_assembler_pool.h

@@ -18,7 +18,6 @@ extern const rb_data_type_t vm_assembler_pool_data_type;
 #define VMAssemblerPool_Get_Struct(obj, sval) TypedData_Get_Struct(obj, vm_assembler_pool_t, &vm_assembler_pool_data_type, sval)
 
 void liquid_define_vm_assembler_pool(void);
-void vm_assembler_pool_gc_mark(vm_assembler_pool_t *pool);
 VALUE vm_assembler_pool_new(void);
 vm_assembler_t *vm_assembler_pool_alloc_assembler(vm_assembler_pool_t *pool);
 void vm_assembler_pool_free_assembler(vm_assembler_t *assembler);

data/lib/liquid/c/version.rb

@@ -2,6 +2,6 @@
 
 module Liquid
   module C
-    VERSION = "4.1.0"
+    VERSION = "4.2.0"
   end
 end

data/lib/liquid/c.rb

@@ -86,7 +86,7 @@ Liquid::ParseContext.class_eval do
 
   def parse_expression(markup)
     if liquid_c_nodes_disabled?
-      Liquid::Expression.ruby_parse(markup)
+      Liquid::Expression.parse(markup)
     else
       Liquid::C::Expression.lax_parse(markup)
     end
@@ -207,22 +207,17 @@ Liquid::C::Expression.class_eval do
     def lax_parse(markup)
       strict_parse(markup)
     rescue Liquid::SyntaxError
-      Liquid::Expression.ruby_parse(markup)
+      Liquid::Expression.parse(markup)
     end
-  end
-end
-
-Liquid::Expression.class_eval do
-  class << self
-    alias_method :ruby_parse, :parse
 
-    def c_parse(markup)
-      Liquid::C::Expression.lax_parse(markup)
-    end
+    # Default to strict parsing, since Liquid::C::Expression.parse should only really
+    # be used with constant expressions.  Otherwise, prefer parse_context.parse_expression.
+    alias_method :parse, :strict_parse
   end
 end
 
 Liquid::Context.class_eval do
+  alias_method :ruby_parse_evaluate, :[]
   alias_method :ruby_evaluate, :evaluate
   alias_method :ruby_find_variable, :find_variable
   alias_method :ruby_strict_variables=, :strict_variables=
@@ -232,6 +227,10 @@ Liquid::Context.class_eval do
   def c_find_variable_kwarg(key, raise_on_not_found: true)
     c_find_variable(key, raise_on_not_found)
   end
+
+  def c_parse_evaluate(expression)
+    c_evaluate(Liquid::C::Expression.lax_parse(expression))
+  end
 end
 
 Liquid::ResourceLimits.class_eval do
@@ -256,15 +255,15 @@ module Liquid
       def enabled=(value)
         @enabled = value
         if value
+          Liquid::Context.send(:alias_method, :[], :c_parse_evaluate)
           Liquid::Context.send(:alias_method, :evaluate, :c_evaluate)
           Liquid::Context.send(:alias_method, :find_variable, :c_find_variable_kwarg)
           Liquid::Context.send(:alias_method, :strict_variables=, :c_strict_variables=)
-          Liquid::Expression.singleton_class.send(:alias_method, :parse, :c_parse)
         else
+          Liquid::Context.send(:alias_method, :[], :ruby_parse_evaluate)
           Liquid::Context.send(:alias_method, :evaluate, :ruby_evaluate)
           Liquid::Context.send(:alias_method, :find_variable, :ruby_find_variable)
           Liquid::Context.send(:alias_method, :strict_variables=, :ruby_strict_variables=)
-          Liquid::Expression.singleton_class.send(:alias_method, :parse, :ruby_parse)
         end
       end
     end

data/test/test_helper.rb

@@ -1,14 +1,21 @@
 # frozen_string_literal: true
 
-at_exit { GC.start }
-
 require "minitest/autorun"
 require "liquid/c"
 
 if GC.respond_to?(:verify_compaction_references)
   # This method was added in Ruby 3.0.0. Calling it this way asks the GC to
   # move objects around, helping to find object movement bugs.
-  GC.verify_compaction_references(double_heap: true, toward: :empty)
+  begin
+    GC.verify_compaction_references(double_heap: true, toward: :empty)
+  rescue NotImplementedError
+    puts "W: GC compaction not suppported by platform"
+  end
+end
+
+# Enable auto-compaction in the GC if supported.
+if GC.respond_to?(:auto_compact=)
+  GC.auto_compact = true
 end
 
 GC.stress = true if ENV["GC_STRESS"]

data/test/unit/block_test.rb

@@ -2,7 +2,7 @@
 
 require "test_helper"
 
-class BlockTest < MiniTest::Test
+class BlockTest < Minitest::Test
   def test_no_allocation_of_trimmed_strings
     template = Liquid::Template.parse("{{ a -}}     {{- b }}")
     assert_equal(2, template.root.nodelist.size)
@@ -127,4 +127,11 @@ class BlockTest < MiniTest::Test
       Liquid::Template.file_system = old_file_system
     end
   end
+
+  def test_assign_filter_argument_exception
+    source = "{% assign v = 'IN' | truncate: 123, liquid_error %}{{ v | default: 'err swallowed' }}"
+    template = Liquid::Template.parse(source)
+    output = template.render({ "liquid_error" => -> { raise Liquid::Error, "var lookup error" } })
+    assert_equal("err swallowed", output)
+  end
 end

data/test/unit/context_test.rb

@@ -29,6 +29,8 @@ class ContextTest < Minitest::Test
   end
 
   def test_evaluating_a_variable_entirely_within_c
+    skip("TracePoint :call not yet supported") if RUBY_ENGINE == "truffleruby"
+
     context = Liquid::Context.new({ "var" => 42 })
     lookup = Liquid::C::Expression.strict_parse("var")
     context.evaluate(lookup) # memoize vm_internal_new calls

data/test/unit/expression_test.rb

@@ -2,7 +2,7 @@
 
 require "test_helper"
 
-class ExpressionTest < MiniTest::Test
+class ExpressionTest < Minitest::Test
   def test_constant_literals
     assert_equal(true, Liquid::C::Expression.strict_parse("true"))
     assert_equal(false, Liquid::C::Expression.strict_parse("false"))
@@ -159,7 +159,7 @@ class ExpressionTest < MiniTest::Test
   end
 
   def test_disable_c_nodes
-    context = Liquid::Context.new({ "x" => 123 })
+    context = Liquid::Context.new({ "x" => 123, "y" => { 123 => 42 } })
 
     expr = Liquid::ParseContext.new.parse_expression("x")
     assert_instance_of(Liquid::C::Expression, expr)
@@ -168,6 +168,11 @@ class ExpressionTest < MiniTest::Test
     expr = Liquid::ParseContext.new(disable_liquid_c_nodes: true).parse_expression("x")
     assert_instance_of(Liquid::VariableLookup, expr)
     assert_equal(123, context.evaluate(expr))
+
+    expr = Liquid::ParseContext.new(disable_liquid_c_nodes: true).parse_expression("y[x]")
+    assert_instance_of(Liquid::VariableLookup, expr)
+    assert_instance_of(Liquid::VariableLookup, expr.lookups.first)
+    assert_equal(42, context.evaluate(expr))
   end
 
   private

data/test/unit/raw_test.rb

@@ -13,7 +13,81 @@ class RawTest < Minitest::Test
   Liquid::Template.register_tag("raw_wrapper", RawWrapper)
 
   def test_derived_class
-    output = Liquid::Template.parse("{% raw_wrapper %}body{% endraw_wrapper %}").render!
-    assert_equal("<body>", output)
+    [
+      "{% raw_wrapper %}body{% endraw_wrapper %}",
+      "{% raw_wrapper %}body{%endraw_wrapper%}",
+      "{% raw_wrapper %}body{%- endraw_wrapper -%}",
+      "{% raw_wrapper %}body{%- endraw_wrapper %}",
+      "{% raw_wrapper %}body{% endraw_wrapper -%}",
+    ].each do |template|
+      output = Liquid::Template.parse(template).render!
+
+      assert_equal(
+        "<body>",
+        output,
+        "Template: #{template}"
+      )
+    end
+  end
+
+  def test_allows_extra_string_after_tag_delimiter
+    output = Liquid::Template.parse("{% raw %}message{% endraw this_is_allowed %}").render
+    assert_equal("message", output)
+
+    output = Liquid::Template.parse("{% raw %}message{%   endraw r%}").render
+    assert_equal("message", output)
+  end
+
+  def test_ignores_incomplete_tag_delimter
+    output = Liquid::Template.parse("{% raw %}{% endraw {% endraw %}").render
+    assert_equal("{% endraw ", output)
+
+    output = Liquid::Template.parse("{% raw %}{%endraw{% endraw %}").render
+    assert_equal("{%endraw", output)
+
+    output = Liquid::Template.parse("{% raw %}{%- endraw {% endraw %}").render
+    assert_equal("{%- endraw ", output)
+  end
+
+  def test_does_not_allow_nbsp_in_tag_delimiter
+    # these are valid
+    Liquid::Template.parse("{% raw %}body{%endraw%}")
+    Liquid::Template.parse("{% raw %}body{% endraw-%}")
+    Liquid::Template.parse("{% raw %}body{% endraw -%}")
+    Liquid::Template.parse("{% raw %}body{%-endraw %}")
+    Liquid::Template.parse("{% raw %}body{%- endraw %}")
+    Liquid::Template.parse("{% raw %}body{%-endraw-%}")
+    Liquid::Template.parse("{% raw %}body{%- endraw -%}")
+    Liquid::Template.parse("{% raw %}body{% endraw\u00A0%}")
+    Liquid::Template.parse("{% raw %}body{% endraw \u00A0%}")
+    Liquid::Template.parse("{% raw %}body{% endraw\u00A0 %}")
+    Liquid::Template.parse("{% raw %}body{% endraw \u00A0 %}")
+    Liquid::Template.parse("{% raw %}body{% endraw \u00A0 endraw %}")
+    Liquid::Template.parse("{% raw %}body{% endraw\u00A0endraw %}")
+
+    [
+      "{%\u00A0endraw%}",
+      "{%\u00A0 endraw%}",
+      "{% \u00A0endraw%}",
+      "{% \u00A0 endraw%}",
+      "{%\u00A0endraw\u00A0%}",
+      "{% - endraw %}",
+      "{% endnot endraw %}",
+    ].each do |bad_delimiter|
+      exception = assert_raises(
+        Liquid::SyntaxError,
+        "#{bad_delimiter.inspect} did not raise Liquid::SyntaxError"
+      ) do
+        Liquid::Template.parse(
+          "{% raw %}body#{bad_delimiter}"
+        )
+      end
+
+      assert_equal(
+        exception.message,
+        "Liquid syntax error: 'raw' tag was never closed",
+        "#{bad_delimiter.inspect} raised the wrong exception message",
+      )
+    end
   end
 end

data/test/unit/tokenizer_test.rb

@@ -6,7 +6,7 @@ require "test_helper"
 class TokenizerTest < Minitest::Test
   def test_tokenizer_nil
     tokenizer = new_tokenizer(nil)
-    assert_nil(tokenizer.send(:shift))
+    assert_nil(tokenizer.shift)
   end
 
   def test_tokenize_strings
@@ -60,10 +60,10 @@ class TokenizerTest < Minitest::Test
   def test_utf8_compatible_source
     source = String.new("ascii", encoding: Encoding::ASCII)
     tokenizer = new_tokenizer(source)
-    output = tokenizer.send(:shift)
+    output = tokenizer.shift
     assert_equal(Encoding::UTF_8, output.encoding)
     assert_equal(source, output)
-    assert_nil(tokenizer.send(:shift))
+    assert_nil(tokenizer.shift)
   end
 
   def test_non_utf8_compatible_source
@@ -105,7 +105,7 @@ class TokenizerTest < Minitest::Test
   def tokenize(source, for_liquid_tag: false, trimmed: false)
     tokenizer = Liquid::C::Tokenizer.new(source, 1, for_liquid_tag)
     tokens = []
-    while (t = trimmed ? tokenizer.send(:shift_trimmed) : tokenizer.send(:shift))
+    while (t = trimmed ? tokenizer.send(:shift_trimmed) : tokenizer.shift)
       tokens << t
     end
     tokens

data/test/unit/variable_test.rb

@@ -254,6 +254,73 @@ class VariableTest < Minitest::Test
     assert_equal("2", output)
   end
 
+  def test_encoding_error_message_with_multi_byte_characters
+    # 2 byte character
+    exc = assert_raises(Liquid::SyntaxError) do
+      variable_strict_parse("\u00A0")
+    end
+    assert_equal(
+      "Liquid syntax error: Unexpected character \u00A0 in \"{{\u00a0}}\"",
+      exc.message
+    )
+
+    # 3 byte character
+    exc = assert_raises(Liquid::SyntaxError) do
+      variable_strict_parse("\u3042")
+    end
+    assert_equal(
+      "Liquid syntax error: Unexpected character \u3042 in \"{{\u3042}}\"",
+      exc.message
+    )
+
+    # 4 byte character
+    exc = assert_raises(Liquid::SyntaxError) do
+      variable_strict_parse("\u{1F600}")
+    end
+    assert_equal(
+      "Liquid syntax error: Unexpected character \u{1F600} in \"{{\u{1F600}}}\"",
+      exc.message
+    )
+  end
+
+  def test_invalid_utf8_sequence
+    # 2 byte character with 1 byte missing
+    exc = assert_raises(ArgumentError) do
+      variable_strict_parse("\xC0")
+    end
+    assert_equal("invalid byte sequence in UTF-8", exc.message)
+
+    # 3 byte character with 1 byte missing
+    exc = assert_raises(ArgumentError) do
+      variable_strict_parse("\xE0\x01")
+    end
+    assert_equal("invalid byte sequence in UTF-8", exc.message)
+
+    # 3 byte character with 2 byte missing
+    exc = assert_raises(ArgumentError) do
+      variable_strict_parse("\xE0")
+    end
+    assert_equal("invalid byte sequence in UTF-8", exc.message)
+
+    # 4 byte character with 1 byte missing
+    exc = assert_raises(ArgumentError) do
+      variable_strict_parse("\xF0\x01\x01")
+    end
+    assert_equal("invalid byte sequence in UTF-8", exc.message)
+
+    # 4 byte character with 2 byte missing
+    exc = assert_raises(ArgumentError) do
+      variable_strict_parse("\xF0\x01")
+    end
+    assert_equal("invalid byte sequence in UTF-8", exc.message)
+
+    # 4 byte character with 3 byte missing
+    exc = assert_raises(ArgumentError) do
+      variable_strict_parse("\xF0")
+    end
+    assert_equal("invalid byte sequence in UTF-8", exc.message)
+  end
+
   private
 
   def variable_strict_parse(markup)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: liquid-c
 version: !ruby/object:Gem::Version
-  version: 4.1.0
+  version: 4.2.0
 platform: ruby
 authors:
 - Justin Li
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-02-08 00:00:00.000000000 Z
+date: 2024-01-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: liquid
@@ -103,6 +103,7 @@ extensions:
 - ext/liquid_c/extconf.rb
 extra_rdoc_files: []
 files:
+- ".github/workflows/cla.yml"
 - ".github/workflows/liquid.yml"
 - ".gitignore"
 - ".rubocop.yml"
@@ -126,6 +127,8 @@ files:
 - ext/liquid_c/lexer.h
 - ext/liquid_c/liquid.c
 - ext/liquid_c/liquid.h
+- ext/liquid_c/liquid_vm.c
+- ext/liquid_c/liquid_vm.h
 - ext/liquid_c/parse_context.c
 - ext/liquid_c/parse_context.h
 - ext/liquid_c/parser.c
@@ -143,8 +146,6 @@ files:
 - ext/liquid_c/variable.h
 - ext/liquid_c/variable_lookup.c
 - ext/liquid_c/variable_lookup.h
-- ext/liquid_c/vm.c
-- ext/liquid_c/vm.h
 - ext/liquid_c/vm_assembler.c
 - ext/liquid_c/vm_assembler.h
 - ext/liquid_c/vm_assembler_pool.c
@@ -191,7 +192,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.20
+rubygems_version: 3.5.4
 signing_key: 
 specification_version: 4
 summary: Liquid performance extension in C