liquid-c 4.0.1 → 4.2.0

data/ext/liquid_c/resource_limits.c

@@ -0,0 +1,279 @@
+#include "liquid.h"
+#include "resource_limits.h"
+
+VALUE cLiquidResourceLimits;
+
+static void resource_limits_free(void *ptr)
+{
+    resource_limits_t *resource_limits = ptr;
+    xfree(resource_limits);
+}
+
+static size_t resource_limits_memsize(const void *ptr)
+{
+    return sizeof(resource_limits_t);
+}
+
+const rb_data_type_t resource_limits_data_type = {
+    "liquid_resource_limits",
+    { NULL, resource_limits_free, resource_limits_memsize },
+    NULL, NULL, RUBY_TYPED_FREE_IMMEDIATELY
+};
+
+static void resource_limits_reset(resource_limits_t *resource_limit)
+{
+    resource_limit->reached_limit = true;
+    resource_limit->last_capture_length = -1;
+    resource_limit->render_score = 0;
+    resource_limit->assign_score = 0;
+}
+
+static VALUE resource_limits_allocate(VALUE klass)
+{
+    resource_limits_t *resource_limits;
+
+    VALUE obj = TypedData_Make_Struct(klass, resource_limits_t, &resource_limits_data_type, resource_limits);
+
+    resource_limits_reset(resource_limits);
+
+    return obj;
+}
+
+static VALUE resource_limits_render_length_limit_method(VALUE self)
+{
+    resource_limits_t *resource_limits;
+    ResourceLimits_Get_Struct(self, resource_limits);
+
+    return LONG2NUM(resource_limits->render_length_limit);
+}
+
+static VALUE resource_limits_set_render_length_limit_method(VALUE self, VALUE render_length_limit)
+{
+    resource_limits_t *resource_limits;
+    ResourceLimits_Get_Struct(self, resource_limits);
+
+     if (render_length_limit == Qnil) {
+        resource_limits->render_length_limit = LONG_MAX;
+    } else {
+        resource_limits->render_length_limit = NUM2LONG(render_length_limit);
+    }
+
+    return Qnil;
+}
+
+static VALUE resource_limits_render_score_limit_method(VALUE self)
+{
+    resource_limits_t *resource_limits;
+    ResourceLimits_Get_Struct(self, resource_limits);
+
+    return LONG2NUM(resource_limits->render_score_limit);
+}
+
+static VALUE resource_limits_set_render_score_limit_method(VALUE self, VALUE render_score_limit)
+{
+    resource_limits_t *resource_limits;
+    ResourceLimits_Get_Struct(self, resource_limits);
+
+    if (render_score_limit == Qnil) {
+        resource_limits->render_score_limit = LONG_MAX;
+    } else {
+        resource_limits->render_score_limit = NUM2LONG(render_score_limit);
+    }
+
+    return Qnil;
+}
+
+static VALUE resource_limits_assign_score_limit_method(VALUE self)
+{
+    resource_limits_t *resource_limits;
+    ResourceLimits_Get_Struct(self, resource_limits);
+
+    return LONG2NUM(resource_limits->assign_score_limit);
+}
+
+static VALUE resource_limits_set_assign_score_limit_method(VALUE self, VALUE assign_score_limit)
+{
+    resource_limits_t *resource_limits;
+    ResourceLimits_Get_Struct(self, resource_limits);
+
+    if (assign_score_limit == Qnil) {
+        resource_limits->assign_score_limit = LONG_MAX;
+    } else {
+        resource_limits->assign_score_limit = NUM2LONG(assign_score_limit);
+    }
+
+    return Qnil;
+}
+
+static VALUE resource_limits_render_score_method(VALUE self)
+{
+    resource_limits_t *resource_limits;
+    ResourceLimits_Get_Struct(self, resource_limits);
+
+    return LONG2NUM(resource_limits->render_score);
+}
+
+static VALUE resource_limits_assign_score_method(VALUE self)
+{
+    resource_limits_t *resource_limits;
+    ResourceLimits_Get_Struct(self, resource_limits);
+
+    return LONG2NUM(resource_limits->assign_score);
+}
+
+static VALUE resource_limits_initialize_method(VALUE self, VALUE render_length_limit,
+                                               VALUE render_score_limit, VALUE assign_score_limit)
+{
+    resource_limits_set_render_length_limit_method(self, render_length_limit);
+    resource_limits_set_render_score_limit_method(self, render_score_limit);
+    resource_limits_set_assign_score_limit_method(self, assign_score_limit);
+
+    return Qnil;
+}
+
+__attribute__((noreturn))
+void resource_limits_raise_limits_reached(resource_limits_t *resource_limit)
+{
+    resource_limit->reached_limit = true;
+    rb_raise(cMemoryError, "Memory limits exceeded");
+}
+
+void resource_limits_increment_render_score(resource_limits_t *resource_limits, long amount)
+{
+    resource_limits->render_score = resource_limits->render_score + amount;
+
+    if (resource_limits->render_score > resource_limits->render_score_limit) {
+        resource_limits_raise_limits_reached(resource_limits);
+    }
+}
+
+static VALUE resource_limits_increment_render_score_method(VALUE self, VALUE amount)
+{
+    resource_limits_t *resource_limits;
+    ResourceLimits_Get_Struct(self, resource_limits);
+
+    resource_limits_increment_render_score(resource_limits, NUM2LONG(amount));
+
+    return Qnil;
+}
+
+static void resource_limits_increment_assign_score(resource_limits_t *resource_limits, long amount)
+{
+    resource_limits->assign_score = resource_limits->assign_score + amount;
+
+    if (resource_limits->assign_score > resource_limits->assign_score_limit) {
+        resource_limits_raise_limits_reached(resource_limits);
+    }
+}
+
+static VALUE resource_limits_increment_assign_score_method(VALUE self, VALUE amount)
+{
+    resource_limits_t *resource_limits;
+    ResourceLimits_Get_Struct(self, resource_limits);
+
+    resource_limits_increment_assign_score(resource_limits, NUM2LONG(amount));
+
+    return Qnil;
+}
+
+void resource_limits_increment_write_score(resource_limits_t *resource_limits, VALUE output)
+{
+    long captured = RSTRING_LEN(output);
+
+    if (resource_limits->last_capture_length >= 0) {
+        long increment = captured - resource_limits->last_capture_length;
+        resource_limits->last_capture_length = captured;
+        resource_limits_increment_assign_score(resource_limits, increment);
+    } else if (captured > resource_limits->render_length_limit) {
+        resource_limits_raise_limits_reached(resource_limits);
+    }
+}
+
+static VALUE resource_limits_increment_write_score_method(VALUE self, VALUE output)
+{
+    Check_Type(output, T_STRING);
+
+    resource_limits_t *resource_limits;
+    ResourceLimits_Get_Struct(self, resource_limits);
+
+    resource_limits_increment_write_score(resource_limits, output);
+
+    return Qnil;
+}
+
+static VALUE resource_limits_raise_limits_reached_method(VALUE self)
+{
+    resource_limits_t *resource_limits;
+    ResourceLimits_Get_Struct(self, resource_limits);
+
+    resource_limits_raise_limits_reached(resource_limits);
+}
+
+static VALUE resource_limits_reached_method(VALUE self)
+{
+    resource_limits_t *resource_limits;
+    ResourceLimits_Get_Struct(self, resource_limits);
+
+    return resource_limits->reached_limit ? Qtrue : Qfalse;
+}
+
+struct capture_ensure_t {
+    resource_limits_t *resource_limits;
+    long old_capture_length;
+};
+
+static VALUE capture_ensure(VALUE data)
+{
+    struct capture_ensure_t *ensure_data = (struct capture_ensure_t *)data;
+    ensure_data->resource_limits->last_capture_length = ensure_data->old_capture_length;
+
+    return Qnil;
+}
+
+static VALUE resource_limits_with_capture_method(VALUE self)
+{
+    resource_limits_t *resource_limits;
+    ResourceLimits_Get_Struct(self, resource_limits);
+
+    struct capture_ensure_t ensure_data = {
+        .resource_limits = resource_limits,
+        .old_capture_length = resource_limits->last_capture_length
+    };
+
+    resource_limits->last_capture_length = 0;
+
+    return rb_ensure(rb_yield, Qundef, capture_ensure, (VALUE)&ensure_data);
+}
+
+
+static VALUE resource_limits_reset_method(VALUE self)
+{
+    resource_limits_t *resource_limits;
+    ResourceLimits_Get_Struct(self, resource_limits);
+    resource_limits_reset(resource_limits);
+    return Qnil;
+}
+
+void liquid_define_resource_limits(void)
+{
+    cLiquidResourceLimits = rb_define_class_under(mLiquidC, "ResourceLimits", rb_cObject);
+    rb_global_variable(&cLiquidResourceLimits);
+
+    rb_define_alloc_func(cLiquidResourceLimits, resource_limits_allocate);
+    rb_define_method(cLiquidResourceLimits, "initialize", resource_limits_initialize_method, 3);
+    rb_define_method(cLiquidResourceLimits, "render_length_limit", resource_limits_render_length_limit_method, 0);
+    rb_define_method(cLiquidResourceLimits, "render_length_limit=", resource_limits_set_render_length_limit_method, 1);
+    rb_define_method(cLiquidResourceLimits, "render_score_limit", resource_limits_render_score_limit_method, 0);
+    rb_define_method(cLiquidResourceLimits, "render_score_limit=", resource_limits_set_render_score_limit_method, 1);
+    rb_define_method(cLiquidResourceLimits, "assign_score_limit", resource_limits_assign_score_limit_method, 0);
+    rb_define_method(cLiquidResourceLimits, "assign_score_limit=", resource_limits_set_assign_score_limit_method, 1);
+    rb_define_method(cLiquidResourceLimits, "render_score", resource_limits_render_score_method, 0);
+    rb_define_method(cLiquidResourceLimits, "assign_score", resource_limits_assign_score_method, 0);
+    rb_define_method(cLiquidResourceLimits, "increment_render_score", resource_limits_increment_render_score_method, 1);
+    rb_define_method(cLiquidResourceLimits, "increment_assign_score", resource_limits_increment_assign_score_method, 1);
+    rb_define_method(cLiquidResourceLimits, "increment_write_score", resource_limits_increment_write_score_method, 1);
+    rb_define_method(cLiquidResourceLimits, "raise_limits_reached", resource_limits_raise_limits_reached_method, 0);
+    rb_define_method(cLiquidResourceLimits, "reached?", resource_limits_reached_method, 0);
+    rb_define_method(cLiquidResourceLimits, "reset", resource_limits_reset_method, 0);
+    rb_define_method(cLiquidResourceLimits, "with_capture", resource_limits_with_capture_method, 0);
+}

data/ext/liquid_c/resource_limits.h

@@ -0,0 +1,23 @@
+#ifndef LIQUID_RESOURCE_LIMITS
+#define LIQUID_RESOURCE_LIMITS
+
+typedef struct resource_limits {
+    long render_length_limit;
+    long render_score_limit;
+    long assign_score_limit;
+    bool reached_limit;
+    long last_capture_length;
+    long render_score;
+    long assign_score;
+} resource_limits_t;
+
+extern VALUE cLiquidResourceLimits;
+extern const rb_data_type_t resource_limits_data_type;
+#define ResourceLimits_Get_Struct(obj, sval) TypedData_Get_Struct(obj, resource_limits_t, &resource_limits_data_type, sval)
+
+void liquid_define_resource_limits(void);
+void resource_limits_raise_limits_reached(resource_limits_t *resource_limit);
+void resource_limits_increment_render_score(resource_limits_t *resource_limits, long amount);
+void resource_limits_increment_write_score(resource_limits_t *resource_limits, VALUE output);
+
+#endif

data/ext/liquid_c/stringutil.h

@@ -0,0 +1,44 @@
+#if !defined(LIQUID_UTIL_H)
+#define LIQUID_UTIL_H
+
+inline static const char *read_while(const char *start, const char *end, int (func)(int))
+{
+    while (start < end && func((unsigned char) *start)) start++;
+    return start;
+}
+
+inline static const char *read_while_reverse(const char *start, const char *end, int (func)(int))
+{
+    end--;
+    while (start <= end && func((unsigned char) *end)) end--;
+    end++;
+    return end;
+}
+
+inline static int count_newlines(const char *start, const char *end)
+{
+    int count = 0;
+    while (start < end) {
+        if (*start == '\n') count++;
+        start++;
+    }
+    return count;
+}
+
+inline static int is_non_newline_space(int c)
+{
+    return rb_isspace(c) && c != '\n';
+}
+
+inline static int not_newline(int c)
+{
+    return c != '\n';
+}
+
+inline static bool is_word_char(char c)
+{
+    return ISALNUM(c) || c == '_';
+}
+
+#endif
+

data/ext/liquid_c/tokenizer.c

@@ -1,5 +1,7 @@
+#include <assert.h>
 #include "liquid.h"
 #include "tokenizer.h"
+#include "stringutil.h"
 
 VALUE cLiquidTokenizer;
 
@@ -35,40 +37,89 @@ static VALUE tokenizer_allocate(VALUE klass)
 
     obj = TypedData_Make_Struct(klass, tokenizer_t, &tokenizer_data_type, tokenizer);
     tokenizer->source = Qnil;
+    tokenizer->bug_compatible_whitespace_trimming = false;
+    tokenizer->raw_tag_body = NULL;
+    tokenizer->raw_tag_body_len = 0;
     return obj;
 }
 
-static VALUE tokenizer_initialize_method(VALUE self, VALUE source, VALUE line_numbers)
+static VALUE tokenizer_initialize_method(VALUE self, VALUE source, VALUE start_line_number, VALUE for_liquid_tag)
 {
     tokenizer_t *tokenizer;
 
     Check_Type(source, T_STRING);
+    check_utf8_encoding(source, "source");
+
+#define MAX_SOURCE_CODE_BYTES ((1 << 24) - 1)
+    if (RSTRING_LEN(source) > MAX_SOURCE_CODE_BYTES) {
+        rb_enc_raise(utf8_encoding, rb_eArgError, "Source too large, max %d bytes", MAX_SOURCE_CODE_BYTES);
+    }
+#undef MAX_SOURCE_CODE_BYTES
+
     Tokenizer_Get_Struct(self, tokenizer);
     source = rb_str_dup_frozen(source);
     tokenizer->source = source;
     tokenizer->cursor = RSTRING_PTR(source);
-    tokenizer->length = RSTRING_LEN(source);
-    tokenizer->lstrip_flag = 0;
+    tokenizer->cursor_end = tokenizer->cursor + RSTRING_LEN(source);
+    tokenizer->lstrip_flag = false;
     // tokenizer->line_number keeps track of the current line number or it is 0
     // to indicate that line numbers aren't being calculated
-    tokenizer->line_number = RTEST(line_numbers) ? 1 : 0;
+    tokenizer->line_number = FIX2UINT(start_line_number);
+    tokenizer->for_liquid_tag = RTEST(for_liquid_tag);
     return Qnil;
 }
 
-void tokenizer_next(tokenizer_t *tokenizer, token_t *token)
+// Internal function to setup an existing tokenizer from C for a liquid tag.
+// This overwrites the passed in tokenizer, so a copy of the struct should
+// be used to reset the tokenizer after parsing the liquid tag.
+void tokenizer_setup_for_liquid_tag(tokenizer_t *tokenizer, const char *cursor, const char *cursor_end, int line_number)
 {
-    if (tokenizer->length <= 0) {
-        memset(token, 0, sizeof(*token));
-        return;
+    tokenizer->cursor = cursor;
+    tokenizer->cursor_end = cursor_end;
+    tokenizer->lstrip_flag = false;
+    tokenizer->line_number = line_number;
+    tokenizer->for_liquid_tag = true;
+}
+
+// Tokenizes contents of {% liquid ... %}
+static void tokenizer_next_for_liquid_tag(tokenizer_t *tokenizer, token_t *token)
+{
+    const char *end = tokenizer->cursor_end;
+    const char *start = tokenizer->cursor;
+    const char *start_trimmed = read_while(start, end, is_non_newline_space);
+
+    token->str_full = start;
+    token->str_trimmed = start_trimmed;
+
+    const char *end_full = read_while(start_trimmed, end, not_newline);
+    if (end_full < end) {
+        tokenizer->cursor = end_full + 1;
+        if (tokenizer->line_number)
+            tokenizer->line_number++;
+    } else {
+        tokenizer->cursor = end_full;
     }
 
+    const char *end_trimmed = read_while_reverse(start_trimmed, end_full, rb_isspace);
+
+    token->len_trimmed = end_trimmed - start_trimmed;
+    token->len_full = end_full - token->str_full;
+
+    if (token->len_trimmed == 0) {
+        token->type = TOKEN_BLANK_LIQUID_TAG_LINE;
+    } else {
+        token->type = TOKEN_TAG;
+    }
+}
+
+// Tokenizes contents of a full Liquid template
+static void tokenizer_next_for_template(tokenizer_t *tokenizer, token_t *token)
+{
     const char *cursor = tokenizer->cursor;
-    const char *last = cursor + tokenizer->length - 1;
+    const char *last = tokenizer->cursor_end - 1;
 
-    token->str = cursor;
+    token->str_full = cursor;
     token->type = TOKEN_RAW;
-    token->lstrip = 0;
-    token->rstrip = 0;
 
     while (cursor < last) {
         if (*cursor++ != '{')
@@ -78,17 +129,17 @@ void tokenizer_next(tokenizer_t *tokenizer, token_t *token)
         if (c != '%' && c != '{')
             continue;
         if (cursor <= last && *cursor == '-') {
-          cursor++;
-          token->rstrip = 1;
+            cursor++;
+            token->rstrip = 1;
         }
         if (cursor - tokenizer->cursor > (ptrdiff_t)(2 + token->rstrip)) {
             token->type = TOKEN_RAW;
             cursor -= 2 + token->rstrip;
             token->lstrip = tokenizer->lstrip_flag;
-            tokenizer->lstrip_flag = 0;
+            tokenizer->lstrip_flag = false;
             goto found;
         }
-        tokenizer->lstrip_flag = 0;
+        tokenizer->lstrip_flag = false;
         token->type = TOKEN_INVALID;
         token->lstrip = token->rstrip;
         token->rstrip = 0;
@@ -103,12 +154,12 @@ void tokenizer_next(tokenizer_t *tokenizer, token_t *token)
                     continue;
                 token->type = TOKEN_TAG;
                 if(cursor[-3] == '-')
-                    token->rstrip = tokenizer->lstrip_flag = 1;
+                    token->rstrip = tokenizer->lstrip_flag = true;
                 goto found;
             }
             // unterminated tag
             cursor = tokenizer->cursor + 2;
-            tokenizer->lstrip_flag = 0;
+            tokenizer->lstrip_flag = false;
             goto found;
         } else {
             while (cursor < last) {
@@ -121,31 +172,51 @@ void tokenizer_next(tokenizer_t *tokenizer, token_t *token)
                 }
                 token->type = TOKEN_VARIABLE;
                 if(cursor[-3] == '-')
-                    token->rstrip = tokenizer->lstrip_flag = 1;
+                    token->rstrip = tokenizer->lstrip_flag = true;
                 goto found;
             }
             // unterminated variable
             cursor = tokenizer->cursor + 2;
-            tokenizer->lstrip_flag = 0;
+            tokenizer->lstrip_flag = false;
             goto found;
         }
     }
     cursor = last + 1;
     token->lstrip = tokenizer->lstrip_flag;
-    tokenizer->lstrip_flag = 0;
+    tokenizer->lstrip_flag = false;
 found:
-    token->length = cursor - tokenizer->cursor;
-    tokenizer->cursor += token->length;
-    tokenizer->length -= token->length;
+    token->len_full = cursor - token->str_full;
+
+    token->str_trimmed = token->str_full;
+    token->len_trimmed = token->len_full;
+
+    if (token->type == TOKEN_VARIABLE || token->type == TOKEN_TAG) {
+        token->str_trimmed += 2 + token->lstrip;
+        token->len_trimmed -= 2 + token->lstrip + 2;
+        if (token->rstrip && token->len_trimmed)
+            token->len_trimmed--;
+    }
+
+    assert(token->len_trimmed >= 0);
+
+    tokenizer->cursor += token->len_full;
 
     if (tokenizer->line_number) {
-        const char *cursor = token->str;
-        const char *end = token->str + token->length;
-        while (cursor < end) {
-            if (*cursor == '\n')
-                tokenizer->line_number++;
-            cursor++;
-        }
+        tokenizer->line_number += count_newlines(token->str_full, token->str_full + token->len_full);
+    }
+}
+
+void tokenizer_next(tokenizer_t *tokenizer, token_t *token)
+{
+    memset(token, 0, sizeof(*token));
+
+    if (tokenizer->cursor >= tokenizer->cursor_end) {
+        return;
+    }
+    if (tokenizer->for_liquid_tag) {
+        tokenizer_next_for_liquid_tag(tokenizer, token);
+    } else {
+        tokenizer_next_for_template(tokenizer, token);
     }
 }
 
@@ -159,7 +230,25 @@ static VALUE tokenizer_shift_method(VALUE self)
     if (!token.type)
         return Qnil;
 
-    return rb_enc_str_new(token.str, token.length, utf8_encoding);
+    // When sent back to Ruby, tokens are the raw string including whitespace
+    // and tag delimiters. It should be possible to reconstruct the exact
+    // template from the tokens.
+    return rb_enc_str_new(token.str_full, token.len_full, utf8_encoding);
+}
+
+static VALUE tokenizer_shift_trimmed_method(VALUE self)
+{
+    tokenizer_t *tokenizer;
+    Tokenizer_Get_Struct(self, tokenizer);
+
+    token_t token;
+    tokenizer_next(tokenizer, &token);
+    if (!token.type)
+        return Qnil;
+
+    // This method doesn't include whitespace and tag delimiters. It allows for
+    // testing the output of tokenizer_next as used by rb_block_parse.
+    return rb_enc_str_new(token.str_trimmed, token.len_trimmed, utf8_encoding);
 }
 
 static VALUE tokenizer_line_number_method(VALUE self)
@@ -173,12 +262,37 @@ static VALUE tokenizer_line_number_method(VALUE self)
     return UINT2NUM(tokenizer->line_number);
 }
 
-void init_liquid_tokenizer()
+static VALUE tokenizer_for_liquid_tag_method(VALUE self)
+{
+    tokenizer_t *tokenizer;
+    Tokenizer_Get_Struct(self, tokenizer);
+
+    return tokenizer->for_liquid_tag ? Qtrue : Qfalse;
+}
+
+
+// Temporary to test rollout of the fix for this bug
+static VALUE tokenizer_bug_compatible_whitespace_trimming(VALUE self) {
+    tokenizer_t *tokenizer;
+    Tokenizer_Get_Struct(self, tokenizer);
+
+    tokenizer->bug_compatible_whitespace_trimming = true;
+    return Qnil;
+}
+
+void liquid_define_tokenizer(void)
 {
     cLiquidTokenizer = rb_define_class_under(mLiquidC, "Tokenizer", rb_cObject);
+    rb_global_variable(&cLiquidTokenizer);
+
     rb_define_alloc_func(cLiquidTokenizer, tokenizer_allocate);
-    rb_define_method(cLiquidTokenizer, "initialize", tokenizer_initialize_method, 2);
-    rb_define_method(cLiquidTokenizer, "shift", tokenizer_shift_method, 0);
+    rb_define_method(cLiquidTokenizer, "initialize", tokenizer_initialize_method, 3);
     rb_define_method(cLiquidTokenizer, "line_number", tokenizer_line_number_method, 0);
+    rb_define_method(cLiquidTokenizer, "for_liquid_tag", tokenizer_for_liquid_tag_method, 0);
+    rb_define_method(cLiquidTokenizer, "bug_compatible_whitespace_trimming!", tokenizer_bug_compatible_whitespace_trimming, 0);
+    rb_define_method(cLiquidTokenizer, "shift", tokenizer_shift_method, 0);
+
+    // For testing the internal token representation.
+    rb_define_private_method(cLiquidTokenizer, "shift_trimmed", tokenizer_shift_trimmed_method, 0);
 }
 

data/ext/liquid_c/tokenizer.h

@@ -6,31 +6,42 @@ enum token_type {
     TOKEN_INVALID,
     TOKEN_RAW,
     TOKEN_TAG,
-    TOKEN_VARIABLE
+    TOKEN_VARIABLE,
+    TOKEN_BLANK_LIQUID_TAG_LINE
 };
 
 typedef struct token {
     enum token_type type;
-    const char *str;
-    long length;
-    unsigned int lstrip;
-    unsigned int rstrip;
+
+    // str_trimmed contains no tag delimiters
+    const char *str_trimmed, *str_full;
+    long len_trimmed, len_full;
+
+    bool lstrip, rstrip;
 } token_t;
 
 typedef struct tokenizer {
     VALUE source;
-    const char *cursor;
-    long length;
+    const char *cursor, *cursor_end;
     unsigned int line_number;
-    unsigned int lstrip_flag;
+    bool lstrip_flag;
+    bool for_liquid_tag;
+
+    // Temporary to test rollout of the fix for this bug
+    bool bug_compatible_whitespace_trimming;
+
+    char *raw_tag_body;
+    unsigned int raw_tag_body_len;
 } tokenizer_t;
 
 extern VALUE cLiquidTokenizer;
 extern const rb_data_type_t tokenizer_data_type;
 #define Tokenizer_Get_Struct(obj, sval) TypedData_Get_Struct(obj, tokenizer_t, &tokenizer_data_type, sval)
 
-void init_liquid_tokenizer();
+void liquid_define_tokenizer(void);
 void tokenizer_next(tokenizer_t *tokenizer, token_t *token);
 
+void tokenizer_setup_for_liquid_tag(tokenizer_t *tokenizer, const char *cursor, const char *cursor_end, int line_number);
+
 #endif
 

data/ext/liquid_c/usage.c

@@ -0,0 +1,18 @@
+#include "usage.h"
+
+static VALUE cLiquidUsage;
+static ID id_increment;
+
+void usage_increment(const char *name)
+{
+    VALUE name_str = rb_str_new_cstr(name);
+    rb_funcall(cLiquidUsage, id_increment, 1, name_str);
+}
+
+void liquid_define_usage(void)
+{
+    cLiquidUsage = rb_const_get(mLiquid, rb_intern("Usage"));
+    rb_global_variable(&cLiquidUsage);
+
+    id_increment = rb_intern("increment");
+}