liquid-c 4.0.1 → 4.1.0

data/ext/liquid_c/tokenizer.c

@@ -1,5 +1,7 @@
+#include <assert.h>
 #include "liquid.h"
 #include "tokenizer.h"
+#include "stringutil.h"
 
 VALUE cLiquidTokenizer;
 
@@ -35,40 +37,89 @@ static VALUE tokenizer_allocate(VALUE klass)
 
     obj = TypedData_Make_Struct(klass, tokenizer_t, &tokenizer_data_type, tokenizer);
     tokenizer->source = Qnil;
+    tokenizer->bug_compatible_whitespace_trimming = false;
+    tokenizer->raw_tag_body = NULL;
+    tokenizer->raw_tag_body_len = 0;
     return obj;
 }
 
-static VALUE tokenizer_initialize_method(VALUE self, VALUE source, VALUE line_numbers)
+static VALUE tokenizer_initialize_method(VALUE self, VALUE source, VALUE start_line_number, VALUE for_liquid_tag)
 {
     tokenizer_t *tokenizer;
 
     Check_Type(source, T_STRING);
+    check_utf8_encoding(source, "source");
+
+#define MAX_SOURCE_CODE_BYTES ((1 << 24) - 1)
+    if (RSTRING_LEN(source) > MAX_SOURCE_CODE_BYTES) {
+        rb_enc_raise(utf8_encoding, rb_eArgError, "Source too large, max %d bytes", MAX_SOURCE_CODE_BYTES);
+    }
+#undef MAX_SOURCE_CODE_BYTES
+
     Tokenizer_Get_Struct(self, tokenizer);
     source = rb_str_dup_frozen(source);
     tokenizer->source = source;
     tokenizer->cursor = RSTRING_PTR(source);
-    tokenizer->length = RSTRING_LEN(source);
-    tokenizer->lstrip_flag = 0;
+    tokenizer->cursor_end = tokenizer->cursor + RSTRING_LEN(source);
+    tokenizer->lstrip_flag = false;
     // tokenizer->line_number keeps track of the current line number or it is 0
     // to indicate that line numbers aren't being calculated
-    tokenizer->line_number = RTEST(line_numbers) ? 1 : 0;
+    tokenizer->line_number = FIX2UINT(start_line_number);
+    tokenizer->for_liquid_tag = RTEST(for_liquid_tag);
     return Qnil;
 }
 
-void tokenizer_next(tokenizer_t *tokenizer, token_t *token)
+// Internal function to setup an existing tokenizer from C for a liquid tag.
+// This overwrites the passed in tokenizer, so a copy of the struct should
+// be used to reset the tokenizer after parsing the liquid tag.
+void tokenizer_setup_for_liquid_tag(tokenizer_t *tokenizer, const char *cursor, const char *cursor_end, int line_number)
 {
-    if (tokenizer->length <= 0) {
-        memset(token, 0, sizeof(*token));
-        return;
+    tokenizer->cursor = cursor;
+    tokenizer->cursor_end = cursor_end;
+    tokenizer->lstrip_flag = false;
+    tokenizer->line_number = line_number;
+    tokenizer->for_liquid_tag = true;
+}
+
+// Tokenizes contents of {% liquid ... %}
+static void tokenizer_next_for_liquid_tag(tokenizer_t *tokenizer, token_t *token)
+{
+    const char *end = tokenizer->cursor_end;
+    const char *start = tokenizer->cursor;
+    const char *start_trimmed = read_while(start, end, is_non_newline_space);
+
+    token->str_full = start;
+    token->str_trimmed = start_trimmed;
+
+    const char *end_full = read_while(start_trimmed, end, not_newline);
+    if (end_full < end) {
+        tokenizer->cursor = end_full + 1;
+        if (tokenizer->line_number)
+            tokenizer->line_number++;
+    } else {
+        tokenizer->cursor = end_full;
     }
 
+    const char *end_trimmed = read_while_reverse(start_trimmed, end_full, rb_isspace);
+
+    token->len_trimmed = end_trimmed - start_trimmed;
+    token->len_full = end_full - token->str_full;
+
+    if (token->len_trimmed == 0) {
+        token->type = TOKEN_BLANK_LIQUID_TAG_LINE;
+    } else {
+        token->type = TOKEN_TAG;
+    }
+}
+
+// Tokenizes contents of a full Liquid template
+static void tokenizer_next_for_template(tokenizer_t *tokenizer, token_t *token)
+{
     const char *cursor = tokenizer->cursor;
-    const char *last = cursor + tokenizer->length - 1;
+    const char *last = tokenizer->cursor_end - 1;
 
-    token->str = cursor;
+    token->str_full = cursor;
     token->type = TOKEN_RAW;
-    token->lstrip = 0;
-    token->rstrip = 0;
 
     while (cursor < last) {
         if (*cursor++ != '{')
@@ -78,17 +129,17 @@ void tokenizer_next(tokenizer_t *tokenizer, token_t *token)
         if (c != '%' && c != '{')
             continue;
         if (cursor <= last && *cursor == '-') {
-          cursor++;
-          token->rstrip = 1;
+            cursor++;
+            token->rstrip = 1;
         }
         if (cursor - tokenizer->cursor > (ptrdiff_t)(2 + token->rstrip)) {
             token->type = TOKEN_RAW;
             cursor -= 2 + token->rstrip;
             token->lstrip = tokenizer->lstrip_flag;
-            tokenizer->lstrip_flag = 0;
+            tokenizer->lstrip_flag = false;
             goto found;
         }
-        tokenizer->lstrip_flag = 0;
+        tokenizer->lstrip_flag = false;
         token->type = TOKEN_INVALID;
         token->lstrip = token->rstrip;
         token->rstrip = 0;
@@ -103,12 +154,12 @@ void tokenizer_next(tokenizer_t *tokenizer, token_t *token)
                     continue;
                 token->type = TOKEN_TAG;
                 if(cursor[-3] == '-')
-                    token->rstrip = tokenizer->lstrip_flag = 1;
+                    token->rstrip = tokenizer->lstrip_flag = true;
                 goto found;
             }
             // unterminated tag
             cursor = tokenizer->cursor + 2;
-            tokenizer->lstrip_flag = 0;
+            tokenizer->lstrip_flag = false;
             goto found;
         } else {
             while (cursor < last) {
@@ -121,31 +172,51 @@ void tokenizer_next(tokenizer_t *tokenizer, token_t *token)
                 }
                 token->type = TOKEN_VARIABLE;
                 if(cursor[-3] == '-')
-                    token->rstrip = tokenizer->lstrip_flag = 1;
+                    token->rstrip = tokenizer->lstrip_flag = true;
                 goto found;
             }
             // unterminated variable
             cursor = tokenizer->cursor + 2;
-            tokenizer->lstrip_flag = 0;
+            tokenizer->lstrip_flag = false;
             goto found;
         }
     }
     cursor = last + 1;
     token->lstrip = tokenizer->lstrip_flag;
-    tokenizer->lstrip_flag = 0;
+    tokenizer->lstrip_flag = false;
 found:
-    token->length = cursor - tokenizer->cursor;
-    tokenizer->cursor += token->length;
-    tokenizer->length -= token->length;
+    token->len_full = cursor - token->str_full;
+
+    token->str_trimmed = token->str_full;
+    token->len_trimmed = token->len_full;
+
+    if (token->type == TOKEN_VARIABLE || token->type == TOKEN_TAG) {
+        token->str_trimmed += 2 + token->lstrip;
+        token->len_trimmed -= 2 + token->lstrip + 2;
+        if (token->rstrip && token->len_trimmed)
+            token->len_trimmed--;
+    }
+
+    assert(token->len_trimmed >= 0);
+
+    tokenizer->cursor += token->len_full;
 
     if (tokenizer->line_number) {
-        const char *cursor = token->str;
-        const char *end = token->str + token->length;
-        while (cursor < end) {
-            if (*cursor == '\n')
-                tokenizer->line_number++;
-            cursor++;
-        }
+        tokenizer->line_number += count_newlines(token->str_full, token->str_full + token->len_full);
+    }
+}
+
+void tokenizer_next(tokenizer_t *tokenizer, token_t *token)
+{
+    memset(token, 0, sizeof(*token));
+
+    if (tokenizer->cursor >= tokenizer->cursor_end) {
+        return;
+    }
+    if (tokenizer->for_liquid_tag) {
+        tokenizer_next_for_liquid_tag(tokenizer, token);
+    } else {
+        tokenizer_next_for_template(tokenizer, token);
     }
 }
 
@@ -159,7 +230,25 @@ static VALUE tokenizer_shift_method(VALUE self)
     if (!token.type)
         return Qnil;
 
-    return rb_enc_str_new(token.str, token.length, utf8_encoding);
+    // When sent back to Ruby, tokens are the raw string including whitespace
+    // and tag delimiters. It should be possible to reconstruct the exact
+    // template from the tokens.
+    return rb_enc_str_new(token.str_full, token.len_full, utf8_encoding);
+}
+
+static VALUE tokenizer_shift_trimmed_method(VALUE self)
+{
+    tokenizer_t *tokenizer;
+    Tokenizer_Get_Struct(self, tokenizer);
+
+    token_t token;
+    tokenizer_next(tokenizer, &token);
+    if (!token.type)
+        return Qnil;
+
+    // This method doesn't include whitespace and tag delimiters. It allows for
+    // testing the output of tokenizer_next as used by rb_block_parse.
+    return rb_enc_str_new(token.str_trimmed, token.len_trimmed, utf8_encoding);
 }
 
 static VALUE tokenizer_line_number_method(VALUE self)
@@ -173,12 +262,37 @@ static VALUE tokenizer_line_number_method(VALUE self)
     return UINT2NUM(tokenizer->line_number);
 }
 
-void init_liquid_tokenizer()
+static VALUE tokenizer_for_liquid_tag_method(VALUE self)
+{
+    tokenizer_t *tokenizer;
+    Tokenizer_Get_Struct(self, tokenizer);
+
+    return tokenizer->for_liquid_tag ? Qtrue : Qfalse;
+}
+
+
+// Temporary to test rollout of the fix for this bug
+static VALUE tokenizer_bug_compatible_whitespace_trimming(VALUE self) {
+    tokenizer_t *tokenizer;
+    Tokenizer_Get_Struct(self, tokenizer);
+
+    tokenizer->bug_compatible_whitespace_trimming = true;
+    return Qnil;
+}
+
+void liquid_define_tokenizer(void)
 {
     cLiquidTokenizer = rb_define_class_under(mLiquidC, "Tokenizer", rb_cObject);
+    rb_global_variable(&cLiquidTokenizer);
+
     rb_define_alloc_func(cLiquidTokenizer, tokenizer_allocate);
-    rb_define_method(cLiquidTokenizer, "initialize", tokenizer_initialize_method, 2);
-    rb_define_method(cLiquidTokenizer, "shift", tokenizer_shift_method, 0);
+    rb_define_method(cLiquidTokenizer, "initialize", tokenizer_initialize_method, 3);
     rb_define_method(cLiquidTokenizer, "line_number", tokenizer_line_number_method, 0);
+    rb_define_method(cLiquidTokenizer, "for_liquid_tag", tokenizer_for_liquid_tag_method, 0);
+    rb_define_method(cLiquidTokenizer, "bug_compatible_whitespace_trimming!", tokenizer_bug_compatible_whitespace_trimming, 0);
+
+    // For testing the internal token representation.
+    rb_define_private_method(cLiquidTokenizer, "shift", tokenizer_shift_method, 0);
+    rb_define_private_method(cLiquidTokenizer, "shift_trimmed", tokenizer_shift_trimmed_method, 0);
 }
 

data/ext/liquid_c/tokenizer.h

@@ -6,31 +6,42 @@ enum token_type {
     TOKEN_INVALID,
     TOKEN_RAW,
     TOKEN_TAG,
-    TOKEN_VARIABLE
+    TOKEN_VARIABLE,
+    TOKEN_BLANK_LIQUID_TAG_LINE
 };
 
 typedef struct token {
     enum token_type type;
-    const char *str;
-    long length;
-    unsigned int lstrip;
-    unsigned int rstrip;
+
+    // str_trimmed contains no tag delimiters
+    const char *str_trimmed, *str_full;
+    long len_trimmed, len_full;
+
+    bool lstrip, rstrip;
 } token_t;
 
 typedef struct tokenizer {
     VALUE source;
-    const char *cursor;
-    long length;
+    const char *cursor, *cursor_end;
     unsigned int line_number;
-    unsigned int lstrip_flag;
+    bool lstrip_flag;
+    bool for_liquid_tag;
+
+    // Temporary to test rollout of the fix for this bug
+    bool bug_compatible_whitespace_trimming;
+
+    char *raw_tag_body;
+    unsigned int raw_tag_body_len;
 } tokenizer_t;
 
 extern VALUE cLiquidTokenizer;
 extern const rb_data_type_t tokenizer_data_type;
 #define Tokenizer_Get_Struct(obj, sval) TypedData_Get_Struct(obj, tokenizer_t, &tokenizer_data_type, sval)
 
-void init_liquid_tokenizer();
+void liquid_define_tokenizer(void);
 void tokenizer_next(tokenizer_t *tokenizer, token_t *token);
 
+void tokenizer_setup_for_liquid_tag(tokenizer_t *tokenizer, const char *cursor, const char *cursor_end, int line_number);
+
 #endif
 

data/ext/liquid_c/usage.c

@@ -0,0 +1,18 @@
+#include "usage.h"
+
+static VALUE cLiquidUsage;
+static ID id_increment;
+
+void usage_increment(const char *name)
+{
+    VALUE name_str = rb_str_new_cstr(name);
+    rb_funcall(cLiquidUsage, id_increment, 1, name_str);
+}
+
+void liquid_define_usage(void)
+{
+    cLiquidUsage = rb_const_get(mLiquid, rb_intern("Usage"));
+    rb_global_variable(&cLiquidUsage);
+
+    id_increment = rb_intern("increment");
+}

data/ext/liquid_c/usage.h

@@ -0,0 +1,9 @@
+#ifndef LIQUID_USAGE_H
+#define LIQUID_USAGE_H
+
+#include "liquid.h"
+
+void liquid_define_usage(void);
+void usage_increment(const char *name);
+
+#endif

data/ext/liquid_c/variable.c

@@ -1,25 +1,43 @@
 #include "liquid.h"
 #include "variable.h"
 #include "parser.h"
+#include "expression.h"
+#include "vm.h"
+
 #include <stdio.h>
 
-static VALUE rb_variable_parse(VALUE self, VALUE markup, VALUE filters)
-{
-    StringValue(markup);
-    char *start = RSTRING_PTR(markup);
+static ID id_rescue_strict_parse_syntax_error;
+
+static ID id_ivar_parse_context;
+static ID id_ivar_name;
+static ID id_ivar_filters;
 
+VALUE cLiquidCVariableExpression;
+
+static VALUE frozen_empty_array;
+
+static VALUE try_variable_strict_parse(VALUE uncast_args)
+{
+    variable_parse_args_t *parse_args = (void *)uncast_args;
     parser_t p;
-    init_parser(&p, start, start + RSTRING_LEN(markup));
+    init_parser(&p, parse_args->markup, parse_args->markup_end);
+    vm_assembler_t *code = parse_args->code;
 
-    if (p.cur.type == TOKEN_EOS)
+    if (p.cur.type == TOKEN_EOS) {
+        vm_assembler_add_push_nil(code);
         return Qnil;
+    }
 
-    VALUE name = parse_expression(&p);
+    parse_and_compile_expression(&p, code);
 
     while (parser_consume(&p, TOKEN_PIPE).type) {
-        lexer_token_t filter_name = parser_must_consume(&p, TOKEN_IDENTIFIER);
+        lexer_token_t filter_name_token = parser_must_consume(&p, TOKEN_IDENTIFIER);
+        VALUE filter_name = token_to_rsym(filter_name_token);
 
-        VALUE filter_args = rb_ary_new(), keyword_args = Qnil, filter;
+        size_t arg_count = 0;
+        size_t keyword_arg_count = 0;
+        VALUE push_keywords_obj = Qnil;
+        vm_assembler_t *push_keywords_code = NULL;
 
         if (parser_consume(&p, TOKEN_COLON).type) {
             do {
@@ -27,28 +45,186 @@ static VALUE rb_variable_parse(VALUE self, VALUE markup, VALUE filters)
                     VALUE key = token_to_rstr(parser_consume_any(&p));
                     parser_consume_any(&p);
 
-                    if (keyword_args == Qnil) keyword_args = rb_hash_new();
-                    rb_hash_aset(keyword_args, key, parse_expression(&p));
+                    keyword_arg_count++;
+
+                    if (push_keywords_obj == Qnil) {
+                        expression_t *push_keywords_expr;
+                        // use an object to automatically free on an exception
+                        push_keywords_obj = expression_new(cLiquidCExpression, &push_keywords_expr);
+                        rb_obj_hide(push_keywords_obj);
+                        push_keywords_code = &push_keywords_expr->code;
+                    }
+
+                    vm_assembler_add_push_const(push_keywords_code, key);
+                    parse_and_compile_expression(&p, push_keywords_code);
                 } else {
-                    rb_ary_push(filter_args, parse_expression(&p));
+                    parse_and_compile_expression(&p, code);
+                    arg_count++;
                 }
             } while (parser_consume(&p, TOKEN_COMMA).type);
         }
 
-        if (keyword_args == Qnil) {
-            filter = rb_ary_new3(2, token_to_rstr(filter_name), filter_args);
-        } else {
-            filter = rb_ary_new3(3, token_to_rstr(filter_name), filter_args, keyword_args);
+        if (keyword_arg_count) {
+            arg_count++;
+            if (keyword_arg_count > 255)
+                rb_enc_raise(utf8_encoding, cLiquidSyntaxError, "Too many filter keyword arguments");
+
+            vm_assembler_concat(code, push_keywords_code);
+            vm_assembler_add_hash_new(code, keyword_arg_count);
+
+            RB_GC_GUARD(push_keywords_obj);
         }
-        rb_ary_push(filters, filter);
+        vm_assembler_add_filter(code, filter_name, arg_count);
     }
 
     parser_must_consume(&p, TOKEN_EOS);
-    return name;
+
+    return Qnil;
+}
+
+typedef struct variable_strict_parse_rescue {
+    variable_parse_args_t *parse_args;
+    size_t instructions_size;
+    size_t constants_size;
+    size_t stack_size;
+} variable_strict_parse_rescue_t;
+
+static VALUE variable_strict_parse_rescue(VALUE uncast_args, VALUE exception)
+{
+    variable_strict_parse_rescue_t *rescue_args = (void *)uncast_args;
+    variable_parse_args_t *parse_args = rescue_args->parse_args;
+    vm_assembler_t *code = parse_args->code;
+
+    // undo partial strict parse
+    uint8_t *last_constants_data_end = (uint8_t *)code->constants.data + rescue_args->constants_size;
+    VALUE *const_ptr = (VALUE *)last_constants_data_end;
+    st_table *constants_table = code->constants_table;
+
+    while((uint8_t *)const_ptr < code->constants.data_end) {
+        st_data_t key = (st_data_t)const_ptr[0];
+        st_delete(constants_table, &key, 0);
+        const_ptr++;
+    }
+
+    code->instructions.data_end = code->instructions.data + rescue_args->instructions_size;
+    code->constants.data_end = last_constants_data_end;
+    code->stack_size = rescue_args->stack_size;
+
+    if (rb_obj_is_kind_of(exception, cLiquidSyntaxError) == Qfalse)
+        rb_exc_raise(exception);
+
+    VALUE markup_obj = rb_enc_str_new(parse_args->markup, parse_args->markup_end - parse_args->markup, utf8_encoding);
+    VALUE variable_obj = rb_funcall(
+        cLiquidVariable, id_rescue_strict_parse_syntax_error, 3,
+        exception, markup_obj, parse_args->parse_context
+    );
+
+    // lax parse
+    code->protected_stack_size = code->stack_size;
+    rb_funcall(variable_obj, id_compile_evaluate, 1, parse_args->code_obj);
+    if (code->stack_size != code->protected_stack_size + 1) {
+        rb_raise(rb_eRuntimeError, "Liquid::Variable#compile_evaluate didn't leave exactly 1 new element on the stack");
+    }
+
+    return Qnil;
+}
+
+void internal_variable_compile_evaluate(variable_parse_args_t *parse_args)
+{
+    vm_assembler_t *code = parse_args->code;
+    variable_strict_parse_rescue_t rescue_args = {
+        .parse_args = parse_args,
+        .instructions_size = c_buffer_size(&code->instructions),
+        .constants_size = c_buffer_size(&code->constants),
+        .stack_size = code->stack_size,
+    };
+    rb_rescue(try_variable_strict_parse, (VALUE)parse_args, variable_strict_parse_rescue, (VALUE)&rescue_args);
 }
 
-void init_liquid_variable(void)
+void internal_variable_compile(variable_parse_args_t *parse_args, unsigned int line_number)
 {
-    rb_define_singleton_method(cLiquidVariable, "c_strict_parse", rb_variable_parse, 2);
+    vm_assembler_t *code = parse_args->code;
+    vm_assembler_add_render_variable_rescue(code, line_number);
+    internal_variable_compile_evaluate(parse_args);
+    vm_assembler_add_pop_write(code);
+}
+
+static VALUE variable_strict_parse_method(VALUE self, VALUE markup)
+{
+    StringValue(markup);
+    check_utf8_encoding(markup, "markup");
+
+    VALUE parse_context = rb_ivar_get(self, id_ivar_parse_context);
+
+    expression_t *expression;
+    VALUE expression_obj = expression_new(cLiquidCVariableExpression, &expression);
+
+    variable_parse_args_t parse_args = {
+        .markup = RSTRING_PTR(markup),
+        .markup_end = RSTRING_END(markup),
+        .code = &expression->code,
+        .code_obj = expression_obj,
+        .parse_context = parse_context,
+    };
+    try_variable_strict_parse((VALUE)&parse_args);
+    RB_GC_GUARD(markup);
+    assert(expression->code.stack_size == 1);
+    vm_assembler_add_leave(&expression->code);
+
+    rb_ivar_set(self, id_ivar_name, expression_obj);
+    rb_ivar_set(self, id_ivar_filters, frozen_empty_array);
+
+    return Qnil;
+}
+
+typedef struct {
+    expression_t *expression;
+    VALUE context;
+} variable_expression_evaluate_args_t;
+
+static VALUE try_variable_expression_evaluate(VALUE uncast_args)
+{
+    variable_expression_evaluate_args_t *args = (void *)uncast_args;
+    return liquid_vm_evaluate(args->context, &args->expression->code);
+}
+
+static VALUE rescue_variable_expression_evaluate(VALUE uncast_args, VALUE exception)
+{
+    variable_expression_evaluate_args_t *args = (void *)uncast_args;
+    vm_t *vm = vm_from_context(args->context);
+    exception = vm_translate_if_filter_argument_error(vm, exception);
+    rb_exc_raise(exception);
+}
+
+VALUE internal_variable_expression_evaluate(expression_t *expression, VALUE context)
+{
+    variable_expression_evaluate_args_t args = { expression, context };
+    return rb_rescue(try_variable_expression_evaluate, (VALUE)&args, rescue_variable_expression_evaluate, (VALUE)&args);
+}
+
+static VALUE variable_expression_evaluate_method(VALUE self, VALUE context)
+{
+    expression_t *expression;
+    Expression_Get_Struct(self, expression);
+    return internal_variable_expression_evaluate(expression, context);
+}
+
+void liquid_define_variable(void)
+{
+    id_rescue_strict_parse_syntax_error = rb_intern("rescue_strict_parse_syntax_error");
+
+    id_ivar_parse_context = rb_intern("@parse_context");
+    id_ivar_name = rb_intern("@name");
+    id_ivar_filters = rb_intern("@filters");
+
+    frozen_empty_array = rb_ary_new();
+    rb_ary_freeze(frozen_empty_array);
+    rb_global_variable(&frozen_empty_array);
+
+    rb_define_method(cLiquidVariable, "c_strict_parse", variable_strict_parse_method, 1);
+
+    cLiquidCVariableExpression = rb_define_class_under(mLiquidC, "VariableExpression", cLiquidCExpression);
+    rb_global_variable(&cLiquidCVariableExpression);
+    rb_define_method(cLiquidCVariableExpression, "evaluate", variable_expression_evaluate_method, 1);
 }
 

data/ext/liquid_c/variable.h

@@ -1,7 +1,24 @@
 #if !defined(LIQUID_VARIABLE_H)
 #define LIQUID_VARIABLE_H
 
-void init_liquid_variable(void);
+#include "vm_assembler.h"
+#include "block.h"
+#include "expression.h"
+
+extern VALUE cLiquidCVariableExpression;
+
+typedef struct variable_parse_args {
+    const char *markup;
+    const char *markup_end;
+    vm_assembler_t *code;
+    VALUE code_obj;
+    VALUE parse_context;
+} variable_parse_args_t;
+
+void liquid_define_variable(void);
+void internal_variable_compile(variable_parse_args_t *parse_args, unsigned int line_number);
+void internal_variable_compile_evaluate(variable_parse_args_t *parse_args);
+VALUE internal_variable_expression_evaluate(expression_t *expression, VALUE context);
 
 #endif
 

data/ext/liquid_c/variable_lookup.c

@@ -0,0 +1,44 @@
+#include "liquid.h"
+#include "context.h"
+
+static ID id_has_key, id_aref, id_fetch, id_to_liquid_value;
+
+VALUE variable_lookup_key(VALUE context, VALUE object, VALUE key, bool is_command)
+{
+    if (rb_obj_class(key) != rb_cString) {
+        VALUE key_value = rb_check_funcall(key, id_to_liquid_value, 0, 0);
+
+        if (key_value != Qundef) {
+            key = key_value;
+        }
+    }
+
+    if (rb_respond_to(object, id_aref) && (
+        (rb_respond_to(object, id_has_key) && rb_funcall(object, id_has_key, 1, key)) ||
+        (rb_obj_is_kind_of(key, rb_cInteger) && rb_respond_to(object, id_fetch))
+    )) {
+        VALUE next_object = rb_funcall(object, id_aref, 1, key);
+        next_object = materialize_proc(context, object, key, next_object);
+        return value_to_liquid_and_set_context(next_object, context);
+    }
+
+    if (is_command) {
+        Check_Type(key, T_STRING);
+        ID intern_key = rb_intern(RSTRING_PTR(key));
+        if (rb_respond_to(object, intern_key)) {
+            VALUE next_object = rb_funcall(object, intern_key, 0);
+            return value_to_liquid_and_set_context(next_object, context);
+        }
+    }
+
+    context_maybe_raise_undefined_variable(context, key);
+    return Qnil;
+}
+
+void liquid_define_variable_lookup(void)
+{
+    id_has_key = rb_intern("key?");
+    id_aref = rb_intern("[]");
+    id_fetch = rb_intern("fetch");
+    id_to_liquid_value = rb_intern("to_liquid_value");
+}