liquid-c 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1f1e4b7a91ef500fc48563ecdff875658c045b0e
-  data.tar.gz: 460cbca81b9ff70a9c20e4051cdc62a26a1e2c7b
+  metadata.gz: ceaa0d7db7673b4b6e73fb7e669a2728dbafb1b3
+  data.tar.gz: 018116a6bedfc9305a3e64aaa6624c4dfb52d7b3
 SHA512:
-  metadata.gz: 1ecf742609e18e26a982897562f3ff0ef0c06e77045d12b98ad36fda3081e4350faffb9bae860ce1ecc060d8c6138b7fd482c323d763538dc38b13bc5738d1f8
-  data.tar.gz: 817638febdcb9a54399ab8aec528091b1c24f66510816e455f09b85bce9b2c1f7a5e559a89ab5611327b1d0f780c9d62fcb36ad0b231eaba35554f551bcdc80e
+  metadata.gz: 8a415cb28d0ad601e63e5ae6e979e762caee118f8019063383da5e24a34034d6b22e0973f0efa16f747b858ea2520299681ae9b32010c547e79877227a3b9cdd
+  data.tar.gz: 1913e1d901f48241a36d25b5eb4a72c6c1e61ccc98f17494d7d75d4e613bac0dc251b49957472abe1f410ce62575817906023182e3c01d64dc7af0ae5a934bb3

data/Gemfile

@@ -6,6 +6,7 @@ gem 'liquid', github: 'Shopify/liquid', branch: 'master'
 
 group :test do
   gem 'spy', '0.4.1'
+  gem 'benchmark-ips'
 end
 
 group :development do

data/README.md

@@ -1,5 +1,5 @@
 # Liquid::C
-[![Build Status](https://api.travis-ci.org/Shopify/liquid-c.png?branch=master)](https://travis-ci.org/Shopify/liquid-c)
+[![Build Status](https://travis-ci.org/Shopify/liquid-c.svg?branch=master)](https://travis-ci.org/Shopify/liquid-c)
 
 Partial native implementation of the liquid ruby gem in C.
 

data/Rakefile

@@ -59,18 +59,11 @@ namespace :profile do
 end
 
 namespace :compare do
-  include Benchmark
-  desc "Compare Liquid to Liquid + Liquid-C"
-  task :run do
-    bare = Benchmark.measure do
-      ruby "./performance.rb bare profile lax"
-    end
-    liquid_c = Benchmark.measure do
-      ruby "./performance.rb c profile lax"
-    end
-    Benchmark.benchmark(CAPTION, 10, FORMAT, "Liquid:", "Liquid-C:") do |x|
-      [bare, liquid_c]
+  %w(lax warn strict).each do |type|
+    desc "Compare Liquid to Liquid-C in #{type} mode"
+    task type.to_sym do
+      ruby "./performance.rb bare benchmark #{type}"
+      ruby "./performance.rb c benchmark #{type}"
     end
-    puts "Ratio: #{liquid_c.real / bare.real * 100}%"
   end
 end

data/ext/liquid_c/block.c

@@ -0,0 +1,122 @@
+#include "liquid.h"
+#include "tokenizer.h"
+#include <stdio.h>
+
+static ID
+    intern_raise_missing_variable_terminator,
+    intern_raise_missing_tag_terminator,
+    intern_nodelist,
+    intern_blank,
+    intern_is_blank,
+    intern_clear,
+    intern_tags,
+    intern_parse,
+    intern_square_brackets;
+
+static int is_id(int c)
+{
+    return rb_isalnum(c) || c == '_';
+}
+
+inline static const char *read_while(const char *start, const char *end, int (func)(int))
+{
+    while (start < end && func((unsigned char) *start)) start++;
+    return start;
+}
+
+static VALUE rb_block_parse(VALUE self, VALUE tokens, VALUE options)
+{
+    tokenizer_t *tokenizer;
+    Tokenizer_Get_Struct(tokens, tokenizer);
+
+    token_t token;
+    VALUE tags = Qnil;
+    VALUE nodelist = rb_ivar_get(self, intern_nodelist);
+
+    while (true) {
+        tokenizer_next(tokenizer, &token);
+
+        switch (token.type) {
+            case TOKENIZER_TOKEN_NONE:
+                return rb_yield_values(2, Qnil, Qnil);
+
+            case TOKEN_INVALID:
+            {
+                VALUE str = rb_enc_str_new(token.str, token.length, utf8_encoding);
+
+                ID raise_method_id = intern_raise_missing_variable_terminator;
+                if (token.str[1] == '%') raise_method_id = intern_raise_missing_tag_terminator;
+
+                return rb_funcall(self, raise_method_id, 2, str, options);
+            }
+            case TOKEN_RAW:
+            {
+                VALUE str = rb_enc_str_new(token.str, token.length, utf8_encoding);
+                rb_ary_push(nodelist, str);
+
+                if (rb_ivar_get(self, intern_blank) == Qtrue) {
+                    const char *end = token.str + token.length;
+
+                    if (read_while(token.str, end, rb_isspace) < end)
+                        rb_ivar_set(self, intern_blank, Qfalse);
+                }
+                break;
+            }
+            case TOKEN_VARIABLE:
+            {
+                VALUE args[2] = {rb_enc_str_new(token.str + 2, token.length - 4, utf8_encoding), options};
+                VALUE var = rb_class_new_instance(2, args, cLiquidVariable);
+                rb_ary_push(nodelist, var);
+                rb_ivar_set(self, intern_blank, Qfalse);
+                break;
+            }
+            case TOKEN_TAG:
+            {
+                const char *start = token.str + 2, *end = token.str + token.length - 2;
+
+                // Imitate \s*(\w+)\s*(.*)? regex
+                const char *name_start = read_while(start, end, rb_isspace);
+                const char *name_end = read_while(name_start, end, is_id);
+
+                VALUE tag_name = rb_enc_str_new(name_start, name_end - name_start, utf8_encoding);
+
+                if (tags == Qnil)
+                    tags = rb_funcall(cLiquidTemplate, intern_tags, 0);
+
+                VALUE tag_class = rb_funcall(tags, intern_square_brackets, 1, tag_name);
+
+                const char *markup_start = read_while(name_end, end, rb_isspace);
+                VALUE markup = rb_enc_str_new(markup_start, end - markup_start, utf8_encoding);
+
+                if (tag_class == Qnil)
+                    return rb_yield_values(2, tag_name, markup);
+
+                VALUE new_tag = rb_funcall(tag_class, intern_parse, 4, tag_name, markup, tokens, options);
+
+                if (rb_ivar_get(self, intern_blank) == Qtrue && !RTEST(rb_funcall(new_tag, intern_is_blank, 0)))
+                    rb_ivar_set(self, intern_blank, Qfalse);
+
+                rb_ary_push(nodelist, new_tag);
+                break;
+            }
+        }
+    }
+    return Qnil;
+}
+
+void init_liquid_block()
+{
+    intern_raise_missing_variable_terminator = rb_intern("raise_missing_variable_terminator");
+    intern_raise_missing_tag_terminator = rb_intern("raise_missing_tag_terminator");
+    intern_nodelist = rb_intern("@nodelist");
+    intern_blank = rb_intern("@blank");
+    intern_is_blank = rb_intern("blank?");
+    intern_clear = rb_intern("clear");
+    intern_tags = rb_intern("tags");
+    intern_parse = rb_intern("parse");
+    intern_square_brackets = rb_intern("[]");
+
+    VALUE cLiquidBlockBody = rb_const_get(mLiquid, rb_intern("BlockBody"));
+    rb_define_method(cLiquidBlockBody, "c_parse", rb_block_parse, 2);
+}
+

data/ext/liquid_c/block.h

@@ -0,0 +1,7 @@
+#if !defined(LIQUID_BLOCK_H)
+#define LIQUID_BLOCK_H
+
+void init_liquid_block();
+
+#endif
+

data/ext/liquid_c/lexer.c

@@ -0,0 +1,148 @@
+#include "liquid.h"
+#include "lexer.h"
+#include <stdio.h>
+
+const char *symbol_names[TOKEN_END] = {
+    [TOKEN_NONE] = "none",
+    [TOKEN_COMPARISON] = "comparison",
+    [TOKEN_STRING] = "string",
+    [TOKEN_NUMBER] = "number",
+    [TOKEN_IDENTIFIER] = "id",
+    [TOKEN_DOTDOT] = "dotdot",
+    [TOKEN_EOS] = "end_of_string",
+    [TOKEN_PIPE] = "pipe",
+    [TOKEN_DOT] = "dot",
+    [TOKEN_COLON] = "colon",
+    [TOKEN_COMMA] = "comma",
+    [TOKEN_OPEN_SQUARE] = "open_square",
+    [TOKEN_CLOSE_SQUARE] = "close_square",
+    [TOKEN_OPEN_ROUND] = "open_round",
+    [TOKEN_CLOSE_ROUND] = "close_round",
+    [TOKEN_QUESTION] = "question",
+    [TOKEN_DASH] = "dash"
+};
+
+inline static int is_identifier(char c)
+{
+    return ISALNUM(c) || c == '_' || c == '-';
+}
+
+inline static int is_special(char c)
+{
+    switch (c) {
+        case '|': case '.': case ':': case ',':
+        case '[': case ']': case '(': case ')':
+        case '?': case '-':
+            return 1;
+    }
+    return 0;
+}
+
+// Returns a pointer to the character after the end of the match.
+inline static const char *prefix_end(const char *cur, const char *end, const char *pattern)
+{
+    size_t pattern_len = strlen(pattern);
+
+    if (pattern_len > (size_t)(end - cur)) return NULL;
+    if (memcmp(cur, pattern, pattern_len) != 0) return NULL;
+
+    return cur + pattern_len;
+}
+
+inline static const char *scan_past(const char *cur, const char *end, char target)
+{
+    const char *match = memchr(cur + 1, target, end - cur - 1);
+    return match ? match + 1 : NULL;
+}
+
+#define RETURN_TOKEN(t, n) { \
+    const char *tok_end = str + (n); \
+    token->type = (t); \
+    token->val = str; \
+    if (str != start) token->flags |= TOKEN_SPACE_PREFIX; \
+    if (tok_end < end && ISSPACE(*tok_end)) token->flags |= TOKEN_SPACE_SUFFIX; \
+    return (token->val_end = tok_end); \
+}
+
+// Reads one token from start, and fills it into the token argument.
+// Returns the start of the next token if any, otherwise the end of the string.
+const char *lex_one(const char *start, const char *end, lexer_token_t *token)
+{
+    // str references the start of the token, after whitespace is skipped.
+    // cur references the currently processing character during iterative lexing.
+    const char *str = start, *cur;
+
+    while (str < end && ISSPACE(*str)) ++str;
+
+    token->val = token->val_end = NULL;
+    token->flags = 0;
+
+    if (str >= end) return str;
+
+    char c = *str;  // First character of the token.
+    char cn = '\0'; // Second character if available, for lookahead.
+    if (str + 1 < end) cn = str[1];
+
+    switch (c) {
+        case '<':
+            RETURN_TOKEN(TOKEN_COMPARISON, cn == '>' || cn == '=' ? 2 : 1);
+        case '>':
+            RETURN_TOKEN(TOKEN_COMPARISON, cn == '=' ? 2 : 1);
+        case '=':
+        case '!':
+            if (cn == '=') RETURN_TOKEN(TOKEN_COMPARISON, 2);
+            break;
+        case '.':
+            if (cn == '.') RETURN_TOKEN(TOKEN_DOTDOT, 2);
+            break;
+    }
+
+    if ((cur = prefix_end(str, end, "contains")))
+        RETURN_TOKEN(TOKEN_COMPARISON, cur - str);
+
+    if (c == '\'' || c == '"') {
+        cur = scan_past(str, end, c);
+
+        if (cur) {
+            // Quote was properly terminated.
+            RETURN_TOKEN(TOKEN_STRING, cur - str);
+        }
+    }
+
+    if (ISDIGIT(c) || c == '-') {
+        int has_dot = 0;
+        cur = str;
+        while (++cur < end) {
+            if (!has_dot && *cur == '.') {
+                has_dot = 1;
+            } else if (!ISDIGIT(*cur)) {
+                break;
+            }
+        }
+        cur--; // Point to last digit (or dot).
+
+        if (*cur == '.') {
+            cur--; // Ignore any trailing dot.
+            has_dot = 0;
+        }
+        if (*cur != '-') {
+            if (has_dot) token->flags |= TOKEN_FLOAT_NUMBER;
+            RETURN_TOKEN(TOKEN_NUMBER, cur + 1 - str);
+        }
+    }
+
+    if (ISALPHA(c) || c == '_') {
+        cur = str;
+        while (++cur < end && is_identifier(*cur)) {}
+        if (cur < end && *cur == '?') cur++;
+        RETURN_TOKEN(TOKEN_IDENTIFIER, cur - str);
+    }
+
+    if (is_special(c)) RETURN_TOKEN(c, 1);
+
+    rb_enc_raise(utf8_encoding, cLiquidSyntaxError, "Unexpected character %c", c);
+    return NULL;
+}
+
+#undef RETURN_TOKEN
+

data/ext/liquid_c/lexer.h

@@ -0,0 +1,46 @@
+#if !defined(LIQUID_LEXER_H)
+#define LIQUID_LEXER_H
+
+enum lexer_token_type {
+    TOKEN_NONE,
+    TOKEN_COMPARISON,
+    TOKEN_STRING,
+    TOKEN_NUMBER,
+    TOKEN_IDENTIFIER,
+    TOKEN_DOTDOT,
+    TOKEN_EOS,
+
+    TOKEN_PIPE = '|',
+    TOKEN_DOT = '.',
+    TOKEN_COLON = ':',
+    TOKEN_COMMA = ',',
+    TOKEN_OPEN_SQUARE = '[',
+    TOKEN_CLOSE_SQUARE = ']',
+    TOKEN_OPEN_ROUND = '(',
+    TOKEN_CLOSE_ROUND = ')',
+    TOKEN_QUESTION = '?',
+    TOKEN_DASH = '-',
+
+    TOKEN_END = 256
+};
+
+#define TOKEN_SPACE_PREFIX 0x1
+#define TOKEN_SPACE_SUFFIX 0x2
+#define TOKEN_SPACE_AFFIX (TOKEN_SPACE_PREFIX | TOKEN_SPACE_SUFFIX)
+#define TOKEN_FLOAT_NUMBER 0x4
+
+typedef struct lexer_token {
+    unsigned char type, flags;
+    const char *val, *val_end;
+} lexer_token_t;
+
+extern const char *symbol_names[TOKEN_END];
+
+const char *lex_one(const char *str, const char *end, lexer_token_t *token);
+
+inline static VALUE token_to_rstr(lexer_token_t token) {
+    return rb_enc_str_new(token.val, token.val_end - token.val, utf8_encoding);
+}
+
+#endif
+

data/ext/liquid_c/liquid.c

@@ -1,12 +1,24 @@
 #include "liquid.h"
 #include "tokenizer.h"
+#include "variable.h"
+#include "lexer.h"
+#include "parser.h"
+#include "block.h"
 
-VALUE mLiquid;
+VALUE mLiquid, cLiquidSyntaxError, cLiquidVariable, cLiquidTemplate;
 rb_encoding *utf8_encoding;
 
 void Init_liquid_c(void)
 {
     utf8_encoding = rb_utf8_encoding();
     mLiquid = rb_define_module("Liquid");
+    cLiquidSyntaxError = rb_const_get(mLiquid, rb_intern("SyntaxError"));
+    cLiquidVariable = rb_const_get(mLiquid, rb_intern("Variable"));
+    cLiquidTemplate = rb_const_get(mLiquid, rb_intern("Template"));
+
     init_liquid_tokenizer();
+    init_liquid_parser();
+    init_liquid_variable();
+    init_liquid_block();
 }
+

data/ext/liquid_c/liquid.h

@@ -5,7 +5,8 @@
 #include <ruby/encoding.h>
 #include <stdbool.h>
 
-extern VALUE mLiquid;
+extern VALUE mLiquid, cLiquidSyntaxError, cLiquidVariable, cLiquidTemplate;
 extern rb_encoding *utf8_encoding;
 
 #endif
+

data/ext/liquid_c/parser.c

@@ -0,0 +1,198 @@
+#include "liquid.h"
+#include "parser.h"
+#include "lexer.h"
+
+static VALUE cLiquidRangeLookup, cLiquidVariableLookup, cRange, symBlank, symEmpty;
+static ID idToI, idEvaluate;
+
+void init_parser(parser_t *p, const char *str, const char *end)
+{
+    p->str_end = end;
+    p->cur.type = p->next.type = TOKEN_EOS;
+    p->str = lex_one(str, end, &p->cur);
+    p->str = lex_one(p->str, end, &p->next);
+}
+
+lexer_token_t parser_consume_any(parser_t *p)
+{
+    lexer_token_t cur = p->cur;
+    p->cur = p->next;
+    p->next.type = TOKEN_EOS;
+    p->str = lex_one(p->str, p->str_end, &p->next);
+    return cur;
+}
+
+lexer_token_t parser_must_consume(parser_t *p, unsigned char type)
+{
+    if (p->cur.type != type) {
+        rb_enc_raise(utf8_encoding, cLiquidSyntaxError, "Expected %s but found %s",
+                 symbol_names[type], symbol_names[p->cur.type]);
+    }
+    return parser_consume_any(p);
+}
+
+lexer_token_t parser_consume(parser_t *p, unsigned char type)
+{
+    if (p->cur.type != type) {
+        lexer_token_t zero = {0};
+        return zero;
+    }
+    return parser_consume_any(p);
+}
+
+inline static int rstring_eq(VALUE rstr, const char *str) {
+    size_t str_len = strlen(str);
+
+    return TYPE(rstr) == T_STRING &&
+           str_len == (size_t)RSTRING_LEN(rstr) &&
+           memcmp(RSTRING_PTR(rstr), str, str_len) == 0;
+}
+
+static VALUE parse_number(parser_t *p)
+{
+    VALUE out;
+    lexer_token_t token = parser_must_consume(p, TOKEN_NUMBER);
+
+    // Set up sentinel for rb_cstr operations.
+    char tmp = *token.val_end;
+    *(char *)token.val_end = '\0';
+
+    if (token.flags & TOKEN_FLOAT_NUMBER) {
+        out = DBL2NUM(rb_cstr_to_dbl(token.val, 1));
+    } else {
+        out = rb_cstr_to_inum(token.val, 10, 1);
+    }
+
+    *(char *)token.val_end = tmp;
+    return out;
+}
+
+static VALUE parse_range(parser_t *p)
+{
+    parser_must_consume(p, TOKEN_OPEN_ROUND);
+
+    VALUE args[2];
+    args[0] = parse_expression(p);
+    parser_must_consume(p, TOKEN_DOTDOT);
+
+    args[1] = parse_expression(p);
+    parser_must_consume(p, TOKEN_CLOSE_ROUND);
+
+    if (rb_respond_to(args[0], idEvaluate) || rb_respond_to(args[1], idEvaluate))
+        return rb_class_new_instance(2, args, cLiquidRangeLookup);
+
+    return rb_class_new_instance(2, args, cRange);
+}
+
+static VALUE parse_variable(parser_t *p)
+{
+    VALUE name, lookups = rb_ary_new(), lookup;
+    unsigned long long command_flags = 0;
+
+    if (parser_consume(p, TOKEN_OPEN_SQUARE).type) {
+        name = parse_expression(p);
+        parser_must_consume(p, TOKEN_CLOSE_SQUARE);
+    } else {
+        name = token_to_rstr(parser_must_consume(p, TOKEN_IDENTIFIER));
+    }
+
+    while (true) {
+        if (p->cur.type == TOKEN_OPEN_SQUARE) {
+            parser_consume_any(p);
+            lookup = parse_expression(p);
+            parser_must_consume(p, TOKEN_CLOSE_SQUARE);
+
+            rb_ary_push(lookups, lookup);
+        } else if (p->cur.type == TOKEN_DOT) {
+            int has_space_affix = parser_consume_any(p).flags & TOKEN_SPACE_AFFIX;
+            lookup = token_to_rstr(parser_must_consume(p, TOKEN_IDENTIFIER));
+
+            if (has_space_affix)
+                rb_enc_raise(utf8_encoding, cLiquidSyntaxError, "Unexpected dot");
+
+            if (rstring_eq(lookup, "size") || rstring_eq(lookup, "first") || rstring_eq(lookup, "last"))
+                command_flags |= 1 << RARRAY_LEN(lookups);
+
+            rb_ary_push(lookups, lookup);
+        } else {
+            break;
+        }
+    }
+
+    if (RARRAY_LEN(lookups) == 0 && TYPE(name) == T_STRING) {
+        if (rstring_eq(name, "nil") || rstring_eq(name, "null")) return Qnil;
+        if (rstring_eq(name, "true")) return Qtrue;
+        if (rstring_eq(name, "false")) return Qfalse;
+        if (rstring_eq(name, "blank")) return symBlank;
+        if (rstring_eq(name, "empty")) return symEmpty;
+    }
+
+    VALUE args[4] = {Qfalse, name, lookups, INT2FIX(command_flags)};
+    return rb_class_new_instance(4, args, cLiquidVariableLookup);
+}
+
+VALUE parse_expression(parser_t *p)
+{
+    switch (p->cur.type) {
+        case TOKEN_IDENTIFIER:
+        case TOKEN_OPEN_SQUARE:
+            return parse_variable(p);
+
+        case TOKEN_NUMBER:
+            return parse_number(p);
+
+        case TOKEN_OPEN_ROUND:
+            return parse_range(p);
+
+        case TOKEN_STRING:
+        {
+            lexer_token_t token = parser_consume_any(p);
+            token.val++;
+            token.val_end--;
+            return token_to_rstr(token);
+        }
+    }
+
+    if (p->cur.type == TOKEN_EOS) {
+        rb_enc_raise(utf8_encoding, cLiquidSyntaxError, "[:%s] is not a valid expression", symbol_names[p->cur.type]);
+    } else {
+        rb_enc_raise(utf8_encoding, cLiquidSyntaxError, "[:%s, \"%.*s\"] is not a valid expression",
+                 symbol_names[p->cur.type], (int)(p->cur.val_end - p->cur.val), p->cur.val);
+    }
+    return Qnil;
+}
+
+static VALUE rb_parse_expression(VALUE self, VALUE markup)
+{
+    StringValue(markup);
+    char *start = RSTRING_PTR(markup);
+
+    parser_t p;
+    init_parser(&p, start, start + RSTRING_LEN(markup));
+
+    if (p.cur.type == TOKEN_EOS)
+        return Qnil;
+
+    VALUE expr = parse_expression(&p);
+
+    if (p.cur.type != TOKEN_EOS)
+        rb_enc_raise(utf8_encoding, cLiquidSyntaxError, "[:%s] is not a valid expression", symbol_names[p.cur.type]);
+
+    return expr;
+}
+
+void init_liquid_parser(void)
+{
+    idToI = rb_intern("to_i");
+    idEvaluate = rb_intern("evaluate");
+    symBlank = ID2SYM(rb_intern("blank?"));
+    symEmpty = ID2SYM(rb_intern("empty?"));
+
+    cLiquidRangeLookup = rb_const_get(mLiquid, rb_intern("RangeLookup"));
+    cRange = rb_const_get(rb_cObject, rb_intern("Range"));
+    cLiquidVariableLookup = rb_const_get(mLiquid, rb_intern("VariableLookup"));
+
+    VALUE cLiquidExpression = rb_const_get(mLiquid, rb_intern("Expression"));
+    rb_define_singleton_method(cLiquidExpression, "c_parse", rb_parse_expression, 1);
+}
+

data/ext/liquid_c/parser.h

@@ -0,0 +1,22 @@
+#if !defined(LIQUID_PARSER_H)
+#define LIQUID_PARSER_H
+
+#include "lexer.h"
+
+typedef struct parser {
+    lexer_token_t cur, next;
+    const char *str, *str_end;
+} parser_t;
+
+void init_parser(parser_t *parser, const char *str, const char *end);
+
+lexer_token_t parser_must_consume(parser_t *parser, unsigned char type);
+lexer_token_t parser_consume(parser_t *parser, unsigned char type);
+lexer_token_t parser_consume_any(parser_t *parser);
+
+VALUE parse_expression(parser_t *parser);
+
+void init_liquid_parser(void);
+
+#endif
+

data/ext/liquid_c/tokenizer.c

@@ -62,7 +62,7 @@ void tokenizer_next(tokenizer_t *tokenizer, token_t *token)
     const char *last = cursor + tokenizer->length - 1;
 
     token->str = cursor;
-    token->type = TOKEN_STRING;
+    token->type = TOKEN_RAW;
 
     while (cursor < last) {
         if (*cursor++ != '{')
@@ -72,7 +72,7 @@ void tokenizer_next(tokenizer_t *tokenizer, token_t *token)
         if (c != '%' && c != '{')
             continue;
         if (cursor - tokenizer->cursor > 2) {
-            token->type = TOKEN_STRING;
+            token->type = TOKEN_RAW;
             cursor -= 2;
             goto found;
         }
@@ -123,7 +123,7 @@ static VALUE tokenizer_shift_method(VALUE self)
 
     token_t token;
     tokenizer_next(tokenizer, &token);
-    if (token.type == TOKEN_NONE)
+    if (!token.type)
         return Qnil;
 
     return rb_enc_str_new(token.str, token.length, utf8_encoding);
@@ -136,3 +136,4 @@ void init_liquid_tokenizer()
     rb_define_method(cLiquidTokenizer, "initialize", tokenizer_initialize_method, 1);
     rb_define_method(cLiquidTokenizer, "shift", tokenizer_shift_method, 0);
 }
+

data/ext/liquid_c/tokenizer.h

@@ -2,9 +2,9 @@
 #define LIQUID_TOKENIZER_H
 
 enum token_type {
-    TOKEN_NONE,
+    TOKENIZER_TOKEN_NONE = 0,
     TOKEN_INVALID,
-    TOKEN_STRING,
+    TOKEN_RAW,
     TOKEN_TAG,
     TOKEN_VARIABLE
 };
@@ -29,3 +29,4 @@ void init_liquid_tokenizer();
 void tokenizer_next(tokenizer_t *tokenizer, token_t *token);
 
 #endif
+

data/ext/liquid_c/variable.c

@@ -0,0 +1,54 @@
+#include "liquid.h"
+#include "variable.h"
+#include "parser.h"
+#include <stdio.h>
+
+static VALUE rb_variable_parse(VALUE self, VALUE markup, VALUE filters)
+{
+    StringValue(markup);
+    char *start = RSTRING_PTR(markup);
+
+    parser_t p;
+    init_parser(&p, start, start + RSTRING_LEN(markup));
+
+    if (p.cur.type == TOKEN_EOS)
+        return Qnil;
+
+    VALUE name = parse_expression(&p);
+
+    while (parser_consume(&p, TOKEN_PIPE).type) {
+        lexer_token_t filter_name = parser_must_consume(&p, TOKEN_IDENTIFIER);
+
+        VALUE filter_args = rb_ary_new(), keyword_args = Qnil, filter;
+
+        if (parser_consume(&p, TOKEN_COLON).type) {
+            do {
+                if (p.cur.type == TOKEN_IDENTIFIER && p.next.type == TOKEN_COLON) {
+                    VALUE key = token_to_rstr(parser_consume_any(&p));
+                    parser_consume_any(&p);
+
+                    if (keyword_args == Qnil) keyword_args = rb_hash_new();
+                    rb_hash_aset(keyword_args, key, parse_expression(&p));
+                } else {
+                    rb_ary_push(filter_args, parse_expression(&p));
+                }
+            } while (parser_consume(&p, TOKEN_COMMA).type);
+        }
+
+        if (keyword_args == Qnil) {
+            filter = rb_ary_new3(2, token_to_rstr(filter_name), filter_args);
+        } else {
+            filter = rb_ary_new3(3, token_to_rstr(filter_name), filter_args, keyword_args);
+        }
+        rb_ary_push(filters, filter);
+    }
+
+    parser_must_consume(&p, TOKEN_EOS);
+    return name;
+}
+
+void init_liquid_variable(void)
+{
+    rb_define_singleton_method(cLiquidVariable, "c_strict_parse", rb_variable_parse, 2);
+}
+

data/ext/liquid_c/variable.h

@@ -0,0 +1,7 @@
+#if !defined(LIQUID_VARIABLE_H)
+#define LIQUID_VARIABLE_H
+
+void init_liquid_variable(void);
+
+#endif
+

data/lib/liquid/c.rb

@@ -2,17 +2,98 @@ require 'liquid/c/version'
 require 'liquid'
 require 'liquid_c'
 
+module Liquid
+  module C
+    @enabled = true
+
+    class << self
+      attr_accessor :enabled
+    end
+  end
+end
+
 Liquid::Template.class_eval do
   private
 
   alias_method :ruby_tokenize, :tokenize
 
   def tokenize(source)
-    if @line_numbers
+    if Liquid::C.enabled && !@line_numbers
+      Liquid::Tokenizer.new(source.to_s)
+    else
       ruby_tokenize(source)
+    end
+  end
+end
+
+Liquid::BlockBody.class_eval do
+  alias_method :ruby_parse, :parse
+
+  def parse(tokens, options)
+    if Liquid::C.enabled && !options[:line_numbers] && !options[:profile]
+      c_parse(tokens, options) { |t, m| yield t, m }
     else
-      Liquid::Tokenizer.new(source.to_s)
+      ruby_parse(tokens, options) { |t, m| yield t, m }
     end
   end
 end
 
+Liquid::Variable.class_eval do
+  alias_method :ruby_lax_parse, :lax_parse
+  alias_method :ruby_strict_parse, :strict_parse
+
+  def lax_parse(markup)
+    stats = @options[:stats_callbacks]
+    stats[:variable_parse].call if stats
+
+    if Liquid::C.enabled
+      begin
+        return strict_parse(markup)
+      rescue Liquid::SyntaxError
+        stats[:variable_fallback].call if stats
+      end
+    end
+
+    ruby_lax_parse(markup)
+  end
+
+  def strict_parse(markup)
+    if Liquid::C.enabled
+      @name = Liquid::Variable.c_strict_parse(markup, @filters = [])
+    else
+      ruby_strict_parse(markup)
+    end
+  end
+end
+
+Liquid::VariableLookup.class_eval do
+  alias_method :ruby_initialize, :initialize
+
+  def initialize(markup, name = nil, lookups = nil, command_flags = nil)
+    if Liquid::C.enabled && markup == false
+      @name = name
+      @lookups = lookups
+      @command_flags = command_flags
+    else
+      ruby_initialize(markup)
+    end
+  end
+end
+
+Liquid::Expression.class_eval do
+  class << self
+    alias_method :ruby_parse, :parse
+
+    def parse(markup)
+      return nil unless markup
+
+      if Liquid::C.enabled
+        begin
+          return c_parse(markup)
+        rescue Liquid::SyntaxError
+        end
+      end
+      ruby_parse(markup)
+    end
+  end
+end

data/lib/liquid/c/version.rb

@@ -1,5 +1,5 @@
 module Liquid
   module C
-    VERSION = "0.0.2"
+    VERSION = "0.0.3"
   end
 end

data/test/unit/variable_test.rb

@@ -0,0 +1,106 @@
+# encoding: utf-8
+require 'test_helper'
+
+class VariableTest < MiniTest::Unit::TestCase
+  def test_variable_parse
+    assert_equal [lookup('hello'), []], variable_parse('hello')
+    assert_equal ['world', []], variable_parse(' "world" ')
+    assert_equal [lookup('hello["world"]'), []], variable_parse(' hello["world"] ')
+    assert_equal [nil, []], variable_parse('')
+    assert_equal [lookup('question?'), []], variable_parse('question?')
+    assert_equal [lookup('[meta]'), []], variable_parse('[meta]')
+    assert_equal [lookup('a-b'), []], variable_parse('a-b')
+    assert_equal [lookup('a-2'), []], variable_parse('a-2')
+  end
+
+  def test_strictness
+    assert_raises(Liquid::SyntaxError) { variable_parse(' hello["world\']" ') }
+    assert_raises(Liquid::SyntaxError) { variable_parse('-..') }
+    assert_raises(Liquid::SyntaxError) { variable_parse('question?mark') }
+    assert_raises(Liquid::SyntaxError) { variable_parse('123.foo') }
+    assert_raises(Liquid::SyntaxError) { variable_parse(' | nothing') }
+
+    ['a .b', 'a. b', 'a . b'].each do |var|
+      assert_raises(Liquid::SyntaxError) { variable_parse(var) }
+    end
+
+    ['a -b', 'a- b', 'a - b'].each do |var|
+      assert_raises(Liquid::SyntaxError) { variable_parse(var) }
+    end
+  end
+
+  def test_literals
+    assert_equal [true, []], variable_parse('true')
+    assert_equal [nil, []], variable_parse('nil')
+    assert_equal [123.4, []], variable_parse('123.4')
+
+    assert_equal [lookup('[blank]'), []], variable_parse('[blank]')
+    assert_equal [lookup(false, true, [:blank?], 0), []], variable_parse('[true][blank]')
+    assert_equal [lookup('[true][blank]'), []], variable_parse('[true][blank]')
+    assert_equal [lookup('x["size"]'), []], variable_parse('x["size"]')
+  end
+
+  def test_variable_filter
+    name = lookup('name')
+    assert_equal [name, [['filter', []]]], variable_parse(' name | filter ')
+    assert_equal [name, [['filter1', []], ['filter2', []]]], variable_parse(' name | filter1 | filter2 ')
+  end
+
+  def test_variable_filter_args
+    name = lookup('name')
+    abc = lookup('abc')
+
+    assert_equal [name, [['filter', [abc]]]], variable_parse(' name | filter: abc ')
+
+    assert_equal [name, [['filter1', [abc]], ['filter2', [abc]]]],
+      variable_parse(' name | filter1: abc | filter2: abc ')
+
+    assert_equal [name, [['filter', [lookup('a')], {'b' => lookup('c'), 'd' => lookup('e')}]]],
+      variable_parse('name | filter : a , b : c , d : e')
+
+    assert_raises Liquid::SyntaxError do
+      variable_parse('name | filter : a : b : c : d : e')
+    end
+  end
+
+  def test_unicode_strings
+    assert_equal ['å߀êùｉｄｈｔлｓԁѵ߀ｒáƙìｓｔɦｅƅêｓｔｐｃｍáѕｔｅｒｒãｃêｃհèｒｒ', []],
+      variable_parse('"å߀êùｉｄｈｔлｓԁѵ߀ｒáƙìｓｔɦｅƅêｓｔｐｃｍáѕｔｅｒｒãｃêｃհèｒｒ"')
+  end
+
+  def test_broken_unicode_errors
+    err = assert_raises(Liquid::SyntaxError) do
+      Liquid::Template.parse("test {{ \xC2\xA0 test }}", error_mode: :strict)
+    end
+    assert err.message
+  end
+
+  def test_callbacks
+    variable_parses = 0
+    variable_fallbacks = 0
+
+    callbacks = {
+      variable_parse: lambda { variable_parses += 1 },
+      variable_fallback: lambda { variable_fallbacks += 1 }
+    }
+
+    Liquid::Variable.new('abc', error_mode: :lax, stats_callbacks: callbacks)
+    assert_equal 1, variable_parses
+    assert_equal 0, variable_fallbacks
+
+    Liquid::Variable.new('@!#', error_mode: :lax, stats_callbacks: callbacks)
+    assert_equal 2, variable_parses
+    assert_equal 1, variable_fallbacks
+  end
+
+  private
+
+  def variable_parse(markup)
+    name = Liquid::Variable.c_strict_parse(markup, filters = [])
+    [name, filters]
+  end
+
+  def lookup(*args)
+    Liquid::VariableLookup.new(*args)
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: liquid-c
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - Dylan Thacker-Smith
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-10-08 00:00:00.000000000 Z
+date: 2014-12-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: liquid
@@ -94,11 +94,19 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- ext/liquid_c/block.c
+- ext/liquid_c/block.h
 - ext/liquid_c/extconf.rb
+- ext/liquid_c/lexer.c
+- ext/liquid_c/lexer.h
 - ext/liquid_c/liquid.c
 - ext/liquid_c/liquid.h
+- ext/liquid_c/parser.c
+- ext/liquid_c/parser.h
 - ext/liquid_c/tokenizer.c
 - ext/liquid_c/tokenizer.h
+- ext/liquid_c/variable.c
+- ext/liquid_c/variable.h
 - lib/liquid/c.rb
 - lib/liquid/c/version.rb
 - liquid-c.gemspec
@@ -106,6 +114,7 @@ files:
 - test/liquid_test.rb
 - test/test_helper.rb
 - test/unit/tokenizer_test.rb
+- test/unit/variable_test.rb
 homepage: ''
 licenses:
 - MIT
@@ -126,7 +135,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.0
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: Liquid performance extension in C
@@ -134,3 +143,4 @@ test_files:
 - test/liquid_test.rb
 - test/test_helper.rb
 - test/unit/tokenizer_test.rb
+- test/unit/variable_test.rb