liquid-c 3.0.0 → 4.0.0.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f2140b7f3e49dc914ab2d474dee3bca94c6f5923
-  data.tar.gz: d0d2ef0d9a4ab5d291b560e9d35c3391a7b456d8
+  metadata.gz: c625a10bf29068a8d124df52ea012868d936f2ea
+  data.tar.gz: 537c2e49a4a6b65c693e7f51d2432d0f969dcb3f
 SHA512:
-  metadata.gz: b019264cea1628bbdb4194685d660a378107ce68561a549324d7171f088491219fa0b95970175cb9aa7078e68c2603c58dde78a5e61beac03b0cd49d2dc8082e
-  data.tar.gz: ddf7b182c306787f2c31451b3af77f6dae6b22c83c79fddfbaba5e4243a1f54c3419607f3e4629a24fd13bc79ea1d462ed1c07eb77e3424094bdaf57bddb19c0
+  metadata.gz: 5390b71dc89234cd139d6e3234473cc00c200e37380ba8a140c1e49c58fe1d7db68dd98ee79d8d023cf4994c9c7b472031504bde9b2bc27312f9f0bb04a56902
+  data.tar.gz: 99565b5eb0e6458afa0b9d7608b67160b1972892c6fa2bf2a4850eca80428b25f49743a57e9380ce6294448442f462109c917c7a702f647597a78d4443215402

data/.travis.yml

@@ -1,6 +1,12 @@
 language: ruby
+
 rvm:
-  - 2.0.0
-  - 2.1.0
-gemfile:
-  - Gemfile
+  - 2.0
+  - 2.1
+  - 2.2
+  - ruby-head
+
+sudo: false
+
+notifications:
+  disable: true

data/Gemfile

@@ -4,6 +4,7 @@ gemspec
 
 gem 'liquid', github: 'Shopify/liquid', branch: 'master'
 
+
 group :test do
   gem 'spy', '0.4.1'
   gem 'benchmark-ips'

data/ext/liquid_c/block.c

@@ -9,9 +9,10 @@ static ID
     intern_blank,
     intern_is_blank,
     intern_clear,
-    intern_tags,
+    intern_registered_tags,
     intern_parse,
-    intern_square_brackets;
+    intern_square_brackets,
+    intern_set_line_number;
 
 static int is_id(int c)
 {
@@ -24,6 +25,14 @@ inline static const char *read_while(const char *start, const char *end, int (fu
     return start;
 }
 
+inline static const char *read_while_end(const char *start, const char *end, int (func)(int))
+{
+    end--;
+    while (start < end && func((unsigned char) *end)) end--;
+    end++;
+    return end;
+}
+
 static VALUE rb_block_parse(VALUE self, VALUE tokens, VALUE options)
 {
     tokenizer_t *tokenizer;
@@ -34,6 +43,9 @@ static VALUE rb_block_parse(VALUE self, VALUE tokens, VALUE options)
     VALUE nodelist = rb_ivar_get(self, intern_nodelist);
 
     while (true) {
+        if (tokenizer->line_number != 0) {
+            rb_funcall(options, intern_set_line_number, 1, UINT2NUM(tokenizer->line_number));
+        }
         tokenizer_next(tokenizer, &token);
 
         switch (token.type) {
@@ -51,7 +63,14 @@ static VALUE rb_block_parse(VALUE self, VALUE tokens, VALUE options)
             }
             case TOKEN_RAW:
             {
-                VALUE str = rb_enc_str_new(token.str, token.length, utf8_encoding);
+                const char *start = token.str, *end = token.str + token.length, *token_start = start, *token_end = end;
+
+                if(token.lstrip)
+                    token_start = read_while(start, end, rb_isspace);
+                if(token.rstrip)
+                    token_end = read_while_end(token_start, end, rb_isspace);
+                    
+                VALUE str = rb_enc_str_new(token_start, token_end - token_start, utf8_encoding);
                 rb_ary_push(nodelist, str);
 
                 if (rb_ivar_get(self, intern_blank) == Qtrue) {
@@ -64,7 +83,7 @@ static VALUE rb_block_parse(VALUE self, VALUE tokens, VALUE options)
             }
             case TOKEN_VARIABLE:
             {
-                VALUE args[2] = {rb_enc_str_new(token.str + 2, token.length - 4, utf8_encoding), options};
+                VALUE args[2] = {rb_enc_str_new(token.str + 2 + token.lstrip, token.length - 4 - token.lstrip - token.rstrip, utf8_encoding), options};
                 VALUE var = rb_class_new_instance(2, args, cLiquidVariable);
                 rb_ary_push(nodelist, var);
                 rb_ivar_set(self, intern_blank, Qfalse);
@@ -72,7 +91,7 @@ static VALUE rb_block_parse(VALUE self, VALUE tokens, VALUE options)
             }
             case TOKEN_TAG:
             {
-                const char *start = token.str + 2, *end = token.str + token.length - 2;
+                const char *start = token.str + 2 + token.lstrip, *end = token.str + token.length - 2 - token.rstrip;
 
                 // Imitate \s*(\w+)\s*(.*)? regex
                 const char *name_start = read_while(start, end, rb_isspace);
@@ -81,7 +100,7 @@ static VALUE rb_block_parse(VALUE self, VALUE tokens, VALUE options)
                 VALUE tag_name = rb_enc_str_new(name_start, name_end - name_start, utf8_encoding);
 
                 if (tags == Qnil)
-                    tags = rb_funcall(cLiquidTemplate, intern_tags, 0);
+                    tags = rb_funcall(self, intern_registered_tags, 0);
 
                 VALUE tag_class = rb_funcall(tags, intern_square_brackets, 1, tag_name);
 
@@ -112,9 +131,10 @@ void init_liquid_block()
     intern_blank = rb_intern("@blank");
     intern_is_blank = rb_intern("blank?");
     intern_clear = rb_intern("clear");
-    intern_tags = rb_intern("tags");
+    intern_registered_tags = rb_intern("registered_tags");
     intern_parse = rb_intern("parse");
     intern_square_brackets = rb_intern("[]");
+    intern_set_line_number = rb_intern("line_number=");
 
     VALUE cLiquidBlockBody = rb_const_get(mLiquid, rb_intern("BlockBody"));
     rb_define_method(cLiquidBlockBody, "c_parse", rb_block_parse, 2);

data/ext/liquid_c/liquid.c

@@ -5,13 +5,14 @@
 #include "parser.h"
 #include "block.h"
 
-VALUE mLiquid, cLiquidSyntaxError, cLiquidVariable, cLiquidTemplate;
+VALUE mLiquid, mLiquidC, cLiquidSyntaxError, cLiquidVariable, cLiquidTemplate;
 rb_encoding *utf8_encoding;
 
 void Init_liquid_c(void)
 {
     utf8_encoding = rb_utf8_encoding();
     mLiquid = rb_define_module("Liquid");
+    mLiquidC = rb_define_module_under(mLiquid, "C");
     cLiquidSyntaxError = rb_const_get(mLiquid, rb_intern("SyntaxError"));
     cLiquidVariable = rb_const_get(mLiquid, rb_intern("Variable"));
     cLiquidTemplate = rb_const_get(mLiquid, rb_intern("Template"));

data/ext/liquid_c/liquid.h

@@ -5,7 +5,7 @@
 #include <ruby/encoding.h>
 #include <stdbool.h>
 
-extern VALUE mLiquid, cLiquidSyntaxError, cLiquidVariable, cLiquidTemplate;
+extern VALUE mLiquid, mLiquidC, cLiquidSyntaxError, cLiquidVariable, cLiquidTemplate;
 extern rb_encoding *utf8_encoding;
 
 #endif

data/ext/liquid_c/parser.c

@@ -2,7 +2,7 @@
 #include "parser.h"
 #include "lexer.h"
 
-static VALUE cLiquidRangeLookup, cLiquidVariableLookup, cRange, symBlank, symEmpty;
+static VALUE cLiquidRangeLookup, cLiquidVariableLookup, cRange, vLiquidExpressionLiterals;
 static ID idToI, idEvaluate;
 
 void init_parser(parser_t *p, const char *str, const char *end)
@@ -119,12 +119,10 @@ static VALUE parse_variable(parser_t *p)
         }
     }
 
-    if (RARRAY_LEN(lookups) == 0 && TYPE(name) == T_STRING) {
-        if (rstring_eq(name, "nil") || rstring_eq(name, "null")) return Qnil;
-        if (rstring_eq(name, "true")) return Qtrue;
-        if (rstring_eq(name, "false")) return Qfalse;
-        if (rstring_eq(name, "blank")) return symBlank;
-        if (rstring_eq(name, "empty")) return symEmpty;
+    if (RARRAY_LEN(lookups) == 0) {
+        VALUE undefined = FIXNUM_P(-1);
+        VALUE literal = rb_hash_lookup2(vLiquidExpressionLiterals, name, undefined);
+        if (literal != undefined) return literal;
     }
 
     VALUE args[4] = {Qfalse, name, lookups, INT2FIX(command_flags)};
@@ -185,8 +183,6 @@ void init_liquid_parser(void)
 {
     idToI = rb_intern("to_i");
     idEvaluate = rb_intern("evaluate");
-    symBlank = ID2SYM(rb_intern("blank?"));
-    symEmpty = ID2SYM(rb_intern("empty?"));
 
     cLiquidRangeLookup = rb_const_get(mLiquid, rb_intern("RangeLookup"));
     cRange = rb_const_get(rb_cObject, rb_intern("Range"));
@@ -194,5 +190,7 @@ void init_liquid_parser(void)
 
     VALUE cLiquidExpression = rb_const_get(mLiquid, rb_intern("Expression"));
     rb_define_singleton_method(cLiquidExpression, "c_parse", rb_parse_expression, 1);
+
+    vLiquidExpressionLiterals = rb_const_get(cLiquidExpression, rb_intern("LITERALS"));
 }
 

data/ext/liquid_c/tokenizer.c

@@ -38,7 +38,7 @@ static VALUE tokenizer_allocate(VALUE klass)
     return obj;
 }
 
-static VALUE tokenizer_initialize_method(VALUE self, VALUE source)
+static VALUE tokenizer_initialize_method(VALUE self, VALUE source, VALUE line_numbers)
 {
     tokenizer_t *tokenizer;
 
@@ -48,6 +48,10 @@ static VALUE tokenizer_initialize_method(VALUE self, VALUE source)
     tokenizer->source = source;
     tokenizer->cursor = RSTRING_PTR(source);
     tokenizer->length = RSTRING_LEN(source);
+    tokenizer->lstrip_flag = 0;
+    // tokenizer->line_number keeps track of the current line number or it is 0
+    // to indicate that line numbers aren't being calculated
+    tokenizer->line_number = RTEST(line_numbers) ? 1 : 0;
     return Qnil;
 }
 
@@ -63,6 +67,8 @@ void tokenizer_next(tokenizer_t *tokenizer, token_t *token)
 
     token->str = cursor;
     token->type = TOKEN_RAW;
+    token->lstrip = 0;
+    token->rstrip = 0;
 
     while (cursor < last) {
         if (*cursor++ != '{')
@@ -71,12 +77,20 @@ void tokenizer_next(tokenizer_t *tokenizer, token_t *token)
         char c = *cursor++;
         if (c != '%' && c != '{')
             continue;
-        if (cursor - tokenizer->cursor > 2) {
+        if (cursor <= last && *cursor == '-') {
+          cursor++;
+          token->rstrip = 1;
+        }
+        if (cursor - tokenizer->cursor > (ptrdiff_t)(2 + token->rstrip)) {
             token->type = TOKEN_RAW;
-            cursor -= 2;
+            cursor -= 2 + token->rstrip;
+            token->lstrip = tokenizer->lstrip_flag;
+            tokenizer->lstrip_flag = 0;
             goto found;
         }
         token->type = TOKEN_INVALID;
+        token->lstrip = token->rstrip;
+        token->rstrip = 0;
         if (c == '%') {
             while (cursor < last) {
                 if (*cursor++ != '%')
@@ -87,10 +101,13 @@ void tokenizer_next(tokenizer_t *tokenizer, token_t *token)
                 if (c != '}')
                     continue;
                 token->type = TOKEN_TAG;
+                if(cursor[-3] == '-')
+                    token->rstrip = tokenizer->lstrip_flag = 1;
                 goto found;
             }
             // unterminated tag
             cursor = tokenizer->cursor + 2;
+            tokenizer->lstrip_flag = 0;
             goto found;
         } else {
             while (cursor < last) {
@@ -102,18 +119,33 @@ void tokenizer_next(tokenizer_t *tokenizer, token_t *token)
                     goto found;
                 }
                 token->type = TOKEN_VARIABLE;
+                if(cursor[-3] == '-')
+                    token->rstrip = tokenizer->lstrip_flag = 1;
                 goto found;
             }
             // unterminated variable
             cursor = tokenizer->cursor + 2;
+            tokenizer->lstrip_flag = 0;
             goto found;
         }
     }
     cursor = last + 1;
+    token->lstrip = tokenizer->lstrip_flag;
+    tokenizer->lstrip_flag = 0;
 found:
     token->length = cursor - tokenizer->cursor;
     tokenizer->cursor += token->length;
     tokenizer->length -= token->length;
+
+    if (tokenizer->line_number) {
+        const char *cursor = token->str;
+        const char *end = token->str + token->length;
+        while (cursor < end) {
+            if (*cursor == '\n')
+                tokenizer->line_number++;
+            cursor++;
+        }
+    }
 }
 
 static VALUE tokenizer_shift_method(VALUE self)
@@ -129,11 +161,23 @@ static VALUE tokenizer_shift_method(VALUE self)
     return rb_enc_str_new(token.str, token.length, utf8_encoding);
 }
 
+static VALUE tokenizer_line_number_method(VALUE self)
+{
+    tokenizer_t *tokenizer;
+    Tokenizer_Get_Struct(self, tokenizer);
+
+    if (tokenizer->line_number == 0)
+        return Qnil;
+
+    return UINT2NUM(tokenizer->line_number);
+}
+
 void init_liquid_tokenizer()
 {
-    cLiquidTokenizer = rb_define_class_under(mLiquid, "Tokenizer", rb_cObject);
+    cLiquidTokenizer = rb_define_class_under(mLiquidC, "Tokenizer", rb_cObject);
     rb_define_alloc_func(cLiquidTokenizer, tokenizer_allocate);
-    rb_define_method(cLiquidTokenizer, "initialize", tokenizer_initialize_method, 1);
+    rb_define_method(cLiquidTokenizer, "initialize", tokenizer_initialize_method, 2);
     rb_define_method(cLiquidTokenizer, "shift", tokenizer_shift_method, 0);
+    rb_define_method(cLiquidTokenizer, "line_number", tokenizer_line_number_method, 0);
 }
 

data/ext/liquid_c/tokenizer.h

@@ -13,12 +13,16 @@ typedef struct token {
     enum token_type type;
     const char *str;
     long length;
+    unsigned int lstrip;
+    unsigned int rstrip;
 } token_t;
 
 typedef struct tokenizer {
     VALUE source;
     const char *cursor;
     long length;
+    unsigned int line_number;
+    unsigned int lstrip_flag;
 } tokenizer_t;
 
 extern VALUE cLiquidTokenizer;

data/lib/liquid/c.rb

@@ -12,16 +12,12 @@ module Liquid
   end
 end
 
-Liquid::Template.class_eval do
-  private
-
-  alias_method :ruby_tokenize, :tokenize
-
-  def tokenize(source)
-    if Liquid::C.enabled && !@line_numbers
-      Liquid::Tokenizer.new(source.to_s)
+Liquid::Tokenizer.class_eval do
+  def self.new(source, line_numbers = false)
+    if Liquid::C.enabled
+      Liquid::C::Tokenizer.new(source.to_s, line_numbers)
     else
-      ruby_tokenize(source)
+      super
     end
   end
 end
@@ -30,7 +26,7 @@ Liquid::BlockBody.class_eval do
   alias_method :ruby_parse, :parse
 
   def parse(tokens, options)
-    if Liquid::C.enabled && !options[:line_numbers] && !options[:profile]
+    if Liquid::C.enabled && !options[:profile]
       c_parse(tokens, options) { |t, m| yield t, m }
     else
       ruby_parse(tokens, options) { |t, m| yield t, m }
@@ -43,7 +39,7 @@ Liquid::Variable.class_eval do
   alias_method :ruby_strict_parse, :strict_parse
 
   def lax_parse(markup)
-    stats = @options[:stats_callbacks]
+    stats = options[:stats_callbacks]
     stats[:variable_parse].call if stats
 
     if Liquid::C.enabled

data/lib/liquid/c/version.rb

@@ -1,5 +1,5 @@
 module Liquid
   module C
-    VERSION = "3.0.0"
+    VERSION = "4.0.0.rc1"
   end
 end

data/liquid-c.gemspec

@@ -7,7 +7,7 @@ Gem::Specification.new do |spec|
   spec.name          = "liquid-c"
   spec.version       = Liquid::C::VERSION
   spec.authors       = ["Justin Li", "Dylan Thacker-Smith"]
-  spec.email         = ["jli@shopify.com", "Dylan.Smith@shopify.com"]
+  spec.email         = ["gems@shopify.com"]
   spec.summary       = "Liquid performance extension in C"
   spec.homepage      = ""
   spec.license       = "MIT"
@@ -23,5 +23,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "bundler", "~> 1.5"
   spec.add_development_dependency "rake"
   spec.add_development_dependency 'rake-compiler'
+  spec.add_development_dependency 'minitest'
   spec.add_development_dependency 'stackprof' if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new("2.1.0")
 end

data/test/unit/tokenizer_test.rb

@@ -33,7 +33,7 @@ class TokenizerTest < MiniTest::Unit::TestCase
   private
 
   def tokenize(source)
-    tokenizer = Liquid::Tokenizer.new(source)
+    tokenizer = Liquid::C::Tokenizer.new(source, false)
     tokens = []
     while t = tokenizer.shift
       tokens << t

data/test/unit/variable_test.rb

@@ -35,7 +35,7 @@ class VariableTest < MiniTest::Unit::TestCase
     assert_equal [123.4, []], variable_parse('123.4')
 
     assert_equal [lookup('[blank]'), []], variable_parse('[blank]')
-    assert_equal [lookup(false, true, [:blank?], 0), []], variable_parse('[true][blank]')
+    assert_equal [lookup(false, true, [Liquid::Expression::LITERALS['blank']], 0), []], variable_parse('[true][blank]')
     assert_equal [lookup('[true][blank]'), []], variable_parse('[true][blank]')
     assert_equal [lookup('x["size"]'), []], variable_parse('x["size"]')
   end
@@ -84,17 +84,21 @@ class VariableTest < MiniTest::Unit::TestCase
       variable_fallback: lambda { variable_fallbacks += 1 }
     }
 
-    Liquid::Variable.new('abc', error_mode: :lax, stats_callbacks: callbacks)
+    create_variable('abc', error_mode: :lax, stats_callbacks: callbacks)
     assert_equal 1, variable_parses
     assert_equal 0, variable_fallbacks
 
-    Liquid::Variable.new('@!#', error_mode: :lax, stats_callbacks: callbacks)
+    create_variable('@!#', error_mode: :lax, stats_callbacks: callbacks)
     assert_equal 2, variable_parses
     assert_equal 1, variable_fallbacks
   end
 
   private
 
+  def create_variable(markup, options={})
+    Liquid::Variable.new(markup, Liquid::ParseContext.new(options))
+  end
+
   def variable_parse(markup)
     name = Liquid::Variable.c_strict_parse(markup, filters = [])
     [name, filters]

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: liquid-c
 version: !ruby/object:Gem::Version
-  version: 3.0.0
+  version: 4.0.0.rc1
 platform: ruby
 authors:
 - Justin Li
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-01-31 00:00:00.000000000 Z
+date: 2016-09-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: liquid
@@ -67,6 +67,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: stackprof
   requirement: !ruby/object:Gem::Requirement
@@ -83,8 +97,7 @@ dependencies:
         version: '0'
 description: 
 email:
-- jli@shopify.com
-- Dylan.Smith@shopify.com
+- gems@shopify.com
 executables: []
 extensions:
 - ext/liquid_c/extconf.rb
@@ -132,12 +145,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.4.5
 signing_key: 
 specification_version: 4
 summary: Liquid performance extension in C