liquid-c 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 2431d1b61b78e144d34a423a60c5463fb8c21a87
+  data.tar.gz: 7cf1e00087a7c80cb6b19d099756149d3d136155
+SHA512:
+  metadata.gz: 29ebcbdcc043203d454b979ae8b136263b14d8dd885ea0556d699c7aa006375798eb408c8db048258869b91bd1cf6e519f37a1fc6609f590c078bbea15893c9d
+  data.tar.gz: 3b2628f520d13b1b487da6d473c9357b631cb860bdf74de3d288d3345584a954b2eaa71ef221d8aac4e21b5474be3d3570c709e7e846afdcbeeb8031ea7ad24d

data/.gitignore

@@ -0,0 +1,8 @@
+*.gem
+*.rbc
+Gemfile.lock
+pkg
+tmp
+*.o
+*.bundle
+ext/*/Makefile

data/.travis.yml

@@ -0,0 +1,7 @@
+language: ruby
+rvm:
+  - 1.9.3
+  - 2.0.0
+  - 2.1.0
+gemfile:
+  - Gemfile

data/Gemfile

@@ -0,0 +1,5 @@
+source 'https://rubygems.org'
+
+gemspec
+
+gem 'liquid', github: 'Shopify/liquid', branch: 'master'

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Shopify
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,36 @@
+# Liquid::C
+[![Build Status](https://api.travis-ci.org/Shopify/liquid-c.png?branch=master)](https://travis-ci.org/Shopify/liquid-c)
+
+Partial native implementation of the liquid ruby gem in C.
+
+## Installation
+
+Add these lines to your application's Gemfile:
+
+    gem 'liquid', github: 'Shopify/liquid', branch: 'master'
+    gem 'liquid-c', github: 'Shopify/liquid-c', branch: 'master'
+
+And then execute:
+
+    $ bundle
+
+## Usage
+
+    require 'liquid/c'
+
+then just use the documented API for the liquid Gem.
+
+## Restrictions
+
+* Input strings are assumed to be UTF-8 encoded strings
+* Tag#parse(tokens) is given a Liquid::Tokenizer object, instead
+  of an array of strings, which only implements the shift method
+  to get the next token.
+
+## Contributing
+
+1. Fork it ( http://github.com/Shopify/liquid-c/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,57 @@
+require 'rake'
+require 'rake/testtask'
+require 'bundler/gem_tasks'
+require 'rake/extensiontask'
+
+Rake::ExtensionTask.new("liquid_c")
+
+task :default => :test
+
+task :test => ['test:unit', 'test:liquid']
+
+namespace :test do
+  Rake::TestTask.new(:unit => :compile) do |t|
+    t.libs << 'lib' << 'test'
+    t.test_files = FileList['test/unit/**/*_test.rb']
+  end
+
+  desc 'run test suite with default parser'
+  Rake::TestTask.new(:base_liquid => :compile) do |t|
+    t.libs << 'lib'
+    t.test_files = ['test/liquid_test.rb']
+  end
+
+  desc 'runs test suite with both strict and lax parsers'
+  task :liquid do
+    ENV['LIQUID_PARSER_MODE'] = 'lax'
+    Rake::Task['test:base_liquid'].invoke
+    ENV['LIQUID_PARSER_MODE'] = 'strict'
+    Rake::Task['test:base_liquid'].reenable
+    Rake::Task['test:base_liquid'].invoke
+  end
+end
+
+namespace :benchmark do
+  desc "Run the liquid benchmark with lax parsing"
+  task :run do
+    ruby "./performance.rb benchmark lax"
+  end
+
+  desc "Run the liquid benchmark with strict parsing"
+  task :strict do
+    ruby "./performance.rb benchmark strict"
+  end
+end
+
+
+namespace :profile do
+  desc "Run the liquid profile/performance coverage"
+  task :run do
+    ruby "./performance.rb profile lax"
+  end
+
+  desc "Run the liquid profile/performance coverage with strict parsing"
+  task :strict do
+    ruby "./performance.rb profile strict"
+  end
+end

data/ext/liquid_c/extconf.rb

@@ -0,0 +1,4 @@
+require 'mkmf'
+$CFLAGS << ' -Wall -Werror'
+$warnflags.gsub!(/-Wdeclaration-after-statement/, "")
+create_makefile("liquid_c")

data/ext/liquid_c/liquid.c

@@ -0,0 +1,9 @@
+#include "liquid.h"
+
+VALUE mLiquid;
+
+void Init_liquid_c(void)
+{
+    mLiquid = rb_define_module("Liquid");
+    init_liquid_tokenizer();
+}

data/ext/liquid_c/liquid.h

@@ -0,0 +1,11 @@
+#ifndef LIQUID_H
+#define LIQUID_H
+
+#include <ruby.h>
+#include <stdbool.h>
+
+#include "tokenizer.h"
+
+extern VALUE mLiquid;
+
+#endif

data/ext/liquid_c/tokenizer.c

@@ -0,0 +1,138 @@
+#include "liquid.h"
+
+VALUE cLiquidTokenizer;
+rb_encoding *utf8_encoding;
+
+static void tokenizer_mark(void *ptr) {
+    tokenizer_t *tokenizer = ptr;
+    rb_gc_mark(tokenizer->source);
+}
+
+static void tokenizer_free(void *ptr)
+{
+    tokenizer_t *tokenizer = ptr;
+    xfree(tokenizer);
+}
+
+static size_t tokenizer_memsize(const void *ptr)
+{
+    return ptr ? sizeof(tokenizer_t) : 0;
+}
+
+const rb_data_type_t tokenizer_data_type = {
+    "liquid_tokenizer",
+    {tokenizer_mark, tokenizer_free, tokenizer_memsize,},
+#ifdef RUBY_TYPED_FREE_IMMEDIATELY
+    NULL, NULL, RUBY_TYPED_FREE_IMMEDIATELY
+#endif
+};
+
+static VALUE tokenizer_allocate(VALUE klass)
+{
+    VALUE obj;
+    tokenizer_t *tokenizer;
+
+    obj = TypedData_Make_Struct(klass, tokenizer_t, &tokenizer_data_type, tokenizer);
+    tokenizer->source = Qnil;
+    return obj;
+}
+
+static VALUE tokenizer_initialize_method(VALUE self, VALUE source)
+{
+    tokenizer_t *tokenizer;
+
+    Check_Type(source, T_STRING);
+    Tokenizer_Get_Struct(self, tokenizer);
+    source = rb_str_dup_frozen(source);
+    tokenizer->source = source;
+    tokenizer->cursor = RSTRING_PTR(source);
+    tokenizer->length = RSTRING_LEN(source);
+    return Qnil;
+}
+
+void tokenizer_next(tokenizer_t *tokenizer, token_t *token)
+{
+    if (tokenizer->length <= 0) {
+        memset(token, 0, sizeof(*token));
+        return;
+    }
+
+    const char *cursor = tokenizer->cursor;
+    const char *last = cursor + tokenizer->length - 1;
+
+    token->str = cursor;
+    token->type = TOKEN_STRING;
+
+    while (cursor < last) {
+        if (*cursor++ != '{')
+            continue;
+
+        char c = *cursor++;
+        if (c != '%' && c != '{')
+            continue;
+        if (cursor - tokenizer->cursor > 2) {
+            token->type = TOKEN_STRING;
+            cursor -= 2;
+            goto found;
+        }
+        token->type = TOKEN_INVALID;
+        if (c == '%') {
+            while (cursor < last) {
+                if (*cursor++ != '%')
+                    continue;
+                c = *cursor++;
+                while (c == '%' && cursor <= last)
+                    c = *cursor++;
+                if (c != '}')
+                    continue;
+                token->type = TOKEN_TAG;
+                goto found;
+            }
+            // unterminated tag
+            cursor = tokenizer->cursor + 2;
+            goto found;
+        } else {
+            while (cursor < last) {
+                if (*cursor++ != '}')
+                    continue;
+                if (*cursor++ != '}') {
+                    // variable incomplete end, used to end raw tags
+                    cursor--;
+                    goto found;
+                }
+                token->type = TOKEN_VARIABLE;
+                goto found;
+            }
+            // unterminated variable
+            cursor = tokenizer->cursor + 2;
+            goto found;
+        }
+    }
+    cursor = last + 1;
+found:
+    token->length = cursor - tokenizer->cursor;
+    tokenizer->cursor += token->length;
+    tokenizer->length -= token->length;
+}
+
+static VALUE tokenizer_shift_method(VALUE self)
+{
+    tokenizer_t *tokenizer;
+    Tokenizer_Get_Struct(self, tokenizer);
+
+    token_t token;
+    tokenizer_next(tokenizer, &token);
+    if (token.type == TOKEN_NONE)
+        return Qnil;
+
+    return rb_enc_str_new(token.str, token.length, utf8_encoding);
+}
+
+void init_liquid_tokenizer()
+{
+    cLiquidTokenizer = rb_define_class_under(mLiquid, "Tokenizer", rb_cObject);
+    rb_define_alloc_func(cLiquidTokenizer, tokenizer_allocate);
+    rb_define_method(cLiquidTokenizer, "initialize", tokenizer_initialize_method, 1);
+    rb_define_method(cLiquidTokenizer, "shift", tokenizer_shift_method, 0);
+    utf8_encoding = rb_utf8_encoding();
+}

data/ext/liquid_c/tokenizer.h

@@ -0,0 +1,34 @@
+#ifndef LIQUID_TOKENIZER_H
+#define LIQUID_TOKENIZER_H
+
+#include <ruby.h>
+#include <ruby/encoding.h>
+
+enum token_type {
+    TOKEN_NONE,
+    TOKEN_INVALID,
+    TOKEN_STRING,
+    TOKEN_TAG,
+    TOKEN_VARIABLE
+};
+
+typedef struct token {
+    enum token_type type;
+    const char *str;
+    long length;
+} token_t;
+
+typedef struct tokenizer {
+    VALUE source;
+    const char *cursor;
+    long length;
+} tokenizer_t;
+
+extern VALUE cLiquidTokenizer;
+extern const rb_data_type_t tokenizer_data_type;
+#define Tokenizer_Get_Struct(obj, sval) TypedData_Get_Struct(obj, tokenizer_t, &tokenizer_data_type, sval)
+
+void init_liquid_tokenizer();
+void tokenizer_next(tokenizer_t *tokenizer, token_t *token);
+
+#endif

data/lib/liquid/c.rb

@@ -0,0 +1,19 @@
+require 'liquid/c/version'
+require 'liquid'
+
+# workaround bundler issue #2947 which could cause an old extension to be used
+require 'rbconfig'
+ext_path = File.expand_path("../../liquid_c.#{RbConfig::CONFIG['DLEXT']}", __FILE__)
+if File.exists?(ext_path)
+  require ext_path
+else
+  require 'liquid_c'
+end
+
+Liquid::Template.class_eval do
+  private
+
+  def tokenize(source)
+    Liquid::Tokenizer.new(source.to_s)
+  end
+end

data/lib/liquid/c/version.rb

@@ -0,0 +1,5 @@
+module Liquid
+  module C
+    VERSION = "0.0.1"
+  end
+end

data/liquid-c.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'liquid/c/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "liquid-c"
+  spec.version       = Liquid::C::VERSION
+  spec.authors       = ["Dylan Thacker-Smith"]
+  spec.email         = ["Dylan.Smith@shopify.com"]
+  spec.summary       = "Liquid performance extension in C"
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.extensions    = ['ext/liquid_c/extconf.rb']
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency 'liquid', '~> 3.0.0'
+
+  spec.add_development_dependency "bundler", "~> 1.5"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency 'rake-compiler'
+  spec.add_development_dependency 'stackprof' if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new("2.1.0")
+end

data/performance.rb

@@ -0,0 +1,6 @@
+require 'liquid'
+require 'liquid/c'
+liquid_lib_dir = $LOAD_PATH.detect{ |p| File.exists?(File.join(p, 'liquid.rb')) }
+
+script = ARGV.shift or abort("unspecified performance script")
+require File.join(File.dirname(liquid_lib_dir), "performance/#{script}")

data/test/liquid_test.rb

@@ -0,0 +1,11 @@
+liquid_lib_dir = $LOAD_PATH.detect{ |p| File.exists?(File.join(p, 'liquid.rb')) }
+liquid_test_dir = File.join(File.dirname(liquid_lib_dir), 'test')
+$LOAD_PATH << liquid_test_dir
+
+require 'test_helper'
+require 'liquid/c'
+
+test_files = FileList[File.join(liquid_test_dir, 'integration/**/*_test.rb')]
+test_files.each do |test_file|
+  require test_file
+end

data/test/test_helper.rb

@@ -0,0 +1,2 @@
+require 'minitest/autorun'
+require 'liquid/c'

data/test/unit/tokenizer_test.rb

@@ -0,0 +1,43 @@
+# encoding: utf-8
+require 'test_helper'
+
+class TokenizerTest < MiniTest::Unit::TestCase
+  def test_tokenize_strings
+    assert_equal [' '], tokenize(' ')
+    assert_equal ['hello world'], tokenize('hello world')
+  end
+
+  def test_tokenize_variables
+    assert_equal ['{{funk}}'], tokenize('{{funk}}')
+    assert_equal [' ', '{{funk}}', ' '], tokenize(' {{funk}} ')
+    assert_equal [' ', '{{funk}}', ' ', '{{so}}', ' ', '{{brother}}', ' '], tokenize(' {{funk}} {{so}} {{brother}} ')
+    assert_equal [' ', '{{  funk  }}', ' '], tokenize(' {{  funk  }} ')
+  end
+
+  def test_tokenize_blocks
+    assert_equal ['{%comment%}'], tokenize('{%comment%}')
+    assert_equal [' ', '{%comment%}', ' '], tokenize(' {%comment%} ')
+
+    assert_equal [' ', '{%comment%}', ' ', '{%endcomment%}', ' '], tokenize(' {%comment%} {%endcomment%} ')
+    assert_equal ['  ', '{% comment %}', ' ', '{% endcomment %}', ' '], tokenize("  {% comment %} {% endcomment %} ")
+  end
+
+  def test_utf8_encoded_template
+    source = 'auswählen'
+    assert_equal Encoding::UTF_8, source.encoding
+    output = tokenize(source)
+    assert_equal [Encoding::UTF_8], output.map(&:encoding)
+    assert_equal [source], output
+  end
+
+  private
+
+  def tokenize(source)
+    tokenizer = Liquid::Tokenizer.new(source)
+    tokens = []
+    while t = tokenizer.shift
+      tokens << t
+    end
+    tokens
+  end
+end

metadata

@@ -0,0 +1,136 @@
+--- !ruby/object:Gem::Specification
+name: liquid-c
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Dylan Thacker-Smith
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-10-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: liquid
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake-compiler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: stackprof
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- Dylan.Smith@shopify.com
+executables: []
+extensions:
+- ext/liquid_c/extconf.rb
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- ext/liquid_c/extconf.rb
+- ext/liquid_c/liquid.c
+- ext/liquid_c/liquid.h
+- ext/liquid_c/tokenizer.c
+- ext/liquid_c/tokenizer.h
+- lib/liquid/c.rb
+- lib/liquid/c/version.rb
+- liquid-c.gemspec
+- performance.rb
+- test/liquid_test.rb
+- test/test_helper.rb
+- test/unit/tokenizer_test.rb
+homepage: ''
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.0
+signing_key: 
+specification_version: 4
+summary: Liquid performance extension in C
+test_files:
+- test/liquid_test.rb
+- test/test_helper.rb
+- test/unit/tokenizer_test.rb