liquid-c 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 2431d1b61b78e144d34a423a60c5463fb8c21a87
4
+ data.tar.gz: 7cf1e00087a7c80cb6b19d099756149d3d136155
5
+ SHA512:
6
+ metadata.gz: 29ebcbdcc043203d454b979ae8b136263b14d8dd885ea0556d699c7aa006375798eb408c8db048258869b91bd1cf6e519f37a1fc6609f590c078bbea15893c9d
7
+ data.tar.gz: 3b2628f520d13b1b487da6d473c9357b631cb860bdf74de3d288d3345584a954b2eaa71ef221d8aac4e21b5474be3d3570c709e7e846afdcbeeb8031ea7ad24d
@@ -0,0 +1,8 @@
1
+ *.gem
2
+ *.rbc
3
+ Gemfile.lock
4
+ pkg
5
+ tmp
6
+ *.o
7
+ *.bundle
8
+ ext/*/Makefile
@@ -0,0 +1,7 @@
1
+ language: ruby
2
+ rvm:
3
+ - 1.9.3
4
+ - 2.0.0
5
+ - 2.1.0
6
+ gemfile:
7
+ - Gemfile
data/Gemfile ADDED
@@ -0,0 +1,5 @@
1
+ source 'https://rubygems.org'
2
+
3
+ gemspec
4
+
5
+ gem 'liquid', github: 'Shopify/liquid', branch: 'master'
@@ -0,0 +1,22 @@
1
+ Copyright (c) 2014 Shopify
2
+
3
+ MIT License
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ "Software"), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
@@ -0,0 +1,36 @@
1
+ # Liquid::C
2
+ [![Build Status](https://api.travis-ci.org/Shopify/liquid-c.png?branch=master)](https://travis-ci.org/Shopify/liquid-c)
3
+
4
+ Partial native implementation of the liquid ruby gem in C.
5
+
6
+ ## Installation
7
+
8
+ Add these lines to your application's Gemfile:
9
+
10
+ gem 'liquid', github: 'Shopify/liquid', branch: 'master'
11
+ gem 'liquid-c', github: 'Shopify/liquid-c', branch: 'master'
12
+
13
+ And then execute:
14
+
15
+ $ bundle
16
+
17
+ ## Usage
18
+
19
+ require 'liquid/c'
20
+
21
+ then just use the documented API for the liquid Gem.
22
+
23
+ ## Restrictions
24
+
25
+ * Input strings are assumed to be UTF-8 encoded strings
26
+ * Tag#parse(tokens) is given a Liquid::Tokenizer object, instead
27
+ of an array of strings, which only implements the shift method
28
+ to get the next token.
29
+
30
+ ## Contributing
31
+
32
+ 1. Fork it ( http://github.com/Shopify/liquid-c/fork )
33
+ 2. Create your feature branch (`git checkout -b my-new-feature`)
34
+ 3. Commit your changes (`git commit -am 'Add some feature'`)
35
+ 4. Push to the branch (`git push origin my-new-feature`)
36
+ 5. Create new Pull Request
@@ -0,0 +1,57 @@
1
+ require 'rake'
2
+ require 'rake/testtask'
3
+ require 'bundler/gem_tasks'
4
+ require 'rake/extensiontask'
5
+
6
+ Rake::ExtensionTask.new("liquid_c")
7
+
8
+ task :default => :test
9
+
10
+ task :test => ['test:unit', 'test:liquid']
11
+
12
+ namespace :test do
13
+ Rake::TestTask.new(:unit => :compile) do |t|
14
+ t.libs << 'lib' << 'test'
15
+ t.test_files = FileList['test/unit/**/*_test.rb']
16
+ end
17
+
18
+ desc 'run test suite with default parser'
19
+ Rake::TestTask.new(:base_liquid => :compile) do |t|
20
+ t.libs << 'lib'
21
+ t.test_files = ['test/liquid_test.rb']
22
+ end
23
+
24
+ desc 'runs test suite with both strict and lax parsers'
25
+ task :liquid do
26
+ ENV['LIQUID_PARSER_MODE'] = 'lax'
27
+ Rake::Task['test:base_liquid'].invoke
28
+ ENV['LIQUID_PARSER_MODE'] = 'strict'
29
+ Rake::Task['test:base_liquid'].reenable
30
+ Rake::Task['test:base_liquid'].invoke
31
+ end
32
+ end
33
+
34
+ namespace :benchmark do
35
+ desc "Run the liquid benchmark with lax parsing"
36
+ task :run do
37
+ ruby "./performance.rb benchmark lax"
38
+ end
39
+
40
+ desc "Run the liquid benchmark with strict parsing"
41
+ task :strict do
42
+ ruby "./performance.rb benchmark strict"
43
+ end
44
+ end
45
+
46
+
47
+ namespace :profile do
48
+ desc "Run the liquid profile/performance coverage"
49
+ task :run do
50
+ ruby "./performance.rb profile lax"
51
+ end
52
+
53
+ desc "Run the liquid profile/performance coverage with strict parsing"
54
+ task :strict do
55
+ ruby "./performance.rb profile strict"
56
+ end
57
+ end
@@ -0,0 +1,4 @@
1
+ require 'mkmf'
2
+ $CFLAGS << ' -Wall -Werror'
3
+ $warnflags.gsub!(/-Wdeclaration-after-statement/, "")
4
+ create_makefile("liquid_c")
@@ -0,0 +1,9 @@
1
+ #include "liquid.h"
2
+
3
+ VALUE mLiquid;
4
+
5
+ void Init_liquid_c(void)
6
+ {
7
+ mLiquid = rb_define_module("Liquid");
8
+ init_liquid_tokenizer();
9
+ }
@@ -0,0 +1,11 @@
1
+ #ifndef LIQUID_H
2
+ #define LIQUID_H
3
+
4
+ #include <ruby.h>
5
+ #include <stdbool.h>
6
+
7
+ #include "tokenizer.h"
8
+
9
+ extern VALUE mLiquid;
10
+
11
+ #endif
@@ -0,0 +1,138 @@
1
+ #include "liquid.h"
2
+
3
+ VALUE cLiquidTokenizer;
4
+ rb_encoding *utf8_encoding;
5
+
6
+ static void tokenizer_mark(void *ptr) {
7
+ tokenizer_t *tokenizer = ptr;
8
+ rb_gc_mark(tokenizer->source);
9
+ }
10
+
11
+ static void tokenizer_free(void *ptr)
12
+ {
13
+ tokenizer_t *tokenizer = ptr;
14
+ xfree(tokenizer);
15
+ }
16
+
17
+ static size_t tokenizer_memsize(const void *ptr)
18
+ {
19
+ return ptr ? sizeof(tokenizer_t) : 0;
20
+ }
21
+
22
+ const rb_data_type_t tokenizer_data_type = {
23
+ "liquid_tokenizer",
24
+ {tokenizer_mark, tokenizer_free, tokenizer_memsize,},
25
+ #ifdef RUBY_TYPED_FREE_IMMEDIATELY
26
+ NULL, NULL, RUBY_TYPED_FREE_IMMEDIATELY
27
+ #endif
28
+ };
29
+
30
+ static VALUE tokenizer_allocate(VALUE klass)
31
+ {
32
+ VALUE obj;
33
+ tokenizer_t *tokenizer;
34
+
35
+ obj = TypedData_Make_Struct(klass, tokenizer_t, &tokenizer_data_type, tokenizer);
36
+ tokenizer->source = Qnil;
37
+ return obj;
38
+ }
39
+
40
+ static VALUE tokenizer_initialize_method(VALUE self, VALUE source)
41
+ {
42
+ tokenizer_t *tokenizer;
43
+
44
+ Check_Type(source, T_STRING);
45
+ Tokenizer_Get_Struct(self, tokenizer);
46
+ source = rb_str_dup_frozen(source);
47
+ tokenizer->source = source;
48
+ tokenizer->cursor = RSTRING_PTR(source);
49
+ tokenizer->length = RSTRING_LEN(source);
50
+ return Qnil;
51
+ }
52
+
53
+ void tokenizer_next(tokenizer_t *tokenizer, token_t *token)
54
+ {
55
+ if (tokenizer->length <= 0) {
56
+ memset(token, 0, sizeof(*token));
57
+ return;
58
+ }
59
+
60
+ const char *cursor = tokenizer->cursor;
61
+ const char *last = cursor + tokenizer->length - 1;
62
+
63
+ token->str = cursor;
64
+ token->type = TOKEN_STRING;
65
+
66
+ while (cursor < last) {
67
+ if (*cursor++ != '{')
68
+ continue;
69
+
70
+ char c = *cursor++;
71
+ if (c != '%' && c != '{')
72
+ continue;
73
+ if (cursor - tokenizer->cursor > 2) {
74
+ token->type = TOKEN_STRING;
75
+ cursor -= 2;
76
+ goto found;
77
+ }
78
+ token->type = TOKEN_INVALID;
79
+ if (c == '%') {
80
+ while (cursor < last) {
81
+ if (*cursor++ != '%')
82
+ continue;
83
+ c = *cursor++;
84
+ while (c == '%' && cursor <= last)
85
+ c = *cursor++;
86
+ if (c != '}')
87
+ continue;
88
+ token->type = TOKEN_TAG;
89
+ goto found;
90
+ }
91
+ // unterminated tag
92
+ cursor = tokenizer->cursor + 2;
93
+ goto found;
94
+ } else {
95
+ while (cursor < last) {
96
+ if (*cursor++ != '}')
97
+ continue;
98
+ if (*cursor++ != '}') {
99
+ // variable incomplete end, used to end raw tags
100
+ cursor--;
101
+ goto found;
102
+ }
103
+ token->type = TOKEN_VARIABLE;
104
+ goto found;
105
+ }
106
+ // unterminated variable
107
+ cursor = tokenizer->cursor + 2;
108
+ goto found;
109
+ }
110
+ }
111
+ cursor = last + 1;
112
+ found:
113
+ token->length = cursor - tokenizer->cursor;
114
+ tokenizer->cursor += token->length;
115
+ tokenizer->length -= token->length;
116
+ }
117
+
118
+ static VALUE tokenizer_shift_method(VALUE self)
119
+ {
120
+ tokenizer_t *tokenizer;
121
+ Tokenizer_Get_Struct(self, tokenizer);
122
+
123
+ token_t token;
124
+ tokenizer_next(tokenizer, &token);
125
+ if (token.type == TOKEN_NONE)
126
+ return Qnil;
127
+
128
+ return rb_enc_str_new(token.str, token.length, utf8_encoding);
129
+ }
130
+
131
+ void init_liquid_tokenizer()
132
+ {
133
+ cLiquidTokenizer = rb_define_class_under(mLiquid, "Tokenizer", rb_cObject);
134
+ rb_define_alloc_func(cLiquidTokenizer, tokenizer_allocate);
135
+ rb_define_method(cLiquidTokenizer, "initialize", tokenizer_initialize_method, 1);
136
+ rb_define_method(cLiquidTokenizer, "shift", tokenizer_shift_method, 0);
137
+ utf8_encoding = rb_utf8_encoding();
138
+ }
@@ -0,0 +1,34 @@
1
+ #ifndef LIQUID_TOKENIZER_H
2
+ #define LIQUID_TOKENIZER_H
3
+
4
+ #include <ruby.h>
5
+ #include <ruby/encoding.h>
6
+
7
+ enum token_type {
8
+ TOKEN_NONE,
9
+ TOKEN_INVALID,
10
+ TOKEN_STRING,
11
+ TOKEN_TAG,
12
+ TOKEN_VARIABLE
13
+ };
14
+
15
+ typedef struct token {
16
+ enum token_type type;
17
+ const char *str;
18
+ long length;
19
+ } token_t;
20
+
21
+ typedef struct tokenizer {
22
+ VALUE source;
23
+ const char *cursor;
24
+ long length;
25
+ } tokenizer_t;
26
+
27
+ extern VALUE cLiquidTokenizer;
28
+ extern const rb_data_type_t tokenizer_data_type;
29
+ #define Tokenizer_Get_Struct(obj, sval) TypedData_Get_Struct(obj, tokenizer_t, &tokenizer_data_type, sval)
30
+
31
+ void init_liquid_tokenizer();
32
+ void tokenizer_next(tokenizer_t *tokenizer, token_t *token);
33
+
34
+ #endif
@@ -0,0 +1,19 @@
1
+ require 'liquid/c/version'
2
+ require 'liquid'
3
+
4
+ # workaround bundler issue #2947 which could cause an old extension to be used
5
+ require 'rbconfig'
6
+ ext_path = File.expand_path("../../liquid_c.#{RbConfig::CONFIG['DLEXT']}", __FILE__)
7
+ if File.exists?(ext_path)
8
+ require ext_path
9
+ else
10
+ require 'liquid_c'
11
+ end
12
+
13
+ Liquid::Template.class_eval do
14
+ private
15
+
16
+ def tokenize(source)
17
+ Liquid::Tokenizer.new(source.to_s)
18
+ end
19
+ end
@@ -0,0 +1,5 @@
1
+ module Liquid
2
+ module C
3
+ VERSION = "0.0.1"
4
+ end
5
+ end
@@ -0,0 +1,27 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'liquid/c/version'
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = "liquid-c"
8
+ spec.version = Liquid::C::VERSION
9
+ spec.authors = ["Dylan Thacker-Smith"]
10
+ spec.email = ["Dylan.Smith@shopify.com"]
11
+ spec.summary = "Liquid performance extension in C"
12
+ spec.homepage = ""
13
+ spec.license = "MIT"
14
+
15
+ spec.extensions = ['ext/liquid_c/extconf.rb']
16
+ spec.files = `git ls-files -z`.split("\x0")
17
+ spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
18
+ spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
19
+ spec.require_paths = ["lib"]
20
+
21
+ spec.add_dependency 'liquid', '~> 3.0.0'
22
+
23
+ spec.add_development_dependency "bundler", "~> 1.5"
24
+ spec.add_development_dependency "rake"
25
+ spec.add_development_dependency 'rake-compiler'
26
+ spec.add_development_dependency 'stackprof' if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new("2.1.0")
27
+ end
@@ -0,0 +1,6 @@
1
+ require 'liquid'
2
+ require 'liquid/c'
3
+ liquid_lib_dir = $LOAD_PATH.detect{ |p| File.exists?(File.join(p, 'liquid.rb')) }
4
+
5
+ script = ARGV.shift or abort("unspecified performance script")
6
+ require File.join(File.dirname(liquid_lib_dir), "performance/#{script}")
@@ -0,0 +1,11 @@
1
+ liquid_lib_dir = $LOAD_PATH.detect{ |p| File.exists?(File.join(p, 'liquid.rb')) }
2
+ liquid_test_dir = File.join(File.dirname(liquid_lib_dir), 'test')
3
+ $LOAD_PATH << liquid_test_dir
4
+
5
+ require 'test_helper'
6
+ require 'liquid/c'
7
+
8
+ test_files = FileList[File.join(liquid_test_dir, 'integration/**/*_test.rb')]
9
+ test_files.each do |test_file|
10
+ require test_file
11
+ end
@@ -0,0 +1,2 @@
1
+ require 'minitest/autorun'
2
+ require 'liquid/c'
@@ -0,0 +1,43 @@
1
+ # encoding: utf-8
2
+ require 'test_helper'
3
+
4
+ class TokenizerTest < MiniTest::Unit::TestCase
5
+ def test_tokenize_strings
6
+ assert_equal [' '], tokenize(' ')
7
+ assert_equal ['hello world'], tokenize('hello world')
8
+ end
9
+
10
+ def test_tokenize_variables
11
+ assert_equal ['{{funk}}'], tokenize('{{funk}}')
12
+ assert_equal [' ', '{{funk}}', ' '], tokenize(' {{funk}} ')
13
+ assert_equal [' ', '{{funk}}', ' ', '{{so}}', ' ', '{{brother}}', ' '], tokenize(' {{funk}} {{so}} {{brother}} ')
14
+ assert_equal [' ', '{{ funk }}', ' '], tokenize(' {{ funk }} ')
15
+ end
16
+
17
+ def test_tokenize_blocks
18
+ assert_equal ['{%comment%}'], tokenize('{%comment%}')
19
+ assert_equal [' ', '{%comment%}', ' '], tokenize(' {%comment%} ')
20
+
21
+ assert_equal [' ', '{%comment%}', ' ', '{%endcomment%}', ' '], tokenize(' {%comment%} {%endcomment%} ')
22
+ assert_equal [' ', '{% comment %}', ' ', '{% endcomment %}', ' '], tokenize(" {% comment %} {% endcomment %} ")
23
+ end
24
+
25
+ def test_utf8_encoded_template
26
+ source = 'auswählen'
27
+ assert_equal Encoding::UTF_8, source.encoding
28
+ output = tokenize(source)
29
+ assert_equal [Encoding::UTF_8], output.map(&:encoding)
30
+ assert_equal [source], output
31
+ end
32
+
33
+ private
34
+
35
+ def tokenize(source)
36
+ tokenizer = Liquid::Tokenizer.new(source)
37
+ tokens = []
38
+ while t = tokenizer.shift
39
+ tokens << t
40
+ end
41
+ tokens
42
+ end
43
+ end
metadata ADDED
@@ -0,0 +1,136 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: liquid-c
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ platform: ruby
6
+ authors:
7
+ - Dylan Thacker-Smith
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2014-10-06 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: liquid
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: 3.0.0
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: 3.0.0
27
+ - !ruby/object:Gem::Dependency
28
+ name: bundler
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - "~>"
32
+ - !ruby/object:Gem::Version
33
+ version: '1.5'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: '1.5'
41
+ - !ruby/object:Gem::Dependency
42
+ name: rake
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - ">="
46
+ - !ruby/object:Gem::Version
47
+ version: '0'
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - ">="
53
+ - !ruby/object:Gem::Version
54
+ version: '0'
55
+ - !ruby/object:Gem::Dependency
56
+ name: rake-compiler
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - ">="
60
+ - !ruby/object:Gem::Version
61
+ version: '0'
62
+ type: :development
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - ">="
67
+ - !ruby/object:Gem::Version
68
+ version: '0'
69
+ - !ruby/object:Gem::Dependency
70
+ name: stackprof
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - ">="
74
+ - !ruby/object:Gem::Version
75
+ version: '0'
76
+ type: :development
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - ">="
81
+ - !ruby/object:Gem::Version
82
+ version: '0'
83
+ description:
84
+ email:
85
+ - Dylan.Smith@shopify.com
86
+ executables: []
87
+ extensions:
88
+ - ext/liquid_c/extconf.rb
89
+ extra_rdoc_files: []
90
+ files:
91
+ - ".gitignore"
92
+ - ".travis.yml"
93
+ - Gemfile
94
+ - LICENSE.txt
95
+ - README.md
96
+ - Rakefile
97
+ - ext/liquid_c/extconf.rb
98
+ - ext/liquid_c/liquid.c
99
+ - ext/liquid_c/liquid.h
100
+ - ext/liquid_c/tokenizer.c
101
+ - ext/liquid_c/tokenizer.h
102
+ - lib/liquid/c.rb
103
+ - lib/liquid/c/version.rb
104
+ - liquid-c.gemspec
105
+ - performance.rb
106
+ - test/liquid_test.rb
107
+ - test/test_helper.rb
108
+ - test/unit/tokenizer_test.rb
109
+ homepage: ''
110
+ licenses:
111
+ - MIT
112
+ metadata: {}
113
+ post_install_message:
114
+ rdoc_options: []
115
+ require_paths:
116
+ - lib
117
+ required_ruby_version: !ruby/object:Gem::Requirement
118
+ requirements:
119
+ - - ">="
120
+ - !ruby/object:Gem::Version
121
+ version: '0'
122
+ required_rubygems_version: !ruby/object:Gem::Requirement
123
+ requirements:
124
+ - - ">="
125
+ - !ruby/object:Gem::Version
126
+ version: '0'
127
+ requirements: []
128
+ rubyforge_project:
129
+ rubygems_version: 2.2.0
130
+ signing_key:
131
+ specification_version: 4
132
+ summary: Liquid performance extension in C
133
+ test_files:
134
+ - test/liquid_test.rb
135
+ - test/test_helper.rb
136
+ - test/unit/tokenizer_test.rb