brianmario-yajl-ruby 0.1.0

data/README.rdoc

@@ -0,0 +1,55 @@
+= YAJL C Bindings for Ruby (work in progress)
+
+This gem (although not in gem form just yet) is a C binding to the excellent YAJL JSON parsing library.
+
+You can read more info at the projects website http://lloydforge.org/projects/yajl or check out it's codes at http://github.com/lloyd/yajl.
+
+== Example of use
+
+File IO
+
+ json_contents = File.new('test.json', 'r')
+ hash = Yajl::Native.parse(json)
+ 
+OR StringIO
+
+ json_contents = StringIO.new
+ hash = Yajl::Native.parse(json)
+
+OR maybe STDIN
+
+ cat someJsonFile.json | ruby -ryajl -e "puts Yajl::Native.parse(STDIN).inspect"
+
+There are a lot more possibilities, some of which I'm going to write other gems/plugins for.
+
+Some ideas are parsing logs in JSON format, an ActiveSupport patch, Rack middleware, use with ohai, JSON API clients, etc...
+
+== How to install
+
+First, Yajl uses CMake to build itself (yes, the author realizes this isn't the norm for open source and is willing and ready to accept patches, fork away kids!) so you'll need to grab it first from http://www.cmake.org.
+
+After you've got that, grab the latest version of Yajl itself from the Githubs at http://github.com/lloyd/yajl.
+
+After you have that installed, you should be able to install it like any other gem hosted here like so:
+
+(more instructions here: http://gems.github.com)
+
+ sudo gem install brianmario-yajl-ruby
+
+== Benchmarks
+
+I'll update this readme with some actual data soon.
+
+After I finished implementation - this library performs close to the same as the current JSON.parse (C gem) does on small/medium files.
+
+But on larger files, and higher amounts of iteration, this library was around 2x faster than JSON.parse.
+
+The main benefit of this library is in it's memory usage.
+
+Since it's able to parse the stream in chunks, it's memory requirements are very, very low.
+
+Again, I'll post some actual data on this; but in my testing here's what my ruby executable was using to parse a 10MB JSON file 100 times:
+
+ JSON.parse ~60MB
+ 
+ Yajl::Native.parse ~30MB

data/Rakefile

@@ -0,0 +1,15 @@
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "yajl-ruby"
+    gem.summary = "Ruby C bindings to the excellent Yajl JSON stream-based parser library."
+    gem.email = "seniorlopez@gmail.com"
+    gem.homepage = "http://github.com/brianmario/yajl-ruby"
+    gem.description = "Ruby C bindings to the excellent Yajl JSON stream-based parser library."
+    gem.authors = ["Brian Lopez"]
+    gem.files.include %w(lib/jeweler/templates/.document lib/jeweler/templates/.gitignore)
+    # gem.rubyforge_project = "yajl-ruby"
+  end
+rescue LoadError
+  puts "Jeweler, or one of its dependencies, is not available. Install it with: sudo gem install technicalpickles-jeweler -s http://gems.github.com"
+end

data/VERSION.yml

@@ -0,0 +1,4 @@
+--- 
+:patch: 0
+:major: 0
+:minor: 2

data/ext/extconf.rb

@@ -0,0 +1,6 @@
+require 'mkmf'
+dir_config('yajl')
+have_header('yajl/yajl_parse.h')
+have_header('yajl/yajl_gen.h')
+have_library('yajl')
+create_makefile("yajl")

data/ext/yajl.c

@@ -0,0 +1,150 @@
+#include <yajl/yajl_parse.h>
+#include <yajl/yajl_gen.h>
+#include <ruby.h>
+
+void set_static_value(void * ctx, VALUE val) {
+    VALUE len = RARRAY((VALUE)ctx)->len;
+    
+    if (len > 0) {
+        VALUE lastEntry = rb_ary_entry((VALUE)ctx, len-1);
+        VALUE hash;
+        switch (TYPE(lastEntry)) {
+            case T_ARRAY:
+                rb_ary_push(lastEntry, val);
+                if (TYPE(val) == T_HASH || TYPE(val) == T_ARRAY) {
+                    rb_ary_push((VALUE)ctx, val);
+                }
+                break;
+            case T_HASH:
+                rb_hash_aset(lastEntry, val, Qnil);
+                rb_ary_push((VALUE)ctx, val);
+                break;
+            case T_STRING:
+                hash = rb_ary_entry((VALUE)ctx, len-2);
+                if (TYPE(hash) == T_HASH) {
+                    rb_hash_aset(hash, lastEntry, val);
+                    rb_ary_pop((VALUE)ctx);
+                    if (TYPE(val) == T_HASH) {
+                        rb_ary_push((VALUE)ctx, val);
+                    }
+                }
+                break;
+        }
+    } else {
+        rb_ary_push((VALUE)ctx, val);
+    }
+}
+
+static int found_null(void * ctx) {
+    set_static_value(ctx, Qnil);
+    return 1;
+}
+
+static int found_boolean(void * ctx, int boolean) {
+    set_static_value(ctx, boolean ? Qtrue : Qfalse);
+    return 1;
+}
+
+static int found_integer(void * ctx, long integerVal) {
+    set_static_value(ctx, LONG2FIX(integerVal));
+    return 1;
+}
+
+static int found_double(void * ctx, double doubleVal) {
+    set_static_value(ctx, rb_float_new(doubleVal));
+    return 1;
+}
+
+static int found_string(void * ctx, const unsigned char * stringVal, unsigned int stringLen) {
+    set_static_value(ctx, rb_str_new((char *)stringVal, stringLen));
+    return 1;
+}
+
+static int found_hash_key(void * ctx, const unsigned char * stringVal, unsigned int stringLen) {
+    set_static_value(ctx, rb_str_new((char *)stringVal, stringLen));
+    return 1;
+}
+
+static int found_start_hash(void * ctx) {
+    set_static_value(ctx, rb_hash_new());
+    return 1;
+}
+
+static int found_end_hash(void * ctx) {
+    if (RARRAY((VALUE)ctx)->len > 1) {
+        rb_ary_pop((VALUE)ctx);
+    }
+    return 1;
+}
+
+static int found_start_array(void * ctx) {
+    set_static_value(ctx, rb_ary_new());
+    return 1;
+}
+
+static int found_end_array(void * ctx) {
+    if (RARRAY((VALUE)ctx)->len > 1) {
+        rb_ary_pop((VALUE)ctx);
+    }
+    return 1;
+}
+
+static yajl_callbacks callbacks = {
+    found_null,
+    found_boolean,
+    found_integer,
+    found_double,
+    NULL,
+    found_string,
+    found_start_hash,
+    found_hash_key,
+    found_end_hash,
+    found_start_array,
+    found_end_array
+};
+
+ID intern_io_read, intern_eof;
+yajl_parser_config cfg = {1, 1};
+
+static VALUE t_parse(VALUE self, VALUE io) {
+    yajl_handle hand;
+    yajl_status stat;
+    int bufferSize = 8192;
+    intern_io_read = rb_intern("read");
+    intern_eof = rb_intern("eof?");
+    VALUE ctx = rb_ary_new();
+    
+    // allocate our parser
+    hand = yajl_alloc(&callbacks, &cfg, NULL, (void *)ctx);
+    VALUE parsed = rb_str_new("", 0);
+    VALUE rbufsize = INT2FIX(bufferSize);
+    
+    // now parse from the IO
+    while (rb_funcall(io, intern_eof, 0) == Qfalse) {
+        rb_funcall(io, intern_io_read, 2, rbufsize, parsed);
+        
+        stat = yajl_parse(hand, (const unsigned char *)RSTRING(parsed)->ptr, RSTRING(parsed)->len);
+        
+        if (stat != yajl_status_ok && stat != yajl_status_insufficient_data) {
+            unsigned char * str = yajl_get_error(hand, 1, (const unsigned char *)RSTRING_PTR(parsed), RSTRING_LEN(parsed));
+            fprintf(stderr, (const char *) str);
+            yajl_free_error(hand, str);
+            break;
+        }
+    }
+    
+    // parse any remaining buffered data
+    stat = yajl_parse_complete(hand);
+    yajl_free(hand);
+
+    return rb_ary_pop(ctx);
+}
+
+VALUE mYajl;
+VALUE mNative;
+
+void Init_yajl() {
+    mYajl = rb_define_module("Yajl");
+    mNative = rb_define_module_under(mYajl, "Native");
+    rb_define_module_function(mNative, "parse", t_parse, 1);
+}

metadata

@@ -0,0 +1,57 @@
+--- !ruby/object:Gem::Specification 
+name: brianmario-yajl-ruby
+version: !ruby/object:Gem::Version 
+  version: 0.1.0
+platform: ruby
+authors: 
+- Brian Lopez
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-04-22 00:00:00 -07:00
+default_executable: 
+dependencies: []
+
+description: ""
+email: seniorlopez@gmail.com
+executables: []
+
+extensions: 
+- ext/extconf.rb
+extra_rdoc_files: []
+
+files: 
+- README.rdoc
+- Rakefile
+- VERSION.yml
+- ext/extconf.rb
+- ext/yajl.c
+has_rdoc: true
+homepage: http://github.com/brianmario/yajl-ruby
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- ext
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.2.0
+signing_key: 
+specification_version: 3
+summary: Ruby C bindings to the excellent Yajl JSON stream-based parser library.
+test_files: []
+