brianmario-yajl-ruby 0.4.0 → 0.4.1

data/CHANGELOG.rdoc

@@ -1,6 +1,12 @@
 = Changelog
 
-0.4.0 (?)
+0.4.1 (April 30th, 2009)
+ * fixed a typo in the stream.rb benchmark file
+ * fixed a bug in Yajl::Stream.parse that was causing "strange" Ruby malloc errors on large files, with large strings
+ * added Yajl::GzipStreamReader as a wrapper around Zlib::GzipReader to allow for standard IO#read behavior
+ ** this allows Yajl::Stream to read off of a Gzip stream directly
+ 
+0.4.0 (April 29th, 2009)
  * NOTE: Breaking API change:
  ** refactored Stream parsing methods out of Yajl::Native into Yajl::Stream
  ** removed Yajl::Native namespace/module

data/README.rdoc

@@ -82,37 +82,37 @@ After I finished implementation - this library performs close to the same as the
 But on larger files, and higher amounts of iteration, this library was around 2x faster than JSON.parse.
 
 The main benefit of this library is in it's memory usage.
-
 Since it's able to parse the stream in chunks, it's memory requirements are very, very low.
 
-Again, I'll post some actual data on this; but in my testing here's what my ruby executable was using to parse a 10MB JSON file 100 times:
+Here's what parsing a 2.43MB JSON file off the filesystem 20 times looks like:
 
- JSON.parse ~60MB
- 
- Yajl::Native.parse ~30MB
+=== Memory Usage
+
+==== Average
 
-== Status
+* Yajl::Stream.parse: 32MB
+* JSON.parse: 54MB
+* ActiveSupport::JSON.decode: 63MB
 
-This library is still being refined and should by no means be used in production.
-That being said, I'd like to invite you to clone/fork and test the crap out of it! ;)
+==== Peak
 
-== TODO
+* Yajl::Stream.parse: 32MB
+* JSON.parse: 57MB
+* ActiveSupport::JSON.decode: 67MB
 
-While parsing is working for the most part, there are still some edge cases I need to tackle.
-Here's a list of things I need to get done to improve the awesomeness of this library.
+=== Parse Time
 
-* Write up a test suite 
-* Fix known parsing issues with certain JSON strings (Yajl itself works fine, it's the logic in this library that needs a little tweaking)
-* Write more docs
-* Change API so it can accept a String or an IO
+* Yajl::Stream.parse: 4.54s
+* JSON.parse: 5.47s
+* ActiveSupport::JSON.decode: 64.42s
 
 == Special Thanks
 
 I've had a lot of inspiration, and a lot of help. Thanks to everyone who's been a part of this and those to come!
 
 * Lloyd Hilaiel (http://github.com/lloyd) - for writing Yajl!!
-* Josh Ferguson (http://github.com/besquared) - for peer-pressuring me into getting back into C; it worked ;)
-* Jonathan Novak (http://github.com/cypriss) - C hacking help
-* Tom Smith (http://github.com/rtomsmith) - C hacking help
+* Josh Ferguson (http://github.com/besquared) - for peer-pressuring me into getting back into C; it worked ;) Also tons of support over IM
+* Jonathan Novak (http://github.com/cypriss) - pointer-hacking help
+* Tom Smith (http://github.com/rtomsmith) - pointer-hacking help
 * Rick (http://github.com/technoweenie) - for making an ActiveSupport patch with support for this library and teasing me that it might go into Rails 3. You sure lit a fire under my ass and I got a ton of work done because of it! :)
 * The entire Github Crew - my inspiration, time spent writing this, finding Yajl, So many-MANY other things wouldn't have been possible without this awesome service. I owe you guys some whiskey at Kilowatt.

data/VERSION.yml

@@ -1,4 +1,4 @@
 --- 
-:patch: 0
+:patch: 1
 :major: 0
 :minor: 4

data/benchmark/stream.rb

@@ -16,7 +16,7 @@ times = ARGV[1] ? ARGV[1].to_i : 1
 puts "Starting benchmark parsing #{File.size(filename)} bytes of JSON data #{times} times\n\n"
 Benchmark.bm { |x|
   x.report {
-    puts "Yajl::Native.parse (C)"
+    puts "Yajl::Stream.parse (C)"
     times.times {
       json.rewind
       Yajl::Stream.parse(json)

data/ext/yajl.c

@@ -57,9 +57,7 @@ static int found_boolean(void * ctx, int boolean) {
 }
 
 static int found_number(void * ctx, const char * numberVal, unsigned int numberLen) {
-    if (strstr(numberVal, ".") != NULL
-        || strstr(numberVal, "e") != NULL
-        || strstr(numberVal, "E") != NULL) {
+    if (strstr(numberVal, ".") != NULL || strstr(numberVal, "e") != NULL || strstr(numberVal, "E") != NULL) {
         set_static_value(ctx, rb_Float(rb_str_new(numberVal, numberLen)));
     } else {
         set_static_value(ctx, rb_Integer(rb_str_new(numberVal, numberLen)));
@@ -156,7 +154,7 @@ static VALUE t_parse(VALUE self, VALUE io) {
     
     // now parse from the IO
     while (rb_funcall(io, intern_eof, 0) == Qfalse) {
-        parsed = rb_funcall(io, intern_io_read, 1, rbufsize);
+        rb_funcall(io, intern_io_read, 2, rbufsize, parsed);
         
         stat = yajl_parse(streamParser, (const unsigned char *)RSTRING_PTR(parsed), RSTRING_LEN(parsed));
         

data/ext/yajl.h

@@ -9,7 +9,7 @@ static ID intern_io_read, intern_eof, intern_respond_to, intern_call;
 static int readBufferSize = READ_BUFSIZE;
 static yajl_parser_config cfg = {1, 1};
 
-yajl_handle streamParser = NULL, chunkedParser = NULL;
+yajl_handle streamParser, chunkedParser;
 VALUE context = Qnil;
 VALUE parse_complete_callback = Qnil;
 

data/lib/yajl/http_stream.rb

@@ -54,7 +54,7 @@ module Yajl
       if response_head[:headers]["Content-Type"].include?(MIME_TYPE)
         case response_head[:headers]["Content-Encoding"]
         when "gzip"
-          socket = Zlib::GzipReader.new(socket)
+          socket = Yajl::GzipStreamReader.new(socket)
         end
         return Yajl::Stream.parse(socket)
       else
@@ -64,4 +64,15 @@ module Yajl
       socket.close
     end
   end
+  
+  # === Yajl::GzipStreamReader
+  #
+  # This is a wrapper around Zlib::GzipReader to allow it's #read method to adhere
+  # to the IO spec, allowing for two parameters (length, and buffer)
+  class GzipStreamReader < ::Zlib::GzipReader
+    def read(len, buffer=nil)
+      buffer.gsub!(/.*/, '')
+      buffer << super(len)
+    end
+  end
 end

data/yajl-ruby.gemspec

@@ -2,11 +2,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{yajl-ruby}
-  s.version = "0.4.0"
+  s.version = "0.4.1"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Brian Lopez"]
-  s.date = %q{2009-04-29}
+  s.date = %q{2009-04-30}
   s.email = %q{seniorlopez@gmail.com}
   s.extensions = ["ext/extconf.rb"]
   s.extra_rdoc_files = [

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: brianmario-yajl-ruby
 version: !ruby/object:Gem::Version 
-  version: 0.4.0
+  version: 0.4.1
 platform: ruby
 authors: 
 - Brian Lopez
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-04-29 00:00:00 -07:00
+date: 2009-04-30 00:00:00 -07:00
 default_executable: 
 dependencies: []