yajl-ruby 0.5.7 → 0.5.8

data/.gitignore

@@ -3,4 +3,5 @@ benchmark/subjects/contacts.*
 *.o
 *.dylib
 *.bundle
-TODO.txt
+TODO.txt
+tmp/*

data/CHANGELOG.md

@@ -1,5 +1,14 @@
 # Changelog
 
+## 0.5.8 (July 6th, 2009)
+* Bugfix in Yajl::HttpStream for proper handling of the Content-type header (Rob Sharp)
+* Yajl::Encoder now has an on_progress callback setter, which can be used to harness the encoder's streaming ability.
+** The passed Proc/lambda will be called, and passed every chunk (currently 8kb) of the encoded JSON string as it's being encoded.
+* API CHANGE WARNING: Yajl::Encoder.encode's block will now be used as (and work the same as) the on_progress callback
+** This means the block will be passed chunks of the JSON string at a time, giving the caller the ability to start processing the encoded data while it's still being encoded.
+* fixed grammatical error in README (Neil Berkman)
+* Added some encoder examples
+
 ## 0.5.7 (June 23rd, 2009)
 * You can now pass parser options (like :symbolize_keys for example) to Yajl::HttpStream.get
 * Refactored spec tests a bit, DRYing up the Yajl::HttpStream specs quite a bit.

data/README.rdoc

@@ -167,7 +167,7 @@ This includes the following API:
  JSON.parse, JSON.generate, JSON.pretty_generate, JSON.load, JSON.dump
  and all of the #to_json instance method overrides for Ruby's primitive objects
 
-Once the compatibility API is enabled, you're existing or new project should work as if the JSON gem itself were being used. Only you'll be using Yajl ;)
+Once the compatibility API is enabled, your existing or new project should work as if the JSON gem itself were being used. Only you'll be using Yajl ;)
 
 There are a lot more possibilities that I'd love to see other gems/plugins for someday.
 
@@ -267,4 +267,4 @@ I've had a lot of inspiration, and a lot of help. Thanks to everyone who's been
 * Rick http://github.com/technoweenie - for making an ActiveSupport patch with support for this library and teasing me that it might go into Rails 3. You sure lit a fire under my ass and I got a ton of work done because of it! :)
 * The entire Github Crew - http://github.com/ - my inspiration, time spent writing this, finding Yajl, So many-MANY other things wouldn't have been possible without this awesome service. I owe you guys some whiskey at Kilowatt.
 * benburkert - http://github.com/benburkert
-* Aman Gupta - http://github.com/tmm1 - tons of suggestions and inspiration for the most recent features, and hopefully more to come ;)
+* Aman Gupta - http://github.com/tmm1 - tons of suggestions and inspiration for the most recent features, and hopefully more to come ;)

data/VERSION.yml

@@ -1,4 +1,4 @@
 --- 
-:patch: 7
+:patch: 8
 :major: 0
 :minor: 5

data/examples/encoding/chunked_encoding.rb

@@ -0,0 +1,27 @@
+# encoding: UTF-8
+
+require 'rubygems'
+require 'yajl'
+
+obj = {
+  :a_test => "of encoding in one pass" * 512,
+  :a_test2 => "of encoding in one pass" * 512,
+  :a_test3 => "of encoding in one pass" * 512,
+  :a_test4 => "of encoding in one pass" * 512,
+  :which_will => "simply return a string when finished" * 512,
+  :which_will2 => "simply return a string when finished" * 512,
+  :which_will3 => "simply return a string when finished" * 512,
+  :which_will4 => "simply return a string when finished" * 512,
+  :as_easy_as => 123
+}
+
+chunks = 0
+total_size = 0
+
+Yajl::Encoder.encode(obj) do |chunk|
+  chunks += 1
+  total_size += chunk.size
+  STDOUT << chunk
+end
+
+puts "\n\nEncoder generated #{total_size} bytes of data, in #{chunks} chunks"

data/examples/encoding/one_shot.rb

@@ -0,0 +1,13 @@
+# encoding: UTF-8
+
+require 'rubygems'
+require 'yajl'
+
+obj = {
+  :a_test => "of encoding in one pass",
+  :which_will => "simply return a string when finished",
+  :as_easy_as => 123
+}
+
+str = Yajl::Encoder.encode(obj)
+puts str

data/examples/encoding/to_an_io.rb

@@ -0,0 +1,12 @@
+# encoding: UTF-8
+
+require 'rubygems'
+require 'yajl'
+
+obj = {
+  :a_test => "of encoding directly to an IO stream",
+  :which_will => "simply return a string when finished",
+  :as_easy_as => 123
+}
+
+Yajl::Encoder.encode(obj, STDOUT)

data/examples/parsing/from_file.rb

@@ -10,5 +10,5 @@ end
 
 json = File.new(file, 'r')
 
-hash = Yajl::Parser.new.parse(json)
+hash = Yajl::Parser.parse(json)
 puts hash.inspect

data/examples/parsing/from_stdin.rb

@@ -5,5 +5,5 @@ require 'yajl'
 
 # Usage: cat benchmark/subjects/item.json | ruby examples/from_stdin.rb
 
-hash = Yajl::Parser.new.parse(STDIN)
+hash = Yajl::Parser.parse(STDIN)
 puts hash.inspect

data/examples/parsing/from_string.rb

@@ -11,5 +11,5 @@ end
 
 json = StringIO.new(string)
 
-hash = Yajl::Parser.new.parse(json)
+hash = Yajl::Parser.parse(json)
 puts hash.inspect

data/ext/yajl_ext.c

@@ -60,25 +60,41 @@ inline void yajl_set_static_value(void * ctx, VALUE val) {
     }
 }
 
-void yajl_encode_part(yajl_gen hand, VALUE obj, VALUE io) {
+static void yajl_encoder_wrapper_free(void * wrapper) {
+    struct yajl_encoder_wrapper * w = wrapper;
+    yajl_gen_free(w->encoder);
+    free(w);
+}
+
+static void yajl_encoder_wrapper_mark(void * wrapper) {
+    struct yajl_encoder_wrapper * w = wrapper;
+    rb_gc_mark(w->on_progress_callback);
+}
+
+void yajl_encode_part(void * wrapper, VALUE obj, VALUE io) {
     VALUE str, outBuff, otherObj;
+    struct yajl_encoder_wrapper * w = wrapper;
     yajl_gen_status status;
     int idx = 0;
     const unsigned char * buffer;
     unsigned int len;
     
-    if (io != Qnil) {
-        status = yajl_gen_get_buf(hand, &buffer, &len);
+    if (io != Qnil || w->on_progress_callback != Qnil) {
+        status = yajl_gen_get_buf(w->encoder, &buffer, &len);
         if (len >= WRITE_BUFSIZE) {
             outBuff = rb_str_new((const char *)buffer, len);
-            rb_io_write(io, outBuff);
-            yajl_gen_clear(hand);
+            if (io != Qnil) {
+                rb_io_write(io, outBuff);
+            } else if (w->on_progress_callback != Qnil) {
+                rb_funcall(w->on_progress_callback, intern_call, 1, outBuff);
+            }
+            yajl_gen_clear(w->encoder);
         }
     }
     
     switch (TYPE(obj)) {
         case T_HASH:
-            status = yajl_gen_map_open(hand);
+            status = yajl_gen_map_open(w->encoder);
             
             // TODO: itterate through keys in the hash
             VALUE keys = rb_funcall(obj, intern_keys, 0);
@@ -87,38 +103,38 @@ void yajl_encode_part(yajl_gen hand, VALUE obj, VALUE io) {
                 entry = rb_ary_entry(keys, idx);
                 keyStr = rb_funcall(entry, intern_to_s, 0); // key must be a string
                 // the key
-                yajl_encode_part(hand, keyStr, io);
+                yajl_encode_part(w, keyStr, io);
                 // the value
-                yajl_encode_part(hand, rb_hash_aref(obj, entry), io);
+                yajl_encode_part(w, rb_hash_aref(obj, entry), io);
             }
             
-            status = yajl_gen_map_close(hand);
+            status = yajl_gen_map_close(w->encoder);
             break;
         case T_ARRAY:
-            status = yajl_gen_array_open(hand);
+            status = yajl_gen_array_open(w->encoder);
             for(idx=0; idx<RARRAY_LEN(obj); idx++) {
                 otherObj = rb_ary_entry(obj, idx);
-                yajl_encode_part(hand, otherObj, io);
+                yajl_encode_part(w, otherObj, io);
             }
-            status = yajl_gen_array_close(hand);
+            status = yajl_gen_array_close(w->encoder);
             break;
         case T_NIL:
-            status = yajl_gen_null(hand);
+            status = yajl_gen_null(w->encoder);
             break;
         case T_TRUE:
-            status = yajl_gen_bool(hand, 1);
+            status = yajl_gen_bool(w->encoder, 1);
             break;
         case T_FALSE:
-            status = yajl_gen_bool(hand, 0);
+            status = yajl_gen_bool(w->encoder, 0);
             break;
         case T_FIXNUM:
         case T_FLOAT:
         case T_BIGNUM:
             str = rb_funcall(obj, intern_to_s, 0);
-            status = yajl_gen_number(hand, RSTRING_PTR(str), (unsigned int)RSTRING_LEN(str));
+            status = yajl_gen_number(w->encoder, RSTRING_PTR(str), (unsigned int)RSTRING_LEN(str));
             break;
         case T_STRING:
-            status = yajl_gen_string(hand, (const unsigned char *)RSTRING_PTR(obj), (unsigned int)RSTRING_LEN(obj));
+            status = yajl_gen_string(w->encoder, (const unsigned char *)RSTRING_PTR(obj), (unsigned int)RSTRING_LEN(obj));
             break;
         default:
             if (rb_respond_to(obj, intern_to_json)) {
@@ -126,7 +142,7 @@ void yajl_encode_part(yajl_gen hand, VALUE obj, VALUE io) {
             } else {
                 str = rb_funcall(obj, intern_to_s, 0);
             }
-            status = yajl_gen_string(hand, (const unsigned char *)RSTRING_PTR(str), (unsigned int)RSTRING_LEN(str));
+            status = yajl_gen_string(w->encoder, (const unsigned char *)RSTRING_PTR(str), (unsigned int)RSTRING_LEN(str));
             break;
     }
 }
@@ -348,7 +364,7 @@ static VALUE rb_yajl_parser_parse(int argc, VALUE * argv, VALUE self) {
         Check_Type(rbufsize, T_FIXNUM);
     }
     if (!NIL_P(blk)) {
-        rb_yajl_set_complete_cb(self, blk);
+        rb_yajl_parser_set_complete_cb(self, blk);
     }
     
     if (TYPE(input) == T_STRING) {
@@ -407,11 +423,11 @@ static VALUE rb_yajl_parser_parse_chunk(VALUE self, VALUE chunk) {
  *
  * call-seq: on_parse_complete = Proc.new { |obj| ... }
  *
- * This callback setter allows you to pass a Proc/lambda or any other object that response to #call.
+ * This callback setter allows you to pass a Proc/lambda or any other object that responds to #call.
  *
  * It will pass a single parameter, the ruby object built from the last parsed JSON object
  */
-static VALUE rb_yajl_set_complete_cb(VALUE self, VALUE callback) {
+static VALUE rb_yajl_parser_set_complete_cb(VALUE self, VALUE callback) {
     struct yajl_parser_wrapper * wrapper;
     GetParser(self, wrapper);
     wrapper->parse_complete_callback = callback;
@@ -436,8 +452,8 @@ static VALUE rb_yajl_set_complete_cb(VALUE self, VALUE callback) {
  * :indent is the character(s) used to indent the output string.
  */
 static VALUE rb_yajl_encoder_new(int argc, VALUE * argv, VALUE klass) {
+    struct yajl_encoder_wrapper * wrapper;
     yajl_gen_config cfg;
-    yajl_gen encoder;
     VALUE opts, obj, indent;
     const char * indentString = "  ";
     int beautify = 0;
@@ -457,8 +473,9 @@ static VALUE rb_yajl_encoder_new(int argc, VALUE * argv, VALUE klass) {
     }
     cfg = (yajl_gen_config){beautify, indentString};
     
-    encoder = yajl_gen_alloc(&cfg, NULL);
-    obj = Data_Wrap_Struct(klass, 0, yajl_gen_free, encoder);
+    obj = Data_Make_Struct(klass, struct yajl_encoder_wrapper, yajl_encoder_wrapper_mark, yajl_encoder_wrapper_free, wrapper);
+    wrapper->encoder = yajl_gen_alloc(&cfg, NULL);
+    wrapper->on_progress_callback = Qnil;
     rb_obj_call_init(obj, 0, 0);
     return obj;
 }
@@ -493,22 +510,26 @@ static VALUE rb_yajl_encoder_init(int argc, VALUE * argv, VALUE self) {
  * ruby objects to encode. This is how streaming is accomplished.
  */
 static VALUE rb_yajl_encoder_encode(int argc, VALUE * argv, VALUE self) {
-    yajl_gen encoder;
+    struct yajl_encoder_wrapper * wrapper;
     const unsigned char * buffer;
     unsigned int len;
     VALUE obj, io, blk, outBuff;
     
-    GetEncoder(self, encoder);
+    GetEncoder(self, wrapper);
     
     rb_scan_args(argc, argv, "11&", &obj, &io, &blk);
     
+    if (blk != Qnil) {
+        wrapper->on_progress_callback = blk;
+    }
+    
     // begin encode process
-    yajl_encode_part(encoder, obj, io);
+    yajl_encode_part(wrapper, obj, io);
 
     // just make sure we output the remaining buffer
-    yajl_gen_get_buf(encoder, &buffer, &len);
+    yajl_gen_get_buf(wrapper->encoder, &buffer, &len);
     outBuff = rb_str_new((const char *)buffer, len);
-    yajl_gen_clear(encoder);
+    yajl_gen_clear(wrapper->encoder);
     if (io != Qnil) {
         rb_io_write(io, outBuff);
         return Qnil;
@@ -521,6 +542,23 @@ static VALUE rb_yajl_encoder_encode(int argc, VALUE * argv, VALUE self) {
     return Qnil;
 }
 
+/*
+ * Document-method: on_progress
+ *
+ * call-seq: on_progress = Proc.new {|str| ...}
+ *
+ * This callback setter allows you to pass a Proc/lambda or any other object that responds to #call.
+ *
+ * It will pass the caller a chunk of the encode buffer after it's reached it's internal max buffer size (defaults to 8kb).
+ * For example, encoding a large object that would normally result in 24288 bytes of data will result in 3 calls to this callback (assuming the 8kb default encode buffer).
+ */
+static VALUE rb_yajl_encoder_set_progress_cb(VALUE self, VALUE callback) {
+    struct yajl_encoder_wrapper * wrapper;
+    GetEncoder(self, wrapper);
+    wrapper->on_progress_callback = callback;
+    return Qnil;
+}
+
 
 // JSON Gem compatibility
 
@@ -750,12 +788,13 @@ void Init_yajl_ext() {
     rb_define_method(cParser, "parse", rb_yajl_parser_parse, -1);
     rb_define_method(cParser, "parse_chunk", rb_yajl_parser_parse_chunk, -1);
     rb_define_method(cParser, "<<", rb_yajl_parser_parse_chunk, 1);
-    rb_define_method(cParser, "on_parse_complete=", rb_yajl_set_complete_cb, 1);
+    rb_define_method(cParser, "on_parse_complete=", rb_yajl_parser_set_complete_cb, 1);
     
     cEncoder = rb_define_class_under(mYajl, "Encoder", rb_cObject);
     rb_define_singleton_method(cEncoder, "new", rb_yajl_encoder_new, -1);
     rb_define_method(cEncoder, "initialize", rb_yajl_encoder_init, -1);
     rb_define_method(cEncoder, "encode", rb_yajl_encoder_encode, -1);
+    rb_define_method(cEncoder, "on_progress=", rb_yajl_encoder_set_progress_cb, 1);
     
     rb_define_singleton_method(cEncoder, "enable_json_gem_compatability", rb_yajl_encoder_enable_json_gem_ext, 0);
     

data/ext/yajl_ext.h

@@ -10,11 +10,11 @@ static ID intern_io_read, intern_eof, intern_call, intern_keys, intern_to_s, int
           sym_allow_comments, sym_check_utf8, sym_pretty, sym_indent, sym_symbolize_keys;
 
 #define GetParser(obj, sval) (sval = (struct yajl_parser_wrapper*)DATA_PTR(obj));
-#define GetEncoder(obj, sval) (sval = (yajl_gen*)DATA_PTR(obj));
+#define GetEncoder(obj, sval) (sval = (struct yajl_encoder_wrapper*)DATA_PTR(obj));
 
 inline void yajl_check_and_fire_callback(void * ctx);
 inline void yajl_set_static_value(void * ctx, VALUE val);
-void yajl_encode_part(yajl_gen hand, VALUE obj, VALUE io);
+void yajl_encode_part(void * wrapper, VALUE obj, VALUE io);
 void yajl_parse_chunk(const unsigned char * chunk, unsigned int len, yajl_handle parser);
 
 static int yajl_found_null(void * ctx);
@@ -26,6 +26,7 @@ static int yajl_found_start_hash(void * ctx);
 static int yajl_found_end_hash(void * ctx);
 static int yajl_found_start_array(void * ctx);
 static int yajl_found_end_array(void * ctx);
+
 static yajl_callbacks callbacks = {
     yajl_found_null,
     yajl_found_boolean,
@@ -49,18 +50,26 @@ struct yajl_parser_wrapper {
     int symbolizeKeys;
     yajl_handle parser;
 };
-static void yajl_parser_wrapper_free(void * wrapper);
-static void yajl_parser_wrapper_mark(void * wrapper);
+
+struct yajl_encoder_wrapper {
+    VALUE on_progress_callback;
+    yajl_gen encoder;
+};
 
 static VALUE rb_yajl_parser_new(int argc, VALUE * argv, VALUE self);
 static VALUE rb_yajl_parser_init(int argc, VALUE * argv, VALUE self);
 static VALUE rb_yajl_parser_parse(int argc, VALUE * argv, VALUE self);
 static VALUE rb_yajl_parser_parse_chunk(VALUE self, VALUE chunk);
-static VALUE rb_yajl_set_complete_cb(VALUE self, VALUE callback);
+static VALUE rb_yajl_parser_set_complete_cb(VALUE self, VALUE callback);
+static void yajl_parser_wrapper_free(void * wrapper);
+static void yajl_parser_wrapper_mark(void * wrapper);
 
 static VALUE rb_yajl_encoder_new(int argc, VALUE * argv, VALUE klass);
 static VALUE rb_yajl_encoder_init(int argc, VALUE * argv, VALUE self);
 static VALUE rb_yajl_encoder_encode(int argc, VALUE * argv, VALUE self);
+static VALUE rb_yajl_encoder_set_progress_cb(VALUE self, VALUE callback);
+static void yajl_encoder_wrapper_free(void * wrapper);
+static void yajl_encoder_wrapper_mark(void * wrapper);
 
 static VALUE rb_yajl_json_ext_object_to_json(int argc, VALUE * argv, VALUE self);
 static VALUE rb_yajl_json_ext_hash_to_json(int argc, VALUE * argv, VALUE self);

data/lib/yajl.rb

@@ -13,7 +13,7 @@ require 'yajl_ext'
 #
 # Ruby bindings to the excellent Yajl (Yet Another JSON Parser) ANSI C library.
 module Yajl
-  VERSION = "0.5.7"
+  VERSION = "0.5.8"
   
   class Parser
     # A helper method for parse-and-forget use-cases
@@ -34,9 +34,9 @@ module Yajl
     # A helper method for encode-and-forget use-cases
     #
     # Examples:
-    #   Yajl::Encoder.encode(obj[, io, :pretty => true, :indent => "\t"])
+    #   Yajl::Encoder.encode(obj[, io, :pretty => true, :indent => "\t", &block])
     #
-    #   output = Yajl::Encoder.encode(obj[, :pretty => true, :indent => "\t"])
+    #   output = Yajl::Encoder.encode(obj[, :pretty => true, :indent => "\t", &block])
     #
     # +obj+ is a ruby object to encode to JSON format
     #
@@ -48,6 +48,8 @@ module Yajl
     # :pretty accepts a boolean and will enable/disable "pretty printing" the resulting output
     #
     # :indent accepts a string and will be used as the indent character(s) during the pretty print process
+    #
+    # If a block is passed, it will be used as (and work the same as) the +on_progress+ callback
     def self.encode(obj, *args, &block)
       # TODO: this code smells, any ideas?
       options = {}

data/lib/yajl/http_stream.rb

@@ -77,7 +77,7 @@ module Yajl
           raise Exception, "Chunked responses detected, but no block given to handle the chunks."
         end
       else
-        content_type = response_head[:headers]["Content-Type"].split('; ')
+        content_type = response_head[:headers]["Content-Type"].split(';')
         content_type = content_type.first
         if ALLOWED_MIME_TYPES.include?(content_type)
           case response_head[:headers]["Content-Encoding"]
@@ -98,4 +98,4 @@ module Yajl
       socket.close
     end
   end
-end
+end

data/spec/encoding/encoding_spec.rb

@@ -52,7 +52,7 @@ describe "Yajl JSON encoder" do
         hash = Yajl::Parser.parse(input)
         output = ''
         encoder.encode(hash) do |json_str|
-          output = json_str
+          output << json_str
         end
         hash2 = Yajl::Parser.parse(output)
         input.close

data/yajl-ruby.gemspec

@@ -2,11 +2,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{yajl-ruby}
-  s.version = "0.5.7"
+  s.version = "0.5.8"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Brian Lopez", "Lloyd Hilaiel"]
-  s.date = %q{2009-06-23}
+  s.date = %q{2009-07-06}
   s.email = %q{seniorlopez@gmail.com}
   s.extensions = ["ext/extconf.rb"]
   s.extra_rdoc_files = [
@@ -34,6 +34,9 @@ Gem::Specification.new do |s|
      "benchmark/subjects/twitter_search.json",
      "benchmark/subjects/twitter_stream.json",
      "benchmark/subjects/unicode.json",
+     "examples/encoding/chunked_encoding.rb",
+     "examples/encoding/one_shot.rb",
+     "examples/encoding/to_an_io.rb",
      "examples/http/twitter_search_api.rb",
      "examples/http/twitter_stream_api.rb",
      "examples/parsing/from_file.rb",
@@ -160,6 +163,9 @@ Gem::Specification.new do |s|
      "spec/parsing/fixtures_spec.rb",
      "spec/parsing/one_off_spec.rb",
      "spec/spec_helper.rb",
+     "examples/encoding/chunked_encoding.rb",
+     "examples/encoding/one_shot.rb",
+     "examples/encoding/to_an_io.rb",
      "examples/http/twitter_search_api.rb",
      "examples/http/twitter_stream_api.rb",
      "examples/parsing/from_file.rb",

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: yajl-ruby
 version: !ruby/object:Gem::Version 
-  version: 0.5.7
+  version: 0.5.8
 platform: ruby
 authors: 
 - Brian Lopez
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-06-23 00:00:00 -07:00
+date: 2009-07-06 00:00:00 -07:00
 default_executable: 
 dependencies: []
 
@@ -44,6 +44,9 @@ files:
 - benchmark/subjects/twitter_search.json
 - benchmark/subjects/twitter_stream.json
 - benchmark/subjects/unicode.json
+- examples/encoding/chunked_encoding.rb
+- examples/encoding/one_shot.rb
+- examples/encoding/to_an_io.rb
 - examples/http/twitter_search_api.rb
 - examples/http/twitter_stream_api.rb
 - examples/parsing/from_file.rb
@@ -192,6 +195,9 @@ test_files:
 - spec/parsing/fixtures_spec.rb
 - spec/parsing/one_off_spec.rb
 - spec/spec_helper.rb
+- examples/encoding/chunked_encoding.rb
+- examples/encoding/one_shot.rb
+- examples/encoding/to_an_io.rb
 - examples/http/twitter_search_api.rb
 - examples/http/twitter_stream_api.rb
 - examples/parsing/from_file.rb