yajl-ruby 0.6.3 → 0.6.4

data/CHANGELOG.md

@@ -1,5 +1,18 @@
 # Changelog
 
+## 0.6.4 (November 4th, 2009)
+* All specs pass on Rubinius :)
+* Fixed a bug in Yajl::Encoder which allowed direct, unescaped encoding of NaN, Infinity and -Infinity.
+  It will now properly throw a Yajl::EncodeError exception if either of these values are found unescaped.
+* Update bundled Yajl library to 1.0.7
+* Conditionally define RSTRING_* and RARRAY_* for older versions of ruby (1.8.5 is still the default on CentOS)
+* Bugfix for JSON gem exception classes to more accurately match those of the actual JSON gem
+* A few small speed optimizations
+* Updated specs to not run bzip2 related examples if unable to load the bzip2 library
+* Finally added UTF-8 checking specs
+* Removed needless calls to ID2SYM all over the place
+* Updated benchmark scripts to bring the GC into the picture a little more
+
 ## 0.6.3 (August 25th, 2009)
 * Fixed a bug in the JSON gem compatibility API where strings weren't being properly escaped
 

data/README.rdoc

@@ -2,7 +2,7 @@
 
 This gem is a C binding to the excellent YAJL JSON parsing and generation library.
 
-You can read more info at the projects website http://lloydforge.org/projects/yajl or check out it's codes at http://github.com/lloyd/yajl.
+You can read more info at the projects website http://lloyd.github.com/yajl or check out it's codes at http://github.com/lloyd/yajl.
 
 == Features
 
@@ -19,11 +19,13 @@ You can read more info at the projects website http://lloydforge.org/projects/ya
 
 == How to install
 
-Install it like any other gem hosted at the Githubs like so:
+First make sure you've got Gemcutter in your sources list:
 
-(more instructions here: http://gems.github.com)
+ gem sources -a http://gemcutter.org
 
- sudo gem install brianmario-yajl-ruby
+Then go ahead and install it as usual:
+
+ sudo gem install yajl-ruby
 
 == Example of use
 
@@ -216,10 +218,10 @@ There are a lot more possibilities that I'd love to see other gems/plugins for s
 
 Some ideas are:
 * parsing logs in JSON format
-* a Rails plugin (http://github.com/technoweenie/yajl-rails)
+* a Rails plugin - DONE! (http://github.com/technoweenie/yajl-rails)
 * builtin support in Rails 3?
-* Rack middleware (ideally the JSON body could be handed to the parser while it's still being received)
-* use with ohai (http://github.com/brianmario/ohai)
+* Rack middleware (ideally the JSON body could be handed to the parser while it's still being received, this is apparently possible with Unicorn)
+* use with ohai - DONE! (http://github.com/opscode/ohai/commit/f89baccc3b9ab587d23e0b6257f6fedffe223c02)
 * JSON API clients (http://github.com/brianmario/crack, http://github.com/brianmario/freckle-api)
 * Patch Marshal#load and Marshal#dump to use JSON? ;)
 * etc...

data/VERSION.yml

@@ -1,4 +1,4 @@
 --- 
-:minor: 6
-:patch: 3
+:patch: 4
 :major: 0
+:minor: 6

data/benchmark/encode.rb

@@ -12,9 +12,9 @@ json = File.new(filename, 'r')
 hash = Yajl::Parser.new.parse(json)
 json.close
 
-times = ARGV[1] ? ARGV[1].to_i : 1
+times = ARGV[1] ? ARGV[1].to_i : 1000
 puts "Starting benchmark encoding #{filename} #{times} times\n\n"
-Benchmark.bm { |x|
+Benchmark.bmbm { |x|
   io_encoder = Yajl::Encoder.new
   x.report {
     puts "Yajl::Encoder#encode (to an IO)"
@@ -30,7 +30,7 @@ Benchmark.bm { |x|
     }
   }
   x.report {
-    puts "JSON's #to_json"
+    puts "JSON.generate"
     times.times {
       JSON.generate(hash)
     }

data/benchmark/encode_json_and_marshal.rb

@@ -5,14 +5,14 @@ require 'yajl_ext'
 require 'stringio'
 require 'json'
 
-times = ARGV[0] ? ARGV[0].to_i : 1
+times = ARGV[0] ? ARGV[0].to_i : 1000
 filename = 'benchmark/subjects/ohai.json'
 json = File.new(filename, 'r')
 hash = Yajl::Parser.new.parse(json)
 json.close
 
 puts "Starting benchmark encoding #{filename} #{times} times\n\n"
-Benchmark.bm { |x|
+Benchmark.bmbm { |x|
   encoder = Yajl::Encoder.new
   x.report {
     puts "Yajl::Encoder#encode"

data/benchmark/encode_json_and_yaml.rb

@@ -11,9 +11,9 @@ json = File.new(filename, 'r')
 hash = Yajl::Parser.new.parse(json)
 json.close
 
-times = ARGV[0] ? ARGV[0].to_i : 1
+times = ARGV[0] ? ARGV[0].to_i : 1000
 puts "Starting benchmark encoding #{filename} into JSON #{times} times\n\n"
-Benchmark.bm { |x|
+Benchmark.bmbm { |x|
   encoder = Yajl::Encoder.new
   x.report {
     puts "Yajl::Encoder#encode"
@@ -30,14 +30,13 @@ Benchmark.bm { |x|
 }
 
 # YAML Section
-filename = 'benchmark/subjects/contacts.yml'
+filename = 'benchmark/subjects/ohai.yml'
 yml = File.new(filename, 'r')
 data = YAML.load_stream(yml)
 yml.close
 
-times = ARGV[0] ? ARGV[0].to_i : 1
 puts "Starting benchmark encoding #{filename} into YAML #{times} times\n\n"
-Benchmark.bm { |x|
+Benchmark.bmbm { |x|
   x.report {
     puts "YAML.dump"
     times.times {

data/benchmark/http.rb

@@ -4,7 +4,7 @@ require 'benchmark'
 require 'yajl/http_stream'
 require 'yajl/gzip'
 require 'yajl/deflate'
-require 'yajl/bzip2'
+require 'yajl/bzip2' unless defined?(Bzip2)
 require 'json'
 require 'uri'
 require 'net/http'
@@ -14,7 +14,7 @@ uri = URI.parse('http://search.twitter.com/search.json?q=github')
 
 times = ARGV[0] ? ARGV[0].to_i : 1
 puts "Starting benchmark parsing #{uri.to_s} #{times} times\n\n"
-Benchmark.bm { |x|
+Benchmark.bmbm { |x|
   x.report {
     puts "Yajl::HttpStream.get"
     times.times {

data/benchmark/parse.rb

@@ -5,17 +5,14 @@ require 'yajl_ext'
 require 'json'
 require 'activesupport'
 
-filename = ARGV[0] || 'benchmark/subjects/ohai.json'
+filename = ARGV[0] || 'benchmark/subjects/twitter_search.json'
 json = File.new(filename, 'r')
 
-# warm up the filesystem
-json.read
-json.rewind
-
-times = ARGV[1] ? ARGV[1].to_i : 1
+times = ARGV[1] ? ARGV[1].to_i : 1000
 puts "Starting benchmark parsing #{File.size(filename)} bytes of JSON data #{times} times\n\n"
-Benchmark.bm { |x|
+Benchmark.bmbm { |x|
   io_parser = Yajl::Parser.new
+  io_parser.on_parse_complete = lambda {|obj|} if times > 1
   x.report {
     puts "Yajl::Parser#parse (from an IO)"
     times.times {
@@ -24,6 +21,7 @@ Benchmark.bm { |x|
     }
   }
   string_parser = Yajl::Parser.new
+  string_parser.on_parse_complete = lambda {|obj|} if times > 1
   x.report {
     puts "Yajl::Parser#parse (from a String)"
     times.times {

data/benchmark/parse_json_and_marshal.rb

@@ -10,22 +10,18 @@ marshal_filename = 'benchmark/subjects/ohai.marshal_dump'
 json = File.new(filename, 'r')
 marshal_file = File.new(marshal_filename, 'r')
 
-# warm up the filesystem
-json.read
-json.rewind
-marshal_file.read
-marshal_file.rewind
-
 hash = {}
 
-times = ARGV[0] ? ARGV[0].to_i : 1
+times = ARGV[0] ? ARGV[0].to_i : 1000
 puts "Starting benchmark parsing #{File.size(filename)} bytes of JSON data #{times} times\n\n"
-Benchmark.bm { |x|
+Benchmark.bmbm { |x|
   x.report {
     puts "Yajl::Parser#parse"
+    yajl = Yajl::Parser.new
+    yajl.on_parse_complete = lambda {|obj|} if times > 1
     times.times {
       json.rewind
-      hash = Yajl::Parser.new.parse(json)
+      hash = yajl.parse(json)
     }
   }
   x.report {

data/benchmark/parse_json_and_yaml.rb

@@ -9,14 +9,11 @@ require 'yaml'
 filename = 'benchmark/subjects/ohai.json'
 json = File.new(filename, 'r')
 
-# warm up the filesystem
-json.read
-json.rewind
-
-times = ARGV[0] ? ARGV[0].to_i : 1
+times = ARGV[0] ? ARGV[0].to_i : 1000
 puts "Starting benchmark parsing #{File.size(filename)} bytes of JSON data #{times} times\n\n"
-Benchmark.bm { |x|
+Benchmark.bmbm { |x|
   parser = Yajl::Parser.new
+  parser.on_parse_complete = lambda {|obj|} if times > 1
   x.report {
     puts "Yajl::Parser#parse"
     times.times {
@@ -35,16 +32,11 @@ Benchmark.bm { |x|
 json.close
 
 # YAML section
-filename = 'benchmark/subjects/contacts.yml'
+filename = 'benchmark/subjects/ohai.yml'
 yaml = File.new(filename, 'r')
 
-# warm up the filesystem
-yaml.read
-yaml.rewind
-
-times = ARGV[0] ? ARGV[0].to_i : 1
 puts "Starting benchmark parsing #{File.size(filename)} bytes of YAML data #{times} times\n\n"
-Benchmark.bm { |x|
+Benchmark.bmbm { |x|
   x.report {
     puts "YAML.load_stream"
     times.times {

data/benchmark/parse_stream.rb

@@ -3,22 +3,16 @@ require 'rubygems'
 require 'benchmark'
 require 'yajl_ext'
 require 'json'
-# require 'activesupport'
+require 'activesupport'
 
 filename = 'benchmark/subjects/twitter_stream.json'
 json = File.new(filename, 'r')
 
-# warm up the filesystem
-json.read
-json.rewind
-
-times = ARGV[0] ? ARGV[0].to_i : 1
+times = ARGV[0] ? ARGV[0].to_i : 100
 puts "Starting benchmark parsing JSON stream (#{File.size(filename)} bytes of JSON data with 430 JSON separate strings) #{times} times\n\n"
-Benchmark.bm { |x|
+Benchmark.bmbm { |x|
   parser = Yajl::Parser.new
-  parser.on_parse_complete = lambda { |obj|
-    # no-op
-  }
+  parser.on_parse_complete = lambda {|obj|}
   x.report {
     puts "Yajl::Parser#parse"
     times.times {
@@ -35,14 +29,14 @@ Benchmark.bm { |x|
       end
     }
   }
-  # x.report {
-  #   puts "ActiveSupport::JSON.decode"
-  #   times.times {
-  #     json.rewind
-  #     while chunk = json.gets
-  #       ActiveSupport::JSON.decode(chunk)
-  #     end
-  #   }
-  # }
+  x.report {
+    puts "ActiveSupport::JSON.decode"
+    times.times {
+      json.rewind
+      while chunk = json.gets
+        ActiveSupport::JSON.decode(chunk)
+      end
+    }
+  }
 }
 json.close

data/ext/api/yajl_gen.h

@@ -57,7 +57,10 @@ extern "C" {
          *  state */
         yajl_gen_in_error_state,
         /** A complete JSON document has been generated */
-        yajl_gen_generation_complete                
+        yajl_gen_generation_complete,                
+        /** yajl_gen_double was passed an invalid floating point value
+         *  (infinity or NaN). */
+        yajl_gen_invalid_number
     } yajl_gen_status;
 
     /** an opaque handle to a generator */
@@ -90,6 +93,9 @@ extern "C" {
     void YAJL_API yajl_gen_free(yajl_gen handle);
 
     yajl_gen_status YAJL_API yajl_gen_integer(yajl_gen hand, long int number);
+    /** generate a floating point number.  number may not be infinity or
+     *  NaN, as these have no representation in JSON.  In these cases the
+     *  generator will return 'yajl_gen_invalid_number' */
     yajl_gen_status YAJL_API yajl_gen_double(yajl_gen hand, double number);
     yajl_gen_status YAJL_API yajl_gen_number(yajl_gen hand,
                                              const char * num,

data/ext/yajl_ext.c

@@ -1,11 +1,11 @@
 #include "yajl_ext.h"
 
-// Helpers for building objects
+/* Helpers for building objects */
 inline void yajl_check_and_fire_callback(void * ctx) {
     struct yajl_parser_wrapper * wrapper;
     GetParser((VALUE)ctx, wrapper);
     
-    // No need to do any of this if the callback isn't even setup
+    /* No need to do any of this if the callback isn't even setup */
     if (wrapper->parse_complete_callback != Qnil) {
         int len = RARRAY_LEN(wrapper->builderStack);
         if (len == 1 && wrapper->nestedArrayLevel == 0 && wrapper->nestedHashLevel == 0) {
@@ -98,15 +98,15 @@ void yajl_encode_part(void * wrapper, VALUE obj, VALUE io) {
         case T_HASH:
             status = yajl_gen_map_open(w->encoder);
             
-            // TODO: itterate through keys in the hash
+            /* TODO: itterate through keys in the hash */
             VALUE keys = rb_funcall(obj, intern_keys, 0);
             VALUE entry, keyStr;
             for(idx=0; idx<RARRAY_LEN(keys); idx++) {
                 entry = rb_ary_entry(keys, idx);
-                keyStr = rb_funcall(entry, intern_to_s, 0); // key must be a string
-                // the key
+                keyStr = rb_funcall(entry, intern_to_s, 0); /* key must be a string */
+                /* the key */
                 yajl_encode_part(w, keyStr, io);
-                // the value
+                /* the value */
                 yajl_encode_part(w, rb_hash_aref(obj, entry), io);
             }
             
@@ -133,6 +133,9 @@ void yajl_encode_part(void * wrapper, VALUE obj, VALUE io) {
         case T_FLOAT:
         case T_BIGNUM:
             str = rb_funcall(obj, intern_to_s, 0);
+            if (strcmp(RSTRING_PTR(str), "NaN") == 0 || strcmp(RSTRING_PTR(str), "Infinity") == 0 || strcmp(RSTRING_PTR(str), "-Infinity") == 0) {
+                rb_raise(cEncodeError, "'%s' is an invalid number", RSTRING_PTR(str));
+            }
             status = yajl_gen_number(w->encoder, RSTRING_PTR(str), (unsigned int)RSTRING_LEN(str));
             break;
         case T_STRING:
@@ -141,7 +144,7 @@ void yajl_encode_part(void * wrapper, VALUE obj, VALUE io) {
         default:
             if (rb_respond_to(obj, intern_to_json)) {
                 str = rb_funcall(obj, intern_to_json, 0);
-                quote_strings = 0; // this lets us append on to the buffer without Yajl quoting it again
+                quote_strings = 0; /* this lets us append on to the buffer without Yajl quoting it again */
             } else {
                 str = rb_funcall(obj, intern_to_s, 0);
             }
@@ -174,7 +177,7 @@ void yajl_parse_chunk(const unsigned char * chunk, unsigned int len, yajl_handle
     }
 }
 
-// YAJL Callbacks
+/* YAJL Callbacks */
 static int yajl_found_null(void * ctx) {
     yajl_set_static_value(ctx, Qnil);
     yajl_check_and_fire_callback(ctx);
@@ -188,15 +191,19 @@ static int yajl_found_boolean(void * ctx, int boolean) {
 }
 
 static int yajl_found_number(void * ctx, const char * numberVal, unsigned int numberLen) {
-    VALUE subString = rb_str_new(numberVal, numberLen);
-    char * cSubString = RSTRING_PTR(subString);
+    char * cSubString = ALLOC_N(char, numberLen+1);
+    if (cSubString) {
+        memcpy(cSubString, numberVal, numberLen);
+    }
+    cSubString[numberLen] = '\0';
     
-    if (strstr(cSubString, ".") != NULL || strstr(cSubString, "e") != NULL || strstr(cSubString, "E") != NULL) {
-            yajl_set_static_value(ctx, rb_Float(subString));
+    if (strchr(cSubString, '.') != NULL || strchr(cSubString, 'e') != NULL || strchr(cSubString, 'E') != NULL) {
+        yajl_set_static_value(ctx, rb_float_new(atof(cSubString)));
     } else {
-        yajl_set_static_value(ctx, rb_Integer(subString));
+        yajl_set_static_value(ctx, INT2FIX(atoi(cSubString)));
     }
     yajl_check_and_fire_callback(ctx);
+    free(cSubString);
     return 1;
 }
 
@@ -210,10 +217,9 @@ static int yajl_found_hash_key(void * ctx, const unsigned char * stringVal, unsi
     struct yajl_parser_wrapper * wrapper;
     GetParser((VALUE)ctx, wrapper);
     VALUE keyStr = rb_str_new((const char *)stringVal, stringLen);
-    
+
     if (wrapper->symbolizeKeys) {
-        ID key = rb_intern(RSTRING_PTR(keyStr));
-        yajl_set_static_value(ctx, ID2SYM(key));
+        yajl_set_static_value(ctx, rb_funcall(keyStr, intern_to_sym, 0));
     } else {
         yajl_set_static_value(ctx, keyStr);
     }
@@ -260,7 +266,7 @@ static int yajl_found_end_array(void * ctx) {
 }
 
 
-// Ruby Interface
+/* Ruby Interface */
 
 /*
  * Document-class: Yajl::Parser
@@ -287,17 +293,17 @@ static VALUE rb_yajl_parser_new(int argc, VALUE * argv, VALUE klass) {
     VALUE opts, obj;
     int allowComments = 1, checkUTF8 = 1, symbolizeKeys = 0;
     
-    // Scan off config vars
+    /* Scan off config vars */
     if (rb_scan_args(argc, argv, "01", &opts) == 1) {
         Check_Type(opts, T_HASH);
         
-        if (rb_hash_aref(opts, ID2SYM(sym_allow_comments)) == Qfalse) {
+        if (rb_hash_aref(opts, sym_allow_comments) == Qfalse) {
             allowComments = 0;
         }
-        if (rb_hash_aref(opts, ID2SYM(sym_check_utf8)) == Qfalse) {
+        if (rb_hash_aref(opts, sym_check_utf8) == Qfalse) {
             checkUTF8 = 0;
         }
-        if (rb_hash_aref(opts, ID2SYM(sym_symbolize_keys)) == Qtrue) {
+        if (rb_hash_aref(opts, sym_symbolize_keys) == Qtrue) {
             symbolizeKeys = 1;
         }
     }
@@ -354,12 +360,11 @@ static VALUE rb_yajl_parser_init(int argc, VALUE * argv, VALUE self) {
 static VALUE rb_yajl_parser_parse(int argc, VALUE * argv, VALUE self) {
     yajl_status stat;
     struct yajl_parser_wrapper * wrapper;
-    VALUE parsed, rbufsize, input, blk;
+    VALUE rbufsize, input, blk;
     
     GetParser(self, wrapper);
-    parsed = rb_str_new2("");
     
-    // setup our parameters
+    /* setup our parameters */
     rb_scan_args(argc, argv, "11&", &input, &rbufsize, &blk);
     if (NIL_P(rbufsize)) {
         rbufsize = INT2FIX(READ_BUFSIZE);
@@ -373,6 +378,7 @@ static VALUE rb_yajl_parser_parse(int argc, VALUE * argv, VALUE self) {
     if (TYPE(input) == T_STRING) {
         yajl_parse_chunk((const unsigned char *)RSTRING_PTR(input), RSTRING_LEN(input), wrapper->parser);
     } else if (rb_respond_to(input, intern_eof)) {
+        VALUE parsed = rb_str_new2("");
         while (rb_funcall(input, intern_eof, 0) != Qtrue) {
             rb_funcall(input, intern_io_read, 2, rbufsize, parsed);
             yajl_parse_chunk((const unsigned char *)RSTRING_PTR(parsed), RSTRING_LEN(parsed), wrapper->parser);
@@ -381,7 +387,7 @@ static VALUE rb_yajl_parser_parse(int argc, VALUE * argv, VALUE self) {
         rb_raise(cParseError, "input must be a string or IO");
     }
     
-    // parse any remaining buffered data
+    /* parse any remaining buffered data */
     stat = yajl_parse_complete(wrapper->parser);
     
     if (wrapper->parse_complete_callback != Qnil) {
@@ -467,13 +473,13 @@ static VALUE rb_yajl_encoder_new(int argc, VALUE * argv, VALUE klass) {
     const char * indentString = "  ";
     int beautify = 0;
     
-    // Scan off config vars
+    /* Scan off config vars */
     if (rb_scan_args(argc, argv, "01", &opts) == 1) {
         Check_Type(opts, T_HASH);
         
-        if (rb_hash_aref(opts, ID2SYM(sym_pretty)) == Qtrue) {
+        if (rb_hash_aref(opts, sym_pretty) == Qtrue) {
             beautify = 1;
-            indent = rb_hash_aref(opts, ID2SYM(sym_indent));
+            indent = rb_hash_aref(opts, sym_indent);
             if (indent != Qnil) {
                 Check_Type(indent, T_STRING);
                 indentString = RSTRING_PTR(indent);
@@ -485,8 +491,8 @@ static VALUE rb_yajl_encoder_new(int argc, VALUE * argv, VALUE klass) {
     obj = Data_Make_Struct(klass, struct yajl_encoder_wrapper, yajl_encoder_wrapper_mark, yajl_encoder_wrapper_free, wrapper);
     wrapper->encoder = yajl_gen_alloc(&cfg, NULL);
     wrapper->on_progress_callback = Qnil;
-    if (opts != Qnil && rb_funcall(opts, intern_has_key, 1, ID2SYM(sym_terminator)) == Qtrue) {
-        wrapper->terminator = rb_hash_aref(opts, ID2SYM(sym_terminator));
+    if (opts != Qnil && rb_funcall(opts, intern_has_key, 1, sym_terminator) == Qtrue) {
+        wrapper->terminator = rb_hash_aref(opts, sym_terminator);
     } else {
         wrapper->terminator = 0;
     }
@@ -543,10 +549,10 @@ static VALUE rb_yajl_encoder_encode(int argc, VALUE * argv, VALUE self) {
         wrapper->on_progress_callback = blk;
     }
     
-    // begin encode process
+    /* begin encode process */
     yajl_encode_part(wrapper, obj, io);
 
-    // just make sure we output the remaining buffer
+    /* just make sure we output the remaining buffer */
     yajl_gen_get_buf(wrapper->encoder, &buffer, &len);
     outBuff = rb_str_new((const char *)buffer, len);
     yajl_gen_clear(wrapper->encoder);
@@ -590,7 +596,7 @@ static VALUE rb_yajl_encoder_set_progress_cb(VALUE self, VALUE callback) {
 }
 
 
-// JSON Gem compatibility
+/* JSON Gem compatibility */
 
 /*
  * Document-class: Hash
@@ -783,7 +789,7 @@ static VALUE rb_yajl_encoder_enable_json_gem_ext(VALUE klass) {
 }
 
 
-// Ruby Extension initializer
+/* Ruby Extension initializer */
 void Init_yajl_ext() {
     mYajl = rb_define_module("Yajl");
     
@@ -812,12 +818,13 @@ void Init_yajl_ext() {
     intern_keys = rb_intern("keys");
     intern_to_s = rb_intern("to_s");
     intern_to_json = rb_intern("to_json");
-    
-    sym_allow_comments = rb_intern("allow_comments");
-    sym_check_utf8 = rb_intern("check_utf8");
-    sym_pretty = rb_intern("pretty");
-    sym_indent = rb_intern("indent");
-    sym_terminator = rb_intern("terminator");
-    sym_symbolize_keys = rb_intern("symbolize_keys");
+    intern_to_sym = rb_intern("to_sym");
     intern_has_key = rb_intern("has_key?");
+    
+    sym_allow_comments = ID2SYM(rb_intern("allow_comments"));
+    sym_check_utf8 = ID2SYM(rb_intern("check_utf8"));
+    sym_pretty = ID2SYM(rb_intern("pretty"));
+    sym_indent = ID2SYM(rb_intern("indent"));
+    sym_terminator = ID2SYM(rb_intern("terminator"));
+    sym_symbolize_keys = ID2SYM(rb_intern("symbolize_keys"));
 }