brianmario-yajl-ruby 0.4.8 → 0.5.0

data/ext/yajl_ext.h

@@ -2,45 +2,58 @@
 #include "api/yajl_gen.h"
 #include <ruby.h>
 
-#define READ_BUFSIZE 4096
-
-static VALUE cParseError, mYajl, mStream, mChunked;
-static ID intern_io_read, intern_eof, intern_respond_to, intern_call, intern_keys, intern_to_s;
-static int readBufferSize = READ_BUFSIZE;
-static yajl_parser_config cfg = {1, 1};
-
-yajl_handle streamParser, chunkedParser;
-VALUE context = Qnil;
-VALUE parse_complete_callback = Qnil;
-
-void check_and_fire_callback(void * ctx);
-void set_static_value(void * ctx, VALUE val);
-
-static int found_null(void * ctx);
-static int found_boolean(void * ctx, int boolean);
-static int found_number(void * ctx, const char * numberVal, unsigned int numberLen);
-static int found_string(void * ctx, const unsigned char * stringVal, unsigned int stringLen);
-static int found_hash_key(void * ctx, const unsigned char * stringVal, unsigned int stringLen);
-static int found_start_hash(void * ctx);
-static int found_end_hash(void * ctx);
-static int found_start_array(void * ctx);
-static int found_end_array(void * ctx);
+#define READ_BUFSIZE 65536
 
+static VALUE cParseError, mYajl, cParser, cEncoder;
+static ID intern_io_read, intern_eof, intern_call, intern_keys, intern_to_s,
+          sym_allow_comments, sym_check_utf8, sym_pretty, sym_indent;
+
+#define GetParser(obj, sval) (sval = (struct yajl_parser_wrapper*)DATA_PTR(obj));
+#define GetEncoder(obj, sval) (sval = (yajl_gen*)DATA_PTR(obj));
+
+void yajl_check_and_fire_callback(void * ctx);
+void yajl_set_static_value(void * ctx, VALUE val);
+void yajl_encode_part(yajl_gen hand, VALUE obj, VALUE io);
+
+static int yajl_found_null(void * ctx);
+static int yajl_found_boolean(void * ctx, int boolean);
+static int yajl_found_number(void * ctx, const char * numberVal, unsigned int numberLen);
+static int yajl_found_string(void * ctx, const unsigned char * stringVal, unsigned int stringLen);
+static int yajl_found_hash_key(void * ctx, const unsigned char * stringVal, unsigned int stringLen);
+static int yajl_found_start_hash(void * ctx);
+static int yajl_found_end_hash(void * ctx);
+static int yajl_found_start_array(void * ctx);
+static int yajl_found_end_array(void * ctx);
 static yajl_callbacks callbacks = {
-    found_null,
-    found_boolean,
+    yajl_found_null,
+    yajl_found_boolean,
     NULL,
     NULL,
-    found_number,
-    found_string,
-    found_start_hash,
-    found_hash_key,
-    found_end_hash,
-    found_start_array,
-    found_end_array
+    yajl_found_number,
+    yajl_found_string,
+    yajl_found_start_hash,
+    yajl_found_hash_key,
+    yajl_found_end_hash,
+    yajl_found_start_array,
+    yajl_found_end_array
 };
 
-static VALUE t_setParseComplete(VALUE self, VALUE callback);
-static VALUE t_parseSome(VALUE self, VALUE string);
-static VALUE t_parse(VALUE self, VALUE io);
-static VALUE t_encode(VALUE self, VALUE obj, VALUE io);
+struct yajl_parser_wrapper {
+    VALUE builderStack;
+    VALUE parse_complete_callback;
+    int nestedArrayLevel;
+    int nestedHashLevel;
+    yajl_handle parser;
+};
+static void yajl_parser_wrapper_free(void * wrapper);
+static void yajl_parser_wrapper_mark(void * wrapper);
+
+static VALUE rb_yajl_parser_new(int argc, VALUE * argv, VALUE self);
+static VALUE rb_yajl_parser_init(int argc, VALUE * argv, VALUE self);
+static VALUE rb_yajl_parser_parse(int argc, VALUE * argv, VALUE self);
+static VALUE rb_yajl_parser_parse_chunk(VALUE self, VALUE chunk);
+static VALUE rb_yajl_set_complete_cb(VALUE self, VALUE callback);
+
+static VALUE rb_yajl_encoder_new(int argc, VALUE * argv, VALUE klass);
+static VALUE rb_yajl_encoder_init(int argc, VALUE * argv, VALUE self);
+static VALUE rb_yajl_encoder_encode(VALUE self, VALUE obj, VALUE io);

data/ext/yajl_lex.c

@@ -129,6 +129,13 @@ yajl_lex_alloc(yajl_alloc_funcs * alloc,
     return lxr;
 }
 
+yajl_lexer
+yajl_lex_realloc(yajl_lexer orig) {
+    yajl_lexer newLxr = yajl_lex_alloc(orig->alloc, orig->allowComments, orig->validateUTF8);
+    yajl_lex_free(orig);
+    return newLxr;
+}
+
 void
 yajl_lex_free(yajl_lexer lxr)
 {

data/ext/yajl_lex.h

@@ -67,6 +67,8 @@ yajl_lexer yajl_lex_alloc(yajl_alloc_funcs * alloc,
                           unsigned int allowComments,
                           unsigned int validateUTF8);
 
+yajl_lexer yajl_lex_realloc(yajl_lexer orig);
+
 void yajl_lex_free(yajl_lexer lexer);
 
 /**

data/ext/yajl_parser.c

@@ -307,7 +307,9 @@ yajl_do_parse(yajl_handle hand, unsigned int * offset,
             {
                 yajl_state s = yajl_bs_current(hand->stateStack);
                 if (s == yajl_state_start) {
-                    yajl_bs_set(hand->stateStack, yajl_state_parse_complete);
+                    // HACK: is this even safe to do?
+                    // yajl_bs_set(hand->stateStack, yajl_state_parse_complete);
+                    yajl_reset_parser(hand);
                 } else if (s == yajl_state_map_need_val) {
                     yajl_bs_set(hand->stateStack, yajl_state_map_got_val);
                 } else { 

data/lib/yajl.rb

@@ -13,28 +13,29 @@ require 'yajl_ext'
 #
 # Ruby bindings to the excellent Yajl (Yet Another JSON Parser) ANSI C library.
 module Yajl
-  VERSION = "0.4.8"
+  VERSION = "0.5.0"
   
-  # == Yajl::Chunked
-  #
-  # This module contains methods for parsing JSON in chunks.
-  # The use case here is that the caller may not be able to get access to the IO to which
-  # JSON content is being received. Rendering Yajl::Stream dead to them.
-  #
-  # With the methods in this module, the caller will be able to pass in chunks of JSON content
-  # until a full object has been parsed from said content.
-  #
-  # In order for this process to work correctly, the caller needs to specify a callback which
-  # is passed the constructed object. The only requirement currently of this callback is that
-  # it respond to #call and accept a single parameter (the object that was created from parsing).
-  module Chunked; end
-  
-  # == Yajl::Stream
+  # == Yajl::Parser
   #
   # This module contains methods for parsing JSON directly from an IO object.
   #
   # The only basic requirment currently is that the IO object respond to #read(len) and eof?
   #
   # The IO is parsed until a complete JSON object has been read and a ruby object will be returned.
-  module Stream; end
+  class Parser; end
+  
+  # Deprecated
+  module Stream
+    # Deprecated
+    def self.parse(io)
+      STDERR.puts "WARNING: Yajl::Stream has be deprecated and will most likely be gone in the next release. Use the Yajl::Parser class instead."
+      Parser.new.parse(io)
+    end
+    
+    # Deprecated
+    def self.encode(obj, io)
+      STDERR.puts "WARNING: Yajl::Stream has be deprecated and will most likely be gone in the next release. Use the Yajl::Encoder class instead."
+      Encoder.new.encode(obj, io)
+    end
+  end
 end

data/lib/yajl/bzip2.rb

@@ -1,6 +1,6 @@
 # encoding: UTF-8
 
-require 'yajl' unless defined?(Yajl::Stream)
+require 'yajl' unless defined?(Yajl::Parser)
 
 begin
   require 'bzip2' unless defined?(Bzip2)

data/lib/yajl/bzip2/stream_reader.rb

@@ -15,7 +15,7 @@ module Yajl
       end
 
       def self.parse(io)
-        Yajl::Stream.parse(new(io))
+        Yajl::Parser.new.parse(new(io))
       end
     end
   end

data/lib/yajl/bzip2/stream_writer.rb

@@ -4,7 +4,7 @@ module Yajl
     # === Yajl::Bzip2::StreamWriter
     class StreamWriter < ::Bzip2::Writer
       def self.encode(obj, io)
-        Yajl::Stream.encode(obj, new(io))
+        Yajl::Encoder.new.encode(obj, new(io))
       end
     end
   end

data/lib/yajl/deflate.rb

@@ -1,6 +1,6 @@
 # encoding: UTF-8
 
-require 'yajl' unless defined?(Yajl::Stream)
+require 'yajl' unless defined?(Yajl::Parser)
 require 'zlib' unless defined?(Zlib)
 require 'yajl/deflate/stream_reader.rb'
 require 'yajl/deflate/stream_writer.rb'

data/lib/yajl/deflate/stream_reader.rb

@@ -19,7 +19,7 @@ module Yajl
       alias :eof? :finished?
       
       def self.parse(io, options=nil)
-        Yajl::Stream.parse(new(io, options))
+        Yajl::Parser.new.parse(new(io, options))
       end
     end
   end

data/lib/yajl/deflate/stream_writer.rb

@@ -9,7 +9,7 @@ module Yajl
       end
       
       def self.encode(obj, io)
-        Yajl::Stream.encode(obj, new(io))
+        Yajl::Encoder.new.encode(obj, new(io))
       end
     end
   end

data/lib/yajl/gzip.rb

@@ -1,6 +1,6 @@
 # encoding: UTF-8
 
-require 'yajl' unless defined?(Yajl::Stream)
+require 'yajl' unless defined?(Yajl::Parser)
 require 'zlib' unless defined?(Zlib)
 require 'yajl/gzip/stream_reader.rb'
 require 'yajl/gzip/stream_writer.rb'

data/lib/yajl/gzip/stream_reader.rb

@@ -15,7 +15,7 @@ module Yajl
       end
       
       def self.parse(io)
-        Yajl::Stream.parse(new(io))
+        Yajl::Parser.new.parse(new(io))
       end
     end
   end

data/lib/yajl/gzip/stream_writer.rb

@@ -4,7 +4,7 @@ module Yajl
     # === Yajl::Gzip::StreamWriter
     class StreamWriter < ::Zlib::GzipWriter
       def self.encode(obj, io)
-        Yajl::Stream.encode(obj, new(io))
+        Yajl::Encoder.new.encode(obj, new(io))
       end
     end
   end

data/lib/yajl/http_stream.rb

@@ -1,6 +1,6 @@
 # encoding: UTF-8
 require 'socket' unless defined?(Socket)
-require 'yajl' unless defined?(Yajl::Stream)
+require 'yajl' unless defined?(Yajl::Parser)
 
 module Yajl
   # == Yajl::HttpStream
@@ -24,7 +24,7 @@ module Yajl
     # 3. the response is read until the end of the headers
     # 4. the _socket itself_ is passed directly to Yajl, for direct parsing off the stream;
     #    As it's being received over the wire!
-    def self.get(uri, opts = {})
+    def self.get(uri, opts = {}, &block)
       user_agent = opts.has_key?(['User-Agent']) ? opts['User-Agent'] : "Yajl::HttpStream #{Yajl::VERSION}"
       
       socket = TCPSocket.new(uri.host, uri.port)
@@ -61,9 +61,25 @@ module Yajl
           end
         end
       end
-      
+      parser = Yajl::Parser.new
       if response_head[:headers]["Transfer-Encoding"] == 'chunked'
-        raise Exception, "Chunked responses not supported yet (I'm working on this)"
+        if block_given?
+          parser.on_parse_complete = block
+          chunkLeft = 0
+          while !socket.eof? && (size = socket.gets.hex)
+            next if size == 0
+            json = socket.read(size)
+            chunkLeft = size-json.size
+            if chunkLeft == 0
+              parser << json
+            else
+              # received only part of the chunk, grab the rest
+              parser << socket.read(chunkLeft)
+            end
+          end
+        else
+          raise Exception, "Chunked responses detected, but no block given to handle the chunks."
+        end
       else
         content_type = response_head[:headers]["Content-Type"].split('; ')
         content_type = content_type.first
@@ -76,7 +92,7 @@ module Yajl
           when "bzip2"
             return Yajl::Bzip2::StreamReader.parse(socket)
           else
-            return Yajl::Stream.parse(socket)
+            return Yajl::Parser.new.parse(socket)
           end
         else
           raise InvalidContentType, "The response MIME type #{content_type}"

data/spec/encoding/encoding_spec.rb

@@ -6,18 +6,23 @@ describe "Yajl JSON encoder" do
   
   FILES.each do |file|
     it "should encode #{File.basename(file)}" do
-      input = File.new(File.expand_path(file), 'r')
-      hash = Yajl::Stream.parse(input)
+      # we don't care about testing the stream subject as it has multiple JSON strings in it
+      if File.basename(file) != 'twitter_stream.json'
+        input = File.new(File.expand_path(file), 'r')
+        io = StringIO.new
+        parser = Yajl::Parser.new
+        encoder = Yajl::Encoder.new
       
-      io = StringIO.new
-      output = Yajl::Stream.encode(hash, io)
-      io.rewind
-      hash2 = Yajl::Stream.parse(io)
+        hash = parser.parse(input)
+        output = encoder.encode(hash, io)
+        io.rewind
+        hash2 = parser.parse(io)
       
-      io.close
-      input.close
+        io.close
+        input.close
       
-      hash.should == hash2
+        hash.should == hash2
+      end
     end
   end
 end

data/spec/http/http_spec.rb

@@ -21,7 +21,7 @@ describe "Yajl HTTP GET request" do
     @gzip = File.new(File.expand_path(File.dirname(__FILE__) + '/fixtures/http.gzip.dump'), 'r')
     
     parse_off_headers(@raw)
-    @raw_template_hash = Yajl::Stream.parse(@raw)
+    @raw_template_hash = Yajl::Parser.new.parse(@raw)
     @raw.rewind
   end
   
@@ -32,10 +32,6 @@ describe "Yajl HTTP GET request" do
     @gzip.close unless @gzip.closed?
   end
   
-  after(:each) do
-    GC.start
-  end
-  
   it "should parse a raw response" do
     file = File.expand_path(File.dirname(__FILE__) + '/http/http.raw.dump')
     uri = 'file://'+file

data/spec/parsing/active_support_spec.rb

@@ -37,14 +37,16 @@ describe "ActiveSupport test cases" do
   TESTS.each do |json, expected|
     it "should be able to parse #{json}" do
       lambda {
-        Yajl::Stream.parse(StringIO.new(json)).should == expected
+        parser = Yajl::Parser.new
+        parser.parse(StringIO.new(json)).should == expected
       }.should_not raise_error(Yajl::ParseError)
     end
   end
 
   it "should fail parsing {: 1}" do
     lambda {
-      Yajl::Stream.parse(StringIO.new("{: 1}"))
-      }.should raise_error(Yajl::ParseError)
+      parser = Yajl::Parser.new
+      parser.parse(StringIO.new("{: 1}"))
+    }.should raise_error(Yajl::ParseError)
   end
 end

data/spec/parsing/chunked_spec.rb

@@ -0,0 +1,72 @@
+# encoding: UTF-8
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper.rb')
+require 'stringio'
+
+describe "Chunked parser" do
+  before(:all) do
+    @final = [{"abc" => 123}, {"def" => 456}]
+  end
+  
+  before(:each) do
+    @callback = lambda { |hash|
+      # no-op
+    }
+    @parser = Yajl::Parser.new
+    @parser.on_parse_complete = @callback
+  end
+  
+  it "should parse a single chunk" do
+    @callback.should_receive(:call).with(@final)
+    @parser << '[{"abc": 123},{"def": 456}]'
+  end
+  
+  it "should parse a single chunk, 3 times" do
+    @callback.should_receive(:call).with(@final).exactly(3).times
+    @parser << '[{"abc": 123},{"def": 456}]'
+    @parser << '[{"abc": 123},{"def": 456}]'
+    @parser << '[{"abc": 123},{"def": 456}]'
+  end
+  
+  it "should parse in two chunks" do
+    @callback.should_receive(:call).with(@final)
+    @parser << '[{"abc": 123},'
+    @parser << '{"def": 456}]'
+  end
+  
+  it "should parse in 2 chunks, twice" do
+    @callback.should_receive(:call).with(@final).exactly(2).times
+    @parser << '[{"abc": 123},'
+    @parser << '{"def": 456}]'
+    @parser << '[{"abc": 123},'
+    @parser << '{"def": 456}]'
+  end
+  
+  it "should parse 2 JSON strings, in 3 chunks" do
+    @callback.should_receive(:call).with(@final).exactly(2).times
+    @parser << '[{"abc": 123},'
+    @parser << '{"def": 456}][{"abc": 123},{"def":'
+    @parser << ' 456}]'
+  end
+  
+  it "should parse 2 JSON strings in 1 chunk" do
+    @callback.should_receive(:call).with(@final).exactly(2).times
+    @parser << '[{"abc": 123},{"def": 456}][{"abc": 123},{"def": 456}]'
+  end
+  
+  it "should parse 2 JSON strings from an IO" do
+    @callback.should_receive(:call).with(@final).exactly(2).times
+    @parser.parse(StringIO.new('[{"abc": 123},{"def": 456}][{"abc": 123},{"def": 456}]'))
+  end
+  
+  it "should parse a JSON string an IO and fire callback once" do
+    @callback.should_receive(:call).with(@final)
+    @parser.parse(StringIO.new('[{"abc": 123},{"def": 456}]'))
+  end
+  
+  it "should parse twitter_stream.json and fire callback 430 times" do
+    path = File.expand_path(File.dirname(__FILE__) + '/../../benchmark/subjects/twitter_stream.json')
+    json = File.new(path, 'r')
+    @callback.should_receive(:call).exactly(430).times
+    @parser.parse(json)
+  end
+end