brianmario-yajl-ruby 0.4.3 → 0.4.4

data/.gitignore

@@ -1,4 +1,5 @@
 Makefile
 *.o
 *.dylib
-*.bundle
+*.bundle
+TODO.txt

data/CHANGELOG.rdoc

@@ -1,5 +1,21 @@
 = Changelog
 
+0.4.4 (May 12th, 2009)
+ * NOTE: Breaking API change:
+ ** renamed Yajl::GzipStreamReader to Yajl::Gzip::StreamReader
+ * added Yajl::Bzip2::StreamReader
+ ** depends on the bzip2-ruby gem if you want to use it, if not Yajl::Bzip2 won't be loaded
+ * added Yajl::Deflate::StreamReader
+ ** actually uses Zlib::Inflate for stream decompression
+ * added parse(io) class methods to Yajl::Gzip::StreamReader and Yajl::Bzip2::StreamReader as
+   a helper for parsing compressed streams.
+ * updated Yajl::HttpStream to request responses compressed as deflate and bzip2 in addition
+   to gzip
+ * fixed a bug regarding parsing Integers as Floats (so 123456 would have be parsed and returned as 123456.0)
+ * fixed a bug which caused a segfault in ruby's GC during string replacement in Yajl::Gzip and Yajl::Bzip2's
+   StreamReader#read methods
+ * added support for user-specified User-Agent strings in Yajl::HttpStream
+
 0.4.3 (May 2nd, 2009)
  * adding text/plain as an allowed mime-type for Yajl::HttpStream for webservers that respond
    with it instead of application/json (ahem...Yelp...)

data/README.rdoc

@@ -15,16 +15,16 @@ Then maybe parse some JSON from:
 a File IO
 
  json_contents = File.new('test.json', 'r')
- hash = Yajl::Native.parse(json)
+ hash = Yajl::Stream.parse(json)
  
 or maybe a StringIO
 
  json_contents = StringIO.new
- hash = Yajl::Native.parse(json)
+ hash = Yajl::Stream.parse(json)
 
 or maybe STDIN
 
- cat someJsonFile.json | ruby -ryajl -e "puts Yajl::Native.parse(STDIN).inspect"
+ cat someJsonFile.json | ruby -ryajl -e "puts Yajl::Stream.parse(STDIN).inspect"
 
 
 Or lets say you didn't have access to the IO object that contained JSON data, but instead

data/VERSION.yml

@@ -1,4 +1,4 @@
 --- 
-:patch: 3
-:major: 0
 :minor: 4
+:patch: 4
+:major: 0

data/benchmark/http.rb

@@ -1,14 +1,17 @@
 # encoding: UTF-8
 require 'rubygems'
 require 'benchmark'
-require 'yajl'
 require 'yajl/http_stream'
+require 'yajl/gzip'
+require 'yajl/deflate'
+# require 'yajl/bzip2'
 require 'json'
 require 'activesupport'
 require 'uri'
 require 'net/http'
 
 uri = URI.parse('http://search.twitter.com/search.json?q=github')
+# uri = URI.parse('http://localhost/yajl-ruby.git/benchmark/subjects/contacts.json')
 
 times = ARGV[0] ? ARGV[0].to_i : 1
 puts "Starting benchmark parsing #{uri.to_s} #{times} times\n\n"

data/ext/yajl.c

@@ -57,23 +57,24 @@ static int found_boolean(void * ctx, int boolean) {
 }
 
 static int found_number(void * ctx, const char * numberVal, unsigned int numberLen) {
-    if (strstr(numberVal, ".") != NULL || strstr(numberVal, "e") != NULL || strstr(numberVal, "E") != NULL) {
-        set_static_value(ctx, rb_Float(rb_str_new(numberVal, numberLen)));
+    VALUE subString = rb_str_new(numberVal, numberLen);
+    if (strstr(RSTRING_PTR(subString), ".") != NULL || strstr(RSTRING_PTR(subString), "e") != NULL || strstr(RSTRING_PTR(subString), "E") != NULL) {
+        set_static_value(ctx, rb_Float(subString));
     } else {
-        set_static_value(ctx, rb_Integer(rb_str_new(numberVal, numberLen)));
+        set_static_value(ctx, rb_Integer(subString));
     }
     check_and_fire_callback(ctx);
     return 1;
 }
 
 static int found_string(void * ctx, const unsigned char * stringVal, unsigned int stringLen) {
-    set_static_value(ctx, rb_str_new((char *)stringVal, stringLen));
+    set_static_value(ctx, rb_str_new((const char *)stringVal, stringLen));
     check_and_fire_callback(ctx);
     return 1;
 }
 
 static int found_hash_key(void * ctx, const unsigned char * stringVal, unsigned int stringLen) {
-    set_static_value(ctx, rb_str_new((char *)stringVal, stringLen));
+    set_static_value(ctx, rb_str_new((const char *)stringVal, stringLen));
     return 1;
 }
 
@@ -153,7 +154,7 @@ static VALUE t_parse(VALUE self, VALUE io) {
     VALUE rbufsize = INT2FIX(readBufferSize);
     
     // now parse from the IO
-    while (rb_funcall(io, intern_eof, 0) == Qfalse) {
+    while (rb_funcall(io, intern_eof, 0) != Qtrue) {
         rb_funcall(io, intern_io_read, 2, rbufsize, parsed);
         
         stat = yajl_parse(streamParser, (const unsigned char *)RSTRING_PTR(parsed), RSTRING_LEN(parsed));

data/lib/yajl/bzip2/stream_reader.rb

@@ -0,0 +1,22 @@
+# encoding: UTF-8
+module Yajl
+  module Bzip2
+    # === Yajl::Bzip2::StreamReader
+    #
+    # This is a wrapper around Bzip::Reader to allow it's #read method to adhere
+    # to the IO spec, allowing for two parameters (length, and buffer)
+    class StreamReader < ::Bzip2::Reader
+      def read(len=nil, buffer=nil)
+        unless buffer.nil?
+          buffer.replace super(len)
+          return buffer
+        end
+        super(len)
+      end
+
+      def self.parse(io)
+        Yajl::Stream.parse(new(io))
+      end
+    end
+  end
+end

data/lib/yajl/bzip2.rb

@@ -0,0 +1,10 @@
+# encoding: UTF-8
+
+require 'yajl.rb' unless defined?(Yajl::Stream)
+
+begin
+  require 'bzip2' unless defined?(Bzip2)
+  require 'yajl/bzip2/stream_reader.rb'
+rescue LoadError => e
+  raise "Unable to load the bzip2 library. Is the bzip2-ruby gem installed?"
+end

data/lib/yajl/deflate/stream_reader.rb

@@ -0,0 +1,26 @@
+# encoding: UTF-8
+module Yajl
+  module Deflate
+    # === Yajl::Deflate::StreamReader
+    #
+    # This is a wrapper around Zlib::Inflate, creating a #read method that adheres
+    # to the IO spec, allowing for two parameters (length, and buffer)
+    class StreamReader < ::Zlib::Inflate
+      def initialize(io, options)
+        @io = io
+        super(options)
+      end
+      
+      def read(len=nil, buffer=nil)
+        buffer.replace inflate(@io.read(len)) and return unless buffer.nil?
+        inflate(@io.read(len))
+      end
+      
+      alias :eof? :finished?
+      
+      def self.parse(io, options=nil)
+        Yajl::Stream.parse(new(io, options))
+      end
+    end
+  end
+end

data/lib/yajl/deflate.rb

@@ -0,0 +1,5 @@
+# encoding: UTF-8
+
+require 'yajl.rb' unless defined?(Yajl::Stream)
+require 'zlib' unless defined?(Zlib)
+require 'yajl/deflate/stream_reader.rb'

data/lib/yajl/gzip/stream_reader.rb

@@ -0,0 +1,22 @@
+# encoding: UTF-8
+module Yajl
+  module Gzip
+    # === Yajl::GzipStreamReader
+    #
+    # This is a wrapper around Zlib::GzipReader to allow it's #read method to adhere
+    # to the IO spec, allowing for two parameters (length, and buffer)
+    class StreamReader < ::Zlib::GzipReader
+      def read(len=nil, buffer=nil)
+        unless buffer.nil?
+          buffer.replace super(len)
+          return buffer
+        end
+        super(len)
+      end
+      
+      def self.parse(io)
+        Yajl::Stream.parse(new(io))
+      end
+    end
+  end
+end

data/lib/yajl/gzip.rb

@@ -0,0 +1,5 @@
+# encoding: UTF-8
+
+require 'yajl.rb' unless defined?(Yajl::Stream)
+require 'zlib' unless defined?(Zlib)
+require 'yajl/gzip/stream_reader.rb'

data/lib/yajl/http_stream.rb

@@ -1,7 +1,6 @@
 # encoding: UTF-8
 require 'socket' unless defined?(Socket)
-require 'zlib' unless defined?(Zlib)
-require 'yajl.rb' unless defined?(Yajl)
+require 'yajl.rb' unless defined?(Yajl::Stream)
 
 module Yajl
   # == Yajl::HttpStream
@@ -25,16 +24,20 @@ module Yajl
     # 3. the response is read until the end of the headers
     # 4. the _socket itself_ is passed directly to Yajl, for direct parsing off the stream;
     #    As it's being received over the wire!
-    def self.get(uri)
-      socket = Socket.new(Socket::Constants::AF_INET, Socket::Constants::SOCK_STREAM, 0)
-      sockaddr = Socket.pack_sockaddr_in(uri.port, uri.host)
-      socket.connect(sockaddr)
+    def self.get(uri, opts = {})
+      user_agent = opts.has_key?(['User-Agent']) ? opts['User-Agent'] : "Yajl::HttpStream #{Yajl::VERSION}"
+      
+      socket = TCPSocket.new(uri.host, uri.port)
       request = "GET #{uri.path}#{uri.query ? "?"+uri.query : nil} HTTP/1.0\r\n"
       request << "Host: #{uri.host}\r\n"
       request << "Authorization: Basic #{[userinfo].pack('m')}\r\n" unless uri.userinfo.nil?
-      request << "User-Agent: Yajl::HttpStream #{Yajl::VERSION}\r\n"
+      request << "User-Agent: #{user_agent}\r\n"
       request << "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\n"
-      request << "Accept-Encoding: gzip\r\n"
+      encodings = []
+      encodings << "bzip2" if defined?(Yajl::Bzip2)
+      encodings << "gzip" if defined?(Yajl::Gzip)
+      encodings << "deflate" if defined?(Yajl::Deflate)
+      request << "Accept-Encoding: #{encodings.join(',')}\r\n" if encodings.any?
       request << "Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7\r\n"
       request << "\r\n\r\n"
       socket.write(request)
@@ -59,13 +62,18 @@ module Yajl
       end
       
       content_type = response_head[:headers]["Content-Type"].split('; ')
-      content_type = content_type[0] if content_type.size > 1
+      content_type = content_type.first
       if ALLOWED_MIME_TYPES.include?(content_type)
         case response_head[:headers]["Content-Encoding"]
         when "gzip"
-          socket = Yajl::GzipStreamReader.new(socket)
+          return Yajl::Gzip::StreamReader.parse(socket)
+        when "deflate"
+          return Yajl::Deflate::StreamReader.parse(socket, -Zlib::MAX_WBITS)
+        when "bzip2"
+          return Yajl::Bzip2::StreamReader.parse(socket)
+        else
+          return Yajl::Stream.parse(socket)
         end
-        return Yajl::Stream.parse(socket)
       else
         raise InvalidContentType, "The response MIME type #{content_type}"
       end
@@ -73,16 +81,4 @@ module Yajl
       socket.close
     end
   end
-  
-  # === Yajl::GzipStreamReader
-  #
-  # This is a wrapper around Zlib::GzipReader to allow it's #read method to adhere
-  # to the IO spec, allowing for two parameters (length, and buffer)
-  class GzipStreamReader < ::Zlib::GzipReader
-    def read(len=nil, buffer=nil)
-      buffer.gsub!(/.*/, '') unless buffer.nil?
-      buffer << super(len) and return unless buffer.nil?
-      super(len)
-    end
-  end
 end

data/lib/yajl.rb

@@ -1,12 +1,19 @@
 # encoding: UTF-8
 require 'yajl.bundle'
-require 'yajl/http_stream.rb' unless defined?(Yajl::HttpStream)
+
+# = Extras
+# We're not going to load these auotmatically, because you might not need them ;)
+#
+# require 'yajl/http_stream.rb' unless defined?(Yajl::HttpStream)
+# require 'yajl/gzip.rb' unless defined?(Yajl::Gzip)
+# require 'yajl/deflate.rb' unless defined?(Yajl::Deflate)
+# require 'yajl/bzip2.rb' unless defined?(Yajl::Bzip2)
 
 # = Yajl
 #
 # Ruby bindings to the excellent Yajl (Yet Another JSON Parser) ANSI C library.
 module Yajl
-  VERSION = "0.3.4"
+  VERSION = "0.4.4"
   
   # == Yajl::Chunked
   #