oortle-yajl-ruby 0.5.8

data/ext/yajl_parser.h

@@ -0,0 +1,79 @@
+/*
+ * Copyright 2007-2009, Lloyd Hilaiel.
+ * 
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ * 
+ *  1. Redistributions of source code must retain the above copyright
+ *     notice, this list of conditions and the following disclaimer.
+ * 
+ *  2. Redistributions in binary form must reproduce the above copyright
+ *     notice, this list of conditions and the following disclaimer in
+ *     the documentation and/or other materials provided with the
+ *     distribution.
+ * 
+ *  3. Neither the name of Lloyd Hilaiel nor the names of its
+ *     contributors may be used to endorse or promote products derived
+ *     from this software without specific prior written permission.
+ * 
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
+ * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
+ * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */ 
+
+#ifndef __YAJL_PARSER_H__
+#define __YAJL_PARSER_H__
+
+#include "api/yajl_parse.h"
+#include "yajl_bytestack.h"
+#include "yajl_buf.h"
+
+
+typedef enum {
+    yajl_state_start = 0,
+    yajl_state_parse_complete,
+    yajl_state_parse_error,
+    yajl_state_lexical_error,
+    yajl_state_map_start,
+    yajl_state_map_sep,    
+    yajl_state_map_need_val,
+    yajl_state_map_got_val,
+    yajl_state_map_need_key,
+    yajl_state_array_start,
+    yajl_state_array_got_val,
+    yajl_state_array_need_val
+} yajl_state;
+
+struct yajl_handle_t {
+    const yajl_callbacks * callbacks;
+    void * ctx;
+    yajl_lexer lexer;
+    const char * parseError;
+    unsigned int errorOffset;
+    /* temporary storage for decoded strings */
+    yajl_buf decodeBuf;
+    /* a stack of states.  access with yajl_state_XXX routines */
+    yajl_bytestack stateStack;
+    /* memory allocation routines */
+    yajl_alloc_funcs alloc;
+};
+
+yajl_status
+yajl_do_parse(yajl_handle handle, unsigned int * offset,
+              const unsigned char * jsonText, unsigned int jsonTextLen);
+
+unsigned char *
+yajl_render_error_string(yajl_handle hand, const unsigned char * jsonText,
+                         unsigned int jsonTextLen, int verbose);
+
+
+#endif

data/lib/yajl.rb

@@ -0,0 +1,80 @@
+# encoding: UTF-8
+require 'yajl_ext'
+
+# = Extras
+# We're not going to load these auotmatically, because you might not need them ;)
+#
+# require 'yajl/http_stream.rb' unless defined?(Yajl::HttpStream)
+# require 'yajl/gzip.rb' unless defined?(Yajl::Gzip)
+# require 'yajl/deflate.rb' unless defined?(Yajl::Deflate)
+# require 'yajl/bzip2.rb' unless defined?(Yajl::Bzip2)
+
+# = Yajl
+#
+# Ruby bindings to the excellent Yajl (Yet Another JSON Parser) ANSI C library.
+module Yajl
+  VERSION = "0.5.7"
+  
+  class Parser
+    # A helper method for parse-and-forget use-cases
+    #
+    # +io+ is the stream to parse JSON from
+    #
+    # The +options+ hash allows you to set two parsing options - :allow_comments and :check_utf8
+    #
+    # :allow_comments accepts a boolean will enable/disable checks for in-line comments in the JSON stream
+    #
+    # :check_utf8 accepts a boolean will enable/disable UTF8 validation for the JSON stream
+    def self.parse(io, options={}, read_bufsize=nil, &block)
+      new(options).parse(io, read_bufsize, &block)
+    end
+  end
+  
+  class Encoder
+    # A helper method for encode-and-forget use-cases
+    #
+    # Examples:
+    #   Yajl::Encoder.encode(obj[, io, :pretty => true, :indent => "\t"])
+    #
+    #   output = Yajl::Encoder.encode(obj[, :pretty => true, :indent => "\t"])
+    #
+    # +obj+ is a ruby object to encode to JSON format
+    #
+    # +io+ is the optional IO stream to encode the ruby object to.
+    # If +io+ isn't passed, the resulting JSON string is returned. If +io+ is passed, nil is returned.
+    #
+    # The +options+ hash allows you to set two encoding options - :pretty and :indent
+    #
+    # :pretty accepts a boolean and will enable/disable "pretty printing" the resulting output
+    #
+    # :indent accepts a string and will be used as the indent character(s) during the pretty print process
+    def self.encode(obj, *args, &block)
+      # TODO: this code smells, any ideas?
+      options = {}
+      io = nil
+      args.each do |arg|
+        if arg.is_a?(Hash)
+          options = arg
+        elsif arg.respond_to?(:read)
+          io = arg
+        end
+      end if args.any?
+      new(options).encode(obj, io, &block)
+    end
+  end
+  
+  # DEPRECATED - See Yajl::Parser and Yajl::Encoder
+  module Stream
+    # DEPRECATED - See Yajl::Parser
+    def self.parse(io)
+      STDERR.puts "WARNING: Yajl::Stream has be deprecated and will most likely be gone in the next release. Use the Yajl::Parser class instead."
+      Parser.new.parse(io)
+    end
+    
+    # DEPRECATED - See Yajl::Encoder
+    def self.encode(obj, io)
+      STDERR.puts "WARNING: Yajl::Stream has be deprecated and will most likely be gone in the next release. Use the Yajl::Encoder class instead."
+      Encoder.new.encode(obj, io)
+    end
+  end
+end

data/lib/yajl/bzip2.rb

@@ -0,0 +1,11 @@
+# encoding: UTF-8
+
+require 'yajl' unless defined?(Yajl::Parser)
+
+begin
+  require 'bzip2' unless defined?(Bzip2)
+  require 'yajl/bzip2/stream_reader.rb'
+  require 'yajl/bzip2/stream_writer.rb'
+rescue LoadError => e
+  raise "Unable to load the bzip2 library. Is the bzip2-ruby gem installed?"
+end

data/lib/yajl/bzip2/stream_reader.rb

@@ -0,0 +1,29 @@
+# encoding: UTF-8
+module Yajl
+  module Bzip2
+    # This is a wrapper around Bzip::Reader to allow it's #read method to adhere
+    # to the IO spec, allowing for two parameters (length, and buffer)
+    class StreamReader < ::Bzip2::Reader
+      
+      # A helper method to allow use similar to IO#read
+      def read(len=nil, buffer=nil)
+        unless buffer.nil?
+          buffer.replace super(len)
+          return buffer
+        end
+        super(len)
+      end
+
+      # Helper method for one-off parsing from a bzip2-compressed stream
+      #
+      # See Yajl::Parser#parse for parameter documentation
+      def self.parse(input, options={}, buffer_size=nil, &block)
+        if input.is_a?(String)
+          input = StringIO.new(input)
+        end
+        
+        Yajl::Parser.new(options).parse(new(input), buffer_size, &block)
+      end
+    end
+  end
+end

data/lib/yajl/bzip2/stream_writer.rb

@@ -0,0 +1,15 @@
+# encoding: UTF-8
+module Yajl
+  module Bzip2
+    # A wrapper around the Bzip2::Writer class for easier JSON stream encoding
+    class StreamWriter < ::Bzip2::Writer
+      
+      # A helper method for encoding to a bzip2-compressed stream
+      #
+      # Look up Yajl::Encoder#encode for parameter documentation
+      def self.encode(obj, io)
+        Yajl::Encoder.new.encode(obj, new(io))
+      end
+    end
+  end
+end

data/lib/yajl/deflate.rb

@@ -0,0 +1,6 @@
+# encoding: UTF-8
+
+require 'yajl' unless defined?(Yajl::Parser)
+require 'zlib' unless defined?(Zlib)
+require 'yajl/deflate/stream_reader.rb'
+require 'yajl/deflate/stream_writer.rb'

data/lib/yajl/deflate/stream_reader.rb

@@ -0,0 +1,38 @@
+# encoding: UTF-8
+module Yajl
+  module Deflate
+    # This is a wrapper around Zlib::Inflate, creating a #read method that adheres
+    # to the IO spec, allowing for two parameters (length, and buffer)
+    class StreamReader < ::Zlib::Inflate
+      
+      # Wrapper to the initialize method so we can set the initial IO to parse from.
+      def initialize(io, options)
+        @io = io
+        super(options)
+      end
+      
+      # A helper method to allow use similar to IO#read
+      def read(len=nil, buffer=nil)
+        buffer.replace inflate(@io.read(len)) and return unless buffer.nil?
+        inflate(@io.read(len))
+      end
+      alias :eof? :finished?
+      
+      # Helper method for one-off parsing from a deflate-compressed stream
+      #
+      # See Yajl::Parser#parse for parameter documentation
+      def self.parse(input, options={}, buffer_size=nil, &block)
+        if input.is_a?(String)
+          input = StringIO.new(input)
+        end
+        
+        if options.is_a?(Hash)
+          deflate_options = options.delete(:deflate_options)
+          Yajl::Parser.new(options).parse(new(input, deflate_options), buffer_size, &block)
+        elsif options.is_a?(Fixnum)
+          Yajl::Parser.new.parse(new(input, options), buffer_size, &block)
+        end
+      end
+    end
+  end
+end

data/lib/yajl/deflate/stream_writer.rb

@@ -0,0 +1,21 @@
+# encoding: UTF-8
+module Yajl
+  module Deflate
+    # A wrapper around the Zlib::Deflate class for easier JSON stream parsing
+    class StreamWriter < ::Zlib::Deflate
+      
+      # A helper method to allow use similar to IO#write
+      def write(str)
+        deflate(str)
+        str.size unless str.nil?
+      end
+      
+      # A helper method for one-off encoding to a deflate-compressed stream
+      #
+      # Look up Yajl::Encoder#encode for parameter documentation
+      def self.encode(obj, io)
+        Yajl::Encoder.new.encode(obj, new(io))
+      end
+    end
+  end
+end

data/lib/yajl/gzip.rb

@@ -0,0 +1,6 @@
+# encoding: UTF-8
+
+require 'yajl' unless defined?(Yajl::Parser)
+require 'zlib' unless defined?(Zlib)
+require 'yajl/gzip/stream_reader.rb'
+require 'yajl/gzip/stream_writer.rb'

data/lib/yajl/gzip/stream_reader.rb

@@ -0,0 +1,28 @@
+# encoding: UTF-8
+module Yajl
+  module Gzip
+    # This is a wrapper around Zlib::GzipReader to allow it's #read method to adhere
+    # to the IO spec, allowing for two parameters (length, and buffer)
+    class StreamReader < ::Zlib::GzipReader
+      
+      # Wrapper method to allow use similar to IO#read
+      def read(len=nil, buffer=nil)
+        unless buffer.nil?
+          buffer.replace super(len)
+          return buffer
+        end
+        super(len)
+      end
+      
+      # Helper method for one-off parsing from a gzip-compressed stream
+      #
+      # See Yajl::Parser#parse for parameter documentation
+      def self.parse(input, options={}, buffer_size=nil, &block)
+        if input.is_a?(String)
+          input = StringIO.new(input)
+        end
+        Yajl::Parser.new(options).parse(new(input), buffer_size, &block)
+      end
+    end
+  end
+end

data/lib/yajl/gzip/stream_writer.rb

@@ -0,0 +1,14 @@
+# encoding: UTF-8
+module Yajl
+  module Gzip
+    # Wraper around the Zlib::GzipWriter class
+    class StreamWriter < ::Zlib::GzipWriter
+      # A helper method for one-off encoding to a gzip-compressed stream
+      #
+      # Look up Yajl::Encoder#encode for parameter documentation
+      def self.encode(obj, io)
+        Yajl::Encoder.new.encode(obj, new(io))
+      end
+    end
+  end
+end

data/lib/yajl/http_stream.rb

@@ -0,0 +1,150 @@
+# encoding: UTF-8
+require 'socket' unless defined?(Socket)
+require 'yajl' unless defined?(Yajl::Parser)
+require 'uri' unless defined?(URI)
+
+module Yajl
+  # This module is for making HTTP requests to which the response bodies (and possibly requests in the near future)
+  # are streamed directly into Yajl.
+  class HttpStream
+    
+    # This Exception is thrown when an HTTP response isn't in ALLOWED_MIME_TYPES
+    # and therefore cannot be parsed.
+    class InvalidContentType < Exception; end
+    
+    # The mime-type we expect the response to be. If it's anything else, we can't parse it
+    # and an InvalidContentType is raised.
+    ALLOWED_MIME_TYPES = ["application/json", "text/plain"]
+    
+    # Makes a basic HTTP GET request to the URI provided
+    # 1. a raw socket is opened to the server/host provided
+    # 2. the request is made using HTTP/1.0, Accept-encoding: gzip (deflate support coming soon, too)
+    # 3. the response is read until the end of the headers
+    # 4. the _socket itself_ is passed directly to Yajl, for direct parsing off the stream; As it's being received over the wire!
+    def self.get(uri, opts = {}, &block)
+      user_agent = opts.has_key?('User-Agent') ? opts.delete(['User-Agent']) : "Yajl::HttpStream #{Yajl::VERSION}"
+      
+      Thread.current[:yajl_socket] = socket = TCPSocket.new(uri.host, uri.port)
+      request = "GET #{uri.path}#{uri.query ? "?"+uri.query : nil} HTTP/1.1\r\n"
+      request << "Host: #{uri.host}\r\n"
+      request << "Authorization: Basic #{[uri.userinfo].pack('m').chomp}\r\n" unless uri.userinfo.nil?
+      request << "User-Agent: #{user_agent}\r\n"
+      request << "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\r\n"
+      request << "Connection: close\r\n"
+      encodings = []
+      encodings << "bzip2" if defined?(Yajl::Bzip2)
+      encodings << "gzip" if defined?(Yajl::Gzip)
+      encodings << "deflate" if defined?(Yajl::Deflate)
+      request << "Accept-Encoding: #{encodings.join(',')}\r\n" if encodings.any?
+      request << "Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7\r\n"
+      request << "\r\n\r\n"
+      socket.write(request)
+
+      handle_response(socket, opts, &block)
+    end
+    
+    # Makes a basic HTTP POST request to the URI provided
+    # 1. a raw socket is opened to the server/host provided
+    # 2. the request is made using HTTP/1.0, Accept-encoding: gzip (deflate support coming soon, too)
+    # 3. the response is read until the end of the headers
+    # 4. the _socket itself_ is passed directly to Yajl, for direct parsing off the stream; As it's being received over the wire!
+  def self.post(uri, body={}, opts = {}, &block)
+      content = body.keys.collect {|param| "#{URI.escape(param.to_s)}=#{URI.escape(body[param].to_s)}"}.join('&')
+
+      user_agent = opts.has_key?('User-Agent') ? opts.delete(['User-Agent']) : "Yajl::HttpStream #{Yajl::VERSION}"
+      
+      Thread.current[:yajl_socket] = socket = TCPSocket.new(uri.host, uri.port)
+      request = "POST #{uri.path}#{uri.query ? "?"+uri.query : nil} HTTP/1.1\r\n"
+      request << "Host: #{uri.host}\r\n"
+      request << "Authorization: Basic #{[uri.userinfo].pack('m').chomp}\r\n" unless uri.userinfo.nil?
+      request << "User-Agent: #{user_agent}\r\n"
+      request << "Accept: */*\r\n"
+      encodings = []
+      encodings << "bzip2" if defined?(Yajl::Bzip2)
+      encodings << "gzip" if defined?(Yajl::Gzip)
+      encodings << "deflate" if defined?(Yajl::Deflate)
+      request << "Accept-Encoding: #{encodings.join(',')}\r\n" if encodings.any?
+      request << "Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7\r\n"
+      request << "Content-Type: application/x-www-form-urlencoded\r\n"
+      request << "Content-Length: #{content.length}"
+      request << "\r\n\r\n"
+      request << content
+      socket.write(request)
+      
+      handle_response(socket, opts, &block)
+    end
+
+    # Terminate a running HTTPStream instance given the thread it's running in
+    def self.terminate(thread)      
+      thread[:yajl_intentional_termination] = true
+      thread[:yajl_socket].close
+    end
+
+    private
+
+    def self.handle_response(socket, opts, &block)
+      response_head = {}
+      response_head[:headers] = {}
+
+      socket.each_line do |line|
+        if line == "\r\n" # end of the headers
+          break
+        else
+          header = line.split(": ")
+          if header.size == 1
+            header = header[0].split(" ")
+            response_head[:version] = header[0]
+            response_head[:code] = header[1].to_i
+            response_head[:msg] = header[2]
+            # this is the response code line
+          else
+            response_head[:headers][header[0]] = header[1].strip
+          end
+        end
+      end
+      parser = Yajl::Parser.new(opts)
+      if response_head[:headers]["Transfer-Encoding"] == 'chunked'
+        if block_given?
+          parser.on_parse_complete = block
+          chunkLeft = 0
+          while !socket.eof? && (size = socket.gets.hex)
+            next if size == 0
+            json = socket.read(size)
+            chunkLeft = size-json.size
+            if chunkLeft == 0
+              parser << json
+            else
+              # received only part of the chunk, grab the rest
+              parser << socket.read(chunkLeft)
+            end
+          end
+        else
+          raise Exception, "Chunked responses detected, but no block given to handle the chunks."
+        end
+      else
+        content_type = response_head[:headers]["Content-Type"].split(';')
+        content_type = content_type.first
+        if ALLOWED_MIME_TYPES.include?(content_type)
+          case response_head[:headers]["Content-Encoding"]
+          when "gzip"
+            return Yajl::Gzip::StreamReader.parse(socket, opts)
+          when "deflate"
+            return Yajl::Deflate::StreamReader.parse(socket, opts.merge({:deflate_options => -Zlib::MAX_WBITS}))
+          when "bzip2"
+            return Yajl::Bzip2::StreamReader.parse(socket, opts)
+          else
+            return Yajl::Parser.new(opts).parse(socket)
+          end
+        else
+          raise InvalidContentType, "The response MIME type #{content_type}"
+        end
+      end
+    rescue IOError => e
+      raise e unless Thread.current[:yajl_intentional_termination]
+      Thread.current[:yajl_intentional_termination] = nil
+    ensure
+      Thread.current[:yajl_socket] = nil
+      socket.close unless socket.closed?
+    end
+  end
+end