json-stream 0.1.3 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6d404ae0a1e8e03bb1551ead94eda3e413c1f492
-  data.tar.gz: 0f14c4b8a6de9f53b4bddaf824a0c0184f969399
+  metadata.gz: 15b32baaa4333e97f2eae71f21a03af04e8275ea
+  data.tar.gz: ea3b178e62a46ae093e5d362abd554fa3c361241
 SHA512:
-  metadata.gz: 3640958bef5a32726c09c723a459d15986564bc7afe70bfc0db9faa15037bddd6539fa04a8d79bb95674b56a08901771f33118d3c43e862cc51984d9eca12b39
-  data.tar.gz: 260eac9e6ff3b440fce89231e0f013bb490ed5666ace3e2c5ee28c3d17cd2eb36a589a5340c42862cf4317a9737e80a976e3ce805b4979ed9dc1df3e9b39ae70
+  metadata.gz: c88c6208cdb0597e6ba9ad1d9a86d62605f2329176eabfc15eb43f227a61efae3c1ff66cd1efe340e73245258e325076b6ecdacf6a3f5b870d03ec03d35c986c
+  data.tar.gz: cb403d67b89ec3f609b5b9e61b9f12c6a93d654206dbed37b5e9d20a8b94ecd1bd717bfb7f72bf129e3ee923904c4ac033fc90da569879c28e9140b5aa6103bd

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2010-2013 David Graham
+Copyright (c) 2010-2014 David Graham
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/README.md

@@ -65,25 +65,28 @@ def receive_data(data)
 end
 ```
 
-Notice how the parser accepts chunks of the JSON document and parses up
-to the end of the available buffer. Passing in more data resumes the
-parse from the prior state. When an interesting state change happens, the
-parser notifies all registered callback procs of the event.
+The parser accepts chunks of the JSON document and parses up to the end of the
+available buffer. Passing in more data resumes the parse from the prior state.
+When an interesting state change happens, the parser notifies all registered
+callback procs of the event.
 
 The event callback is where we can do interesting data filtering and passing
 to other processes. The above example simply prints state changes, but
-imagine the callbacks looking for an array named "rows" and processing sets
+imagine the callbacks looking for an array named `rows` and processing sets
 of these row objects in small batches. Millions of rows, streaming over the
 network, can be processed in constant memory space this way.
 
 ## Dependencies
 
 * ruby >= 1.9.2
+* jruby >= 1.7
 
-## Contact
+## Alternatives
 
-Project contact: David Graham <david.malcom.graham@gmail.com>
+* [json](https://github.com/flori/json)
+* [yajl-ruby](https://github.com/brianmario/yajl-ruby)
+* [yajl-ffi](https://github.com/dgraham/yajl-ffi)
 
 ## License
 
-JSON::Stream is released under the MIT license.  Check the LICENSE file for details.
+JSON::Stream is released under the MIT license. Check the LICENSE file for details.

data/Rakefile

@@ -12,8 +12,8 @@ task :build => [:pkg] do
 end
 
 Rake::TestTask.new(:test) do |test|
-  test.libs << 'test'
-  test.pattern = 'test/**/*_test.rb'
+  test.libs << 'spec'
+  test.pattern = 'spec/**/*_spec.rb'
   test.warning = true
 end
 

data/json-stream.gemspec

@@ -12,9 +12,9 @@ Gem::Specification.new do |s|
   s.license      = 'MIT'
 
   s.files        = Dir['[A-Z]*', 'json-stream.gemspec', '{lib}/**/*']
-  s.test_files   = Dir['test/**/*']
+  s.test_files   = Dir['spec/**/*']
   s.require_path = 'lib'
 
-  s.add_development_dependency 'rake'
+  s.add_development_dependency 'rake', '~> 10.3'
   s.required_ruby_version = '>= 1.9.2'
 end

data/lib/json/stream/buffer.rb

@@ -2,42 +2,54 @@
 
 module JSON
   module Stream
-
     # A character buffer that expects a UTF-8 encoded stream of bytes.
     # This handles truncated multi-byte characters properly so we can just
     # feed it binary data and receive a properly formatted UTF-8 String as
-    # output. See here for UTF-8 parsing details:
-    # http://en.wikipedia.org/wiki/UTF-8
-    # http://tools.ietf.org/html/rfc3629#section-3
+    # output.
+    #
+    # More UTF-8 parsing details are available at:
+    #
+    #   http://en.wikipedia.org/wiki/UTF-8
+    #   http://tools.ietf.org/html/rfc3629#section-3
     class Buffer
       def initialize
-        @state, @buf, @need = :start, [], 0
+        @state = :start
+        @buf = []
+        @need = 0
       end
 
       # Fill the buffer with a String of binary UTF-8 encoded bytes. Returns
       # as much of the data in a UTF-8 String as we have. Truncated multi-byte
       # characters are saved in the buffer until the next call to this method
       # where we expect to receive the rest of the multi-byte character.
+      #
+      # data - The partial binary encoded String data.
+      #
+      # Raises JSON::Stream::ParserError if the UTF-8 byte sequence is malformed.
+      #
+      # Returns a UTF-8 encoded String.
       def <<(data)
         bytes = []
-        data.bytes.each do |b|
+        data.each_byte do |byte|
           case @state
           when :start
-            if b < 128
-              bytes << b
-            elsif b >= 192
+            if byte < 128
+              bytes << byte
+            elsif byte >= 192
               @state = :multi_byte
-              @buf << b
-              @need = case
-                when b >= 240 then 4
-                when b >= 224 then 3
-                when b >= 192 then 2 end
+              @buf << byte
+              @need =
+                case
+                when byte >= 240 then 4
+                when byte >= 224 then 3
+                when byte >= 192 then 2
+                end
             else
               error('Expected start of multi-byte or single byte char')
             end
           when :multi_byte
-            if b > 127 && b < 192
-              @buf << b
+            if byte > 127 && byte < 192
+              @buf << byte
               if @buf.size == @need
                 bytes += @buf.slice!(0, @buf.size)
                 @state = :start
@@ -52,6 +64,27 @@ module JSON
         end
       end
 
+      # Determine if the buffer contains partial UTF-8 continuation bytes that
+      # are waiting on subsequent completion bytes before a full codepoint is
+      # formed.
+      #
+      # Examples
+      #
+      #   bytes = "é".bytes
+      #
+      #   buffer << bytes[0]
+      #   buffer.empty?
+      #   # => false
+      #
+      #   buffer << bytes[1]
+      #   buffer.empty?
+      #   # => true
+      #
+      # Returns true if the buffer is empty.
+      def empty?
+        @buf.empty?
+      end
+
       private
 
       def error(message)

data/lib/json/stream/builder.rb

@@ -2,17 +2,15 @@
 
 module JSON
   module Stream
-    # A parser listener that builds a full, in memory, object graph from a
-    # JSON document. Typically, we would use the json gem's JSON.parse() method
-    # when we have the full JSON document because it's much faster than this.
-    # JSON::Stream is typically used when we have a huge JSON document streaming
-    # to us and we don't want to hold the entire parsed object in memory.
-    # Regardless, this is a good example of how to write parser callbacks.
+    # A parser listener that builds a full, in memory, object from a JSON
+    # document. This is similar to using the json gem's `JSON.parse` method.
     #
-    # parser = JSON::Stream::Parser.new
-    # builder = JSON::Stream::Builder.new(parser)
-    # parser << json
-    # obj = builder.result
+    # Examples
+    #
+    #   parser = JSON::Stream::Parser.new
+    #   builder = JSON::Stream::Builder.new(parser)
+    #   parser << '{"answer": 42, "question": false}'
+    #   obj = builder.result
     class Builder
       METHODS = %w[start_document end_document start_object end_object start_array end_array key value]
 
@@ -25,66 +23,49 @@ module JSON
       end
 
       def start_document
-        @stack, @result = [], nil
+        @stack = []
+        @keys = []
+        @result = nil
       end
 
       def end_document
-        @result = @stack.pop.obj
+        @result = @stack.pop
       end
 
       def start_object
-        @stack.push(ObjectNode.new)
+        @stack.push({})
       end
 
       def end_object
-        unless @stack.size == 1
-          node = @stack.pop
-          @stack[-1] << node.obj
+        return if @stack.size == 1
+        node = @stack.pop
+
+        case @stack.last
+        when Hash
+          @stack.last[@keys.pop] = node
+        when Array
+          @stack.last << node
         end
       end
       alias :end_array :end_object
 
       def start_array
-        @stack.push(ArrayNode.new)
+        @stack.push([])
       end
 
       def key(key)
-        @stack[-1] << key
+        @keys << key
       end
 
       def value(value)
-        @stack[-1] << value
-      end
-    end
-
-    class ArrayNode
-      attr_reader :obj
-
-      def initialize
-        @obj = []
-      end
-
-      def <<(node)
-        @obj << node
-        self
-      end
-    end
-
-    class ObjectNode
-      attr_reader :obj
-
-      def initialize
-        @obj, @key = {}, nil
-      end
-
-      def <<(node)
-        if @key
-          @obj[@key] = node
-          @key = nil
+        case @stack.last
+        when Hash
+          @stack.last[@keys.pop] = value
+        when Array
+          @stack.last << value
         else
-          @key = node
+          @stack << value
         end
-        self
       end
     end
   end

data/lib/json/stream/parser.rb

@@ -2,15 +2,24 @@
 
 module JSON
   module Stream
-    class ParserError < RuntimeError; end
+    # Raised on any invalid JSON text.
+    ParserError = Class.new(RuntimeError)
 
-    # A streaming JSON parser that generates SAX-like events for
-    # state changes. Use the json gem for small documents. Use this
-    # for huge documents that won't fit in memory.
+    # A streaming JSON parser that generates SAX-like events for state changes.
+    # Use the json gem for small documents. Use this for huge documents that
+    # won't fit in memory.
+    #
+    # Examples
+    #
+    #   parser = JSON::Stream::Parser.new
+    #   parser.key {|key| puts key }
+    #   parser.value {|value| puts value }
+    #   parser << '{"answer":'
+    #   parser << ' 42}'
     class Parser
-      BUF_SIZE      = 512
+      BUF_SIZE      = 4096
       CONTROL       = /[\x00-\x1F]/
-      WS            = /\s/
+      WS            = /[ \n\t\r]/
       HEX           = /[0-9a-fA-F]/
       DIGIT         = /[0-9]/
       DIGIT_1_9     = /[1-9]/
@@ -40,6 +49,17 @@ module JSON
       # Parses a full JSON document from a String or an IO stream and returns
       # the parsed object graph. For parsing small JSON documents with small
       # memory requirements, use the json gem's faster JSON.parse method instead.
+      #
+      # json - The String or IO containing JSON data.
+      #
+      # Examples
+      #
+      #   JSON::Stream::Parser.parse('{"hello": "world"}')
+      #   # => {"hello": "world"}
+      #
+      # Raises a JSON::Stream::ParserError if the JSON data is malformed.
+      #
+      # Returns a Hash.
       def self.parse(json)
         stream = json.is_a?(String) ? StringIO.new(json) : json
         parser = Parser.new
@@ -47,71 +67,138 @@ module JSON
         while (buf = stream.read(BUF_SIZE)) != nil
           parser << buf
         end
-        raise ParserError, "unexpected eof" unless builder.result
+        parser.finish
         builder.result
       ensure
         stream.close
       end
 
-      %w[start_document end_document start_object end_object
-          start_array end_array key value].each do |name|
+      # Drain any remaining buffered characters into the parser to complete
+      # the parsing of the document.
+      #
+      # This is only required when parsing a document containing a single
+      # numeric value, integer or float. The parser has no other way to
+      # detect when it should no longer expect additional characters with
+      # which to complete the parse, so it must be signaled by a call to
+      # this method.
+      #
+      # If you're parsing more typical object or array documents, there's no
+      # need to call `finish` because the parse will complete when the final
+      # closing `]` or `}` character is scanned.
+      #
+      # Raises a JSON::Stream::ParserError if the JSON data is malformed.
+      #
+      # Returns nothing.
+      def finish
+        # Partial multi-byte character waiting for completion bytes.
+        error('Unexpected end-of-file') unless @utf8.empty?
 
-        define_method(name) do |&block|
-          @listeners[name] << block
-        end
+        # Partial array, object, or string.
+        error('Unexpected end-of-file') unless @stack.empty?
 
-        define_method("notify_#{name}") do |*args|
-          @listeners[name].each do |block|
-            block.call(*args)
-          end
+        case @state
+        when :end_document
+          # done, do nothing
+        when :in_float
+          end_value(@buf.to_f)
+        when :in_exponent
+          error('Unexpected end-of-file') unless @buf =~ DIGIT_END
+          end_value(@buf.to_f)
+        when :start_zero
+          end_value(@buf.to_i)
+        when :start_int
+          end_value(@buf.to_i)
+        else
+          error('Unexpected end-of-file')
         end
-        private "notify_#{name}"
       end
 
       # Create a new parser with an optional initialization block where
-      # we can register event callbacks. For example:
-      # parser = JSON::Stream::Parser.new do
-      #   start_document { puts "start document" }
-      #   end_document   { puts "end document" }
-      #   start_object   { puts "start object" }
-      #   end_object     { puts "end object" }
-      #   start_array    { puts "start array" }
-      #   end_array      { puts "end array" }
-      #   key            {|k| puts "key: #{k}" }
-      #   value          {|v| puts "value: #{v}" }
-      # end
+      # we can register event callbacks.
+      #
+      # Examples
+      #
+      #   parser = JSON::Stream::Parser.new do
+      #     start_document { puts "start document" }
+      #     end_document   { puts "end document" }
+      #     start_object   { puts "start object" }
+      #     end_object     { puts "end object" }
+      #     start_array    { puts "start array" }
+      #     end_array      { puts "end array" }
+      #     key            {|k| puts "key: #{k}" }
+      #     value          {|v| puts "value: #{v}" }
+      #   end
       def initialize(&block)
         @state = :start_document
         @utf8 = Buffer.new
-        @listeners = Hash.new {|h, k| h[k] = [] }
-        @stack, @unicode, @buf, @pos = [], "", "", -1
+        @listeners = {
+          start_document: [],
+          end_document: [],
+          start_object: [],
+          end_object: [],
+          start_array: [],
+          end_array: [],
+          key: [],
+          value: []
+        }
+
+        # Track parse stack.
+        @stack = []
+        @unicode = ""
+        @buf = ""
+        @pos = -1
+
+        # Register any observers in the block.
         instance_eval(&block) if block_given?
       end
 
+      def start_document(&block)
+        @listeners[:start_document] << block
+      end
+
+      def end_document(&block)
+        @listeners[:end_document] << block
+      end
+
+      def start_object(&block)
+        @listeners[:start_object] << block
+      end
+
+      def end_object(&block)
+        @listeners[:end_object] << block
+      end
+
+      def start_array(&block)
+        @listeners[:start_array] << block
+      end
+
+      def end_array(&block)
+        @listeners[:end_array] << block
+      end
+
+      def key(&block)
+        @listeners[:key] << block
+      end
+
+      def value(&block)
+        @listeners[:value] << block
+      end
+
       # Pass data into the parser to advance the state machine and
       # generate callback events. This is well suited for an EventMachine
       # receive_data loop.
+      #
+      # data - The String of partial JSON data to parse.
+      #
+      # Raises a JSON::Stream::ParserError if the JSON data is malformed.
+      #
+      # Returns nothing.
       def <<(data)
         (@utf8 << data).each_char do |ch|
           @pos += 1
           case @state
           when :start_document
-            case ch
-            when LEFT_BRACE
-              @state = :start_object
-              @stack.push(:object)
-              notify_start_document
-              notify_start_object
-            when LEFT_BRACKET
-              @state = :start_array
-              @stack.push(:array)
-              notify_start_document
-              notify_start_array
-            when WS
-              # ignore
-            else
-              error("Expected object or array start")
-            end
+            start_value(ch)
           when :start_object
             case ch
             when RIGHT_BRACE
@@ -122,17 +209,16 @@ module JSON
             when WS
               # ignore
             else
-              error("Expected object key start")
+              error('Expected object key start')
             end
           when :start_string
             case ch
             when QUOTE
               if @stack.pop == :string
-                @state = :end_value
-                notify_value(@buf)
+                end_value(@buf)
               else # :key
                 @state = :end_key
-                notify_key(@buf)
+                notify(:key, @buf)
               end
               @buf = ""
             when BACKSLASH
@@ -165,7 +251,7 @@ module JSON
             when U
               @state = :unicode_escape
             else
-              error("Expected escaped character")
+              error('Expected escaped character')
             end
           when :unicode_escape
             case ch
@@ -174,7 +260,7 @@ module JSON
               if @unicode.size == 4
                 codepoint = @unicode.slice!(0, 4).hex
                 if codepoint >= 0xD800 && codepoint <= 0xDBFF
-                  error('Expected low surrogate pair half') if @stack[-1].is_a?(Fixnum)
+                  error('Expected low surrogate pair half') if @stack.last.is_a?(Fixnum)
                   @state = :start_surrogate_pair
                   @stack.push(codepoint)
                 elsif codepoint >= 0xDC00 && codepoint <= 0xDFFF
@@ -225,8 +311,7 @@ module JSON
               @state = :start_exponent
               @buf << ch
             else
-              @state = :end_value
-              notify_value(@buf.to_i)
+              end_value(@buf.to_i)
               @buf = ""
               @pos -= 1
               redo
@@ -247,8 +332,7 @@ module JSON
               @state = :start_exponent
               @buf << ch
             else
-              @state = :end_value
-              notify_value(@buf.to_f)
+              end_value(@buf.to_f)
               @buf = ""
               @pos -= 1
               redo
@@ -267,9 +351,7 @@ module JSON
               @buf << ch
             else
               error('Expected 0-9 digit') unless @buf =~ DIGIT_END
-              @state = :end_value
-              num = @buf.include?('.') ? @buf.to_f : @buf.to_i
-              notify_value(num)
+              end_value(@buf.to_f)
               @buf = ""
               @pos -= 1
               redo
@@ -285,8 +367,7 @@ module JSON
               @state = :start_exponent
               @buf << ch
             else
-              @state = :end_value
-              notify_value(@buf.to_i)
+              end_value(@buf.to_i)
               @buf = ""
               @pos -= 1
               redo
@@ -304,7 +385,7 @@ module JSON
             when WS
               # ignore
             else
-              error("Expected colon key separator")
+              error('Expected colon key separator')
             end
           when :key_sep
             start_value(ch)
@@ -328,10 +409,10 @@ module JSON
             when WS
               # ignore
             else
-              error("Expected comma or object or array close")
+              error('Expected comma or object or array close')
             end
           when :value_sep
-            if @stack[-1] == :object
+            if @stack.last == :object
               case ch
               when QUOTE
                 @state = :start_string
@@ -339,59 +420,120 @@ module JSON
               when WS
                 # ignore
               else
-                error("Expected object key start")
+                error('Expected object key start')
               end
             else
               start_value(ch)
             end
           when :end_document
-            error("Unexpected data") unless ch =~ WS
+            error('Unexpected data') unless ch =~ WS
           end
         end
       end
 
       private
 
+      # Invoke all registered observer procs for the event type.
+      #
+      # type - The Symbol listener name.
+      # args - The argument list to pass into the observer procs.
+      #
+      # Examples
+      #
+      #    # broadcast events for {"answer": 42}
+      #    notify(:start_object)
+      #    notify(:key, "answer")
+      #    notify(:value, 42)
+      #    notify(:end_object)
+      #
+      # Returns nothing.
+      def notify(type, *args)
+        @listeners[type].each do |block|
+          block.call(*args)
+        end
+      end
+
+      # Complete an object or array container value type.
+      #
+      # type - The Symbol, :object or :array, of the expected type.
+      #
+      # Raises a JSON::Stream::ParserError if the expected container type
+      #   was not completed.
+      #
+      # Returns nothing.
       def end_container(type)
         @state = :end_value
         if @stack.pop == type
-          send("notify_end_#{type}")
+          case type
+          when :object then notify(:end_object)
+          when :array  then notify(:end_array)
+          end
         else
           error("Expected end of #{type}")
         end
-        if @stack.empty?
-          @state = :end_document
-          notify_end_document
-        end
+        notify_end_document if @stack.empty?
+      end
+
+      # Broadcast an `end_document` event to observers after a complete JSON
+      # value document (object, array, number, string, true, false, null) has
+      # been parsed from the text. This is the final event sent to observers
+      # and signals the parse has finished.
+      #
+      # Returns nothing.
+      def notify_end_document
+        @state = :end_document
+        notify(:end_document)
       end
 
+      # Parse one of the three allowed keywords: true, false, null.
+      #
+      # word  - The String keyword ('true', 'false', 'null').
+      # value - The Ruby value (true, false, nil).
+      # re    - The Regexp of allowed keyword characters.
+      # ch    - The current String character being parsed.
+      #
+      # Raises a JSON::Stream::ParserError if the character does not belong
+      #   in the expected keyword.
+      #
+      # Returns nothing.
       def keyword(word, value, re, ch)
         if ch =~ re
           @buf << ch
         else
           error("Expected #{word} keyword")
         end
+
         if @buf.size == word.size
           if @buf == word
-            @state = :end_value
             @buf = ""
-            notify_value(value)
+            end_value(value)
           else
             error("Expected #{word} keyword")
           end
         end
       end
 
+      # Process the first character of one of the seven possible JSON
+      # values: object, array, string, true, false, null, number.
+      #
+      # ch - The current character String.
+      #
+      # Raises a JSON::Stream::ParserError if the character does not signal
+      #   the start of a value.
+      #
+      # Returns nothing.
       def start_value(ch)
         case ch
         when LEFT_BRACE
+          notify(:start_document) if @stack.empty?
           @state = :start_object
           @stack.push(:object)
-          notify_start_object
+          notify(:start_object)
         when LEFT_BRACKET
+          notify(:start_document) if @stack.empty?
           @state = :start_array
           @stack.push(:array)
-          notify_start_array
+          notify(:start_array)
         when QUOTE
           @state = :start_string
           @stack.push(:string)
@@ -416,10 +558,23 @@ module JSON
         when WS
           # ignore
         else
-          error("Expected value")
+          error('Expected value')
         end
       end
 
+      # Advance the state machine and notify `value` observers that a
+      # string, number or keyword (true, false, null) value was parsed.
+      #
+      # value - The object to broadcast to observers.
+      #
+      # Returns nothing.
+      def end_value(value)
+        @state = :end_value
+        notify(:start_document) if @stack.empty?
+        notify(:value, value)
+        notify_end_document if @stack.empty?
+      end
+
       def error(message)
         raise ParserError, "#{message}: char #{@pos}"
       end