json-stream 0.2.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 15b32baaa4333e97f2eae71f21a03af04e8275ea
-  data.tar.gz: ea3b178e62a46ae093e5d362abd554fa3c361241
+  metadata.gz: 62af90ebe18d5c8a58ca9b75695bf2c403b4c4a0
+  data.tar.gz: d3176279af8156702e63ffdf4c963ab6f0e25cb3
 SHA512:
-  metadata.gz: c88c6208cdb0597e6ba9ad1d9a86d62605f2329176eabfc15eb43f227a61efae3c1ff66cd1efe340e73245258e325076b6ecdacf6a3f5b870d03ec03d35c986c
-  data.tar.gz: cb403d67b89ec3f609b5b9e61b9f12c6a93d654206dbed37b5e9d20a8b94ecd1bd717bfb7f72bf129e3ee923904c4ac033fc90da569879c28e9140b5aa6103bd
+  metadata.gz: 5d757855dcd79878bb9d5cbdd74c05252bde3286259b88315e95b3302ce60bc24bbf2d6f6e0ab32a5507b55bbdeae3b63ab405368880cfb29b1981f3cb291146
+  data.tar.gz: 0a90cf028ed263a4dc571c4834e1b033d9c23b17aaaef36a7275d44e804a69c1c7c8090ea119411858608bfb45a8c6ada7b2e4e6c8862294ef54ef091b1030e0

data/README.md

@@ -2,9 +2,10 @@
 
 JSON::Stream is a JSON parser, based on a finite state machine, that generates
 events for each state change. This allows streaming both the JSON document into
-memory and the parsed object graph out of memory to some other process. This
-is much like an XML SAX parser that generates events during parsing. There is
-no requirement for the document, or the object graph, to be fully buffered in
+memory and the parsed object graph out of memory to some other process.
+
+This is much like an XML SAX parser that generates events during parsing. There
+is no requirement for the document, or the object graph, to be fully buffered in
 memory. This is best suited for huge JSON documents that won't fit in memory.
 For example, streaming and processing large map/reduce views from Apache
 CouchDB.

data/lib/json/stream/buffer.rb

@@ -14,7 +14,7 @@ module JSON
     class Buffer
       def initialize
         @state = :start
-        @buf = []
+        @buffer = []
         @need = 0
       end
 
@@ -29,6 +29,12 @@ module JSON
       #
       # Returns a UTF-8 encoded String.
       def <<(data)
+        # Avoid state machine for complete UTF-8.
+        if @buffer.empty?
+          data.force_encoding(Encoding::UTF_8)
+          return data if data.valid_encoding?
+        end
+
         bytes = []
         data.each_byte do |byte|
           case @state
@@ -37,7 +43,7 @@ module JSON
               bytes << byte
             elsif byte >= 192
               @state = :multi_byte
-              @buf << byte
+              @buffer << byte
               @need =
                 case
                 when byte >= 240 then 4
@@ -49,9 +55,9 @@ module JSON
             end
           when :multi_byte
             if byte > 127 && byte < 192
-              @buf << byte
-              if @buf.size == @need
-                bytes += @buf.slice!(0, @buf.size)
+              @buffer << byte
+              if @buffer.size == @need
+                bytes += @buffer.slice!(0, @buffer.size)
                 @state = :start
               end
             else
@@ -59,8 +65,10 @@ module JSON
             end
           end
         end
-        bytes.pack('C*').force_encoding(Encoding::UTF_8).tap do |str|
-          error('Invalid UTF-8 byte sequence') unless str.valid_encoding?
+
+        # Build UTF-8 encoded string from completed codepoints.
+        bytes.pack('C*').force_encoding(Encoding::UTF_8).tap do |text|
+          error('Invalid UTF-8 byte sequence') unless text.valid_encoding?
         end
       end
 
@@ -82,7 +90,7 @@ module JSON
       #
       # Returns true if the buffer is empty.
       def empty?
-        @buf.empty?
+        @buffer.empty?
       end
 
       private

data/lib/json/stream/builder.rb

@@ -38,13 +38,15 @@ module JSON
 
       def end_object
         return if @stack.size == 1
+
         node = @stack.pop
+        top = @stack[-1]
 
-        case @stack.last
+        case top
         when Hash
-          @stack.last[@keys.pop] = node
+          top[@keys.pop] = node
         when Array
-          @stack.last << node
+          top << node
         end
       end
       alias :end_array :end_object
@@ -58,11 +60,12 @@ module JSON
       end
 
       def value(value)
-        case @stack.last
+        top = @stack[-1]
+        case top
         when Hash
-          @stack.last[@keys.pop] = value
+          top[@keys.pop] = value
         when Array
-          @stack.last << value
+          top << value
         else
           @stack << value
         end

data/lib/json/stream/parser.rb

@@ -73,46 +73,6 @@ module JSON
         stream.close
       end
 
-      # Drain any remaining buffered characters into the parser to complete
-      # the parsing of the document.
-      #
-      # This is only required when parsing a document containing a single
-      # numeric value, integer or float. The parser has no other way to
-      # detect when it should no longer expect additional characters with
-      # which to complete the parse, so it must be signaled by a call to
-      # this method.
-      #
-      # If you're parsing more typical object or array documents, there's no
-      # need to call `finish` because the parse will complete when the final
-      # closing `]` or `}` character is scanned.
-      #
-      # Raises a JSON::Stream::ParserError if the JSON data is malformed.
-      #
-      # Returns nothing.
-      def finish
-        # Partial multi-byte character waiting for completion bytes.
-        error('Unexpected end-of-file') unless @utf8.empty?
-
-        # Partial array, object, or string.
-        error('Unexpected end-of-file') unless @stack.empty?
-
-        case @state
-        when :end_document
-          # done, do nothing
-        when :in_float
-          end_value(@buf.to_f)
-        when :in_exponent
-          error('Unexpected end-of-file') unless @buf =~ DIGIT_END
-          end_value(@buf.to_f)
-        when :start_zero
-          end_value(@buf.to_i)
-        when :start_int
-          end_value(@buf.to_i)
-        else
-          error('Unexpected end-of-file')
-        end
-      end
-
       # Create a new parser with an optional initialization block where
       # we can register event callbacks.
       #
@@ -201,11 +161,11 @@ module JSON
             start_value(ch)
           when :start_object
             case ch
-            when RIGHT_BRACE
-              end_container(:object)
             when QUOTE
               @state = :start_string
               @stack.push(:key)
+            when RIGHT_BRACE
+              end_container(:object)
             when WS
               # ignore
             else
@@ -260,7 +220,7 @@ module JSON
               if @unicode.size == 4
                 codepoint = @unicode.slice!(0, 4).hex
                 if codepoint >= 0xD800 && codepoint <= 0xDBFF
-                  error('Expected low surrogate pair half') if @stack.last.is_a?(Fixnum)
+                  error('Expected low surrogate pair half') if @stack[-1].is_a?(Fixnum)
                   @state = :start_surrogate_pair
                   @stack.push(codepoint)
                 elsif codepoint >= 0xDC00 && codepoint <= 0xDFFF
@@ -402,17 +362,17 @@ module JSON
             case ch
             when COMMA
               @state = :value_sep
-            when RIGHT_BRACKET
-              end_container(:array)
             when RIGHT_BRACE
               end_container(:object)
+            when RIGHT_BRACKET
+              end_container(:array)
             when WS
               # ignore
             else
               error('Expected comma or object or array close')
             end
           when :value_sep
-            if @stack.last == :object
+            if @stack[-1] == :object
               case ch
               when QUOTE
                 @state = :start_string
@@ -431,6 +391,46 @@ module JSON
         end
       end
 
+      # Drain any remaining buffered characters into the parser to complete
+      # the parsing of the document.
+      #
+      # This is only required when parsing a document containing a single
+      # numeric value, integer or float. The parser has no other way to
+      # detect when it should no longer expect additional characters with
+      # which to complete the parse, so it must be signaled by a call to
+      # this method.
+      #
+      # If you're parsing more typical object or array documents, there's no
+      # need to call `finish` because the parse will complete when the final
+      # closing `]` or `}` character is scanned.
+      #
+      # Raises a JSON::Stream::ParserError if the JSON data is malformed.
+      #
+      # Returns nothing.
+      def finish
+        # Partial multi-byte character waiting for completion bytes.
+        error('Unexpected end-of-file') unless @utf8.empty?
+
+        # Partial array, object, or string.
+        error('Unexpected end-of-file') unless @stack.empty?
+
+        case @state
+        when :end_document
+          # done, do nothing
+        when :in_float
+          end_value(@buf.to_f)
+        when :in_exponent
+          error('Unexpected end-of-file') unless @buf =~ DIGIT_END
+          end_value(@buf.to_f)
+        when :start_zero
+          end_value(@buf.to_i)
+        when :start_int
+          end_value(@buf.to_i)
+        else
+          error('Unexpected end-of-file')
+        end
+      end
+
       private
 
       # Invoke all registered observer procs for the event type.

data/lib/json/stream/version.rb

@@ -2,6 +2,6 @@
 
 module JSON
   module Stream
-    VERSION = '0.2.0'
+    VERSION = '0.2.1'
   end
 end

data/spec/buffer_spec.rb

@@ -51,6 +51,13 @@ describe JSON::Stream::Buffer do
     assert_equal "\u{10102}", subject << "\x82"
   end
 
+  it 'rejects valid utf-8 followed by partial two byte sequence' do
+    assert_equal '[', subject << '['
+    assert_equal '"', subject << '"'
+    assert_equal '', subject << "\xC3"
+    -> { subject << '"' }.must_raise JSON::Stream::ParserError
+  end
+
   it 'rejects invalid two byte start characters' do
     -> { subject << "\xC3\xC3" }.must_raise JSON::Stream::ParserError
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: json-stream
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - David Graham
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-07-06 00:00:00.000000000 Z
+date: 2014-07-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake