cae-multipart_parser 1.1.0 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 58b85fbc0eb96b1b5d3d124d74b726df472fb1ed
-  data.tar.gz: 34cf8eabe80ace9e5df827b27b1355cd377d0e2d
+  metadata.gz: ba2747bd66a2c381c5d6b0845d5005af0c193638
+  data.tar.gz: 59fa00307201bb96569afcf18146e0466abc3dad
 SHA512:
-  metadata.gz: 5ec5356efc56b49734c768631fda770bf8e51dd3aa6edd60cc9fa4d24bd604ba077fdac53268e86e398e1215e84415603fc54f750932f0ecf722989bf06824bd
-  data.tar.gz: 39a4915b557153bd5f39630471d2a03b29b58e496cf7ca7fdbadbabe8f7d9b001baa8e48dd018b99389baaade121731e575fc715e709e29936bd60cbc8fbf2af
+  metadata.gz: 69c00279283004fa1f1145a8b0b66afc915ef510628f8fd530d60c881fb01755ceed5114705d1a94a0d535cd11d2f55c03e2f4d94931d509c920d2790c45095a
+  data.tar.gz: 2accc6e0407347dfebe6154a06dc484a3d2ab67bd918722218d26ffddc9df6a4c4b5417d18f7fec30af787aaeb55ae485fefec6a90952b78d1d7d477ffd2a17a

data/README.md

@@ -1,7 +1,6 @@
 # Event-driven HTTP Multipart Parser
 
-This is based on https://github.com/danabr/multipart-parser, with
-modifications to suit my use-case.
+This is based on https://github.com/danabr/multipart-parser, with modifications to suit my use-case.
 
 It currently depends on finding a `Content-Length` part header to avoid having to scan the entire body. It will raise `Cae::MultipartParser::Parser::ContentLengthUnsetError` if this header is not present.
 
@@ -12,15 +11,25 @@ It currently depends on finding a `Content-Length` part header to avoid having t
 parser = Cae::MultipartParser::Parser.new(boundary: boundary)
 
 parser.parse fh do |part|
-	part.on(:headers) do |headers|
-		# headers is a Hash
+	# part.headers and part.content_length should be set now
+	# headers are underscored and uppercased:
+	if part.headers['CONTENT_TYPE'] == 'text/html'
+		# ...
 	end
-	part.on(:data) do |data|
-		# data is a chunk of body data
-		# this may be called multiple times
+
+	# content_length is an integer:
+	if part.content_length < 1024
+		# ...
 	end
-	part.on(:end) do
-		# part is finished, there will be no more data callbacks
+
+	while part.body.read(chunksize, buf)
+		# buf contains up to chunksize bytes of data.
+		# Do not assume if less than chunksize is returned, you're done,
+		# for internal as-yet-to-be-fixed reasons.
+		# Only stop when #read returns nil. Note that #read is NOT IO#read,
+		# but is mostly compatible.
 	end
+
+	# all the part body has now been read
 end
-```
+```

data/lib/cae/multipart_parser/parser.rb

@@ -44,6 +44,8 @@ module Cae
           while i < length
             c = buffer[i]
 
+            #p "state=#{@state} index=#{@index} chars=#{buffer[i, 40]}"
+
             case @state
             when :start
               if @index == @boundary_length - 2
@@ -88,29 +90,17 @@ module Cae
               # this must populate #content_length
               @part.parse_header_str @headers
 
-              @part_data_remaining = @part.content_length
-              raise ContentLengthUnsetError if @part_data_remaining == 0
-
-              # allow caller to setup callbacks
-              yield @part
+              raise ContentLengthUnsetError if @part.content_length == 0
 
-              @part.callback :headers, @part.headers
-
-              data_start = i
-              @index = 0
-              @state = :part_data
-              next # keep i pointing at current char for :part_data
-
-            when :part_data
               chunk_remaining = length - data_start
-              cb_len = @part_data_remaining > chunk_remaining ? chunk_remaining : @part_data_remaining
-
-              @part.callback :data, buffer[data_start, cb_len]
+              cb_len = @part.content_length > chunk_remaining ? chunk_remaining : @part.content_length
+              @part.body = Part::Body.new(io, @part.content_length, buffer[i, cb_len])
+              yield @part
 
-              @part_data_remaining -= cb_len
               i += cb_len
 
-              @state = :boundary if @part_data_remaining == 0
+              @index = 0
+              @state = :boundary
               next # we've bumped i already, don't increment it
 
             when :boundary

data/lib/cae/multipart_parser/part.rb

@@ -1,30 +1,34 @@
 # vim: et sw=2 ts=2 sts=2
+
+require 'cae/multipart_parser/part/body'
+
 module Cae
   module MultipartParser
     class Part
 
       attr_reader :headers
+      attr_accessor :body
 
       def initialize
         @callbacks = {}
         @headers = {}
+        @body = nil
       end
 
       def parse_header_str(str)
-        # munge multiline headers back into one line.
+        # Munge multiline headers back into one line.
         str = str.gsub /\r\n\s+/, ' '
 
-        # split header string into a hash
-        str.split(/\r\n/).each do |h|
-          key, value = h.split ':'
+        # Split header string into a hash. Returns the initial hash.
+        str.split(/\r\n/).each_with_object(@headers) do |line, headers|
+          key, value = line.split ':'
 
-          # normalize content-length -> CONTENT_LENGTH
+          # normalize Content-Length -> CONTENT_LENGTH
           key.upcase!
           key.tr! '-', '_'
 
-          @headers[key] = value.lstrip
+          headers[key] = value.lstrip
         end
-        @headers
       end
 
       def content_length

data/lib/cae/multipart_parser/part/body.rb

@@ -0,0 +1,49 @@
+# vim: et sw=2 ts=2 sts=2
+module Cae
+  module MultipartParser
+    class Part
+      class Body
+
+        def initialize(fh, read_limit, read_buffer = nil)
+          # Backing filehandle
+          @fh = fh
+
+          # After we've read this amount, act like the backing filehandle is empty
+          @read_limit = read_limit
+
+          # If anything is in this, we'll return it on the first read()
+          @read_buffer = read_buffer
+        end
+
+        def read(length, outbuf = nil)
+          # Check we're not trying to read more bytes than are available
+          length = @read_limit > length ? length : @read_limit
+
+          # Early nil return if there's nothing available. This technically
+          # breaks compatibility with IO#read, but I don't care.
+          # (IO#read returns an empty string if passed length is 0; we'll return nil)
+          return nil if length == 0
+
+          # if there's anything in @read_buffer, return it before doing a real read
+          if @read_buffer
+            # initialise outbuf if it wasn't passed in.
+            outbuf ||= String.new
+
+            # copy contents into outbuf, being careful NOT to change the object_id
+            outbuf.clear
+            outbuf << @read_buffer[0, length]
+
+            # advance buffer pointer; if there's none left, @read_buffer will be nil
+            @read_limit -= outbuf.length
+            @read_buffer = @read_buffer[length, @read_limit]
+
+            return outbuf
+          end
+
+          @fh.read(length, outbuf).tap{|o| @read_limit -= o.length }
+        end
+
+      end
+    end
+  end
+end

data/lib/cae/multipart_parser/version.rb

@@ -1,5 +1,5 @@
 module Cae
   module MultipartParser
-    VERSION = "1.1.0"
+    VERSION = "2.0.0"
   end
 end

data/spec/parser_spec.rb

@@ -18,56 +18,40 @@ describe Cae::MultipartParser::Parser do
       body = generate_body(boundary, [part])
       fh = StringIO.new body
 
-      r = parser.parse fh do |part|
-      end
-
+      r = parser.parse(fh){ }
       r.must_equal body.length
     end
 
-    it "calls the :headers callback with a hash" do
+    it "sets part#headers" do
       part = SecureRandom.random_bytes(1024) # random data
       fh = StringIO.new generate_body(boundary, [part])
 
       headers = nil
-      parser.parse fh do |part|
-        part.on(:headers){|h| headers = h }
-      end
+      parser.parse(fh){|part| headers = part.headers }
       headers.must_be_kind_of Hash
     end
 
-    it "calls the :data callback with the original data" do
+    it "passes the original data to the part#body handle" do
       part = SecureRandom.random_bytes(1024 * 1024) # 1MB of random data
       fh = StringIO.new generate_body(boundary, [part])
       ret = ''
       parser.parse fh do |part|
-        part.on(:data){|data| ret << data }
+        part.body.must_be_kind_of Cae::MultipartParser::Part::Body
+        while x = part.body.read(1024)
+          ret << x
+        end
       end
 
       ret.must_equal part
     end
 
-    it "calls the :end callback after the part is done" do
+    it "yields for each part" do
       part = SecureRandom.random_bytes(1024) # random data
-      fh = StringIO.new generate_body(boundary, [part])
+      parts = [part, part]
+      fh = StringIO.new generate_body(boundary, parts)
       done = 0
-      parser.parse fh do |part|
-        part.on(:end){ done += 1 }
-      end
-
-      done.must_equal 1
-    end
-
-    it "calls callbacks after each part is done" do
-      part = SecureRandom.random_bytes(1024) # random data
-      fh = StringIO.new generate_body(boundary, [part, part])
-      headers, done = 0, 0
-      parser.parse fh do |part|
-        part.on(:headers){ headers += 1 }
-        part.on(:end){ done += 1 }
-      end
-
-      headers.must_equal 2
-      done.must_equal 2
+      parser.parse(fh){|part| done += 1 }
+      done.must_equal parts.count
     end
   end
 end

data/spec/part_body_spec.rb

@@ -0,0 +1,69 @@
+# vim: et sw=2 ts=2 sts=2
+
+require File.expand_path("spec_helper", File.dirname(__FILE__))
+
+def read_all(fh, chunksize, outbuf = nil)
+  str = String.new
+  outbuf = String.new
+  while fh.read(chunksize, outbuf)
+    str << outbuf
+  end
+  str
+end
+
+describe Cae::MultipartParser::Part::Body do
+  let(:fh_data) { '12345' }
+  let(:initial_buffer) { nil }
+  let(:size_limit) { 5 }
+  let(:expected) { ((initial_buffer || '') + fh_data)[0, size_limit] }
+  let(:fh) { StringIO.new fh_data }
+  let(:body) do
+    Cae::MultipartParser::Part::Body.new(fh, size_limit, initial_buffer)
+  end
+
+  (1..15).each do |chunksize|
+    it "should work with read chunk size #{chunksize}" do
+      read_all(body, chunksize).must_equal expected
+    end
+  end
+
+  describe "with an initial buffer" do
+    let(:initial_buffer) { 'abcde' }
+    let(:size_limit) { 10 }
+
+    (1..15).each do |chunksize|
+      it "should work with read chunk size #{chunksize}" do
+        read_all(body, chunksize).must_equal expected
+      end
+    end
+
+    it "should not reallocate if passed an outbuf" do
+      outbuf = String.new
+      refute_changes(->{outbuf.object_id}) do
+        while body.read(1, outbuf)
+          # no-op
+        end
+      end
+    end
+
+    it "should return nil when empty" do
+      read_all(body, size_limit) # empty the "file"
+      body.read(1).must_equal nil
+    end
+
+    describe "with a shorter size_limit than data available" do
+      let(:size_limit) { 5 }
+      it "should stop returning data when size_limit is hit" do
+        read_all(body, 1).must_equal expected
+      end
+
+      it "should return nil when 'empty'" do
+        read_all(body, size_limit) # empty the "file"
+        body.read(1).must_equal nil
+      end
+    end
+
+  end
+
+
+end

data/spec/spec_helper.rb

@@ -23,3 +23,11 @@ def generate_body(boundary, arr)
   req = Net::HTTP::Post::Multipart.new '/', parts, {}, boundary
   req.body_stream.read
 end
+
+
+def refute_changes(what)
+  old = what.call
+  yield
+  assert_equal old, what.call
+end
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cae-multipart_parser
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 2.0.0
 platform: ruby
 authors:
 - Chris Elsworth
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-09-05 00:00:00.000000000 Z
+date: 2015-09-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: multipart-post
@@ -54,8 +54,10 @@ files:
 - lib/cae/multipart_parser.rb
 - lib/cae/multipart_parser/parser.rb
 - lib/cae/multipart_parser/part.rb
+- lib/cae/multipart_parser/part/body.rb
 - lib/cae/multipart_parser/version.rb
 - spec/parser_spec.rb
+- spec/part_body_spec.rb
 - spec/part_spec.rb
 - spec/spec_helper.rb
 homepage: https://github.com/celsworth/cae-multipart_parser
@@ -84,6 +86,7 @@ specification_version: 4
 summary: Event-driven HTTP Multipart parser
 test_files:
 - spec/parser_spec.rb
+- spec/part_body_spec.rb
 - spec/part_spec.rb
 - spec/spec_helper.rb
 has_rdoc: