cae-multipart_parser 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: b40160cadbf8cc8a3f697358faf9fbc86189f69a
+  data.tar.gz: 9c53814388b57e07c4fe9067f1bcac1c642d5498
+SHA512:
+  metadata.gz: 01ed9067615cf6c444f92558b6d6232ff3b431afbd09264fbc5348f1a83183cb7e8c20d3b095f0e825fad8c1c788c12b19719cf554fa57232227693e3181c47b
+  data.tar.gz: 9cd5bf6885f0a60098497089797c9dc0345279a2458d69b8f11e198662f70d084af9a8bd9ca522a73e35e63881db17f38ee462564710050fd535fac6e8905d88

data/.gitignore

@@ -0,0 +1,15 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log
+*.swp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in cae-logger.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2015 Chris Elsworth
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,26 @@
+# Event-driven HTTP Multipart Parser
+
+This is based on https://github.com/danabr/multipart-parser, with
+modifications to suit my use-case.
+
+It currently depends on finding a `Content-Length` part header to avoid having to scan the entire body. It will raise `Cae::MultipartParser::Parser::ContentLengthUnsetError` if this header is not present.
+
+
+## Usage
+
+```ruby
+parser = Cae::MultipartParser::Parser.new(boundary: boundary)
+
+parser.parse fh do |part|
+	part.on(:headers) do |headers|
+		# headers is a Hash
+	end
+	part.on(:data) do |data|
+		# data is a chunk of body data
+		# this may be called multiple times
+	end
+	part.on(:end) do
+		# part is finished, there will be no more data callbacks
+	end
+end
+```

data/Rakefile

@@ -0,0 +1,14 @@
+### Specs
+
+spec = proc do |env|
+	env.each{|k,v| ENV[k] = v}
+	sh "#{FileUtils::RUBY} -rubygems -I lib -e 'ARGV.each{|f| require f}' ./spec/*_spec.rb"
+	env.each{|k,v| ENV.delete(k) }
+end
+
+desc "Run specs"
+task "spec" do
+	spec.call({})
+end
+
+task :default => :spec

data/cae-multipart_parser.gemspec

@@ -0,0 +1,22 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'cae/multipart_parser/version'
+
+Gem::Specification.new do |spec|
+	spec.name          = "cae-multipart_parser"
+	spec.version       = Cae::MultipartParser::VERSION
+	spec.authors       = ["Chris Elsworth"]
+	spec.email         = ["chris@shagged.org"]
+	spec.summary       = "Event-driven HTTP Multipart parser"
+	spec.homepage      = "https://github.com/celsworth/cae-multipart_parser"
+	spec.license       = "MIT"
+
+	spec.files         = `git ls-files -z`.split("\x0")
+	spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+	spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+	spec.require_paths = ["lib"]
+
+	spec.add_development_dependency "multipart-post"
+	spec.add_development_dependency "minitest"
+end

data/lib/cae/multipart_parser/parser.rb

@@ -0,0 +1,165 @@
+# vim: et sw=2 ts=2 sts=2
+
+module Cae
+  module MultipartParser
+    class Parser
+
+      CR = "\r".freeze
+      LF = "\n".freeze
+      DASH = "-".freeze
+
+      BOUNDARY_PREFIX = (CR + LF + DASH + DASH).freeze
+
+      ContentLengthUnsetError = Class.new(StandardError)
+
+      attr_accessor :read_buffer_size
+
+      def initialize(opts = {})
+        # remember offsets into our state in between calls
+        @index = 0
+
+        @state = :start
+
+        # default 2MB read buffer
+        @read_buffer_size = 2 * 1024 * 1024
+
+        @boundary = BOUNDARY_PREFIX + opts[:boundary]
+        @boundary_length = @boundary.length
+      end
+
+      # Parse data from the IO +io+, calling callbacks appropriately.
+      #
+      # @return [Integer] the number of bytes parsed.
+      def parse(io)
+        parsed = 0
+
+        buffer = String.new
+        while io.read(@read_buffer_size, buffer)
+          length = buffer.length
+          i = 0
+          data_start = 0
+
+          #p "parsing #{length} chars: #{buffer}"
+
+          while i < length
+            c = buffer[i]
+
+            #p "state=#{@state}: i=#{i}. index=#{@index}, d_start=#{data_start} chars=#{buffer[i, 50]}"
+
+            case @state
+            when :start
+              if @index == @boundary_length - 2
+                break unless c == CR
+                @index += 1
+              elsif @index == @boundary_length - 1
+                break unless c == LF
+                # reached end of boundary, we're into the first part
+                @state = :headers_start
+              else
+                # there is no leading \r\n on the first boundary, hence index+2
+                break unless c == @boundary[@index + 2] # Unexpected character
+                @index += 1
+              end
+
+            when :headers_start
+              @part = Part.new
+              @state = :headers
+              @index = 0
+              @headers = ''
+              next # keep i pointing at current char for :headers
+
+            when :headers
+              if (c == CR && @index == 0) || (c == LF && @index == 1)
+                # keep \r\n to split on later
+                @headers << c
+                @index += 1
+              elsif c == CR && @index == 2
+                # don't keep final \r, update state to check for final \n
+                @state = :headers_almost_done
+              else
+                # normal header char, reset index and keep char
+                @headers << c
+                @index = 0
+              end
+
+            when :headers_almost_done
+              break unless c == LF # Unexpected character
+              @state = :part_start
+
+            when :part_start
+              # this must populate #content_length
+              @part.parse_header_str @headers
+
+              @part_data_remaining = @part.content_length
+              raise ContentLengthUnsetError if @part_data_remaining == 0
+
+              # allow caller to setup callbacks
+              yield @part
+
+              @part.callback :headers, @part.headers
+
+              data_start = i
+              @index = 0
+              @state = :part_data
+              next # keep i pointing at current char for :part_data
+
+            when :part_data
+              chunk_remaining = length - data_start
+              cb_len = @part_data_remaining > chunk_remaining ? chunk_remaining : @part_data_remaining
+
+              @part.callback :data, buffer[data_start, cb_len]
+
+              @part_data_remaining -= cb_len
+              i += cb_len
+
+              @state = :boundary if @part_data_remaining == 0
+              next # we've bumped i already, don't increment it
+
+            when :boundary
+              break unless c == @boundary[@index] # unexpected character
+              @index += 1
+              @state = :boundary_almost_done if @index == @boundary_length
+
+            when :boundary_almost_done
+              # work out whether this is a part boundary or the final boundary
+              if c == CR
+                @state = :boundary_part_almost_done
+              elsif c == DASH
+                @state = :boundary_last_almost_done
+              else
+                break # unexpected character, this isn't a boundary after all
+              end
+              @index += 1
+
+            when :boundary_part_almost_done
+              # final character of an inter-part boundary must be LF
+              break unless c == LF # unexpected character
+              @part.callback :end
+              @state = :headers_start
+
+            when :boundary_last_almost_done
+              # final character of final boundary must be -
+              break unless c == DASH # unexpected character
+              @part.callback :end
+              @state = :end
+
+            end # case
+
+            i += 1
+
+          end # while
+
+          if i != length
+            raise "unexpected char at #{i} (#{buffer[i].inspect})"
+          end
+
+          parsed += length
+
+        end # while
+
+        parsed
+      end
+
+    end
+  end
+end

data/lib/cae/multipart_parser/part.rb

@@ -0,0 +1,44 @@
+# vim: et sw=2 ts=2 sts=2
+module Cae
+  module MultipartParser
+    class Part
+
+      attr_reader :headers
+
+      def initialize
+        @callbacks = {}
+        @headers = {}
+      end
+
+      def parse_header_str(str)
+        # munge multiline headers back into one line.
+        str = str.gsub /\r\n\s+/, ' '
+
+        # split header string into a hash
+        str.split(/\r\n/).each do |h|
+          key, value = h.split ':'
+
+          # normalize content-length -> Content-Length
+          key = key.split('-').map(&:capitalize).join('-')
+
+          @headers[key] = value.lstrip
+        end
+        @headers
+      end
+
+      def content_length
+        @headers['Content-Length'].to_i
+      end
+
+      def on(event, &callback)
+        @callbacks[event] = callback
+      end
+
+      # Parser will call :headers, :data, :end
+      def callback(event, arg = nil)
+        @callbacks[event].call(arg) if @callbacks.has_key?(event)
+      end
+
+    end
+  end
+end

data/lib/cae/multipart_parser/version.rb

@@ -0,0 +1,5 @@
+module Cae
+  module MultipartParser
+    VERSION = "1.0.0"
+  end
+end

data/lib/cae/multipart_parser.rb

@@ -0,0 +1,2 @@
+require 'cae/multipart_parser/parser'
+require 'cae/multipart_parser/part'

data/spec/parser_spec.rb

@@ -0,0 +1,73 @@
+# vim: et sw=2 ts=2 sts=2
+
+require File.expand_path("spec_helper", File.dirname(__FILE__))
+
+describe Cae::MultipartParser::Parser do
+  let(:boundary) do
+    # create a random boundary
+    ("-" * 24) + SecureRandom.random_bytes(8).unpack('H*').first
+  end
+
+  let(:parser) do
+    Cae::MultipartParser::Parser.new(boundary: boundary)
+  end
+
+  describe "#parse" do
+    it "returns the number of bytes parsed" do
+      part = SecureRandom.random_bytes(1024) # random data
+      body = generate_body(boundary, [part])
+      fh = StringIO.new body
+
+      r = parser.parse fh do |part|
+      end
+
+      r.must_equal body.length
+    end
+
+    it "calls the :headers callback with a hash" do
+      part = SecureRandom.random_bytes(1024) # random data
+      fh = StringIO.new generate_body(boundary, [part])
+
+      headers = nil
+      parser.parse fh do |part|
+        part.on(:headers){|h| headers = h }
+      end
+      headers.must_be_kind_of Hash
+    end
+
+    it "calls the :data callback with the original data" do
+      part = SecureRandom.random_bytes(1024 * 1024) # 1MB of random data
+      fh = StringIO.new generate_body(boundary, [part])
+      ret = ''
+      parser.parse fh do |part|
+        part.on(:data){|data| ret << data }
+      end
+
+      ret.must_equal part
+    end
+
+    it "calls the :end callback after the part is done" do
+      part = SecureRandom.random_bytes(1024) # random data
+      fh = StringIO.new generate_body(boundary, [part])
+      done = 0
+      parser.parse fh do |part|
+        part.on(:end){ done += 1 }
+      end
+
+      done.must_equal 1
+    end
+
+    it "calls callbacks after each part is done" do
+      part = SecureRandom.random_bytes(1024) # random data
+      fh = StringIO.new generate_body(boundary, [part, part])
+      headers, done = 0, 0
+      parser.parse fh do |part|
+        part.on(:headers){ headers += 1 }
+        part.on(:end){ done += 1 }
+      end
+
+      headers.must_equal 2
+      done.must_equal 2
+    end
+  end
+end

data/spec/part_spec.rb

@@ -0,0 +1,82 @@
+# vim: et sw=2 ts=2 sts=2
+
+require File.expand_path("spec_helper", File.dirname(__FILE__))
+
+describe Cae::MultipartParser::Part do
+  let(:part) { Cae::MultipartParser::Part.new }
+
+  describe '#parse_header_str' do
+    it "parses a simple input" do
+      input = "Content-Type: text/html"
+      part.parse_header_str(input).must_equal({
+        'Content-Type' => 'text/html'
+      })
+    end
+
+    it "ignores trailing CRLF" do
+      input = "Content-Type: text/html\r\n"
+      part.parse_header_str(input).must_equal({
+        'Content-Type' => 'text/html'
+      })
+    end
+
+    it "parses multiple lines" do
+      input = "Content-Type: text/html\r\nContent-Length: 100"
+      part.parse_header_str(input).must_equal({
+        'Content-Type' => 'text/html',
+        'Content-Length' => '100'
+      })
+    end
+
+    it "parses multiline headers" do
+      input = "Content-Type: multipart/form-data;\r\n\tboundary=foo"
+      part.parse_header_str(input).must_equal({
+        'Content-Type' => 'multipart/form-data; boundary=foo'
+      })
+    end
+
+    it "parses multiple multiline headers" do
+      input = "Content-Type: multipart/form-data;\r\n\tboundary=foo\r\nContent-Type2: multipart/form-data;\r\n\tboundary=bar"
+      part.parse_header_str(input).must_equal({
+        'Content-Type' => 'multipart/form-data; boundary=foo',
+        'Content-Type2' => 'multipart/form-data; boundary=bar'
+      })
+    end
+  end
+
+  describe '#content_length' do
+    it "considers Content-Length to be zero if unset" do
+      input = "Content-Type: text/html"
+      part.parse_header_str(input)
+      part.content_length.must_equal 0
+    end
+
+    it "exposes Content-Length if present in headers" do
+      input = "Content-Type: text/html\r\nContent-Length: 100"
+      part.parse_header_str(input)
+      part.content_length.must_equal 100
+    end
+  end
+
+  describe "#callback" do
+    it "ignores an unregistered callback type" do
+      part.callback(:unregistered).must_equal nil
+    end
+
+    it "calls a registered callback type" do
+      foo = 0
+      cb = ->(arg){ foo = arg }
+      part.on(:registered, &cb)
+      part.callback(:registered, 1)
+      foo.must_equal 1
+    end
+
+    it "defaults to a nil arg" do
+      foo = true
+      cb = ->(arg){ foo = arg }
+      part.on(:registered, &cb)
+      part.callback(:registered)
+      foo.must_equal nil
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,25 @@
+# vim: et sw=2 ts=2 sts=2
+
+$:.unshift(File.expand_path("../lib", File.dirname(__FILE__)))
+
+require "rubygems"
+
+gem 'minitest'
+require "minitest/autorun"
+
+require 'cae/multipart_parser'
+
+require 'securerandom'
+require 'net/http/post/multipart'
+
+# create a multipart body out of the given array
+def generate_body(boundary, arr)
+  parts = {}
+  arr.each_with_index do |part, idx|
+    fileno = "file#{idx}"
+    parts[fileno] = UploadIO.new(StringIO.new(part), fileno, 'application/binary')
+  end
+
+  req = Net::HTTP::Post::Multipart.new '/', parts, {}, boundary
+  req.body_stream.read
+end

metadata

@@ -0,0 +1,89 @@
+--- !ruby/object:Gem::Specification
+name: cae-multipart_parser
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Chris Elsworth
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-09-03 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: multipart-post
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- chris@shagged.org
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- cae-multipart_parser.gemspec
+- lib/cae/multipart_parser.rb
+- lib/cae/multipart_parser/parser.rb
+- lib/cae/multipart_parser/part.rb
+- lib/cae/multipart_parser/version.rb
+- spec/parser_spec.rb
+- spec/part_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/celsworth/cae-multipart_parser
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5.1
+signing_key: 
+specification_version: 4
+summary: Event-driven HTTP Multipart parser
+test_files:
+- spec/parser_spec.rb
+- spec/part_spec.rb
+- spec/spec_helper.rb
+has_rdoc: