rack-large-uploads 0.0.1 → 0.0.2

data/README.md

@@ -11,6 +11,16 @@ Based largely on the [`Rack::Uploads` middleware](https://github.com/mutle/rack-
 but with greater expectations regarding conventions, and specific support for
 large files (dealing with memory issues).
 
+Implements **(experimental)** handling of chunked uploads, only passing control
+to, e.g., Rails once all uploads for the request are complete. This is very
+experimental, because it is currently:
+
+  * dependent upon headers and filename value as set by blueimp's jQuery
+    fileupload plugin
+  * dependent on the presence of `uploader` request param, which is a UUID or
+    similar, and is used to collect the upload chunks
+  * untested
+
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -49,7 +59,7 @@ example nginx configuration:
       # depends on the nginx upload module (http://www.grid.net.ru/nginx/upload.en.html)
       upload_pass         @application;
 
-      # NOTE: if there is no upload file in the request, nginx generates a 405.
+      # NOTE: if the request is something other than POST, nginx generates a 405.
       # use that to pass the request on to the endpoint, instead of try_files
       # which fails to play nicely with upload_pass.
       #
@@ -57,10 +67,15 @@ example nginx configuration:
       # http://www.nickager.com/blog/File-upload-using-Nginx-and-Seaside---step-1
       error_page          405 415 = @application;
 
+      # NOTE: cleaning up 201 & 202 means we need to be sure our app is doing
+      # some processing right away. Rack::LargeUploads sends 202 responses for
+      # each chunk received during chunked uploads, and the chunk has been
+      # appended to the combined file by the time we're back with the response.
+      upload_cleanup      201 202 400 404 499 500-505;
       upload_store        /path/to/app/tmp/uploads 1;
       upload_store_access user:rw group:rw all:rw;
 
-      upload_pass_args       on;         # NOTE: handles URI params, not form content.
+      upload_pass_args       on;          # NOTE: handles URI params, not form content.
       upload_pass_form_field "^[a-z_].*"; # ...and here is how we resolve that last.
 
       # match the request params expected by ActionDispatch

data/lib/rack/large-uploads.rb

@@ -1,12 +1,14 @@
-require "rack/large-uploads/version"
+require 'fileutils'
+require 'rack/large-uploads/version'
 
 # TODO: remove these dependencies:
-require "action_dispatch"
-require "active_support/core_ext"
+require 'action_dispatch'
+require 'active_support/core_ext'
 
 module Rack
   class LargeUploads
-    autoload :UploadedFile, 'rack/large-uploads/uploaded_file'
+    autoload :UploadedChunk, 'rack/large-uploads/uploaded_chunk'
+    autoload :UploadedFile,  'rack/large-uploads/uploaded_file'
 
     def initialize(app, options = {}, &block)
       @app     = app
@@ -23,18 +25,28 @@ module Rack
 
     def call(env)
       request = Rack::Request.new(env)
+      params  = request.params
 
       if request.post? && request.form_data?
-        files = extract_files(request, request.params)
+        files = extract_files(request, params)
       end
 
-      filter(:before, files) if files.present?
-      response = @app.call(env)
-      filter(:after, files) if files.present?
+      if files.present?
+        if files.all? { |f| f.is_a?(Rack::LargeUploads::UploadedFile) }
+          filter(:before, env, files)
+          response = @app.call(env)
+          filter(:after, env, files)
 
-      response
+          return response
+        else
+          return [202, {}, []]
+        end
+      else
+        @app.call(env)
+      end
     end
 
+
     def before(&block)
       @filters[:before] = block
     end
@@ -45,8 +57,8 @@ module Rack
 
     private
 
-      def filter(position, files)
-        @filters[position] && @filters[position].call(files)
+      def filter(position, env, files)
+        @filters[position] && @filters[position].call(env, files)
       end
 
       def extract_files(request, params, files = [])
@@ -61,6 +73,10 @@ module Rack
         files
       end
 
+      # TODO:
+      #   * make chunked check configurable (:filename value at least, maybe other)
+      #   * make chunked storage path configurable
+      #   * make 'uploader' param configurable, and per-upload rather than per-request
       def file_from(request, key, value)
         # direct to rails...
         # ----->
@@ -83,13 +99,42 @@ module Rack
         #     "md5"     =>"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
         #     "size"    =>"5242880000"
         #   }
+
         if value.is_a?(Hash)
           attributes = HashWithIndifferentAccess.new(value)
           tempfile   = attributes[:tempfile]
 
           if tempfile.present?
-            tempfile = ::File.new(tempfile) if tempfile.is_a?(String)
-            return Rack::LargeUploads::UploadedFile.new(attributes.merge({ :tempfile => tempfile }))
+            # check for "chunked" upload
+            if attributes[:filename] == 'blob'
+              temppath = tempfile.is_a?(String) ? tempfile : tempfile.path
+              storage  = ::File.expand_path(::File.join(temppath, '../../chunked'))
+              uploader = request.params['uploader']
+              combined = ::File.join(storage, uploader)
+              fullsize = request.env['HTTP_X_FILE_SIZE'].to_i
+
+              # append chunk to "combined" File
+              FileUtils.mkdir_p(storage)
+              ::File.open(combined, 'ab') do |f|
+                f.write(::File.read(tempfile))
+              end
+
+              # finished?
+              if ::File.size(combined) == fullsize
+                attributes = {
+                  :filename => request.env['HTTP_X_FILE_NAME'],
+                  :size     => request.env['HTTP_X_FILE_SIZE'],
+                  :type     => request.env['HTTP_X_FILE_TYPE'],
+                  :tempfile => ::File.new(combined)
+                }
+
+                return Rack::LargeUploads::UploadedFile.new(attributes)
+              else
+                return Rack::LargeUploads::UploadedChunk.new(attributes)
+              end
+            else
+              return Rack::LargeUploads::UploadedFile.new(attributes)
+            end
           end
         end
 

data/lib/rack/large-uploads/uploaded_chunk.rb

@@ -0,0 +1,7 @@
+module Rack
+  class LargeUploads
+    class UploadedChunk < ActionDispatch::Http::UploadedFile
+      # stay tuned
+    end
+  end
+end

data/lib/rack/large-uploads/uploaded_file.rb

@@ -2,6 +2,9 @@ module Rack
   class LargeUploads
     class UploadedFile < ActionDispatch::Http::UploadedFile
       def initialize(hash)
+        tempfile = hash[:tempfile]
+        hash[:tempfile] = ::File.new(tempfile) if tempfile.is_a?(String)
+
         @uploaded_md5  = hash.delete(:md5)
         @uploaded_size = hash.delete(:size)
         super(hash)

data/lib/rack/large-uploads/version.rb

@@ -1,5 +1,5 @@
 module Rack
   class LargeUploads
-    VERSION = "0.0.1"
+    VERSION = "0.0.2"
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rack-large-uploads
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-03-30 00:00:00.000000000 Z
+date: 2012-09-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
-  requirement: &70180264879460 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,15 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70180264879460
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &70180264895180 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +37,15 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70180264895180
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70180264894620 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +53,15 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70180264894620
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rr
-  requirement: &70180264894040 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +69,15 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70180264894040
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: simplecov
-  requirement: &70180264893460 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +85,15 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70180264893460
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: actionpack
-  requirement: &70180264892740 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +101,15 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70180264892740
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: activesupport
-  requirement: &70180264892100 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,7 +117,12 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70180264892100
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: ! 'Rack middleware for handling large file uploads. Integrates nicely
   with the Nginx upload module: http://www.grid.net.ru/nginx/upload.en.html'
 email:
@@ -107,6 +142,7 @@ files:
 - bootstrap.gems
 - lib/rack-large-uploads.rb
 - lib/rack/large-uploads.rb
+- lib/rack/large-uploads/uploaded_chunk.rb
 - lib/rack/large-uploads/uploaded_file.rb
 - lib/rack/large-uploads/version.rb
 - rack-large-uploads.gemspec
@@ -132,10 +168,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -1884482723306409026
+      hash: -2149263862982449584
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.11
+rubygems_version: 1.8.24
 signing_key: 
 specification_version: 3
 summary: Rack middleware for handling large file uploads