s3_direct_upload 0.0.5 → 0.0.6

data/README.md

@@ -144,6 +144,13 @@ jQuery ->
 
 ### Javascript Events Hooks
 
+#### First upload started
+`s3_uploads_start` is fired once when any batch of uploads is starting.
+```coffeescript
+$('#myS3Uploader').bind 's3_uploads_start', (e) ->
+  alert("Uploads have started")
+```
+
 #### Successfull upload
 When a file has been successfully to S3, the `s3_upload_complete` is triggered on the form. A `content` object is passed along with the following attributes :
 
@@ -173,6 +180,33 @@ $(document).bind 's3_uploads_complete', ->
     alert("All Uploads completed")
 ```
 
+## Cleaning old uploads on S3
+You may be processing the files upon upload and reuploading them to another
+bucket or directory. If so you can remove the originali files by running a
+rake task.
+
+First, add the fog gem to your `Gemfile` and run `bundle`:
+```ruby
+  require 'fog'
+```
+
+Then, run the rake task to delete uploads older than 2 days:
+```
+  $ rake s3_direct_upload:clean_remote_uploads
+  Deleted file with key: "uploads/20121210T2139Z_03846cb0329b6a8eba481ec689135701/06 - PCR_RYA014-25.jpg"
+  Deleted file with key: "uploads/20121210T2139Z_03846cb0329b6a8eba481ec689135701/05 - PCR_RYA014-24.jpg"
+  $
+```
+
+Optionally customize the prefix used for cleaning (default is `uploads/#{2.days.ago.strftime('%Y%m%d')}`):
+**config/initalizers/s3_direct_upload.rb**
+```ruby
+S3DirectUpload.config do |c|
+  # ...
+  c.prefix_to_clean = "my_path/#{1.week.ago.strftime('%y%m%d')}"
+end
+```
+
 ## Contributing / TODO
 This is just a simple gem that only really provides some javascript and a form helper. 
 This gem could go all sorts of ways based on what people want and how people contribute. 

data/app/assets/javascripts/s3_direct_upload.js.coffee

@@ -35,6 +35,9 @@ $.fn.S3Uploader = (options) ->
           $uploadForm.append(data.context)
           data.submit()
 
+      start: (e) ->
+        $uploadForm.trigger("s3_uploads_start", [e])
+
       progress: (e, data) ->
         if data.context
           progress = parseInt(data.loaded / data.total * 100, 10)
@@ -49,11 +52,9 @@ $.fn.S3Uploader = (options) ->
           $.post(to, content)
 
         data.context.remove() if data.context && settings.remove_completed_progress_bar # remove progress bar
-        $uploadForm.trigger("s3_upload_complete", [content])
 
         current_files.splice($.inArray(data, current_files), 1) # remove that element from the array
-        if current_files.length == 0
-          $(document).trigger("s3_uploads_complete")
+        $uploadForm.trigger("s3_uploads_complete", [content]) unless current_files.length
 
       fail: (e, data) ->
         content = build_content_object $uploadForm, data.files[0], data.result
@@ -69,25 +70,20 @@ $.fn.S3Uploader = (options) ->
           name: "Content-Type"
           value: fileType
 
-        data[1].value = settings.path + data[1].value
-
+        data[1].value = settings.path + data[1].value #the key
         data
-
+        
   build_content_object = ($uploadForm, file, result) ->
-    domain = $uploadForm.attr('action')
     content = {}
     if result # Use the S3 response to set the URL to avoid character encodings bugs
-      path             = $('Key', result).text()
-      split_path       = path.split('/')
-      content.url      = domain + path
-      content.filename = split_path[split_path.length - 1]
-      content.filepath = split_path.slice(0, split_path.length - 1).join('/')
-    else # IE8 and IE9 return a null result object so we use the file object instead
-      path             = settings.path + $uploadForm.find('input[name=key]').val().replace('/${filename}', '')
-      content.url      = domain + path + '/' + file.name
-      content.filename = file.name
-      content.filepath = path
-
+      content.url      = $(result).find("Location").text()
+      content.filepath = $('<a />').attr('href', content.url)[0].pathname
+    else # IE <= 9 return a null result object so we use the file object instead
+      domain           = $uploadForm.attr('action')
+      content.filepath = settings.path + $uploadForm.find('input[name=key]').val().replace('/${filename}', '')
+      content.url      = domain + content.filepath + '/' + encodeURIComponent(file.name)
+
+    content.filename   = file.name
     content.filesize   = file.size if 'size' of file
     content.filetype   = file.type if 'type' of file
     content = $.extend content, settings.additional_data if settings.additional_data

data/lib/s3_direct_upload.rb

@@ -8,5 +8,6 @@ require 'digest/sha1'
 require 's3_direct_upload/config_aws'
 require 's3_direct_upload/form_helper'
 require 's3_direct_upload/engine' if defined?(Rails)
+require 's3_direct_upload/railtie' if defined?(Rails)
 
 ActionView::Base.send(:include, S3DirectUpload::UploadHelper) if defined?(ActionView::Base)

data/lib/s3_direct_upload/config_aws.rb

@@ -4,7 +4,7 @@ module S3DirectUpload
   class Config
     include Singleton
 
-    ATTRIBUTES = [:access_key_id, :secret_access_key, :bucket]
+    ATTRIBUTES = [:access_key_id, :secret_access_key, :bucket, :prefix_to_clean]
 
     attr_accessor *ATTRIBUTES
   end
@@ -15,4 +15,4 @@ module S3DirectUpload
     end
     Config.instance
   end
-end
+end

data/lib/s3_direct_upload/form_helper.rb

@@ -41,19 +41,20 @@ module S3DirectUpload
         {
           :key => @options[:key] || key,
           :acl => @options[:acl],
+          "AWSAccessKeyId" => @options[:aws_access_key_id],
           :policy => policy,
           :signature => signature,
-          "AWSAccessKeyId" => @options[:aws_access_key_id],
-          success_action_status: "201"
+          :success_action_status => "201",
+          'X-Requested-With' => 'xhr'
         }
       end
 
       def key
-        @key ||= "uploads/#{SecureRandom.hex}/${filename}"
+        @key ||= "uploads/#{DateTime.now.utc.strftime("%Y%m%dT%H%MZ")}_#{SecureRandom.hex}/${filename}"
       end
 
       def url
-        "https://#{@options[:bucket]}.s3.amazonaws.com/"
+        "https://s3.amazonaws.com/#{@options[:bucket]}/"
       end
 
       def policy
@@ -66,6 +67,7 @@ module S3DirectUpload
           conditions: [
             ["starts-with", "$utf8", ""],
             ["starts-with", "$key", ""],
+            ["starts-with", "$x-requested-with", ""],
             ["content-length-range", 0, @options[:max_file_size]],
             ["starts-with","$Content-Type",""],
             {bucket: @options[:bucket]},
@@ -85,4 +87,4 @@ module S3DirectUpload
       end
     end
   end
-end
+end

data/lib/s3_direct_upload/railtie.rb

@@ -0,0 +1,7 @@
+module S3DirectUpload
+  class Railtie < Rails::Railtie
+    initializer "railtie.configure_rails_initialization" do |app|
+      app.middleware.use JQuery::FileUpload::Rails::Middleware
+    end
+  end
+end

data/lib/s3_direct_upload/version.rb

@@ -1,3 +1,3 @@
 module S3DirectUpload
-  VERSION = "0.0.5"
+  VERSION = "0.0.6"
 end

data/lib/tasks/s3_direct_upload.rake

@@ -0,0 +1,57 @@
+namespace :s3_direct_upload do
+  desc "Removes old uploads from specified s3 bucket/directory -- Useful when uploads are processed into another directory"
+  task :clean_remote_uploads do
+    require 'thread'
+    require 'fog'
+
+    s3     = Fog::Storage::AWS.new(aws_access_key_id: S3DirectUpload.config.access_key_id, aws_secret_access_key: S3DirectUpload.config.secret_access_key)
+    bucket = S3DirectUpload.config.bucket
+    prefix = S3DirectUpload.config.prefix_to_clean || "uploads/#{2.days.ago.strftime('%Y%m%d')}"
+
+    queue         = Queue.new
+    semaphore     = Mutex.new
+    threads       = []
+    thread_count  = 20
+    total_listed  = 0
+    total_deleted = 0
+
+    threads << Thread.new do
+      Thread.current[:name] = "get files"
+      # Get all the files from this bucket. Fog handles pagination internally.
+      s3.directories.get("#{bucket}").files.all({prefix: prefix}).each do |file|
+        queue.enq(file)
+        total_listed += 1
+      end
+      # Add a final EOF message to signal the deletion threads to stop.
+      thread_count.times { queue.enq(:EOF) }
+    end
+
+    # Delete all the files in the queue until EOF with N threads.
+    thread_count.times do |count|
+      threads << Thread.new(count) do |number|
+        Thread.current[:name] = "delete files(#{number})"
+        # Dequeue until EOF.
+        file = nil
+        while file != :EOF
+          # Dequeue the latest file and delete it. (Will block until it gets a new file.)
+          file = queue.deq
+          unless file == :EOF
+            file.destroy
+            puts %Q{Deleted file with key: "#{file.key}"}
+          end
+          # Increment the global synchronized counter.
+          semaphore.synchronize {total_deleted += 1}
+        end
+      end
+    end
+
+    # Wait for the threads to finish.
+    threads.each do |t|
+      begin
+        t.join
+      rescue RuntimeError => e
+        puts "Failure on thread #{t[:name]}: #{e.message}"
+      end
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: s3_direct_upload
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.6
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-12-06 00:00:00.000000000 Z
+date: 2013-01-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -50,7 +50,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 3.2.1
+        version: 3.2.5
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -58,7 +58,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 3.2.1
+        version: 3.2.5
 - !ruby/object:Gem::Dependency
   name: jquery-fileupload-rails
   requirement: !ruby/object:Gem::Requirement
@@ -66,7 +66,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.3.5
+        version: 0.4.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -74,7 +74,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 0.3.5
+        version: 0.4.0
 description: Direct Upload to Amazon S3 With CORS and jquery-file-upload
 email:
 - w@waynehoover.com
@@ -85,8 +85,10 @@ files:
 - lib/s3_direct_upload/config_aws.rb
 - lib/s3_direct_upload/engine.rb
 - lib/s3_direct_upload/form_helper.rb
+- lib/s3_direct_upload/railtie.rb
 - lib/s3_direct_upload/version.rb
 - lib/s3_direct_upload.rb
+- lib/tasks/s3_direct_upload.rake
 - app/assets/javascripts/s3_direct_upload.js.coffee
 - app/assets/stylesheets/s3_direct_upload_progress_bars.css.scss
 - LICENSE
@@ -103,12 +105,18 @@ required_ruby_version: !ruby/object:Gem::Requirement
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: 865831684437857007
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+      segments:
+      - 0
+      hash: 865831684437857007
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.23