s3_multi_upload 0.0.2 → 0.0.4

data/bin/s3_multi_upload

@@ -11,8 +11,9 @@ module S3_Multi_Upload
     method_option :bucket,            :required => true
     method_option :file,              :required => true
     method_option :threads,           :required => true, :type => :numeric, :default => 5
-    method_option :chunk_size,        :required => true, :type => :hash,    :default => {10 => :mb}
+    method_option :chunk_size,        :required => true, :type => :hash,    :default => {10 => 'mb'}
     method_option :progress_bar,      :type => :boolean, :default => true
+    method_option :checksum,          :type => :boolean, :default => true
     method_option :key,               :desc => 's3 object key (default: file name)'
 
     def upload

data/lib/s3_multi_upload.rb

@@ -1,10 +1,11 @@
 require 'pathname'
 require 'aws-sdk'
 require 'progressbar'
+require 'base64'
 
 module S3_Multi_Upload
   class Upload
-    attr_accessor :options, :file, :s3, :bucket, :object, :progress
+    attr_accessor :options, :file, :queue, :mutex, :s3, :bucket, :object, :progress
 
     def initialize options
 
@@ -13,6 +14,8 @@ module S3_Multi_Upload
 
       @options = options
       @file    = Pathname.new options[:file]
+      @queue   = Queue.new
+      @mutex   = Mutex.new
 
       @s3      = AWS::S3.new
       @bucket  = @s3.buckets.create options[:bucket]
@@ -38,10 +41,6 @@ module S3_Multi_Upload
       normalize *options[:chunk_size].first
     end
 
-    def queue
-      @queue ||= Queue.new
-    end
-
     def enqueue
       (file.size.to_f / chunk_size).ceil.times do |index|
         queue << [chunk_size * index, index + 1]
@@ -50,24 +49,35 @@ module S3_Multi_Upload
 
     def upload
       object.multipart_upload do |upload|
-        threads = []
-
-        options[:threads].times do
-          threads << Thread.new do
+        options[:threads].times.collect do
+          Thread.new do
             until queue.empty?
               offset, index = queue.deq :asynchronously rescue nil
 
               unless offset.nil?
-                upload.add_part :data        => file.read(chunk_size, offset),
-                                :part_number => index
+                upload_parameters = {
+                  :data        => file.read(chunk_size, offset),
+                  :part_number => index,
+                }
+
+                if options[:checksum]
+                  digest         = Digest::MD5.digest(upload_parameters[:data])
+                  encoded_digest = Base64.encode64(digest).strip
 
-                progress.inc if options[:progress_bar]
+                  upload_parameters[:content_md5] = encoded_digest
+                end
+
+                upload.add_part upload_parameters
+
+                if options[:progress_bar]
+                  mutex.synchronize do
+                    progress.inc
+                  end
+                end
               end
             end
           end
-        end
-
-        threads.each &:join
+        end.each(&:join)
       end
     end
 

metadata

@@ -1,15 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: s3_multi_upload
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.4
   prerelease: 
 platform: ruby
 authors:
 - dallas marlow
+- michael pilat
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-08-03 00:00:00.000000000 Z
+date: 2013-01-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk
@@ -62,6 +63,7 @@ dependencies:
 description: 
 email:
 - dallasmarlow@gmail.com
+- mike@mikepilat.com
 executables:
 - s3_multi_upload
 extensions: []
@@ -89,7 +91,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.24
+rubygems_version: 1.8.23
 signing_key: 
 specification_version: 3
 summary: s3 multipart uploads in parallel