s3_multi_upload 0.0.1

data/bin/s3_multi_upload

@@ -0,0 +1,26 @@
+#!/usr/bin/env ruby
+$:.unshift File.join File.dirname(__FILE__), *%w[.. lib]
+%w[thor s3_multi_upload].each {|l| require l}
+
+module S3_Multi_Upload
+  class Task < Thor
+
+    desc 'upload', 'upload a file to s3 in parallel'
+    method_option :access_key_id,     :required => true
+    method_option :secret_access_key, :required => true
+    method_option :bucket,            :required => true
+    method_option :file,              :required => true
+    method_option :threads,           :required => true, :type => :numeric, :default => 5
+    method_option :chunk_size,        :required => true, :type => :hash,    :default => {10 => :mb}
+    method_option :progress_bar,      :type => :boolean, :default => true
+    method_option :key,               :desc => 's3 object key (default: file name)'
+
+    def upload
+      Upload.new(options).process
+    end
+
+    default_task :upload
+  end
+end
+
+S3_Multi_Upload::Task.start

data/lib/s3_multi_upload.rb

@@ -0,0 +1,87 @@
+require 'pathname'
+require 'aws-sdk'
+require 'progressbar'
+
+module S3_Multi_Upload
+  class Upload
+    attr_accessor :options, :file, :s3, :bucket, :object, :progress
+
+    def initialize options
+
+      AWS.config :access_key_id     => options[:access_key_id],
+                 :secret_access_key => options[:secret_access_key]
+
+      @options = options
+      @file    = Pathname.new options[:file]
+
+      @s3      = AWS::S3.new
+      @bucket  = @s3.buckets.create options[:bucket]
+      @object  = @bucket.objects[options[:key] || @file.basename]
+
+      enqueue
+    end
+
+    def normalize value, unit = nil
+      case unit.to_sym.downcase
+      when nil, :b, :byte, :bytes
+        value.to_f
+      when :k, :kb, :kilobyte, :kilobytes
+        value.to_f * 1024
+      when :m, :mb, :megabyte, :megabytes
+        value.to_f * 1024 ** 2
+      when :g, :gb, :gigabyte, :gigabytes
+        value.to_f * 1024 ** 3
+      end
+    end 
+
+    def chunk_size
+      normalize *options[:chunk_size].first
+    end
+
+    def queue
+      @queue ||= Queue.new
+    end
+
+    def enqueue
+      (file.size.to_f / chunk_size).ceil.times do |index|
+        queue << [chunk_size * index, index + 1]
+      end
+    end
+
+    def upload
+      object.multipart_upload do |upload|
+        threads = []
+
+        options[:threads].times do
+          threads << Thread.new do
+            until queue.empty?
+              offset, index = queue.deq :asynchronously rescue nil
+              
+              unless offset.nil?
+                upload.add_part :data        => file.read(chunk_size, offset),
+                                :part_number => index
+
+                progress.inc if options[:progress_bar]
+              end
+            end
+          end
+        end
+
+        threads.each &:join
+      end
+    end
+
+    def progress
+      @progress ||= ProgressBar.new :upload, queue.size
+    end
+
+    def process
+      value, unit = *options[:chunk_size].first
+      puts "uploading #{file} to s3://#{options[:bucket]}/#{object.key} using #{options[:threads]} threads in chunks of #{value} #{unit}"
+      progress if options[:progress_bar]
+      abort 'upload failed' unless upload
+      progress.finish if options[:progress_bar]
+    end
+
+  end
+end

metadata

@@ -0,0 +1,82 @@
+--- !ruby/object:Gem::Specification
+name: s3_multi_upload
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- dallas marlow
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-04-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: &70295147055820 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70295147055820
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: &70295147055280 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70295147055280
+- !ruby/object:Gem::Dependency
+  name: progressbar
+  requirement: &70295147054780 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70295147054780
+description: 
+email:
+- dallasmarlow@gmail.com
+executables:
+- s3_multi_upload
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/s3_multi_upload
+- lib/s3_multi_upload.rb
+homepage: https://github.com/dallasmarlow/s3_multi_upload
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.10
+signing_key: 
+specification_version: 3
+summary: s3 multipart uploads in parallel
+test_files: []
+has_rdoc: