s3_multi_upload 0.0.2 → 0.0.4
Sign up to get free protection for your applications and to get access to all the features.
- data/bin/s3_multi_upload +2 -1
- data/lib/s3_multi_upload.rb +25 -15
- metadata +5 -3
data/bin/s3_multi_upload
CHANGED
@@ -11,8 +11,9 @@ module S3_Multi_Upload
|
|
11
11
|
method_option :bucket, :required => true
|
12
12
|
method_option :file, :required => true
|
13
13
|
method_option :threads, :required => true, :type => :numeric, :default => 5
|
14
|
-
method_option :chunk_size, :required => true, :type => :hash, :default => {10 =>
|
14
|
+
method_option :chunk_size, :required => true, :type => :hash, :default => {10 => 'mb'}
|
15
15
|
method_option :progress_bar, :type => :boolean, :default => true
|
16
|
+
method_option :checksum, :type => :boolean, :default => true
|
16
17
|
method_option :key, :desc => 's3 object key (default: file name)'
|
17
18
|
|
18
19
|
def upload
|
data/lib/s3_multi_upload.rb
CHANGED
@@ -1,10 +1,11 @@
|
|
1
1
|
require 'pathname'
|
2
2
|
require 'aws-sdk'
|
3
3
|
require 'progressbar'
|
4
|
+
require 'base64'
|
4
5
|
|
5
6
|
module S3_Multi_Upload
|
6
7
|
class Upload
|
7
|
-
attr_accessor :options, :file, :s3, :bucket, :object, :progress
|
8
|
+
attr_accessor :options, :file, :queue, :mutex, :s3, :bucket, :object, :progress
|
8
9
|
|
9
10
|
def initialize options
|
10
11
|
|
@@ -13,6 +14,8 @@ module S3_Multi_Upload
|
|
13
14
|
|
14
15
|
@options = options
|
15
16
|
@file = Pathname.new options[:file]
|
17
|
+
@queue = Queue.new
|
18
|
+
@mutex = Mutex.new
|
16
19
|
|
17
20
|
@s3 = AWS::S3.new
|
18
21
|
@bucket = @s3.buckets.create options[:bucket]
|
@@ -38,10 +41,6 @@ module S3_Multi_Upload
|
|
38
41
|
normalize *options[:chunk_size].first
|
39
42
|
end
|
40
43
|
|
41
|
-
def queue
|
42
|
-
@queue ||= Queue.new
|
43
|
-
end
|
44
|
-
|
45
44
|
def enqueue
|
46
45
|
(file.size.to_f / chunk_size).ceil.times do |index|
|
47
46
|
queue << [chunk_size * index, index + 1]
|
@@ -50,24 +49,35 @@ module S3_Multi_Upload
|
|
50
49
|
|
51
50
|
def upload
|
52
51
|
object.multipart_upload do |upload|
|
53
|
-
threads
|
54
|
-
|
55
|
-
options[:threads].times do
|
56
|
-
threads << Thread.new do
|
52
|
+
options[:threads].times.collect do
|
53
|
+
Thread.new do
|
57
54
|
until queue.empty?
|
58
55
|
offset, index = queue.deq :asynchronously rescue nil
|
59
56
|
|
60
57
|
unless offset.nil?
|
61
|
-
|
62
|
-
|
58
|
+
upload_parameters = {
|
59
|
+
:data => file.read(chunk_size, offset),
|
60
|
+
:part_number => index,
|
61
|
+
}
|
62
|
+
|
63
|
+
if options[:checksum]
|
64
|
+
digest = Digest::MD5.digest(upload_parameters[:data])
|
65
|
+
encoded_digest = Base64.encode64(digest).strip
|
63
66
|
|
64
|
-
|
67
|
+
upload_parameters[:content_md5] = encoded_digest
|
68
|
+
end
|
69
|
+
|
70
|
+
upload.add_part upload_parameters
|
71
|
+
|
72
|
+
if options[:progress_bar]
|
73
|
+
mutex.synchronize do
|
74
|
+
progress.inc
|
75
|
+
end
|
76
|
+
end
|
65
77
|
end
|
66
78
|
end
|
67
79
|
end
|
68
|
-
end
|
69
|
-
|
70
|
-
threads.each &:join
|
80
|
+
end.each(&:join)
|
71
81
|
end
|
72
82
|
end
|
73
83
|
|
metadata
CHANGED
@@ -1,15 +1,16 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: s3_multi_upload
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.4
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
8
8
|
- dallas marlow
|
9
|
+
- michael pilat
|
9
10
|
autorequire:
|
10
11
|
bindir: bin
|
11
12
|
cert_chain: []
|
12
|
-
date:
|
13
|
+
date: 2013-01-20 00:00:00.000000000 Z
|
13
14
|
dependencies:
|
14
15
|
- !ruby/object:Gem::Dependency
|
15
16
|
name: aws-sdk
|
@@ -62,6 +63,7 @@ dependencies:
|
|
62
63
|
description:
|
63
64
|
email:
|
64
65
|
- dallasmarlow@gmail.com
|
66
|
+
- mike@mikepilat.com
|
65
67
|
executables:
|
66
68
|
- s3_multi_upload
|
67
69
|
extensions: []
|
@@ -89,7 +91,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
89
91
|
version: '0'
|
90
92
|
requirements: []
|
91
93
|
rubyforge_project:
|
92
|
-
rubygems_version: 1.8.
|
94
|
+
rubygems_version: 1.8.23
|
93
95
|
signing_key:
|
94
96
|
specification_version: 3
|
95
97
|
summary: s3 multipart uploads in parallel
|