s3_uploader 0.0.5 → 0.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +15 -0
- data/README.md +2 -3
- data/lib/s3_uploader/s3_uploader.rb +34 -20
- data/lib/s3_uploader/version.rb +1 -1
- data/spec/s3uploader_spec.rb +50 -31
- metadata +24 -16
checksums.yaml
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
---
|
2
|
+
!binary "U0hBMQ==":
|
3
|
+
metadata.gz: !binary |-
|
4
|
+
ZDRkNjdiMDIyYmU3ZWE0NDc3OTNlYjU3MTU3NzM5NjhiYTFkYTcxOA==
|
5
|
+
data.tar.gz: !binary |-
|
6
|
+
M2YyODEyNjliNDc4YTkwNDcwYmIzZTdhN2QyYTFkYWU0MDM2MGE1Mw==
|
7
|
+
SHA512:
|
8
|
+
metadata.gz: !binary |-
|
9
|
+
MGQyMDhmZWEyNzMwYzQxMzhkYmYwYWM0NWZhYzY1ZTQ1MmQ0ZTg2M2VlMDFh
|
10
|
+
NjVlYzg5NDc2ZDFiYjk1NGFjYzY5NjcxYTE4YmFjODc0MmRhYjcwYzQ1NGJh
|
11
|
+
MWQwZmE4YjMyNDU2MDA0MmI2ZWM0YjI4ZGY5OGVjZjM5Y2ZjNmE=
|
12
|
+
data.tar.gz: !binary |-
|
13
|
+
ZTU0ZGRhNTU4NDg1MDAwOWZlNDFiZDU1YWQ3NmYxMDI4MDlmYmExN2IxZDNl
|
14
|
+
YmU5NjNhOThmYmZiNzVmYzdjYWYxY2VhNjQ0Mzk4N2RjNjE0M2E2MTlkNGEz
|
15
|
+
N2JlMGIxZGYxMjgxOTU3MDEwOWM3OGMxMmNkNzhiM2JkOGY5NmU=
|
data/README.md
CHANGED
@@ -48,9 +48,8 @@ Again, it uses S3_KEY and S3_SECRET environment variables if non provided in par
|
|
48
48
|
|
49
49
|
## TODO
|
50
50
|
|
51
|
-
1.
|
52
|
-
2.
|
53
|
-
3. Add optional time, size and number of files uploaded report at end of process
|
51
|
+
1. Allow regex pattern matching to select files to upload
|
52
|
+
2. Add optional time, size and number of files uploaded report at end of process
|
54
53
|
|
55
54
|
## Contributing
|
56
55
|
|
@@ -1,4 +1,5 @@
|
|
1
1
|
module S3Uploader
|
2
|
+
KILO_SIZE = 1024.0
|
2
3
|
def self.upload_directory(source, bucket, options = {})
|
3
4
|
options = {
|
4
5
|
:destination_dir => '',
|
@@ -13,19 +14,11 @@ module S3Uploader
|
|
13
14
|
|
14
15
|
raise 'Source must be a directory' unless File.directory?(source)
|
15
16
|
|
16
|
-
source = source.chop if source.end_with?('/')
|
17
|
-
if options[:destination_dir] != '' and !options[:destination_dir].end_with?('/')
|
18
|
-
options[:destination_dir] = "#{options[:destination_dir]}/"
|
19
|
-
end
|
20
|
-
|
21
|
-
files = Queue.new
|
22
|
-
Dir.glob("#{source}/**/*").select{ |f| !File.directory?(f) }.each do |f|
|
23
|
-
files << f
|
24
|
-
end
|
25
|
-
|
26
17
|
if options[:connection]
|
27
18
|
connection = options[:connection]
|
28
19
|
else
|
20
|
+
raise "Missing access keys" if options[:s3_key].nil? or options[:s3_secret].nil?
|
21
|
+
|
29
22
|
connection = Fog::Storage.new({
|
30
23
|
:provider => 'AWS',
|
31
24
|
:aws_access_key_id => options[:s3_key],
|
@@ -34,8 +27,21 @@ module S3Uploader
|
|
34
27
|
})
|
35
28
|
end
|
36
29
|
|
30
|
+
source = source.chop if source.end_with?('/')
|
31
|
+
if options[:destination_dir] != '' and !options[:destination_dir].end_with?('/')
|
32
|
+
options[:destination_dir] = "#{options[:destination_dir]}/"
|
33
|
+
end
|
34
|
+
total_size = 0
|
35
|
+
files = Queue.new
|
36
|
+
Dir.glob("#{source}/**/*").select{ |f| !File.directory?(f) }.each do |f|
|
37
|
+
files << f
|
38
|
+
total_size += File.size(f)
|
39
|
+
|
40
|
+
end
|
41
|
+
|
37
42
|
directory = connection.directories.new(:key => bucket)
|
38
43
|
|
44
|
+
start = Time.now
|
39
45
|
total_files = files.size
|
40
46
|
file_number = 0
|
41
47
|
@mutex = Mutex.new
|
@@ -44,24 +50,32 @@ module S3Uploader
|
|
44
50
|
options[:threads].times do |i|
|
45
51
|
threads[i] = Thread.new {
|
46
52
|
|
47
|
-
|
53
|
+
until files.empty?
|
48
54
|
@mutex.synchronize do
|
49
55
|
file_number += 1
|
56
|
+
Thread.current["file_number"] = file_number
|
50
57
|
end
|
51
|
-
file = files.pop
|
52
|
-
|
53
|
-
|
54
|
-
|
58
|
+
file = files.pop rescue nil
|
59
|
+
if file
|
60
|
+
key = file.gsub(source, '')[1..-1]
|
61
|
+
dest = "#{options[:destination_dir]}#{key}"
|
62
|
+
log.info("[#{Thread.current["file_number"]}/#{total_files}] Uploading #{key} to s3://#{bucket}/#{dest}")
|
55
63
|
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
64
|
+
directory.files.create(
|
65
|
+
:key => dest,
|
66
|
+
:body => File.open(file),
|
67
|
+
:public => options[:public]
|
68
|
+
)
|
69
|
+
end
|
61
70
|
end
|
62
71
|
}
|
63
72
|
end
|
64
73
|
threads.each { |t| t.join }
|
65
74
|
|
75
|
+
finish = Time.now
|
76
|
+
elapsed = finish.to_f - start.to_f
|
77
|
+
mins, secs = elapsed.divmod 60.0
|
78
|
+
log.info("Uploaded %d (%.#{0}f KB) in %d:%04.2f" % [total_files, total_size / KILO_SIZE, mins.to_i, secs])
|
79
|
+
|
66
80
|
end
|
67
81
|
end
|
data/lib/s3_uploader/version.rb
CHANGED
data/spec/s3uploader_spec.rb
CHANGED
@@ -1,37 +1,56 @@
|
|
1
1
|
require 'spec_helper'
|
2
2
|
|
3
|
-
describe S3Uploader do
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
:provider => 'AWS',
|
9
|
-
:aws_access_key_id => '11111111111',
|
10
|
-
:aws_secret_access_key => 'XXXXXXXXXXXXXXXXXXXXXXXXXXX'
|
11
|
-
})
|
12
|
-
|
13
|
-
@connection.directories.create(
|
14
|
-
:key => 'mybucket',
|
15
|
-
:public => true
|
16
|
-
)
|
17
|
-
|
18
|
-
@tmp_directory = File.join(Dir.tmpdir, 'test_s3_uploader')
|
19
|
-
create_test_files(@tmp_directory, 10)
|
20
|
-
create_test_files(File.join(@tmp_directory, 'subdir1'), 5)
|
21
|
-
@logger = Logger.new(StringIO.new)
|
22
|
-
end
|
23
|
-
|
24
|
-
it "should upload all files in a directory" do
|
25
|
-
|
26
|
-
@logger.should_receive(:info).exactly(15).times.with(/Uploading/)
|
27
|
-
|
28
|
-
S3Uploader.upload_directory(@tmp_directory, 'mybucket',
|
3
|
+
describe S3Uploader do
|
4
|
+
|
5
|
+
it 'when called with missing access keys it should raise an exception' do
|
6
|
+
lambda {
|
7
|
+
S3Uploader.upload_directory('/tmp', 'mybucket',
|
29
8
|
{ :destination_dir => 'test1/',
|
30
|
-
:
|
31
|
-
:
|
9
|
+
:s3_key => nil,
|
10
|
+
:s3_secret => nil
|
32
11
|
})
|
33
|
-
|
34
|
-
|
12
|
+
}.should raise_error('Missing access keys')
|
13
|
+
end
|
14
|
+
|
15
|
+
it 'when called with source not directory it should raise an exception' do
|
16
|
+
lambda {
|
17
|
+
S3Uploader.upload_directory('/xzzaz1232', 'mybucket')
|
18
|
+
}.should raise_error('Source must be a directory')
|
19
|
+
end
|
20
|
+
|
21
|
+
|
22
|
+
before :all do
|
23
|
+
Fog.mock!
|
24
|
+
|
25
|
+
@connection = Fog::Storage.new({
|
26
|
+
:provider => 'AWS',
|
27
|
+
:aws_access_key_id => '11111111111',
|
28
|
+
:aws_secret_access_key => 'XXXXXXXXXXXXXXXXXXXXXXXXXXX'
|
29
|
+
})
|
30
|
+
|
31
|
+
@connection.directories.create(
|
32
|
+
:key => 'mybucket',
|
33
|
+
:public => true
|
34
|
+
)
|
35
|
+
|
36
|
+
@tmp_directory = File.join(Dir.tmpdir, 'test_s3_uploader')
|
37
|
+
create_test_files(@tmp_directory, 10)
|
38
|
+
create_test_files(File.join(@tmp_directory, 'subdir1'), 5)
|
39
|
+
@logger = Logger.new(STDOUT)
|
40
|
+
end
|
41
|
+
|
42
|
+
it "should upload all files in a directory" do
|
43
|
+
puts @tmp_directory
|
44
|
+
# @logger.should_receive(:info).exactly(15).times.with(/Uploading/)
|
45
|
+
# @logger.should_receive(:info).exactly(1).times.with(/Uploaded/)
|
46
|
+
|
47
|
+
S3Uploader.upload_directory(@tmp_directory, 'mybucket',
|
48
|
+
{ :destination_dir => 'test1/',
|
49
|
+
:logger => @logger,
|
50
|
+
:connection => @connection
|
51
|
+
})
|
52
|
+
|
53
|
+
end
|
35
54
|
|
36
55
|
end
|
37
56
|
|
@@ -39,6 +58,6 @@ def create_test_files(directory, number_of_files)
|
|
39
58
|
FileUtils.mkdir_p directory
|
40
59
|
|
41
60
|
number_of_files.times do |i|
|
42
|
-
Open3.popen3(
|
61
|
+
Open3.popen3("dd if=/dev/zero of=#{directory}/file#{i}.txt count=1024 bs=1024")
|
43
62
|
end
|
44
63
|
end
|
metadata
CHANGED
@@ -1,49 +1,57 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: s3_uploader
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
5
|
-
prerelease:
|
4
|
+
version: 0.0.6
|
6
5
|
platform: ruby
|
7
6
|
authors:
|
8
7
|
- Christian Hein
|
9
8
|
autorequire:
|
10
9
|
bindir: bin
|
11
10
|
cert_chain: []
|
12
|
-
date:
|
11
|
+
date: 2013-09-29 00:00:00.000000000 Z
|
13
12
|
dependencies:
|
14
13
|
- !ruby/object:Gem::Dependency
|
15
14
|
name: fog
|
16
|
-
requirement:
|
17
|
-
none: false
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
18
16
|
requirements:
|
19
17
|
- - ! '>='
|
20
18
|
- !ruby/object:Gem::Version
|
21
19
|
version: '0'
|
22
20
|
type: :runtime
|
23
21
|
prerelease: false
|
24
|
-
version_requirements:
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ! '>='
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
25
27
|
- !ruby/object:Gem::Dependency
|
26
28
|
name: rspec
|
27
|
-
requirement:
|
28
|
-
none: false
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
29
30
|
requirements:
|
30
31
|
- - ! '>='
|
31
32
|
- !ruby/object:Gem::Version
|
32
33
|
version: '0'
|
33
34
|
type: :development
|
34
35
|
prerelease: false
|
35
|
-
version_requirements:
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ! '>='
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
36
41
|
- !ruby/object:Gem::Dependency
|
37
42
|
name: rake
|
38
|
-
requirement:
|
39
|
-
none: false
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
40
44
|
requirements:
|
41
45
|
- - ! '>='
|
42
46
|
- !ruby/object:Gem::Version
|
43
47
|
version: '0'
|
44
48
|
type: :development
|
45
49
|
prerelease: false
|
46
|
-
version_requirements:
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ! '>='
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
47
55
|
description: S3 multithreaded directory uploader
|
48
56
|
email:
|
49
57
|
- chrishein@gmail.com
|
@@ -66,28 +74,28 @@ files:
|
|
66
74
|
- spec/spec_helper.rb
|
67
75
|
homepage: ''
|
68
76
|
licenses: []
|
77
|
+
metadata: {}
|
69
78
|
post_install_message:
|
70
79
|
rdoc_options: []
|
71
80
|
require_paths:
|
72
81
|
- lib
|
73
82
|
required_ruby_version: !ruby/object:Gem::Requirement
|
74
|
-
none: false
|
75
83
|
requirements:
|
76
84
|
- - ! '>='
|
77
85
|
- !ruby/object:Gem::Version
|
78
86
|
version: '0'
|
79
87
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
80
|
-
none: false
|
81
88
|
requirements:
|
82
89
|
- - ! '>='
|
83
90
|
- !ruby/object:Gem::Version
|
84
91
|
version: '0'
|
85
92
|
requirements: []
|
86
93
|
rubyforge_project:
|
87
|
-
rubygems_version: 1.
|
94
|
+
rubygems_version: 2.1.2
|
88
95
|
signing_key:
|
89
|
-
specification_version:
|
96
|
+
specification_version: 4
|
90
97
|
summary: S3 multithreaded directory uploader
|
91
98
|
test_files:
|
92
99
|
- spec/s3uploader_spec.rb
|
93
100
|
- spec/spec_helper.rb
|
101
|
+
has_rdoc:
|