s3_uploader 0.0.7 → 0.0.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +8 -8
- data/README.md +11 -5
- data/lib/s3_uploader/s3_uploader.rb +18 -16
- data/lib/s3_uploader/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
---
|
2
2
|
!binary "U0hBMQ==":
|
3
3
|
metadata.gz: !binary |-
|
4
|
-
|
4
|
+
MDFkOWViMjI4OWI3YjMyOTIwMzc2Y2E4ZTNiNzgyNjJiMTc2NWE5NA==
|
5
5
|
data.tar.gz: !binary |-
|
6
|
-
|
6
|
+
ZjAwZjNiOTZhYmE3ZDllYTI1Mjk5OGY2ODg3MjQzNjlhOGJhY2M0OQ==
|
7
7
|
SHA512:
|
8
8
|
metadata.gz: !binary |-
|
9
|
-
|
10
|
-
|
11
|
-
|
9
|
+
NGVkNmRjNWNjYzc3NzY2OGY5YjQwYWYwZThjYmQ4ZjMzOGNmZmU5OGI2ODcx
|
10
|
+
MmJmN2NkMTIwNWY2NTU3M2VhN2JmNWYzOTVhNDQ1MTUxZjI5NTE3MGU3ZmE1
|
11
|
+
NTY0MjA5NzI1MjRjZDJkM2NjN2Q2MGMzMTcwODE5NDU3YWIzYTU=
|
12
12
|
data.tar.gz: !binary |-
|
13
|
-
|
14
|
-
|
15
|
-
|
13
|
+
NTE1MjQyYjFhZGRlNDcwYjRlNDYyZDBiYWU0MGUwYjUzNGQ4M2M4NWFhZTdl
|
14
|
+
N2EwODBkNTliZTllZGZjZTk5YjNiODlkZjA0N2E3N2VmYTkxODI2MGNhZTE1
|
15
|
+
ODVlNDMwYzBiZDQ0OGQ1ZTI5ODE3Y2EyZjhjNDY0NmE2MjI2MDc=
|
data/README.md
CHANGED
@@ -32,19 +32,24 @@ Or install it yourself as:
|
|
32
32
|
:s3_secret => YOUR_SECRET_KEY,
|
33
33
|
:destination_dir => 'test/',
|
34
34
|
:region => 'eu-west-1',
|
35
|
-
:threads => 4
|
35
|
+
:threads => 4,
|
36
|
+
:metadata => { 'Cache-Control' => 'max-age=315576000' }
|
37
|
+
})
|
36
38
|
|
37
39
|
If no keys are provided, it uses S3_KEY and S3_SECRET environment variables. us-east-1 is the default region.
|
38
40
|
|
39
41
|
S3Uploader.upload_directory('/tmp/test', 'mybucket', { :destination_dir => 'test/', :threads => 4 })
|
40
|
-
|
42
|
+
|
41
43
|
Or as a command line binary
|
42
|
-
|
44
|
+
|
43
45
|
s3uploader -r eu-west-1 -k YOUR_KEY -s YOUR_SECRET_KEY -d test/ -t 4 /tmp/test mybucket
|
44
|
-
|
46
|
+
|
45
47
|
Again, it uses S3_KEY and S3_SECRET environment variables if non provided in parameters.
|
46
|
-
|
48
|
+
|
47
49
|
s3uploader -d test/ -t 4 /tmp/test mybucket
|
50
|
+
|
51
|
+
|
52
|
+
Metadata headers are documented [here](http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPUT.html)
|
48
53
|
|
49
54
|
## TODO
|
50
55
|
|
@@ -62,6 +67,7 @@ Again, it uses S3_KEY and S3_SECRET environment variables if non provided in par
|
|
62
67
|
## Contributors
|
63
68
|
|
64
69
|
* [Mark Wagner](https://github.com/theSociableme)
|
70
|
+
* [Brandon Hilkert](https://github.com/brandonhilkert)
|
65
71
|
|
66
72
|
## License
|
67
73
|
|
@@ -7,18 +7,19 @@ module S3Uploader
|
|
7
7
|
:s3_key => ENV['S3_KEY'],
|
8
8
|
:s3_secret => ENV['S3_SECRET'],
|
9
9
|
:public => false,
|
10
|
-
:region => 'us-east-1'
|
10
|
+
:region => 'us-east-1',
|
11
|
+
:metadata => {}
|
11
12
|
}.merge(options)
|
12
|
-
|
13
|
+
|
13
14
|
log = options[:logger] || Logger.new(STDOUT)
|
14
|
-
|
15
|
+
|
15
16
|
raise 'Source must be a directory' unless File.directory?(source)
|
16
|
-
|
17
|
+
|
17
18
|
if options[:connection]
|
18
19
|
connection = options[:connection]
|
19
20
|
else
|
20
21
|
raise "Missing access keys" if options[:s3_key].nil? or options[:s3_secret].nil?
|
21
|
-
|
22
|
+
|
22
23
|
connection = Fog::Storage.new({
|
23
24
|
:provider => 'AWS',
|
24
25
|
:aws_access_key_id => options[:s3_key],
|
@@ -26,7 +27,7 @@ module S3Uploader
|
|
26
27
|
:region => options[:region]
|
27
28
|
})
|
28
29
|
end
|
29
|
-
|
30
|
+
|
30
31
|
source = source.chop if source.end_with?('/')
|
31
32
|
if options[:destination_dir] != '' and !options[:destination_dir].end_with?('/')
|
32
33
|
options[:destination_dir] = "#{options[:destination_dir]}/"
|
@@ -36,20 +37,20 @@ module S3Uploader
|
|
36
37
|
Dir.glob("#{source}/**/*").select{ |f| !File.directory?(f) }.each do |f|
|
37
38
|
files << f
|
38
39
|
total_size += File.size(f)
|
39
|
-
|
40
|
+
|
40
41
|
end
|
41
|
-
|
42
|
+
|
42
43
|
directory = connection.directories.new(:key => bucket)
|
43
|
-
|
44
|
+
|
44
45
|
start = Time.now
|
45
46
|
total_files = files.size
|
46
47
|
file_number = 0
|
47
48
|
@mutex = Mutex.new
|
48
|
-
|
49
|
+
|
49
50
|
threads = []
|
50
51
|
options[:threads].times do |i|
|
51
52
|
threads[i] = Thread.new {
|
52
|
-
|
53
|
+
|
53
54
|
until files.empty?
|
54
55
|
@mutex.synchronize do
|
55
56
|
file_number += 1
|
@@ -60,22 +61,23 @@ module S3Uploader
|
|
60
61
|
key = file.gsub(source, '')[1..-1]
|
61
62
|
dest = "#{options[:destination_dir]}#{key}"
|
62
63
|
log.info("[#{Thread.current["file_number"]}/#{total_files}] Uploading #{key} to s3://#{bucket}/#{dest}")
|
63
|
-
|
64
|
+
|
64
65
|
directory.files.create(
|
65
66
|
:key => dest,
|
66
67
|
:body => File.open(file),
|
67
|
-
:public => options[:public]
|
68
|
+
:public => options[:public],
|
69
|
+
:metadata => options[:metadata]
|
68
70
|
)
|
69
71
|
end
|
70
|
-
end
|
72
|
+
end
|
71
73
|
}
|
72
74
|
end
|
73
75
|
threads.each { |t| t.join }
|
74
|
-
|
76
|
+
|
75
77
|
finish = Time.now
|
76
78
|
elapsed = finish.to_f - start.to_f
|
77
79
|
mins, secs = elapsed.divmod 60.0
|
78
80
|
log.info("Uploaded %d (%.#{0}f KB) in %d:%04.2f" % [total_files, total_size / KILO_SIZE, mins.to_i, secs])
|
79
|
-
|
81
|
+
|
80
82
|
end
|
81
83
|
end
|
data/lib/s3_uploader/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: s3_uploader
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.8
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Christian Hein
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2013-
|
11
|
+
date: 2013-11-06 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fog
|