s3cp 1.0.2 → 1.0.3
Sign up to get free protection for your applications and to get access to all the features.
- data/History.txt +18 -0
- data/lib/s3cp/s3cp.rb +29 -9
- data/lib/s3cp/s3du.rb +33 -29
- data/lib/s3cp/s3ls.rb +35 -31
- data/lib/s3cp/version.rb +1 -1
- metadata +4 -4
data/History.txt
CHANGED
@@ -1,3 +1,21 @@
|
|
1
|
+
=== 1.0.3 / (2012-05-11)
|
2
|
+
|
3
|
+
* Added: Exponential backoff for `s3cp` command based on the formula:
|
4
|
+
|
5
|
+
delay = initial_retry_delay * (factor ^ retries)
|
6
|
+
|
7
|
+
where:
|
8
|
+
- `initial_retry_delay` is 1 second by default.
|
9
|
+
- `factor` is 1.4142 by default
|
10
|
+
(retry delay doubles every two retries)
|
11
|
+
- max. number of retries is now 20 by default.
|
12
|
+
|
13
|
+
(which means s3cp retries for roughly 58 minutes by default)
|
14
|
+
|
15
|
+
* Fixed: `s3ls` and `s3du` now properly handle Errno::EPIPE
|
16
|
+
|
17
|
+
* Fixed: `s3du` would not display last character of files
|
18
|
+
|
1
19
|
=== 1.0.2 / 2012-03-9
|
2
20
|
|
3
21
|
* Added: `s3up` now outputs uploaded file size to STDERR, e.g.,
|
data/lib/s3cp/s3cp.rb
CHANGED
@@ -31,10 +31,12 @@ require 's3cp/utils'
|
|
31
31
|
options = {}
|
32
32
|
options[:verbose] = $stdout.isatty ? true : false
|
33
33
|
options[:headers] = []
|
34
|
-
options[:overwrite] = ENV["
|
34
|
+
options[:overwrite] = ENV["S3CP_OVERWRITE"] ? (ENV["S3CP_OVERWRITE"] =~ /y|yes|true|1|^\s*$/i ? true : false) : true
|
35
35
|
options[:checksum] = ENV["S3CP_CHECKSUM"] ? (ENV["S3CP_CHECKSUM"] =~ /y|yes|true|1|^\s*$/i ? true : false) : true
|
36
|
-
options[:retries] = ENV["S3CP_RETRIES"] ? ENV["S3CP_RETRIES"].to_i :
|
36
|
+
options[:retries] = ENV["S3CP_RETRIES"] ? ENV["S3CP_RETRIES"].to_i : 18
|
37
37
|
options[:retry_delay] = ENV["S3CP_RETRY_DELAY"] ? ENV["S3CP_RETRY_DELAY"].to_i : 1
|
38
|
+
options[:retry_backoff] = ENV["S3CP_BACKOFF"] ? ENV["S3CP_BACKOFF"].to_f : 1.4142 # double every 2 tries
|
39
|
+
|
38
40
|
options[:include_regex] = []
|
39
41
|
options[:exclude_regex] = []
|
40
42
|
options[:sync] = false
|
@@ -82,6 +84,10 @@ op = OptionParser.new do |opts|
|
|
82
84
|
options[:retry_delay] = delay.to_i
|
83
85
|
end
|
84
86
|
|
87
|
+
opts.on("--retry-backoff FACTOR", "Exponential backoff factor (default #{options[:retry_backoff]})") do |factor|
|
88
|
+
options[:retry_backoff] = factor.to_f
|
89
|
+
end
|
90
|
+
|
85
91
|
opts.on("--no-checksum", "Disable checksum checking") do
|
86
92
|
options[:checksum] = false
|
87
93
|
end
|
@@ -266,8 +272,9 @@ def local_to_s3(bucket_to, key, file, options = {})
|
|
266
272
|
fail "Unable to upload to s3://#{bucket_to}/#{key} after #{retries} attempts."
|
267
273
|
end
|
268
274
|
if retries > 0
|
269
|
-
|
270
|
-
|
275
|
+
delay = options[:retry_delay] * (options[:retry_backoff] ** retries)
|
276
|
+
STDERR.puts "Sleeping #{delay} seconds. Will retry #{options[:retries] - retries} more time(s)."
|
277
|
+
sleep delay
|
271
278
|
end
|
272
279
|
|
273
280
|
f = File.open(file)
|
@@ -285,7 +292,7 @@ def local_to_s3(bucket_to, key, file, options = {})
|
|
285
292
|
@progress_bar.inc result.length if result
|
286
293
|
result
|
287
294
|
rescue => e
|
288
|
-
puts e
|
295
|
+
STDERR.puts e
|
289
296
|
raise e
|
290
297
|
end
|
291
298
|
end
|
@@ -300,8 +307,12 @@ def local_to_s3(bucket_to, key, file, options = {})
|
|
300
307
|
|
301
308
|
if options[:checksum]
|
302
309
|
actual_md5 = s3_checksum(bucket_to, key)
|
303
|
-
|
304
|
-
|
310
|
+
if actual_md5.is_a? String
|
311
|
+
if actual_md5 != expected_md5
|
312
|
+
STDERR.puts "Warning: invalid MD5 checksum. Expected: #{expected_md5} Actual: #{actual_md5}"
|
313
|
+
end
|
314
|
+
else
|
315
|
+
STDERR.puts "Warning: invalid MD5 checksum in metadata: #{actual_md5.inspect}; skipped checksum verification."
|
305
316
|
actual_md5 = nil
|
306
317
|
end
|
307
318
|
end
|
@@ -328,8 +339,9 @@ def s3_to_local(bucket_from, key_from, dest, options = {})
|
|
328
339
|
fail "Unable to download s3://#{bucket_from}/#{key_from} after #{retries} attempts."
|
329
340
|
end
|
330
341
|
if retries > 0
|
331
|
-
|
332
|
-
|
342
|
+
delay = options[:retry_delay] * (options[:retry_backoff] ** retries)
|
343
|
+
STDERR.puts "Sleeping #{delay} seconds. Will retry #{options[:retries] - retries} more time(s)."
|
344
|
+
sleep delay
|
333
345
|
end
|
334
346
|
begin
|
335
347
|
expected_md5 = if options[:checksum] || options[:sync]
|
@@ -377,6 +389,14 @@ def s3_to_local(bucket_from, key_from, dest, options = {})
|
|
377
389
|
raise e unless options[:checksum]
|
378
390
|
STDERR.puts e
|
379
391
|
end
|
392
|
+
|
393
|
+
if options[:checksum] && expected_md5 != nil
|
394
|
+
actual_md5 = md5(dest)
|
395
|
+
if actual_md5 != expected_md5
|
396
|
+
STDERR.puts "Warning: invalid MD5 checksum. Expected: #{expected_md5} Actual: #{actual_md5}"
|
397
|
+
end
|
398
|
+
end
|
399
|
+
|
380
400
|
retries += 1
|
381
401
|
end until options[:checksum] == false || expected_md5.nil? || md5(dest) == expected_md5
|
382
402
|
end
|
data/lib/s3cp/s3du.rb
CHANGED
@@ -102,39 +102,43 @@ def print(key, size)
|
|
102
102
|
puts ("%#{7 + @options[:precision]}s " % size) + key
|
103
103
|
end
|
104
104
|
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
105
|
+
begin
|
106
|
+
@s3.interface.incrementally_list_bucket(@bucket, s3_options) do |page|
|
107
|
+
|
108
|
+
entries = page[:contents]
|
109
|
+
entries.each do |entry|
|
110
|
+
key = entry[:key]
|
111
|
+
size = entry[:size]
|
112
|
+
|
113
|
+
if options[:regex].nil? || options[:regex].match(key)
|
114
|
+
current_key = if actual_depth
|
115
|
+
pos = nth_occurrence(key, "/", actual_depth)
|
116
|
+
(pos != -1) ? key[0..pos-1] : key
|
117
|
+
end
|
118
|
+
|
119
|
+
if (last_key && last_key != current_key)
|
120
|
+
print(last_key, subtotal_size)
|
121
|
+
subtotal_size = size
|
122
|
+
else
|
123
|
+
subtotal_size += size
|
124
|
+
end
|
125
|
+
|
126
|
+
last_key = current_key
|
127
|
+
total_size += size
|
116
128
|
end
|
117
|
-
|
118
|
-
if (last_key && last_key != current_key)
|
119
|
-
print(last_key, subtotal_size)
|
120
|
-
subtotal_size = size
|
121
|
-
else
|
122
|
-
subtotal_size += size
|
123
|
-
end
|
124
|
-
|
125
|
-
last_key = current_key
|
126
|
-
total_size += size
|
127
129
|
end
|
128
130
|
end
|
129
|
-
end
|
130
131
|
|
131
|
-
if last_key != nil
|
132
|
-
|
133
|
-
end
|
132
|
+
if last_key != nil
|
133
|
+
print(last_key, subtotal_size)
|
134
|
+
end
|
134
135
|
|
135
|
-
if options[:depth] > 0
|
136
|
-
|
137
|
-
else
|
138
|
-
|
136
|
+
if options[:depth] > 0
|
137
|
+
print("", total_size)
|
138
|
+
else
|
139
|
+
puts S3CP.format_filesize(total_size, :unit => options[:unit], :precision => options[:precision])
|
140
|
+
end
|
141
|
+
rescue Errno::EPIPE
|
142
|
+
# ignore
|
139
143
|
end
|
140
144
|
|
data/lib/s3cp/s3ls.rb
CHANGED
@@ -103,39 +103,43 @@ s3_options[:prefix] = @key
|
|
103
103
|
s3_options["max-keys"] = options[:max_keys] if options[:max_keys] && !options[:delimiter]
|
104
104
|
s3_options[:delimiter] = options[:delimiter] if options[:delimiter]
|
105
105
|
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
106
|
+
begin
|
107
|
+
@s3.interface.incrementally_list_bucket(@bucket, s3_options) do |page|
|
108
|
+
entries = []
|
109
|
+
if options[:delimiter]
|
110
|
+
entries << { :key => page[:contents][0][:key] } if page[:contents].length > 0 && entries.length > 0
|
111
|
+
page[:common_prefixes].each do |entry|
|
112
|
+
entries << { :key => entry }
|
113
|
+
end
|
114
|
+
entries << { :key => nil }
|
112
115
|
end
|
113
|
-
entries
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
response
|
136
|
-
end
|
137
|
-
exit if response == 'n'
|
116
|
+
entries += page[:contents]
|
117
|
+
entries.each do |entry|
|
118
|
+
key = entry[:key] ? "s3://#{@bucket}/#{entry[:key]}" : "---"
|
119
|
+
if options[:long_format] && entry[:last_modified] && entry[:size]
|
120
|
+
last_modified = DateTime.parse(entry[:last_modified])
|
121
|
+
size = entry[:size]
|
122
|
+
size = S3CP.format_filesize(size, :unit => options[:unit], :precision => options[:precision])
|
123
|
+
size = ("%#{7 + options[:precision]}s " % size)
|
124
|
+
puts "#{last_modified.strftime(options[:date_format])} #{size} #{key}"
|
125
|
+
else
|
126
|
+
puts key
|
127
|
+
end
|
128
|
+
rows += 1
|
129
|
+
keys += 1
|
130
|
+
if options[:max_keys] && keys >= options[:max_keys]
|
131
|
+
exit
|
132
|
+
end
|
133
|
+
if options[:rows_per_page] && (rows % options[:rows_per_page] == 0)
|
134
|
+
begin
|
135
|
+
print "Continue? (Y/n) "
|
136
|
+
response = STDIN.gets.chomp.downcase
|
137
|
+
end until response == 'n' || response == 'y' || response == ''
|
138
|
+
exit if response == 'n'
|
139
|
+
end
|
138
140
|
end
|
139
141
|
end
|
142
|
+
rescue Errno::EPIPE
|
143
|
+
# ignore
|
140
144
|
end
|
141
145
|
|
data/lib/s3cp/version.rb
CHANGED
metadata
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: s3cp
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
4
|
+
hash: 17
|
5
5
|
prerelease:
|
6
6
|
segments:
|
7
7
|
- 1
|
8
8
|
- 0
|
9
|
-
-
|
10
|
-
version: 1.0.
|
9
|
+
- 3
|
10
|
+
version: 1.0.3
|
11
11
|
platform: ruby
|
12
12
|
authors:
|
13
13
|
- Alex Boisvert
|
@@ -15,7 +15,7 @@ autorequire:
|
|
15
15
|
bindir: bin
|
16
16
|
cert_chain: []
|
17
17
|
|
18
|
-
date: 2012-
|
18
|
+
date: 2012-05-11 00:00:00 Z
|
19
19
|
dependencies:
|
20
20
|
- !ruby/object:Gem::Dependency
|
21
21
|
prerelease: false
|