s3cp 0.2.2 → 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/History.txt +7 -1
- data/lib/s3cp/#Untitled-7# +288 -0
- data/lib/s3cp/s3cp.rb +176 -88
- data/lib/s3cp/version.rb +1 -1
- metadata +5 -4
data/History.txt
CHANGED
@@ -0,0 +1,288 @@
|
|
1
|
+
diff --git a/History.txt b/History.txt
|
2
|
+
index 8de2725..37eaec8 100644
|
3
|
+
--- a/History.txt
|
4
|
+
+++ b/History.txt
|
5
|
+
@@ -1,5 +1,9 @@
|
6
|
+
=== 0.2.3 / (Pending)
|
7
|
+
|
8
|
+
+* Added: --include and --exclude REGEX support for s3cp
|
9
|
+
+
|
10
|
+
+* Added: --sync mode for s3cp
|
11
|
+
+
|
12
|
+
=== 0.2.2 / (2012-02-23)
|
13
|
+
|
14
|
+
* Added: Progress bars during upload/download if $stdout.isatty
|
15
|
+
diff --git a/lib/s3cp/s3cp.rb b/lib/s3cp/s3cp.rb
|
16
|
+
index 0fabcdd..a1423f5 100644
|
17
|
+
--- a/lib/s3cp/s3cp.rb
|
18
|
+
+++ b/lib/s3cp/s3cp.rb
|
19
|
+
@@ -35,6 +35,9 @@ options[:overwrite] = ENV["S3CP_RETRIES"] ? (ENV["S3CP_OVERWRITE"] =~ /y|y
|
20
|
+
options[:checksum] = ENV["S3CP_CHECKSUM"] ? (ENV["S3CP_CHECKSUM"] =~ /y|yes|true|1|^\s*$/i ? true : false) : true
|
21
|
+
options[:retries] = ENV["S3CP_RETRIES"] ? ENV["S3CP_RETRIES"].to_i : 5
|
22
|
+
options[:retry_delay] = ENV["S3CP_RETRY_DELAY"] ? ENV["S3CP_RETRY_DELAY"].to_i : 1
|
23
|
+
+options[:include_regex] = []
|
24
|
+
+options[:exclude_regex] = []
|
25
|
+
+options[:sync] = false
|
26
|
+
|
27
|
+
op = OptionParser.new do |opts|
|
28
|
+
opts.banner = <<-BANNER
|
29
|
+
@@ -67,6 +70,10 @@ op = OptionParser.new do |opts|
|
30
|
+
options[:overwrite] = false
|
31
|
+
end
|
32
|
+
|
33
|
+
+ opts.on("--sync", "Sync mode: use checksum to determine if files need copying.") do
|
34
|
+
+ options[:sync] = true
|
35
|
+
+ end
|
36
|
+
+
|
37
|
+
opts.on("--max-attempts N", "Number of attempts to upload/download until checksum matches (default #{options[:retries]})") do |attempts|
|
38
|
+
options[:retries] = attempts.to_i
|
39
|
+
end
|
40
|
+
@@ -90,6 +97,14 @@ op = OptionParser.new do |opts|
|
41
|
+
opts.separator " AMZ headers: \'x-amz-acl: public-read\'"
|
42
|
+
opts.separator ""
|
43
|
+
|
44
|
+
+ opts.on("-i REGEX", "--include REGEX", "Copy only files matching the following regular expression.") do |regex|
|
45
|
+
+ options[:include_regex] << regex
|
46
|
+
+ end
|
47
|
+
+
|
48
|
+
+ opts.on("-x REGEX", "--exclude REGEX", "Do not copy files matching provided regular expression.") do |regex|
|
49
|
+
+ options[:exclude_regex] << regex
|
50
|
+
+ end
|
51
|
+
+
|
52
|
+
opts.on("--verbose", "Verbose mode") do
|
53
|
+
options[:verbose] = true
|
54
|
+
end
|
55
|
+
@@ -110,6 +125,16 @@ if ARGV.size < 2
|
56
|
+
exit
|
57
|
+
end
|
58
|
+
|
59
|
+
+if options[:include_regex].any? && !options[:recursive]
|
60
|
+
+ puts "-i (--include regex) option requires -r (recursive) option."
|
61
|
+
+ exit(1)
|
62
|
+
+end
|
63
|
+
+
|
64
|
+
+if options[:exclude_regex].any? && !options[:recursive]
|
65
|
+
+ puts "-x (--exclude regex) option requires -r (recursive) option."
|
66
|
+
+ exit(1)
|
67
|
+
+end
|
68
|
+
+
|
69
|
+
destination = ARGV.last
|
70
|
+
sources = ARGV[0..-2]
|
71
|
+
|
72
|
+
@@ -144,6 +169,16 @@ end
|
73
|
+
@bucket = $1
|
74
|
+
@prefix = $2
|
75
|
+
|
76
|
+
+@includes = options[:include_regex].map { |s| Regexp.new(s) }
|
77
|
+
+@excludes = options[:exclude_regex].map { |s| Regexp.new(s) }
|
78
|
+
+
|
79
|
+
+def match(path)
|
80
|
+
+ matching = true
|
81
|
+
+ return false if @includes.any? && !@includes.any? { |regex| regex.match(path) }
|
82
|
+
+ return false if @excludes.any? && @excludes.any? { |regex| regex.match(path) }
|
83
|
+
+ true
|
84
|
+
+end
|
85
|
+
+
|
86
|
+
@s3 = S3CP.connect()
|
87
|
+
|
88
|
+
def direction(from, to)
|
89
|
+
@@ -217,68 +252,90 @@ end
|
90
|
+
|
91
|
+
def local_to_s3(bucket_to, key, file, options = {})
|
92
|
+
log(with_headers("Copy #{file} to s3://#{bucket_to}/#{key}"))
|
93
|
+
- if options[:checksum]
|
94
|
+
- expected_md5 = md5(file)
|
95
|
+
+
|
96
|
+
+ expected_md5 = if options[:checksum] || options[:sync]
|
97
|
+
+ md5(file)
|
98
|
+
end
|
99
|
+
- retries = 0
|
100
|
+
- begin
|
101
|
+
- if retries == options[:retries]
|
102
|
+
- fail "Unable to upload to s3://#{bucket_to}/#{key} after #{retries} attempts."
|
103
|
+
- end
|
104
|
+
- if retries > 0
|
105
|
+
- STDERR.puts "Warning: failed checksum for s3://#{bucket_to}/#{bucket_to}. Retrying #{options[:retries] - retries} more time(s)."
|
106
|
+
- sleep options[:retry_delay]
|
107
|
+
+
|
108
|
+
+ actual_md5 = if options[:sync]
|
109
|
+
+ md5 = s3_checksum(bucket_to, key)
|
110
|
+
+ case md5
|
111
|
+
+ when :not_found
|
112
|
+
+ nil
|
113
|
+
+ when :invalid
|
114
|
+
+ STDERR.puts "Warning: invalid MD5 checksum in metadata; file will be force-copied."
|
115
|
+
+ nil
|
116
|
+
+ else
|
117
|
+
+ md5
|
118
|
+
end
|
119
|
+
+ end
|
120
|
+
|
121
|
+
- f = File.open(file)
|
122
|
+
+ if actual_md5.nil? || (options[:sync] && expected_md5 != actual_md5)
|
123
|
+
+ retries = 0
|
124
|
+
begin
|
125
|
+
- if $stdout.isatty
|
126
|
+
- f = Proxy.new(f)
|
127
|
+
- progress_bar = ProgressBar.new(File.basename(file), File.size(file)).tap do |p|
|
128
|
+
- p.file_transfer_mode
|
129
|
+
- end
|
130
|
+
- class << f
|
131
|
+
- attr_accessor :progress_bar
|
132
|
+
- def read(length, buffer=nil)
|
133
|
+
- begin
|
134
|
+
- result = @target.read(length, buffer)
|
135
|
+
- @progress_bar.inc result.length if result
|
136
|
+
- result
|
137
|
+
- rescue => e
|
138
|
+
- puts e
|
139
|
+
- raise e
|
140
|
+
+ if retries == options[:retries]
|
141
|
+
+ fail "Unable to upload to s3://#{bucket_to}/#{key} after #{retries} attempts."
|
142
|
+
+ end
|
143
|
+
+ if retries > 0
|
144
|
+
+ STDERR.puts "Warning: failed checksum for s3://#{bucket_to}/#{bucket_to}. Retrying #{options[:retries] - retries} more time(s)."
|
145
|
+
+ sleep options[:retry_delay]
|
146
|
+
+ end
|
147
|
+
+
|
148
|
+
+ f = File.open(file)
|
149
|
+
+ begin
|
150
|
+
+ if $stdout.isatty
|
151
|
+
+ f = Proxy.new(f)
|
152
|
+
+ progress_bar = ProgressBar.new(File.basename(file), File.size(file)).tap do |p|
|
153
|
+
+ p.file_transfer_mode
|
154
|
+
+ end
|
155
|
+
+ class << f
|
156
|
+
+ attr_accessor :progress_bar
|
157
|
+
+ def read(length, buffer=nil)
|
158
|
+
+ begin
|
159
|
+
+ result = @target.read(length, buffer)
|
160
|
+
+ @progress_bar.inc result.length if result
|
161
|
+
+ result
|
162
|
+
+ rescue => e
|
163
|
+
+ puts e
|
164
|
+
+ raise e
|
165
|
+
+ end
|
166
|
+
end
|
167
|
+
end
|
168
|
+
+ f.progress_bar = progress_bar
|
169
|
+
+ else
|
170
|
+
+ progress_bar = nil
|
171
|
+
end
|
172
|
+
- f.progress_bar = progress_bar
|
173
|
+
- else
|
174
|
+
- progress_bar = nil
|
175
|
+
- end
|
176
|
+
|
177
|
+
- meta = @s3.interface.put(bucket_to, key, f, @headers)
|
178
|
+
- progress_bar.finish if progress_bar
|
179
|
+
+ meta = @s3.interface.put(bucket_to, key, f, @headers)
|
180
|
+
+ progress_bar.finish if progress_bar
|
181
|
+
|
182
|
+
- if options[:checksum]
|
183
|
+
- metadata = @s3.interface.head(bucket_to, key)
|
184
|
+
- actual_md5 = metadata["etag"] or fail "Unable to get etag/md5 for #{bucket_to}:#{key}"
|
185
|
+
- actual_md5 = actual_md5.sub(/^"/, "").sub(/"$/, "") # strip beginning and trailing quotes
|
186
|
+
- if actual_md5 =~ /-/
|
187
|
+
- STDERR.puts "Warning: invalid MD5 checksum in metadata; skipped checksum verification."
|
188
|
+
- actual_md5 = nil
|
189
|
+
+ if options[:checksum]
|
190
|
+
+ actual_md5 = s3_checksum(bucket_to, key)
|
191
|
+
+ unless actual_md5.is_a? String
|
192
|
+
+ STDERR.puts "Warning: invalid MD5 checksum in metadata; skipped checksum verification."
|
193
|
+
+ actual_md5 = nil
|
194
|
+
+ end
|
195
|
+
end
|
196
|
+
+ rescue => e
|
197
|
+
+ raise e unless options[:checksum]
|
198
|
+
+ STDERR.puts e
|
199
|
+
+ ensure
|
200
|
+
+ f.close()
|
201
|
+
end
|
202
|
+
- rescue => e
|
203
|
+
- raise e unless options[:checksum]
|
204
|
+
- STDERR.puts e
|
205
|
+
- ensure
|
206
|
+
- f.close()
|
207
|
+
- end
|
208
|
+
- retries += 1
|
209
|
+
+ retries += 1
|
210
|
+
end until options[:checksum] == false || actual_md5.nil? || expected_md5 == actual_md5
|
211
|
+
+ else
|
212
|
+
+ log "Already synchronized."
|
213
|
+
+ end
|
214
|
+
end
|
215
|
+
|
216
|
+
def s3_to_local(bucket_from, key_from, dest, options = {})
|
217
|
+
log("Copy s3://#{bucket_from}/#{key_from} to #{dest}")
|
218
|
+
+
|
219
|
+
+ actual_md5 = if options[:sync] && File.exist?(dest)
|
220
|
+
+ md5(dest)
|
221
|
+
+ end
|
222
|
+
+
|
223
|
+
retries = 0
|
224
|
+
begin
|
225
|
+
if retries == options[:retries]
|
226
|
+
@@ -292,34 +349,38 @@ def s3_to_local(bucket_from, key_from, dest, options = {})
|
227
|
+
|
228
|
+
f = File.new(dest, "wb")
|
229
|
+
begin
|
230
|
+
- size = nil
|
231
|
+
- if options[:checksum]
|
232
|
+
- metadata = @s3.interface.head(bucket_from, key_from)
|
233
|
+
- expected_md5 = metadata["etag"] or fail "Unable to get etag/md5 for #{bucket_from}:#{key_from}"
|
234
|
+
- expected_md5 = expected_md5.sub(/^"/, "").sub(/"$/, "") # strip beginning and trailing quotes
|
235
|
+
- if expected_md5 =~ /-/
|
236
|
+
+ expected_md5 = if options[:checksum] || options[:sync]
|
237
|
+
+ md5 = s3_checksum(bucket_from, key_from)
|
238
|
+
+ if options[:sync] && !md5.is_a?(String)
|
239
|
+
+ STDERR.puts "Warning: invalid MD5 checksum in metadata; file will be force-copied."
|
240
|
+
+ nil
|
241
|
+
+ elsif !md5.is_a? String
|
242
|
+
STDERR.puts "Warning: invalid MD5 checksum in metadata; skipped checksum verification."
|
243
|
+
- expected_md5 = nil
|
244
|
+
+ nil
|
245
|
+
+ else
|
246
|
+
+ md5
|
247
|
+
end
|
248
|
+
- size = metadata["content-length"].to_i
|
249
|
+
- elsif $stdout.isatty
|
250
|
+
- metadata = @s3.interface.head(bucket_from, key_from)
|
251
|
+
- size = metadata["content-length"].to_i
|
252
|
+
end
|
253
|
+
- begin
|
254
|
+
- progress_bar = if size
|
255
|
+
- ProgressBar.new(File.basename(key_from), size).tap do |p|
|
256
|
+
- p.file_transfer_mode
|
257
|
+
+
|
258
|
+
+ if (expected_md5 == nil) || (options[:sync] && expected_md5 != actual_md5)
|
259
|
+
+ begin
|
260
|
+
+ progress_bar = if $stdout.isatty
|
261
|
+
+ size = s3_size(bucket_from, key_from)
|
262
|
+
+ ProgressBar.new(File.basename(key_from), size).tap do |p|
|
263
|
+
+ p.file_transfer_mode
|
264
|
+
+ end
|
265
|
+
end
|
266
|
+
+ @s3.interface.get(bucket_from, key_from) do |chunk|
|
267
|
+
+ f.write(chunk)
|
268
|
+
+ progress_bar.inc chunk.size if progress_bar
|
269
|
+
+ end
|
270
|
+
+ progress_bar.finish
|
271
|
+
+ rescue => e
|
272
|
+
+ progress_bar.halt if progress_bar
|
273
|
+
+ raise e
|
274
|
+
end
|
275
|
+
- @s3.interface.get(bucket_from, key_from) do |chunk|
|
276
|
+
- f.write(chunk)
|
277
|
+
- progress_bar.inc chunk.size if progress_bar
|
278
|
+
- end
|
279
|
+
- progress_bar.finish
|
280
|
+
- rescue => e
|
281
|
+
- progress_bar.halt if progress_bar
|
282
|
+
- raise e
|
283
|
+
+ else
|
284
|
+
+ log("Already synchronized")
|
285
|
+
end
|
286
|
+
rescue => e
|
287
|
+
raise e unless options[:checksum]
|
288
|
+
|
data/lib/s3cp/s3cp.rb
CHANGED
@@ -35,6 +35,9 @@ options[:overwrite] = ENV["S3CP_RETRIES"] ? (ENV["S3CP_OVERWRITE"] =~ /y|y
|
|
35
35
|
options[:checksum] = ENV["S3CP_CHECKSUM"] ? (ENV["S3CP_CHECKSUM"] =~ /y|yes|true|1|^\s*$/i ? true : false) : true
|
36
36
|
options[:retries] = ENV["S3CP_RETRIES"] ? ENV["S3CP_RETRIES"].to_i : 5
|
37
37
|
options[:retry_delay] = ENV["S3CP_RETRY_DELAY"] ? ENV["S3CP_RETRY_DELAY"].to_i : 1
|
38
|
+
options[:include_regex] = []
|
39
|
+
options[:exclude_regex] = []
|
40
|
+
options[:sync] = false
|
38
41
|
|
39
42
|
op = OptionParser.new do |opts|
|
40
43
|
opts.banner = <<-BANNER
|
@@ -67,6 +70,10 @@ op = OptionParser.new do |opts|
|
|
67
70
|
options[:overwrite] = false
|
68
71
|
end
|
69
72
|
|
73
|
+
opts.on("--sync", "Sync mode: use checksum to determine if files need copying.") do
|
74
|
+
options[:sync] = true
|
75
|
+
end
|
76
|
+
|
70
77
|
opts.on("--max-attempts N", "Number of attempts to upload/download until checksum matches (default #{options[:retries]})") do |attempts|
|
71
78
|
options[:retries] = attempts.to_i
|
72
79
|
end
|
@@ -90,6 +97,14 @@ op = OptionParser.new do |opts|
|
|
90
97
|
opts.separator " AMZ headers: \'x-amz-acl: public-read\'"
|
91
98
|
opts.separator ""
|
92
99
|
|
100
|
+
opts.on("-i REGEX", "--include REGEX", "Copy only files matching the following regular expression.") do |regex|
|
101
|
+
options[:include_regex] << regex
|
102
|
+
end
|
103
|
+
|
104
|
+
opts.on("-x REGEX", "--exclude REGEX", "Do not copy files matching provided regular expression.") do |regex|
|
105
|
+
options[:exclude_regex] << regex
|
106
|
+
end
|
107
|
+
|
93
108
|
opts.on("--verbose", "Verbose mode") do
|
94
109
|
options[:verbose] = true
|
95
110
|
end
|
@@ -110,6 +125,16 @@ if ARGV.size < 2
|
|
110
125
|
exit
|
111
126
|
end
|
112
127
|
|
128
|
+
if options[:include_regex].any? && !options[:recursive]
|
129
|
+
puts "-i (--include regex) option requires -r (recursive) option."
|
130
|
+
exit(1)
|
131
|
+
end
|
132
|
+
|
133
|
+
if options[:exclude_regex].any? && !options[:recursive]
|
134
|
+
puts "-x (--exclude regex) option requires -r (recursive) option."
|
135
|
+
exit(1)
|
136
|
+
end
|
137
|
+
|
113
138
|
destination = ARGV.last
|
114
139
|
sources = ARGV[0..-2]
|
115
140
|
|
@@ -144,6 +169,16 @@ end
|
|
144
169
|
@bucket = $1
|
145
170
|
@prefix = $2
|
146
171
|
|
172
|
+
@includes = options[:include_regex].map { |s| Regexp.new(s) }
|
173
|
+
@excludes = options[:exclude_regex].map { |s| Regexp.new(s) }
|
174
|
+
|
175
|
+
def match(path)
|
176
|
+
matching = true
|
177
|
+
return false if @includes.any? && !@includes.any? { |regex| regex.match(path) }
|
178
|
+
return false if @excludes.any? && @excludes.any? { |regex| regex.match(path) }
|
179
|
+
true
|
180
|
+
end
|
181
|
+
|
147
182
|
@s3 = S3CP.connect()
|
148
183
|
|
149
184
|
def direction(from, to)
|
@@ -217,68 +252,86 @@ end
|
|
217
252
|
|
218
253
|
def local_to_s3(bucket_to, key, file, options = {})
|
219
254
|
log(with_headers("Copy #{file} to s3://#{bucket_to}/#{key}"))
|
220
|
-
|
221
|
-
|
255
|
+
|
256
|
+
expected_md5 = if options[:checksum] || options[:sync]
|
257
|
+
md5(file)
|
222
258
|
end
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
|
227
|
-
|
228
|
-
|
229
|
-
|
230
|
-
|
259
|
+
|
260
|
+
actual_md5 = if options[:sync]
|
261
|
+
md5 = s3_checksum(bucket_to, key)
|
262
|
+
case md5
|
263
|
+
when :not_found
|
264
|
+
nil
|
265
|
+
when :invalid
|
266
|
+
STDERR.puts "Warning: invalid MD5 checksum in metadata; file will be force-copied."
|
267
|
+
nil
|
268
|
+
else
|
269
|
+
md5
|
231
270
|
end
|
271
|
+
end
|
232
272
|
|
233
|
-
|
273
|
+
if actual_md5.nil? || (options[:sync] && expected_md5 != actual_md5)
|
274
|
+
retries = 0
|
234
275
|
begin
|
235
|
-
if
|
236
|
-
|
237
|
-
|
238
|
-
|
239
|
-
|
240
|
-
|
241
|
-
|
242
|
-
|
243
|
-
|
244
|
-
|
245
|
-
|
246
|
-
|
247
|
-
|
248
|
-
|
249
|
-
|
276
|
+
if retries == options[:retries]
|
277
|
+
fail "Unable to upload to s3://#{bucket_to}/#{key} after #{retries} attempts."
|
278
|
+
end
|
279
|
+
if retries > 0
|
280
|
+
STDERR.puts "Warning: failed checksum for s3://#{bucket_to}/#{bucket_to}. Retrying #{options[:retries] - retries} more time(s)."
|
281
|
+
sleep options[:retry_delay]
|
282
|
+
end
|
283
|
+
|
284
|
+
f = File.open(file)
|
285
|
+
begin
|
286
|
+
if $stdout.isatty
|
287
|
+
f = Proxy.new(f)
|
288
|
+
progress_bar = ProgressBar.new(File.basename(file), File.size(file)).tap do |p|
|
289
|
+
p.file_transfer_mode
|
290
|
+
end
|
291
|
+
class << f
|
292
|
+
attr_accessor :progress_bar
|
293
|
+
def read(length, buffer=nil)
|
294
|
+
begin
|
295
|
+
result = @target.read(length, buffer)
|
296
|
+
@progress_bar.inc result.length if result
|
297
|
+
result
|
298
|
+
rescue => e
|
299
|
+
puts e
|
300
|
+
raise e
|
301
|
+
end
|
250
302
|
end
|
251
303
|
end
|
304
|
+
f.progress_bar = progress_bar
|
305
|
+
else
|
306
|
+
progress_bar = nil
|
252
307
|
end
|
253
|
-
f.progress_bar = progress_bar
|
254
|
-
else
|
255
|
-
progress_bar = nil
|
256
|
-
end
|
257
308
|
|
258
|
-
|
259
|
-
|
309
|
+
meta = @s3.interface.put(bucket_to, key, f, @headers)
|
310
|
+
progress_bar.finish if progress_bar
|
260
311
|
|
261
|
-
|
262
|
-
|
263
|
-
|
264
|
-
|
265
|
-
|
266
|
-
|
267
|
-
actual_md5 = nil
|
312
|
+
if options[:checksum]
|
313
|
+
actual_md5 = s3_checksum(bucket_to, key)
|
314
|
+
unless actual_md5.is_a? String
|
315
|
+
STDERR.puts "Warning: invalid MD5 checksum in metadata; skipped checksum verification."
|
316
|
+
actual_md5 = nil
|
317
|
+
end
|
268
318
|
end
|
319
|
+
rescue => e
|
320
|
+
raise e unless options[:checksum]
|
321
|
+
STDERR.puts e
|
322
|
+
ensure
|
323
|
+
f.close()
|
269
324
|
end
|
270
|
-
|
271
|
-
raise e unless options[:checksum]
|
272
|
-
STDERR.puts e
|
273
|
-
ensure
|
274
|
-
f.close()
|
275
|
-
end
|
276
|
-
retries += 1
|
325
|
+
retries += 1
|
277
326
|
end until options[:checksum] == false || actual_md5.nil? || expected_md5 == actual_md5
|
327
|
+
else
|
328
|
+
log "Already synchronized."
|
329
|
+
end
|
278
330
|
end
|
279
331
|
|
280
332
|
def s3_to_local(bucket_from, key_from, dest, options = {})
|
281
333
|
log("Copy s3://#{bucket_from}/#{key_from} to #{dest}")
|
334
|
+
|
282
335
|
retries = 0
|
283
336
|
begin
|
284
337
|
if retries == options[:retries]
|
@@ -289,43 +342,51 @@ def s3_to_local(bucket_from, key_from, dest, options = {})
|
|
289
342
|
STDERR.puts "Warning: failed checksum for s3://#{bucket_from}/#{key_from}. Retrying #{options[:retries] - retries} more time(s)."
|
290
343
|
sleep options[:retry_delay]
|
291
344
|
end
|
292
|
-
|
293
|
-
f = File.new(dest, "wb")
|
294
345
|
begin
|
295
|
-
|
296
|
-
|
297
|
-
|
298
|
-
|
299
|
-
|
300
|
-
|
346
|
+
expected_md5 = if options[:checksum] || options[:sync]
|
347
|
+
md5 = s3_checksum(bucket_from, key_from)
|
348
|
+
if options[:sync] && !md5.is_a?(String)
|
349
|
+
STDERR.puts "Warning: invalid MD5 checksum in metadata; file will be force-copied."
|
350
|
+
nil
|
351
|
+
elsif !md5.is_a? String
|
301
352
|
STDERR.puts "Warning: invalid MD5 checksum in metadata; skipped checksum verification."
|
302
|
-
|
353
|
+
nil
|
354
|
+
else
|
355
|
+
md5
|
303
356
|
end
|
304
|
-
size = metadata["content-length"].to_i
|
305
|
-
elsif $stdout.isatty
|
306
|
-
metadata = @s3.interface.head(bucket_from, key_from)
|
307
|
-
size = metadata["content-length"].to_i
|
308
357
|
end
|
309
|
-
|
310
|
-
|
311
|
-
|
312
|
-
|
358
|
+
|
359
|
+
actual_md5 = if options[:sync] && File.exist?(dest)
|
360
|
+
md5(dest)
|
361
|
+
end
|
362
|
+
|
363
|
+
if !options[:sync] || (expected_md5 != actual_md5)
|
364
|
+
f = File.new(dest, "wb")
|
365
|
+
begin
|
366
|
+
progress_bar = if $stdout.isatty
|
367
|
+
size = s3_size(bucket_from, key_from)
|
368
|
+
ProgressBar.new(File.basename(key_from), size).tap do |p|
|
369
|
+
p.file_transfer_mode
|
370
|
+
end
|
313
371
|
end
|
372
|
+
@s3.interface.get(bucket_from, key_from) do |chunk|
|
373
|
+
f.write(chunk)
|
374
|
+
progress_bar.inc chunk.size if progress_bar
|
375
|
+
end
|
376
|
+
progress_bar.finish
|
377
|
+
rescue => e
|
378
|
+
progress_bar.halt if progress_bar
|
379
|
+
raise e
|
380
|
+
ensure
|
381
|
+
f.close()
|
314
382
|
end
|
315
|
-
|
316
|
-
|
317
|
-
|
318
|
-
end
|
319
|
-
progress_bar.finish
|
320
|
-
rescue => e
|
321
|
-
progress_bar.halt if progress_bar
|
322
|
-
raise e
|
383
|
+
else
|
384
|
+
log("Already synchronized")
|
385
|
+
return
|
323
386
|
end
|
324
387
|
rescue => e
|
325
388
|
raise e unless options[:checksum]
|
326
389
|
STDERR.puts e
|
327
|
-
ensure
|
328
|
-
f.close()
|
329
390
|
end
|
330
391
|
retries += 1
|
331
392
|
end until options[:checksum] == false || expected_md5.nil? || md5(dest) == expected_md5
|
@@ -337,14 +398,33 @@ def s3_exist?(bucket, key)
|
|
337
398
|
(metadata != nil)
|
338
399
|
end
|
339
400
|
|
401
|
+
def s3_checksum(bucket, key)
|
402
|
+
begin
|
403
|
+
metadata = @s3.interface.head(bucket, key)
|
404
|
+
return :not_found unless metadata
|
405
|
+
rescue => e
|
406
|
+
return :not_found if e.is_a?(RightAws::AwsError) && e.http_code == "404"
|
407
|
+
raise e
|
408
|
+
end
|
409
|
+
|
410
|
+
md5 = metadata["etag"] or fail "Unable to get etag/md5 for #{bucket_to}:#{key}"
|
411
|
+
return :invalid unless md5
|
412
|
+
|
413
|
+
md5 = md5.sub(/^"/, "").sub(/"$/, "") # strip beginning and trailing quotes
|
414
|
+
return :invalid if md5 =~ /-/
|
415
|
+
|
416
|
+
md5
|
417
|
+
end
|
418
|
+
|
419
|
+
def s3_size(bucket, key)
|
420
|
+
metadata = @s3.interface.head(bucket, key)
|
421
|
+
metadata["content-length"].to_i
|
422
|
+
end
|
423
|
+
|
340
424
|
def copy(from, to, options)
|
341
425
|
bucket_from, key_from = S3CP.bucket_and_key(from)
|
342
426
|
bucket_to, key_to = S3CP.bucket_and_key(to)
|
343
427
|
|
344
|
-
#puts "bucket_from #{bucket_from.inspect} key_from #{key_from.inspect}"
|
345
|
-
#puts "bucket_to #{bucket_to.inspect} key_from #{key_to.inspect}"
|
346
|
-
#puts "direction #{direction(from, to)}"
|
347
|
-
|
348
428
|
case direction(from, to)
|
349
429
|
when :s3_to_s3
|
350
430
|
if options[:recursive]
|
@@ -353,8 +433,10 @@ def copy(from, to, options)
|
|
353
433
|
page[:contents].each { |entry| keys << entry[:key] }
|
354
434
|
end
|
355
435
|
keys.each do |key|
|
356
|
-
|
357
|
-
|
436
|
+
if match(key)
|
437
|
+
dest = no_slash(key_to) + '/' + relative(key_from, key)
|
438
|
+
s3_to_s3(bucket_from, key, bucket_to, dest) unless !options[:overwrite] && s3_exist?(bucket_to, dest)
|
439
|
+
end
|
358
440
|
end
|
359
441
|
else
|
360
442
|
s3_to_s3(bucket_from, key_from, bucket_to, key_to) unless !options[:overwrite] && s3_exist?(bucket_to, key_to)
|
@@ -363,9 +445,10 @@ def copy(from, to, options)
|
|
363
445
|
if options[:recursive]
|
364
446
|
files = Dir[from + "/**/*"]
|
365
447
|
files.each do |f|
|
366
|
-
|
367
|
-
|
368
|
-
|
448
|
+
if File.file?(f) && match(f)
|
449
|
+
key = no_slash(key_to) + '/' + relative(from, f)
|
450
|
+
local_to_s3(bucket_to, key, File.expand_path(f), options) unless !options[:overwrite] && s3_exist?(bucket_to, key)
|
451
|
+
end
|
369
452
|
end
|
370
453
|
else
|
371
454
|
local_to_s3(bucket_to, key_to, File.expand_path(from), options) unless !options[:overwrite] && s3_exist?(bucket_to, key_to)
|
@@ -377,12 +460,14 @@ def copy(from, to, options)
|
|
377
460
|
page[:contents].each { |entry| keys << entry[:key] }
|
378
461
|
end
|
379
462
|
keys.each do |key|
|
380
|
-
|
381
|
-
|
382
|
-
|
383
|
-
|
384
|
-
|
385
|
-
|
463
|
+
if match(key)
|
464
|
+
dest = File.expand_path(to) + '/' + relative(key_from, key)
|
465
|
+
dest = File.join(dest, File.basename(key)) if File.directory?(dest)
|
466
|
+
dir = File.dirname(dest)
|
467
|
+
FileUtils.mkdir_p dir unless File.exist? dir
|
468
|
+
fail "Destination path is not a directory: #{dir}" unless File.directory?(dir)
|
469
|
+
s3_to_local(bucket_from, key, dest, options) unless !options[:overwrite] && File.exist?(dest)
|
470
|
+
end
|
386
471
|
end
|
387
472
|
else
|
388
473
|
dest = File.expand_path(to)
|
@@ -390,6 +475,9 @@ def copy(from, to, options)
|
|
390
475
|
s3_to_local(bucket_from, key_from, dest, options) unless !options[:overwrite] && File.exist?(dest)
|
391
476
|
end
|
392
477
|
when :local_to_local
|
478
|
+
if options[:include_regex].any? || options[:exclude_regex].any?
|
479
|
+
fail "Include/exclude not supported on local-to-local copies"
|
480
|
+
end
|
393
481
|
if options[:recursive]
|
394
482
|
FileUtils.cp_r from, to
|
395
483
|
else
|
data/lib/s3cp/version.rb
CHANGED
metadata
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: s3cp
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
4
|
+
hash: 17
|
5
5
|
prerelease:
|
6
6
|
segments:
|
7
7
|
- 0
|
8
8
|
- 2
|
9
|
-
-
|
10
|
-
version: 0.2.
|
9
|
+
- 3
|
10
|
+
version: 0.2.3
|
11
11
|
platform: ruby
|
12
12
|
authors:
|
13
13
|
- Alex Boisvert
|
@@ -15,7 +15,7 @@ autorequire:
|
|
15
15
|
bindir: bin
|
16
16
|
cert_chain: []
|
17
17
|
|
18
|
-
date: 2012-02-
|
18
|
+
date: 2012-02-24 00:00:00 Z
|
19
19
|
dependencies:
|
20
20
|
- !ruby/object:Gem::Dependency
|
21
21
|
prerelease: false
|
@@ -154,6 +154,7 @@ files:
|
|
154
154
|
- lib/s3cp/utils.rb
|
155
155
|
- lib/s3cp/s3mod.rb
|
156
156
|
- lib/s3cp/s3cat.rb
|
157
|
+
- lib/s3cp/#Untitled-7#
|
157
158
|
- lib/s3cp/completion.rb
|
158
159
|
- History.txt
|
159
160
|
- README.md
|