s3cp 0.1.15 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/History.txt +18 -1
- data/bin/s3cp_complete +10 -0
- data/bin/s3dir +4 -0
- data/bin/s3stat +4 -0
- data/lib/s3cp/completion.rb +154 -0
- data/lib/s3cp/s3cp.rb +26 -12
- data/lib/s3cp/s3stat.rb +37 -0
- data/lib/s3cp/version.rb +1 -1
- metadata +15 -7
data/History.txt
CHANGED
@@ -1,4 +1,21 @@
|
|
1
|
-
=== 0.1
|
1
|
+
=== 0.2.1 / (Pending)
|
2
|
+
|
3
|
+
=== 0.2.0 / (2012-02-20)
|
4
|
+
|
5
|
+
* Added: s3stat command to display S3 object properties
|
6
|
+
* Added: s3dir as a shortcut for "s3ls --delimiter / ..."
|
7
|
+
(default to character in S3CP_DELIMITER environment variable or "/" if not defined)
|
8
|
+
* Added: s3cp defaults can now be set using environment variables
|
9
|
+
S3CP_OVERWRITE, S3CP_CHECKSUM, S3CP_RETRIES, S3CP_RETRY_DELAY
|
10
|
+
* Added: Support for Bash command-line completion of S3 URLs (see below).
|
11
|
+
* Fixed: Skip checksum verification for S3 objects with invalid MD5's
|
12
|
+
(details @ https://forums.aws.amazon.com/message.jspa?messageID=234538)
|
13
|
+
|
14
|
+
To install Bash completion for S3 URLs, add the following to ~/.bashrc:
|
15
|
+
|
16
|
+
for cmd in [ s3cat s3cp s3dir s3ls s3mod s3rm s3stat ]; do
|
17
|
+
complete -C s3cp_complete $cmd
|
18
|
+
done
|
2
19
|
|
3
20
|
=== 0.1.15 / (2012-02-17)
|
4
21
|
|
data/bin/s3cp_complete
ADDED
@@ -0,0 +1,10 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
#
|
3
|
+
# A script to allow Bash to complete an s3cp command-line.
|
4
|
+
#
|
5
|
+
# To install for Bash 2.0 or better, add the following to ~/.bashrc:
|
6
|
+
#
|
7
|
+
# $ for cmd in [ s3cat s3cp s3dir s3ls s3mod s3rm s3stat ]; do complete -C s3cp_complete $cmd; done
|
8
|
+
#
|
9
|
+
require 's3cp/completion'
|
10
|
+
|
data/bin/s3dir
ADDED
data/bin/s3stat
ADDED
@@ -0,0 +1,154 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'extensions/kernel' if RUBY_VERSION =~ /1.8/
|
3
|
+
require 'right_aws'
|
4
|
+
require 'optparse'
|
5
|
+
require 'date'
|
6
|
+
require 'highline/import'
|
7
|
+
require 'tempfile'
|
8
|
+
|
9
|
+
require 's3cp/utils'
|
10
|
+
|
11
|
+
cmd_line = ENV['COMP_LINE']
|
12
|
+
position = ENV['COMP_POINT'].to_i
|
13
|
+
|
14
|
+
url = begin
|
15
|
+
start = position
|
16
|
+
start -= 1 while start >= 1 && cmd_line[start-1].chr != ' '
|
17
|
+
cmd_line[start..position-1]
|
18
|
+
end
|
19
|
+
|
20
|
+
cmd = ARGV[0]
|
21
|
+
arg1 = ARGV[1]
|
22
|
+
arg2 = ARGV[2]
|
23
|
+
|
24
|
+
DEBUG = true
|
25
|
+
|
26
|
+
@s3 = S3CP.connect()
|
27
|
+
|
28
|
+
if DEBUG
|
29
|
+
@debug = File.open("/tmp/s3cp_complete", "wb")
|
30
|
+
end
|
31
|
+
|
32
|
+
def debug(str)
|
33
|
+
@debug.puts(str) if @debug
|
34
|
+
end
|
35
|
+
|
36
|
+
debug "url #{url}"
|
37
|
+
|
38
|
+
delimiter = ENV["S3CP_DELIMITER"] || "/"
|
39
|
+
debug "delimiter #{delimiter}"
|
40
|
+
|
41
|
+
bucket, prefix = S3CP.bucket_and_key(url)
|
42
|
+
if bucket == "s3"
|
43
|
+
bucket = ""
|
44
|
+
prefix = nil
|
45
|
+
end
|
46
|
+
|
47
|
+
debug "bucket #{bucket.inspect} prefix #{prefix.inspect}"
|
48
|
+
|
49
|
+
@legacy_format = (url =~ /\:\//) ? false : true
|
50
|
+
debug "legacy_format #{@legacy_format} "
|
51
|
+
|
52
|
+
recursive = cmd_line.split(" ").include?("-r")
|
53
|
+
debug "recursive #{recursive}"
|
54
|
+
|
55
|
+
@dirs_only = (cmd =~ /s3dir$/) || (cmd =~ /s3cp$/ && recursive) || (cmd =~ /s3rm$/ && recursive)
|
56
|
+
debug "dirs_only #{@dirs_only}"
|
57
|
+
|
58
|
+
def print_keys(bucket, keys)
|
59
|
+
keys.each do |key|
|
60
|
+
if @legacy_format
|
61
|
+
debug key
|
62
|
+
puts key
|
63
|
+
else
|
64
|
+
debug "//#{bucket}/#{key}"
|
65
|
+
puts "//#{bucket}/#{key}"
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
def print_buckets(buckets)
|
71
|
+
buckets = buckets.map { |b| @legacy_format ? b + ":" : b + "/" }
|
72
|
+
buckets << buckets[0] + " " if buckets.size == 1
|
73
|
+
buckets.each do |bucket|
|
74
|
+
if @legacy_format
|
75
|
+
debug bucket
|
76
|
+
puts bucket
|
77
|
+
else
|
78
|
+
debug "//#{bucket}"
|
79
|
+
puts "//#{bucket}"
|
80
|
+
end
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
if (prefix && prefix.length > 0) || (url =~ /s3\:\/\/[^\/]+\//) || (url =~ /\:$/)
|
85
|
+
# complete s3 key name
|
86
|
+
bucket, prefix = S3CP.bucket_and_key(url)
|
87
|
+
fail "Your URL looks funny, doesn't it?" unless bucket
|
88
|
+
|
89
|
+
result = nil
|
90
|
+
|
91
|
+
# try directory first
|
92
|
+
dir_options = Hash.new
|
93
|
+
dir_options[:prefix] = prefix
|
94
|
+
dir_options[:delimiter] = delimiter
|
95
|
+
begin
|
96
|
+
@s3.interface.incrementally_list_bucket(bucket, dir_options) do |page|
|
97
|
+
entries = page[:common_prefixes]
|
98
|
+
entries << page[:contents][0][:key] if page[:contents].length > 0 && entries.length > 0
|
99
|
+
result = entries
|
100
|
+
end
|
101
|
+
rescue => e
|
102
|
+
debug "exception #{e}"
|
103
|
+
result = []
|
104
|
+
end
|
105
|
+
|
106
|
+
debug "result1 #{result.inspect}"
|
107
|
+
|
108
|
+
# there may be longer matches
|
109
|
+
if (result.size == 0) || (result.size == 1)
|
110
|
+
prefix = result[0] if result.size == 1
|
111
|
+
file_options = Hash.new
|
112
|
+
file_options[:prefix] = prefix
|
113
|
+
file_options["max-keys"] = 100
|
114
|
+
short_keys = Hash.new
|
115
|
+
all_keys = []
|
116
|
+
begin
|
117
|
+
@s3.interface.incrementally_list_bucket(bucket, file_options) do |page|
|
118
|
+
entries = page[:contents]
|
119
|
+
entries.each do |entry|
|
120
|
+
key = entry[:key]
|
121
|
+
pos = prefix.length-1
|
122
|
+
pos += 1 while pos+1 < key.length && key[pos+1].chr == delimiter
|
123
|
+
short_key = key[0..pos]
|
124
|
+
short_keys[short_key] = key
|
125
|
+
all_keys << key
|
126
|
+
end
|
127
|
+
end
|
128
|
+
rescue => e
|
129
|
+
debug "exception #{e}"
|
130
|
+
result = []
|
131
|
+
end
|
132
|
+
result = @dirs_only ? short_keys.keys.sort : all_keys
|
133
|
+
debug "result2 #{result.inspect}"
|
134
|
+
end
|
135
|
+
|
136
|
+
debug "final #{result.inspect}"
|
137
|
+
|
138
|
+
print_keys(bucket, result)
|
139
|
+
else
|
140
|
+
# complete bucket name
|
141
|
+
bucket ||= url
|
142
|
+
begin
|
143
|
+
buckets = @s3.interface.list_all_my_buckets()
|
144
|
+
bucket_names = buckets.map { |b| b[:name] }
|
145
|
+
matching = bucket_names.select { |b| b =~ /^#{bucket}/ }
|
146
|
+
print_buckets(matching)
|
147
|
+
rescue => e
|
148
|
+
debug "exception #{e}"
|
149
|
+
result = []
|
150
|
+
end
|
151
|
+
end
|
152
|
+
|
153
|
+
@debug.close() if DEBUG
|
154
|
+
|
data/lib/s3cp/s3cp.rb
CHANGED
@@ -13,10 +13,10 @@ require 's3cp/utils'
|
|
13
13
|
options = {}
|
14
14
|
options[:verbose] = $stdout.isatty ? true : false
|
15
15
|
options[:headers] = []
|
16
|
-
options[:overwrite]
|
17
|
-
options[:checksum]
|
18
|
-
options[:retries]
|
19
|
-
options[:retry_delay] = 1
|
16
|
+
options[:overwrite] = ENV["S3CP_RETRIES"] ? (ENV["S3CP_OVERWRITE"] =~ /y|yes|true|1|^\s*$/i ? true : false) : true
|
17
|
+
options[:checksum] = ENV["S3CP_CHECKSUM"] ? (ENV["S3CP_CHECKSUM"] =~ /y|yes|true|1|^\s*$/i ? true : false) : true
|
18
|
+
options[:retries] = ENV["S3CP_RETRIES"] ? ENV["S3CP_RETRIES"].to_i : 5
|
19
|
+
options[:retry_delay] = ENV["S3CP_RETRY_DELAY"] ? ENV["S3CP_RETRY_DELAY"].to_i : 1
|
20
20
|
|
21
21
|
op = OptionParser.new do |opts|
|
22
22
|
opts.banner = <<-BANNER
|
@@ -50,7 +50,7 @@ op = OptionParser.new do |opts|
|
|
50
50
|
end
|
51
51
|
|
52
52
|
opts.on("--max-attempts N", "Number of attempts to upload/download until checksum matches (default #{options[:retries]})") do |attempts|
|
53
|
-
options[:
|
53
|
+
options[:retries] = attempts.to_i
|
54
54
|
end
|
55
55
|
|
56
56
|
opts.on("--retry-delay SECONDS", "Time to wait (in seconds) between retries (default #{options[:retry_delay]})") do |delay|
|
@@ -189,10 +189,13 @@ def local_to_s3(bucket_to, key, file, options = {})
|
|
189
189
|
end
|
190
190
|
retries = 0
|
191
191
|
begin
|
192
|
-
if retries == options[:
|
193
|
-
fail "Unable to upload to s3://#{
|
192
|
+
if retries == options[:retries]
|
193
|
+
fail "Unable to upload to s3://#{bucket_to}/#{key} after #{retries} attempts."
|
194
|
+
end
|
195
|
+
if retries > 0
|
196
|
+
STDERR.puts "Warning: failed checksum for s3://#{bucket_to}/#{bucket_to}. Retrying #{options[:retries] - retries} more time(s)."
|
197
|
+
sleep options[:retry_delay]
|
194
198
|
end
|
195
|
-
sleep options[:retry_delay] if retries > 0
|
196
199
|
|
197
200
|
f = File.open(file)
|
198
201
|
begin
|
@@ -202,6 +205,10 @@ def local_to_s3(bucket_to, key, file, options = {})
|
|
202
205
|
metadata = @s3.interface.head(bucket_to, key)
|
203
206
|
actual_md5 = metadata["etag"] or fail "Unable to get etag/md5 for #{bucket_to}:#{key}"
|
204
207
|
actual_md5 = actual_md5.sub(/^"/, "").sub(/"$/, "") # strip beginning and trailing quotes
|
208
|
+
if actual_md5 =~ /-/
|
209
|
+
STDERR.puts "Warning: invalid MD5 checksum in metadata; skipped checksum verification."
|
210
|
+
actual_md5 = nil
|
211
|
+
end
|
205
212
|
end
|
206
213
|
rescue => e
|
207
214
|
raise e unless options[:checksum]
|
@@ -210,18 +217,21 @@ def local_to_s3(bucket_to, key, file, options = {})
|
|
210
217
|
f.close()
|
211
218
|
end
|
212
219
|
retries += 1
|
213
|
-
|
220
|
+
end until options[:checksum] == false || actual_md5.nil? || expected_md5 == actual_md5
|
214
221
|
end
|
215
222
|
|
216
223
|
def s3_to_local(bucket_from, key_from, dest, options = {})
|
217
224
|
log("Copy s3://#{bucket_from}/#{key_from} to #{dest}")
|
218
225
|
retries = 0
|
219
226
|
begin
|
220
|
-
if retries == options[:
|
227
|
+
if retries == options[:retries]
|
221
228
|
File.delete(dest) if File.exist?(dest)
|
222
229
|
fail "Unable to download s3://#{bucket_from}/#{key_from} after #{retries} attempts."
|
223
230
|
end
|
224
|
-
|
231
|
+
if retries > 0
|
232
|
+
STDERR.puts "Warning: failed checksum for s3://#{bucket_from}/#{key_from}. Retrying #{options[:retries] - retries} more time(s)."
|
233
|
+
sleep options[:retry_delay]
|
234
|
+
end
|
225
235
|
|
226
236
|
f = File.new(dest, "wb")
|
227
237
|
begin
|
@@ -229,6 +239,10 @@ def s3_to_local(bucket_from, key_from, dest, options = {})
|
|
229
239
|
metadata = @s3.interface.head(bucket_from, key_from)
|
230
240
|
expected_md5 = metadata["etag"] or fail "Unable to get etag/md5 for #{bucket_from}:#{key_from}"
|
231
241
|
expected_md5 = expected_md5.sub(/^"/, "").sub(/"$/, "") # strip beginning and trailing quotes
|
242
|
+
if expected_md5 =~ /-/
|
243
|
+
STDERR.puts "Warning: invalid MD5 checksum in metadata; skipped checksum verification."
|
244
|
+
expected_md5 = nil
|
245
|
+
end
|
232
246
|
end
|
233
247
|
@s3.interface.get(bucket_from, key_from) do |chunk|
|
234
248
|
f.write(chunk)
|
@@ -240,7 +254,7 @@ def s3_to_local(bucket_from, key_from, dest, options = {})
|
|
240
254
|
f.close()
|
241
255
|
end
|
242
256
|
retries += 1
|
243
|
-
end until options[:checksum] == false || md5(dest) == expected_md5
|
257
|
+
end until options[:checksum] == false || expected_md5.nil? || md5(dest) == expected_md5
|
244
258
|
end
|
245
259
|
|
246
260
|
def s3_exist?(bucket, key)
|
data/lib/s3cp/s3stat.rb
ADDED
@@ -0,0 +1,37 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'extensions/kernel' if RUBY_VERSION =~ /1.8/
|
3
|
+
require 'right_aws'
|
4
|
+
require 'optparse'
|
5
|
+
require 's3cp/utils'
|
6
|
+
|
7
|
+
op = OptionParser.new do |opts|
|
8
|
+
opts.banner = "s3stat [path]"
|
9
|
+
|
10
|
+
opts.on_tail("-h", "--help", "Show this message") do
|
11
|
+
puts op
|
12
|
+
exit
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
op.parse!(ARGV)
|
17
|
+
|
18
|
+
if ARGV.size < 1
|
19
|
+
puts op
|
20
|
+
exit
|
21
|
+
end
|
22
|
+
|
23
|
+
source = ARGV[0]
|
24
|
+
permission = ARGV.last
|
25
|
+
|
26
|
+
@s3 = S3CP.connect()
|
27
|
+
|
28
|
+
def get_metadata(bucket, key)
|
29
|
+
metadata = @s3.interface.head(bucket, key)
|
30
|
+
metadata.sort.each do |k,v|
|
31
|
+
puts "#{"%20s" % k} #{v}"
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
bucket,key = S3CP.bucket_and_key(source)
|
36
|
+
get_metadata(bucket, key)
|
37
|
+
|
data/lib/s3cp/version.rb
CHANGED
metadata
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: s3cp
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
4
|
+
hash: 23
|
5
5
|
prerelease:
|
6
6
|
segments:
|
7
7
|
- 0
|
8
|
-
-
|
9
|
-
-
|
10
|
-
version: 0.
|
8
|
+
- 2
|
9
|
+
- 0
|
10
|
+
version: 0.2.0
|
11
11
|
platform: ruby
|
12
12
|
authors:
|
13
13
|
- Alex Boisvert
|
@@ -15,7 +15,7 @@ autorequire:
|
|
15
15
|
bindir: bin
|
16
16
|
cert_chain: []
|
17
17
|
|
18
|
-
date: 2012-02-
|
18
|
+
date: 2012-02-21 00:00:00 Z
|
19
19
|
dependencies:
|
20
20
|
- !ruby/object:Gem::Dependency
|
21
21
|
prerelease: false
|
@@ -116,11 +116,14 @@ description:
|
|
116
116
|
email:
|
117
117
|
- alex.boisvert@gmail.com
|
118
118
|
executables:
|
119
|
-
- s3ls
|
120
|
-
- s3cp
|
121
119
|
- s3cat
|
120
|
+
- s3cp
|
121
|
+
- s3cp_complete
|
122
|
+
- s3dir
|
123
|
+
- s3ls
|
122
124
|
- s3mod
|
123
125
|
- s3rm
|
126
|
+
- s3stat
|
124
127
|
extensions: []
|
125
128
|
|
126
129
|
extra_rdoc_files:
|
@@ -130,17 +133,22 @@ files:
|
|
130
133
|
- lib/s3cp/s3rm.rb
|
131
134
|
- lib/s3cp/s3ls.rb
|
132
135
|
- lib/s3cp/version.rb
|
136
|
+
- lib/s3cp/s3stat.rb
|
133
137
|
- lib/s3cp/s3cp.rb
|
134
138
|
- lib/s3cp/utils.rb
|
135
139
|
- lib/s3cp/s3mod.rb
|
136
140
|
- lib/s3cp/s3cat.rb
|
141
|
+
- lib/s3cp/completion.rb
|
137
142
|
- History.txt
|
138
143
|
- README.md
|
139
144
|
- bin/s3mod
|
140
145
|
- bin/s3ls
|
141
146
|
- bin/s3cp
|
147
|
+
- bin/s3cp_complete
|
148
|
+
- bin/s3stat
|
142
149
|
- bin/s3cat
|
143
150
|
- bin/s3rm
|
151
|
+
- bin/s3dir
|
144
152
|
homepage:
|
145
153
|
licenses: []
|
146
154
|
|