s3cp 0.1.15 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,4 +1,21 @@
1
- === 0.1.16 / (Pending)
1
+ === 0.2.1 / (Pending)
2
+
3
+ === 0.2.0 / (2012-02-20)
4
+
5
+ * Added: s3stat command to display S3 object properties
6
+ * Added: s3dir as a shortcut for "s3ls --delimiter / ..."
7
+ (default to character in S3CP_DELIMITER environment variable or "/" if not defined)
8
+ * Added: s3cp defaults can now be set using environment variables
9
+ S3CP_OVERWRITE, S3CP_CHECKSUM, S3CP_RETRIES, S3CP_RETRY_DELAY
10
+ * Added: Support for Bash command-line completion of S3 URLs (see below).
11
+ * Fixed: Skip checksum verification for S3 objects with invalid MD5's
12
+ (details @ https://forums.aws.amazon.com/message.jspa?messageID=234538)
13
+
14
+ To install Bash completion for S3 URLs, add the following to ~/.bashrc:
15
+
16
+ for cmd in [ s3cat s3cp s3dir s3ls s3mod s3rm s3stat ]; do
17
+ complete -C s3cp_complete $cmd
18
+ done
2
19
 
3
20
  === 0.1.15 / (2012-02-17)
4
21
 
@@ -0,0 +1,10 @@
1
+ #!/usr/bin/env ruby
2
+ #
3
+ # A script to allow Bash to complete an s3cp command-line.
4
+ #
5
+ # To install for Bash 2.0 or better, add the following to ~/.bashrc:
6
+ #
7
+ # $ for cmd in [ s3cat s3cp s3dir s3ls s3mod s3rm s3stat ]; do complete -C s3cp_complete $cmd; done
8
+ #
9
+ require 's3cp/completion'
10
+
@@ -0,0 +1,4 @@
1
+ #!/usr/bin/env ruby
2
+ ARGV.unshift "--delimiter", ENV["S3CP_DELIMITER"] || "/"
3
+ require 's3cp/s3ls'
4
+
@@ -0,0 +1,4 @@
1
+ #!/usr/bin/env ruby
2
+ require 's3cp/s3stat'
3
+
4
+
@@ -0,0 +1,154 @@
1
+ require 'rubygems'
2
+ require 'extensions/kernel' if RUBY_VERSION =~ /1.8/
3
+ require 'right_aws'
4
+ require 'optparse'
5
+ require 'date'
6
+ require 'highline/import'
7
+ require 'tempfile'
8
+
9
+ require 's3cp/utils'
10
+
11
+ cmd_line = ENV['COMP_LINE']
12
+ position = ENV['COMP_POINT'].to_i
13
+
14
+ url = begin
15
+ start = position
16
+ start -= 1 while start >= 1 && cmd_line[start-1].chr != ' '
17
+ cmd_line[start..position-1]
18
+ end
19
+
20
+ cmd = ARGV[0]
21
+ arg1 = ARGV[1]
22
+ arg2 = ARGV[2]
23
+
24
+ DEBUG = true
25
+
26
+ @s3 = S3CP.connect()
27
+
28
+ if DEBUG
29
+ @debug = File.open("/tmp/s3cp_complete", "wb")
30
+ end
31
+
32
+ def debug(str)
33
+ @debug.puts(str) if @debug
34
+ end
35
+
36
+ debug "url #{url}"
37
+
38
+ delimiter = ENV["S3CP_DELIMITER"] || "/"
39
+ debug "delimiter #{delimiter}"
40
+
41
+ bucket, prefix = S3CP.bucket_and_key(url)
42
+ if bucket == "s3"
43
+ bucket = ""
44
+ prefix = nil
45
+ end
46
+
47
+ debug "bucket #{bucket.inspect} prefix #{prefix.inspect}"
48
+
49
+ @legacy_format = (url =~ /\:\//) ? false : true
50
+ debug "legacy_format #{@legacy_format} "
51
+
52
+ recursive = cmd_line.split(" ").include?("-r")
53
+ debug "recursive #{recursive}"
54
+
55
+ @dirs_only = (cmd =~ /s3dir$/) || (cmd =~ /s3cp$/ && recursive) || (cmd =~ /s3rm$/ && recursive)
56
+ debug "dirs_only #{@dirs_only}"
57
+
58
+ def print_keys(bucket, keys)
59
+ keys.each do |key|
60
+ if @legacy_format
61
+ debug key
62
+ puts key
63
+ else
64
+ debug "//#{bucket}/#{key}"
65
+ puts "//#{bucket}/#{key}"
66
+ end
67
+ end
68
+ end
69
+
70
+ def print_buckets(buckets)
71
+ buckets = buckets.map { |b| @legacy_format ? b + ":" : b + "/" }
72
+ buckets << buckets[0] + " " if buckets.size == 1
73
+ buckets.each do |bucket|
74
+ if @legacy_format
75
+ debug bucket
76
+ puts bucket
77
+ else
78
+ debug "//#{bucket}"
79
+ puts "//#{bucket}"
80
+ end
81
+ end
82
+ end
83
+
84
+ if (prefix && prefix.length > 0) || (url =~ /s3\:\/\/[^\/]+\//) || (url =~ /\:$/)
85
+ # complete s3 key name
86
+ bucket, prefix = S3CP.bucket_and_key(url)
87
+ fail "Your URL looks funny, doesn't it?" unless bucket
88
+
89
+ result = nil
90
+
91
+ # try directory first
92
+ dir_options = Hash.new
93
+ dir_options[:prefix] = prefix
94
+ dir_options[:delimiter] = delimiter
95
+ begin
96
+ @s3.interface.incrementally_list_bucket(bucket, dir_options) do |page|
97
+ entries = page[:common_prefixes]
98
+ entries << page[:contents][0][:key] if page[:contents].length > 0 && entries.length > 0
99
+ result = entries
100
+ end
101
+ rescue => e
102
+ debug "exception #{e}"
103
+ result = []
104
+ end
105
+
106
+ debug "result1 #{result.inspect}"
107
+
108
+ # there may be longer matches
109
+ if (result.size == 0) || (result.size == 1)
110
+ prefix = result[0] if result.size == 1
111
+ file_options = Hash.new
112
+ file_options[:prefix] = prefix
113
+ file_options["max-keys"] = 100
114
+ short_keys = Hash.new
115
+ all_keys = []
116
+ begin
117
+ @s3.interface.incrementally_list_bucket(bucket, file_options) do |page|
118
+ entries = page[:contents]
119
+ entries.each do |entry|
120
+ key = entry[:key]
121
+ pos = prefix.length-1
122
+ pos += 1 while pos+1 < key.length && key[pos+1].chr == delimiter
123
+ short_key = key[0..pos]
124
+ short_keys[short_key] = key
125
+ all_keys << key
126
+ end
127
+ end
128
+ rescue => e
129
+ debug "exception #{e}"
130
+ result = []
131
+ end
132
+ result = @dirs_only ? short_keys.keys.sort : all_keys
133
+ debug "result2 #{result.inspect}"
134
+ end
135
+
136
+ debug "final #{result.inspect}"
137
+
138
+ print_keys(bucket, result)
139
+ else
140
+ # complete bucket name
141
+ bucket ||= url
142
+ begin
143
+ buckets = @s3.interface.list_all_my_buckets()
144
+ bucket_names = buckets.map { |b| b[:name] }
145
+ matching = bucket_names.select { |b| b =~ /^#{bucket}/ }
146
+ print_buckets(matching)
147
+ rescue => e
148
+ debug "exception #{e}"
149
+ result = []
150
+ end
151
+ end
152
+
153
+ @debug.close() if DEBUG
154
+
@@ -13,10 +13,10 @@ require 's3cp/utils'
13
13
  options = {}
14
14
  options[:verbose] = $stdout.isatty ? true : false
15
15
  options[:headers] = []
16
- options[:overwrite] = true
17
- options[:checksum] = true
18
- options[:retries] = 5
19
- options[:retry_delay] = 1
16
+ options[:overwrite] = ENV["S3CP_RETRIES"] ? (ENV["S3CP_OVERWRITE"] =~ /y|yes|true|1|^\s*$/i ? true : false) : true
17
+ options[:checksum] = ENV["S3CP_CHECKSUM"] ? (ENV["S3CP_CHECKSUM"] =~ /y|yes|true|1|^\s*$/i ? true : false) : true
18
+ options[:retries] = ENV["S3CP_RETRIES"] ? ENV["S3CP_RETRIES"].to_i : 5
19
+ options[:retry_delay] = ENV["S3CP_RETRY_DELAY"] ? ENV["S3CP_RETRY_DELAY"].to_i : 1
20
20
 
21
21
  op = OptionParser.new do |opts|
22
22
  opts.banner = <<-BANNER
@@ -50,7 +50,7 @@ op = OptionParser.new do |opts|
50
50
  end
51
51
 
52
52
  opts.on("--max-attempts N", "Number of attempts to upload/download until checksum matches (default #{options[:retries]})") do |attempts|
53
- options[:max_attempts] = attempts.to_i
53
+ options[:retries] = attempts.to_i
54
54
  end
55
55
 
56
56
  opts.on("--retry-delay SECONDS", "Time to wait (in seconds) between retries (default #{options[:retry_delay]})") do |delay|
@@ -189,10 +189,13 @@ def local_to_s3(bucket_to, key, file, options = {})
189
189
  end
190
190
  retries = 0
191
191
  begin
192
- if retries == options[:max_attempts]
193
- fail "Unable to upload to s3://#{bucket_from}/#{key_from} after #{retries} attempts."
192
+ if retries == options[:retries]
193
+ fail "Unable to upload to s3://#{bucket_to}/#{key} after #{retries} attempts."
194
+ end
195
+ if retries > 0
196
+ STDERR.puts "Warning: failed checksum for s3://#{bucket_to}/#{bucket_to}. Retrying #{options[:retries] - retries} more time(s)."
197
+ sleep options[:retry_delay]
194
198
  end
195
- sleep options[:retry_delay] if retries > 0
196
199
 
197
200
  f = File.open(file)
198
201
  begin
@@ -202,6 +205,10 @@ def local_to_s3(bucket_to, key, file, options = {})
202
205
  metadata = @s3.interface.head(bucket_to, key)
203
206
  actual_md5 = metadata["etag"] or fail "Unable to get etag/md5 for #{bucket_to}:#{key}"
204
207
  actual_md5 = actual_md5.sub(/^"/, "").sub(/"$/, "") # strip beginning and trailing quotes
208
+ if actual_md5 =~ /-/
209
+ STDERR.puts "Warning: invalid MD5 checksum in metadata; skipped checksum verification."
210
+ actual_md5 = nil
211
+ end
205
212
  end
206
213
  rescue => e
207
214
  raise e unless options[:checksum]
@@ -210,18 +217,21 @@ def local_to_s3(bucket_to, key, file, options = {})
210
217
  f.close()
211
218
  end
212
219
  retries += 1
213
- end until options[:checksum] == false || expected_md5 == actual_md5
220
+ end until options[:checksum] == false || actual_md5.nil? || expected_md5 == actual_md5
214
221
  end
215
222
 
216
223
  def s3_to_local(bucket_from, key_from, dest, options = {})
217
224
  log("Copy s3://#{bucket_from}/#{key_from} to #{dest}")
218
225
  retries = 0
219
226
  begin
220
- if retries == options[:max_attempts]
227
+ if retries == options[:retries]
221
228
  File.delete(dest) if File.exist?(dest)
222
229
  fail "Unable to download s3://#{bucket_from}/#{key_from} after #{retries} attempts."
223
230
  end
224
- sleep options[:retry_delay] if retries > 0
231
+ if retries > 0
232
+ STDERR.puts "Warning: failed checksum for s3://#{bucket_from}/#{key_from}. Retrying #{options[:retries] - retries} more time(s)."
233
+ sleep options[:retry_delay]
234
+ end
225
235
 
226
236
  f = File.new(dest, "wb")
227
237
  begin
@@ -229,6 +239,10 @@ def s3_to_local(bucket_from, key_from, dest, options = {})
229
239
  metadata = @s3.interface.head(bucket_from, key_from)
230
240
  expected_md5 = metadata["etag"] or fail "Unable to get etag/md5 for #{bucket_from}:#{key_from}"
231
241
  expected_md5 = expected_md5.sub(/^"/, "").sub(/"$/, "") # strip beginning and trailing quotes
242
+ if expected_md5 =~ /-/
243
+ STDERR.puts "Warning: invalid MD5 checksum in metadata; skipped checksum verification."
244
+ expected_md5 = nil
245
+ end
232
246
  end
233
247
  @s3.interface.get(bucket_from, key_from) do |chunk|
234
248
  f.write(chunk)
@@ -240,7 +254,7 @@ def s3_to_local(bucket_from, key_from, dest, options = {})
240
254
  f.close()
241
255
  end
242
256
  retries += 1
243
- end until options[:checksum] == false || md5(dest) == expected_md5
257
+ end until options[:checksum] == false || expected_md5.nil? || md5(dest) == expected_md5
244
258
  end
245
259
 
246
260
  def s3_exist?(bucket, key)
@@ -0,0 +1,37 @@
1
+ require 'rubygems'
2
+ require 'extensions/kernel' if RUBY_VERSION =~ /1.8/
3
+ require 'right_aws'
4
+ require 'optparse'
5
+ require 's3cp/utils'
6
+
7
+ op = OptionParser.new do |opts|
8
+ opts.banner = "s3stat [path]"
9
+
10
+ opts.on_tail("-h", "--help", "Show this message") do
11
+ puts op
12
+ exit
13
+ end
14
+ end
15
+
16
+ op.parse!(ARGV)
17
+
18
+ if ARGV.size < 1
19
+ puts op
20
+ exit
21
+ end
22
+
23
+ source = ARGV[0]
24
+ permission = ARGV.last
25
+
26
+ @s3 = S3CP.connect()
27
+
28
+ def get_metadata(bucket, key)
29
+ metadata = @s3.interface.head(bucket, key)
30
+ metadata.sort.each do |k,v|
31
+ puts "#{"%20s" % k} #{v}"
32
+ end
33
+ end
34
+
35
+ bucket,key = S3CP.bucket_and_key(source)
36
+ get_metadata(bucket, key)
37
+
@@ -1,4 +1,4 @@
1
1
 
2
2
  module S3CP
3
- VERSION = "0.1.15"
3
+ VERSION = "0.2.0"
4
4
  end
metadata CHANGED
@@ -1,13 +1,13 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: s3cp
3
3
  version: !ruby/object:Gem::Version
4
- hash: 5
4
+ hash: 23
5
5
  prerelease:
6
6
  segments:
7
7
  - 0
8
- - 1
9
- - 15
10
- version: 0.1.15
8
+ - 2
9
+ - 0
10
+ version: 0.2.0
11
11
  platform: ruby
12
12
  authors:
13
13
  - Alex Boisvert
@@ -15,7 +15,7 @@ autorequire:
15
15
  bindir: bin
16
16
  cert_chain: []
17
17
 
18
- date: 2012-02-18 00:00:00 Z
18
+ date: 2012-02-21 00:00:00 Z
19
19
  dependencies:
20
20
  - !ruby/object:Gem::Dependency
21
21
  prerelease: false
@@ -116,11 +116,14 @@ description:
116
116
  email:
117
117
  - alex.boisvert@gmail.com
118
118
  executables:
119
- - s3ls
120
- - s3cp
121
119
  - s3cat
120
+ - s3cp
121
+ - s3cp_complete
122
+ - s3dir
123
+ - s3ls
122
124
  - s3mod
123
125
  - s3rm
126
+ - s3stat
124
127
  extensions: []
125
128
 
126
129
  extra_rdoc_files:
@@ -130,17 +133,22 @@ files:
130
133
  - lib/s3cp/s3rm.rb
131
134
  - lib/s3cp/s3ls.rb
132
135
  - lib/s3cp/version.rb
136
+ - lib/s3cp/s3stat.rb
133
137
  - lib/s3cp/s3cp.rb
134
138
  - lib/s3cp/utils.rb
135
139
  - lib/s3cp/s3mod.rb
136
140
  - lib/s3cp/s3cat.rb
141
+ - lib/s3cp/completion.rb
137
142
  - History.txt
138
143
  - README.md
139
144
  - bin/s3mod
140
145
  - bin/s3ls
141
146
  - bin/s3cp
147
+ - bin/s3cp_complete
148
+ - bin/s3stat
142
149
  - bin/s3cat
143
150
  - bin/s3rm
151
+ - bin/s3dir
144
152
  homepage:
145
153
  licenses: []
146
154