s3cp 0.1.15 → 0.2.0

data/History.txt

@@ -1,4 +1,21 @@
-=== 0.1.16 / (Pending)
+=== 0.2.1 / (Pending)
+
+=== 0.2.0 / (2012-02-20)
+
+* Added: s3stat command to display S3 object properties
+* Added: s3dir as a shortcut for "s3ls --delimiter / ..."
+         (default to character in S3CP_DELIMITER environment variable or "/" if not defined)
+* Added: s3cp defaults can now be set using environment variables
+         S3CP_OVERWRITE, S3CP_CHECKSUM, S3CP_RETRIES, S3CP_RETRY_DELAY
+* Added: Support for Bash command-line completion of S3 URLs (see below).
+* Fixed: Skip checksum verification for S3 objects with invalid MD5's
+         (details @ https://forums.aws.amazon.com/message.jspa?messageID=234538)
+
+To install Bash completion for S3 URLs, add the following to ~/.bashrc:
+
+  for cmd in [ s3cat s3cp s3dir s3ls s3mod s3rm s3stat ]; do
+    complete -C s3cp_complete $cmd
+  done
 
 === 0.1.15 / (2012-02-17)
 

data/bin/s3cp_complete

@@ -0,0 +1,10 @@
+#!/usr/bin/env ruby
+#
+# A script to allow Bash to complete an s3cp command-line.
+#
+# To install for Bash 2.0 or better, add the following to ~/.bashrc:
+#
+#     $ for cmd in [ s3cat s3cp s3dir s3ls s3mod s3rm s3stat ]; do complete -C s3cp_complete $cmd; done
+#
+require 's3cp/completion'
+

data/bin/s3dir

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+ARGV.unshift "--delimiter", ENV["S3CP_DELIMITER"] || "/"
+require 's3cp/s3ls'
+

data/bin/s3stat

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+require 's3cp/s3stat'
+
+

data/lib/s3cp/completion.rb

@@ -0,0 +1,154 @@
+require 'rubygems'
+require 'extensions/kernel' if RUBY_VERSION =~ /1.8/
+require 'right_aws'
+require 'optparse'
+require 'date'
+require 'highline/import'
+require 'tempfile'
+
+require 's3cp/utils'
+
+cmd_line = ENV['COMP_LINE']
+position = ENV['COMP_POINT'].to_i
+
+url = begin
+  start = position
+  start -= 1 while start >= 1 && cmd_line[start-1].chr != ' '
+  cmd_line[start..position-1]
+end
+
+cmd = ARGV[0]
+arg1 = ARGV[1]
+arg2  = ARGV[2]
+
+DEBUG = true
+
+@s3 = S3CP.connect()
+
+if DEBUG
+  @debug = File.open("/tmp/s3cp_complete", "wb")
+end
+
+def debug(str)
+  @debug.puts(str) if @debug
+end
+
+debug "url #{url}"
+
+delimiter = ENV["S3CP_DELIMITER"] || "/"
+debug "delimiter #{delimiter}"
+
+bucket, prefix = S3CP.bucket_and_key(url)
+if bucket == "s3"
+  bucket = ""
+  prefix = nil
+end
+
+debug "bucket #{bucket.inspect} prefix #{prefix.inspect}"
+
+@legacy_format = (url =~ /\:\//) ? false : true
+debug "legacy_format #{@legacy_format} "
+
+recursive = cmd_line.split(" ").include?("-r")
+debug "recursive #{recursive}"
+
+@dirs_only = (cmd =~ /s3dir$/) || (cmd =~ /s3cp$/ && recursive) || (cmd =~ /s3rm$/ && recursive)
+debug "dirs_only #{@dirs_only}"
+
+def print_keys(bucket, keys)
+  keys.each do |key|
+    if @legacy_format
+      debug key
+      puts key
+    else
+      debug "//#{bucket}/#{key}"
+      puts "//#{bucket}/#{key}"
+    end
+  end
+end
+
+def print_buckets(buckets)
+  buckets = buckets.map { |b| @legacy_format ? b + ":" : b + "/"  }
+  buckets << buckets[0] + " " if buckets.size == 1
+  buckets.each do |bucket|
+    if @legacy_format
+      debug bucket
+      puts bucket
+    else
+      debug "//#{bucket}"
+      puts "//#{bucket}"
+    end
+  end
+end
+
+if (prefix && prefix.length > 0) || (url =~ /s3\:\/\/[^\/]+\//) || (url =~ /\:$/)
+  # complete s3 key name
+  bucket, prefix = S3CP.bucket_and_key(url)
+  fail "Your URL looks funny, doesn't it?" unless bucket
+
+  result = nil
+
+  # try directory first
+  dir_options = Hash.new
+  dir_options[:prefix] = prefix
+  dir_options[:delimiter] = delimiter
+  begin
+    @s3.interface.incrementally_list_bucket(bucket, dir_options) do |page|
+      entries = page[:common_prefixes]
+      entries << page[:contents][0][:key] if page[:contents].length > 0 && entries.length > 0
+      result = entries
+    end
+  rescue => e
+    debug "exception #{e}"
+    result = []
+  end
+
+  debug "result1 #{result.inspect}"
+
+  # there may be longer matches
+  if (result.size == 0) || (result.size == 1)
+    prefix = result[0] if result.size == 1
+    file_options = Hash.new
+    file_options[:prefix] = prefix
+    file_options["max-keys"] = 100
+    short_keys = Hash.new
+    all_keys = []
+    begin
+      @s3.interface.incrementally_list_bucket(bucket, file_options) do |page|
+        entries = page[:contents]
+        entries.each do |entry|
+          key = entry[:key]
+          pos = prefix.length-1
+          pos += 1 while pos+1 < key.length && key[pos+1].chr == delimiter
+          short_key = key[0..pos]
+          short_keys[short_key] = key
+          all_keys << key
+        end
+      end
+    rescue => e
+      debug "exception #{e}"
+      result = []
+    end
+    result = @dirs_only ? short_keys.keys.sort : all_keys
+    debug "result2 #{result.inspect}"
+  end
+
+  debug "final #{result.inspect}"
+
+  print_keys(bucket, result)
+else
+  # complete bucket name
+  bucket ||= url
+  begin
+    buckets = @s3.interface.list_all_my_buckets()
+    bucket_names = buckets.map { |b| b[:name] }
+    matching = bucket_names.select { |b| b =~ /^#{bucket}/ }
+    print_buckets(matching)
+  rescue => e
+    debug "exception #{e}"
+    result = []
+  end
+end
+
+@debug.close() if DEBUG
+

data/lib/s3cp/s3cp.rb

@@ -13,10 +13,10 @@ require 's3cp/utils'
 options = {}
 options[:verbose] = $stdout.isatty ? true : false
 options[:headers] = []
-options[:overwrite] = true
-options[:checksum] = true
-options[:retries] = 5
-options[:retry_delay] = 1
+options[:overwrite]   = ENV["S3CP_RETRIES"]     ? (ENV["S3CP_OVERWRITE"] =~ /y|yes|true|1|^\s*$/i ? true : false) : true
+options[:checksum]    = ENV["S3CP_CHECKSUM"]    ? (ENV["S3CP_CHECKSUM"]  =~ /y|yes|true|1|^\s*$/i ? true : false) : true
+options[:retries]     = ENV["S3CP_RETRIES"]     ? ENV["S3CP_RETRIES"].to_i     : 5
+options[:retry_delay] = ENV["S3CP_RETRY_DELAY"] ? ENV["S3CP_RETRY_DELAY"].to_i : 1
 
 op = OptionParser.new do |opts|
   opts.banner = <<-BANNER
@@ -50,7 +50,7 @@ op = OptionParser.new do |opts|
   end
 
   opts.on("--max-attempts N", "Number of attempts to upload/download until checksum matches (default #{options[:retries]})") do |attempts|
-    options[:max_attempts] = attempts.to_i
+    options[:retries] = attempts.to_i
   end
 
   opts.on("--retry-delay SECONDS", "Time to wait (in seconds) between retries (default #{options[:retry_delay]})") do |delay|
@@ -189,10 +189,13 @@ def local_to_s3(bucket_to, key, file, options = {})
   end
   retries = 0
   begin
-    if retries == options[:max_attempts]
-      fail "Unable to upload to s3://#{bucket_from}/#{key_from} after #{retries} attempts."
+    if retries == options[:retries]
+      fail "Unable to upload to s3://#{bucket_to}/#{key} after #{retries} attempts."
+    end
+    if retries > 0
+      STDERR.puts "Warning: failed checksum for s3://#{bucket_to}/#{bucket_to}. Retrying #{options[:retries] - retries} more time(s)."
+      sleep options[:retry_delay]
     end
-    sleep options[:retry_delay] if retries > 0
 
     f = File.open(file)
     begin
@@ -202,6 +205,10 @@ def local_to_s3(bucket_to, key, file, options = {})
         metadata = @s3.interface.head(bucket_to, key)
         actual_md5 = metadata["etag"] or fail "Unable to get etag/md5 for #{bucket_to}:#{key}"
         actual_md5 = actual_md5.sub(/^"/, "").sub(/"$/, "") # strip beginning and trailing quotes
+        if actual_md5 =~ /-/
+          STDERR.puts "Warning: invalid MD5 checksum in metadata; skipped checksum verification."
+          actual_md5 = nil
+        end
       end
     rescue => e
       raise e unless options[:checksum]
@@ -210,18 +217,21 @@ def local_to_s3(bucket_to, key, file, options = {})
       f.close()
     end
     retries += 1
-  end until options[:checksum] == false || expected_md5 == actual_md5
+    end until options[:checksum] == false || actual_md5.nil? || expected_md5 == actual_md5
 end
 
 def s3_to_local(bucket_from, key_from, dest, options = {})
   log("Copy s3://#{bucket_from}/#{key_from} to #{dest}")
   retries = 0
   begin
-    if retries == options[:max_attempts]
+    if retries == options[:retries]
       File.delete(dest) if File.exist?(dest)
       fail "Unable to download s3://#{bucket_from}/#{key_from} after #{retries} attempts."
     end
-    sleep options[:retry_delay] if retries > 0
+    if retries > 0
+      STDERR.puts "Warning: failed checksum for s3://#{bucket_from}/#{key_from}. Retrying #{options[:retries] - retries} more time(s)."
+      sleep options[:retry_delay]
+    end
 
     f = File.new(dest, "wb")
     begin
@@ -229,6 +239,10 @@ def s3_to_local(bucket_from, key_from, dest, options = {})
         metadata = @s3.interface.head(bucket_from, key_from)
         expected_md5 = metadata["etag"] or fail "Unable to get etag/md5 for #{bucket_from}:#{key_from}"
         expected_md5 = expected_md5.sub(/^"/, "").sub(/"$/, "") # strip beginning and trailing quotes
+        if expected_md5 =~ /-/
+          STDERR.puts "Warning: invalid MD5 checksum in metadata; skipped checksum verification."
+          expected_md5 = nil
+        end
       end
       @s3.interface.get(bucket_from, key_from) do |chunk|
         f.write(chunk)
@@ -240,7 +254,7 @@ def s3_to_local(bucket_from, key_from, dest, options = {})
       f.close()
     end
     retries += 1
-  end until options[:checksum] == false || md5(dest) == expected_md5
+  end until options[:checksum] == false || expected_md5.nil? || md5(dest) == expected_md5
 end
 
 def s3_exist?(bucket, key)

data/lib/s3cp/s3stat.rb

@@ -0,0 +1,37 @@
+require 'rubygems'
+require 'extensions/kernel' if RUBY_VERSION =~ /1.8/
+require 'right_aws'
+require 'optparse'
+require 's3cp/utils'
+
+op = OptionParser.new do |opts|
+  opts.banner = "s3stat [path]"
+
+  opts.on_tail("-h", "--help", "Show this message") do
+    puts op
+    exit
+  end
+end
+
+op.parse!(ARGV)
+
+if ARGV.size < 1
+  puts op
+  exit
+end
+
+source  = ARGV[0]
+permission = ARGV.last
+
+@s3 = S3CP.connect()
+
+def get_metadata(bucket, key)
+  metadata = @s3.interface.head(bucket, key)
+  metadata.sort.each do |k,v|
+    puts "#{"%20s" % k} #{v}"
+  end
+end
+
+bucket,key = S3CP.bucket_and_key(source)
+get_metadata(bucket, key)
+

data/lib/s3cp/version.rb

@@ -1,4 +1,4 @@
 
 module S3CP
-  VERSION = "0.1.15"
+  VERSION = "0.2.0"
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: s3cp
 version: !ruby/object:Gem::Version 
-  hash: 5
+  hash: 23
   prerelease: 
   segments: 
   - 0
-  - 1
-  - 15
-  version: 0.1.15
+  - 2
+  - 0
+  version: 0.2.0
 platform: ruby
 authors: 
 - Alex Boisvert
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-02-18 00:00:00 Z
+date: 2012-02-21 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   prerelease: false
@@ -116,11 +116,14 @@ description:
 email: 
 - alex.boisvert@gmail.com
 executables: 
-- s3ls
-- s3cp
 - s3cat
+- s3cp
+- s3cp_complete
+- s3dir
+- s3ls
 - s3mod
 - s3rm
+- s3stat
 extensions: []
 
 extra_rdoc_files: 
@@ -130,17 +133,22 @@ files:
 - lib/s3cp/s3rm.rb
 - lib/s3cp/s3ls.rb
 - lib/s3cp/version.rb
+- lib/s3cp/s3stat.rb
 - lib/s3cp/s3cp.rb
 - lib/s3cp/utils.rb
 - lib/s3cp/s3mod.rb
 - lib/s3cp/s3cat.rb
+- lib/s3cp/completion.rb
 - History.txt
 - README.md
 - bin/s3mod
 - bin/s3ls
 - bin/s3cp
+- bin/s3cp_complete
+- bin/s3stat
 - bin/s3cat
 - bin/s3rm
+- bin/s3dir
 homepage: 
 licenses: []