s3cp 1.0.2 → 1.0.3

data/History.txt

@@ -1,3 +1,21 @@
+=== 1.0.3 / (2012-05-11)
+
+* Added:    Exponential backoff for `s3cp` command based on the formula:
+
+              delay = initial_retry_delay * (factor ^ retries)
+
+            where:
+                    - `initial_retry_delay` is 1 second by default.
+                    - `factor` is 1.4142 by default
+                      (retry delay doubles every two retries)
+                    - max. number of retries is now 20 by default.
+
+            (which means s3cp retries for roughly 58 minutes by default)
+
+* Fixed:  `s3ls` and `s3du` now properly handle Errno::EPIPE
+
+* Fixed:  `s3du` would not display last character of files
+
 === 1.0.2 / 2012-03-9
 
 * Added:  `s3up` now outputs uploaded file size to STDERR, e.g.,

data/lib/s3cp/s3cp.rb

@@ -31,10 +31,12 @@ require 's3cp/utils'
 options = {}
 options[:verbose] = $stdout.isatty ? true : false
 options[:headers] = []
-options[:overwrite]   = ENV["S3CP_RETRIES"]     ? (ENV["S3CP_OVERWRITE"] =~ /y|yes|true|1|^\s*$/i ? true : false) : true
+options[:overwrite]   = ENV["S3CP_OVERWRITE"]   ? (ENV["S3CP_OVERWRITE"] =~ /y|yes|true|1|^\s*$/i ? true : false) : true
 options[:checksum]    = ENV["S3CP_CHECKSUM"]    ? (ENV["S3CP_CHECKSUM"]  =~ /y|yes|true|1|^\s*$/i ? true : false) : true
-options[:retries]     = ENV["S3CP_RETRIES"]     ? ENV["S3CP_RETRIES"].to_i     : 5
+options[:retries]     = ENV["S3CP_RETRIES"]     ? ENV["S3CP_RETRIES"].to_i     : 18
 options[:retry_delay] = ENV["S3CP_RETRY_DELAY"] ? ENV["S3CP_RETRY_DELAY"].to_i : 1
+options[:retry_backoff] = ENV["S3CP_BACKOFF"]   ? ENV["S3CP_BACKOFF"].to_f     : 1.4142 # double every 2 tries
+
 options[:include_regex] = []
 options[:exclude_regex] = []
 options[:sync] = false
@@ -82,6 +84,10 @@ op = OptionParser.new do |opts|
     options[:retry_delay] = delay.to_i
   end
 
+  opts.on("--retry-backoff FACTOR", "Exponential backoff factor (default #{options[:retry_backoff]})") do |factor|
+    options[:retry_backoff] = factor.to_f
+  end
+
   opts.on("--no-checksum", "Disable checksum checking") do
     options[:checksum] = false
   end
@@ -266,8 +272,9 @@ def local_to_s3(bucket_to, key, file, options = {})
         fail "Unable to upload to s3://#{bucket_to}/#{key} after #{retries} attempts."
       end
       if retries > 0
-        STDERR.puts "Warning: failed checksum for s3://#{bucket_to}/#{bucket_to}. Retrying #{options[:retries] - retries} more time(s)."
-        sleep options[:retry_delay]
+        delay = options[:retry_delay] * (options[:retry_backoff] ** retries)
+        STDERR.puts "Sleeping #{delay} seconds.  Will retry #{options[:retries] - retries} more time(s)."
+        sleep delay
       end
 
       f = File.open(file)
@@ -285,7 +292,7 @@ def local_to_s3(bucket_to, key, file, options = {})
                 @progress_bar.inc result.length if result
                 result
               rescue => e
-                puts e
+                STDERR.puts e
                 raise e
               end
             end
@@ -300,8 +307,12 @@ def local_to_s3(bucket_to, key, file, options = {})
 
         if options[:checksum]
           actual_md5 = s3_checksum(bucket_to, key)
-          unless actual_md5.is_a? String
-            STDERR.puts "Warning: invalid MD5 checksum in metadata; skipped checksum verification."
+          if actual_md5.is_a? String
+            if actual_md5 != expected_md5
+              STDERR.puts "Warning: invalid MD5 checksum.  Expected: #{expected_md5} Actual: #{actual_md5}"
+            end
+          else
+            STDERR.puts "Warning: invalid MD5 checksum in metadata: #{actual_md5.inspect}; skipped checksum verification."
             actual_md5 = nil
           end
         end
@@ -328,8 +339,9 @@ def s3_to_local(bucket_from, key_from, dest, options = {})
       fail "Unable to download s3://#{bucket_from}/#{key_from} after #{retries} attempts."
     end
     if retries > 0
-      STDERR.puts "Warning: failed checksum for s3://#{bucket_from}/#{key_from}. Retrying #{options[:retries] - retries} more time(s)."
-      sleep options[:retry_delay]
+      delay = options[:retry_delay] * (options[:retry_backoff] ** retries)
+      STDERR.puts "Sleeping #{delay} seconds.  Will retry #{options[:retries] - retries} more time(s)."
+      sleep delay
     end
     begin
       expected_md5 = if options[:checksum] || options[:sync]
@@ -377,6 +389,14 @@ def s3_to_local(bucket_from, key_from, dest, options = {})
       raise e unless options[:checksum]
       STDERR.puts e
     end
+
+    if options[:checksum] && expected_md5 != nil
+      actual_md5 = md5(dest)
+      if actual_md5 != expected_md5
+        STDERR.puts "Warning: invalid MD5 checksum.  Expected: #{expected_md5} Actual: #{actual_md5}"
+      end
+    end
+
     retries += 1
   end until options[:checksum] == false || expected_md5.nil? || md5(dest) == expected_md5
 end

data/lib/s3cp/s3du.rb

@@ -102,39 +102,43 @@ def print(key, size)
   puts ("%#{7 + @options[:precision]}s " % size) + key
 end
 
-@s3.interface.incrementally_list_bucket(@bucket, s3_options) do |page|
-
-  entries = page[:contents]
-  entries.each do |entry|
-    key  = entry[:key]
-    size = entry[:size]
-
-    if options[:regex].nil? || options[:regex].match(key)
-      current_key = if actual_depth
-        pos = nth_occurrence(key, "/", actual_depth)
-        pos ? key[0..pos-1] : prefix
+begin
+  @s3.interface.incrementally_list_bucket(@bucket, s3_options) do |page|
+
+    entries = page[:contents]
+    entries.each do |entry|
+      key  = entry[:key]
+      size = entry[:size]
+
+      if options[:regex].nil? || options[:regex].match(key)
+        current_key = if actual_depth
+          pos = nth_occurrence(key, "/", actual_depth)
+          (pos != -1) ? key[0..pos-1] : key
+        end
+
+        if (last_key && last_key != current_key)
+          print(last_key, subtotal_size)
+          subtotal_size = size
+        else
+          subtotal_size += size
+        end
+
+        last_key = current_key
+        total_size += size
       end
-
-      if (last_key && last_key != current_key)
-        print(last_key, subtotal_size)
-        subtotal_size = size
-      else
-        subtotal_size += size
-      end
-
-      last_key = current_key
-      total_size += size
     end
   end
-end
 
-if last_key != nil
-  print(last_key, subtotal_size)
-end
+  if last_key != nil
+    print(last_key, subtotal_size)
+  end
 
-if options[:depth] > 0
-  print("", total_size)
-else
-  puts S3CP.format_filesize(total_size, :unit => options[:unit], :precision => options[:precision])
+  if options[:depth] > 0
+    print("", total_size)
+  else
+    puts S3CP.format_filesize(total_size, :unit => options[:unit], :precision => options[:precision])
+  end
+rescue Errno::EPIPE
+  # ignore
 end
 

data/lib/s3cp/s3ls.rb

@@ -103,39 +103,43 @@ s3_options[:prefix] = @key
 s3_options["max-keys"] = options[:max_keys] if options[:max_keys] && !options[:delimiter]
 s3_options[:delimiter] = options[:delimiter] if options[:delimiter]
 
-@s3.interface.incrementally_list_bucket(@bucket, s3_options) do |page|
-  entries = []
-  if options[:delimiter]
-    entries << { :key => page[:contents][0][:key] } if page[:contents].length > 0 && entries.length > 0
-    page[:common_prefixes].each do |entry|
-      entries << { :key => entry }
+begin
+  @s3.interface.incrementally_list_bucket(@bucket, s3_options) do |page|
+    entries = []
+    if options[:delimiter]
+      entries << { :key => page[:contents][0][:key] } if page[:contents].length > 0 && entries.length > 0
+      page[:common_prefixes].each do |entry|
+        entries << { :key => entry }
+      end
+      entries << { :key => nil }
     end
-    entries << { :key => nil }
-  end
-  entries += page[:contents]
-  entries.each do |entry|
-    key = entry[:key] ? "s3://#{@bucket}/#{entry[:key]}" : "---"
-    if options[:long_format] && entry[:last_modified] && entry[:size]
-      last_modified = DateTime.parse(entry[:last_modified])
-      size = entry[:size]
-      size = S3CP.format_filesize(size, :unit => options[:unit], :precision => options[:precision])
-      size = ("%#{7 + options[:precision]}s " % size)
-      puts "#{last_modified.strftime(options[:date_format])} #{size} #{key}"
-    else
-      puts key
-    end
-    rows += 1
-    keys += 1
-    if options[:max_keys] && keys >= options[:max_keys]
-      exit
-    end
-    if options[:rows_per_page] && (rows % options[:rows_per_page] == 0)
-      begin
-        print "Continue? (Y/n) "
-        response = STDIN.gets.chomp.downcase
-      end until response == 'n' || response == 'y' || response == ''
-      exit if response == 'n'
+    entries += page[:contents]
+    entries.each do |entry|
+      key = entry[:key] ? "s3://#{@bucket}/#{entry[:key]}" : "---"
+      if options[:long_format] && entry[:last_modified] && entry[:size]
+        last_modified = DateTime.parse(entry[:last_modified])
+        size = entry[:size]
+        size = S3CP.format_filesize(size, :unit => options[:unit], :precision => options[:precision])
+        size = ("%#{7 + options[:precision]}s " % size)
+        puts "#{last_modified.strftime(options[:date_format])} #{size} #{key}"
+      else
+        puts key
+      end
+      rows += 1
+      keys += 1
+      if options[:max_keys] && keys >= options[:max_keys]
+        exit
+      end
+      if options[:rows_per_page] && (rows % options[:rows_per_page] == 0)
+        begin
+          print "Continue? (Y/n) "
+          response = STDIN.gets.chomp.downcase
+        end until response == 'n' || response == 'y' || response == ''
+        exit if response == 'n'
+      end
     end
   end
+rescue Errno::EPIPE
+  # ignore
 end
 

data/lib/s3cp/version.rb

@@ -16,5 +16,5 @@
 # the License.
 
 module S3CP
-  VERSION = "1.0.2"
+  VERSION = "1.0.3"
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: s3cp
 version: !ruby/object:Gem::Version 
-  hash: 19
+  hash: 17
   prerelease: 
   segments: 
   - 1
   - 0
-  - 2
-  version: 1.0.2
+  - 3
+  version: 1.0.3
 platform: ruby
 authors: 
 - Alex Boisvert
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-03-09 00:00:00 Z
+date: 2012-05-11 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   prerelease: false