s3cp 1.1.21 → 1.1.22

data/History.txt

@@ -1,3 +1,33 @@
+=== 1.1.22 (2013-03-07)
+
+* Changed: s3cp now adds a new "md5" custom metadata to any uploaded file
+           using multi-part protocol with --checksum option to allow for
+           proper content integrity checking and synchronization (--sync)
+           on (large / multi-part) files since the etag metadata is only a
+           valid MD5 value for files uploaded as a single part.
+
+           As a reminder, --checksum is on by default and multi-part upload
+           kicks in by default for files > 16MB.  These defaults are
+           configurable through the command-line, environment variables or
+           $HOME/.s3cp.
+
+           On the command line,
+
+             # Turn things off
+             % export S3CP_CHECKSUM=false
+             % export S3CP_MULTIPART=false
+
+             # Configure multipart threshold for files > 64MB
+             % export S3CP_MULTIPART=67108864
+
+           In your in $HOME/.s3cp,
+
+             ENV["S3CP_MULTIPART"] = "false"
+
+             # Use multi-part upload only for files > 128MB
+             AWS.config.s3_multipart_threshold = (128 * 1024 * 1024)
+
+
 === 1.1.21 (2013-03-05)
 
 * Fixed:  s3cp --sync would not correctly copy a file from S3 to local file

data/lib/s3cp/s3cp.rb

@@ -27,7 +27,14 @@ options[:checksum]    = ENV["S3CP_CHECKSUM"]    ? (ENV["S3CP_CHECKSUM"]  =~ /y|y
 options[:retries]     = ENV["S3CP_RETRIES"]     ? ENV["S3CP_RETRIES"].to_i     : 18
 options[:retry_delay] = ENV["S3CP_RETRY_DELAY"] ? ENV["S3CP_RETRY_DELAY"].to_i : 1
 options[:retry_backoff] = ENV["S3CP_BACKOFF"]   ? ENV["S3CP_BACKOFF"].to_f     : 1.4142 # double every 2 tries
-
+options[:multipart] = case ENV["S3CP_MULTIPART"]
+  when /n|no|false/i
+    false
+  when /\d+/
+    ENV["S3CP_MULTIPART"].to_i
+  else
+    true
+  end
 options[:include_regex] = []
 options[:exclude_regex] = []
 options[:sync] = false
@@ -275,7 +282,7 @@ def local_to_s3(bucket_to, key, file, options = {})
     when :not_found
       nil
     when :invalid
-      STDERR.puts "Warning: invalid MD5 checksum in metadata; file will be force-copied."
+      STDERR.puts "Warning: No MD5 checksum available and ETag not suitable due to multi-part upload; file will be force-copied."
       nil
     else
       md5
@@ -295,17 +302,31 @@ def local_to_s3(bucket_to, key, file, options = {})
       end
 
       begin
+        obj = @s3.buckets[bucket_to].objects[key]
+
         s3_options = {}
         S3CP.set_header_options(s3_options, @headers)
         s3_options[:acl] = options[:acl]
         s3_options[:content_length] = File.size(file)
 
+        multipart_threshold = options[:multipart].is_a?(Fixnum) ?  options[:multipart] : AWS.config.s3_multipart_threshold
+        if (expected_md5 != nil) && (File.size(file) >= multipart_threshold) && options[:multipart]
+          meta = s3_options[:metadata] || {}
+          meta[:md5] = expected_md5
+          s3_options[:metadata] = meta
+        end
+
+        if options[:multipart]
+          s3_options[:multipart_threshold] = multipart_threshold
+        else
+          s3_options[:single_request] = true
+        end
+
         progress_bar = ProgressBar.new(File.basename(file), File.size(file)).tap do |p|
           p.file_transfer_mode
         end
 
         File.open(file) do |io|
-          obj = @s3.buckets[bucket_to].objects[key]
           obj.write(ProxyIO.new(io, progress_bar), s3_options)
         end
 
@@ -427,13 +448,14 @@ def s3_checksum(bucket, key)
     raise e
   end
 
-  md5 = metadata[:etag] or fail "Unable to get etag/md5 for #{bucket_to}:#{key}"
-  return :invalid unless md5
-
-  md5 = md5.sub(/^"/, "").sub(/"$/, "") # strip beginning and trailing quotes
-  return :invalid if md5 =~ /-/
-
-  md5
+  case
+  when metadata[:meta] && metadata[:meta]["md5"]
+    metadata[:meta]["md5"]
+  when metadata[:etag] && metadata[:etag] !~ /-/
+    metadata[:etag].sub(/^"/, "").sub(/"$/, "") # strip beginning and trailing quotes
+  else
+    :invalid
+  end
 end
 
 def key_path(prefix, key)

data/lib/s3cp/version.rb

@@ -16,5 +16,5 @@
 # the License.
 
 module S3CP
-  VERSION = "1.1.21"
+  VERSION = "1.1.22"
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: s3cp
 version: !ruby/object:Gem::Version 
-  hash: 57
+  hash: 63
   prerelease: 
   segments: 
   - 1
   - 1
-  - 21
-  version: 1.1.21
+  - 22
+  version: 1.1.22
 platform: ruby
 authors: 
 - Alex Boisvert
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2013-03-06 00:00:00 Z
+date: 2013-03-08 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   version_requirements: &id001 !ruby/object:Gem::Requirement