s3cp 0.2.3 → 0.2.4

data/History.txt

@@ -1,4 +1,31 @@
-=== 0.2.4 / (Pending)
+=== 0.2.5 / (Pending)
+
+=== 0.2.4 / 2012-02-27
+
+* Fixed: Handling of relative paths in s3cp.
+
+  % s3cp -r bucket:path/to/files /local/path
+
+  will now produce local files:
+
+     /local/path/files/A
+     /local/path/files/B
+     /local/path/files/C
+     /local/path/files2/D
+     /local/path/files2/E
+     ...
+
+  whereas,
+
+  % s3cp -r bucket:path/to/files/ /local/path
+
+
+    /local/path/A
+    /local/path/B
+    /local/path/C
+    ...
+
+  (note the trailing slash on "bucket:path/to/files")
 
 === 0.2.3 / (2012-02-24)
 

data/lib/s3cp/s3cp.rb

@@ -198,8 +198,11 @@ def no_slash(path)
   path.match(/^\//) ? path[1..-1] : path
 end
 
+# relative("path/to/", "path/to/file") => "file"
+# relative("path/to",  "path/to/file") => "to/file"
 def relative(base, path)
-  no_slash(path[base.length..-1])
+  dir = base.rindex("/") ? base[0..base.rindex("/")] : ""
+  no_slash(path[dir.length..-1])
 end
 
 def log(msg)

data/lib/s3cp/version.rb

@@ -16,5 +16,5 @@
 # the License.
 
 module S3CP
-  VERSION = "0.2.3"
+  VERSION = "0.2.4"
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: s3cp
 version: !ruby/object:Gem::Version 
-  hash: 17
+  hash: 31
   prerelease: 
   segments: 
   - 0
   - 2
-  - 3
-  version: 0.2.3
+  - 4
+  version: 0.2.4
 platform: ruby
 authors: 
 - Alex Boisvert
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-02-24 00:00:00 Z
+date: 2012-02-28 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   prerelease: false
@@ -154,7 +154,6 @@ files:
 - lib/s3cp/utils.rb
 - lib/s3cp/s3mod.rb
 - lib/s3cp/s3cat.rb
-- lib/s3cp/#Untitled-7#
 - lib/s3cp/completion.rb
 - History.txt
 - README.md

data/lib/s3cp/#Untitled-7#

@@ -1,288 +0,0 @@
-diff --git a/History.txt b/History.txt
-index 8de2725..37eaec8 100644
---- a/History.txt
-+++ b/History.txt
-@@ -1,5 +1,9 @@
- === 0.2.3 / (Pending)
- 
-+* Added: --include and --exclude REGEX support for s3cp
-+
-+* Added: --sync mode for s3cp
-+
- === 0.2.2 / (2012-02-23)
- 
- * Added: Progress bars during upload/download if $stdout.isatty
-diff --git a/lib/s3cp/s3cp.rb b/lib/s3cp/s3cp.rb
-index 0fabcdd..a1423f5 100644
---- a/lib/s3cp/s3cp.rb
-+++ b/lib/s3cp/s3cp.rb
-@@ -35,6 +35,9 @@ options[:overwrite]   = ENV["S3CP_RETRIES"]     ? (ENV["S3CP_OVERWRITE"] =~ /y|y
- options[:checksum]    = ENV["S3CP_CHECKSUM"]    ? (ENV["S3CP_CHECKSUM"]  =~ /y|yes|true|1|^\s*$/i ? true : false) : true
- options[:retries]     = ENV["S3CP_RETRIES"]     ? ENV["S3CP_RETRIES"].to_i     : 5
- options[:retry_delay] = ENV["S3CP_RETRY_DELAY"] ? ENV["S3CP_RETRY_DELAY"].to_i : 1
-+options[:include_regex] = []
-+options[:exclude_regex] = []
-+options[:sync] = false
- 
- op = OptionParser.new do |opts|
-   opts.banner = <<-BANNER
-@@ -67,6 +70,10 @@ op = OptionParser.new do |opts|
-     options[:overwrite] = false
-   end
- 
-+  opts.on("--sync", "Sync mode: use checksum to determine if files need copying.") do
-+    options[:sync] = true
-+  end
-+
-   opts.on("--max-attempts N", "Number of attempts to upload/download until checksum matches (default #{options[:retries]})") do |attempts|
-     options[:retries] = attempts.to_i
-   end
-@@ -90,6 +97,14 @@ op = OptionParser.new do |opts|
-   opts.separator "               AMZ headers: \'x-amz-acl: public-read\'"
-   opts.separator ""
- 
-+  opts.on("-i REGEX", "--include REGEX", "Copy only files matching the following regular expression.") do |regex|
-+    options[:include_regex] << regex
-+  end
-+
-+  opts.on("-x REGEX", "--exclude REGEX", "Do not copy files matching provided regular expression.") do |regex|
-+    options[:exclude_regex] << regex
-+  end
-+
-   opts.on("--verbose", "Verbose mode") do
-     options[:verbose] = true
-   end
-@@ -110,6 +125,16 @@ if ARGV.size < 2
-   exit
- end
- 
-+if options[:include_regex].any? && !options[:recursive]
-+  puts "-i (--include regex) option requires -r (recursive) option."
-+  exit(1)
-+end
-+
-+if options[:exclude_regex].any? && !options[:recursive]
-+  puts "-x (--exclude regex) option requires -r (recursive) option."
-+  exit(1)
-+end
-+
- destination = ARGV.last
- sources = ARGV[0..-2]
- 
-@@ -144,6 +169,16 @@ end
- @bucket = $1
- @prefix = $2
- 
-+@includes = options[:include_regex].map { |s| Regexp.new(s) }
-+@excludes = options[:exclude_regex].map { |s| Regexp.new(s) }
-+
-+def match(path)
-+  matching = true
-+  return false if @includes.any? && !@includes.any? { |regex| regex.match(path) }
-+  return false if @excludes.any? &&  @excludes.any? { |regex| regex.match(path) }
-+  true
-+end
-+
- @s3 = S3CP.connect()
- 
- def direction(from, to)
-@@ -217,68 +252,90 @@ end
- 
- def local_to_s3(bucket_to, key, file, options = {})
-   log(with_headers("Copy #{file} to s3://#{bucket_to}/#{key}"))
--  if options[:checksum]
--    expected_md5 = md5(file)
-+
-+  expected_md5 = if options[:checksum] || options[:sync]
-+     md5(file)
-   end
--  retries = 0
--  begin
--    if retries == options[:retries]
--      fail "Unable to upload to s3://#{bucket_to}/#{key} after #{retries} attempts."
--    end
--    if retries > 0
--      STDERR.puts "Warning: failed checksum for s3://#{bucket_to}/#{bucket_to}. Retrying #{options[:retries] - retries} more time(s)."
--      sleep options[:retry_delay]
-+
-+  actual_md5 = if options[:sync]
-+    md5 = s3_checksum(bucket_to, key)
-+    case md5
-+    when :not_found
-+      nil
-+    when :invalid
-+      STDERR.puts "Warning: invalid MD5 checksum in metadata; file will be force-copied."
-+      nil
-+    else
-+      md5
-     end
-+  end
- 
--    f = File.open(file)
-+  if actual_md5.nil? || (options[:sync] && expected_md5 != actual_md5)
-+    retries = 0
-     begin
--      if $stdout.isatty
--        f = Proxy.new(f)
--        progress_bar = ProgressBar.new(File.basename(file), File.size(file)).tap do |p|
--          p.file_transfer_mode
--        end
--        class << f
--         attr_accessor :progress_bar
--          def read(length, buffer=nil)
--            begin
--              result = @target.read(length, buffer)
--              @progress_bar.inc result.length if result
--              result
--            rescue => e
--              puts e
--              raise e
-+      if retries == options[:retries]
-+        fail "Unable to upload to s3://#{bucket_to}/#{key} after #{retries} attempts."
-+      end
-+      if retries > 0
-+        STDERR.puts "Warning: failed checksum for s3://#{bucket_to}/#{bucket_to}. Retrying #{options[:retries] - retries} more time(s)."
-+        sleep options[:retry_delay]
-+      end
-+
-+      f = File.open(file)
-+      begin
-+        if $stdout.isatty
-+          f = Proxy.new(f)
-+          progress_bar = ProgressBar.new(File.basename(file), File.size(file)).tap do |p|
-+            p.file_transfer_mode
-+          end
-+          class << f
-+           attr_accessor :progress_bar
-+            def read(length, buffer=nil)
-+              begin
-+                result = @target.read(length, buffer)
-+                @progress_bar.inc result.length if result
-+                result
-+              rescue => e
-+                puts e
-+                raise e
-+              end
-             end
-           end
-+          f.progress_bar = progress_bar
-+        else
-+          progress_bar = nil
-         end
--        f.progress_bar = progress_bar
--      else
--        progress_bar = nil
--      end
- 
--      meta = @s3.interface.put(bucket_to, key, f, @headers)
--      progress_bar.finish if progress_bar
-+        meta = @s3.interface.put(bucket_to, key, f, @headers)
-+        progress_bar.finish if progress_bar
- 
--      if options[:checksum]
--        metadata = @s3.interface.head(bucket_to, key)
--        actual_md5 = metadata["etag"] or fail "Unable to get etag/md5 for #{bucket_to}:#{key}"
--        actual_md5 = actual_md5.sub(/^"/, "").sub(/"$/, "") # strip beginning and trailing quotes
--        if actual_md5 =~ /-/
--          STDERR.puts "Warning: invalid MD5 checksum in metadata; skipped checksum verification."
--          actual_md5 = nil
-+        if options[:checksum]
-+          actual_md5 = s3_checksum(bucket_to, key)
-+          unless actual_md5.is_a? String
-+            STDERR.puts "Warning: invalid MD5 checksum in metadata; skipped checksum verification."
-+            actual_md5 = nil
-+          end
-         end
-+      rescue => e
-+        raise e unless options[:checksum]
-+        STDERR.puts e
-+      ensure
-+        f.close()
-       end
--    rescue => e
--      raise e unless options[:checksum]
--      STDERR.puts e
--    ensure
--      f.close()
--    end
--    retries += 1
-+      retries += 1
-     end until options[:checksum] == false || actual_md5.nil? || expected_md5 == actual_md5
-+  else
-+    log "Already synchronized."
-+  end
- end
- 
- def s3_to_local(bucket_from, key_from, dest, options = {})
-   log("Copy s3://#{bucket_from}/#{key_from} to #{dest}")
-+
-+  actual_md5 = if options[:sync] && File.exist?(dest)
-+    md5(dest)
-+  end
-+
-   retries = 0
-   begin
-     if retries == options[:retries]
-@@ -292,34 +349,38 @@ def s3_to_local(bucket_from, key_from, dest, options = {})
- 
-     f = File.new(dest, "wb")
-     begin
--      size = nil
--      if options[:checksum]
--        metadata = @s3.interface.head(bucket_from, key_from)
--        expected_md5 = metadata["etag"] or fail "Unable to get etag/md5 for #{bucket_from}:#{key_from}"
--        expected_md5 = expected_md5.sub(/^"/, "").sub(/"$/, "") # strip beginning and trailing quotes
--        if expected_md5 =~ /-/
-+      expected_md5 = if options[:checksum] || options[:sync]
-+        md5 = s3_checksum(bucket_from, key_from)
-+        if options[:sync] && !md5.is_a?(String)
-+          STDERR.puts "Warning: invalid MD5 checksum in metadata; file will be force-copied."
-+          nil
-+        elsif !md5.is_a? String
-           STDERR.puts "Warning: invalid MD5 checksum in metadata; skipped checksum verification."
--          expected_md5 = nil
-+          nil
-+        else
-+          md5
-         end
--        size = metadata["content-length"].to_i
--      elsif $stdout.isatty
--        metadata = @s3.interface.head(bucket_from, key_from)
--        size = metadata["content-length"].to_i
-       end
--      begin
--        progress_bar = if size
--          ProgressBar.new(File.basename(key_from), size).tap do |p|
--            p.file_transfer_mode
-+
-+      if (expected_md5 == nil) || (options[:sync] && expected_md5 != actual_md5)
-+        begin
-+          progress_bar = if $stdout.isatty
-+            size = s3_size(bucket_from, key_from)
-+            ProgressBar.new(File.basename(key_from), size).tap do |p|
-+              p.file_transfer_mode
-+            end
-           end
-+          @s3.interface.get(bucket_from, key_from) do |chunk|
-+            f.write(chunk)
-+            progress_bar.inc chunk.size if progress_bar
-+          end
-+          progress_bar.finish
-+        rescue => e
-+          progress_bar.halt if progress_bar
-+          raise e
-         end
--        @s3.interface.get(bucket_from, key_from) do |chunk|
--          f.write(chunk)
--          progress_bar.inc chunk.size if progress_bar
--        end
--        progress_bar.finish
--      rescue => e
--        progress_bar.halt if progress_bar
--        raise e
-+      else
-+        log("Already synchronized")
-       end
-     rescue => e
-       raise e unless options[:checksum]
-