s3cp 1.1.31 → 1.1.32

data/History.txt

@@ -1,3 +1,44 @@
+=== 1.1.32 (2013-08-26)
+
+* Added:  New --mkdir option on s3cp allowing to recreate the source directory
+          structure from any point in the source path(s).
+
+          The --mkdir option accepts a regular expression to match against the
+          source path(s) and will recreate the source directory structure from
+          the match point.
+
+          Assuming the following bucket content,
+
+          s3://bucket/foo/bar/1
+          s3://bucket/foo/bar/2
+          s3://bucket/foo/baz/3
+          s3://bucket/foo/baz/4
+
+          then running:
+
+          % s3cp --recursive --mkdir /foo/  s3://bucket/foo/bar .
+
+          would create the following local directories and files:
+
+          ./foo/bar/1
+          ./foo/bar/2
+
+          If the regular expression contains a capturing group, the destination
+          path(s) will be relative to the group's starting position instead of
+          the regular expression's matching position:
+
+          % s3cp --recursive --mkdir "/foo/(.)"  s3://bucket/foo/ .
+
+          would create the following local content:
+
+          ./bar/1
+          ./bar/2
+          ./baz/3
+          ./baz/4
+
+          Note: The --mkdir option is currently only valid when copying from S3
+          to the local filesystem.
+
 === 1.1.31 (2013-05-16)
 
 * Fixed:  Passing ACLs to s3cp using --header 'x-amz-acl: public-read' was broken

data/lib/s3cp/s3cat.rb

@@ -31,8 +31,12 @@ op = OptionParser.new do |opts|
     options[:debug] = true
   end
 
-  opts.on("--tty", "TTY mode") do |tty|
-    options[:tty] = tty
+  opts.on("--tty", "Force TTY mode") do
+    options[:tty] = true
+  end
+
+  opts.on("--no-tty", "Disable TTY mode") do
+    options[:tty] = false
   end
 
   opts.on("--edit", "Edit mode") do |edit|

data/lib/s3cp/s3cp.rb

@@ -39,6 +39,7 @@ options[:include_regex] = []
 options[:exclude_regex] = []
 options[:sync] = false
 options[:move] = false
+options[:mkdir] = nil
 
 op = OptionParser.new do |opts|
   opts.banner = <<-BANNER
@@ -79,6 +80,10 @@ op = OptionParser.new do |opts|
     options[:move] = true
   end
 
+  opts.on("--mkdir MATCH", "Recreate directory structure starting at `MATCH` from source path(s)") do |prefix|
+    options[:mkdir] = Regexp.new(prefix)
+  end
+
   opts.on("--max-attempts N", "Number of attempts to upload/download until checksum matches (default #{options[:retries]})") do |attempts|
     options[:retries] = attempts.to_i
   end
@@ -234,6 +239,12 @@ def relative(base, path)
   no_slash(path[dir.length..-1])
 end
 
+def mkdir_relative(regex, key)
+  m = regex.match(key) or return
+  pos = m.captures.empty? ? m.begin(0) : m.begin(1)
+  suffix = key[pos..-1]
+end
+
 def log(msg)
   puts msg if @verbose
 end
@@ -535,8 +546,13 @@ def copy(from, to, options)
       end
       keys.each do |key|
         if match(key)
-          dest = File.expand_path(to) + '/' + relative(key_from, key)
-          dest = File.join(dest, File.basename(key)) if File.directory?(dest)
+          dest = if options[:mkdir]
+            suffix = mkdir_relative(options[:mkdir], key) or next
+            dest = File.join(File.expand_path(to), suffix)
+          else
+            dest = File.expand_path(to) + '/' + relative(key_from, key)
+            dest = File.join(dest, File.basename(key)) if File.directory?(dest)
+          end
           dir = File.dirname(dest)
           FileUtils.mkdir_p dir unless File.exist? dir
           fail "Destination path is not a directory: #{dir}" unless File.directory?(dir)
@@ -548,8 +564,16 @@ def copy(from, to, options)
         end
       end
     else
-      dest = File.expand_path(to)
-      dest = File.join(dest, File.basename(key_from)) if File.directory?(dest)
+      dest = if options[:mkdir]
+        suffix = mkdir_relative(options[:mkdir], key_from) or
+          fail("Error: The key '#{key_from}' does not match the --mkdir regular expression '#{options[:mkdir]}'")
+        dest = File.join(File.expand_path(to), suffix)
+        dir = File.dirname(dest)
+        FileUtils.mkdir_p dir unless File.exist? dir
+        dest
+      else
+        File.join(File.expand_path(to), File.basename(key_from)) if File.directory?(dest)
+      end
       if !options[:overwrite] && File.exist?(dest)
         $stderr.puts "Skipping #{dest} - already exists."
       else
@@ -575,3 +599,4 @@ S3CP.standard_exception_handling(options) do
     copy(source, destination, options)
   end
 end
+

data/lib/s3cp/s3ls.rb

@@ -17,10 +17,36 @@
 
 require 's3cp/utils'
 
+require "rubygems"
+require "highline/system_extensions"
+
+def silence_streams(*streams)
+  on_hold = streams.collect { |stream| stream.dup }
+  streams.each do |stream|
+    stream.reopen(RUBY_PLATFORM =~ /mswin/ ? 'NUL:' : '/dev/null')
+    stream.sync = true
+  end
+  yield
+ensure
+  streams.each_with_index do |stream, i|
+    stream.reopen(on_hold[i])
+  end
+end
+
+silence_streams(STDERR) do
+  p "size: " + HighLine::SystemExtensions.terminal_size.inspect
+end
+
 # Parse arguments
 options = {}
 options[:date_format] = ENV['S3CP_DATE_FORMAT'] || '%x %X'
-options[:rows_per_page] = ($terminal.output_rows - 1) if $stdout.isatty rescue nil
+#options[:rows_per_page] = ($terminal.output_rows - 1) if $stdout.isatty rescue nil
+silence_streams(STDERR) do
+  if termsize = HighLine::SystemExtensions.terminal_size
+    options[:rows_per_page] = termsize[0]
+  end
+end
+
 options[:precision] = 0
 
 op = OptionParser.new do |opts|

data/lib/s3cp/version.rb

@@ -16,5 +16,5 @@
 # the License.
 
 module S3CP
-  VERSION = "1.1.31"
+  VERSION = "1.1.32"
 end

data/lib/s3cp/version.rb~

@@ -0,0 +1,20 @@
+# Copyright (C) 2010-2012 Alex Boisvert and Bizo Inc. / All rights reserved.
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more contributor
+# license agreements.  See the NOTICE file  distributed with this work for
+# additional information regarding copyright ownership.  The ASF licenses this
+# file to you under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License.  You may obtain a copy of
+# the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+module S3CP
+  VERSION = "1.1.31"
+end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: s3cp
 version: !ruby/object:Gem::Version 
-  hash: 45
+  hash: 83
   prerelease: 
   segments: 
   - 1
   - 1
-  - 31
-  version: 1.1.31
+  - 32
+  version: 1.1.32
 platform: ruby
 authors: 
 - Alex Boisvert
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2013-05-16 00:00:00 Z
+date: 2013-08-26 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   version_requirements: &id001 !ruby/object:Gem::Requirement 
@@ -28,9 +28,9 @@ dependencies:
         - 0
         - 6
         version: "0.6"
-  requirement: *id001
   type: :runtime
   prerelease: false
+  requirement: *id001
   name: extensions
 - !ruby/object:Gem::Dependency 
   version_requirements: &id002 !ruby/object:Gem::Requirement 
@@ -44,9 +44,9 @@ dependencies:
         - 5
         - 1
         version: 1.5.1
-  requirement: *id002
   type: :runtime
   prerelease: false
+  requirement: *id002
   name: highline
 - !ruby/object:Gem::Dependency 
   version_requirements: &id003 !ruby/object:Gem::Requirement 
@@ -60,9 +60,9 @@ dependencies:
         - 8
         - 3
         version: 1.8.3
-  requirement: *id003
   type: :runtime
   prerelease: false
+  requirement: *id003
   name: aws-sdk
 - !ruby/object:Gem::Dependency 
   version_requirements: &id004 !ruby/object:Gem::Requirement 
@@ -76,12 +76,28 @@ dependencies:
         - 10
         - 0
         version: 0.10.0
-  requirement: *id004
   type: :runtime
   prerelease: false
+  requirement: *id004
   name: progressbar
 - !ruby/object:Gem::Dependency 
   version_requirements: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 23
+        segments: 
+        - 1
+        - 5
+        - 10
+        version: 1.5.10
+  type: :runtime
+  prerelease: false
+  requirement: *id005
+  name: nokogiri
+- !ruby/object:Gem::Dependency 
+  version_requirements: &id006 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
@@ -92,12 +108,12 @@ dependencies:
         - 5
         - 0
         version: 2.5.0
-  requirement: *id005
   type: :development
   prerelease: false
+  requirement: *id006
   name: rspec
 - !ruby/object:Gem::Dependency 
-  version_requirements: &id006 !ruby/object:Gem::Requirement 
+  version_requirements: &id007 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
@@ -108,9 +124,9 @@ dependencies:
         - 8
         - 7
         version: 0.8.7
-  requirement: *id006
   type: :development
   prerelease: false
+  requirement: *id007
   name: rake
 description: 
 email: 
@@ -149,6 +165,7 @@ files:
 - lib/s3cp/version.rb
 - lib/s3cp/s3cp.rb
 - lib/s3cp/s3stat.rb
+- lib/s3cp/version.rb~
 - lib/s3cp/utils.rb
 - lib/s3cp/s3lifecycle.rb
 - lib/s3cp/s3rm.rb