s3cp 0.2.1 → 0.2.2

data/History.txt

@@ -1,4 +1,11 @@
-=== 0.2.2 / (Pending)
+=== 0.2.3 / (Pending)
+
+=== 0.2.2 / (2012-02-23)
+
+* Added: Progress bars during upload/download if $stdout.isatty
+
+* Fixed: s3cat now handles broken pipes properly
+         e.g. "s3cat bucket:some/file | head" will now terminate early.
 
 === 0.2.1 / (2012-02-20)
 

data/README.md

@@ -1,7 +1,7 @@
 S3CP: Commands-line tools for Amazon S3 file manipulation
 =============================================================
 
-Just a few simple command-line utilities to list, copy, view S3 files, e.g.,  `s3cp`, `s3ls`, `s3cat`.
+Just a few simple command-line utilities to list, copy, view S3 files, e.g.,  `s3cp`, `s3ls`, `s3cat`, `s3rm`, etc.
 
 ### Installing ###
 
@@ -26,9 +26,11 @@ If you want to hack on s3cp and build the gem yourself, you will need Bundler (h
     export AWS_SECRET_ACCESS_KEY=...
 
     s3ls s3://mybucket/path/to/some/files
+    s3dir s3://mybucket/path/to/some/files
     s3cat s3://mybucket/path/to/some/file.txt
     s3cp local_file.bin s3://mybucket/some/path
     s3mod s3://mybucket/path/to/some/file.txt public-read
+    s3stat s3://mybucket/path/to/some/file.txt
 
 Use the `-h` option to learn about command-line options.
 
@@ -36,6 +38,14 @@ All commands support both `s3://bucket/path/to/file` and the legacy `bucket:path
 
 Commands are also TTY-aware;  when run in an interactive shell, their behavior will change.  For example, `s3cat` will launch your favorite `PAGER` or `less` (the default pager) whereas `s3ls` will display N items at a time, where N is the number of display lines on your terminal and pause between pages.
 
+### Bash completion for S3 URLs ###
+
+To install Bash completion for S3 URLs, add the following to ~/.bashrc:
+
+    for cmd in [ s3cat s3cp s3dir s3ls s3mod s3rm s3stat ]; do
+      complete -C s3cp_complete $cmd
+    done
+
 ### Usage ###
 
     $ s3cp

data/lib/s3cp/completion.rb

@@ -1,3 +1,20 @@
+# Copyright (C) 2010-2012 Alex Boisvert and Bizo Inc. / All rights reserved.
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more contributor
+# license agreements.  See the NOTICE file  distributed with this work for
+# additional information regarding copyright ownership.  The ASF licenses this
+# file to you under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License.  You may obtain a copy of
+# the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
 require 'rubygems'
 require 'extensions/kernel' if RUBY_VERSION =~ /1.8/
 require 'right_aws'
@@ -14,7 +31,7 @@ position = ENV['COMP_POINT'].to_i
 url = begin
   start = position
   start -= 1 while start >= 1 && cmd_line[start-1].chr != ' '
-  cmd_line[start..position-1]
+  cmd_line[start..position-1].gsub("\"", "")
 end
 
 cmd = ARGV[0]
@@ -35,6 +52,9 @@ end
 
 debug "url #{url}"
 
+debug "arg1 #{arg1.inspect}"
+debug "arg2 #{arg2.inspect}"
+
 delimiter = ENV["S3CP_DELIMITER"] || "/"
 debug "delimiter #{delimiter}"
 

data/lib/s3cp/s3cat.rb

@@ -1,3 +1,20 @@
+# Copyright (C) 2010-2012 Alex Boisvert and Bizo Inc. / All rights reserved.
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more contributor
+# license agreements.  See the NOTICE file  distributed with this work for
+# additional information regarding copyright ownership.  The ASF licenses this
+# file to you under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License.  You may obtain a copy of
+# the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
 require 'rubygems'
 require 'extensions/kernel' if RUBY_VERSION =~ /1.8/
 require 'right_aws'
@@ -5,6 +22,7 @@ require 'optparse'
 require 'date'
 require 'highline/import'
 require 'tempfile'
+require 'progressbar'
 
 require 's3cp/utils'
 
@@ -50,12 +68,21 @@ fail "Your URL looks funny, doesn't it?" unless @bucket
 
 if options[:tty]
   # store contents to file to display with PAGER
+  metadata = @s3.head(@bucket, @prefix)
+  size = metadata["content-length"].to_i
+
+  progress_bar = ProgressBar.new(File.basename(@prefix), size).tap do |p|
+    p.file_transfer_mode
+  end
+
   file = Tempfile.new('s3cat')
   out = File.new(file.path, "wb")
   begin
     @s3.get(@bucket, @prefix) do |chunk|
       out.write(chunk)
+      progress_bar.inc chunk.size
     end
+    progress_bar.finish
   ensure
     out.close()
   end
@@ -63,7 +90,11 @@ if options[:tty]
   file.delete()
 else
   @s3.get(@bucket, @prefix) do |chunk|
-    STDOUT.print(chunk)
+    begin
+      STDOUT.print(chunk)
+    rescue Errno::EPIPE
+      break
+    end
   end
 end
 

data/lib/s3cp/s3cp.rb

@@ -1,3 +1,20 @@
+# Copyright (C) 2010-2012 Alex Boisvert and Bizo Inc. / All rights reserved.
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more contributor
+# license agreements.  See the NOTICE file  distributed with this work for
+# additional information regarding copyright ownership.  The ASF licenses this
+# file to you under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License.  You may obtain a copy of
+# the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
 require 'rubygems'
 require 'extensions/kernel' if RUBY_VERSION =~ /1.8/
 require 'right_aws'
@@ -6,6 +23,7 @@ require 'date'
 require 'highline/import'
 require 'fileutils'
 require 'digest'
+require 'progressbar'
 
 require 's3cp/utils'
 
@@ -95,6 +113,21 @@ end
 destination = ARGV.last
 sources = ARGV[0..-2]
 
+class Proxy
+  instance_methods.each { |m| undef_method m unless m =~ /(^__|^send$|^object_id$)/ }
+
+  def initialize(target)
+    @target = target
+  end
+
+  protected
+
+  def method_missing(name, *args, &block)
+    #puts "method_missing! #{name} #{args.inspect}"
+    @target.send(name, *args, &block)
+  end
+end
+
 def s3?(url)
   S3CP.bucket_and_key(url)[0]
 end
@@ -199,7 +232,31 @@ def local_to_s3(bucket_to, key, file, options = {})
 
     f = File.open(file)
     begin
+      if $stdout.isatty
+        f = Proxy.new(f)
+        progress_bar = ProgressBar.new(File.basename(file), File.size(file)).tap do |p|
+          p.file_transfer_mode
+        end
+        class << f
+         attr_accessor :progress_bar
+          def read(length, buffer=nil)
+            begin
+              result = @target.read(length, buffer)
+              @progress_bar.inc result.length if result
+              result
+            rescue => e
+              puts e
+              raise e
+            end
+          end
+        end
+        f.progress_bar = progress_bar
+      else
+        progress_bar = nil
+      end
+
       meta = @s3.interface.put(bucket_to, key, f, @headers)
+      progress_bar.finish if progress_bar
 
       if options[:checksum]
         metadata = @s3.interface.head(bucket_to, key)
@@ -235,6 +292,7 @@ def s3_to_local(bucket_from, key_from, dest, options = {})
 
     f = File.new(dest, "wb")
     begin
+      size = nil
       if options[:checksum]
         metadata = @s3.interface.head(bucket_from, key_from)
         expected_md5 = metadata["etag"] or fail "Unable to get etag/md5 for #{bucket_from}:#{key_from}"
@@ -243,9 +301,25 @@ def s3_to_local(bucket_from, key_from, dest, options = {})
           STDERR.puts "Warning: invalid MD5 checksum in metadata; skipped checksum verification."
           expected_md5 = nil
         end
+        size = metadata["content-length"].to_i
+      elsif $stdout.isatty
+        metadata = @s3.interface.head(bucket_from, key_from)
+        size = metadata["content-length"].to_i
       end
-      @s3.interface.get(bucket_from, key_from) do |chunk|
-        f.write(chunk)
+      begin
+        progress_bar = if size
+          ProgressBar.new(File.basename(key_from), size).tap do |p|
+            p.file_transfer_mode
+          end
+        end
+        @s3.interface.get(bucket_from, key_from) do |chunk|
+          f.write(chunk)
+          progress_bar.inc chunk.size if progress_bar
+        end
+        progress_bar.finish
+      rescue => e
+        progress_bar.halt if progress_bar
+        raise e
       end
     rescue => e
       raise e unless options[:checksum]

data/lib/s3cp/s3ls.rb

@@ -1,3 +1,20 @@
+# Copyright (C) 2010-2012 Alex Boisvert and Bizo Inc. / All rights reserved.
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more contributor
+# license agreements.  See the NOTICE file  distributed with this work for
+# additional information regarding copyright ownership.  The ASF licenses this
+# file to you under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License.  You may obtain a copy of
+# the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
 require 'rubygems'
 require 'extensions/kernel' if RUBY_VERSION =~ /1.8/
 require 'right_aws'

data/lib/s3cp/s3mod.rb

@@ -1,3 +1,20 @@
+# Copyright (C) 2010-2012 Alex Boisvert and Bizo Inc. / All rights reserved.
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more contributor
+# license agreements.  See the NOTICE file  distributed with this work for
+# additional information regarding copyright ownership.  The ASF licenses this
+# file to you under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License.  You may obtain a copy of
+# the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
 require 'rubygems'
 require 'extensions/kernel' if RUBY_VERSION =~ /1.8/
 require 'right_aws'

data/lib/s3cp/s3rm.rb

@@ -1,3 +1,20 @@
+# Copyright (C) 2010-2012 Alex Boisvert and Bizo Inc. / All rights reserved.
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more contributor
+# license agreements.  See the NOTICE file  distributed with this work for
+# additional information regarding copyright ownership.  The ASF licenses this
+# file to you under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License.  You may obtain a copy of
+# the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
 require 'rubygems'
 require 'extensions/kernel' if RUBY_VERSION =~ /1.8/
 require 'right_aws'

data/lib/s3cp/s3stat.rb

@@ -1,3 +1,20 @@
+# Copyright (C) 2010-2012 Alex Boisvert and Bizo Inc. / All rights reserved.
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more contributor
+# license agreements.  See the NOTICE file  distributed with this work for
+# additional information regarding copyright ownership.  The ASF licenses this
+# file to you under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License.  You may obtain a copy of
+# the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
 require 'rubygems'
 require 'extensions/kernel' if RUBY_VERSION =~ /1.8/
 require 'right_aws'

data/lib/s3cp/utils.rb

@@ -1,3 +1,20 @@
+# Copyright (C) 2010-2012 Alex Boisvert and Bizo Inc. / All rights reserved.
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more contributor
+# license agreements.  See the NOTICE file  distributed with this work for
+# additional information regarding copyright ownership.  The ASF licenses this
+# file to you under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License.  You may obtain a copy of
+# the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
 module S3CP
   extend self
 

data/lib/s3cp/version.rb

@@ -1,4 +1,20 @@
+# Copyright (C) 2010-2012 Alex Boisvert and Bizo Inc. / All rights reserved.
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more contributor
+# license agreements.  See the NOTICE file  distributed with this work for
+# additional information regarding copyright ownership.  The ASF licenses this
+# file to you under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License.  You may obtain a copy of
+# the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+# License for the specific language governing permissions and limitations under
+# the License.
 
 module S3CP
-  VERSION = "0.2.1"
+  VERSION = "0.2.2"
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: s3cp
 version: !ruby/object:Gem::Version 
-  hash: 21
+  hash: 19
   prerelease: 
   segments: 
   - 0
   - 2
-  - 1
-  version: 0.2.1
+  - 2
+  version: 0.2.2
 platform: ruby
 authors: 
 - Alex Boisvert
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-02-21 00:00:00 Z
+date: 2012-02-23 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   prerelease: false
@@ -82,8 +82,24 @@ dependencies:
   name: right_http_connection
 - !ruby/object:Gem::Dependency 
   prerelease: false
-  type: :development
+  type: :runtime
   requirement: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 55
+        segments: 
+        - 0
+        - 10
+        - 0
+        version: 0.10.0
+  version_requirements: *id005
+  name: progressbar
+- !ruby/object:Gem::Dependency 
+  prerelease: false
+  type: :development
+  requirement: &id006 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
@@ -94,12 +110,12 @@ dependencies:
         - 5
         - 0
         version: 2.5.0
-  version_requirements: *id005
+  version_requirements: *id006
   name: rspec
 - !ruby/object:Gem::Dependency 
   prerelease: false
   type: :development
-  requirement: &id006 !ruby/object:Gem::Requirement 
+  requirement: &id007 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
@@ -110,7 +126,7 @@ dependencies:
         - 8
         - 7
         version: 0.8.7
-  version_requirements: *id006
+  version_requirements: *id007
   name: rake
 description: 
 email: