s3cp 0.1.1 → 0.1.2

data/History.txt

@@ -1,3 +1,14 @@
+=== 0.1.3 / (Pending)
+
+=== 0.1.2 / 2011-09-29
+
+* s3ls now displays entries with s3://@bucket/ prefix   (Josh Carver)
+* s3cp now supports passig headers (--headers)          (Donnie Flood)
+
+=== 0.1.1 / 2011-09-22
+
+* Added s3mod command (Josh Carver)
+
 === 0.1 / 2011-04-05
 
 * First release

data/lib/s3cp/s3cp.rb

@@ -10,11 +10,35 @@ require 's3cp/utils'
 
 # Parse arguments
 options = {}
-options[:verbose] = true if $stdout.isatty
+options[:verbose] = $stdout.isatty ? true : false
+options[:headers] = []
 
 op = OptionParser.new do |opts|
-  opts.banner = "s3cp [path]"
+  opts.banner = <<-BANNER
+    s3cp supports 4 coyping use cases:
+      1. Copy from local machine to S3
+      2. Copy from S3 to local machine
+      3. Copy from S3 to S3
+      4. Copy from local machine to another path on local machine (for completeness)
+    
+    Local to S3:
+      s3cp LOCAL_PATH S3_PATH
+      
+    S3 to Local:
+      s3cp S3_PATH LOCAL_PATH
+      
+    S3 to S3:
+      s3cp S3_PATH S3_PATH2
+      
+    Local to Local:
+      s3cp LOCAL_PATH LOCAL_PATH2
+      
+  BANNER
   opts.separator ''
+  
+  opts.on("-h", '--headers \'Header1: Header1Value\',\'Header2: Header2Value\'', Array, "Headers to set on the item in S3.  This can include http headers like \'Content-Type: image/jpg\' or AMZ headers like: \'x-amz-acl: public-read\'" ) do |h|
+    options[:headers] = h
+  end
 
   opts.on("-r", "Recursive mode") do
     options[:recursive] = true
@@ -52,6 +76,10 @@ if options[:debug]
   puts "Options: \n#{options.inspect}"
 end
 
+if options[:verbose]
+  @verbose = true
+end
+
 @bucket = $1
 @prefix = $2
 
@@ -78,6 +106,63 @@ def relative(base, path)
   no_slash(path[base.length..-1])
 end
 
+def log(msg)
+  puts msg if @verbose
+end
+
+def headers_array_to_hash(header_array)
+  headers = {}
+  header_array.each do |header|
+    header_parts = header.split(": ", 2)
+    if header_parts.size == 2
+      headers[header_parts[0]] = header_parts[1] 
+    else
+      log("Header ignored because of error splitting [#{header}].  Expected colon delimiter; e.g. Header: Value")
+    end
+  end
+  headers
+end
+@headers = headers_array_to_hash(options[:headers])
+
+def with_headers(msg)
+  unless @headers.empty?
+    msg += " with headers:"
+    msg += @headers.collect{|k,v| "'#{k}: #{v}'"}.join(", ")
+  end
+  msg
+end
+
+def s3_to_s3(bucket_from, key, bucket_to, dest)  
+  log(with_headers("Copy s3://#{bucket_from}/#{key} to s3://#{bucket_to}/#{dest}"))
+  if @headers.empty?
+    @s3.interface.copy(bucket_from, key, bucket_to, dest)
+  else
+    @s3.interface.copy(bucket_from, key, bucket_to, dest, :copy, @headers)
+  end
+end
+
+def local_to_s3(bucket_to, key, file)
+  log(with_headers("Copy #{file} to s3://#{bucket_to}/#{key}"))
+  f = File.open(file)
+  begin
+    @s3.interface.put(bucket_to, key, f, @headers)
+  ensure
+    f.close()
+  end
+end
+
+def s3_to_local(bucket_from, key_from, dest)
+  log("Copy s3://#{bucket_from}/#{key_from} to #{dest}")
+  f = File.new(dest, File::CREAT|File::RDWR)
+  begin
+    @s3.interface.get(bucket_from, key_from) do |chunk|
+      f.write(chunk)
+    end
+  ensure
+    f.close()
+  end
+end
+
 def copy(from, to, options)
   bucket_from, key_from = S3CP.bucket_and_key(from)
   bucket_to, key_to = S3CP.bucket_and_key(to)
@@ -95,12 +180,10 @@ def copy(from, to, options)
       end
       keys.each do |key|
         dest = no_slash(key_to) + '/' + relative(key_from, key)
-        puts "Copy s3://#{bucket_from}/#{key} to s3://#{bucket_to}/#{dest}"
-        @s3.interface.copy(bucket_from, key, bucket_to, dest)
+        s3_to_s3(bucket_from, key, bucket_to, dest)
       end
     else
-      puts "Copy s3://#{bucket_from}/#{key_from} to s3://#{bucket_to}/#{key_to}"
-      @s3.interface.copy(bucket_from, key_from, bucket_to, key_to)
+      s3_to_s3(bucket_from, key_from, bucket_to, key_to)
     end
   when :local_to_s3
     if options[:recursive]
@@ -108,18 +191,10 @@ def copy(from, to, options)
       files.each do |f|
         f = File.expand_path(f)
         key = no_slash(key_to) + '/' + relative(from, f)
-        puts "Copy #{f} to s3://#{bucket_to}/#{key}"
-        @s3.interface.put(bucket_to, key,  File.open(f))
+        local_to_s3(bucket_to, key, f)
       end
     else
-      f = File.expand_path(from)
-      puts "Copy #{f} to s3://#{bucket_to}/#{key_to}"
-      f = File.open(f)
-      begin
-        @s3.interface.put(bucket_to, key_to, f)
-      ensure
-        f.close()
-      end
+      local_to_s3(bucket_to, key_to, File.expand_path(from))
     end
   when :s3_to_local
     if options[:recursive]
@@ -130,31 +205,15 @@ def copy(from, to, options)
       keys.each do |key|
         dest = File.expand_path(to) + '/' + relative(key_from, key)
         dest = File.join(dest, File.basename(key)) if File.directory?(dest)
-        puts "Copy s3://#{bucket_from}/#{key} to #{dest}"
         dir = File.dirname(dest)
         FileUtils.mkdir_p dir unless File.exist? dir
         fail "Destination path is not a directory: #{dir}" unless File.directory?(dir)
-        f = File.new(dest, File::CREAT|File::RDWR)
-        begin
-          @s3.interface.get(bucket_from, key) do |chunk|
-            f.write(chunk)
-          end
-        ensure
-          f.close()
-        end
+        s3_to_local(bucket_from, key, dest)
       end
     else
       dest = File.expand_path(to)
       dest = File.join(dest, File.basename(key_from)) if File.directory?(dest)
-      puts "Copy s3://#{bucket_from}/#{key_from} to #{dest}"
-      f = File.new(dest, File::CREAT|File::RDWR)
-      begin
-        @s3.interface.get(bucket_from, key_from) do |chunk|
-          f.write(chunk)
-        end
-      ensure
-        f.close()
-      end
+      s3_to_local(bucket_from, key_from, dest)
     end
   when :local_to_local
     if options[:recursive]

data/lib/s3cp/s3ls.rb

@@ -64,7 +64,7 @@ end
 rows = 0
 @s3.interface.incrementally_list_bucket(@bucket, :prefix => @key) do |page|
   page[:contents].each do |entry|
-    key = entry[:key]
+    key = "s3://#{@bucket}/#{entry[:key]}"
     last_modified = DateTime.parse(entry[:last_modified])
     if options[:long_format]
       puts "#{last_modified.strftime(options[:date_format])} #{key}"

data/lib/s3cp/version.rb

@@ -1,4 +1,4 @@
 
 module S3CP
-  VERSION = "0.1.1"
+  VERSION = "0.1.2"
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: s3cp
 version: !ruby/object:Gem::Version 
-  hash: 25
+  hash: 31
   prerelease: 
   segments: 
   - 0
   - 1
-  - 1
-  version: 0.1.1
+  - 2
+  version: 0.1.2
 platform: ruby
 authors: 
 - Alex Boisvert
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-09-22 00:00:00 Z
+date: 2011-09-29 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   version_requirements: &id001 !ruby/object:Gem::Requirement