s3cp 1.1.12 → 1.1.13

data/History.txt

@@ -1,3 +1,12 @@
+=== 1.1.13 (2013-01-27)
+
+* Added:   s3stat --acl option to display associated access-control list XML document.
+
+* Added:   s3mod now accepts multiple (wildcard) paths and supports setting both
+           --headers and --acl. (Denis Arh).
+
+* Fixed:   Beter metadata display in s3stat.
+
 === 1.1.12 (2013-01-22)
 
 * Added:   Support for 'Content-Disposition' header (Denis Arh).

data/lib/s3cp/s3mod.rb

@@ -17,17 +17,51 @@
 
 require 's3cp/utils'
 
+# Default options
+options = {
+  :verbose => false,
+  :acl     => nil,
+  :headers => []
+}
+
+# Setup cli params
 op = OptionParser.new do |opts|
-  opts.banner = "s3mod [path] [permission]"
+  opts.banner = "s3mod [path(s)]"
+
+  opts.banner = <<-BANNER
+    s3mod [path(s)] ([permission])
+
+    Multiple paths with multiple wildcards are supported:
+      s3://bucket/path/*foo/*/bar*.jpg
+
+    LEGACY SUPPORT
+
+    This tool still supports old paramater format:
+      % s3mod [path] [permission]
+    You are encouraged to use --acl instead of passing permission as last parameter.
+
+    WARNINGS:
+
+    1) Due to limitations in AWS's S3 API, when you use --headers all ACL
+       properties are removed from the S3 object.
+
+    2) Due to limitations in AWS's S3 API, this command fails on S3 objects
+       larger then 5Gb.
+
+  BANNER
+  opts.separator ''
+
+  opts.on("--verbose", "Verbose mode") do
+    options[:verbose] = true
+  end
 
-  opts.separator ""
-  opts.separator "where [permission] is one of:"
-  opts.separator ""
-  opts.separator " * private"
-  opts.separator " * authenticated-read"
-  opts.separator " * public-read"
-  opts.separator " * public-read-write"
-  opts.separator ""
+  opts.on('--headers \'Header1: Header1Value\',\'Header2: Header2Value\'', Array, "Headers to set on the item in S3." ) do |h|
+    options[:headers] = h
+  end
+
+  opts.on("--acl PERMISSION", "One of 'private', 'authenticated-read', 'public-read', 'public-read-write'") do |permission|
+    options[:acl] = S3CP.validate_acl(permission)
+  end
 
   opts.on_tail("-h", "--help", "Show this message") do
     puts op
@@ -36,22 +70,36 @@ op = OptionParser.new do |opts|
 end
 
 op.parse!(ARGV)
+paths = ARGV
 
-if ARGV.size < 2
+if ARGV.size < 1 || !options[:headers]
   puts op
   exit
 end
 
-def update_permissions(s3, bucket, key, permission)
-  puts "Setting #{permission} on s3://#{bucket}/#{key}"
-  s3.buckets[bucket].objects[key].acl = permission
+# Legacy (support for s3mod [path] [acl] )
+# See if last param starts with "s3://", validate & remove it from list of paths
+if !options[:acl] && paths.size > 1 && S3CP::LEGAL_MODS.include?(paths.last)
+  options[:acl] = S3CP.validate_acl(paths.pop);
 end
 
-source  = ARGV[0]
-permission = S3CP.validate_acl(ARGV.last)
-
 S3CP.load_config()
-
 @s3 = S3CP.connect()
-bucket,key = S3CP.bucket_and_key(source)
-update_permissions(@s3, bucket, key, permission)
+
+paths.each do |path|
+  bucket,key = S3CP.bucket_and_key(path)
+  fail "Invalid bucket/key: #{path}" unless key
+
+  S3CP.objects_by_wildcard(@s3.buckets[bucket], key) { | obj |
+    puts "s3://#{bucket}/#{obj.key}"
+
+    if options[:headers].size > 0
+      current_medata = obj.metadata
+      object_metadata = S3CP.set_header_options(current_medata, S3CP.headers_array_to_hash(options[:headers]))
+    end
+
+    if options[:acl]
+      obj.acl = options[:acl]
+    end
+  }
+end

data/lib/s3cp/s3stat.rb

@@ -17,9 +17,15 @@
 
 require 's3cp/utils'
 
+@options = {}
+
 op = OptionParser.new do |opts|
   opts.banner = "s3stat [path]"
 
+  opts.on("--acl", "Display Access Control List XML document") do
+    @options[:acl] = true
+  end
+
   opts.on_tail("-h", "--help", "Show this message") do
     puts op
     exit
@@ -43,8 +49,15 @@ S3CP.load_config()
 
 @s3 = S3CP.connect().buckets[@bucket]
 
-metadata = @s3.objects[@key].head
+obj = @s3.objects[@key]
+
+metadata = obj.head
 metadata.keys.sort { |k1, k2| k1.to_s <=> k2.to_s}.each do |k|
-  puts "#{"%30s" % k} #{metadata[k]}"
+  puts "#{"%30s" % k} #{metadata[k].is_a?(Hash) ? metadata[k].inspect : metadata[k].to_s}"
 end
 
+if @options[:acl]
+  puts
+  xml = Nokogiri::XML(obj.acl.to_s)
+  puts xml.to_s
+end

data/lib/s3cp/utils.rb

@@ -175,8 +175,9 @@ module S3CP
       end
     end
 
-
-    options[:metadata] = remaining unless remaining.empty?
+    remaining.each do |k,v|
+      options[k] = v
+    end
 
     options
   end
@@ -187,6 +188,31 @@ module S3CP
     end
     permission
   end
+
+  # Yield to block for all objects matching key_regex
+  def objects_by_wildcard(bucket, key_regex, &block)
+    # First, trim multiple wildcards & wildcards on the end
+    key = key_regex.gsub(/\*+/, '*')
+
+    if 0 < key.count('*')
+      key_split = key.split('*', 2);
+      kpfix     = key_split.shift(); # ignore first part as AWS API takes it as a prefix
+      regex     = []
+
+      key_split.each do |kpart|
+        regex.push Regexp.quote(kpart)
+      end
+
+      regex = regex.empty? ? nil : Regexp.new(regex.join('.*') + '$');
+
+      bucket.objects.with_prefix(kpfix).each do |obj|
+        yield obj if regex == nil || regex.match(obj.key[kpfix.size..-1])
+      end
+    else
+      # no wildcards, simple:
+      yield bucket.objects[key]
+    end
+  end
 end
 
 # Monkey-patch S3 object for download streaming

data/lib/s3cp/version.rb

@@ -16,5 +16,5 @@
 # the License.
 
 module S3CP
-  VERSION = "1.1.12"
+  VERSION = "1.1.13"
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: s3cp
 version: !ruby/object:Gem::Version 
-  hash: 11
+  hash: 9
   prerelease: 
   segments: 
   - 1
   - 1
-  - 12
-  version: 1.1.12
+  - 13
+  version: 1.1.13
 platform: ruby
 authors: 
 - Alex Boisvert
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2013-01-22 00:00:00 Z
+date: 2013-01-27 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   version_requirements: &id001 !ruby/object:Gem::Requirement