s3batch 0.1.0

data/lib/s3batch/batch_upload.rb

@@ -0,0 +1,86 @@
+module S3Batch
+
+  class TaskManager
+    def initialize
+      @count = 0
+      @adding_ended = false
+    end
+
+    def add
+      @count += 1
+    end
+
+    def remove
+      @count -= 1
+      EM.stop if @count == 0 && @adding_ended
+    end
+
+    def end_adding
+      @adding_ended = true
+      EM.stop if @count == 0
+    end
+  end
+
+  class UploadCollection
+    def initialize(bucket, dir, pattern, options = {})
+      @bucket = bucket
+      @dir = dir.end_with?('/') ? dir : dir + "/"
+      @pattern = pattern
+      @options = options
+    end
+
+    def upload request_options = {}
+      item = Happening::S3::Item.new(@bucket, '', @options)
+      item.get { |response|
+        keys = parse_keys(response.response)
+        check_md5_and_upload keys
+      }
+    end
+
+    def self.run s3id, s3key, bucket, dir, pattern
+      EM.run {
+        items = UploadCollection.new bucket, dir, pattern, :aws_access_key_id => s3id, :aws_secret_access_key => s3key, :protocol => 'http', :permissions => 'public-read'
+        items.upload 
+      }
+    end
+    
+    private
+    
+      def parse_keys xml
+        h = {}
+        doc = Nokogiri::XML(xml)
+        doc.css("Contents").each { |entry|
+          key = entry.at("Key").content
+          md5 = entry.at("ETag").content.gsub(/\A"/m, "").gsub(/"\Z/m, "")
+          h[key] = md5
+        }
+        return h
+      end
+
+      def check_md5_and_upload keys
+        manager = TaskManager.new
+        on_error = Proc.new {|response| puts "An error occured: #{response.response_header.status}"; manager.remove; }
+        on_success = Proc.new {|response| manager.remove; }
+
+        Dir.glob(@dir + @pattern) {|filename|
+          next unless File.file? filename
+
+          key = filename[@dir.length..-1]
+          content = File.read(filename)
+          md5 = Digest::MD5.hexdigest(content)
+
+          if keys[key] != md5
+            puts "uploading #{key} to #{@bucket}"
+            item = Happening::S3::Item.new(@bucket, key, @options)
+            item.put(content, :on_error => on_error, :on_success => on_success)
+            manager.add
+          else
+            puts "ignore #{key}, no change"
+          end
+        }
+
+        manager.end_adding
+      end
+    
+  end
+end

data/lib/s3batch/happening_patch.rb

@@ -0,0 +1,27 @@
+#Monkey patch
+module Happening
+  class AWS
+    protected
+    alias old_canonical_request_description canonical_request_description
+
+    def canonical_request_description(method, path, headers = {}, expires = nil)
+      description = old_canonical_request_description(method, path, headers, expires)
+      description << '?delete'  if path[/[&?]delete($|&|=)/]
+      description
+    end
+  end
+
+  module S3
+    class Request
+      protected
+      def validate
+      end
+    end
+
+    class Item
+      protected
+      def validate
+      end
+    end
+  end
+end

data/lib/s3batch/multi_delete.rb

@@ -0,0 +1,38 @@
+module S3Batch
+  class DeleteCollection < Happening::S3::Item
+    MULTIPLE_DELETE_LIMIT = 1000
+
+    def initialize(bucket, keys, options = {})
+      super bucket, "NO_USE", options
+      @keys = keys
+    end
+    
+    def path(with_bucket=true)
+      with_bucket ? "/#{bucket}/?delete" : "/?delete"
+    end
+
+    def delete request_options = {}, &blk
+      i = 0
+      while i < @keys.size
+        keys = @keys.slice(i, MULTIPLE_DELETE_LIMIT)
+        data = "<Delete><Quiet>true</Quiet><Object><Key>" + keys.join("</Key></Object><Object><Key>") + "</Key></Object></Delete>"
+        md5 = Base64.encode64(Digest::MD5.digest(data)).strip
+
+        headers = aws.sign("POST", path, {"Content-MD5" => md5})
+        request_options[:on_success] = blk if blk          
+        request_options.update(:headers => headers, :data => data)
+        Happening::S3::Request.new(:post, url, {:ssl => options[:ssl]}.update(request_options)).execute
+        i += MULTIPLE_DELETE_LIMIT
+      end
+    end
+
+    def run s3id, s3key, bucket, keys 
+      EM.run {
+        on_error = Proc.new {|response| puts "An error occured: #{response.response}"; EM.stop }
+        on_success = Proc.new {|response| puts "Deleted!"; EM.stop }
+        items = DeleteCollection.new bucket, keys, :aws_access_key_id => s3id, :aws_secret_access_key => s3key, :protocol => 'http'
+        items.delete(:on_error => on_error, :on_success => on_success) 
+      }
+    end
+  end
+end

data/lib/s3batch.rb

@@ -0,0 +1,18 @@
+require 'happening'
+require 'nokogiri'
+
+require File.expand_path(File.dirname(__FILE__) + '/s3batch/happening_patch')
+require File.expand_path(File.dirname(__FILE__) + '/s3batch/multi_delete')
+require File.expand_path(File.dirname(__FILE__) + '/s3batch/batch_upload')
+
+#ruby s3batch.rb S3ID S3KEY BUCKET DIR
+if $0 == __FILE__
+  s3id = ARGV[0]
+  s3key = ARGV[1]
+  bucket = ARGV[2]
+  dir = ARGV[3]
+  pattern = ARGV[4] || "**/*"
+
+  S3Batch::UploadCollection.run s3id, s3key, bucket, dir, pattern
+end
+

metadata

@@ -0,0 +1,72 @@
+--- !ruby/object:Gem::Specification
+name: s3batch
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+  prerelease: 
+platform: ruby
+authors:
+- Xue Yong Zhi
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-05-30 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: happening
+  requirement: &70551020 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70551020
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: &70550310 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70550310
+description: 
+email:
+- yong@intridea.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/s3batch/multi_delete.rb
+- lib/s3batch/batch_upload.rb
+- lib/s3batch/happening_patch.rb
+- lib/s3batch.rb
+homepage: http://github.com/yong/s3batch
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.10
+signing_key: 
+specification_version: 3
+summary: upload/delete s3 objects in batch
+test_files: []
+has_rdoc: