manage_s3_bucket 0.0.3 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: dde6a371a9ad8dfac1a067d160d4cd1cd424f996
-  data.tar.gz: cf5af5ae4c878c0a79da5c41cd174bbb3b23ebc1
+  metadata.gz: 7ba3626e48782dd839e5dff9b475afbe0f958bd4
+  data.tar.gz: 8fc596f3f870e4f8f560406ed2d10baafdc69122
 SHA512:
-  metadata.gz: 36febb344e664a9f8ffec61c8702032d200cd5a3cd4339a5095a06e24cd567518f40dfb9bf93a1ad31bfbe72877daa13b54938188056b8adb6bbfa9574386ca2
-  data.tar.gz: 6629c227e85346af7792e8633d9228504d43901cce0dbfb089cad2d230819886102045df7857252b6670b06e10eae60993d3c39aeef6b013cd77a3166d511e8a
+  metadata.gz: 51721bbb20836405f279ec1abe7cbae908862fb8e3fd8ebde415e9fbf97209d6aa180e6efcedc83a5b40f13e240206d0301bdc6c0fb536a48ca8e88a7c1e28ff
+  data.tar.gz: 3528b15e0c04f85d32a273784e495e61603ed8d63ca91e7945d49606618b9943670c4e0867234a393ed2b8ad6581eb85d37b45ab46296ed5922458f68f1fafbc

data/lib/manage_s3_bucket.rb

@@ -1,26 +1,118 @@
-require "manage_s3_bucket/version"
+###############################################################################################
+# How to use
+## Get Bucket
+### s3 = Manage::S3::new(@aws_access_key_id, @aws_secret_key_id)
+## Remove Directory
+### s3.remove_path("bucket_name/folder/subfolder")
+## Clear all files and folders from bucket
+### s3.remove_path("bucket_name")
+## Copy path
+### s3.copy_path("backups-all-apps/upload_files/erp/2015.04.06.21.00.01", "backups-all-apps/folder/subfolder")
+### Not too fast as s3s3mirror (https://github.com/cobbzilla/s3s3mirror)
+
+###############################################################################################
+require "manage_s3_bucket"
 require "right_aws_api"
+require 'pp' # show thread errors
+require 'work_queue'
 
 class Manage
   class S3
-    class Bucket
-      def initialize(bucket_name, aws_access_key_id, aws_secret_key_id)
-        @bucket_name       = bucket_name
-        @aws_access_key_id = aws_access_key_id
-        @aws_secret_key_id = aws_secret_key_id
+    # nodoc
+    def initialize(aws_access_key_id, aws_secret_key_id)
+      @aws_access_key_id = aws_access_key_id
+      @aws_secret_key_id = aws_secret_key_id
+    end
+
+    def s3
+      @s3 ||= RightScale::CloudApi::AWS::S3::Manager::new(@aws_access_key_id, @aws_secret_key_id, 'https://s3.amazonaws.com')
+    end
+
+    def extract_path(path)
+      slice_path = path.split('/')
+      [slice_path[0], slice_path[1..-1].join('/')]
+    end
+
+    def list_directory(path, marker = nil)
+      bucket, directory = extract_path(path)
+      opt_list = {
+        'Bucket' => bucket,
+        'prefix' => directory,
+        "max-keys" => 1000
+      }
+      opt_list['marker'] = marker if marker
+      list = s3.ListObjects(opt_list)['ListBucketResult']
+      return list
+    end
+
+    def execute_in_path(path, marker = nil, &block)
+      marker ||= nil
+      bucket, directory = extract_path(path)
+      list = list_directory(path, marker)
+      objects = list['Contents']
+      if objects
+        keys = objects.map { |c| c['Key'] }
+        yield keys
+        if list["IsTruncated"] == "true"
+          marker = keys.last
+          p "Next Page: #{marker}"
+          execute_in_path path, marker do |keys|
+            yield keys
+          end
+        end
       end
-      def s3
-        @s3 ||= RightScale::CloudApi::AWS::S3::Manager::new(@aws_access_key_id, @aws_secret_key_id, 'https://s3.amazonaws.com')
+    end
+
+    def remove_path(path)
+      bucket, directory = extract_path(path)
+      p "Deleting #{path}"
+      execute_in_path path do |keys|
+        s3.DeleteMultipleObjects('Bucket' => bucket, 'Object' => keys)
+      end
+    end
+
+    def copy_key(_s3, source_path, dest_path, key)
+      copied = false
+      i = 1
+      source_bucket, source_directory = extract_path(source_path)
+      dest_bucket, dest_directory     = extract_path(dest_path)
+      new_key = "#{dest_directory}#{key.gsub(source_directory, '')}"
+      while !copied
+        begin
+          _s3.CopyObject(
+            'SourceBucket'      => source_bucket,
+            'SourceObject'      => key,
+            'DestinationBucket' => dest_bucket,
+            'DestinationObject' => new_key
+          )
+          copied = true
+          pp "SUCCESS: Copyed #{key} to #{new_key}"
+        rescue
+          i += 1
+          pp $!
+          sleep 10
+          pp '#'*200
+          pp "ERROR: Trying #{i} to copy #{key} to #{new_key}"
+          _s3 = s3
+          copy_key(_s3, source_path, dest_path, key)
+        end
       end
-      def remove_directory(directory)
-        # deletando keys
-        objects = s3.ListObjects('Bucket' => @bucket_name, 'prefix' => directory)["ListBucketResult"]["Contents"]
-        if objects
-          keys = objects.map{|c| c["Key"]}
-          s3.DeleteMultipleObjects('Bucket' => @bucket_name, "Object" => keys)
-          remove_directory(directory) # s3 removes max 1000 objects, cause this we used recursive method
+    end
+
+    def copy_path(source_path, dest_path)
+      # It's not too fast as s3mirror (java tool)
+      max_threads = 500
+      wq = WorkQueue.new max_threads
+      execute_in_path source_path do |keys|
+        wq.enqueue_b do
+          _s3 = s3
+          keys.each do |key|
+            copy_key(_s3, source_path, dest_path, key)
+          end
         end
       end
     end
   end
 end
+
+

data/lib/manage_s3_bucket/version.rb

@@ -1,3 +1,3 @@
 module ManageS3Bucket
-  VERSION = "0.0.3"
+  VERSION = "0.0.4"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: manage_s3_bucket
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors:
 - Rodrigo
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-04-07 00:00:00.000000000 Z
+date: 2015-04-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: right_aws_api