cloud_files_transfer 0.0.4 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 01c05eb886b60bc720391b5b5b175239462b7c0b
-  data.tar.gz: 0af38c4b6be02f69838c976690b53fe3059754f0
+  metadata.gz: 4f731a4bd53072ba8ae3b2a843678c7a6f37fc5f
+  data.tar.gz: 39ffce01aec214d12aa17a6ed6f73f49a3b9bc75
 SHA512:
-  metadata.gz: f06c8f5516fe239271c6c278c2f28cd9f08dcaa0c5fd81e1541ba319e624c5a4f007f466a0beb6256ca7294a08461c848fa0928209678d49c454e4ba65e1bb8c
-  data.tar.gz: 78ff5243f818edb1b0bd7b34fa2a3883ff26087d1a89b1284a6238a0daf31c0e12bf5883c3488f0cd24926df3e32795a17f448d44dcdbca980266fbfc65f49cb
+  metadata.gz: 653c0a9edf5a6a4b73d965834e0001f8a4348005dfaf4b99879d46dc20435f71ad48460ef22b783cf812d15c0b34fbc98b54798e88430ca05dbd774bd4b8d7ba
+  data.tar.gz: 58a30b332c3607336ffe2b08764e5c1ba148e79d4fdb5f9f1884197cfd1d08bc6ad1f377427c97fa74b4aef9ef955a5ffde238caca0f50ba372498950d888727

data/README.md

@@ -36,7 +36,11 @@ destination:
 
 Then launch the rake task to start the transfer. It doesn't matter where you do it from. It could be faster though if done from a Rackspace instance using service net (snet).
 
-    $ rake cloud_files_assets:transfer
+    $ rake cloudfiles:transfer
+
+By default, it will create 4 threads to transfer simultaneously. You can specify the amount of threads like so:
+
+    $ rake cloudfiles:transfer jobs=8
 
 ## Contributing
 

data/cloud_files_transfer.gemspec

@@ -19,4 +19,5 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_dependency "cloudfiles"
+  spec.add_dependency "colorize"
 end

data/lib/cloud_files_transfer/client.rb

@@ -19,7 +19,7 @@ module CloudFilesTransfer
 
     def create_connection
       c = CloudFiles::Connection.new(username: username, api_key: api_key, snet: snet)
-      puts "Connection established."
+      puts "Connection established.".colorize(:green)
       c
     end
 

data/lib/cloud_files_transfer/transfer.rb

@@ -10,27 +10,49 @@ module CloudFilesTransfer
     end
 
     def self.copy!(origin, destination, path, args={})
-      return puts("#{path} skipped.") if destination.object_exists?(path)
+      return puts("#{path} skipped.".colorize(:light_blue)) if (destination.object_exists?(path) rescue false)
       new(origin, destination, path, args).copy
     end
 
     def copy
       retries = 0
-      puts "Saving #{path}"
       begin
         origin_object = origin.object(path)
         desintation_object = destination.create_object(path)
         desintation_object.write(origin_object.data)
+        success("#{path} saved")
       rescue Exception
         retries = retries + 1
-        unless retries > 5
-          puts "#{path} failed. Retry"
+        unless retries > 2
+          warn("#{path} failed. Retry")
           retry
         else
-          puts "#{path} failed."
+          fail("#{path} failed.")
         end
       end
     end
 
+    private
+
+    def info message
+      colored_print(message, :light_blue)
+    end
+
+    def warn message
+      colored_print(message, :yellow)
+    end
+
+    def success message
+      colored_print(message, :green)
+    end
+
+    def fail message
+      colored_print(message, :red)
+    end
+
+    def colored_print message, color
+      puts(message.colorize(color))
+    end
+
   end
 end

data/lib/cloud_files_transfer/version.rb

@@ -1,3 +1,3 @@
 module CloudFilesTransfer
-  VERSION = "0.0.4"
+  VERSION = "0.0.5"
 end

data/lib/cloud_files_transfer.rb

@@ -1,11 +1,12 @@
 require 'cloudfiles'
+require 'colorize'
 require 'yaml'
 require "cloud_files_transfer/version"
 require 'cloud_files_transfer/client'
-require 'cloud_files_transfer/container'
 require 'cloud_files_transfer/transfer'
 
 module CloudFilesTransfer
+
   class Railtie < Rails::Railtie
 
     rake_tasks do

data/lib/tasks/cloud_files_assets.rake

@@ -1,4 +1,4 @@
-namespace :cloud_files_assets do
+namespace :cloudfiles do
 
   desc "Copy assets from one container to another container"
   task transfer: :environment do
@@ -29,14 +29,26 @@ namespace :cloud_files_assets do
     )
 
     @container_from = @client_from.container
-    @container_to   = CloudFilesTransfer::Container.new(@client_to.container)
+    @container_to   = @client_to.container
     @assets = @container_from.objects
 
-    puts "Starting transfer. #{@assets.size} files to copy."
+    jobs_count = ENV['jobs'].try(:to_i) || 4
 
-    @assets.each do |path|
-      CloudFilesTransfer::Transfer.copy!(@container_from, @container_to, path)
+    puts "#{@assets.size} files to copy."
+    puts "Transfering #{jobs_count} files at a time. Here we go!"
+
+    @assets = @assets.in_groups(jobs_count)
+
+    Thread.abort_on_exception=true
+
+    threads = jobs_count.times.map do |i|
+      Thread.new(i) do |i|
+        @assets[i].each do |path|
+          CloudFilesTransfer::Transfer.copy!(@container_from, @container_to, path)
+        end
+      end
     end
+    threads.each {|t| t.join}
 
     puts "Done."
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cloud_files_transfer
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
 platform: ruby
 authors:
 - Mathieu Gagné
@@ -24,6 +24,20 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: colorize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Transfer Rackspace Cloud Files assets from one container to another or
   from account to another.
 email:
@@ -40,7 +54,6 @@ files:
 - cloud_files_transfer.gemspec
 - lib/cloud_files_transfer.rb
 - lib/cloud_files_transfer/client.rb
-- lib/cloud_files_transfer/container.rb
 - lib/cloud_files_transfer/transfer.rb
 - lib/cloud_files_transfer/version.rb
 - lib/tasks/cloud_files_assets.rake

data/lib/cloud_files_transfer/container.rb

@@ -1,38 +0,0 @@
-module CloudFilesTransfer
-  class Container
-
-    attr_accessor :container, :skip_existing_object
-
-    delegate :object_exists?, :create_object, to: :container
-
-    def initialize(container, args={})
-      @container = container
-      @skip_existing_object = args.fetch(:skip_existing_object, true)
-    end
-
-    def transfer(object)
-      return skip_message(object.name) if skip_existing_object and object_exists?(object.name)
-      retries = 0
-      puts "Saving #{object.name}"
-      begin
-        copy = create_object(object.name)
-        copy.write(object.data)
-      rescue Exception
-        retries = retries + 1
-        unless retries > 5
-          puts "#{object.name} failed. Retry"
-          retry
-        else
-          puts "#{object.name} failed."
-        end
-      end
-    end
-
-    private
-
-    def skip_message(object_name)
-      puts "#{object_name} skipped."
-    end
-
-  end
-end