cloudsync 2.1.1 → 2.2.0

data/VERSION

@@ -1 +1 @@
-2.1.1
+2.2.0

data/cloudsync.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{cloudsync}
-  s.version = "2.1.1"
+  s.version = "2.2.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Cory Forsyth"]
-  s.date = %q{2010-10-14}
+  s.date = %q{2010-10-15}
   s.default_executable = %q{cloudsync}
   s.description = %q{Sync files between various clouds or sftp servers. Available backends are S3, CloudFiles, and SFTP servers. Can sync, mirror, and prune.}
   s.email = %q{cory.forsyth@gmail.com}

data/lib/cloudsync/backend/base.rb

@@ -69,26 +69,21 @@ module Cloudsync
         raise NotImplementedError
       end
     
-      # all_files
-      def all_files
+      def files_to_sync(upload_prefix={})
         raise NotImplementedError
       end
       
-      def files_to_sync(upload_prefix={})
-        all_files
+      # get_file_from_store
+      def get_file_from_store(file)
+        raise NotImplementedError
       end
-      
+
       private
       
       def dry_run?
         return false unless @sync_manager
         @sync_manager.dry_run?
       end
-
-      # get_file_from_store
-      def get_file_from_store(file)
-        raise NotImplementedError
-      end
     end
   end
-end
+end

data/lib/cloudsync/backend/cloudfiles.rb

@@ -49,7 +49,13 @@ module Cloudsync
           container = get_or_create_container(container)
           objects_from_container(container, upload_prefix).each do |path, hash|
             next if hash[:content_type] == "application/directory"
-            files << Cloudsync::File.from_cf_info(container, path, hash, self.to_s)
+
+            file = Cloudsync::File.from_cf_info(container, path, hash, self.to_s)
+            if block_given?
+              yield file
+            else
+              files << file
+            end
           end
           files
         end
@@ -75,6 +81,10 @@ module Cloudsync
         $LOGGER.error("Failed to delete file #{file}")
       end
     
+      def get_file_from_store(file)
+        Cloudsync::File.from_cf_obj( get_obj_from_store(file), self.to_s )
+      end
+
       private
       
       def get_or_create_container(container_name)
@@ -90,6 +100,8 @@ module Cloudsync
       end
       
       def objects_from_container(container, upload_prefix)
+        $LOGGER.debug("Getting files from #{container.name}")
+        
         objects = []
         if upload_prefix[:prefix]
           container.objects_detail(:path => upload_prefix[:prefix]).collect do |path, hash|
@@ -104,6 +116,7 @@ module Cloudsync
         end
         objects
       end
+      
 
       def get_obj_from_store(file)
         @store.container(file.bucket).object(file.upload_path)
@@ -111,10 +124,6 @@ module Cloudsync
         nil
       end
     
-      def get_file_from_store(file)
-        Cloudsync::File.from_cf_obj( get_obj_from_store(file), self.to_s )
-      end
-    
       def get_or_create_obj_from_store(file)
         container = get_or_create_container(file.container)
       

data/lib/cloudsync/backend/s3.rb

@@ -67,18 +67,28 @@ module Cloudsync
       rescue RightAws::AwsError => e
         $LOGGER.error("Caught error: #{e} trying to delete #{file}")
       end
-    
+      
       def files_to_sync(upload_prefix="")
         $LOGGER.info("Getting files to sync [#{self}]")
         
         buckets_to_sync(upload_prefix).inject([]) do |files, bucket|
           objects_from_bucket(bucket, upload_prefix).collect do |key|
-            files << Cloudsync::File.from_s3_obj(key, self.to_s)
+            file = Cloudsync::File.from_s3_obj(key, self.to_s)
+            if block_given?
+              yield file
+            else
+              files << file
+            end
           end
           files
         end
       end
 
+      # Convenience to grab a single file
+      def get_file_from_store(file)
+        Cloudsync::File.from_s3_obj( get_obj_from_store(file), self.to_s )
+      end
+
       private
     
       def buckets_to_sync(upload_prefix="")
@@ -91,6 +101,8 @@ module Cloudsync
       end
       
       def objects_from_bucket(bucket, upload_prefix="")
+        $LOGGER.debug("Getting files from #{bucket}")
+        
         prefix_parts = upload_prefix.split("/")
         prefix_parts.shift
         prefix = prefix_parts.join("/")
@@ -102,17 +114,11 @@ module Cloudsync
         end
       end
     
-      # Convenience to grab a single file
-      def get_file_from_store(file)
-        Cloudsync::File.from_s3_obj( get_obj_from_store(file), self.to_s )
-      end
-    
       def get_or_create_obj_from_store(file)
         @store.bucket(file.bucket, true).key(file.upload_path)
       end
     
       def get_obj_from_store(file)
-        $LOGGER.debug("gofs, buck: #{file.bucket}. upload path: #{file.upload_path}")
         if bucket = @store.bucket(file.bucket)
           key = bucket.key(file.upload_path)
           return key if key.exists?

data/lib/cloudsync/backend/sftp.rb

@@ -63,19 +63,19 @@ module Cloudsync::Backend
       end
     end
     
-    def files_to_sync(upload_prefix={})
+    def files_to_sync(upload_prefix="")
       $LOGGER.info("Getting files to sync [#{self}]")
       files = []
       Net::SSH.start(@host, @username, :password => @password) do |ssh|
         ssh.sftp.connect do |sftp|
           filepaths = sftp.dir.glob(@download_prefix, "**/**").collect {|entry| entry.name}
         
-          files = filepaths.collect do |filepath|
+          filepaths.each do |filepath|
             attrs = sftp.stat!(local_filepath_from_filepath(filepath))
             next unless attrs.file?
 
             e_tag = ssh.exec!(md5sum_cmd(filepath)).split(" ").first
-            Cloudsync::File.new \
+            file = Cloudsync::File.new \
               :path            => filepath,
               :upload_prefix   => @upload_prefix,
               :download_prefix => @download_prefix,
@@ -84,31 +84,18 @@ module Cloudsync::Backend
               :e_tag           => e_tag,
               :backend         => self.to_s,
               :backend_type    => Cloudsync::Backend::Sftp
-          end.compact
+            
+            if block_given?
+              yield file
+            else
+              files << file
+            end
+          end
         end
       end
       files
     end
     
-    # def absolute_path(path)
-    #   @download_prefix + "/" + path
-    # end
-    
-    private
-    
-    def md5sum_cmd(filepath)
-      Escape.shell_command(["md5sum","#{local_filepath_from_filepath(filepath)}"])
-    end
-    
-    def local_filepath_from_filepath(filepath)
-      stripped_path = filepath.sub(/^#{@upload_prefix}\/?/,"")
-      if @download_prefix
-        "#{@download_prefix}/#{stripped_path}"
-      else
-        stripped_path
-      end
-    end
-    
     # get_file_from_store
     def get_file_from_store(file)
       $LOGGER.debug("Looking for local filepath: #{local_filepath_from_filepath(file.full_download_path)}")
@@ -137,5 +124,20 @@ module Cloudsync::Backend
       end
       sftp_file
     end
+    
+    private
+    
+    def md5sum_cmd(filepath)
+      Escape.shell_command(["md5sum","#{local_filepath_from_filepath(filepath)}"])
+    end
+    
+    def local_filepath_from_filepath(filepath)
+      stripped_path = filepath.sub(/^#{@upload_prefix}\/?/,"")
+      if @download_prefix
+        "#{@download_prefix}/#{stripped_path}"
+      else
+        stripped_path
+      end
+    end
   end 
 end

data/lib/cloudsync/sync_manager.rb

@@ -74,11 +74,9 @@ module Cloudsync
       
       $LOGGER.info("[SM]: Prune from #{from_backend} to #{to_backend} started at #{prune_start = Time.now}. Dry-run? #{!!dry_run?}")
       
-      to_backend_files     = to_backend.files_to_sync(from_backend.upload_prefix)
-      total_files          = to_backend_files.size
-      last_decile_complete = 0
-      
-      to_backend_files.each_with_index do |file, index|
+      index                = 1
+
+      to_backend.files_to_sync(from_backend.upload_prefix) do |file|
         $LOGGER.debug("Checking if file #{file} exists on [#{from_backend}]")
         if found_file = from_backend.get_file_from_store(file)
           $LOGGER.debug("Keeping file #{file} because it was found on #{from_backend}.")
@@ -90,10 +88,11 @@ module Cloudsync
           to_backend.delete(file)
         end
         
-        if decile_complete(index, total_files) != last_decile_complete
-          last_decile_complete = decile_complete(index, total_files)
-          $LOGGER.info("[SM]: Prune: Completed #{index} files (skipped: #{file_stats[:skipped].size}, removed: #{file_stats[:removed].size}). #{last_decile_complete * 10}% complete")
+        if index % 1000 == 0
+          $LOGGER.info("[SM]: Prune: Completed #{index} files (skipped: #{file_stats[:skipped].size}, removed: #{file_stats[:removed].size}).")
         end
+        
+        index += 1
       end
       
       $LOGGER.info(["[SM]: Prune from #{from_backend} to #{to_backend} finished at #{Time.now}, took #{Time.now - prune_start}s.",
@@ -106,11 +105,9 @@ module Cloudsync
       file_stats = {:copied => [], :skipped => []}
       $LOGGER.info("[SM]: Sync from #{from_backend} to #{to_backend} started at #{sync_start = Time.now}. Mode: #{mode}. Dry-run? #{!!dry_run?}")
 
-      from_backend_files   = from_backend.files_to_sync(to_backend.upload_prefix)
-      total_files          = from_backend_files.size
-      last_decile_complete = 0
-      
-      from_backend_files.each_with_index do |file, index|
+      index = 1
+
+      from_backend.files_to_sync(to_backend.upload_prefix) do |file|
         if (mode == :sync_all || to_backend.needs_update?(file))
           file_stats[:copied] << file
           from_backend.copy(file, to_backend)
@@ -119,10 +116,11 @@ module Cloudsync
           $LOGGER.debug("Skipping up-to-date file #{file}")
         end
         
-        if decile_complete(index, total_files) != last_decile_complete
-          last_decile_complete = decile_complete(index, total_files)
-          $LOGGER.info("[SM]: Sync from #{from_backend} to #{to_backend}: Completed #{index} files (skipped: #{file_stats[:skipped].size}, copied: #{file_stats[:copied].size}). #{last_decile_complete * 10}% complete")
+        if index % 1000 == 0
+          $LOGGER.info("[SM]: Sync from #{from_backend} to #{to_backend}: Completed #{index} files (skipped: #{file_stats[:skipped].size}, copied: #{file_stats[:copied].size}).")
         end
+        
+        index += 1
       end
       
       $LOGGER.info(["[SM]: Sync from #{from_backend} to #{to_backend} finished at #{Time.now}, took #{Time.now - sync_start}s.",

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: cloudsync
 version: !ruby/object:Gem::Version 
-  hash: 9
+  hash: 7
   prerelease: false
   segments: 
   - 2
-  - 1
-  - 1
-  version: 2.1.1
+  - 2
+  - 0
+  version: 2.2.0
 platform: ruby
 authors: 
 - Cory Forsyth
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-10-14 00:00:00 -04:00
+date: 2010-10-15 00:00:00 -04:00
 default_executable: cloudsync
 dependencies: 
 - !ruby/object:Gem::Dependency