cloudsync 2.2.1 → 2.3.0

data/Rakefile

@@ -11,7 +11,7 @@ begin
     gem.homepage = "http://github.com/megaphone/cloudsync"
     gem.authors = ["Cory Forsyth"]
     gem.add_dependency "right_aws", "~> 2.0.0"
-    gem.add_dependency "cloudfiles", "~> 1.4.8"
+    gem.add_dependency "cloudfiles", "~> 1.4.9"
     gem.add_dependency "commander", "~> 4.0.3"
     gem.add_dependency "net-ssh", "~> 2.0.19"
     gem.add_dependency "net-sftp", "~> 2.0.4"

data/VERSION

@@ -1 +1 @@
-2.2.1
+2.3.0

data/cloudsync.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{cloudsync}
-  s.version = "2.2.1"
+  s.version = "2.3.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Cory Forsyth"]
-  s.date = %q{2010-10-15}
+  s.date = %q{2010-10-20}
   s.default_executable = %q{cloudsync}
   s.description = %q{Sync files between various clouds or sftp servers. Available backends are S3, CloudFiles, and SFTP servers. Can sync, mirror, and prune.}
   s.email = %q{cory.forsyth@gmail.com}
@@ -53,14 +53,14 @@ Gem::Specification.new do |s|
 
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
       s.add_runtime_dependency(%q<right_aws>, ["~> 2.0.0"])
-      s.add_runtime_dependency(%q<cloudfiles>, ["~> 1.4.8"])
+      s.add_runtime_dependency(%q<cloudfiles>, ["~> 1.4.9"])
       s.add_runtime_dependency(%q<commander>, ["~> 4.0.3"])
       s.add_runtime_dependency(%q<net-ssh>, ["~> 2.0.19"])
       s.add_runtime_dependency(%q<net-sftp>, ["~> 2.0.4"])
       s.add_runtime_dependency(%q<escape>, ["~> 0.0.4"])
     else
       s.add_dependency(%q<right_aws>, ["~> 2.0.0"])
-      s.add_dependency(%q<cloudfiles>, ["~> 1.4.8"])
+      s.add_dependency(%q<cloudfiles>, ["~> 1.4.9"])
       s.add_dependency(%q<commander>, ["~> 4.0.3"])
       s.add_dependency(%q<net-ssh>, ["~> 2.0.19"])
       s.add_dependency(%q<net-sftp>, ["~> 2.0.4"])
@@ -68,7 +68,7 @@ Gem::Specification.new do |s|
     end
   else
     s.add_dependency(%q<right_aws>, ["~> 2.0.0"])
-    s.add_dependency(%q<cloudfiles>, ["~> 1.4.8"])
+    s.add_dependency(%q<cloudfiles>, ["~> 1.4.9"])
     s.add_dependency(%q<commander>, ["~> 4.0.3"])
     s.add_dependency(%q<net-ssh>, ["~> 2.0.19"])
     s.add_dependency(%q<net-sftp>, ["~> 2.0.4"])

data/lib/cloudsync/backend/base.rb

@@ -3,6 +3,10 @@ require 'tempfile'
 module Cloudsync
   module Backend
     class Base
+      OBJECT_LIMIT    = 250
+      CONTAINER_LIMIT = 250
+      BUCKET_LIMIT    = CONTAINER_LIMIT
+      
       attr_accessor :store, :sync_manager, :name, :upload_prefix
       
       def initialize(opts = {})
@@ -80,6 +84,13 @@ module Cloudsync
 
       private
       
+      def remove_container_name(string)
+        parts = string.split("/")
+        parts.shift
+        parts.join("/")
+      end
+      alias :remove_bucket_name :remove_container_name
+      
       def dry_run?
         return false unless @sync_manager
         @sync_manager.dry_run?

data/lib/cloudsync/backend/cloudfiles.rb

@@ -42,12 +42,13 @@ module Cloudsync
         $LOGGER.debug("Finished putting #{file} to #{self} (#{Time.now - start_time}s)")
       end
     
-      def files_to_sync(upload_prefix={})
+      def files_to_sync(upload_prefix="")
         $LOGGER.info("Getting files to sync [#{self}]")
-
-        containers_to_sync(upload_prefix).inject([]) do |files, container|
+        
+        files = []
+        containers_to_sync(upload_prefix) do |container|
           container = get_or_create_container(container)
-          objects_from_container(container, upload_prefix).each do |path, hash|
+          objects_from_container(container, remove_container_name(upload_prefix)) do |path, hash|
             next if hash[:content_type] == "application/directory"
 
             file = Cloudsync::File.from_cf_info(container, path, hash, self.to_s)
@@ -96,27 +97,56 @@ module Cloudsync
       end
 
       def containers_to_sync(upload_prefix)
-        upload_prefix[:bucket] ? [upload_prefix[:bucket]] : @store.containers
+        container_name = upload_prefix.split("/").first
+        if container_name
+          yield container_name
+        else
+          last_marker = nil
+          loop do
+            containers = @store.containers(CONTAINER_LIMIT, last_marker)
+            break if containers.empty?
+            containers.each do |container| 
+              last_marker = container
+              yield container
+            end
+          end
+        end
       end
       
-      def objects_from_container(container, upload_prefix)
-        $LOGGER.debug("Getting files from #{container.name}")
-        
-        objects = []
-        if upload_prefix[:prefix]
-          container.objects_detail(:path => upload_prefix[:prefix]).collect do |path, hash|
-            if hash[:content_type] == "application/directory"
-              objects += objects_from_container(container, :prefix => path) 
-            else
-              objects << [path, hash]
+      # cf = Cloudsync::Backend::CloudFiles.new( YAML::load_file("cloudsync.yml")[:cloudfiles]); $LOGGER = Logger.new(STDOUT); count = 0; cf.files_to_sync {|p,h| count += 1 }; puts count
+      # cf = Cloudsync::Backend::CloudFiles.new( YAML::load_file("cloudsync.yml")[:cloudfiles]); $LOGGER = Logger.new(STDOUT); count = 0; paths = []; cf.files_to_sync("mpsounds-adobe.max.trivia") {|f| count += 1; paths << f.path}; puts count
+      
+      
+      # prefix_path must not include the container name at the beginning of the string
+      def objects_from_container(container, prefix_path="", &block)
+        $LOGGER.debug("Getting files from #{container.name} (prefix: #{prefix_path})")
+
+        last_marker     = nil
+        loop do
+          params          = {:limit => OBJECT_LIMIT, :marker => last_marker}
+          params[:path]   = prefix_path if !prefix_path.empty?
+          
+          $LOGGER.debug("OFC #{container.name} (#{prefix_path}) loop: #{params.inspect}")
+          
+          objects_details = container.objects_detail(params)
+
+          $LOGGER.debug("OFC #{container.name} (#{prefix_path}) got #{objects_details.size}. #{objects_details.class}")
+          $LOGGER.debug("-"*50)
+          
+          break if objects_details.empty?
+          
+          objects_details.sort.each do |path, hash|
+            if hash[:content_type] == "application/directory" && !prefix_path.empty?
+              $LOGGER.debug("OFC #{container.name} (#{prefix_path}) recursing into #{path}")
+              objects_from_container(container, path, &block)
+              $LOGGER.debug("OFC #{container.name} (#{prefix_path}) done recursing into #{path}")
             end
+            
+            last_marker = path
+            yield path, hash
           end
-        else
-          objects = container.objects_detail
         end
-        objects
       end
-      
 
       def get_obj_from_store(file)
         @store.container(file.bucket).object(file.upload_path)

data/lib/cloudsync/backend/s3.rb

@@ -72,7 +72,7 @@ module Cloudsync
         $LOGGER.info("Getting files to sync [#{self}]")
         
         buckets_to_sync(upload_prefix).inject([]) do |files, bucket|
-          objects_from_bucket(bucket, upload_prefix).collect do |key|
+          objects_from_bucket(bucket, upload_prefix) do |key|
             file = Cloudsync::File.from_s3_obj(key, self.to_s)
             if block_given?
               yield file
@@ -90,6 +90,9 @@ module Cloudsync
       end
 
       private
+
+      # cf = Cloudsync::Backend::S3.new( YAML::load_file("cloudsync.yml")[:s3]); $LOGGER = Logger.new(STDOUT); count = 0; cf.files_to_sync {|p,h| count += 1 }; puts count
+
     
       def buckets_to_sync(upload_prefix="")
         bucket_name = upload_prefix.split("/").first
@@ -103,14 +106,21 @@ module Cloudsync
       def objects_from_bucket(bucket, upload_prefix="")
         $LOGGER.debug("Getting files from #{bucket}")
         
-        prefix_parts = upload_prefix.split("/")
-        prefix_parts.shift
-        prefix = prefix_parts.join("/")
+        prefix = remove_bucket_name(upload_prefix)
+        params = {'marker' => "", 'max-keys' => OBJECT_LIMIT}
+        params['prefix'] = prefix if !prefix.nil?
         
-        if !prefix.empty?
-          bucket.keys(:prefix => prefix)
-        else
-          bucket.keys
+        loop do
+          $LOGGER.debug(params.inspect)
+          keys = bucket.keys(params)
+          break if keys.empty?
+          
+          keys.each do |key|
+            $LOGGER.debug(key.name)
+            yield key
+          end
+
+          params['marker'] = keys.last.name
         end
       end
     

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: cloudsync
 version: !ruby/object:Gem::Version 
-  hash: 5
+  hash: 3
   prerelease: false
   segments: 
   - 2
-  - 2
-  - 1
-  version: 2.2.1
+  - 3
+  - 0
+  version: 2.3.0
 platform: ruby
 authors: 
 - Cory Forsyth
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-10-15 00:00:00 -04:00
+date: 2010-10-20 00:00:00 -04:00
 default_executable: cloudsync
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -42,12 +42,12 @@ dependencies:
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
-        hash: 23
+        hash: 21
         segments: 
         - 1
         - 4
-        - 8
-        version: 1.4.8
+        - 9
+        version: 1.4.9
   type: :runtime
   version_requirements: *id002
 - !ruby/object:Gem::Dependency