multi_sync 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 580e01265e2768d6a728b4dc3b29adad42874e7b
-  data.tar.gz: 09575ab0fa096363ef1deeeb94a435e6dc1a5605
+  metadata.gz: e9f1a53316b76079921573131ddede125112c3c4
+  data.tar.gz: 36193eacd42d2c64268688cb0a2068c5071f306c
 SHA512:
-  metadata.gz: 5c729e36efd763755ca1d1f85827fc986a7e90823ab1199c900620d8f012f2f7182ce432d91fdf3292063f3ab74f11a146690814551c074329d33c3653dc2b4f
-  data.tar.gz: 3968ae066b80ef8fb122c7b90aa7817ec1e7b8646091d7ff5af8e9832bcaa9da506e8dc746800664e49df5918bc1874a847e6cc1768c41c37f281a0e8677f81a
+  metadata.gz: aadc84bd39bd8fdd4dcddd2c2ee70d3493f49db344cd29df962bb088b5759f76f5d5dd97039f49c858e3cfe02db3078080cc984819146643c6797d17e8087e86
+  data.tar.gz: b6ab4df2946516aea524b84a8d506e272646faa4180a74f1f8eabcd5701c238ad234d964c0df02c95a9ceb90e00ddbe949feeebc88d2df93387a9a40a09f869f

data/.travis.yml

@@ -3,22 +3,20 @@ cache: bundler
 bundler_args: --without development
 rvm:
   - ruby-head
+  - ruby
   - jruby-head
+  - jruby
   - 2.1.0
   - 2.0.0
   - 1.9.3
-  - rbx-19mode
-  - jruby-19mode
-# gemfile:
-#   - gemfiles/Gemfile.middleman-3.1.x
-#   - gemfiles/Gemfile.rails-3.2.x
-#   - Gemfile
+  - rbx-2
 matrix:
   fast_finish: true
   allow_failures:
     - rvm: ruby-head
+    - rvm: ruby
     - rvm: jruby-head
-    - rvm: rbx-19mode
-    - rvm: jruby-19mode
+    - rvm: jruby
+    - rvm: rbx-2
 notifications:
   email: false

data/Gemfile

@@ -12,7 +12,6 @@ end
 
 group :test do
   gem 'rake', '~> 10.0'
-  gem 'fakefs', '~> 0.4.3'
   gem 'rspec'
   gem 'timecop'
 end

data/README.md

@@ -2,7 +2,7 @@
 
 :heavy_exclamation_mark: **currently a functioning WIP thats not quite finished yet but its close!** :heavy_exclamation_mark:
 
-A flexible synchronisation library for your assets.
+Flexible synchronisation for your assets.
 
 `MultiSync` stands on the shoulders of giants. On one side is [Celluloid](http://celluloid.io) allowing for the synchronisation of assets to be highly parallel. On the other is [Fog::Storage](https://github.com/fog/fog) allowing `MulitSync` to support [various well known storage services](#storage-services).
 
@@ -15,7 +15,7 @@ Listed below are examples of how to get setup and started.
 ## Installation
 
 ```ruby
-gem 'multi_sync', '~> 0.0.1'
+gem 'multi_sync', '~> 0.0.2'
 ```
 
 ```ruby
@@ -43,104 +43,132 @@ end
 
 ### Source
 
-A source takes two arguments. The first is a `name` to reference this source by and the second is a `Hash` of configuration detailed below.
+All `source`s takes one argument which is a `Hash` of configuration detailed below. There are currently two type's of `source`s which are
+
+#### Source Types
+
+- `local_source` - Uses all files within the `source_dir`
+- `manifest_source` - Tries to find a `Sprocket`s `manifest.{yml,json}` file within the `source_dir`
 
 | Key | Type | Default | Description |
 | :-- | :--- | :------ | :---------- |
-| `type` | `Symbol` | `nil` | The `type` of source this is (`:local`, `:manifest`) |
-| `source_dir` | `Pathname`, `String` | `nil` | The location this source should use |
-| `resource_options` | `Hash` | `nil` | A hash of options for this source`s resources |
-| `targets` | `Symbol`, `Array` | All targets | The target(s) this source should sync against |
+| `source_dir` | `Pathname`, `String` | `nil` | The location this `source` should use |
+| `resource_options` | `Hash` | `{}` | A hash of options for this `source`'s resources |
+| `targets` | `Symbol`, `Array` | All targets | The `target`(s) this `source` should sync against |
 | `include` | `String` ([shell glob](http://www.ruby-doc.org/core-2.1.1/Dir.html#method-c-glob)) | `**/*` | A shell globe to use for inclusion |
 | `exclude` | `String` ([shell glob](http://www.ruby-doc.org/core-2.1.1/Dir.html#method-c-glob)) | `nil` | A shell globe to use for exclusion |
 ___
 
 ```ruby
-# A source named ':build' which is ':local' and will use all files within '../build'
-source :build, {
-  type: :local,
+# A `local` `source` which will use all files within '../build'
+local_source({
   source_dir: '../build'
-}
+})
 ```
 ___
 
 ```ruby
-# A source named ':assets' which will use a Sprockets ':manifest' within '../public/assets'
-source :assets, {
-  type: :manifest,
+# A `manifest` `source` which will use a Sprockets ':manifest' within '../public/assets'
+manifest_source({
   source_dir: '../public/assets'
-}
+})
 ```
 ___
 
 ```ruby
-# A source named ':video_assets' which is `:local' and will use all files
-# within '../build' including only 'mp4, mpg, mov'
-source :video_assets, {
-  type: :local,
+# A `local` `source` which will use all files
+# within '../build' including only 'mp4, mpg, mov' files
+local_source({
   source_dir: '../build',
   include: '*.{mp4,mpg,mov}'
-}
+})
 ```
 ___
 
 ```ruby
-# A source named ':no_images' which is `:local' and will use all files
-# within '../build' excluding any 'jpg, gif, png'
-source :no_images, {
-  type: :local,
+# A `local` `source` which will use all files
+# within '../build' excluding any 'jpg, gif, png' files
+local_source({
   source_dir: '../build',
   exclude: '*.{jpg,gif,png}'
-}
+})
 ```
 ___
 
 ```ruby
-# A source named ':www' which will use a Sprockets ':manifest'
-# within '../public/assets' excluding any 'jpg, gif, png' files
-# and only synchronising with a target named `:www`
-source :www, {
-  type: :manifest,
-  source_dir: '../public/assets',
-  exclude: '*.{jpg,gif,png}',
-  targets: :www
-}
-```
-___
-
-```ruby
-# A source named ':image_assets' which will use a Sprockets ':manifest'
+# A `manifest` `source` which will use use a Sprockets `manifest`
 # within '../public/assets' including only 'jpg, gif, png' files
-# which sets `cache_control` and `expires` headers and
-# synchronises with the target `:images`
-source :image_assets, {
-  type: :manifest,
+# which sets `cache_control` and `expires` headers
+manifest_source({
   source_dir: '../public/assets',
   include: '*.{jpg,gif,png}',
   resource_options: {
     cache_control: 'public, max-age=31557600',
     expires: CGI.rfc1123_date(Time.now + 31557600)
-  },
-  targets: :images
-}
+  }
+})
 ```
 
 ### Target
 
+All `target`s takes one argument which is a `Hash` of configuration detailed below. There is currently only one `target` type which is:
+
+#### Target Types
+
+- `aws_target` - Synchronises to `aws` (`S3`)
+
+| Key | Type | Default | Description |
+| :-- | :--- | :------ | :---------- |
+| `target_dir` | `Pathname`, `String` | `nil` | the name of the `target`'s directory (eg s3 bucket name) |
+| `destination_dir` | `Pathname`, `String` | `nil` | the name of the `target` destination's directory (eg folder within target) |
+| `credentials` | `Hash` | inherits [Fog Credentials](https://github.com/karlfreeman/multi_sync#fog-credentials-support) | credentionals needed by [Fog](http://fog.io) |
+___
+
+```ruby
+# An `aws` `target` which will sync to the root of a bucket named 's3-bucket-name'
+# in region 'us-east-1', with access_key_id 'xxx', and secret_access_key 'xxx'
+aws_target({
+  target_dir: 's3-bucket-name'
+  credentials: {
+    region: 'us-east-1',
+    aws_access_key_id: 'xxx',
+    aws_secret_access_key: 'xxx'
+  }
+})
+```
+___
+
 ```ruby
-...
+# An `aws` `target` which will sync to a bucket named 's3-bucket-name'
+# using credentials sourced from Fog's credentials (.fog or FOG_RC)
+aws_target({
+  target_dir: 's3-bucket-name'
+})
 ```
+___
 
+```ruby
+# An `aws` `target` which will sync to a bucket named 's3-bucket-name'
+# within a directory named 'directory-within-s3'
+# in region 'us-east-1', with access_key_id 'xxx', and secret_access_key 'xxx'
+aws_target({
+  target_dir: 's3-bucket-name'
+  destination_dir: 'directory-within-s3'
+  credentials: {
+    region: 'us-east-1',
+    aws_access_key_id: 'xxx',
+    aws_secret_access_key: 'xxx'
+  }
+})
+```
 
 ## Supported Libraries
 
-- [Rails](https://github.com/karlfreeman/multi_sync/wiki/rails)
+- [Rails](https://github.com/karlfreeman/multi_sync-rails)
 - Sinatra (WIP)
-- Middleman (WIP)
+- [Middleman](https://github.com/karlfreeman/multi_sync-middleman)
 - Jekyll (WIP)
 - Nanoc (WIP)
-- [POR](https://github.com/karlfreeman/multi_sync/wiki/por)
-- Rake (WIP)
 
 ## Badges
 

data/Rakefile

@@ -19,27 +19,5 @@ begin
 rescue LoadError
 end
 
-namespace :spec do
-
-  desc 'Run specs with middleman'
-  task :middleman do
-    ENV['BUNDLE_GEMFILE'] = 'gemfiles/middleman-3.1.x.gemfile'
-    Rake::Task['spec'].execute
-  end
-
-  desc 'Run specs with rails 3.2'
-  task :rails_3_2 do
-    ENV['BUNDLE_GEMFILE'] = 'gemfiles/rails-3.2.x.gemfile'
-    Rake::Task['spec'].execute
-  end
-
-  desc 'Run specs with rails 4.0'
-  task :rails_4_0 do
-    ENV['BUNDLE_GEMFILE'] = 'gemfiles/rails-4.0.x.gemfile'
-    Rake::Task['spec'].execute
-  end
-
-end
-
 task default: :spec
 task test: :spec

data/lib/multi_sync.rb

@@ -12,8 +12,9 @@ module MultiSync
 
   # a list of libraries, extension file and class name
   REQUIREMENT_MAP = [
-    ['rails', 'multi_sync/extensions/rails'].freeze,
-    ['middleman-core', 'multi_sync/extensions/middleman'].freeze
+    ['rails', 'multi_sync/extensions/rails'],
+    ['middleman-core', 'multi_sync/extensions/middleman'],
+    ['jekyll', 'multi_sync/extensions/jekyll']
   ]
 
   # delegate all MultiSync::Configuration's attribute accessors to the configuration
@@ -25,8 +26,22 @@ module MultiSync
   # delegate all MultiSync::Client's attribute accessors to the configuration
   def_delegators :client, *MultiSync::Client.attribute_set.map(&:name)
 
-  # include some public methods
-  def_delegators :client, :target, :source, :synchronize
+   # include sync method
+  def_delegator :client, :sync
+
+  # create methods for each source (local_source(options), manifest_source(options))
+  MultiSync::Client::SUPPORTED_SOURCE_TYPES.each do |type, clazz|
+    define_singleton_method "#{type}_source" do |options = {}|
+      client.add_source(clazz, options)
+    end
+  end
+
+  # create methods for each target (aws_target(options))
+  MultiSync::Client::SUPPORTED_TARGET_TYPES.each do |type, clazz|
+    define_singleton_method "#{type}_target" do |options = {}|
+      client.add_target(clazz, options)
+    end
+  end
 
   # Configure
   #
@@ -40,7 +55,7 @@ module MultiSync
   #
   # @return [MultiSync]
   def self.run(&block)
-    configure(&block).synchronize
+    configure(&block).sync
   end
 
   # Prepare
@@ -64,6 +79,13 @@ module MultiSync
     @configuration ||= MultiSync::Configuration.new(options)
   end
 
+  # Return the MultiSync::VERSION
+  #
+  # @return [String]
+  def self.version
+    MultiSync::VERSION
+  end
+
   # Reset the MultiSync::Client
   def self.reset_client!
     remove_instance_variable :@client if defined?(@client)

data/lib/multi_sync/attributes/pathname.rb

@@ -9,4 +9,4 @@ module MultiSync
       end
     end
   end
-end
+end

data/lib/multi_sync/client.rb

@@ -1,27 +1,31 @@
-require 'set'
 require 'virtus'
 require 'lazily'
 require 'celluloid'
-%w(sources targets).each do |dir|
+%w(sources targets helpers).each do |dir|
   Dir.glob(File.expand_path("../#{dir}/**/*.rb", __FILE__), &method(:require))
 end
-require 'multi_sync/mixins/pluralize_helper'
 
 module MultiSync
   class Client
     include Virtus.model
-    include MultiSync::Mixins::PluralizeHelper
+    include MultiSync::Helpers::Pluralize
 
     attribute :supervisor, Celluloid::SupervisionGroup
-    attribute :incomplete_jobs, Set, default: Set.new
-    attribute :running_jobs, Set, default: Set.new
-    attribute :complete_jobs, Set, default: Set.new
+    attribute :running_upload_jobs, Array, default: []
+    attribute :running_delete_jobs, Array, default: []
+    attribute :complete_upload_jobs, Array, default: []
+    attribute :complete_delete_jobs, Array, default: []
+    attribute :incomplete_upload_jobs, Array, default: []
+    attribute :incomplete_delete_jobs, Array, default: []
     attribute :sources, Array, default: []
     attribute :sync_attempts, Integer, default: 0
     attribute :file_sync_attempts, Integer, default: 0
     attribute :started_at, Time, required: false
     attribute :finished_at, Time, required: false
 
+    SUPPORTED_SOURCE_TYPES = [[:local, MultiSync::LocalSource], [:manifest, MultiSync::ManifestSource]]
+    SUPPORTED_TARGET_TYPES = [[:local, MultiSync::LocalTarget], [:aws, MultiSync::AwsTarget]]
+
     # Initialize a new Client object
     #
     # @param options [Hash]
@@ -30,94 +34,63 @@ module MultiSync
       super
     end
 
-    def add_target(name, options = {})
-      fail ArgumentError, "Duplicate target names detected, please rename '#{name}' to be unique" if supervisor_actor_names.include?(name)
-      clazz = MultiSync.const_get("#{options[:type].capitalize}Target")
-      supervisor.pool(clazz, as: name, args: [options], size: MultiSync.target_pool_size)
-    rescue NameError
-      MultiSync.warn "Unknown target type: #{options[:type]}"
-      raise ArgumentError, "Unknown target type: #{options[:type]}"
+    #
+    #
+    #
+    def add_target(clazz, options = {})
+      # TODO: friendly pool names?
+      pool_name = Celluloid.uuid
+      supervisor.pool(clazz, as: pool_name, args: [options], size: MultiSync.target_pool_size)
+      pool_name
     end
-    alias_method :target, :add_target
 
-    def add_source(name, options = {})
-      clazz = MultiSync.const_get("#{options[:type].capitalize}Source")
-      sources << clazz.new(options)
-    rescue NameError
-      MultiSync.warn "Unknown source type: #{options[:type]}"
-      raise ArgumentError, "Unknown source type: #{options[:type]}"
+    #
+    #
+    #
+    def add_source(clazz, options = {})
+      source = clazz.new(options)
+      sources << source
+      source
     end
-    alias_method :source, :add_source
-
-    def synchronize
-      MultiSync.debug 'Preventing synchronization as there are no sources found.' && return if sync_pointless?
-      MultiSync.debug 'Starting synchronization...'
 
-      determine_sync if first_run?
-      sync_attempted
+    #
+    #
+    #
+    def sync
+      MultiSync.warn 'Preventing synchronization as there are no sources found.' && return if sync_pointless?
 
-      MultiSync.debug 'Scheduling jobs in the future...'
-      incomplete_jobs.delete_if do | job |
-        running_jobs << { id: job[:id], future: Celluloid::Actor[job[:target_id]].future.send(job[:method], job[:args]), method: job[:method] }
+      if first_run?
+        MultiSync.debug 'Starting synchronization...'
+        determine_sync
+      else
+        MultiSync.debug 'Restarting synchronization...'
       end
 
-      MultiSync.debug 'Fetching jobs from the future...'
-      running_jobs.delete_if do | job |
+      sync_attempted
+
+      MultiSync.debug 'Fetching upload jobs from the future...'
+      (running_upload_jobs | incomplete_upload_jobs).lazily.each do | job |
         begin
-          completed_job = { id: job[:id], response: job[:future].value, method: job[:method] }
+          complete_upload_jobs << job.value
         rescue => error
           self.file_sync_attempts = file_sync_attempts + 1
           MultiSync.warn error.inspect
-          false
-        else
-          complete_jobs << completed_job
-          true
+          incomplete_upload_jobs << job
         end
       end
 
-      finish_sync
-      finalize
-    end
-    alias_method :sync, :synchronize
-
-    def finalize
-      if finished_at
-        elapsed = finished_at.to_f - started_at.to_f
-        minutes, seconds = elapsed.divmod 60.0
-        bytes = get_total_file_size_from_complete_jobs
-        MultiSync.debug "Sync completed in #{pluralize(minutes.round, 'minute')} and #{pluralize(seconds.round, 'second')}"
-        MultiSync.debug "#{pluralize(complete_jobs.length, 'file')} were synchronised (#{pluralize(get_complete_deleted_jobs.length, 'deleted file')} and #{pluralize(get_complete_upload_jobs.length, 'uploaded file')}) from #{pluralize(sources.length, 'source')} to #{pluralize(supervisor.actors.length, 'target')}"
-        if bytes > 1024.0
-          kilobytes = bytes / 1024.0
-          MultiSync.debug "The upload weight totalled %.#{0}f #{pluralize(kilobytes, 'kilobyte', nil, false)}" % kilobytes
-        else
-          MultiSync.debug "The upload weight totalled %.#{0}f #{pluralize(bytes, 'byte', nil, false)}" % bytes
+      MultiSync.debug 'Fetching delete jobs from the future...'
+      (running_delete_jobs | incomplete_delete_jobs).lazily.each do | job |
+        begin
+          complete_delete_jobs << job.value
+        rescue => error
+          self.file_sync_attempts = file_sync_attempts + 1
+          MultiSync.warn error.inspect
+          incomplete_delete_jobs << job
         end
-        MultiSync.debug "#{pluralize(file_sync_attempts, 'failed request')} were detected and re-tried"
-      else
-        MultiSync.debug "Sync failed to complete with #{pluralize(incomplete_jobs.length, 'outstanding file')} to be synchronised"
-        MultiSync.debug "#{pluralize(complete_jobs.length, 'file')} were synchronised (#{pluralize(get_complete_deleted_jobs.length, 'deleted file')} and #{pluralize(get_complete_upload_jobs.length, 'uploaded file')}) from #{pluralize(sources.length, 'source')} to #{pluralize(supervisor.actors.length, 'target')}"
       end
 
-      supervisor.finalize
-    end
-    alias_method :fin, :finalize
-
-    def get_complete_deleted_jobs
-      complete_jobs.select { |job| job[:method] == :delete }
-    end
-
-    def get_complete_upload_jobs
-      complete_jobs.select { |job| job[:method] == :upload }
-    end
-
-    def get_total_file_size_from_complete_jobs
-      total_file_size = 0
-      get_complete_upload_jobs.each do | job |
-        job_content_length = job[:response].content_length || job[:response].determine_content_length || 0
-        total_file_size += job_content_length
-      end
-      total_file_size
+      finish_sync
     end
 
     private
@@ -129,10 +102,9 @@ module MultiSync
 
         starting_synchronizing_msg = "ynchronizing: '#{source.source_dir}'"
         starting_synchronizing_msg.prepend MultiSync.force ? 'Forcefully s' : 'S'
-        MultiSync.info starting_synchronizing_msg
+        MultiSync.debug starting_synchronizing_msg
 
         source_files = source.files
-        source_files.sort! # sort to make sure the source's indexes match the targets
 
         # when no targets are specified, assume all targets
         source.targets = supervisor_actor_names if source.targets.empty?
@@ -147,7 +119,7 @@ module MultiSync
 
           MultiSync.debug 'Fetching files from the target...'
 
-          target_files = Celluloid::Actor[target_id].files
+          target_files = supervisor[target_id].files
           target_files.sort! # sort to make sure the target's indexs match the sources
 
           MultiSync.debug "#{pluralize(target_files.length, 'file')} found from the target"
@@ -162,28 +134,30 @@ module MultiSync
           abandoned_files_msg += ", however we're skipping them as :delete_abandoned_files is false" unless MultiSync.delete_abandoned_files
           MultiSync.debug abandoned_files_msg
 
-          # remove missing_files from source_files (as we know they are missing so don't need to check them)
-          # remove abandoned_files from target_files (as we know they are abandoned so don't need to check them)
+          # remove missing_files from source_files (as we know they're missing so don't need to check for them)
+          # remove abandoned_files from target_files (as we know they're abandoned so don't need to check for them)
           outdated_files.concat determine_outdated_files(source_files - missing_files, target_files - abandoned_files)
           MultiSync.debug "#{outdated_files.length} of the files are outdated"
 
-          # abandoned files
-          if MultiSync.delete_abandoned_files
-            abandoned_files.lazily.each do | file |
-              incomplete_jobs << { id: Celluloid.uuid, target_id: target_id, method: :delete, args: file }
-            end
+          MultiSync.debug 'Scheduling jobs in the future...'
+
+          # outdated files
+          outdated_files.lazily.each do | resource |
+            running_upload_jobs << supervisor[target_id].future.upload(resource)
           end
 
           # missing files
           if MultiSync.upload_missing_files
-            missing_files.lazily.each do | file |
-              incomplete_jobs << { id: Celluloid.uuid, target_id: target_id, method: :upload, args: file }
+            missing_files.lazily.each do | resource |
+              running_upload_jobs << supervisor[target_id].future.upload(resource)
             end
           end
 
-          # outdated files
-          outdated_files.lazily.each do | file |
-            incomplete_jobs << { id: Celluloid.uuid, target_id: target_id, method: :upload, args: file }
+          # abandoned files
+          if MultiSync.delete_abandoned_files
+            abandoned_files.lazily.each do | resource |
+              running_delete_jobs << supervisor[target_id].future.delete(resource)
+            end
           end
 
         end
@@ -191,39 +165,77 @@ module MultiSync
       end
     end
 
+    def sync_attempted
+      self.started_at = Time.now if first_run?
+      self.sync_attempts = sync_attempts.next
+      if sync_attempts > MultiSync.max_sync_attempts
+        MultiSync.warn "Sync was attempted more then #{MultiSync.max_sync_attempts} times"
+        fail ArgumentError, "Sync was attempted more then #{MultiSync.max_sync_attempts} times"
+      end
+    end
+
+    def finish_sync
+      # recurse when there are incomplete_jobs still
+      incomplete_jobs.length != 0 ? sync : self.finished_at = Time.now
+
+      if finished_at
+        elapsed = finished_at.to_f - started_at.to_f
+        minutes, seconds = elapsed.divmod 60.0
+        bytes = complete_upload_jobs_bytes
+        kilobytes = bytes / 1024.0
+        MultiSync.debug "Sync completed in #{pluralize(minutes.round, 'minute')} and #{pluralize(seconds.round, 'second')}"
+        MultiSync.debug 'The combined upload weight was ' + ((bytes > 1024.0) ? pluralize(kilobytes.round, 'kilobyte') : pluralize(bytes.round, 'byte'))
+        MultiSync.debug "#{pluralize(file_sync_attempts, 'failed request')} were detected and re-tried"
+      else
+        MultiSync.debug "Sync failed to complete with #{pluralize(incomplete_jobs.length, 'outstanding file')} to be synchronised"
+      end
+      MultiSync.debug "#{pluralize(complete_jobs.length, 'file')} were synchronised (#{pluralize(complete_delete_jobs.length, 'deleted file')} and #{pluralize(complete_upload_jobs.length, 'uploaded file')}) from #{pluralize(sources.length, 'source')} to #{pluralize(supervisor_actor_names.length, 'target')}"
+
+      supervisor.terminate
+    end
+
+    def complete_jobs
+      complete_upload_jobs | complete_delete_jobs
+    end
+
+    def incomplete_jobs
+      incomplete_upload_jobs | incomplete_delete_jobs
+    end
+
+    def complete_upload_jobs_bytes
+      total_bytes = 0
+      complete_upload_jobs.each do | job |
+        total_bytes += job.content_length || job.determine_content_length || 0
+      end
+      total_bytes
+    end
+
     def determine_missing_files(source_files, target_files)
-      missing_files = (source_files - target_files)
-      missing_files
+      source_files - target_files
     end
 
     def determine_abandoned_files(source_files, target_files)
-      abandoned_files = (target_files - source_files)
-      abandoned_files
+      target_files - source_files
     end
 
     def determine_outdated_files(source_files, target_files)
       outdated_files = []
+      equivalent_files = []
 
       # TODO: replace with celluloid pool of futures
       # check each source file against the matching target_file's etag
       source_files.lazily.each_with_index do |file, i|
-        outdated_files << file unless !MultiSync.force && file.matching_etag?(target_files[i])
+        if !file.matching_etag?(target_files[i]) || MultiSync.force
+          outdated_files << file
+        else
+          equivalent_files << file
+        end
       end
 
-      outdated_files
-    end
+      # TODO: move to a better place
+      MultiSync.debug "#{equivalent_files.length} of the files are identical"
 
-    def sync_attempted
-      self.started_at = Time.now if first_run?
-      self.sync_attempts = sync_attempts.next
-      if sync_attempts > MultiSync.max_sync_attempts
-        MultiSync.warn "Sync was attempted more then #{MultiSync.max_sync_attempts} times"
-        fail ArgumentError, "Sync was attempted more then #{MultiSync.max_sync_attempts} times"
-      end
-    end
-
-    def finish_sync
-      incomplete_jobs.length != 0 ? synchronize : self.finished_at = Time.now
+      outdated_files
     end
 
     def first_run?
@@ -235,7 +247,7 @@ module MultiSync
     end
 
     def supervisor_actor_names
-      supervisor.actors.map { |actor| actor.registered_name }
+      supervisor.actors.map(&:registered_name)
     end
   end
 end