multi_sync 0.0.1

data/lib/multi_sync.rb

@@ -0,0 +1,92 @@
+require 'forwardable'
+require 'multi_sync/client'
+require 'multi_sync/version'
+require 'multi_sync/logging'
+require 'multi_sync/environment'
+require 'multi_sync/configuration'
+
+module MultiSync
+  extend SingleForwardable
+  extend MultiSync::Logging
+  extend MultiSync::Environment
+
+  # a list of libraries, extension file and class name
+  REQUIREMENT_MAP = [
+    ['rails', 'multi_sync/extensions/rails'].freeze,
+    ['middleman-core', 'multi_sync/extensions/middleman'].freeze
+  ]
+
+  # delegate all MultiSync::Configuration's attribute accessors to the configuration
+  def_delegators :configuration, *MultiSync::Configuration.attribute_set.map(&:name)
+
+  # delegate all MultiSync::Configuration's attribute setters to the configuration ( hacky I know... )
+  def_delegators :configuration, *(MultiSync::Configuration.attribute_set.map(&:name).map { |key| "#{key}=".to_sym })
+
+  # delegate all MultiSync::Client's attribute accessors to the configuration
+  def_delegators :client, *MultiSync::Client.attribute_set.map(&:name)
+
+  # include some public methods
+  def_delegators :client, :target, :source, :synchronize
+
+  # Configure
+  #
+  # @return [MultiSync]
+  def self.configure(&block)
+    instance_eval(&block) if block_given?
+    self
+  end
+
+  # Run
+  #
+  # @return [MultiSync]
+  def self.run(&block)
+    configure(&block).synchronize
+  end
+
+  # Prepare
+  #
+  # @return [MultiSync]
+  def self.prepare(&block)
+    configure(&block)
+  end
+
+  # Fetch the MultiSync::Client
+  #
+  # @return [MultiSync::Client]
+  def self.client(options = {})
+    @client ||= MultiSync::Client.new(options)
+  end
+
+  # Fetch the MultiSync::Configuration
+  #
+  # @return [MultiSync::Configuration]
+  def self.configuration(options = {})
+    @configuration ||= MultiSync::Configuration.new(options)
+  end
+
+  # Reset the MultiSync::Client
+  def self.reset_client!
+    remove_instance_variable :@client if defined?(@client)
+  end
+
+  # Reset the MultiSync::Configuration
+  def self.reset_configuration!
+    remove_instance_variable :@configuration if defined?(@configuration)
+  end
+
+  # Reset MultiSync::Configuration and MultiSync::Client
+  def self.reset!
+    self.reset_client!
+    self.reset_configuration!
+  end
+
+  # by rescuing from a LoadError we can sniff out gems in use and try to automagically hook into them
+  REQUIREMENT_MAP.each do |library, extension|
+    begin
+      require library
+      require extension
+    rescue ::LoadError
+      next
+    end
+  end
+end

data/lib/multi_sync/client.rb

@@ -0,0 +1,243 @@
+require 'set'
+require 'virtus'
+require 'lazily'
+require 'celluloid'
+require 'multi_sync/sources/local_source'
+require 'multi_sync/sources/manifest_source'
+require 'multi_sync/targets/aws_target'
+require 'multi_sync/targets/local_target'
+require 'multi_sync/mixins/pluralize_helper'
+
+module MultiSync
+  class Client
+    include Virtus
+    include MultiSync::Mixins::PluralizeHelper
+
+    attribute :supervisor
+    attribute :incomplete_jobs, Set, default: Set.new
+    attribute :running_jobs, Set, default: Set.new
+    attribute :complete_jobs, Set, default: Set.new
+    attribute :sources, Array, default: []
+    attribute :sync_attempts, Integer, default: 0
+    attribute :file_sync_attempts, Integer, default: 0
+    attribute :started_at, Time
+    attribute :finished_at, Time
+
+    # Initialize a new Client object
+    #
+    # @param options [Hash]
+    def initialize(options = {})
+      self.supervisor = Celluloid::SupervisionGroup.run!
+    end
+
+    def add_target(name, options = {})
+      fail ArgumentError, "Duplicate target names detected, please rename '#{name}' to be unique" if supervisor_actor_names.include?(name)
+      clazz = MultiSync.const_get("#{options[:type].capitalize}Target")
+      supervisor.pool(clazz, as: name, args: [options], size: MultiSync.target_pool_size)
+    rescue NameError
+      MultiSync.warn "Unknown target type: #{options[:type]}"
+      raise ArgumentError, "Unknown target type: #{options[:type]}"
+    end
+    alias_method :target, :add_target
+
+    def add_source(name, options = {})
+      clazz = MultiSync.const_get("#{options[:type].capitalize}Source")
+      sources << clazz.new(options)
+    rescue NameError
+      MultiSync.warn "Unknown source type: #{options[:type]}"
+      raise ArgumentError, "Unknown source type: #{options[:type]}"
+    end
+    alias_method :source, :add_source
+
+    def synchronize
+      if sync_pointless?
+        MultiSync.debug "Preventing synchronization as there are #{sources.length} sources to sync..."
+        return
+      else
+        MultiSync.debug 'Starting synchronization...'
+      end
+
+      determine_sync if first_run?
+      sync_attempted
+
+      MultiSync.debug 'Scheduling jobs in the future...'
+      incomplete_jobs.delete_if do | job |
+        running_jobs << { id: job[:id], future: Celluloid::Actor[job[:target_id]].future.send(job[:method], job[:args]), method: job[:method] }
+      end
+
+      MultiSync.debug 'Fetching jobs from the future...'
+      running_jobs.delete_if do | job |
+        begin
+          completed_job = { id: job[:id], response: job[:future].value, method: job[:method] }
+        rescue
+          self.file_sync_attempts = file_sync_attempts + 1
+          false
+        else
+          complete_jobs << completed_job
+          true
+        end
+      end
+
+      finish_sync
+      finalize
+    end
+    alias_method :sync, :synchronize
+
+    def finalize
+      # if finished_at
+      #   elapsed = finished_at.to_f - started_at.to_f
+      #   minutes, seconds = elapsed.divmod 60.0
+      #   kilobytes = get_total_file_size_from_complete_jobs / 1024.0
+      #   MultiSync.debug "Sync completed in #{pluralize(minutes.round, 'minute')} and #{pluralize(seconds.round, 'second')}"
+      #   MultiSync.debug "#{pluralize(complete_jobs.length, 'file')} were synchronised (#{pluralize(get_complete_deleted_jobs.length, 'deleted file')} and #{pluralize(get_complete_upload_jobs.length, 'uploaded file')}) from #{pluralize(sources.length, 'source')} to #{pluralize(supervisor.actors.length, 'target')}"
+      #   MultiSync.debug "The upload weight totalled %.#{0}f #{pluralize(kilobytes, 'KB', 'KB', false)}" % kilobytes
+      #   MultiSync.debug "#{pluralize(file_sync_attempts, 'failed request')} were detected and re-tried"
+      # else
+      #   MultiSync.debug "Sync failed to complete with #{pluralize(incomplete_jobs.length, 'outstanding file')} to be synchronised"
+      #   MultiSync.debug "#{pluralize(complete_jobs.length, 'file')} were synchronised (#{pluralize(get_complete_deleted_jobs.length, 'deleted file')} and #{pluralize(get_complete_upload_jobs.length, 'uploaded file')}) from #{pluralize(sources.length, 'source')} to #{pluralize(supervisor.actors.length, 'target')}"
+      # end
+
+      supervisor.finalize
+    end
+    alias_method :fin, :finalize
+
+    def get_complete_deleted_jobs
+      complete_jobs.select { |job| job[:method] == :delete }
+    end
+
+    def get_complete_upload_jobs
+      complete_jobs.select { |job| job[:method] == :upload }
+    end
+
+    def get_total_file_size_from_complete_jobs
+      total_file_size = 0
+      get_complete_upload_jobs.each do | job |
+        # MultiSync.info job[:response]
+        # MultiSync.info job[:response].determine_content_length
+        if job[:response].content_length
+          total_file_size += job[:response].content_length
+        end
+
+      end
+      total_file_size
+    end
+
+    private
+
+    def determine_sync
+      sources.lazily.each do |source|
+
+        source_files = []
+
+        starting_synchronizing_msg = "ynchronizing: '#{source.source_dir}'"
+        starting_synchronizing_msg.prepend MultiSync.force ? 'Forcefully s' : 'S'
+        MultiSync.info starting_synchronizing_msg
+
+        source_files = source.files
+        source_files.sort! # sort to make sure the source's indexes match the targets
+
+        # when no targets are specified, assume all targets
+        source.targets = supervisor_actor_names if source.targets.empty?
+
+        source.targets.lazily.each do | target_id |
+
+          missing_files = []
+          abandoned_files = []
+          outdated_files = []
+
+          MultiSync.debug "#{pluralize(source_files.length, 'file')} found from the source"
+
+          MultiSync.debug 'Fetching files from the target...'
+
+          target_files = Celluloid::Actor[target_id].files
+          target_files.sort! # sort to make sure the target's indexs match the sources
+
+          MultiSync.debug "#{pluralize(target_files.length, 'file')} found from the target"
+
+          missing_files.concat determine_missing_files(source_files, target_files)
+          missing_files_msg = "#{missing_files.length} of the files are missing"
+          missing_files_msg += ", however we're skipping them as :upload_missing_files is false" unless MultiSync.upload_missing_files
+          MultiSync.debug missing_files_msg
+
+          abandoned_files.concat determine_abandoned_files(source_files, target_files)
+          abandoned_files_msg = "#{abandoned_files.length} of the files are abandoned"
+          abandoned_files_msg += ", however we're skipping them as :delete_abandoned_files is false" unless MultiSync.delete_abandoned_files
+          MultiSync.debug abandoned_files_msg
+
+          # remove missing_files from source_files ( as we know they are missing so don't need to check them )
+          # remove abandoned_files from target_files ( as we know they are abandoned so don't need to check them )
+          outdated_files.concat determine_outdated_files(source_files - missing_files, target_files - abandoned_files)
+          MultiSync.debug "#{outdated_files.length} of the files are outdated"
+
+          # abandoned files
+          if MultiSync.delete_abandoned_files
+            abandoned_files.lazily.each do | file |
+              incomplete_jobs << { id: Celluloid.uuid, target_id: target_id, method: :delete, args: file }
+            end
+          end
+
+          # missing files
+          if MultiSync.upload_missing_files
+            missing_files.lazily.each do | file |
+              incomplete_jobs << { id: Celluloid.uuid, target_id: target_id, method: :upload, args: file }
+            end
+          end
+
+          # outdated files
+          outdated_files.lazily.each do | file |
+            incomplete_jobs << { id: Celluloid.uuid, target_id: target_id, method: :upload, args: file }
+          end
+
+        end
+
+      end
+    end
+
+    def determine_missing_files(source_files, target_files)
+      missing_files = (source_files - target_files)
+      missing_files
+    end
+
+    def determine_abandoned_files(source_files, target_files)
+      abandoned_files = (target_files - source_files)
+      abandoned_files
+    end
+
+    def determine_outdated_files(source_files, target_files)
+      outdated_files = []
+
+      # TODO: replace with celluloid pool of futures
+      # check each source file against the matching target_file's etag
+      source_files.lazily.each_with_index do |file, i|
+        outdated_files << file unless !MultiSync.force && file.matching_etag?(target_files[i])
+      end
+
+      outdated_files
+    end
+
+    def sync_attempted
+      self.started_at = Time.now if first_run?
+      self.sync_attempts = sync_attempts.next
+      if sync_attempts > MultiSync.max_sync_attempts
+        MultiSync.warn "Sync was attempted more then #{MultiSync.max_sync_attempts} times"
+        fail ArgumentError, "Sync was attempted more then #{MultiSync.max_sync_attempts} times"
+      end
+    end
+
+    def finish_sync
+      incomplete_jobs.length != 0 ? synchronize : self.finished_at = Time.now
+    end
+
+    def first_run?
+      sync_attempts == 0
+    end
+
+    def sync_pointless?
+      sources.empty?
+    end
+
+    def supervisor_actor_names
+      supervisor.actors.map { |actor| actor.registered_name }
+    end
+  end
+end

data/lib/multi_sync/configuration.rb

@@ -0,0 +1,36 @@
+require 'virtus'
+require 'fog'
+require 'celluloid'
+
+module MultiSync
+  class Configuration
+    include Virtus
+
+    attribute :verbose, Boolean, default: false
+    attribute :force, Boolean, default: false
+    attribute :run_on_build, Boolean, default: true
+    attribute :delete_abandoned_files, Boolean, default: true
+    attribute :upload_missing_files, Boolean, default: true
+    attribute :target_pool_size, Integer, default: :celluloid_cores
+    attribute :max_sync_attempts, Integer, default: 3
+    attribute :credentials, Hash, default: :fog_credentials
+
+    # Initialize a new Configuration object
+    #
+    # @param options [Hash]
+    def initialize(options = {})
+      Celluloid.logger = MultiSync.test? ? nil : MultiSync.logger
+      options.each_pair do |key, value|
+        send("#{key}=", value) if attributes.keys.include?(key)
+      end
+    end
+
+    def celluloid_cores
+      Celluloid.cores
+    end
+
+    def fog_credentials
+      Fog.credentials
+    end
+  end
+end

data/lib/multi_sync/environment.rb

@@ -0,0 +1,37 @@
+module MultiSync
+  module Environment
+    # Retrieves the current MultiSync environment
+    #
+    # @return [String] the current environment
+    def environment
+      @environment ||= ENV['MULTI_SYNC_ENV'] || ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
+    end
+    alias_method :env, :environment
+
+    # Sets the current MultiSync environment
+    #
+    # @param [String|Symbol] env the environment symbol
+    def environment=(e)
+      @environment = e.to_s
+    end
+    alias_method :env=, :environment=
+
+    # Determines if we are in a particular environment
+    #
+    # @return [Boolean] true if current environment matches, false otherwise
+    def environment?(e)
+      environment == e.to_s
+    end
+    alias_method :env?, :environment?
+
+    # Create methods for the environment shorthands
+    [:test, :development, :staging, :production].each do |e|
+      # Determines if we are in a particular environment
+      #
+      # @return [Boolean] true if current environment matches, false otherwise
+      define_method "#{e}?" do
+        environment? e
+      end
+    end
+  end
+end

data/lib/multi_sync/extensions/middleman.rb

@@ -0,0 +1,22 @@
+module MultiSync
+  module Extensions
+    class Middleman
+      MultiSync.info "Middleman -v #{::Middleman::VERSION} auto-detected"
+
+      class << self
+        def source_dir
+          File.expand_path(File.join(ENV['MM_ROOT'], 'build'))
+        end
+
+        def destination_dir
+          ''
+        end
+      end
+
+      MultiSync.source(:middleman,
+        type: :local,
+        source_dir: MultiSync::Extensions::Middleman.source_dir
+      )
+    end
+  end
+end

data/lib/multi_sync/extensions/rails.rb

@@ -0,0 +1,20 @@
+module MultiSync
+  module Extensions
+    require 'multi_sync/extensions/rails/railtie' if defined?(::Rails::Railtie)
+    require 'multi_sync/extensions/rails/asset_sync'
+
+    class Rails
+      MultiSync.info "Rails -v #{::Rails::VERSION::STRING} auto-detected"
+
+      class << self
+        def source_dir
+          ::Rails.root.join('public', destination_dir)
+        end
+
+        def destination_dir
+          ::Rails.application.config.assets.prefix.sub(/^\//, '')
+        end
+      end
+    end
+  end
+end