multi_sync 0.0.1

data/lib/multi_sync/targets/aws_target.rb

@@ -0,0 +1,80 @@
+require 'multi_sync/target'
+require 'multi_sync/resources/remote_resource'
+
+module MultiSync
+  class AwsTarget < Target
+    # Initialize a new AwsTarget object
+    #
+    # @param options [Hash]
+    def initialize(options = {})
+      super(options)
+      self.connection = ::Fog::Storage.new(credentials.merge(provider: :aws))
+    end
+
+    def files
+      files = []
+
+      directory = connection.directories.get(target_dir.to_s, prefix: destination_dir.to_s)
+      return files if directory.nil?
+
+      directory.files.lazily.each { |file|
+
+        pathname = Pathname.new(file.key)
+
+        # eg directory or overreaching AWS globbing
+        next unless valid_path?(pathname)
+
+        files << MultiSync::RemoteResource.new(
+          file: file,
+          with_root: target_dir + pathname, # pathname seems to already have the prefix (destination_dir)
+          without_root: destination_dir != '' ? pathname.relative_path_from(destination_dir).cleanpath : pathname
+        )
+
+      }
+
+      files
+    end
+
+    def upload(resource)
+      MultiSync.say_status :upload, resource.path_without_root.to_s
+      MultiSync.debug "Upload #{resource.class_name}:'#{resource.path_without_root}' to #{class_name}:'#{File.join('/', target_dir + destination_dir)}'"
+      directory = connection.directories.get(target_dir.to_s)
+      return if directory.nil?
+
+      upload_hash = {
+        key: (destination_dir + resource.path_without_root).to_s,
+        body: resource.body,
+        content_type: resource.content_type,
+        content_md5: Digest::MD5.base64digest(resource.body)
+      }
+
+      MultiSync::Resource::AWS_ATTRIBUTES.each do |attribute_hash|
+        upload_hash[attribute_hash[:name]] = resource.send(attribute_hash[:name])
+      end
+
+      directory.files.create(upload_hash)
+
+      resource
+    end
+
+    def delete(resource)
+      MultiSync.say_status :upload, resource.path_without_root.to_s
+      MultiSync.debug "Delete #{resource.class_name}:'#{resource.path_without_root}' from #{class_name}:'#{File.join('/', target_dir + destination_dir)}'"
+      connection.delete_object(target_dir.to_s, (destination_dir + resource.path_without_root).to_s)
+      resource
+    end
+
+    private
+
+    # directory or overreaching AWS globbing
+    def valid_path?(pathname)
+      # directory
+      return false if pathname.to_s =~ /\/$/
+
+      # overreaching AWS globbing
+      return false if !destination_dir.to_s.empty? && !(pathname.to_s =~ /^#{destination_dir.to_s}\//)
+
+      true
+    end
+  end
+end

data/lib/multi_sync/targets/local_target.rb

@@ -0,0 +1,58 @@
+require 'multi_sync/target'
+require 'multi_sync/resources/remote_resource'
+
+module MultiSync
+  class LocalTarget < Target
+    # Initialize a new LocalTarget object
+    #
+    # @param options [Hash]
+    def initialize(options = {})
+      super(options)
+      self.connection = ::Fog::Storage.new(credentials.merge(provider: :local))
+    end
+
+    def files
+      files = []
+
+      directory = connection.directories.get(destination_dir.to_s)
+      return files if directory.nil?
+
+      directory.files.lazily.each { |file|
+
+        pathname = Pathname.new(file.key)
+
+        # directory
+        next if pathname.directory?
+
+        MultiSync.debug "Found RemoteResource:'#{pathname}' from #{class_name}:'#{File.join(connection.local_root, destination_dir)}'"
+
+        files << MultiSync::RemoteResource.new(
+          file: file,
+          with_root: target_dir + destination_dir + pathname,
+          without_root: pathname
+        )
+
+      }
+
+      files
+    end
+
+    def upload(resource)
+      key = resource.path_without_root.to_s
+      MultiSync.say_status :upload, key
+      MultiSync.debug "Upload #{resource.class_name}:'#{key}' to #{class_name}:'#{File.join(connection.local_root, destination_dir)}'"
+      directory = connection.directories.get(destination_dir.to_s)
+      return if directory.nil?
+      directory.files.create(key: key, body: resource.body)
+      resource
+    end
+
+    def delete(resource)
+      key = resource.path_without_root.to_s
+      MultiSync.say_status :delete, key
+      MultiSync.debug "Delete #{resource.class_name}:'#{key}' from #{class_name}:'#{File.join(connection.local_root, destination_dir)}'"
+      connection.directories.get(destination_dir.to_s).files.get(key).destroy
+      resource
+    end
+  end
+end

data/lib/multi_sync/version.rb

@@ -0,0 +1,3 @@
+module MultiSync
+  VERSION = '0.0.1' unless defined?(MultiSync::VERSION)
+end

data/lib/tasks/multi_sync_rails.rake

@@ -0,0 +1,15 @@
+require 'multi_sync'
+
+namespace :assets do
+  desc 'Synchronize assets'
+  task :sync => :environment do
+    ActiveSupport::Notifications.instrument 'multi_sync.run' do
+      MultiSync::Extensions::AssetSync.check_and_migrate
+      MultiSync.run
+    end
+  end
+end
+
+Rake::Task['assets:precompile'].enhance do
+  Rake::Task['assets:sync'].invoke if defined?(MultiSync) && MultiSync.run_on_build
+end

data/multi_sync.gemspec

@@ -0,0 +1,35 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'multi_sync/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'multi_sync'
+  spec.version       = MultiSync::VERSION
+  spec.authors       = ['Karl Freeman']
+  spec.email         = ['karlfreeman@gmail.com']
+  spec.summary       = %q{A flexible synchronisation library for your assets}
+  spec.description   = %q{A flexible synchronisation library for your assets}
+  spec.homepage      = 'https://github.com/karlfreeman/multi_sync'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ['lib']
+  spec.required_ruby_version = '>= 1.9.3'
+
+  spec.add_dependency 'fog', '~> 1.12'
+  spec.add_dependency 'lazily', '~> 0.1'
+  spec.add_dependency 'virtus', '~> 0.5'
+  spec.add_dependency 'celluloid', '~> 0.15'
+  spec.add_dependency 'multi_mime', '~> 1.0'
+  spec.add_dependency 'multi_json', '~> 1.7'
+
+  spec.add_development_dependency 'bundler', '~> 1.5'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'kramdown', '>= 0.14'
+  spec.add_development_dependency 'rubocop'
+  spec.add_development_dependency 'pry'
+  spec.add_development_dependency 'yard'
+end

data/spec/spec_helper.rb

@@ -0,0 +1,22 @@
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+
+require 'bundler'
+Bundler.setup
+
+%w(support).each do |dir|
+  Dir.glob(File.expand_path("../#{dir}/**/*.rb", __FILE__), &method(:require))
+end
+
+require 'multi_sync'
+
+RSpec.configure do |config|
+  config.expect_with :rspec do |c|
+    c.syntax = :expect
+  end
+
+  config.before(:each) do
+    MultiSync.reset!
+    MultiSync.env = :test
+    MultiSync.verbose = true
+  end
+end

data/spec/support/celluloid.rb

@@ -0,0 +1,14 @@
+require 'celluloid/test'
+Celluloid.shutdown_timeout = 1
+RSpec.configure do |config|
+  config.around do |ex|
+    Celluloid.actor_system = nil
+    Thread.list.each do |thread|
+      next if thread == Thread.current
+      thread.kill
+    end
+    Celluloid.boot
+    ex.run
+    Celluloid.shutdown
+  end
+end

data/spec/support/fakefs.rb

@@ -0,0 +1,7 @@
+require 'fakefs/safe'
+require 'fakefs/spec_helpers'
+require 'pathname'
+require 'fileutils'
+RSpec.configure do |config|
+  config.include FakeFS::SpecHelpers, fakefs: true
+end

data/spec/support/fog.rb

@@ -0,0 +1,25 @@
+require 'fog'
+Fog::Mock.delay = 0
+
+module Fog
+  module SpecHelpers
+    def self.extended(example_group)
+      example_group.before(:each) do
+        Fog.mock!
+        Fog::Mock.reset
+      end
+
+      example_group.after(:each) do
+        Fog.unmock!
+      end
+    end
+
+    def self.included(example_group)
+      example_group.extend self
+    end
+  end
+end
+
+RSpec.configure do |config|
+  config.include Fog::SpecHelpers, fog: true
+end

data/spec/support/pry.rb

@@ -0,0 +1,6 @@
+if defined?(RUBY_ENGINE) && RUBY_ENGINE == 'ruby'
+  begin
+    require 'pry' unless ENV['CI']
+  rescue LoadError
+  end
+end

data/spec/support/timecop.rb

@@ -0,0 +1,2 @@
+require 'timecop'
+Timecop.freeze

data/spec/unit/multi_sync/client_spec.rb

@@ -0,0 +1,301 @@
+require 'spec_helper'
+
+describe MultiSync::Client, fakefs: true do
+
+  before do
+    FileUtils.mkdir_p('/tmp/simple')
+    File.open('/tmp/simple/foo.txt', 'w') do |f| f.write('foo') end
+    File.open('/tmp/simple/bar.txt', 'w') do |f| f.write('bar') end
+    FileUtils.mkdir_p('/tmp/simple/in-a-dir')
+    File.open('/tmp/simple/in-a-dir/baz.html', 'w') do |f| f.write('baz') end
+
+    FileUtils.cp_r('/tmp/simple', '/tmp/simple-with-missing-file')
+    FileUtils.rm_r('/tmp/simple-with-missing-file/foo.txt')
+
+    FileUtils.cp_r('/tmp/simple', '/tmp/simple-with-abandoned-file')
+    File.open('/tmp/simple-with-abandoned-file/baz.txt', 'w') do |f| f.write('baz') end
+
+    FileUtils.cp_r('/tmp/simple', '/tmp/simple-with-outdated-file')
+    File.open('/tmp/simple-with-outdated-file/foo.txt', 'w') do |f| f.write('not-foo') end
+
+    FileUtils.mkdir_p('/tmp/complex')
+    50.times do
+      File.open("/tmp/complex/#{SecureRandom.urlsafe_base64}.txt", 'w') do |f| f.write(SecureRandom.random_bytes) end
+    end
+
+    FileUtils.mkdir_p('/tmp/complex-empty')
+  end
+
+  context :sync do
+    context :local do
+      context 'simple' do
+        it 'should work' do
+          missing_files_target_options = {
+            type: :local,
+            target_dir: '/tmp',
+            destination_dir: 'simple-with-missing-file',
+            credentials: {
+              local_root: '/tmp'
+            }
+          }
+
+          abandoned_files_target_options = {
+            type: :local,
+            target_dir: '/tmp',
+            destination_dir: 'simple-with-abandoned-file',
+            credentials: {
+              local_root: '/tmp'
+            }
+          }
+
+          outdated_files_target_options = {
+            type: :local,
+            target_dir: '/tmp',
+            destination_dir: 'simple-with-outdated-file',
+            credentials: {
+              local_root: '/tmp'
+            }
+          }
+
+          local_source_options = {
+            type: :local,
+            source_dir: '/tmp/simple'
+          }
+
+          missing_files_target = MultiSync::LocalTarget.new(missing_files_target_options)
+          abandoned_files_target = MultiSync::LocalTarget.new(abandoned_files_target_options)
+          outdated_files_target = MultiSync::LocalTarget.new(outdated_files_target_options)
+
+          expect(missing_files_target).to have(2).files
+          expect(abandoned_files_target).to have(4).files
+          expect(outdated_files_target).to have(3).files
+
+          local_source = MultiSync::LocalSource.new(local_source_options)
+          expect(local_source).to have(3).files
+
+          expect(outdated_files_target.files[1].body).to eq 'not-foo'
+
+          MultiSync.run do
+            target :missing_files_target, missing_files_target_options
+            target :abandoned_files_target, abandoned_files_target_options
+            target :outdated_files_target, outdated_files_target_options
+            source :simple, local_source_options.merge(targets: [:missing_files_target, :abandoned_files_target, :outdated_files_target])
+          end
+
+          expect(missing_files_target).to have(3).files
+          expect(abandoned_files_target).to have(3).files
+          expect(outdated_files_target).to have(3).files
+          expect(outdated_files_target.files[1].body).to eq 'foo'
+        end
+      end
+
+      context 'complex' do
+        it 'should work' do
+          complex_empty_target_options = {
+            type: :local,
+            target_dir: '/tmp',
+            destination_dir: 'complex-empty',
+            credentials: {
+              local_root: '/tmp'
+            }
+          }
+
+          local_source_options = {
+            type: :local,
+            source_dir: '/tmp/complex'
+          }
+
+          complex_empty_target = MultiSync::LocalTarget.new(complex_empty_target_options)
+          expect(complex_empty_target).to have(0).files
+
+          local_source = MultiSync::LocalSource.new(local_source_options)
+          expect(local_source).to have(50).files
+
+          MultiSync.run do
+            target :complex_empty_target, complex_empty_target_options
+            source :complex, local_source_options.merge(targets: [:complex_empty_target])
+          end
+
+          expect(complex_empty_target).to have(50).files
+        end
+      end
+    end
+
+    context :aws, fog: true do
+      context 'simple' do
+        before do
+          connection = Fog::Storage.new(
+            provider: :aws,
+            region: 'us-east-1',
+            aws_access_key_id: 'xxx',
+            aws_secret_access_key: 'xxx'
+          )
+
+          directory = connection.directories.create(key: 'multi_sync', public: true)
+
+          %w(simple simple-with-missing-file simple-with-abandoned-file simple-with-outdated-file).each do |fixture_name|
+            Dir.glob("/tmp/#{fixture_name}/**/*").reject { |path| File.directory?(path) }.each do |path|
+              directory.files.create(
+                key: path.gsub('/tmp/', ''),
+                body: File.open(path, 'r'),
+                public: true
+              )
+            end
+          end
+        end
+
+        it 'should work' do
+          missing_files_target_options = {
+            type: :aws,
+            target_dir: 'multi_sync',
+            destination_dir: 'simple-with-missing-file',
+            credentials: {
+              region: 'us-east-1',
+              aws_access_key_id: 'xxx',
+              aws_secret_access_key: 'xxx'
+            }
+          }
+
+          abandoned_files_target_options = {
+            type: :aws,
+            target_dir: 'multi_sync',
+            destination_dir: 'simple-with-abandoned-file',
+            credentials: {
+              region: 'us-east-1',
+              aws_access_key_id: 'xxx',
+              aws_secret_access_key: 'xxx'
+            }
+          }
+
+          outdated_files_target_options = {
+            type: :aws,
+            target_dir: 'multi_sync',
+            destination_dir: 'simple-with-outdated-file',
+            credentials: {
+              region: 'us-east-1',
+              aws_access_key_id: 'xxx',
+              aws_secret_access_key: 'xxx'
+            }
+          }
+
+          local_source_options = {
+            type: :local,
+            source_dir: '/tmp/simple'
+          }
+
+          missing_files_target = MultiSync::AwsTarget.new(missing_files_target_options)
+          abandoned_files_target = MultiSync::AwsTarget.new(abandoned_files_target_options)
+          outdated_files_target = MultiSync::AwsTarget.new(outdated_files_target_options)
+
+          expect(missing_files_target).to have(2).files
+          expect(abandoned_files_target).to have(4).files
+          expect(outdated_files_target).to have(3).files
+
+          local_source = MultiSync::LocalSource.new(local_source_options)
+          expect(local_source).to have(3).files
+
+          expect(outdated_files_target.files[1].body).to eq 'not-foo'
+
+          MultiSync.run do
+            target :missing_files_target, missing_files_target_options
+            target :abandoned_files_target, abandoned_files_target_options
+            target :outdated_files_target, outdated_files_target_options
+            source :simple, local_source_options.merge(targets: [:missing_files_target, :abandoned_files_target, :outdated_files_target])
+          end
+
+          expect(missing_files_target).to have(3).files
+          expect(abandoned_files_target).to have(3).files
+          expect(outdated_files_target).to have(3).files
+          expect(outdated_files_target.files[1].body).to eq 'foo'
+        end
+      end
+
+      context 'complex' do
+        before do
+          connection = Fog::Storage.new(
+            provider: :aws,
+            region: 'us-east-1',
+            aws_access_key_id: 'xxx',
+            aws_secret_access_key: 'xxx'
+          )
+
+          connection.directories.create(key: 'multi_sync', public: true)
+        end
+
+        it 'should work' do
+          complex_empty_target_options = {
+            type: :aws,
+            target_dir: 'multi_sync',
+            destination_dir: 'complex-empty',
+            credentials: {
+              region: 'us-east-1',
+              aws_access_key_id: 'xxx',
+              aws_secret_access_key: 'xxx'
+            }
+          }
+
+          local_source_options = {
+            type: :local,
+            source_dir: '/tmp/complex'
+          }
+
+          complex_empty_target = MultiSync::AwsTarget.new(complex_empty_target_options)
+          expect(complex_empty_target).to have(0).files
+
+          local_source = MultiSync::LocalSource.new(local_source_options)
+          expect(local_source).to have(50).files
+
+          MultiSync.run do
+            target :complex_empty_target, complex_empty_target_options
+            source :complex, local_source_options.merge(targets: [:complex_empty_target])
+          end
+
+          expect(complex_empty_target).to have(50).files
+        end
+      end
+
+      context 'without a destination_dir' do
+        before do
+          connection = Fog::Storage.new(
+            provider: :aws,
+            region: 'us-east-1',
+            aws_access_key_id: 'xxx',
+            aws_secret_access_key: 'xxx'
+          )
+
+          connection.directories.create(key: 'without_destination_dir', public: true)
+        end
+
+        it 'should work' do
+          without_destination_dir_target_options = {
+            type: :aws,
+            target_dir: 'without_destination_dir',
+            credentials: {
+              region: 'us-east-1',
+              aws_access_key_id: 'xxx',
+              aws_secret_access_key: 'xxx'
+            }
+          }
+
+          local_source_options = {
+            type: :local,
+            source_dir: '/tmp/simple'
+          }
+
+          without_destination_dir_target = MultiSync::AwsTarget.new(without_destination_dir_target_options)
+          expect(without_destination_dir_target).to have(0).files
+
+          local_source = MultiSync::LocalSource.new(local_source_options)
+          expect(local_source).to have(3).files
+
+          MultiSync.run do
+            target :without_destination_dir_target, without_destination_dir_target_options
+            source :local, local_source_options.merge(targets: :without_destination_dir_target)
+          end
+
+          expect(without_destination_dir_target).to have(3).files
+        end
+      end
+    end
+  end
+end