backup 3.0.21 → 3.0.22

data/lib/backup/syncer/rsync/base.rb

@@ -4,19 +4,6 @@ module Backup
   module Syncer
     module RSync
       class Base < Syncer::Base
-
-        ##
-        # Path to store the synced files/directories to
-        attr_accessor :path
-
-        ##
-        # Directories to sync
-        attr_writer :directories
-
-        ##
-        # Flag for mirroring the files/directories
-        attr_accessor :mirror
-
         ##
         # Additional options for the rsync cli
         attr_accessor :additional_options
@@ -33,19 +20,6 @@ module Backup
           @additional_options ||= Array.new
         end
 
-        ##
-        # Syntactical suger for the DSL for adding directories
-        def directories(&block)
-          return @directories unless block_given?
-          instance_eval(&block)
-        end
-
-        ##
-        # Adds a path to the @directories array
-        def add(path)
-          @directories << path
-        end
-
         private
 
         ##

data/lib/backup/syncer/s3.rb

@@ -2,127 +2,46 @@
 
 module Backup
   module Syncer
-    class S3 < Base
+    class S3 < Cloud
 
       ##
       # Amazon Simple Storage Service (S3) Credentials
       attr_accessor :access_key_id, :secret_access_key
 
       ##
-      # Amazon S3 bucket name and path to sync to
-      attr_accessor :bucket, :path
+      # The S3 bucket to store files to
+      attr_accessor :bucket
 
       ##
-      # Directories to sync
-      attr_writer :directories
-
-      ##
-      # Flag to enable mirroring
-      attr_accessor :mirror
-
-      ##
-      # Additional options for the s3sync cli
-      attr_accessor :additional_options
-
-      ##
-      # Instantiates a new S3 Syncer object and sets the default configuration
-      # specified in the Backup::Configuration::Syncer::S3.
-      # Then it sets the object defaults if particular properties weren't set.
-      # Finally it'll evaluate the users configuration file and overwrite
-      # anything that's been defined
-      def initialize(&block)
-        load_defaults!
-
-        @path               ||= 'backups'
-        @directories          = Array.new
-        @mirror             ||= false
-        @additional_options ||= []
-
-        instance_eval(&block) if block_given?
-      end
-
-      ##
-      # Sets the Amazon S3 credentials for S3Sync, performs the S3Sync
-      # operation, then unsets the credentials (back to nil values)
-      def perform!
-        set_environment_variables!
-
-        @directories.each do |directory|
-          Logger.message("#{ syncer_name } started syncing '#{ directory }'.")
-          Logger.silent(
-            run("#{ utility(:s3sync) } #{ options } " +
-                "'#{ File.expand_path(directory) }' '#{ bucket }:#{ dest_path }'")
-          )
-        end
-
-        unset_environment_variables!
-      end
-
-      ##
-      # Syntactical suger for the DSL for adding directories
-      def directories(&block)
-        return @directories unless block_given?
-        instance_eval(&block)
-      end
-
-      ##
-      # Adds a path to the @directories array
-      def add(path)
-        @directories << path
-      end
+      # The AWS region of the specified S3 bucket
+      attr_accessor :region
 
       private
 
       ##
-      # Return @path with preceeding '/' slash removed
-      def dest_path
-        @dest_path ||= @path.sub(/^\//, '')
-      end
-
-      ##
-      # Returns all the specified S3Sync options,
-      # concatenated, ready for the CLI
-      def options
-        ([verbose_option, recursive_option, mirror_option] +
-          additional_options).compact.join("\s")
-      end
-
-      ##
-      # Returns S3Sync syntax for enabling mirroring
-      def mirror_option
-        '--delete' if @mirror
-      end
-
-      ##
-      # Returns S3Sync syntax for syncing recursively
-      def recursive_option
-        '--recursive'
+      # Established and creates a new Fog storage object for S3.
+      def connection
+        @connection ||= Fog::Storage.new(
+          :provider              => provider,
+          :aws_access_key_id     => access_key_id,
+          :aws_secret_access_key => secret_access_key,
+          :region                => region
+        )
       end
 
       ##
-      # Returns S3Sync syntax for making output verbose
-      def verbose_option
-        '--verbose'
+      # Creates a new @repository_object (bucket). Fetches it from S3
+      # if it already exists, otherwise it will create it first and fetch use that instead.
+      def repository_object
+        @repository_object ||= connection.directories.get(bucket) ||
+          connection.directories.create(:key => bucket, :location => region)
       end
 
       ##
-      # In order for S3Sync to know what credentials to use, we have to set the
-      # AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment variables, these
-      # evironment variables will be used by S3Sync
-      def set_environment_variables!
-        ENV['AWS_ACCESS_KEY_ID']     = access_key_id
-        ENV['AWS_SECRET_ACCESS_KEY'] = secret_access_key
-        ENV['AWS_CALLING_FORMAT']    = 'SUBDOMAIN'
+      # This is the provider that Fog uses for the Cloud Files
+      def provider
+        "AWS"
       end
-
-      ##
-      # Sets the AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY back to nil
-      def unset_environment_variables!
-        ENV['AWS_ACCESS_KEY_ID']     = nil
-        ENV['AWS_SECRET_ACCESS_KEY'] = nil
-        ENV['AWS_CALLING_FORMAT']    = nil
-      end
-
     end
   end
 end

data/lib/backup/version.rb

@@ -13,7 +13,7 @@ module Backup
     #  Defines the minor version
     # PATCH:
     #  Defines the patch version
-    MAJOR, MINOR, PATCH = 3, 0, 21
+    MAJOR, MINOR, PATCH = 3, 0, 22
 
     ##
     # Returns the major version ( big release based off of multiple minor releases )

data/spec/cli/utility_spec.rb

@@ -187,8 +187,8 @@ describe 'Backup::CLI::Utility' do
           --trigger=TRIGGER
           [--config-path=CONFIG_PATH]  # Path to your Backup configuration directory
           [--databases=DATABASES]      # (mongodb, mysql, postgresql, redis, riak)
-          [--storages=STORAGES]        # (cloudfiles, dropbox, ftp, local, ninefold, rsync, s3, scp, sftp)
-          [--syncers=SYNCERS]          # (rsync_local, rsync_pull, rsync_push, s3)
+          [--storages=STORAGES]        # (cloud_files, dropbox, ftp, local, ninefold, rsync, s3, scp, sftp)
+          [--syncers=SYNCERS]          # (cloud_files, rsync_local, rsync_pull, rsync_push, s3)
           [--encryptors=ENCRYPTORS]    # (gpg, openssl)
           [--compressors=COMPRESSORS]  # (bzip2, gzip, lzma, pbzip2)
           [--notifiers=NOTIFIERS]      # (campfire, hipchat, mail, presently, prowl, twitter)

data/spec/configuration/syncer/cloud_files_spec.rb

@@ -0,0 +1,44 @@
+# encoding: utf-8
+
+require File.expand_path('../../../spec_helper.rb', __FILE__)
+
+describe Backup::Configuration::Syncer::CloudFiles do
+  before do
+    Backup::Configuration::Syncer::CloudFiles.defaults do |cf|
+      cf.username   = 'my-username'
+      cf.api_key    = 'my-api-key'
+      cf.container  = 'my-container'
+      cf.auth_url   = 'my-auth-url'
+      cf.servicenet = true
+      cf.path       = '/backups/'
+      cf.mirror     = true
+    end
+  end
+  after { Backup::Configuration::Syncer::CloudFiles.clear_defaults! }
+
+  it 'should set the default cloud files configuration' do
+    cf = Backup::Configuration::Syncer::CloudFiles
+    cf.username.should   == 'my-username'
+    cf.api_key.should    == 'my-api-key'
+    cf.container.should  == 'my-container'
+    cf.auth_url.should   == 'my-auth-url'
+    cf.servicenet.should == true
+    cf.path.should       == '/backups/'
+    cf.mirror.should     == true
+  end
+
+  describe '#clear_defaults!' do
+    it 'should clear all the defaults, resetting them to nil' do
+      Backup::Configuration::Syncer::CloudFiles.clear_defaults!
+
+      cf = Backup::Configuration::Syncer::CloudFiles
+      cf.username.should   == nil
+      cf.api_key.should    == nil
+      cf.container.should  == nil
+      cf.auth_url.should   == nil
+      cf.servicenet.should == nil
+      cf.path.should       == nil
+      cf.mirror.should     == nil
+    end
+  end
+end

data/spec/configuration/syncer/s3_spec.rb

@@ -9,9 +9,7 @@ describe Backup::Configuration::Syncer::S3 do
       s3.secret_access_key   = 'my_secret_access_key'
       s3.bucket              = 'my-bucket'
       s3.path                = '/backups/'
-      #s3.directories         = 'cannot_have_a_default_value'
       s3.mirror              = true
-      s3.additional_options  = ['--exclude="*.rb"']
     end
   end
   after { Backup::Configuration::Syncer::S3.clear_defaults! }
@@ -23,7 +21,6 @@ describe Backup::Configuration::Syncer::S3 do
     s3.bucket.should              == 'my-bucket'
     s3.path.should                == '/backups/'
     s3.mirror.should              == true
-    s3.additional_options.should  == ['--exclude="*.rb"']
   end
 
   describe '#clear_defaults!' do
@@ -36,7 +33,6 @@ describe Backup::Configuration::Syncer::S3 do
       s3.bucket.should              == nil
       s3.path.should                == nil
       s3.mirror.should              == nil
-      s3.additional_options.should  == nil
     end
   end
 end

data/spec/notifier/twitter_spec.rb

@@ -88,7 +88,7 @@ describe Backup::Notifier::Twitter do
     context 'when status is :success' do
       it 'should send Success message' do
         notifier.expects(:send_message).with(
-          '[Backup::Success] test label (test_trigger)'
+          "[Backup::Success] test label (test_trigger) (@ #{notifier.instance_variable_get("@model").time})"
         )
         notifier.send(:notify!, :success)
       end
@@ -97,7 +97,7 @@ describe Backup::Notifier::Twitter do
     context 'when status is :warning' do
       it 'should send Warning message' do
         notifier.expects(:send_message).with(
-          '[Backup::Warning] test label (test_trigger)'
+          "[Backup::Warning] test label (test_trigger) (@ #{notifier.instance_variable_get("@model").time})"
         )
         notifier.send(:notify!, :warning)
       end
@@ -106,7 +106,7 @@ describe Backup::Notifier::Twitter do
     context 'when status is :failure' do
       it 'should send Failure message' do
         notifier.expects(:send_message).with(
-          '[Backup::Failure] test label (test_trigger)'
+          "[Backup::Failure] test label (test_trigger) (@ #{notifier.instance_variable_get("@model").time})"
         )
         notifier.send(:notify!, :failure)
       end

data/spec/syncer/cloud_files_spec.rb

@@ -0,0 +1,192 @@
+# encoding: utf-8
+require File.expand_path('../../spec_helper.rb', __FILE__)
+
+class Parallel; end
+
+describe Backup::Syncer::CloudFiles do
+  describe '#perform!' do
+    let(:syncer)     { Backup::Syncer::CloudFiles.new }
+    let(:connection) { stub('connection',
+      :directories => stub('directories', :get => container)) }
+    let(:container)     { stub('container', :files => files) }
+    let(:files)      { [] }
+    let(:content)    { stub('content') }
+
+    before :each do
+      Fog::Storage.stubs(:new).returns connection
+      File.stubs(:open).returns content
+      File.stubs(:exist?).returns true
+      files.stubs(:create).returns true
+
+      syncer.directories << 'tmp'
+      syncer.path = 'storage'
+      Backup::Syncer::S3::SyncContext.any_instance.
+        stubs(:`).returns 'MD5(tmp/foo)= 123abcdef'
+    end
+
+    it "respects the concurrency_type setting with threads" do
+      syncer.concurrency_type = :threads
+
+      Parallel.expects(:each).with(anything, {:in_threads => 2}, anything)
+
+      syncer.perform!
+    end
+
+    it "respects the parallel thread count" do
+      syncer.concurrency_type  = :threads
+      syncer.concurrency_level = 10
+
+      Parallel.expects(:each).with(anything, {:in_threads => 10}, anything)
+
+      syncer.perform!
+    end
+
+    it "respects the concurrency_type setting with processors" do
+      syncer.concurrency_type = :processes
+
+      Parallel.expects(:each).with(anything, {:in_processes => 2}, anything)
+
+      syncer.perform!
+    end
+
+    it "respects the parallel thread count" do
+      syncer.concurrency_type  = :processes
+      syncer.concurrency_level = 10
+
+      Parallel.expects(:each).with(anything, {:in_processes => 10}, anything)
+
+      syncer.perform!
+    end
+
+    context 'file exists locally' do
+      it "uploads a file if it does not exist remotely" do
+        files.expects(:create).with(:key => 'storage/tmp/foo', :body => content)
+
+        syncer.perform!
+      end
+
+      it "uploads a file if it exists remotely with a different MD5" do
+        files << stub('file', :key => 'storage/tmp/foo', :etag => 'abcdef123')
+
+        files.expects(:create).with(:key => 'storage/tmp/foo', :body => content)
+
+        syncer.perform!
+      end
+
+      it "does nothing if the file exists remotely with the same MD5" do
+        files << stub('file', :key => 'storage/tmp/foo', :etag => '123abcdef')
+
+        files.expects(:create).never
+
+        syncer.perform!
+      end
+
+      it "skips the file if it no longer exists locally" do
+        File.stubs(:exist?).returns false
+
+        files.expects(:create).never
+
+        syncer.perform!
+      end
+
+      it "respects the given path" do
+        syncer.path = 'box'
+
+        files.expects(:create).with(:key => 'box/tmp/foo', :body => content)
+
+        syncer.perform!
+      end
+
+      it "uploads the content of the local file" do
+        File.expects(:open).with('tmp/foo').returns content
+
+        syncer.perform!
+      end
+
+      it "creates the connection with the provided credentials" do
+        syncer.api_key    = 'my-key'
+        syncer.username   = 'my-name'
+        syncer.auth_url   = 'my-auth'
+        syncer.servicenet = 'my-servicenet'
+
+        Fog::Storage.expects(:new).with(
+          :provider             => 'Rackspace',
+          :rackspace_api_key    => 'my-key',
+          :rackspace_username   => 'my-name',
+          :rackspace_auth_url   => 'my-auth',
+          :rackspace_servicenet => 'my-servicenet'
+        ).returns connection
+
+        syncer.perform!
+      end
+
+      it "uses the container with the given name" do
+        syncer.container = 'leaky'
+
+        connection.directories.expects(:get).with('leaky').returns(container)
+
+        syncer.perform!
+      end
+
+      it "creates the container if one does not exist" do
+        syncer.container = 'leaky'
+        connection.directories.stubs(:get).returns nil
+
+        connection.directories.expects(:create).
+          with(:key => 'leaky').returns(container)
+
+        syncer.perform!
+      end
+
+      it "iterates over each directory" do
+        syncer.directories << 'files'
+
+        Backup::Syncer::CloudFiles::SyncContext.any_instance.expects(:`).
+          with('find tmp -print0 | xargs -0 openssl md5 2> /dev/null').
+          returns 'MD5(tmp/foo)= 123abcdef'
+        Backup::Syncer::CloudFiles::SyncContext.any_instance.expects(:`).
+          with('find files -print0 | xargs -0 openssl md5 2> /dev/null').
+          returns 'MD5(tmp/foo)= 123abcdef'
+
+        syncer.perform!
+      end
+    end
+
+    context 'file does not exist locally' do
+      let(:file) { stub('file', :key => 'storage/tmp/foo',
+        :etag => '123abcdef') }
+
+      before :each do
+        Backup::Syncer::CloudFiles::SyncContext.any_instance.
+          stubs(:`).returns ''
+        files << file
+        File.stubs(:exist?).returns false
+      end
+
+      it "removes the remote file when mirroring is turned on" do
+        syncer.mirror = true
+
+        file.expects(:destroy).once
+
+        syncer.perform!
+      end
+
+      it "leaves the remote file when mirroring is turned off" do
+        syncer.mirror = false
+
+        file.expects(:destroy).never
+
+        syncer.perform!
+      end
+
+      it "does not remove files not under one of the specified directories" do
+        file.stubs(:key).returns 'unsynced/tmp/foo'
+        syncer.mirror = true
+
+        file.expects(:destroy).never
+
+        syncer.perform!
+      end
+    end
+  end
+end