backup 3.0.21 → 3.0.22

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    backup (3.0.21)
+    backup (3.0.22)
       POpen4 (~> 0.1.4)
       thor (~> 0.14.6)
 
@@ -71,6 +71,7 @@ GEM
     net-ssh (2.1.4)
     nokogiri (1.5.0)
     open4 (1.3.0)
+    parallel (0.5.12)
     polyglot (0.3.3)
     prowler (1.3.1)
     rack (1.4.0)
@@ -120,6 +121,7 @@ DEPENDENCIES
   net-scp (~> 1.0.4)
   net-sftp (~> 2.0.5)
   net-ssh (~> 2.1.4)
+  parallel (~> 0.5.12)
   prowler (>= 1.3.1)
   rb-fsevent
   rb-inotify

data/README.md

@@ -18,7 +18,7 @@ Drop me a message for any questions, suggestions, requests, bugs or submit them
 Installation
 ------------
 
-To get the latest stable version (3.0.20)
+To get the latest stable version
 
     gem install backup
 
@@ -94,7 +94,8 @@ Syncers
 -------
 
 - RSync (Push, Pull and Local)
-- Amazon Simple Storage Service (S3)
+- Amazon S3
+- Rackspce Cloud Files
 
 [Syncer Wiki Page](https://github.com/meskyanichi/backup/wiki/Syncers)
 
@@ -238,7 +239,7 @@ First, it will dump the two Databases (MySQL and MongoDB). The MySQL dump will b
 `sample_backup/databases/MySQL/my_sample_mysql_db.sql.gz`. The MongoDB dump will be dumped into
 `sample_backup/databases/MongoDB/`, which will then be packaged into `sample_backup/databases/MongoDB-#####.tar.gz`
 (`#####` will be a simple unique identifier, in case multiple dumps are performed.)
-Next, it will create two _tar_ Archives (user_avatars and logs). Each will be piped through the Gzip Compressor into
+Next, it will create two _tar_ Archives (user\_avatars and logs). Each will be piped through the Gzip Compressor into
 `sample_backup/archives/` as `user_archives.tar.gz` and `logs.tar.gz`.
 Finally, the `sample_backup` directory will be packaged into an uncompressed _tar_ archive, which will be piped through
 the OpenSSL Encryptor to encrypt this final package into `YYYY-MM-DD-hh-mm-ss.sample_backup.tar.enc`. This final

data/lib/backup.rb

@@ -78,8 +78,10 @@ module Backup
   ##
   # Autoload Backup syncer files
   module Syncer
-    autoload :Base,  File.join(SYNCER_PATH, 'base')
-    autoload :S3,    File.join(SYNCER_PATH, 's3')
+    autoload :Base,       File.join(SYNCER_PATH, 'base')
+    autoload :Cloud,      File.join(SYNCER_PATH, 'cloud')
+    autoload :CloudFiles, File.join(SYNCER_PATH, 'cloud_files')
+    autoload :S3,         File.join(SYNCER_PATH, 's3')
     module RSync
       autoload :Base,  File.join(SYNCER_PATH, 'rsync', 'base')
       autoload :Local, File.join(SYNCER_PATH, 'rsync', 'local')
@@ -174,8 +176,10 @@ module Backup
     end
 
     module Syncer
-      autoload :Base,  File.join(CONFIGURATION_PATH, 'syncer', 'base')
-      autoload :S3,    File.join(CONFIGURATION_PATH, 'syncer', 's3')
+      autoload :Base,       File.join(CONFIGURATION_PATH, 'syncer', 'base')
+      autoload :Cloud,      File.join(CONFIGURATION_PATH, 'syncer', 'cloud')
+      autoload :CloudFiles, File.join(CONFIGURATION_PATH, 'syncer', 'cloud_files')
+      autoload :S3,         File.join(CONFIGURATION_PATH, 'syncer', 's3')
       module RSync
         autoload :Base,  File.join(CONFIGURATION_PATH, 'syncer', 'rsync', 'base')
         autoload :Local, File.join(CONFIGURATION_PATH, 'syncer', 'rsync', 'local')

data/lib/backup/config.rb

@@ -111,7 +111,7 @@ module Backup
             # Encryptors
             ['OpenSSL', 'GPG'],
             # Syncers
-            ['S3', { 'RSync' => ['Push', 'Pull', 'Local'] }],
+            ['Rackspace', 'S3', { 'RSync' => ['Push', 'Pull', 'Local'] }],
             # Notifiers
             ['Mail', 'Twitter', 'Campfire', 'Presently', 'Prowl', 'Hipchat']
           ]

data/lib/backup/configuration/syncer/cloud.rb

@@ -0,0 +1,23 @@
+# encoding: utf-8
+
+module Backup
+  module Configuration
+    module Syncer
+      class Cloud < Base
+        class << self
+          ##
+          # Amazon S3 bucket name and path to sync to
+          attr_accessor :bucket, :path
+
+          ##
+          # Directories to sync
+          attr_accessor :directories
+
+          ##
+          # Flag to enable mirroring
+          attr_accessor :mirror
+        end
+      end
+    end
+  end
+end

data/lib/backup/configuration/syncer/cloud_files.rb

@@ -0,0 +1,30 @@
+# encoding: utf-8
+
+module Backup
+  module Configuration
+    module Syncer
+      class CloudFiles < Cloud
+        class << self
+          ##
+          # Rackspace CloudFiles Credentials
+          attr_accessor :api_key, :username
+
+          ##
+          # Rackspace CloudFiles Container
+          attr_accessor :container
+
+          ##
+          # Rackspace AuthURL allows you to connect to a different Rackspace datacenter
+          # - https://auth.api.rackspacecloud.com     (Default: US)
+          # - https://lon.auth.api.rackspacecloud.com (UK)
+          attr_accessor :auth_url
+
+          ##
+          # Improve performance and avoid data transfer costs by setting @servicenet to `true`
+          # This only works if Backup runs on a Rackspace server
+          attr_accessor :servicenet
+        end
+      end
+    end
+  end
+end

data/lib/backup/configuration/syncer/s3.rb

@@ -3,25 +3,19 @@
 module Backup
   module Configuration
     module Syncer
-      class S3 < Base
+      class S3 < Cloud
         class << self
-
           ##
           # Amazon Simple Storage Service (S3) Credentials
           attr_accessor :access_key_id, :secret_access_key
 
           ##
-          # Amazon S3 bucket name and path to sync to
-          attr_accessor :bucket, :path
-
-          ##
-          # Flag to enable mirroring
-          attr_accessor :mirror
+          # The S3 bucket to store files to
+          attr_accessor :bucket
 
           ##
-          # Additional options for the s3sync cli
-          attr_accessor :additional_options
-
+          # The AWS region of the specified S3 bucket
+          attr_accessor :region
         end
       end
     end

data/lib/backup/dependency.rb

@@ -79,6 +79,12 @@ module Backup
           :require => 'hipchat',
           :version => '~> 0.4.1',
           :for => 'Sending notifications to Hipchat'
+        },
+
+        'parallel' => {
+          :require => 'parallel',
+          :version => '~> 0.5.12',
+          :for => 'Adding concurrency to Cloud-based syncers.'
         }
       }
     end

data/lib/backup/notifier/twitter.rb

@@ -49,7 +49,7 @@ module Backup
                when :warning then 'Warning'
                when :failure then 'Failure'
                end
-        message = "[Backup::%s] #{@model.label} (#{@model.trigger})" % name
+        message = "[Backup::%s] #{@model.label} (#{@model.trigger}) (@ #{@model.time})" % name
         send_message(message)
       end
 

data/lib/backup/syncer/base.rb

@@ -6,6 +6,31 @@ module Backup
       include Backup::CLI::Helpers
       include Backup::Configuration::Helpers
 
+      ##
+      # Directories to sync
+      attr_accessor :directories
+
+      ##
+      # Path to store the synced files/directories to
+      attr_accessor :path
+
+      ##
+      # Flag for mirroring the files/directories
+      attr_accessor :mirror
+
+      ##
+      # Syntactical suger for the DSL for adding directories
+      def directories(&block)
+        return @directories unless block_given?
+        instance_eval(&block)
+      end
+
+      ##
+      # Adds a path to the @directories array
+      def add(path)
+        @directories << path
+      end
+
       private
 
       def syncer_name

data/lib/backup/syncer/cloud.rb

@@ -0,0 +1,187 @@
+# encoding: utf-8
+
+##
+# Only load the Fog gem, along with the Parallel gem, when the Backup::Syncer::Cloud class is loaded
+Backup::Dependency.load('fog')
+Backup::Dependency.load('parallel')
+
+module Backup
+  module Syncer
+    class Cloud < Base
+
+      ##
+      # Create a Mutex to synchronize certain parts of the code
+      # in order to prevent race conditions or broken STDOUT.
+      MUTEX = Mutex.new
+
+      ##
+      # Concurrency setting - defaults to false, but can be set to:
+      # - :threads
+      # - :processes
+      attr_accessor :concurrency_type
+
+      ##
+      # Concurrency level - the number of threads or processors to use. Defaults to 2.
+      attr_accessor :concurrency_level
+
+      ##
+      # Instantiates a new Cloud Syncer object and sets the default
+      # configuration specified in the Backup::Configuration::Syncer::S3. Then
+      # it sets the object defaults if particular properties weren't set.
+      # Finally it'll evaluate the users configuration file and overwrite
+      # anything that's been defined.
+      def initialize(&block)
+        load_defaults!
+
+        @path               ||= 'backups'
+        @directories        ||= Array.new
+        @mirror             ||= false
+        @concurrency_type     = false
+        @concurrency_level    = 2
+
+        instance_eval(&block) if block_given?
+
+        @path = path.sub(/^\//, '')
+      end
+
+      ##
+      # Performs the Sync operation
+      def perform!
+        Logger.message("#{ self.class } started the syncing process:")
+
+        directories.each do |directory|
+          SyncContext.new(directory, repository_object, path).
+            sync! mirror, concurrency_type, concurrency_level
+        end
+      end
+
+      private
+
+      class SyncContext
+        attr_reader :directory, :bucket, :path
+
+        ##
+        # Creates a new SyncContext object which handles a single directory
+        # from the Syncer::Base @directories array.
+        def initialize(directory, bucket, path)
+          @directory, @bucket, @path = directory, bucket, path
+        end
+
+        ##
+        # Performs the sync operation using the provided techniques (mirroring/concurrency).
+        def sync!(mirror = false, concurrency_type = false, concurrency_level = 2)
+          block = Proc.new { |relative_path| sync_file relative_path, mirror }
+
+          case concurrency_type
+          when FalseClass
+            all_file_names.each &block
+          when :threads
+            Parallel.each all_file_names, :in_threads => concurrency_level, &block
+          when :processes
+            Parallel.each all_file_names, :in_processes => concurrency_level, &block
+          else
+            raise "Unknown concurrency_type setting: #{concurrency_type.inspect}"
+          end
+        end
+
+        private
+
+        ##
+        # Gathers all the remote and local file name and merges them together, removing
+        # duplicate keys if any, and sorts the in alphabetical order.
+        def all_file_names
+          @all_file_names ||= (local_files.keys | remote_files.keys).sort
+        end
+
+        ##
+        # Returns a Hash of local files (the keys are the filesystem paths,
+        # the values are the LocalFile objects for that given file)
+        def local_files
+          @local_files ||= begin
+            local_hashes.split("\n").collect { |line|
+              LocalFile.new directory, line
+            }.inject({}) { |hash, file|
+              hash[file.relative_path] = file
+              hash
+            }
+          end
+        end
+
+        ##
+        # Returns a String of file paths and their md5 hashes.
+        def local_hashes
+          MUTEX.synchronize { Logger.message("\s\sGenerating checksums for #{ directory }") }
+          `find #{directory} -print0 | xargs -0 openssl md5 2> /dev/null`
+        end
+
+        ##
+        # Returns a Hash of remote files (the keys are the remote paths,
+        # the values are the Fog file objects for that given file)
+        def remote_files
+          @remote_files ||= bucket.files.to_a.select { |file|
+            file.key[%r{^#{remote_base}/}]
+          }.inject({}) { |hash, file|
+            key = file.key.gsub(/^#{remote_base}\//,
+              "#{directory.split('/').last}/")
+            hash[key] = file
+            hash
+          }
+        end
+
+        ##
+        # Creates and returns a String that represents the base remote storage path
+        def remote_base
+          @remote_base ||= [path, directory.split('/').last].select { |part|
+            part && part.strip.length > 0
+          }.join('/')
+        end
+
+        ##
+        # Performs a sync operation on a file. When mirroring is enabled
+        # and a local file has been removed since the last sync, it will also
+        # remove it from the remote location. It will no upload files that
+        # have not changed since the last sync. Checks are done using an md5 hash.
+        # If a file has changed, or has been newly added, the file will be transferred/overwritten.
+        def sync_file(relative_path, mirror)
+          local_file  = local_files[relative_path]
+          remote_file = remote_files[relative_path]
+
+          if local_file && File.exist?(local_file.path)
+            unless remote_file && remote_file.etag == local_file.md5
+              MUTEX.synchronize { Logger.message("\s\s[transferring] #{relative_path}") }
+              bucket.files.create(
+                :key  => "#{path}/#{relative_path}".gsub(/^\//, ''),
+                :body => File.open(local_file.path)
+              )
+            else
+              MUTEX.synchronize { Logger.message("\s\s[skipping] #{relative_path}") }
+            end
+          elsif remote_file && mirror
+            MUTEX.synchronize { Logger.message("\s\s[removing] #{relative_path}") }
+            remote_file.destroy
+          end
+        end
+      end
+
+      class LocalFile
+        attr_reader :directory, :path, :md5
+
+        ##
+        # Creates a new LocalFile object using the given directory and line
+        # from the md5 hash checkup. This object figures out the path, relative_path and md5 hash
+        # for the file.
+        def initialize(directory, line)
+          @directory  = directory
+          @path, @md5 = *line.chomp.match(/^MD5\(([^\)]+)\)= (\w+)$/).captures
+        end
+
+        ##
+        # Returns the relative path to the file.
+        def relative_path
+          @relative_path ||= path.gsub %r{^#{directory}},
+            directory.split('/').last
+        end
+      end
+    end
+  end
+end

data/lib/backup/syncer/cloud_files.rb

@@ -0,0 +1,56 @@
+# encoding: utf-8
+
+module Backup
+  module Syncer
+    class CloudFiles < Cloud
+
+      ##
+      # Rackspace CloudFiles Credentials
+      attr_accessor :api_key, :username
+
+      ##
+      # Rackspace CloudFiles Container
+      attr_accessor :container
+
+      ##
+      # Rackspace AuthURL allows you to connect to a different Rackspace datacenter
+      # - https://auth.api.rackspacecloud.com     (Default: US)
+      # - https://lon.auth.api.rackspacecloud.com (UK)
+      attr_accessor :auth_url
+
+      ##
+      # Improve performance and avoid data transfer costs by setting @servicenet to `true`
+      # This only works if Backup runs on a Rackspace server
+      attr_accessor :servicenet
+
+      private
+
+      ##
+      # Established and creates a new Fog storage object for CloudFiles.
+      def connection
+        @connection ||= Fog::Storage.new(
+          :provider             => provider,
+          :rackspace_username   => username,
+          :rackspace_api_key    => api_key,
+          :rackspace_auth_url   => auth_url,
+          :rackspace_servicenet => servicenet
+        )
+      end
+
+      ##
+      # Creates a new @repository_object (container). Fetches it from Cloud Files
+      # if it already exists, otherwise it will create it first and fetch use that instead.
+      def repository_object
+        @repository_object ||= connection.directories.get(container) ||
+          connection.directories.create(:key => container)
+      end
+
+      ##
+      # This is the provider that Fog uses for the Cloud Files
+      def provider
+        "Rackspace"
+      end
+
+    end
+  end
+end