backup 3.6.0 → 3.7.0

data/lib/backup/syncer/cloud/local_file.rb

@@ -0,0 +1,93 @@
+# encoding: utf-8
+
+module Backup
+  module Syncer
+    module Cloud
+      class LocalFile
+        attr_reader :path, :md5
+
+        class << self
+          include Utilities::Helpers
+
+          # Returns a Hash of LocalFile objects for each file within +dir+.
+          # Hash keys are the file's path relative to +dir+.
+          def find(dir)
+            dir = File.expand_path(dir)
+            hash = {}
+            find_md5(dir).each do |path, md5|
+              file = new(path, md5)
+              hash[path.sub(dir + '/', '')] = file if file
+            end
+            hash
+          end
+
+          # Return a new LocalFile object if it's valid.
+          # Otherwise, log a warning and return nil.
+          def new(*args)
+            file = super
+            if file.invalid?
+              Logger.warn("\s\s[skipping] #{ file.path }\n" +
+                          "\s\sPath Contains Invalid UTF-8 byte sequences")
+              file = nil
+            end
+            file
+          end
+
+          private
+
+          # Returns an Array of file paths and their md5 hashes.
+          #
+          # Lines output from `cmd` are formatted like:
+          #   MD5(/dir/subdir/file)= 7eaabd1f53024270347800d0fdb34357
+          # However, if +dir+ is empty, the following is returned:
+          #   (stdin)= d41d8cd98f00b204e9800998ecf8427e
+          # Which extracts as: ['in', 'd41d8cd98f00b204e9800998ecf8427e']
+          # I'm not sure I can rely on the fact this doesn't begin with 'MD5',
+          # so I'll reject entries with a path that doesn't start with +dir+.
+          #
+          # String#slice avoids `invalid byte sequence in UTF-8` errors
+          # that String#split would raise.
+          #
+          # Utilities#run is not used here because this would produce too much
+          # log output, and Pipeline does not support capturing output.
+          def find_md5(dir)
+            cmd = "#{ utility(:find) } -L '#{ dir }' -type f -print0 | " +
+                  "#{ utility(:xargs) } -0 #{ utility(:openssl) } md5 2> /dev/null"
+            %x[#{ cmd }].lines.map do |line|
+              line.chomp!
+              entry = [line.slice(4..-36), line.slice(-32..-1)]
+              entry[0].to_s.start_with?(dir) ? entry : nil
+            end.compact
+          end
+        end
+
+        # If +path+ contains invalid UTF-8, it will be sanitized
+        # and the LocalFile object will be flagged as invalid.
+        # This is done so @file.path may be logged.
+        def initialize(path, md5)
+          @path = sanitize(path)
+          @md5 = md5
+        end
+
+        def invalid?
+          !!@invalid
+        end
+
+        private
+
+        def sanitize(str)
+          str.each_char.map do |char|
+            begin
+              char.unpack('U')
+              char
+            rescue
+              @invalid = true
+              "\xEF\xBF\xBD" # => "\uFFFD"
+            end
+          end.join
+        end
+
+      end
+    end
+  end
+end

data/lib/backup/syncer/cloud/s3.rb

@@ -1,67 +1,114 @@
 # encoding: utf-8
+require 'backup/cloud_io/s3'
 
 module Backup
   module Syncer
     module Cloud
       class S3 < Base
+        class Error < Backup::Error; end
 
         ##
         # Amazon Simple Storage Service (S3) Credentials
         attr_accessor :access_key_id, :secret_access_key
 
         ##
-        # The S3 bucket to store files to
+        # Amazon S3 bucket name
         attr_accessor :bucket
 
         ##
-        # The AWS region of the specified S3 bucket
+        # Region of the specified S3 bucket
         attr_accessor :region
 
         ##
-        # Instantiates a new Cloud::S3 Syncer.
+        # Encryption algorithm to use for Amazon Server-Side Encryption
         #
-        # Pre-configured defaults specified in
-        # Configuration::Syncer::Cloud::S3
-        # are set via a super() call to Cloud::Base,
-        # which in turn will invoke Syncer::Base.
+        # Supported values:
         #
-        # Once pre-configured defaults and Cloud specific defaults are set,
-        # the block from the user's configuration file is evaluated.
-        def initialize(syncer_id = nil, &block)
+        # - :aes256
+        #
+        # Default: nil
+        attr_accessor :encryption
+
+        ##
+        # Storage class to use for the S3 objects uploaded
+        #
+        # Supported values:
+        #
+        # - :standard (default)
+        # - :reduced_redundancy
+        #
+        # Default: :standard
+        attr_accessor :storage_class
+
+        def initialize(syncer_id = nil)
           super
 
-          instance_eval(&block) if block_given?
-          @path = path.sub(/^\//, '')
+          @storage_class ||= :standard
+
+          check_configuration
         end
 
         private
 
-        ##
-        # Established and creates a new Fog storage object for S3.
-        def connection
-          @connection ||= Fog::Storage.new(
-            :provider              => provider,
-            :aws_access_key_id     => access_key_id,
-            :aws_secret_access_key => secret_access_key,
-            :region                => region
+        def cloud_io
+          @cloud_io ||= CloudIO::S3.new(
+            :access_key_id      => access_key_id,
+            :secret_access_key  => secret_access_key,
+            :bucket             => bucket,
+            :region             => region,
+            :encryption         => encryption,
+            :storage_class      => storage_class,
+            :max_retries        => max_retries,
+            :retry_waitsec      => retry_waitsec,
+            # Syncer can not use multipart upload.
+            :chunk_size         => 0
           )
         end
 
-        ##
-        # Creates a new @repository_object (bucket).
-        # Fetches it from S3 if it already exists,
-        # otherwise it will create it first and fetch use that instead.
-        def repository_object
-          @repository_object ||= connection.directories.get(bucket) ||
-            connection.directories.create(:key => bucket, :location => region)
+        def get_remote_files(remote_base)
+          hash = {}
+          cloud_io.objects(remote_base).each do |object|
+            relative_path = object.key.sub(remote_base + '/', '')
+            hash[relative_path] = object.etag
+          end
+          hash
         end
 
-        ##
-        # This is the provider that Fog uses for the Cloud Files
-        def provider
-          "AWS"
+        def check_configuration
+          required = %w{ access_key_id secret_access_key bucket }
+          raise Error, <<-EOS if required.map {|name| send(name) }.any?(&:nil?)
+            Configuration Error
+            #{ required.map {|name| "##{ name }"}.join(', ') } are all required
+          EOS
+
+          raise Error, <<-EOS if encryption && encryption.to_s.upcase != 'AES256'
+            Configuration Error
+            #encryption must be :aes256 or nil
+          EOS
+
+          classes = ['STANDARD', 'REDUCED_REDUNDANCY']
+          raise Error, <<-EOS unless classes.include?(storage_class.to_s.upcase)
+            Configuration Error
+            #storage_class must be :standard or :reduced_redundancy
+          EOS
         end
 
+        attr_deprecate :concurrency_type, :version => '3.7.0',
+                       :message => 'Use #thread_count instead.',
+                       :action => lambda {|klass, val|
+                         if val == :threads
+                           klass.thread_count = 2 unless klass.thread_count
+                         else
+                           klass.thread_count = 0
+                         end
+                       }
+
+        attr_deprecate :concurrency_level, :version => '3.7.0',
+                       :message => 'Use #thread_count instead.',
+                       :action => lambda {|klass, val|
+                         klass.thread_count = val unless klass.thread_count == 0
+                       }
+
       end # Class S3 < Base
     end # module Cloud
   end

data/lib/backup/syncer/rsync/base.rb

@@ -9,6 +9,13 @@ module Backup
         # Additional String or Array of options for the rsync cli
         attr_accessor :additional_rsync_options
 
+        def initialize(syncer_id = nil, &block)
+          super
+          instance_eval(&block) if block_given?
+
+          @path ||= '~/backups'
+        end
+
         private
 
         ##

data/lib/backup/syncer/rsync/local.rb

@@ -5,11 +5,6 @@ module Backup
     module RSync
       class Local < Base
 
-        def initialize(syncer_id = nil, &block)
-          super
-          instance_eval(&block) if block_given?
-        end
-
         def perform!
           log!(:started)
 

data/lib/backup/syncer/rsync/push.rb

@@ -101,9 +101,8 @@ module Backup
         # Flag for compressing (only compresses for the transfer)
         attr_accessor :compress
 
-        def initialize(syncer_id = nil, &block)
+        def initialize(syncer_id = nil)
           super
-          instance_eval(&block) if block_given?
 
           @mode ||= :ssh
           @port ||= mode == :rsync_daemon ? 873 : 22

data/lib/backup/utilities.rb

@@ -2,14 +2,17 @@
 
 module Backup
   module Utilities
+    class Error < Backup::Error; end
+
     UTILITY = {}
     NAMES = %w{
-      tar cat split find xargs sudo chown
+      tar cat split find xargs sudo chown hostname
       gzip bzip2 lzma pbzip2
       mongo mongodump mysqldump pg_dump pg_dumpall redis-cli riak-admin
       gpg openssl
       rsync ssh
       sendmail exim
+      send_nsca
     }
 
     module DSL
@@ -23,7 +26,7 @@ module Backup
           define_method name.gsub('-', '_'), lambda {|val|
             path = File.expand_path(val)
             unless File.executable?(path)
-              raise Errors::Utilities::NotFoundError, <<-EOS
+              raise Utilities::Error, <<-EOS
                 The path given for '#{ name }' was not found or not executable.
                 Path was: #{ path }
               EOS
@@ -58,14 +61,15 @@ module Backup
       #
       #   Backup::Utilities.configure do
       #     # General Utilites
-      #     tar   '/path/to/tar'
+      #     tar      '/path/to/tar'
       #     tar_dist :gnu   # or :bsd
-      #     cat   '/path/to/cat'
-      #     split '/path/to/split'
-      #     find  '/path/to/find'
-      #     xargs '/path/to/xargs'
-      #     sudo  '/path/to/sudo'
-      #     chown '/path/to/chown'
+      #     cat      '/path/to/cat'
+      #     split    '/path/to/split'
+      #     find     '/path/to/find'
+      #     xargs    '/path/to/xargs'
+      #     sudo     '/path/to/sudo'
+      #     chown    '/path/to/chown'
+      #     hostname '/path/to/hostname'
       #
       #     # Compressors
       #     gzip    '/path/to/gzip'
@@ -93,6 +97,7 @@ module Backup
       #     # Notifiers
       #     sendmail  '/path/to/sendmail'
       #     exim      '/path/to/exim'
+      #     send_nsca '/path/to/send_nsca'
       #   end
       #
       # These paths may be set using absolute paths, or relative to the
@@ -129,11 +134,10 @@ module Backup
       # Raises an error if utility can not be found in the system's $PATH
       def utility(name)
         name = name.to_s.strip
-        raise Errors::Utilities::NotFoundError,
-            'Utility Name Empty' if name.empty?
+        raise Error, 'Utility Name Empty' if name.empty?
 
         UTILITY[name] ||= %x[which '#{ name }' 2>/dev/null].chomp
-        raise(Errors::Utilities::NotFoundError, <<-EOS) if UTILITY[name].empty?
+        raise Error, <<-EOS if UTILITY[name].empty?
           Could not locate '#{ name }'.
           Make sure the specified utility is installed
           and available in your system's $PATH, or specify it's location
@@ -190,8 +194,7 @@ module Backup
             out, err = stdout.read.strip, stderr.read.strip
           end
         rescue Exception => e
-          raise Errors::Utilities::SystemCallError.wrap(
-              e, "Failed to execute '#{ name }'")
+          raise Error.wrap(e, "Failed to execute '#{ name }'")
         ensure
           GC.enable if RUBY_VERSION < '1.9'
         end
@@ -211,7 +214,7 @@ module Backup
 
           return out
         else
-          raise Errors::Utilities::SystemCallError, <<-EOS
+          raise Error, <<-EOS
             '#{ name }' failed with exit status: #{ ps.exitstatus }
             STDOUT Messages: #{ out.empty? ? 'None' : "\n#{ out }" }
             STDERR Messages: #{ err.empty? ? 'None' : "\n#{ err }" }

data/lib/backup/version.rb

@@ -1,5 +1,5 @@
 # encoding: utf-8
 
 module Backup
-  VERSION = '3.6.0'
+  VERSION = '3.7.0'
 end

data/templates/cli/notifier/http_post

@@ -0,0 +1,35 @@
+  ##
+  # HttpPost [Notifier]
+  #
+  # For details, see:
+  # https://github.com/meskyanichi/backup/wiki/Notifiers
+  #
+  notify_by HttpPost do |post|
+    post.on_success = true
+    post.on_warning = true
+    post.on_failure = true
+
+    # URI to post the notification to.
+    # Port may be specified if needed.
+    # If Basic Authentication is required, supply user:pass.
+    post.uri = 'https://user:pass@your.domain.com:8443/path'
+
+    ##
+    # Optional
+    #
+    # Additional headers to send.
+    # post.headers = { 'Authentication' => 'my_auth_info' }
+    #
+    # Additional form params to post.
+    # post.params = { 'auth_token' => 'my_token' }
+    #
+    # Successful response codes. Default: 200
+    # post.success_codes = [200, 201, 204]
+    #
+    # Defaults to true on most systems.
+    # Force with +true+, disable with +false+
+    # post.ssl_verify_peer = false
+    #
+    # Supplied by default. Override with a custom 'cacert.pem' file.
+    # post.ssl_ca_file = '/my/cacert.pem'
+  end

data/templates/cli/notifier/nagios

@@ -0,0 +1,13 @@
+##
+# Nagios [Notifier]
+#
+notify_by Nagios do |nagios|
+  nagios.on_success = true
+  nagios.on_warning = true
+  nagios.on_failure = true
+
+  nagios.nagios_host  = 'nagioshost'
+  nagios.nagios_port  = 5667
+  nagios.service_name = 'My Backup'
+  nagios.service_host = 'backuphost'
+end

data/templates/cli/storage/cloud_files

@@ -1,22 +1,13 @@
   ##
   # Rackspace Cloud Files [Storage]
   #
-  # Available Auth URLs:
-  #
-  #  - https://auth.api.rackspacecloud.com     (US - Default)
-  #  - https://lon.auth.api.rackspacecloud.com (UK)
-  #
-  # Servicenet:
-  #
-  #  Set this to 'true' if Backup runs on a Rackspace server. It will avoid
-  #  transfer charges and it's more performant.
-  #
+  # See the documentation on the Wiki for details.
+  # https://github.com/meskyanichi/backup/wiki/Storages
   store_with CloudFiles do |cf|
-    cf.api_key    = "my_api_key"
-    cf.username   = "my_username"
-    cf.container  = "my_container"
-    cf.path       = "/path/to/my/backups"
-    cf.keep       = 5
-    cf.auth_url   = "lon.auth.api.rackspacecloud.com"
-    cf.servicenet = false
+    cf.api_key            = 'my_api_key'
+    cf.username           = 'my_username'
+    cf.container          = 'my_container'
+    cf.segments_container = 'my_segments_container' # must be different than `container`
+    cf.segment_size       = 5 # MiB
+    cf.path               = 'path/to/backups' # path within the container
   end