backup 3.1.3 → 3.2.0

data/lib/backup/syncer/base.rb

@@ -14,12 +14,20 @@ module Backup
       # Flag for mirroring the files/directories
       attr_accessor :mirror
 
-      def initialize
+      ##
+      # Optional user-defined identifier to differentiate multiple syncers
+      # defined within a single backup model. Currently this is only used
+      # in the log messages.
+      attr_reader :syncer_id
+
+      def initialize(syncer_id = nil)
+        @syncer_id = syncer_id
+
         load_defaults!
 
-        @path               ||= 'backups'
-        @mirror             ||= false
-        @directories          = Array.new
+        @path   ||= '~/backups'
+        @mirror ||= false
+        @directories = Array.new
       end
 
       ##
@@ -29,16 +37,23 @@ module Backup
         instance_eval(&block)
       end
 
-      ##
-      # Adds a path to the @directories array
       def add(path)
-        @directories << path
+        directories << path
       end
 
       private
 
       def syncer_name
-        self.class.to_s.sub('Backup::', '')
+        @syncer_name ||= self.class.to_s.sub('Backup::', '') +
+            (syncer_id ? " (#{ syncer_id })" : '')
+      end
+
+      def log!(action)
+        msg = case action
+              when :started then 'Started...'
+              when :finished then 'Finished!'
+              end
+        Logger.info "#{ syncer_name } #{ msg }"
       end
 
     end

data/lib/backup/syncer/cloud/base.rb

@@ -39,9 +39,10 @@ module Backup
         # If not specified in the pre-configured defaults,
         # the Cloud specific defaults are set here before evaluating
         # any block provided in the user's configuration file.
-        def initialize
+        def initialize(syncer_id = nil)
           super
 
+          @path = path.sub(/^~\//, '')
           @concurrency_type  ||= false
           @concurrency_level ||= 2
         end
@@ -49,8 +50,8 @@ module Backup
         ##
         # Performs the Sync operation
         def perform!
+          log!(:started)
           Logger.info(
-            "#{ syncer_name } started the syncing process:\n" +
             "\s\sConcurrency: #{ @concurrency_type } Level: #{ @concurrency_level }"
           )
 
@@ -60,7 +61,7 @@ module Backup
             ).sync! @mirror, @concurrency_type, @concurrency_level
           end
 
-          Logger.info("#{ syncer_name } Syncing Complete!")
+          log!(:finished)
         end
 
         private

data/lib/backup/syncer/cloud/cloud_files.rb

@@ -36,7 +36,7 @@ module Backup
         #
         # Once pre-configured defaults and Cloud specific defaults are set,
         # the block from the user's configuration file is evaluated.
-        def initialize(&block)
+        def initialize(syncer_id = nil, &block)
           super
 
           instance_eval(&block) if block_given?

data/lib/backup/syncer/cloud/s3.rb

@@ -27,7 +27,7 @@ module Backup
         #
         # Once pre-configured defaults and Cloud specific defaults are set,
         # the block from the user's configuration file is evaluated.
-        def initialize(&block)
+        def initialize(syncer_id = nil, &block)
           super
 
           instance_eval(&block) if block_given?

data/lib/backup/syncer/rsync/base.rb

@@ -4,43 +4,31 @@ module Backup
   module Syncer
     module RSync
       class Base < Syncer::Base
-        ##
-        # Additional options for the rsync cli
-        attr_accessor :additional_options
 
         ##
-        # Instantiates a new RSync Syncer object
-        # and sets the default configuration
-        def initialize
-          super
-
-          @additional_options ||= Array.new
-        end
+        # Additional String or Array of options for the rsync cli
+        attr_accessor :additional_rsync_options
 
         private
 
         ##
-        # Returns the @directories as a space-delimited string of
-        # single-quoted values for use in the `rsync` command line.
-        # Each path is expanded, since these refer to local paths
-        # for both RSync::Local and RSync::Push.
-        # RSync::Pull does not use this method.
-        def directories_option
-          @directories.map do |directory|
-            "'#{ File.expand_path(directory) }'"
-          end.join(' ')
+        # Common base command for Local/Push/Pull
+        def rsync_command
+          utility(:rsync) << ' --archive' << mirror_option <<
+              " #{ Array(additional_rsync_options).join(' ') }".rstrip
         end
 
-        ##
-        # Returns Rsync syntax for enabling mirroring
         def mirror_option
-          '--delete' if @mirror
+          mirror ? ' --delete' : ''
         end
 
         ##
-        # Returns Rsync syntax for invoking "archive" mode
-        def archive_option
-          '--archive'
+        # Each path is expanded, since these refer to local paths and are
+        # being shell-quoted. This will also remove any trailing `/` from
+        # each path, as we don't want rsync's "trailing / on source directories"
+        # behavior. This method is used by RSync::Local and RSync::Push.
+        def paths_to_push
+          directories.map {|dir| "'#{ File.expand_path(dir) }'" }.join(' ')
         end
 
       end

data/lib/backup/syncer/rsync/local.rb

@@ -5,50 +5,36 @@ module Backup
     module RSync
       class Local < Base
 
-        ##
-        # Instantiates a new RSync::Local Syncer.
-        #
-        # Pre-configured defaults specified in
-        # Configuration::Syncer::RSync::Local
-        # are set via a super() call to RSync::Base,
-        # which in turn will invoke Syncer::Base.
-        #
-        # Once pre-configured defaults and RSync specific defaults are set,
-        # the block from the user's configuration file is evaluated.
-        def initialize(&block)
+        def initialize(syncer_id = nil, &block)
           super
-
           instance_eval(&block) if block_given?
         end
 
-        ##
-        # Performs the RSync::Local operation
-        # debug options: -vhP
         def perform!
-          Logger.info(
-            "#{ syncer_name } started syncing the following directories:\n\s\s" +
-            @directories.join("\n\s\s")
-          )
-          run("#{ utility(:rsync) } #{ options } " +
-              "#{ directories_option } '#{ dest_path }'")
+          log!(:started)
+
+          create_dest_path!
+          run("#{ rsync_command } #{ paths_to_push } '#{ dest_path }'")
+
+          log!(:finished)
         end
 
         private
 
-        ##
-        # Return expanded @path
+        # Expand path, since this is local and shell-quoted.
         def dest_path
-          @dest_path ||= File.expand_path(@path)
+          @dest_path ||= File.expand_path(path)
         end
 
-        ##
-        # Returns all the specified Rsync::Local options,
-        # concatenated, ready for the CLI
-        def options
-          ([archive_option, mirror_option] +
-            additional_options).compact.join("\s")
+        def create_dest_path!
+          FileUtils.mkdir_p dest_path
         end
 
+        attr_deprecate :additional_options, :version => '3.2.0',
+                       :message => 'Use #additional_rsync_options instead.',
+                       :action => lambda {|klass, val|
+                         klass.additional_rsync_options = val
+                       }
       end
     end
   end

data/lib/backup/syncer/rsync/pull.rb

@@ -5,19 +5,15 @@ module Backup
     module RSync
       class Pull < Push
 
-        ##
-        # Performs the RSync::Pull operation
-        # debug options: -vhP
         def perform!
+          log!(:started)
           write_password_file!
 
-          @directories.each do |directory|
-            Logger.info("#{ syncer_name } started syncing '#{ directory }'.")
-            run("#{ utility(:rsync) } #{ options } " +
-                "'#{ username }@#{ ip }:#{ directory.sub(/^\~\//, '') }' " +
-                "'#{ dest_path }'")
-          end
+          create_dest_path!
+          run("#{ rsync_command } #{ host_options }#{ paths_to_pull } " +
+              "'#{ dest_path }'")
 
+          log!(:finished)
         ensure
           remove_password_file!
         end
@@ -25,11 +21,51 @@ module Backup
         private
 
         ##
-        # Return expanded @path, since this path is local
+        # Returns the syntax for pulling multiple paths from the remote host.
+        # e.g.
+        #   rsync -a -e "ssh -p 22" host:'path1' :'path2' '/dest'
+        #   rsync -a rsync_user@host::'modname/path1' ::'modname/path2' '/dest'
+        #
+        # Remove any preceeding '~/', since these paths are on the remote.
+        # Also remove any trailing `/`, since we don't want rsync's
+        # "trailing / on source directories" behavior.
+        def paths_to_pull
+          sep = mode == :ssh ? ':' : '::'
+          directories.map {|dir|
+            "#{ sep }'#{ dir.sub(/^~\//, '').sub(/\/$/, '') }'"
+          }.join(' ').sub(/^#{ sep }/, '')
+        end
+
+        # Expand path, since this is local and shell-quoted.
         def dest_path
-          @dest_path ||= File.expand_path(@path)
+          @dest_path ||= File.expand_path(path)
         end
 
+        def create_dest_path!
+          FileUtils.mkdir_p dest_path
+        end
+
+        attr_deprecate :additional_options, :version => '3.2.0',
+                       :message => 'Use #additional_rsync_options instead.',
+                       :action => lambda {|klass, val|
+                         klass.additional_rsync_options = val
+                       }
+
+        attr_deprecate :username, :version => '3.2.0',
+                       :message => 'Use #ssh_user instead.',
+                       :action => lambda {|klass, val|
+                         klass.ssh_user = val
+                       }
+        attr_deprecate :password, :version => '3.2.0',
+                       :message => 'Use #rsync_password instead.',
+                       :action => lambda {|klass, val|
+                         klass.rsync_password = val
+                       }
+        attr_deprecate :ip, :version => '3.2.0',
+                       :message => 'Use #host instead.',
+                       :action => lambda {|klass, val|
+                         klass.host = val
+                       }
       end
     end
   end

data/lib/backup/syncer/rsync/push.rb

@@ -6,54 +6,119 @@ module Backup
       class Push < Base
 
         ##
-        # Server credentials
-        attr_accessor :username, :password
+        # Mode of operation
+        #
+        # [:ssh (default)]
+        #   Connects to the remote via SSH.
+        #   Does not use an rsync daemon on the remote.
+        #
+        # [:ssh_daemon]
+        #   Connects to the remote via SSH.
+        #   Spawns a single-use daemon on the remote, which allows certain
+        #   daemon features (like modules) to be used.
+        #
+        # [:rsync_daemon]
+        #   Connects directly to an rsync daemon via TCP.
+        #   Data transferred is not encrypted.
+        #
+        attr_accessor :mode
 
         ##
-        # Server IP Address and SSH port
-        attr_accessor :ip
+        # Server Address
+        attr_accessor :host
 
         ##
-        # The SSH port to connect to
+        # SSH or RSync port
+        #
+        # For `:ssh` or `:ssh_daemon` mode, this specifies the SSH port to use
+        # and defaults to 22.
+        #
+        # For `:rsync_daemon` mode, this specifies the TCP port to use
+        # and defaults to 873.
         attr_accessor :port
 
         ##
-        # Flag for compressing (only compresses for the transfer)
-        attr_accessor :compress
+        # SSH User
+        #
+        # If the user running the backup is not the same user that needs to
+        # authenticate with the remote server, specify the user here.
+        #
+        # The user must have SSH keys setup for passphrase-less access to the
+        # remote. If the SSH User does not have passphrase-less keys, or no
+        # default keys in their `~/.ssh` directory, you will need to use the
+        # `-i` option in `:additional_ssh_options` to specify the
+        # passphrase-less key to use.
+        #
+        # Used only for `:ssh` and `:ssh_daemon` modes.
+        attr_accessor :ssh_user
 
         ##
-        # Instantiates a new RSync::Push or RSync::Pull Syncer.
+        # Additional SSH Options
         #
-        # Pre-configured defaults specified in
-        # Configuration::Syncer::RSync::Push or
-        # Configuration::Syncer::RSync::Pull
-        # are set via a super() call to RSync::Base,
-        # which in turn will invoke Syncer::Base.
+        # Used to supply a String or Array of options to be passed to the SSH
+        # command in `:ssh` and `:ssh_daemon` modes.
         #
-        # Once pre-configured defaults and RSync specific defaults are set,
-        # the block from the user's configuration file is evaluated.
-        def initialize(&block)
-          super
+        # For example, if you need to supply a specific SSH key for the `ssh_user`,
+        # you would set this to: "-i '/path/to/id_rsa'". Which would produce:
+        #
+        #   rsync -e "ssh -p 22 -i '/path/to/id_rsa'"
+        #
+        # Arguments may be single-quoted, but should not contain any double-quotes.
+        #
+        # Used only for `:ssh` and `:ssh_daemon` modes.
+        attr_accessor :additional_ssh_options
 
-          @port               ||= 22
-          @compress           ||= false
+        ##
+        # RSync User
+        #
+        # If the user running the backup is not the same user that needs to
+        # authenticate with the rsync daemon, specify the user here.
+        #
+        # Used only for `:ssh_daemon` and `:rsync_daemon` modes.
+        attr_accessor :rsync_user
 
+        ##
+        # RSync Password
+        #
+        # If specified, Backup will write the password to a temporary file and
+        # use it with rsync's `--password-file` option for daemon authentication.
+        #
+        # Note that setting this will override `rsync_password_file`.
+        #
+        # Used only for `:ssh_daemon` and `:rsync_daemon` modes.
+        attr_accessor :rsync_password
+
+        ##
+        # RSync Password File
+        #
+        # If specified, this path will be passed to rsync's `--password-file`
+        # option for daemon authentication.
+        #
+        # Used only for `:ssh_daemon` and `:rsync_daemon` modes.
+        attr_accessor :rsync_password_file
+
+        ##
+        # Flag for compressing (only compresses for the transfer)
+        attr_accessor :compress
+
+        def initialize(syncer_id = nil, &block)
+          super
           instance_eval(&block) if block_given?
+
+          @mode ||= :ssh
+          @port ||= mode == :rsync_daemon ? 873 : 22
+          @compress ||= false
         end
 
-        ##
-        # Performs the RSync:Push operation
-        # debug options: -vhP
         def perform!
+          log!(:started)
           write_password_file!
 
-          Logger.info(
-            "#{ syncer_name } started syncing the following directories:\n\s\s" +
-            @directories.join("\n\s\s")
-          )
-          run("#{ utility(:rsync) } #{ options } #{ directories_option } " +
-              "'#{ username }@#{ ip }:#{ dest_path }'")
+          create_dest_path!
+          run("#{ rsync_command } #{ paths_to_push } " +
+              "#{ host_options }'#{ dest_path }'")
 
+          log!(:finished)
         ensure
           remove_password_file!
         end
@@ -61,55 +126,101 @@ module Backup
         private
 
         ##
-        # Return @path with any preceeding "~/" removed
+        # Remove any preceeding '~/', since this is on the remote,
+        # and remove any trailing `/`.
         def dest_path
-          @dest_path ||= @path.sub(/^\~\//, '')
+          @dest_path ||= path.sub(/^~\//, '').sub(/\/$/, '')
         end
 
         ##
-        # Returns all the specified Rsync::[Push/Pull] options,
-        # concatenated, ready for the CLI
-        def options
-          ([archive_option, mirror_option, compress_option, port_option,
-            password_option] + additional_options).compact.join("\s")
+        # Runs a 'mkdir -p' command on the remote to ensure the dest_path exists.
+        # This used because rsync will attempt to create the path, but will only
+        # call 'mkdir' without the '-p' option. This is only applicable in :ssh
+        # mode, and only used if the path would require this.
+        def create_dest_path!
+          return unless mode == :ssh && dest_path.index('/').to_i > 0
+
+          run "#{ utility(:ssh) } #{ ssh_transport_args } #{ host } " +
+                 %Q["mkdir -p '#{ dest_path }'"]
         end
 
         ##
-        # Returns Rsync syntax for compressing the file transfers
-        def compress_option
-          '--compress' if @compress
+        # For Push, this will prepend the #dest_path.
+        # For Pull, this will prepend the first path in #paths_to_pull.
+        def host_options
+          if mode == :ssh
+            "#{ host }:"
+          else
+            user = "#{ rsync_user }@" if rsync_user
+            "#{ user }#{ host }::"
+          end
         end
 
         ##
-        # Returns Rsync syntax for defining a port to connect to
-        def port_option
-          "-e 'ssh -p #{@port}'"
+        # Common base command, plus options for Push/Pull
+        def rsync_command
+          super << compress_option << password_option << transport_options
+        end
+
+        def compress_option
+          compress ? ' --compress' : ''
         end
 
-        ##
-        # Returns Rsync syntax for setting a password (via a file)
         def password_option
-          "--password-file='#{@password_file.path}'" if @password_file
+          return '' if mode == :ssh
+
+          path = @password_file ? @password_file.path : rsync_password_file
+          path ? " --password-file='#{ File.expand_path(path) }'" : ''
         end
 
-        ##
-        # Writes the provided password to a temporary file so that
-        # the rsync utility can read the password from this file
-        def write_password_file!
-          unless @password.nil?
-            @password_file = Tempfile.new('backup-rsync-password')
-            @password_file.write(@password)
-            @password_file.close
+        def transport_options
+          if mode == :rsync_daemon
+            " --port #{ port }"
+          else
+            %Q[ -e "#{ utility(:ssh) } #{ ssh_transport_args }"]
           end
         end
 
-        ##
-        # Removes the previously created @password_file
-        # (temporary file containing the password)
+        def ssh_transport_args
+          args = "-p #{ port } "
+          args << "-l #{ ssh_user } " if ssh_user
+          args << Array(additional_ssh_options).join(' ')
+          args.rstrip
+        end
+
+        def write_password_file!
+          return unless rsync_password && mode != :ssh
+
+          @password_file = Tempfile.new('backup-rsync-password')
+          @password_file.write(rsync_password)
+          @password_file.close
+        end
+
         def remove_password_file!
           @password_file.delete if @password_file
-          @password_file = nil
         end
+
+        attr_deprecate :additional_options, :version => '3.2.0',
+                       :message => 'Use #additional_rsync_options instead.',
+                       :action => lambda {|klass, val|
+                         klass.additional_rsync_options = val
+                       }
+
+        attr_deprecate :username, :version => '3.2.0',
+                       :message => 'Use #ssh_user instead.',
+                       :action => lambda {|klass, val|
+                         klass.ssh_user = val
+                       }
+        attr_deprecate :password, :version => '3.2.0',
+                       :message => 'Use #rsync_password instead.',
+                       :action => lambda {|klass, val|
+                         klass.rsync_password = val
+                       }
+        attr_deprecate :ip, :version => '3.2.0',
+                       :message => 'Use #host instead.',
+                       :action => lambda {|klass, val|
+                         klass.host = val
+                       }
       end
     end
   end