backup 3.7.2 → 3.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6cc0cefe15e7fb9dc06e2204ce90aad256ea474a
-  data.tar.gz: 073692acde24bfe0dc958334a3c3ecfb955c62a2
+  metadata.gz: 697c751e37f7ee0a9c4485ba7c793980a7a5b087
+  data.tar.gz: 2bc9a792c855ed04d7d8e32a95612b7510836a93
 SHA512:
-  metadata.gz: 09747bffc3eae9f9769c5d3b1d86a198ce5d6560ad34877b0f7b491fb514844751e8ce5e3b8c2b5a69786ca0904e09f1f94023ed4bf3739429a10297d9c43398
-  data.tar.gz: 0ae326031c81d8c3ad905427463018bccdee68bade8dc8c7a5398ff241fd6572dd428cba04ed47f0d09f280c8dc390c16922fafb777fd01378952ee0a3925af3
+  metadata.gz: 66fee3c688e30e8c0dca0083d3b65b989578f9cdb42b6d043e6d6880e129bfa2ee08b220ee1b778322f02d52b58ad119a33a46458ced5f4e24663122963f9a1e
+  data.tar.gz: 44d7d2382a074f6f5e83df6b360d26137c6bc2733031cb2b22ce0f5ce1d83b97712366254f248ceb6681aaf5f8d42fcd138484b670bb52acab51d5fdbfe8026b

data/README.md

@@ -8,12 +8,12 @@ was built with modularity, extensibility and simplicity in mind.
 
 ## Installation
 
+**_Do not add `gem backup` to an application's `Gemfile`_**
+
 To install the latest version, run:
 
     $ [sudo] gem install backup
 
-_Do not add `gem backup` to an application's `Gemfile`_
-
 See [Installation](https://github.com/meskyanichi/backup/wiki/Installation) for more information about installing and
 updating your installation of Backup.
 

data/lib/backup/archive.rb

@@ -71,21 +71,23 @@ module Backup
       FileUtils.mkdir_p(path)
 
       pipeline = Pipeline.new
-      pipeline.add(
-        "#{ tar_command } #{ tar_options } -cPf -#{ tar_root } " +
-        "#{ paths_to_exclude } #{ paths_to_package }",
-        tar_success_codes
-      )
-
-      extension = 'tar'
-      @model.compressor.compress_with do |command, ext|
-        pipeline << command
-        extension << ext
-      end if @model.compressor
-
-      pipeline << "#{ utility(:cat) } > " +
-          "'#{ File.join(path, "#{ name }.#{ extension }") }'"
-      pipeline.run
+      with_files_from(paths_to_package) do |files_from|
+        pipeline.add(
+          "#{ tar_command } #{ tar_options } -cPf -#{ tar_root } " +
+          "#{ paths_to_exclude } #{ files_from }",
+          tar_success_codes
+        )
+
+        extension = 'tar'
+        @model.compressor.compress_with do |command, ext|
+          pipeline << command
+          extension << ext
+        end if @model.compressor
+
+        pipeline << "#{ utility(:cat) } > " +
+            "'#{ File.join(path, "#{ name }.#{ extension }") }'"
+        pipeline.run
+      end
 
       if pipeline.success?
         Logger.info "Archive '#{ name }' Complete!"
@@ -107,9 +109,16 @@ module Backup
     end
 
     def paths_to_package
-      options[:paths].map {|path|
-        "'#{ prepare_path(path) }'"
-      }.join(' ')
+      options[:paths].map {|path| prepare_path(path) }
+    end
+
+    def with_files_from(paths)
+      tmpfile = Tempfile.new('backup-archive-paths')
+      paths.each {|path| tmpfile.puts path }
+      tmpfile.close
+      yield "-T '#{ tmpfile.path }'"
+    ensure
+      tmpfile.delete
     end
 
     def paths_to_exclude

data/lib/backup/cloud_io/s3.rb

@@ -13,14 +13,15 @@ module Backup
       MAX_FILE_SIZE       = 1024**3 * 5   # 5 GiB
       MAX_MULTIPART_SIZE  = 1024**4 * 5   # 5 TiB
 
-      attr_reader :access_key_id, :secret_access_key, :region, :bucket,
-                  :chunk_size, :encryption, :storage_class
+      attr_reader :access_key_id, :secret_access_key, :use_iam_profile,
+                  :region, :bucket, :chunk_size, :encryption, :storage_class
 
       def initialize(options = {})
         super
 
         @access_key_id      = options[:access_key_id]
         @secret_access_key  = options[:secret_access_key]
+        @use_iam_profile    = options[:use_iam_profile]
         @region             = options[:region]
         @bucket             = options[:bucket]
         @chunk_size         = options[:chunk_size]
@@ -120,12 +121,16 @@ module Backup
 
       def connection
         @connection ||= begin
-          conn = Fog::Storage.new(
-            :provider               => 'AWS',
-            :aws_access_key_id      => access_key_id,
-            :aws_secret_access_key  => secret_access_key,
-            :region                 => region
-          )
+          opts = { :provider => 'AWS', :region => region }
+          if use_iam_profile
+            opts.merge!(:use_iam_profile => true)
+          else
+            opts.merge!(
+              :aws_access_key_id      => access_key_id,
+              :aws_secret_access_key  => secret_access_key
+            )
+          end
+          conn = Fog::Storage.new(opts)
           conn.sync_clock
           conn
         end

data/lib/backup/logger/logfile.rb

@@ -3,6 +3,8 @@
 module Backup
   class Logger
     class Logfile
+      class Error < Backup::Error; end
+
       class Options
         ##
         # Enable the use of Backup's log file.
@@ -87,7 +89,11 @@ module Backup
           path = File.join(Backup::Config.root_path, path)
         end
         FileUtils.mkdir_p(path)
-        File.join(path, 'backup.log')
+        path = File.join(path, 'backup.log')
+        if File.exist?(path) && !File.writable?(path)
+          raise Error, "Log File at '#{ path }' is not writable"
+        end
+        path
       end
 
       ##
@@ -96,7 +102,7 @@ module Backup
         return unless File.exist?(@logfile)
 
         if File.stat(@logfile).size > @options.max_bytes
-          FileUtils.mv(@logfile, @logfile + '~')
+          FileUtils.cp(@logfile, @logfile + '~')
           File.open(@logfile + '~', 'r') do |io_in|
             File.open(@logfile, 'w') do |io_out|
               io_in.seek(-@options.max_bytes, IO::SEEK_END) && io_in.gets

data/lib/backup/storage/s3.rb

@@ -8,7 +8,7 @@ module Backup
 
       ##
       # Amazon Simple Storage Service (S3) Credentials
-      attr_accessor :access_key_id, :secret_access_key
+      attr_accessor :access_key_id, :secret_access_key, :use_iam_profile
 
       ##
       # Amazon S3 bucket name
@@ -81,6 +81,7 @@ module Backup
         @cloud_io ||= CloudIO::S3.new(
           :access_key_id      => access_key_id,
           :secret_access_key  => secret_access_key,
+          :use_iam_profile    => use_iam_profile,
           :region             => region,
           :bucket             => bucket,
           :encryption         => encryption,
@@ -114,7 +115,11 @@ module Backup
       end
 
       def check_configuration
-        required = %w{ access_key_id secret_access_key bucket }
+        if use_iam_profile
+          required = %w{ bucket }
+        else
+          required = %w{ access_key_id secret_access_key bucket }
+        end
         raise Error, <<-EOS if required.map {|name| send(name) }.any?(&:nil?)
           Configuration Error
           #{ required.map {|name| "##{ name }"}.join(', ') } are all required

data/lib/backup/storage/sftp.rb

@@ -7,7 +7,7 @@ module Backup
 
       ##
       # Server credentials
-      attr_accessor :username, :password
+      attr_accessor :username, :password, :ssh_options
 
       ##
       # Server IP Address and SFTP port
@@ -16,8 +16,9 @@ module Backup
       def initialize(model, storage_id = nil)
         super
 
-        @port ||= 22
-        @path ||= 'backups'
+        @ssh_options ||= {}
+        @port        ||= 22
+        @path        ||= 'backups'
         path.sub!(/^~\//, '')
       end
 
@@ -25,9 +26,7 @@ module Backup
 
       def connection
         Net::SFTP.start(
-          ip, username,
-          :password => password,
-          :port     => port
+          ip, username, { :password => password, :port => port }.merge(ssh_options)
         ) {|sftp| yield sftp }
       end
 

data/lib/backup/syncer/base.rb

@@ -20,13 +20,16 @@ module Backup
       # in the log messages.
       attr_reader :syncer_id
 
+      attr_reader :excludes
+
       def initialize(syncer_id = nil)
         @syncer_id = syncer_id
 
         load_defaults!
 
         @mirror ||= false
-        @directories = Array.new
+        @directories = []
+        @excludes = []
       end
 
       ##
@@ -40,6 +43,13 @@ module Backup
         directories << path
       end
 
+      # For Cloud Syncers, +pattern+ can be a string (with shell-style
+      # wildcards) or a regex.
+      # For RSync, each +pattern+ will be passed to rsync's --exclude option.
+      def exclude(pattern)
+        excludes << pattern
+      end
+
       private
 
       def syncer_name

data/lib/backup/syncer/cloud/base.rb

@@ -64,7 +64,7 @@ module Backup
           remote_files = get_remote_files(remote_base)
 
           Logger.info("Gathering local data for '#{ File.expand_path(dir) }'...")
-          local_files = LocalFile.find(dir)
+          local_files = LocalFile.find(dir, excludes)
 
           relative_paths = (local_files.keys | remote_files.keys).sort
           if relative_paths.empty?

data/lib/backup/syncer/cloud/local_file.rb

@@ -1,22 +1,23 @@
 # encoding: utf-8
+require 'digest/md5'
 
 module Backup
   module Syncer
     module Cloud
       class LocalFile
-        attr_reader :path, :md5
+        attr_reader :path
+        attr_accessor :md5
 
         class << self
-          include Utilities::Helpers
 
-          # Returns a Hash of LocalFile objects for each file within +dir+.
+          # Returns a Hash of LocalFile objects for each file within +dir+,
+          # except those matching any of the +excludes+.
           # Hash keys are the file's path relative to +dir+.
-          def find(dir)
+          def find(dir, excludes = [])
             dir = File.expand_path(dir)
             hash = {}
-            find_md5(dir).each do |path, md5|
-              file = new(path, md5)
-              hash[path.sub(dir + '/', '')] = file if file
+            find_md5(dir, excludes).each do |file|
+              hash[file.path.sub(dir + '/', '')] = file
             end
             hash
           end
@@ -36,37 +37,43 @@ module Backup
           private
 
           # Returns an Array of file paths and their md5 hashes.
-          #
-          # Lines output from `cmd` are formatted like:
-          #   MD5(/dir/subdir/file)= 7eaabd1f53024270347800d0fdb34357
-          # However, if +dir+ is empty, the following is returned:
-          #   (stdin)= d41d8cd98f00b204e9800998ecf8427e
-          # Which extracts as: ['in', 'd41d8cd98f00b204e9800998ecf8427e']
-          # I'm not sure I can rely on the fact this doesn't begin with 'MD5',
-          # so I'll reject entries with a path that doesn't start with +dir+.
-          #
-          # String#slice avoids `invalid byte sequence in UTF-8` errors
-          # that String#split would raise.
-          #
-          # Utilities#run is not used here because this would produce too much
-          # log output, and Pipeline does not support capturing output.
-          def find_md5(dir)
-            cmd = "#{ utility(:find) } -L '#{ dir }' -type f -print0 | " +
-                  "#{ utility(:xargs) } -0 #{ utility(:openssl) } md5 2> /dev/null"
-            %x[#{ cmd }].lines.map do |line|
-              line.chomp!
-              entry = [line.slice(4..-36), line.slice(-32..-1)]
-              entry[0].to_s.start_with?(dir) ? entry : nil
-            end.compact
+          def find_md5(dir, excludes)
+            found = []
+            (Dir.entries(dir) - %w{. ..}).map {|e| File.join(dir, e) }.each do |path|
+              if File.directory?(path)
+                unless exclude?(excludes, path)
+                  found += find_md5(path, excludes)
+                end
+              elsif File.file?(path)
+                if file = new(path)
+                  unless exclude?(excludes, file.path)
+                    file.md5 = Digest::MD5.file(file.path).hexdigest
+                    found << file
+                  end
+                end
+              end
+            end
+            found
+          end
+
+          # Returns true if +path+ matches any of the +excludes+.
+          # Note this can not be called if +path+ includes invalid UTF-8.
+          def exclude?(excludes, path)
+            excludes.any? do |ex|
+              if ex.is_a?(String)
+                File.fnmatch?(ex, path)
+              elsif ex.is_a?(Regexp)
+                ex.match(path)
+              end
+            end
           end
         end
 
         # If +path+ contains invalid UTF-8, it will be sanitized
         # and the LocalFile object will be flagged as invalid.
         # This is done so @file.path may be logged.
-        def initialize(path, md5)
+        def initialize(path)
           @path = sanitize(path)
-          @md5 = md5
         end
 
         def invalid?

data/lib/backup/syncer/cloud/s3.rb

@@ -9,7 +9,7 @@ module Backup
 
         ##
         # Amazon Simple Storage Service (S3) Credentials
-        attr_accessor :access_key_id, :secret_access_key
+        attr_accessor :access_key_id, :secret_access_key, :use_iam_profile
 
         ##
         # Amazon S3 bucket name
@@ -54,6 +54,7 @@ module Backup
           @cloud_io ||= CloudIO::S3.new(
             :access_key_id      => access_key_id,
             :secret_access_key  => secret_access_key,
+            :use_iam_profile    => use_iam_profile,
             :bucket             => bucket,
             :region             => region,
             :encryption         => encryption,
@@ -75,7 +76,11 @@ module Backup
         end
 
         def check_configuration
-          required = %w{ access_key_id secret_access_key bucket }
+          if use_iam_profile
+            required = %w{ bucket }
+          else
+            required = %w{ access_key_id secret_access_key bucket }
+          end
           raise Error, <<-EOS if required.map {|name| send(name) }.any?(&:nil?)
             Configuration Error
             #{ required.map {|name| "##{ name }"}.join(', ') } are all required

data/lib/backup/syncer/rsync/base.rb

@@ -21,7 +21,7 @@ module Backup
         ##
         # Common base command for Local/Push/Pull
         def rsync_command
-          utility(:rsync) << ' --archive' << mirror_option <<
+          utility(:rsync) << ' --archive' << mirror_option << exclude_option <<
               " #{ Array(additional_rsync_options).join(' ') }".rstrip
         end
 
@@ -29,6 +29,10 @@ module Backup
           mirror ? ' --delete' : ''
         end
 
+        def exclude_option
+          excludes.map {|pattern| " --exclude='#{ pattern }'" }.join
+        end
+
         ##
         # Each path is expanded, since these refer to local paths and are
         # being shell-quoted. This will also remove any trailing `/` from

data/lib/backup/utilities.rb

@@ -6,7 +6,7 @@ module Backup
 
     UTILITY = {}
     NAMES = %w{
-      tar cat split find xargs sudo chown hostname
+      tar cat split sudo chown hostname
       gzip bzip2 lzma pbzip2
       mongo mongodump mysqldump pg_dump pg_dumpall redis-cli riak-admin
       gpg openssl
@@ -65,8 +65,6 @@ module Backup
       #     tar_dist :gnu   # or :bsd
       #     cat      '/path/to/cat'
       #     split    '/path/to/split'
-      #     find     '/path/to/find'
-      #     xargs    '/path/to/xargs'
       #     sudo     '/path/to/sudo'
       #     chown    '/path/to/chown'
       #     hostname '/path/to/hostname'

data/lib/backup/version.rb

@@ -1,5 +1,5 @@
 # encoding: utf-8
 
 module Backup
-  VERSION = '3.7.2'
+  VERSION = '3.8.0'
 end

data/templates/cli/storage/s3

@@ -4,8 +4,12 @@
   # See the documentation on the Wiki for details.
   # https://github.com/meskyanichi/backup/wiki/Storages
   store_with S3 do |s3|
+    # AWS Credentials
     s3.access_key_id     = "my_access_key_id"
     s3.secret_access_key = "my_secret_access_key"
+    # Or, to use a IAM Profile:
+    # s3.use_iam_profile = true
+
     s3.region            = "us-east-1"
     s3.bucket            = "bucket-name"
     s3.path              = "path/to/backups"

data/templates/cli/storage/scp

@@ -8,4 +8,7 @@
     server.port       = 22
     server.path       = "~/backups/"
     server.keep       = 5
+
+    # Additional options for the SSH connection.
+    # server.ssh_options = {}
   end

data/templates/cli/storage/sftp

@@ -8,4 +8,7 @@
     server.port       = 22
     server.path       = "~/backups/"
     server.keep       = 5
+
+    # Additional options for the SSH connection.
+    # server.ssh_options = {}
   end

data/templates/cli/syncer/cloud_files

@@ -14,5 +14,11 @@
     cf.directories do |directory|
       directory.add "/path/to/directory/to/sync"
       directory.add "/path/to/other/directory/to/sync"
+
+      # Exclude files/folders from the sync.
+      # The pattern may be a shell glob pattern (see `File.fnmatch`) or a Regexp.
+      # All patterns will be applied when traversing each added directory.
+      directory.exclude '**/*~'
+      directory.exclude /\/tmp$/
     end
   end

data/templates/cli/syncer/rsync_local

@@ -8,5 +8,13 @@
     rsync.directories do |directory|
       directory.add "/var/apps/my_app/public/uploads"
       directory.add "/var/apps/my_app/logs"
+
+      # Exclude files/folders.
+      # Each pattern will be passed to rsync's `--exclude` option.
+      #
+      # Note: rsync is run using the `--archive` option,
+      #       so be sure to read the `FILTER RULES` in `man rsync`.
+      directory.exclude '*~'
+      directory.exclude 'tmp/'
     end
   end

data/templates/cli/syncer/rsync_pull

@@ -18,5 +18,13 @@
     rsync.directories do |directory|
       directory.add "/var/apps/my_app/public/uploads"
       directory.add "/var/apps/my_app/logs"
+
+      # Exclude files/folders.
+      # Each pattern will be passed to rsync's `--exclude` option.
+      #
+      # Note: rsync is run using the `--archive` option,
+      #       so be sure to read the `FILTER RULES` in `man rsync`.
+      directory.exclude '*~'
+      directory.exclude 'tmp/'
     end
   end

data/templates/cli/syncer/rsync_push

@@ -18,5 +18,13 @@
     rsync.directories do |directory|
       directory.add "/var/apps/my_app/public/uploads"
       directory.add "/var/apps/my_app/logs"
+
+      # Exclude files/folders.
+      # Each pattern will be passed to rsync's `--exclude` option.
+      #
+      # Note: rsync is run using the `--archive` option,
+      #       so be sure to read the `FILTER RULES` in `man rsync`.
+      directory.exclude '*~'
+      directory.exclude 'tmp/'
     end
   end

data/templates/cli/syncer/s3

@@ -4,8 +4,12 @@
   # See the documentation on the Wiki for details.
   # https://github.com/meskyanichi/backup/wiki/Syncers
   sync_with Cloud::S3 do |s3|
+    # AWS Credentials
     s3.access_key_id     = "my_access_key_id"
     s3.secret_access_key = "my_secret_access_key"
+    # Or, to use a IAM Profile:
+    # s3.use_iam_profile = true
+
     s3.bucket            = "my-bucket"
     s3.region            = "us-east-1"
     s3.path              = "/backups"
@@ -15,5 +19,11 @@
     s3.directories do |directory|
       directory.add "/path/to/directory/to/sync"
       directory.add "/path/to/other/directory/to/sync"
+
+      # Exclude files/folders from the sync.
+      # The pattern may be a shell glob pattern (see `File.fnmatch`) or a Regexp.
+      # All patterns will be applied when traversing each added directory.
+      directory.exclude '**/*~'
+      directory.exclude /\/tmp$/
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: backup
 version: !ruby/object:Gem::Version
-  version: 3.7.2
+  version: 3.8.0
 platform: ruby
 authors:
 - Michael van Rooijen
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-09-28 00:00:00.000000000 Z
+date: 2013-10-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: builder