backup 4.4.1 → 5.0.0.beta.1

data/lib/backup/cli.rb

@@ -1,5 +1,3 @@
-# encoding: utf-8
-
 ##
 # Build the Backup Command Line Interface using Thor
 module Backup
@@ -27,9 +25,9 @@ module Backup
     #
     # If the --check option is given, `backup check` will be run
     # and no triggers will be performed.
-    desc 'perform', "Performs the backup for the specified trigger(s)."
+    desc "perform", "Performs the backup for the specified trigger(s)."
 
-    long_desc <<-EOS.gsub(/^ +/, '')
+    long_desc <<-EOS.gsub(/^ +/, "")
       Performs the backup for the specified trigger(s).
 
       You may perform multiple backups by providing multiple triggers,
@@ -51,65 +49,65 @@ module Backup
     EOS
 
     method_option :trigger,
-                  :aliases  => ['-t', '--triggers'],
-                  :required => true,
-                  :type     => :string,
-                  :desc     => "Triggers to perform. e.g. 'trigger_a,trigger_b'"
+      aliases: ["-t", "--triggers"],
+      required: true,
+      type: :string,
+      desc: "Triggers to perform. e.g. 'trigger_a,trigger_b'"
 
     method_option :config_file,
-                  :aliases  => '-c',
-                  :type     => :string,
-                  :default  => '',
-                  :desc     => 'Path to your config.rb file.'
+      aliases: "-c",
+      type: :string,
+      default: "",
+      desc: "Path to your config.rb file."
 
     method_option :root_path,
-                  :aliases  => '-r',
-                  :type     => :string,
-                  :default  => '',
-                  :desc     => 'Root path to base all relative path on.'
+      aliases: "-r",
+      type: :string,
+      default: "",
+      desc: "Root path to base all relative path on."
 
     method_option :data_path,
-                  :aliases  => '-d',
-                  :type     => :string,
-                  :default  => '',
-                  :desc     => 'Path to store storage cycling data.'
+      aliases: "-d",
+      type: :string,
+      default: "",
+      desc: "Path to store storage cycling data."
 
     method_option :log_path,
-                  :aliases  => '-l',
-                  :type     => :string,
-                  :default  => '',
-                  :desc     => "Path to store Backup's log file."
+      aliases: "-l",
+      type: :string,
+      default: "",
+      desc: "Path to store Backup's log file."
 
     method_option :tmp_path,
-                  :type     => :string,
-                  :default  => '',
-                  :desc     => 'Path to store temporary data during the backup.'
+      type: :string,
+      default: "",
+      desc: "Path to store temporary data during the backup."
 
     # Note that :quiet, :syslog and :logfile are specified as :string types,
     # so the --no-<option> usage will set the value to nil instead of false.
     method_option :quiet,
-                  :aliases  => '-q',
-                  :type     => :string,
-                  :default  => false,
-                  :banner   => '',
-                  :desc     => 'Disable console log output.'
+      aliases: "-q",
+      type: :boolean,
+      default: false,
+      banner: "",
+      desc: "Disable console log output."
 
     method_option :syslog,
-                  :type     => :string,
-                  :default  => false,
-                  :banner   => '',
-                  :desc     => 'Enable logging to syslog.'
+      type: :boolean,
+      default: false,
+      banner: "",
+      desc: "Enable logging to syslog."
 
     method_option :logfile,
-                  :type     => :string,
-                  :default  => true,
-                  :banner   => '',
-                  :desc     => "Enable Backup's log file."
+      type: :boolean,
+      default: true,
+      banner: "",
+      desc: "Enable Backup's log file."
 
     method_option :check,
-                  :type     => :boolean,
-                  :default  => false,
-                  :desc     => 'Check configuration for errors or warnings.'
+      type: :boolean,
+      default: false,
+      desc: "Check configuration for errors or warnings."
 
     def perform
       check if options[:check] # this will exit()
@@ -119,27 +117,28 @@ module Backup
         # Set logger options
         opts = options
         Logger.configure do
-          console.quiet     = opts[:quiet]
-          logfile.enabled   = opts[:logfile]
-          logfile.log_path  = opts[:log_path]
-          syslog.enabled    = opts[:syslog]
+          console.quiet    = opts[:quiet]
+          logfile.enabled  = opts[:logfile]
+          logfile.log_path = opts[:log_path]
+          syslog.enabled   = opts[:syslog]
         end
 
         # Load the user's +config.rb+ file and all their Models
         Config.load(options)
 
         # Identify all Models to be run for the given +triggers+.
-        triggers = options[:trigger].split(',').map(&:strip)
-        models = triggers.map {|trigger|
+        triggers = options[:trigger].split(",").map(&:strip)
+        models = triggers.uniq.flat_map do |trigger|
           Model.find_by_trigger(trigger)
-        }.flatten.uniq
+        end
 
-        raise Error, "No Models found for trigger(s) " +
-            "'#{ triggers.join(',') }'." if models.empty?
+        if models.empty?
+          raise Error, "No Models found for trigger(s) " \
+            "'#{triggers.join(",")}'."
+        end
 
         # Finalize Logger and begin real-time logging.
         Logger.start!
-
       rescue Exception => err
         Logger.error Error.wrap(err)
         unless Helpers.is_backup_error? err
@@ -148,7 +147,7 @@ module Backup
         # Logger configuration will be ignored
         # and messages will be output to the console only.
         Logger.abort!
-        exit(3)
+        exit 3
       end
 
       until models.empty?
@@ -164,7 +163,7 @@ module Backup
             Logger.info Error.new(<<-EOS)
               Backup will now continue...
               The following triggers will now be processed:
-              (#{ models.map {|m| m.trigger }.join(', ') })
+              (#{models.map(&:trigger).join(", ")})
             EOS
           end
         when 3
@@ -173,7 +172,7 @@ module Backup
             Logger.error FatalError.new(<<-EOS)
               Backup will now exit.
               The following triggers will not be processed:
-              (#{ models.map {|m| m.trigger }.join(', ') })
+              (#{models.map(&:trigger).join(", ")})
             EOS
           end
         end
@@ -203,19 +202,19 @@ module Backup
     #
     # If successful, this method with exit(0).
     # If there are Errors or Warnings, it will exit(1).
-    desc 'check', 'Check for configuration errors or warnings'
+    desc "check", "Check for configuration errors or warnings"
 
-    long_desc <<-EOS.gsub(/^ +/, '')
+    long_desc <<-EOS.gsub(/^ +/, "")
       Loads your 'config.rb' file and all models and reports any
       errors or warnings with your configuration, including missing
       dependencies and the use of any deprecated settings.
     EOS
 
     method_option :config_file,
-                  :aliases  => '-c',
-                  :type     => :string,
-                  :default  => '',
-                  :desc     => "Path to your config.rb file."
+      aliases: "-c",
+      type: :string,
+      default: "",
+      desc: "Path to your config.rb file."
 
     def check
       begin
@@ -228,10 +227,10 @@ module Backup
       end
 
       if Logger.has_warnings? || Logger.has_errors?
-        Logger.error 'Configuration Check Failed.'
+        Logger.error "Configuration Check Failed."
         exit_code = 1
       else
-        Logger.info 'Configuration Check Succeeded.'
+        Logger.info "Configuration Check Succeeded."
         exit_code = 0
       end
 
@@ -245,130 +244,127 @@ module Backup
     # For example:
     #   $ backup generate:model --trigger my_backup --databases='mongodb'
     # will generate a pre-populated model with a base MongoDB setup
-    desc 'generate:model', "Generates a Backup model file."
+    desc "generate:model", "Generates a Backup model file."
 
-    long_desc <<-EOS.gsub(/^ +/, '')
+    long_desc <<-EOS.gsub(/^ +/, "")
       Generates a Backup model file.
 
       If your configuration file is not in the default location at
-      #{ Config.config_file }
+      #{Config.config_file}
       you must specify it's location using '--config-file'.
       If no configuration file exists at this location, one will be created.
 
       The model file will be created as '<config_path>/models/<trigger>.rb'
       Your model file will be created in a 'models/' sub-directory
       where your config file is located. The default location would be:
-      #{ Config.root_path }/models/<trigger>.rb
+      #{Config.root_path}/models/<trigger>.rb
     EOS
 
     method_option :trigger,
-                  :aliases  => '-t',
-                  :required => true,
-                  :type     => :string,
-                  :desc     => 'Trigger name for the Backup model'
+      aliases: "-t",
+      required: true,
+      type: :string,
+      desc: "Trigger name for the Backup model"
 
     method_option :config_file,
-                  :type     => :string,
-                  :desc     => 'Path to your Backup configuration file'
+      type: :string,
+      desc: "Path to your Backup configuration file"
 
     # options with their available values
-    %w{ databases storages syncers encryptor compressor notifiers }.each do |name|
-      path = File.join(Backup::TEMPLATE_PATH, 'cli', name)
-      opts = Dir[path + '/*'].sort.map {|p| File.basename(p) }.join(', ')
-      method_option name, :type => :string, :desc => "(#{ opts })"
+    %w[databases storages syncers encryptor compressor notifiers].each do |name|
+      path = File.join(Backup::TEMPLATE_PATH, "cli", name)
+      opts = Dir[path + "/*"].sort.map { |p| File.basename(p) }.join(", ")
+      method_option name, type: :string, desc: "(#{opts})"
     end
 
     method_option :archives,
-                  :type     => :boolean,
-                  :desc     => 'Model will include tar archives.'
+      type: :boolean,
+      desc: "Model will include tar archives."
 
     method_option :splitter,
-                  :type     => :boolean,
-                  :default  => false,
-                  :desc     => 'Add Splitter to the model'
+      type: :boolean,
+      default: false,
+      desc: "Add Splitter to the model"
 
-    define_method 'generate:model' do
-      opts = options.merge(:trigger => options[:trigger].gsub(/\W/, '_'))
+    define_method "generate:model" do
+      opts = options.merge(trigger: options[:trigger].gsub(/\W/, "_"))
       config_file = opts[:config_file] ?
                     File.expand_path(opts.delete(:config_file)) : Config.config_file
-      models_path = File.join(File.dirname(config_file), 'models')
-      model_file  = File.join(models_path, "#{ opts[:trigger] }.rb")
+      models_path = File.join(File.dirname(config_file), "models")
+      model_file  = File.join(models_path, "#{opts[:trigger]}.rb")
 
       unless File.exist?(config_file)
-        invoke 'generate:config', [], :config_file => config_file
+        invoke "generate:config", [], config_file: config_file
       end
 
       FileUtils.mkdir_p(models_path)
       if Helpers.overwrite?(model_file)
-        File.open(model_file, 'w') do |file|
-          file.write(Backup::Template.new({:options => opts}).result('cli/model'))
+        File.open(model_file, "w") do |file|
+          file.write(Backup::Template.new(options: opts).result("cli/model"))
         end
-        puts "Generated model file: '#{ model_file }'."
+        puts "Generated model file: '#{model_file}'."
       end
     end
 
     ##
     # [Generate:Config]
     # Generates the main configuration file
-    desc 'generate:config', 'Generates the main Backup configuration file'
+    desc "generate:config", "Generates the main Backup configuration file"
 
-    long_desc <<-EOS.gsub(/^ +/, '')
+    long_desc <<-EOS.gsub(/^ +/, "")
       Path to the Backup configuration file to generate.
 
       Defaults to:
 
-      #{ Config.config_file }
+      #{Config.config_file}
     EOS
 
     method_option :config_file,
-                  :type => :string,
-                  :desc => 'Path to the Backup configuration file to generate.'
+      type: :string,
+      desc: "Path to the Backup configuration file to generate."
 
-    define_method 'generate:config' do
+    define_method "generate:config" do
       config_file = options[:config_file] ?
           File.expand_path(options[:config_file]) : Config.config_file
 
       FileUtils.mkdir_p(File.dirname(config_file))
       if Helpers.overwrite?(config_file)
-        File.open(config_file, 'w') do |file|
-          file.write(Backup::Template.new.result('cli/config'))
+        File.open(config_file, "w") do |file|
+          file.write(Backup::Template.new.result("cli/config"))
         end
-        puts "Generated configuration file: '#{ config_file }'."
+        puts "Generated configuration file: '#{config_file}'."
       end
     end
 
     ##
     # [Version]
     # Returns the current version of the Backup gem
-    map '-v' => :version
-    desc 'version', 'Display installed Backup version'
+    map "-v" => :version
+    desc "version", "Display installed Backup version"
     def version
-      puts "Backup #{ Backup::VERSION }"
+      puts "Backup #{Backup::VERSION}"
     end
 
     # This is to avoid Thor's warnings when stubbing methods on the Thor class.
     module Helpers
       class << self
-
         def overwrite?(path)
           return true unless File.exist?(path)
 
-          $stderr.print "A file already exists at '#{ path }'.\n" +
+          $stderr.print "A file already exists at '#{path}'.\n" \
                         "Do you want to overwrite? [y/n] "
           /^[Yy]/ =~ $stdin.gets
         end
 
         def exec!(cmd)
-          puts "Launching: #{ cmd }"
+          puts "Launching: #{cmd}"
           exec(cmd)
         end
 
         def is_backup_error?(error)
           error.class.ancestors.include? Backup::Error
         end
-
       end
     end
-
   end
 end

data/lib/backup/cloud_io/base.rb

@@ -1,5 +1,3 @@
-# encoding: utf-8
-
 module Backup
   module CloudIO
     class Error < Backup::Error; end
@@ -22,20 +20,19 @@ module Backup
         rescue => err
           retries += 1
           raise Error.wrap(err, <<-EOS) if retries > max_retries
-            Max Retries (#{ max_retries }) Exceeded!
-            Operation: #{ operation }
+            Max Retries (#{max_retries}) Exceeded!
+            Operation: #{operation}
             Be sure to check the log messages for each retry attempt.
           EOS
 
           Logger.info Error.wrap(err, <<-EOS)
-            Retry ##{ retries } of #{ max_retries }
-            Operation: #{ operation }
+            Retry ##{retries} of #{max_retries}
+            Operation: #{operation}
           EOS
           sleep(retry_waitsec)
           retry
         end
       end
-
     end
   end
 end

data/lib/backup/cloud_io/cloud_files.rb

@@ -1,7 +1,6 @@
-# encoding: utf-8
-require 'backup/cloud_io/base'
-require 'fog'
-require 'digest/md5'
+require "backup/cloud_io/base"
+require "fog"
+require "digest/md5"
 
 module Backup
   module CloudIO
@@ -13,8 +12,8 @@ module Backup
       SEGMENT_BUFFER  = 1024**2         # 1 MiB
 
       attr_reader :username, :api_key, :auth_url, :region, :servicenet,
-                  :container, :segments_container, :segment_size, :days_to_keep,
-                  :fog_options
+        :container, :segments_container, :segment_size, :days_to_keep,
+        :fog_options
 
       def initialize(options = {})
         super
@@ -41,9 +40,9 @@ module Backup
         if segment_bytes > 0 && file_size > segment_bytes
           raise FileSizeError, <<-EOS if file_size > MAX_SLO_SIZE
             File Too Large
-            File: #{ src }
-            Size: #{ file_size }
-            Max SLO Size is #{ MAX_SLO_SIZE } (5 GiB * 1000 segments)
+            File: #{src}
+            Size: #{file_size}
+            Max SLO Size is #{MAX_SLO_SIZE} (5 GiB * 1000 segments)
           EOS
 
           segment_bytes = adjusted_segment_bytes(segment_bytes, file_size)
@@ -52,9 +51,9 @@ module Backup
         else
           raise FileSizeError, <<-EOS if file_size > MAX_FILE_SIZE
             File Too Large
-            File: #{ src }
-            Size: #{ file_size }
-            Max File Size is #{ MAX_FILE_SIZE } (5 GiB)
+            File: #{src}
+            Size: #{file_size}
+            Max File Size is #{MAX_FILE_SIZE} (5 GiB)
           EOS
 
           put_object(src, dest)
@@ -69,14 +68,14 @@ module Backup
       def objects(prefix)
         objects = []
         resp = nil
-        prefix = prefix.chomp('/')
-        opts = { :prefix => prefix + '/' }
+        prefix = prefix.chomp("/")
+        opts = { prefix: prefix + "/" }
 
         create_containers
 
-        while resp.nil? || resp.body.count == 10000
-          opts.merge!(:marker => objects.last.name) unless objects.empty?
-          with_retries("GET '#{ container }/#{ prefix }/*'") do
+        while resp.nil? || resp.body.count == 10_000
+          opts[:marker] = objects.last.name unless objects.empty?
+          with_retries("GET '#{container}/#{prefix}/*'") do
             resp = connection.get_container(container, opts)
           end
           resp.body.each do |obj_data|
@@ -90,7 +89,7 @@ module Backup
       # Used by Object to fetch metadata if needed.
       def head_object(object)
         resp = nil
-        with_retries("HEAD '#{ container }/#{ object.name }'") do
+        with_retries("HEAD '#{container}/#{object.name}'") do
           resp = connection.head_object(container, object.name)
         end
         resp
@@ -106,14 +105,14 @@ module Backup
         names.map!(&:name) if names.first.is_a?(Object)
 
         until names.empty?
-          _names = names.slice!(0, 10000)
-          with_retries('DELETE Multiple Objects') do
-            resp = connection.delete_multiple_objects(container, _names)
-            resp_status = resp.body['Response Status']
-            raise Error, <<-EOS unless resp_status == '200 OK'
-              #{ resp_status }
+          names_partial = names.slice!(0, 10_000)
+          with_retries("DELETE Multiple Objects") do
+            resp = connection.delete_multiple_objects(container, names_partial)
+            resp_status = resp.body["Response Status"]
+            raise Error, <<-EOS unless resp_status == "200 OK"
+              #{resp_status}
               The server returned the following:
-              #{ resp.body.inspect }
+              #{resp.body.inspect}
             EOS
           end
         end
@@ -126,13 +125,13 @@ module Backup
       # - Missing segments will be ignored.
       def delete_slo(objects)
         Array(objects).each do |object|
-          with_retries("DELETE SLO Manifest '#{ container }/#{ object.name }'") do
+          with_retries("DELETE SLO Manifest '#{container}/#{object.name}'") do
             resp = connection.delete_static_large_object(container, object.name)
-            resp_status = resp.body['Response Status']
-            raise Error, <<-EOS unless resp_status == '200 OK'
-              #{ resp_status }
+            resp_status = resp.body["Response Status"]
+            raise Error, <<-EOS unless resp_status == "200 OK"
+              #{resp_status}
               The server returned the following:
-              #{ resp.body.inspect }
+              #{resp.body.inspect}
             EOS
           end
         end
@@ -142,12 +141,12 @@ module Backup
 
       def connection
         @connection ||= Fog::Storage.new({
-          :provider             => 'Rackspace',
-          :rackspace_username   => username,
-          :rackspace_api_key    => api_key,
-          :rackspace_auth_url   => auth_url,
-          :rackspace_region     => region,
-          :rackspace_servicenet => servicenet
+          provider: "Rackspace",
+          rackspace_username: username,
+          rackspace_api_key: api_key,
+          rackspace_auth_url: auth_url,
+          rackspace_region: region,
+          rackspace_servicenet: servicenet
         }.merge(fog_options || {}))
       end
 
@@ -155,16 +154,16 @@ module Backup
         return if @containers_created
         @containers_created = true
 
-        with_retries('Create Containers') do
+        with_retries("Create Containers") do
           connection.put_container(container)
           connection.put_container(segments_container) if segments_container
         end
       end
 
       def put_object(src, dest)
-        opts = headers.merge('ETag' => Digest::MD5.file(src).hexdigest)
-        with_retries("PUT '#{ container }/#{ dest }'") do
-          File.open(src, 'r') do |file|
+        opts = headers.merge("ETag" => Digest::MD5.file(src).hexdigest)
+        with_retries("PUT '#{container}/#{dest}'") do
+          File.open(src, "r") do |file|
             connection.put_object(container, dest, file, opts)
           end
         end
@@ -176,25 +175,25 @@ module Backup
       # SLO manifest object is uploaded.
       def upload_segments(src, dest, segment_bytes, file_size)
         total_segments = (file_size / segment_bytes.to_f).ceil
-        progress = (0.1..0.9).step(0.1).map {|n| (total_segments * n).floor }
-        Logger.info "\s\sUploading #{ total_segments } SLO Segments..."
+        progress = (0.1..0.9).step(0.1).map { |n| (total_segments * n).floor }
+        Logger.info "\s\sUploading #{total_segments} SLO Segments..."
 
         segments = []
-        File.open(src, 'r') do |file|
+        File.open(src, "r") do |file|
           segment_number = 0
           until file.eof?
             segment_number += 1
-            object = "#{ dest }/#{ segment_number.to_s.rjust(4, '0') }"
+            object = "#{dest}/#{segment_number.to_s.rjust(4, "0")}"
             pos = file.pos
             md5 = segment_md5(file, segment_bytes)
-            opts = headers.merge('ETag' => md5)
+            opts = headers.merge("ETag" => md5)
 
-            with_retries("PUT '#{ segments_container }/#{ object }'") do
+            with_retries("PUT '#{segments_container}/#{object}'") do
               file.seek(pos)
               offset = 0
               connection.put_object(segments_container, object, nil, opts) do
                 # block is called to stream data until it returns ''
-                data = ''
+                data = ""
                 if offset <= segment_bytes - SEGMENT_BUFFER
                   data = file.read(SEGMENT_BUFFER).to_s # nil => ''
                   offset += data.size
@@ -204,13 +203,13 @@ module Backup
             end
 
             segments << {
-              :path => "#{ segments_container }/#{ object }",
-              :etag => md5,
-              :size_bytes => file.pos - pos
+              path: "#{segments_container}/#{object}",
+              etag: md5,
+              size_bytes: file.pos - pos
             }
 
             if i = progress.rindex(segment_number)
-              Logger.info "\s\s...#{ i + 1 }0% Complete..."
+              Logger.info "\s\s...#{i + 1}0% Complete..."
             end
           end
         end
@@ -234,9 +233,9 @@ module Backup
       # are not found. However, each segment's ETag was verified when we
       # uploaded the segments, so this should only retry failed requests.
       def upload_manifest(dest, segments)
-        Logger.info "\s\sStoring SLO Manifest '#{ container }/#{ dest }'"
+        Logger.info "\s\sStoring SLO Manifest '#{container}/#{dest}'"
 
-        with_retries("PUT SLO Manifest '#{ container }/#{ dest }'") do
+        with_retries("PUT SLO Manifest '#{container}/#{dest}'") do
           connection.put_static_obj_manifest(container, dest, segments, headers)
         end
       end
@@ -245,7 +244,7 @@ module Backup
       # This includes non-SLO objects, the SLO manifest and all segments.
       def headers
         headers = {}
-        headers.merge!('X-Delete-At' => delete_at) if delete_at
+        headers["X-Delete-At"] = delete_at if delete_at
         headers
       end
 
@@ -261,10 +260,10 @@ module Backup
         mb += 1 until file_size / (1024**2 * mb).to_f <= 1000
         Logger.warn Error.new(<<-EOS)
           Segment Size Adjusted
-          Your original #segment_size of #{ orig_mb } MiB has been adjusted
-          to #{ mb } MiB in order to satisfy the limit of 1000 segments.
+          Your original #segment_size of #{orig_mb} MiB has been adjusted
+          to #{mb} MiB in order to satisfy the limit of 1000 segments.
           To enforce your chosen #segment_size, you should use the Splitter.
-          e.g. split_into_chunks_of #{ mb * 1000 } (#segment_size * 1000)
+          e.g. split_into_chunks_of #{mb * 1000} (#segment_size * 1000)
         EOS
         1024**2 * mb
       end
@@ -274,16 +273,16 @@ module Backup
 
         def initialize(cloud_io, data)
           @cloud_io = cloud_io
-          @name = data['name']
-          @hash = data['hash']
+          @name = data["name"]
+          @hash = data["hash"]
         end
 
         def slo?
-          !!metadata['X-Static-Large-Object']
+          !!metadata["X-Static-Large-Object"]
         end
 
         def marked_for_deletion?
-          !!metadata['X-Delete-At']
+          !!metadata["X-Delete-At"]
         end
 
         private
@@ -292,7 +291,6 @@ module Backup
           @metadata ||= @cloud_io.head_object(self).headers
         end
       end
-
     end
   end
 end