backup 4.3.0 → 5.0.0.beta.3

data/lib/backup/cloud_io/base.rb

@@ -1,5 +1,3 @@
-# encoding: utf-8
-
 module Backup
   module CloudIO
     class Error < Backup::Error; end
@@ -22,20 +20,19 @@ module Backup
         rescue => err
           retries += 1
           raise Error.wrap(err, <<-EOS) if retries > max_retries
-            Max Retries (#{ max_retries }) Exceeded!
-            Operation: #{ operation }
+            Max Retries (#{max_retries}) Exceeded!
+            Operation: #{operation}
             Be sure to check the log messages for each retry attempt.
           EOS
 
           Logger.info Error.wrap(err, <<-EOS)
-            Retry ##{ retries } of #{ max_retries }
-            Operation: #{ operation }
+            Retry ##{retries} of #{max_retries}
+            Operation: #{operation}
           EOS
           sleep(retry_waitsec)
           retry
         end
       end
-
     end
   end
 end

data/lib/backup/cloud_io/cloud_files.rb

@@ -1,7 +1,6 @@
-# encoding: utf-8
-require 'backup/cloud_io/base'
-require 'fog'
-require 'digest/md5'
+require "backup/cloud_io/base"
+require "fog"
+require "digest/md5"
 
 module Backup
   module CloudIO
@@ -13,8 +12,8 @@ module Backup
       SEGMENT_BUFFER  = 1024**2         # 1 MiB
 
       attr_reader :username, :api_key, :auth_url, :region, :servicenet,
-                  :container, :segments_container, :segment_size, :days_to_keep,
-                  :fog_options
+        :container, :segments_container, :segment_size, :days_to_keep,
+        :fog_options
 
       def initialize(options = {})
         super
@@ -41,9 +40,9 @@ module Backup
         if segment_bytes > 0 && file_size > segment_bytes
           raise FileSizeError, <<-EOS if file_size > MAX_SLO_SIZE
             File Too Large
-            File: #{ src }
-            Size: #{ file_size }
-            Max SLO Size is #{ MAX_SLO_SIZE } (5 GiB * 1000 segments)
+            File: #{src}
+            Size: #{file_size}
+            Max SLO Size is #{MAX_SLO_SIZE} (5 GiB * 1000 segments)
           EOS
 
           segment_bytes = adjusted_segment_bytes(segment_bytes, file_size)
@@ -52,9 +51,9 @@ module Backup
         else
           raise FileSizeError, <<-EOS if file_size > MAX_FILE_SIZE
             File Too Large
-            File: #{ src }
-            Size: #{ file_size }
-            Max File Size is #{ MAX_FILE_SIZE } (5 GiB)
+            File: #{src}
+            Size: #{file_size}
+            Max File Size is #{MAX_FILE_SIZE} (5 GiB)
           EOS
 
           put_object(src, dest)
@@ -69,14 +68,14 @@ module Backup
       def objects(prefix)
         objects = []
         resp = nil
-        prefix = prefix.chomp('/')
-        opts = { :prefix => prefix + '/' }
+        prefix = prefix.chomp("/")
+        opts = { prefix: prefix + "/" }
 
         create_containers
 
-        while resp.nil? || resp.body.count == 10000
-          opts.merge!(:marker => objects.last.name) unless objects.empty?
-          with_retries("GET '#{ container }/#{ prefix }/*'") do
+        while resp.nil? || resp.body.count == 10_000
+          opts[:marker] = objects.last.name unless objects.empty?
+          with_retries("GET '#{container}/#{prefix}/*'") do
             resp = connection.get_container(container, opts)
           end
           resp.body.each do |obj_data|
@@ -90,7 +89,7 @@ module Backup
       # Used by Object to fetch metadata if needed.
       def head_object(object)
         resp = nil
-        with_retries("HEAD '#{ container }/#{ object.name }'") do
+        with_retries("HEAD '#{container}/#{object.name}'") do
           resp = connection.head_object(container, object.name)
         end
         resp
@@ -106,14 +105,14 @@ module Backup
         names.map!(&:name) if names.first.is_a?(Object)
 
         until names.empty?
-          _names = names.slice!(0, 10000)
-          with_retries('DELETE Multiple Objects') do
-            resp = connection.delete_multiple_objects(container, _names)
-            resp_status = resp.body['Response Status']
-            raise Error, <<-EOS unless resp_status == '200 OK'
-              #{ resp_status }
+          names_partial = names.slice!(0, 10_000)
+          with_retries("DELETE Multiple Objects") do
+            resp = connection.delete_multiple_objects(container, names_partial)
+            resp_status = resp.body["Response Status"]
+            raise Error, <<-EOS unless resp_status == "200 OK"
+              #{resp_status}
               The server returned the following:
-              #{ resp.body.inspect }
+              #{resp.body.inspect}
             EOS
           end
         end
@@ -126,13 +125,13 @@ module Backup
       # - Missing segments will be ignored.
       def delete_slo(objects)
         Array(objects).each do |object|
-          with_retries("DELETE SLO Manifest '#{ container }/#{ object.name }'") do
+          with_retries("DELETE SLO Manifest '#{container}/#{object.name}'") do
             resp = connection.delete_static_large_object(container, object.name)
-            resp_status = resp.body['Response Status']
-            raise Error, <<-EOS unless resp_status == '200 OK'
-              #{ resp_status }
+            resp_status = resp.body["Response Status"]
+            raise Error, <<-EOS unless resp_status == "200 OK"
+              #{resp_status}
               The server returned the following:
-              #{ resp.body.inspect }
+              #{resp.body.inspect}
             EOS
           end
         end
@@ -142,12 +141,12 @@ module Backup
 
       def connection
         @connection ||= Fog::Storage.new({
-          :provider             => 'Rackspace',
-          :rackspace_username   => username,
-          :rackspace_api_key    => api_key,
-          :rackspace_auth_url   => auth_url,
-          :rackspace_region     => region,
-          :rackspace_servicenet => servicenet
+          provider: "Rackspace",
+          rackspace_username: username,
+          rackspace_api_key: api_key,
+          rackspace_auth_url: auth_url,
+          rackspace_region: region,
+          rackspace_servicenet: servicenet
         }.merge(fog_options || {}))
       end
 
@@ -155,16 +154,16 @@ module Backup
         return if @containers_created
         @containers_created = true
 
-        with_retries('Create Containers') do
+        with_retries("Create Containers") do
           connection.put_container(container)
           connection.put_container(segments_container) if segments_container
         end
       end
 
       def put_object(src, dest)
-        opts = headers.merge('ETag' => Digest::MD5.file(src).hexdigest)
-        with_retries("PUT '#{ container }/#{ dest }'") do
-          File.open(src, 'r') do |file|
+        opts = headers.merge("ETag" => Digest::MD5.file(src).hexdigest)
+        with_retries("PUT '#{container}/#{dest}'") do
+          File.open(src, "r") do |file|
             connection.put_object(container, dest, file, opts)
           end
         end
@@ -176,25 +175,25 @@ module Backup
       # SLO manifest object is uploaded.
       def upload_segments(src, dest, segment_bytes, file_size)
         total_segments = (file_size / segment_bytes.to_f).ceil
-        progress = (0.1..0.9).step(0.1).map {|n| (total_segments * n).floor }
-        Logger.info "\s\sUploading #{ total_segments } SLO Segments..."
+        progress = (0.1..0.9).step(0.1).map { |n| (total_segments * n).floor }
+        Logger.info "\s\sUploading #{total_segments} SLO Segments..."
 
         segments = []
-        File.open(src, 'r') do |file|
+        File.open(src, "r") do |file|
           segment_number = 0
           until file.eof?
             segment_number += 1
-            object = "#{ dest }/#{ segment_number.to_s.rjust(4, '0') }"
+            object = "#{dest}/#{segment_number.to_s.rjust(4, "0")}"
             pos = file.pos
             md5 = segment_md5(file, segment_bytes)
-            opts = headers.merge('ETag' => md5)
+            opts = headers.merge("ETag" => md5)
 
-            with_retries("PUT '#{ segments_container }/#{ object }'") do
+            with_retries("PUT '#{segments_container}/#{object}'") do
               file.seek(pos)
               offset = 0
               connection.put_object(segments_container, object, nil, opts) do
                 # block is called to stream data until it returns ''
-                data = ''
+                data = ""
                 if offset <= segment_bytes - SEGMENT_BUFFER
                   data = file.read(SEGMENT_BUFFER).to_s # nil => ''
                   offset += data.size
@@ -204,13 +203,13 @@ module Backup
             end
 
             segments << {
-              :path => "#{ segments_container }/#{ object }",
-              :etag => md5,
-              :size_bytes => file.pos - pos
+              path: "#{segments_container}/#{object}",
+              etag: md5,
+              size_bytes: file.pos - pos
             }
 
             if i = progress.rindex(segment_number)
-              Logger.info "\s\s...#{ i + 1 }0% Complete..."
+              Logger.info "\s\s...#{i + 1}0% Complete..."
             end
           end
         end
@@ -234,9 +233,9 @@ module Backup
       # are not found. However, each segment's ETag was verified when we
       # uploaded the segments, so this should only retry failed requests.
       def upload_manifest(dest, segments)
-        Logger.info "\s\sStoring SLO Manifest '#{ container }/#{ dest }'"
+        Logger.info "\s\sStoring SLO Manifest '#{container}/#{dest}'"
 
-        with_retries("PUT SLO Manifest '#{ container }/#{ dest }'") do
+        with_retries("PUT SLO Manifest '#{container}/#{dest}'") do
           connection.put_static_obj_manifest(container, dest, segments, headers)
         end
       end
@@ -245,7 +244,7 @@ module Backup
       # This includes non-SLO objects, the SLO manifest and all segments.
       def headers
         headers = {}
-        headers.merge!('X-Delete-At' => delete_at) if delete_at
+        headers["X-Delete-At"] = delete_at if delete_at
         headers
       end
 
@@ -261,10 +260,10 @@ module Backup
         mb += 1 until file_size / (1024**2 * mb).to_f <= 1000
         Logger.warn Error.new(<<-EOS)
           Segment Size Adjusted
-          Your original #segment_size of #{ orig_mb } MiB has been adjusted
-          to #{ mb } MiB in order to satisfy the limit of 1000 segments.
+          Your original #segment_size of #{orig_mb} MiB has been adjusted
+          to #{mb} MiB in order to satisfy the limit of 1000 segments.
           To enforce your chosen #segment_size, you should use the Splitter.
-          e.g. split_into_chunks_of #{ mb * 1000 } (#segment_size * 1000)
+          e.g. split_into_chunks_of #{mb * 1000} (#segment_size * 1000)
         EOS
         1024**2 * mb
       end
@@ -274,16 +273,16 @@ module Backup
 
         def initialize(cloud_io, data)
           @cloud_io = cloud_io
-          @name = data['name']
-          @hash = data['hash']
+          @name = data["name"]
+          @hash = data["hash"]
         end
 
         def slo?
-          !!metadata['X-Static-Large-Object']
+          !!metadata["X-Static-Large-Object"]
         end
 
         def marked_for_deletion?
-          !!metadata['X-Delete-At']
+          !!metadata["X-Delete-At"]
         end
 
         private
@@ -292,7 +291,6 @@ module Backup
           @metadata ||= @cloud_io.head_object(self).headers
         end
       end
-
     end
   end
 end

data/lib/backup/cloud_io/s3.rb

@@ -1,9 +1,8 @@
-# encoding: utf-8
-require 'backup/cloud_io/base'
-require 'fog'
-require 'digest/md5'
-require 'base64'
-require 'stringio'
+require "backup/cloud_io/base"
+require "fog"
+require "digest/md5"
+require "base64"
+require "stringio"
 
 module Backup
   module CloudIO
@@ -14,8 +13,8 @@ module Backup
       MAX_MULTIPART_SIZE  = 1024**4 * 5   # 5 TiB
 
       attr_reader :access_key_id, :secret_access_key, :use_iam_profile,
-                  :region, :bucket, :chunk_size, :encryption, :storage_class,
-                  :fog_options
+        :region, :bucket, :chunk_size, :encryption, :storage_class,
+        :fog_options
 
       def initialize(options = {})
         super
@@ -39,9 +38,9 @@ module Backup
         if chunk_bytes > 0 && file_size > chunk_bytes
           raise FileSizeError, <<-EOS if file_size > MAX_MULTIPART_SIZE
             File Too Large
-            File: #{ src }
-            Size: #{ file_size }
-            Max Multipart Upload Size is #{ MAX_MULTIPART_SIZE } (5 TiB)
+            File: #{src}
+            Size: #{file_size}
+            Max Multipart Upload Size is #{MAX_MULTIPART_SIZE} (5 TiB)
           EOS
 
           chunk_bytes = adjusted_chunk_bytes(chunk_bytes, file_size)
@@ -51,9 +50,9 @@ module Backup
         else
           raise FileSizeError, <<-EOS if file_size > MAX_FILE_SIZE
             File Too Large
-            File: #{ src }
-            Size: #{ file_size }
-            Max File Size is #{ MAX_FILE_SIZE } (5 GiB)
+            File: #{src}
+            Size: #{file_size}
+            Max File Size is #{MAX_FILE_SIZE} (5 GiB)
           EOS
 
           put_object(src, dest)
@@ -68,15 +67,15 @@ module Backup
       def objects(prefix)
         objects = []
         resp = nil
-        prefix = prefix.chomp('/')
-        opts = { 'prefix' => prefix + '/' }
+        prefix = prefix.chomp("/")
+        opts = { "prefix" => prefix + "/" }
 
-        while resp.nil? || resp.body['IsTruncated']
-          opts.merge!('marker' => objects.last.key) unless objects.empty?
-          with_retries("GET '#{ bucket }/#{ prefix }/*'") do
+        while resp.nil? || resp.body["IsTruncated"]
+          opts["marker"] = objects.last.key unless objects.empty?
+          with_retries("GET '#{bucket}/#{prefix}/*'") do
             resp = connection.get_bucket(bucket, opts)
           end
-          resp.body['Contents'].each do |obj_data|
+          resp.body["Contents"].each do |obj_data|
             objects << Object.new(self, obj_data)
           end
         end
@@ -87,7 +86,7 @@ module Backup
       # Used by Object to fetch metadata if needed.
       def head_object(object)
         resp = nil
-        with_retries("HEAD '#{ bucket }/#{ object.key }'") do
+        with_retries("HEAD '#{bucket}/#{object.key}'") do
           resp = connection.head_object(bucket, object.key)
         end
         resp
@@ -102,18 +101,18 @@ module Backup
         keys = Array(objects_or_keys).dup
         keys.map!(&:key) if keys.first.is_a?(Object)
 
-        opts = { :quiet => true } # only report Errors in DeleteResult
+        opts = { quiet: true } # only report Errors in DeleteResult
         until keys.empty?
-          _keys = keys.slice!(0, 1000)
-          with_retries('DELETE Multiple Objects') do
-            resp = connection.delete_multiple_objects(bucket, _keys, opts.dup)
-            unless resp.body['DeleteResult'].empty?
-              errors = resp.body['DeleteResult'].map do |result|
-                error = result['Error']
-                "Failed to delete: #{ error['Key'] }\n" +
-                "Reason: #{ error['Code'] }: #{ error['Message'] }"
+          keys_partial = keys.slice!(0, 1000)
+          with_retries("DELETE Multiple Objects") do
+            resp = connection.delete_multiple_objects(bucket, keys_partial, opts.dup)
+            unless resp.body["DeleteResult"].empty?
+              errors = resp.body["DeleteResult"].map do |result|
+                error = result["Error"]
+                "Failed to delete: #{error["Key"]}\n" \
+                  "Reason: #{error["Code"]}: #{error["Message"]}"
               end.join("\n")
-              raise Error, "The server returned the following:\n#{ errors }"
+              raise Error, "The server returned the following:\n#{errors}"
             end
           end
         end
@@ -122,41 +121,40 @@ module Backup
       private
 
       def connection
-        @connection ||= begin
-          opts = { :provider => 'AWS', :region => region }
-          if use_iam_profile
-            opts.merge!(:use_iam_profile => true)
-          else
-            opts.merge!(
-              :aws_access_key_id      => access_key_id,
-              :aws_secret_access_key  => secret_access_key
-            )
+        @connection ||=
+          begin
+            opts = { provider: "AWS", region: region }
+            if use_iam_profile
+              opts[:use_iam_profile] = true
+            else
+              opts[:aws_access_key_id] = access_key_id
+              opts[:aws_secret_access_key] = secret_access_key
+            end
+            opts.merge!(fog_options || {})
+            conn = Fog::Storage.new(opts)
+            conn.sync_clock
+            conn
           end
-          opts.merge!(fog_options || {})
-          conn = Fog::Storage.new(opts)
-          conn.sync_clock
-          conn
-        end
       end
 
       def put_object(src, dest)
         md5 = Base64.encode64(Digest::MD5.file(src).digest).chomp
-        options = headers.merge('Content-MD5' => md5)
-        with_retries("PUT '#{ bucket }/#{ dest }'") do
-          File.open(src, 'r') do |file|
+        options = headers.merge("Content-MD5" => md5)
+        with_retries("PUT '#{bucket}/#{dest}'") do
+          File.open(src, "r") do |file|
             connection.put_object(bucket, dest, file, options)
           end
         end
       end
 
       def initiate_multipart(dest)
-        Logger.info "\s\sInitiate Multipart '#{ bucket }/#{ dest }'"
+        Logger.info "\s\sInitiate Multipart '#{bucket}/#{dest}'"
 
         resp = nil
-        with_retries("POST '#{ bucket }/#{ dest }' (Initiate)") do
+        with_retries("POST '#{bucket}/#{dest}' (Initiate)") do
           resp = connection.initiate_multipart_upload(bucket, dest, headers)
         end
-        resp.body['UploadId']
+        resp.body["UploadId"]
       end
 
       # Each part's MD5 is sent to verify the transfer.
@@ -164,26 +162,26 @@ module Backup
       # once the multipart upload is completed.
       def upload_parts(src, dest, upload_id, chunk_bytes, file_size)
         total_parts = (file_size / chunk_bytes.to_f).ceil
-        progress = (0.1..0.9).step(0.1).map {|n| (total_parts * n).floor }
-        Logger.info "\s\sUploading #{ total_parts } Parts..."
+        progress = (0.1..0.9).step(0.1).map { |n| (total_parts * n).floor }
+        Logger.info "\s\sUploading #{total_parts} Parts..."
 
         parts = []
-        File.open(src, 'r') do |file|
+        File.open(src, "r") do |file|
           part_number = 0
           while data = file.read(chunk_bytes)
             part_number += 1
             md5 = Base64.encode64(Digest::MD5.digest(data)).chomp
 
-            with_retries("PUT '#{ bucket }/#{ dest }' Part ##{ part_number }") do
+            with_retries("PUT '#{bucket}/#{dest}' Part ##{part_number}") do
               resp = connection.upload_part(
                 bucket, dest, upload_id, part_number, StringIO.new(data),
-                { 'Content-MD5' => md5 }
+                "Content-MD5" => md5
               )
-              parts << resp.headers['ETag']
+              parts << resp.headers["ETag"]
             end
 
             if i = progress.rindex(part_number)
-              Logger.info "\s\s...#{ i + 1 }0% Complete..."
+              Logger.info "\s\s...#{i + 1}0% Complete..."
             end
           end
         end
@@ -191,13 +189,13 @@ module Backup
       end
 
       def complete_multipart(dest, upload_id, parts)
-        Logger.info "\s\sComplete Multipart '#{ bucket }/#{ dest }'"
+        Logger.info "\s\sComplete Multipart '#{bucket}/#{dest}'"
 
-        with_retries("POST '#{ bucket }/#{ dest }' (Complete)") do
+        with_retries("POST '#{bucket}/#{dest}' (Complete)") do
           resp = connection.complete_multipart_upload(bucket, dest, upload_id, parts)
-          raise Error, <<-EOS if resp.body['Code']
+          raise Error, <<-EOS if resp.body["Code"]
             The server returned the following error:
-            #{ resp.body['Code'] }: #{ resp.body['Message'] }
+            #{resp.body["Code"]}: #{resp.body["Message"]}
           EOS
         end
       end
@@ -206,14 +204,10 @@ module Backup
         headers = {}
 
         enc = encryption.to_s.upcase
-        headers.merge!(
-          { 'x-amz-server-side-encryption' => enc}
-        ) unless enc.empty?
+        headers["x-amz-server-side-encryption"] = enc unless enc.empty?
 
         sc = storage_class.to_s.upcase
-        headers.merge!(
-          { 'x-amz-storage-class' => sc }
-        ) unless sc.empty? || sc == 'STANDARD'
+        headers["x-amz-storage-class"] = sc unless sc.empty? || sc == "STANDARD"
 
         headers
       end
@@ -225,10 +219,10 @@ module Backup
         mb += 1 until file_size / (1024**2 * mb).to_f <= 10_000
         Logger.warn Error.new(<<-EOS)
           Chunk Size Adjusted
-          Your original #chunk_size of #{ orig_mb } MiB has been adjusted
-          to #{ mb } MiB in order to satisfy the limit of 10,000 chunks.
+          Your original #chunk_size of #{orig_mb} MiB has been adjusted
+          to #{mb} MiB in order to satisfy the limit of 10,000 chunks.
           To enforce your chosen #chunk_size, you should use the Splitter.
-          e.g. split_into_chunks_of #{ mb * 10_000 } (#chunk_size * 10_000)
+          e.g. split_into_chunks_of #{mb * 10_000} (#chunk_size * 10_000)
         EOS
         1024**2 * mb
       end
@@ -238,14 +232,14 @@ module Backup
 
         def initialize(cloud_io, data)
           @cloud_io = cloud_io
-          @key  = data['Key']
-          @etag = data['ETag']
-          @storage_class = data['StorageClass']
+          @key  = data["Key"]
+          @etag = data["ETag"]
+          @storage_class = data["StorageClass"]
         end
 
         # currently 'AES256' or nil
         def encryption
-          metadata['x-amz-server-side-encryption']
+          metadata["x-amz-server-side-encryption"]
         end
 
         private
@@ -254,7 +248,6 @@ module Backup
           @metadata ||= @cloud_io.head_object(self).headers
         end
       end
-
     end
   end
 end