backupsss 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7d8134a309ca7b0fd67955ee59b704417951b648
-  data.tar.gz: 41d054c4ebe76355d2f6d9de6a52c081daf732f3
+  metadata.gz: 473324a2ff4a2affcf336b2ac9adaf8314f3a2a8
+  data.tar.gz: efa42347a6d6e9b4bb2d9aa7c0af3a4fac323bb0
 SHA512:
-  metadata.gz: bb3d651fa5f156ea578f16a708ee73d6edb19e4e19136e6ab2eb27ac9739a490c9f12cf2a7419f8c6adf97be8fe24efb9cb316ebbf4e9f13e44a15400e3eae8c
-  data.tar.gz: c738fef618d52401503927e1a14415e6692697f3d321731a46894ba9830d191e1c5b786685dc732bcd55c3928ffff4314d3ad3559e58ade67b97c9f841b21582
+  metadata.gz: 970be5e7c287f045ecaed99f506291df4626746ad662ada92ac7654d37c29144c208abc09845afdbe7ea1c24ca5466124187263891edea6f6b65d8d350816227
+  data.tar.gz: 9f32195679dcc98f7d9944a8003d5bc0c938ef3a8f06a24c44ccc1e2e32a7cfe716f3fcc72412c3a32e4f9a663fabdf88b8ccea0b3b05b4d762732c7c7368c2f

data/.rubocop.yml

@@ -1,2 +1,4 @@
 AllCops:
+  Exclude:
+    - '*.gemspec'
   TargetRubyVersion: 2.1

data/backupsss.gemspec

@@ -35,11 +35,12 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'guard-rspec', '~> 4.6.4'
   spec.add_development_dependency 'guard-rubocop', '~> 1.2'
   spec.add_development_dependency 'guard-bundler', '~> 2.1'
-  spec.add_development_dependency 'rubocop', '~> 0.37'
+  spec.add_development_dependency 'rubocop', '0.46'
   spec.add_development_dependency 'simplecov', '~> 0.11.2'
   spec.add_development_dependency 'simplecov-console', '~> 0.3.0'
   spec.add_development_dependency 'codeclimate-test-reporter', '~> 0.4'
 
-  spec.add_runtime_dependency 'aws-sdk'
-  spec.add_runtime_dependency 'rufus-scheduler'
+  spec.add_runtime_dependency 'aws-sdk', '~> 2.7.0'
+  spec.add_runtime_dependency 'parallel', '~> 1.10.0'
+  spec.add_runtime_dependency 'rufus-scheduler', '~> 3.3.2'
 end

data/lib/backupsss/backup.rb

@@ -1,25 +1,126 @@
+require 'parallel'
+
 module Backupsss
   # A class for delivering a tar to S3
   class Backup
+    MAX_FILE_SIZE = 1024 * 1024 * 100 # 100MB
+
     attr_reader :config, :client, :filename
 
     def initialize(config, client)
-      @config       = config
-      @client       = client
+      @config   = config
+      @client   = client
       @filename = config[:filename]
     end
 
     def put_file(file)
-      client.put_object(bucket_opts.merge(body: file))
+      large_file(file) ? multi_upload(file) : single_upload(file)
     end
 
     private
 
+    def create_multipart_upload
+      $stdout.puts 'Creating a multipart upload'
+
+      client.create_multipart_upload(bucket_opts)
+    end
+
+    def large_file(file)
+      $stdout.puts 'Checking backup size ...'
+      is_lg  = file.size > MAX_FILE_SIZE
+      status = is_lg ? 'greater than' : 'less than or equal to'
+      $stdout.puts "Size of backup is #{status} 100MB"
+
+      is_lg
+    end
+
+    def complete_multipart_upload_request(upload_id, parts)
+      bucket_opts.merge(
+        upload_id: upload_id, multipart_upload: { parts: parts }
+      )
+    end
+
+    def timed_multipart_upload
+      s = Time.now
+      $stdout.puts "Starting multipart upload at #{s}"
+      yield
+      e = Time.now
+      duration = ((e - s) / 60).round(2)
+      output   = ["Completed multipart upload at #{e}"]
+      output << "Completed in #{duration} minutes."
+
+      $stdout.puts output.join("\n")
+    end
+
+    def abort_multipart_message(error, upload_id)
+      "#{error}\n#{upload_id}: Aborting multipart upload"
+    end
+
+    def multi_upload(file)
+      upload_id = create_multipart_upload.upload_id
+
+      bail_multipart_on_fail(upload_id) do
+        timed_multipart_upload do
+          parts = upload_parts(file, upload_id).sort do |a, b|
+            a[:part_number] <=> b[:part_number]
+          end
+
+          req = complete_multipart_upload_request(upload_id, parts)
+          client.complete_multipart_upload(req)
+        end
+      end
+    end
+
+    def upload_parts(file, upload_id)
+      Parallel.map(1..part_count(file), in_threads: 10) do |part|
+        bail_upload_part_on_fail(part, upload_id) do
+          $stdout.puts "#{upload_id}: Uploading part number #{part}\n"
+          r = client.upload_part(upload_part_params(file, part, upload_id))
+          success_msg = "#{upload_id}: Completed uploading part number #{part}"
+          r.on_success { $stdout.puts success_msg }
+
+          { etag: r.etag, part_number: part }
+        end
+      end
+    end
+
+    def bail_multipart_on_fail(upload_id)
+      yield
+    rescue StandardError => e
+      $stdout.puts abort_multipart_message(e, upload_id)
+      client.abort_multipart_upload(bucket_opts.merge(upload_id: upload_id))
+    end
+
+    def bail_upload_part_on_fail(part, upload_id)
+      yield
+    rescue StandardError => e
+      output = ["#{upload_id}: Failed to upload part number #{part}"]
+      output << "because of #{e.message}"
+      output << "#{upload_id}: Aborting remaining parts"
+      raise output.join("\n")
+    end
+
+    def upload_part_params(file, part, upload_id)
+      start = (part - 1) * MAX_FILE_SIZE
+      body  = IO.read(file.path, MAX_FILE_SIZE, start)
+
+      bucket_opts.merge(part_number: part, body: body, upload_id: upload_id)
+    end
+
+    def part_count(file)
+      c = (file.size.to_f / MAX_FILE_SIZE.to_f).ceil
+      $stdout.puts "Uploading backup as #{c} parts"
+
+      c
+    end
+
+    def single_upload(file)
+      client.put_object(bucket_opts.merge(body: file.read))
+    end
+
     def bucket_opts
-      {
-        bucket: config[:s3_bucket],
-        key:    "#{config[:s3_bucket_prefix]}/#{filename}"
-      }
+      { bucket: config[:s3_bucket],
+        key:    "#{config[:s3_bucket_prefix]}/#{filename}" }
     end
   end
 end

data/lib/backupsss/tar.rb

@@ -12,21 +12,34 @@ module Backupsss
     end
 
     def make
-      return nil unless valid_dest? && valid_src?
+      return unless valid_dest? && valid_src?
       _, err, status = Open3.capture3("#{tar_command} #{dest} #{src}")
-      STDERR.puts "tar command stderr:\n#{err}" unless err.empty?
-      check_tar_result(status)
-      File.open(dest)
+      File.open(dest) if valid_exit?(status, err) && valid_file?
     end
 
-    def check_tar_result(status)
-      if status.exitstatus.nonzero?
-        raise "ERROR: #{tar_command} exited #{status.exitstatus}"
-      end
-      unless File.exist?(dest)
-        raise 'ERROR: Tar destination file does not exist'
-      end
-      raise 'ERROR: Tar destionation file is 0 bytes.' if File.size(dest).zero?
+    def valid_exit?(status, err)
+      output = []
+      output << "command.......#{tar_command}"
+      output << "stderr........#{err}" unless err.empty?
+      output << "status........#{status}"
+      output << "exit code.....#{status.to_i}"
+      $stdout.puts output.join("\n")
+
+      return true if success_cases(status.to_i, err)
+      raise "ERROR: #{tar_command} exited #{status.to_i}"
+    end
+
+    def valid_file?
+      raise messages[:no_file] unless File.exist?(dest)
+      raise messages[:zero_byte] if File.size(dest).zero?
+      true
+    end
+
+    def messages
+      {
+        no_file:   'ERROR: Tar destination file does not exist.',
+        zero_byte: 'ERROR: Tar destination file is 0 bytes.'
+      }
     end
 
     def valid_dest?
@@ -47,6 +60,10 @@ module Backupsss
 
     private
 
+    def clean_exit(status)
+      status.zero?
+    end
+
     def dest_dir
       File.dirname(dest)
     end
@@ -56,13 +73,21 @@ module Backupsss
     end
 
     def dir_exists?(dir)
-      File.exist?(dir) || raise_sys_err(dir, Errno::ENOENT::Errno)
+      File.exist?(File.open(dir)) || raise_sys_err(dir, Errno::ENOENT::Errno)
+    end
+
+    def file_changed?(signal_int, err)
+      signal_int == 1 && err.match(/file changed as we read it/)
     end
 
     def src_readable?
       File.readable?(src) || raise_sys_err(src, Errno::EPERM::Errno)
     end
 
+    def success_cases(signal_int, err)
+      clean_exit(signal_int) || file_changed?(signal_int, err)
+    end
+
     def raise_sys_err(dir, err)
       raise SystemCallError.new(dir.to_s, err)
     end

data/lib/backupsss/version.rb

@@ -1,3 +1,3 @@
 module Backupsss
-  VERSION = '0.2.0'.freeze
+  VERSION = '0.3.0'.freeze
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: backupsss
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Reppard Walker
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-11-23 00:00:00.000000000 Z
+date: 2017-01-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -114,16 +114,16 @@ dependencies:
   name: rubocop
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0.37'
+        version: '0.46'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0.37'
+        version: '0.46'
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
@@ -170,30 +170,44 @@ dependencies:
   name: aws-sdk
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 2.7.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 2.7.0
+- !ruby/object:Gem::Dependency
+  name: parallel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.10.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.10.0
 - !ruby/object:Gem::Dependency
   name: rufus-scheduler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 3.3.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 3.3.2
 description: Backup any file or directory as a tar and push the tar to a specificed
   S3 bucket.
 email: