interu-backup 3.0.16

data/lib/backup/storage/object.rb

@@ -0,0 +1,45 @@
+# encoding: utf-8
+
+module Backup
+  module Storage
+    class Object
+
+      ##
+      # Holds the type attribute
+      attr_accessor :storage_file
+
+      ##
+      # Instantiates a new Backup::Storage::Object and stores the
+      # full path to the storage file (yaml) in the @storage_file attribute
+      def initialize(type)
+        @storage_file = File.join(DATA_PATH, TRIGGER, "#{type}.yml")
+      end
+
+      ##
+      # Tries to load an existing YAML file and returns an
+      # array of storage objects. If no file exists, an empty
+      # array gets returned
+      #
+      # If a file is loaded it'll sort the array of objects by @time
+      # descending. The newest backup storage object comes in Backup::Storage::Object.load[0]
+      # and the oldest in Backup::Storage::Object.load[-1]
+      def load
+        if File.exist?(storage_file)
+          YAML.load_file(storage_file).sort { |a,b| b.time <=> a.time }
+        else
+          []
+        end
+      end
+
+      ##
+      # Takes the provided objects and converts it to YAML format.
+      # The YAML data gets written to the storage file
+      def write(objects)
+        File.open(storage_file, 'w') do |file|
+          file.write(objects.to_yaml)
+        end
+      end
+
+    end
+  end
+end

data/lib/backup/storage/rsync.rb

@@ -0,0 +1,129 @@
+# encoding: utf-8
+
+##
+# Require the tempfile Ruby library when Backup::Storage::RSync is loaded
+require 'tempfile'
+
+##
+# Only load the Net::SSH library when the Backup::Storage::RSync class is loaded
+Backup::Dependency.load('net-ssh')
+
+module Backup
+  module Storage
+    class RSync < Base
+      include Backup::CLI
+
+      ##
+      # Server credentials
+      attr_accessor :username, :password
+
+      ##
+      # Server IP Address and SSH port
+      attr_accessor :ip, :port
+
+      ##
+      # Path to store backups to
+      attr_accessor :path
+
+      ##
+      # Creates a new instance of the RSync storage object
+      # First it sets the defaults (if any exist) and then evaluates
+      # the configuration block which may overwrite these defaults
+      def initialize(&block)
+        load_defaults!
+
+        @port ||= 22
+        @path ||= 'backups'
+
+        instance_eval(&block) if block_given?
+        write_password_file!
+
+        @time = TIME
+        @path = path.sub(/^\~\//, '')
+      end
+
+      ##
+      # This is the remote path to where the backup files will be stored
+      def remote_path
+        File.join(path, TRIGGER)
+      end
+
+      ##
+      # Performs the backup transfer
+      def perform!
+        transfer!
+        remove_password_file!
+      end
+
+    private
+
+      ##
+      # Establishes a connection to the remote server and returns the Net::SSH object.
+      # Not doing any instance variable caching because this object gets persisted in YAML
+      # format to a file and will issues. This, however has no impact on performance since it only
+      # gets invoked once per object for a #transfer! and once for a remove! Backups run in the
+      # background anyway so even if it were a bit slower it shouldn't matter.
+      def connection
+        Net::SSH.start(ip, username, :password => @password, :port => port)
+      end
+
+      ##
+      # Transfers the archived file to the specified remote server
+      def transfer!
+        Logger.message("#{ self.class } started transferring \"#{ remote_file }\".")
+        create_remote_directories!
+        run("#{ utility(:rsync) } #{ options } #{ password } '#{ File.join(local_path, local_file) }' '#{ username }@#{ ip }:#{ File.join(remote_path, remote_file[20..-1]) }'")
+      end
+
+      ##
+      # Removes the transferred archive file from the server
+      def remove!
+        response = connection.exec!("rm #{ File.join(remote_path, remote_file) }")
+        if response =~ /No such file or directory/
+          Logger.warn "Could not remove file \"#{ File.join(remote_path, remote_file) }\"."
+        end
+      end
+
+      ##
+      # Creates (if they don't exist yet) all the directories on the remote
+      # server in order to upload the backup file.
+      def create_remote_directories!
+        connection.exec!("mkdir -p '#{ remote_path }'")
+      end
+
+      ##
+      # RSync options
+      # -z     = Compresses the bytes that will be transferred to reduce bandwidth usage
+      # --port = the port to connect to through SSH
+      # -Phv   = debug options
+      def options
+        "-z --port='#{ port }'"
+      end
+
+      ##
+      # Returns Rsync syntax for using a password file
+      def password
+        "--password-file='#{@password_file.path}'" unless @password.nil?
+      end
+
+      ##
+      # Writes the provided password to a temporary file so that
+      # the rsync utility can read the password from this file
+      def write_password_file!
+        unless @password.nil?
+          @password_file = Tempfile.new('backup-rsync-password')
+          @password_file.write(@password)
+          @password_file.close
+        end
+      end
+
+      ##
+      # Removes the previously created @password_file
+      # (temporary file containing the password)
+      def remove_password_file!
+        @password_file.unlink unless @password.nil?
+      end
+
+    end
+  end
+end

data/lib/backup/storage/s3.rb

@@ -0,0 +1,180 @@
+# encoding: utf-8
+
+##
+# Only load the Fog gem when the Backup::Storage::S3 class is loaded
+Backup::Dependency.load('fog')
+
+module Backup
+  module Storage
+    class S3 < Base
+
+      ##
+      # Amazon Simple Storage Service (S3) Credentials
+      attr_accessor :access_key_id, :secret_access_key
+
+      ##
+      # Amazon S3 bucket name and path
+      attr_accessor :bucket, :path
+
+      ##
+      # Region of the specified S3 bucket
+      attr_accessor :region
+
+      ##
+      # Creates a new instance of the Amazon S3 storage object
+      # First it sets the defaults (if any exist) and then evaluates
+      # the configuration block which may overwrite these defaults
+      #
+      # Currently available regions:
+      #   eu-west-1, us-east-1, ap-southeast-1, us-west-1
+      def initialize(&block)
+        load_defaults!
+
+        @path ||= 'backups'
+
+        instance_eval(&block) if block_given?
+
+        @time = TIME
+      end
+
+      ##
+      # This is the remote path to where the backup files will be stored
+      def remote_path
+        File.join(path, TRIGGER).sub(/^\//, '')
+      end
+
+      ##
+      # This is the provider that Fog uses for the S3 Storage
+      def provider
+        'AWS'
+      end
+
+      ##
+      # Performs the backup transfer
+      def perform!
+        transfer!
+        cycle!
+      end
+
+    private
+
+      ##
+      # Establishes a connection to Amazon S3 and returns the Fog object.
+      # Not doing any instance variable caching because this object gets persisted in YAML
+      # format to a file and will issues. This, however has no impact on performance since it only
+      # gets invoked once per object for a #transfer! and once for a remove! Backups run in the
+      # background anyway so even if it were a bit slower it shouldn't matter.
+      def connection
+        Fog::Storage.new(
+          :provider               => provider,
+          :aws_access_key_id      => access_key_id,
+          :aws_secret_access_key  => secret_access_key,
+          :region                 => region
+        )
+      end
+
+      ##
+      # Transfers the archived file to the specified Amazon S3 bucket
+      def transfer!
+        # maximum file size 5GB
+        #max_file_size = 5368709120
+        max_file_size = 15728640
+        # split size must be between 5MB and 5GB
+        max_split_size = max_file_size - 5242880
+
+        begin
+          local_file_path = File.join(local_path, local_file)
+
+          Logger.message("#{ self.class } started transferring \"#{ remote_file }\".")
+          connection.sync_clock
+          if File.stat(local_file_path).size <= max_file_size
+            connection.put_object(
+              bucket,
+              File.join(remote_path, remote_file),
+              File.open(File.join(local_path, local_file))
+            )
+          else
+            Logger.message("#{ self.class } started multipart uploading \"#{ remote_file }\".")
+
+            workspace_path = local_path + "/workspace"
+            create_workspace(workspace_path)
+
+            `split -b #{max_split_size}  #{local_file_path} #{workspace_path}/#{local_file}.0`
+
+            upload_id = initiate_multipart_upload
+            etags = upload_part(workspace_path, upload_id)
+
+            s3_md5 = complete_multipart_upload(etags, upload_id)
+            ## please check etag
+            # if it's differrent from local_file, try to upload again.
+            # ex)
+            # require 'digest/md5'
+            # original_md5 = Digest::MD5.hexdigest(File.open(local_file_path).read)
+
+            remove_workspace(workspace_path)
+          end
+        rescue Excon::Errors::NotFound => e
+          raise "An error occurred while trying to transfer the backup, please make sure the bucket exists.\n #{e.inspect}"
+        end
+      end
+
+      def initiate_multipart_upload
+        res = connection.initiate_multipart_upload(
+          bucket,
+          File.join(remote_path, remote_file)
+        )
+        res.body['UploadId']
+      end
+
+      def upload_part workspace_path, upload_id
+        etags = []
+        split_files = Dir.entries(workspace_path).select{|file| file != ".." and file != "."}.sort
+
+        split_files.each_with_index do |split_file, index|
+          Logger.message("uploading #{index + 1} / #{split_files.size}")
+          res = connection.upload_part(
+            bucket,
+            File.join(remote_path, remote_file),
+            upload_id,
+            index + 1,
+            File.open(File.join(workspace_path, split_file))
+          )
+          etags << res.headers['ETag']
+        end
+        etags
+      end
+
+      def complete_multipart_upload etags, upload_id
+        res = connection.complete_multipart_upload(
+          bucket,
+          File.join(remote_path, remote_file),
+          upload_id,
+          etags
+        )
+        res.body['ETag']
+      end
+
+      def create_workspace workspace_path
+        Dir.mkdir(workspace_path)
+      end
+
+      def remove_workspace workspace_path
+        split_files = Dir.entries(workspace_path).select{|file| file != ".." and file != "."}.sort
+        split_files.each do |split_file|
+          File.delete(File.join(workspace_path, split_file))
+        end
+        Dir.rmdir(workspace_path)
+      end
+
+      ##
+      # Removes the transferred archive file from the Amazon S3 bucket
+      def remove!
+        begin
+          connection.sync_clock
+          connection.delete_object(bucket, File.join(remote_path, remote_file))
+        rescue Excon::Errors::SocketError; end
+      end
+
+    end
+  end
+end

data/lib/backup/storage/scp.rb

@@ -0,0 +1,106 @@
+# encoding: utf-8
+
+##
+# Only load the Net::SSH and Net::SCP library/gems
+# when the Backup::Storage::SCP class is loaded
+Backup::Dependency.load('net-ssh')
+Backup::Dependency.load('net-scp')
+
+
+module Backup
+  module Storage
+    class SCP < Base
+
+      ##
+      # Server credentials
+      attr_accessor :username, :password
+
+      ##
+      # Server IP Address and SCP port
+      attr_accessor :ip, :port
+
+      ##
+      # Path to store backups to
+      attr_accessor :path
+
+      ##
+      # Creates a new instance of the SCP storage object
+      # First it sets the defaults (if any exist) and then evaluates
+      # the configuration block which may overwrite these defaults
+      def initialize(&block)
+        load_defaults!
+
+        @port ||= 22
+        @path ||= 'backups'
+
+        instance_eval(&block) if block_given?
+
+        @time = TIME
+        @path = path.sub(/^\~\//, '')
+      end
+
+      ##
+      # This is the remote path to where the backup files will be stored
+      def remote_path
+        File.join(path, TRIGGER)
+      end
+
+      ##
+      # Performs the backup transfer
+      def perform!
+        transfer!
+        cycle!
+      end
+
+    private
+
+      ##
+      # Establishes a connection to the remote server and returns the Net::SCP object.
+      # Not doing any instance variable caching because this object gets persisted in YAML
+      # format to a file and will issues. This, however has no impact on performance since it only
+      # gets invoked once per object for a #transfer! and once for a remove! Backups run in the
+      # background anyway so even if it were a bit slower it shouldn't matter.
+      #
+      # We will be using Net::SSH, and use Net::SCP through Net::SSH to transfer backups
+      def connection
+        Net::SSH.start(ip, username, :password => password, :port => port)
+      end
+
+      ##
+      # Transfers the archived file to the specified remote server
+      def transfer!
+        Logger.message("#{ self.class } started transferring \"#{ remote_file }\".")
+        create_remote_directories!
+        connection.scp.upload!(
+          File.join(local_path, local_file),
+          File.join(remote_path, remote_file)
+        )
+      end
+
+      ##
+      # Removes the transferred archive file from the server
+      def remove!
+        response = connection.exec!("rm #{ File.join(remote_path, remote_file) }")
+        if response =~ /No such file or directory/
+          Logger.warn "Could not remove file \"#{ File.join(remote_path, remote_file) }\"."
+        end
+      end
+
+      ##
+      # Creates (if they don't exist yet) all the directories on the remote
+      # server in order to upload the backup file. Net::SCP does not support
+      # paths to directories that don't yet exist when creating new directories.
+      # Instead, we split the parts up in to an array (for each '/') and loop through
+      # that to create the directories one by one. Net::SCP raises an exception when
+      # the directory it's trying ot create already exists, so we have rescue it
+      def create_remote_directories!
+        path_parts = Array.new
+        remote_path.split('/').each do |path_part|
+          path_parts << path_part
+          connection.exec!("mkdir '#{ path_parts.join('/') }'")
+        end
+      end
+
+    end
+  end
+end