venet-backup 4.1.3

data/lib/backup/storage/google/google_drive_transfer.rb

@@ -0,0 +1,125 @@
+module Backup
+  class GoogleDriveTransfer
+    include Config::Helpers
+
+    def initialize(client, drive)
+      @client = client
+      @drive = drive
+
+      get_info
+    end
+
+    def get_info
+      list = @client.execute(api_method: @drive.about.get)
+      @about = list.data
+    end
+
+    # File upload
+    #
+    def upload(package, folder_id = nil)
+      package.filenames.each do |filename|
+        src = File.join(Config.tmp_path, filename)
+
+        Logger.info "Storing '#{ src }'..."
+
+        file_folder = create_folder(package.time, folder_id)
+        insert_file(filename, src, file_folder.id)
+      end
+    end
+
+    def create_folder(title, parent_folder_id)
+      parent_folder_id ||= @about.root_folder_id
+
+      folder = @drive.files.insert.request_schema.new({
+        "title" => title,
+        "mimeType" => "application/vnd.google-apps.folder",
+        "parents" => [{
+          "id" => parent_folder_id
+        }],
+      })
+      result = @client.execute!(
+        api_method: @drive.files.insert,
+        body_object: folder
+      )
+
+      result.status == 200 ? result.data : nil
+    end
+
+    def insert_file(filename, src, folder_id)
+      folder_id ||= @about.root_folder_id
+
+      File.open(src, "r") do |file|
+        file_options = {
+          "title" => filename,
+          "parents" => [{
+            "id" => folder_id
+          }]
+        }
+
+        file = @drive.files.insert.request_schema.new(file_options)
+
+        media = Google::APIClient::UploadIO.new(src, MIME::Types.type_for(filename))
+        result = @client.execute(
+          api_method: @drive.files.insert,
+          body_object: file,
+          media: media,
+          parameters: {
+            "uploadType" => "multipart"
+          }
+        )
+
+        result
+      end
+    end
+
+
+    # Delete file
+    #
+    def delete(package, folder_id = nil)
+      folder_id ||= @about.root_folder_id
+
+      del_folder = find_files({
+        "folderId" => folder_id,
+        "q" => "title = '#{package.time}'",
+        "maxResults" => 1
+      })[0]
+
+      delete_folder(folder_id, del_folder.id)
+
+      nil
+    end
+
+    def find_files(params = {})
+      page_token = nil
+
+      begin
+        params["pageToken"] = page_token if page_token.to_s != ""
+
+        result = @client.execute(
+          api_method: @drive.children.list,
+          parameters: params
+        )
+
+        if result.status == 200
+          children = result.data
+          page_token = children.next_page_token
+
+          return children.items
+        else
+          puts "An error occurred: #{result.data['error']['message']}"
+          page_token = nil
+        end
+      end while page_token.to_s != ""
+    end
+
+    def delete_folder(parent_folder_id, del_folder_id)
+      @client.execute!(
+        api_method: @drive.children.delete,
+        parameters: {
+          "folderId" => parent_folder_id,
+          "childId" => del_folder_id
+        }
+      )
+    end
+  end
+end

data/lib/backup/storage/google_drive.rb

@@ -0,0 +1,62 @@
+# encoding: utf-8
+
+require "google/api_client"
+require "google/api_client/auth/file_storage"
+require "backup/storage/google/google_drive_auth"
+require "backup/storage/google/google_drive_transfer"
+
+module Backup
+  module Storage
+    class GoogleDrive < Base
+      include Storage::Cycler
+      class Error < Backup::Error; end
+
+      attr_accessor :client_id, :client_secret, :folder_id, :api_version, :cache_path
+
+      ##
+      # Creates a new instance of the storage object
+      def initialize(model, storage_id = nil)
+        super
+
+        @path           ||= "backups"
+        @cache_path     ||= ".cache"
+        path.sub!(/^\//, "")
+      end
+
+      private
+
+        def connection
+          auth = GoogleDriveAuth.new(
+            client_id: client_id,
+            client_secret: client_secret,
+            api_version: api_version,
+            cache_path: cache_path
+          )
+
+          @client = auth.client
+          @drive = auth.drive
+
+          @connection = GoogleDriveTransfer.new(@client, @drive)
+
+          @connection
+
+        rescue => err
+          raise Error.wrap(err, "Connection Failed")
+        end
+
+        def transfer!
+          connection.upload(package, folder_id)
+
+        rescue => err
+          raise Error.wrap(err, "Upload Failed!")
+        end
+
+        def remove!(package)
+          Logger.info "Removing backup package dated #{ package.time }..."
+
+          connection.delete(package, folder_id)
+        end
+
+    end
+  end
+end

data/lib/backup/storage/local.rb

@@ -0,0 +1,64 @@
+# encoding: utf-8
+
+module Backup
+  module Storage
+    class Local < Base
+      include Storage::Cycler
+      class Error < Backup::Error; end
+
+      def initialize(model, storage_id = nil)
+        super
+
+        @path ||= '~/backups'
+      end
+
+      private
+
+      def transfer!
+        FileUtils.mkdir_p(remote_path)
+
+        transfer_method = package_movable? ? :mv : :cp
+        package.filenames.each do |filename|
+          src = File.join(Config.tmp_path, filename)
+          dest = File.join(remote_path, filename)
+          Logger.info "Storing '#{ dest }'..."
+
+          FileUtils.send(transfer_method, src, dest)
+        end
+      end
+
+      # Called by the Cycler.
+      # Any error raised will be logged as a warning.
+      def remove!(package)
+        Logger.info "Removing backup package dated #{ package.time }..."
+
+        FileUtils.rm_r(remote_path_for(package))
+      end
+
+      # expanded since this is a local path
+      def remote_path(pkg = package)
+        File.expand_path(super)
+      end
+      alias :remote_path_for :remote_path
+
+      ##
+      # If this Local Storage is not the last Storage for the Model,
+      # force the transfer to use a *copy* operation and issue a warning.
+      def package_movable?
+        if self == model.storages.last
+          true
+        else
+          Logger.warn Error.new(<<-EOS)
+            Local File Copy Warning!
+            The final backup file(s) for '#{ model.label }' (#{ model.trigger })
+            will be *copied* to '#{ remote_path }'
+            To avoid this, when using more than one Storage, the 'Local' Storage
+            should be added *last* so the files may be *moved* to their destination.
+          EOS
+          false
+        end
+      end
+
+    end
+  end
+end

data/lib/backup/storage/ninefold.rb

@@ -0,0 +1,74 @@
+# encoding: utf-8
+require 'fog'
+
+module Backup
+  module Storage
+    class Ninefold < Base
+      include Storage::Cycler
+      class Error < Backup::Error; end
+
+      ##
+      # Ninefold Credentials
+      attr_accessor :storage_token, :storage_secret
+
+      def initialize(model, storage_id = nil)
+        super
+
+        @path ||= 'backups'
+        path.sub!(/^\//, '')
+      end
+
+      private
+
+      def connection
+        @connection ||= Fog::Storage.new(
+          :provider                => 'Ninefold',
+          :ninefold_storage_token  => storage_token,
+          :ninefold_storage_secret => storage_secret
+        )
+      end
+
+      def transfer!
+        directory = directory_for(remote_path, true)
+        package.filenames.each do |filename|
+          src = File.join(Config.tmp_path, filename)
+          dest = File.join(remote_path, filename)
+          Logger.info "Storing '#{ dest }'..."
+          File.open(src, 'r') do |file|
+            directory.files.create(:key => filename, :body => file)
+          end
+        end
+      end
+
+      ##
+      # Queries the connection for the directory for the given +remote_path+
+      # Returns nil if not found, or creates the directory if +create+ is true.
+      def directory_for(remote_path, create = false)
+        directory = connection.directories.get(remote_path)
+        if directory.nil? && create
+          directory = connection.directories.create(:key => remote_path)
+        end
+        directory
+      end
+
+      # Called by the Cycler.
+      # Any error raised will be logged as a warning.
+      def remove!(package)
+        Logger.info "Removing backup package dated #{ package.time }..."
+
+        remote_path = remote_path_for(package)
+        directory = directory_for(remote_path)
+
+        raise Error, "Directory at '#{ remote_path }' not found" unless directory
+
+        package.filenames.each do |filename|
+          file = directory.files.get(filename)
+          file.destroy if file
+        end
+
+        directory.destroy
+      end
+
+    end
+  end
+end

data/lib/backup/storage/rsync.rb

@@ -0,0 +1,248 @@
+# encoding: utf-8
+
+module Backup
+  module Storage
+    class RSync < Base
+      include Utilities::Helpers
+
+      ##
+      # Mode of operation
+      #
+      # [:ssh (default)]
+      #   Connects to the remote via SSH.
+      #   Does not use an rsync daemon on the remote.
+      #
+      # [:ssh_daemon]
+      #   Connects to the remote via SSH.
+      #   Spawns a single-use daemon on the remote, which allows certain
+      #   daemon features (like modules) to be used.
+      #
+      # [:rsync_daemon]
+      #   Connects directly to an rsync daemon via TCP.
+      #   Data transferred is not encrypted.
+      #
+      attr_accessor :mode
+
+      ##
+      # Server Address
+      #
+      # If not specified, the storage operation will be local.
+      attr_accessor :host
+
+      ##
+      # SSH or RSync port
+      #
+      # For `:ssh` or `:ssh_daemon` mode, this specifies the SSH port to use
+      # and defaults to 22.
+      #
+      # For `:rsync_daemon` mode, this specifies the TCP port to use
+      # and defaults to 873.
+      attr_accessor :port
+
+      ##
+      # SSH User
+      #
+      # If the user running the backup is not the same user that needs to
+      # authenticate with the remote server, specify the user here.
+      #
+      # The user must have SSH keys setup for passphrase-less access to the
+      # remote. If the SSH User does not have passphrase-less keys, or no
+      # default keys in their `~/.ssh` directory, you will need to use the
+      # `-i` option in `:additional_ssh_options` to specify the
+      # passphrase-less key to use.
+      #
+      # Used only for `:ssh` and `:ssh_daemon` modes.
+      attr_accessor :ssh_user
+
+      ##
+      # Additional SSH Options
+      #
+      # Used to supply a String or Array of options to be passed to the SSH
+      # command in `:ssh` and `:ssh_daemon` modes.
+      #
+      # For example, if you need to supply a specific SSH key for the `ssh_user`,
+      # you would set this to: "-i '/path/to/id_rsa'". Which would produce:
+      #
+      #   rsync -e "ssh -p 22 -i '/path/to/id_rsa'"
+      #
+      # Arguments may be single-quoted, but should not contain any double-quotes.
+      #
+      # Used only for `:ssh` and `:ssh_daemon` modes.
+      attr_accessor :additional_ssh_options
+
+      ##
+      # RSync User
+      #
+      # If the user running the backup is not the same user that needs to
+      # authenticate with the rsync daemon, specify the user here.
+      #
+      # Used only for `:ssh_daemon` and `:rsync_daemon` modes.
+      attr_accessor :rsync_user
+
+      ##
+      # RSync Password
+      #
+      # If specified, Backup will write the password to a temporary file and
+      # use it with rsync's `--password-file` option for daemon authentication.
+      #
+      # Note that setting this will override `rsync_password_file`.
+      #
+      # Used only for `:ssh_daemon` and `:rsync_daemon` modes.
+      attr_accessor :rsync_password
+
+      ##
+      # RSync Password File
+      #
+      # If specified, this path will be passed to rsync's `--password-file`
+      # option for daemon authentication.
+      #
+      # Used only for `:ssh_daemon` and `:rsync_daemon` modes.
+      attr_accessor :rsync_password_file
+
+      ##
+      # Additional String or Array of options for the rsync cli
+      attr_accessor :additional_rsync_options
+
+      ##
+      # Flag for compressing (only compresses for the transfer)
+      attr_accessor :compress
+
+      ##
+      # Path to store the synced backup package file(s) to.
+      #
+      # If no +host+ is specified, then +path+ will be local, and the only
+      # other used option would be +additional_rsync_options+.
+      # +path+ will be expanded, so '~/my_path' will expand to '$HOME/my_path'.
+      #
+      # If a +host+ is specified, this will be a path on the host.
+      # If +mode+ is `:ssh` (default), then any relative path, or path starting
+      # with '~/' will be relative to the directory the ssh_user is logged
+      # into. For `:ssh_daemon` or `:rsync_daemon` modes, this would reference
+      # an rsync module/path.
+      #
+      # In :ssh_daemon and :rsync_daemon modes, +path+ (or path defined by
+      # your rsync module) must already exist.
+      #
+      # In :ssh mode or local operation (no +host+ specified), +path+ will
+      # be created if needed - either locally, or on the remote for :ssh mode.
+      attr_accessor :path
+
+      def initialize(model, storage_id = nil)
+        super
+
+        @mode ||= :ssh
+        @port ||= mode == :rsync_daemon ? 873 : 22
+        @compress ||= false
+        @path ||= '~/backups'
+      end
+
+      private
+
+      def transfer!
+        write_password_file
+        create_remote_path
+
+        package.filenames.each do |filename|
+          src = "'#{ File.join(Config.tmp_path, filename) }'"
+          dest = "#{ host_options }'#{ File.join(remote_path, filename) }'"
+          Logger.info "Syncing to #{ dest }..."
+          run("#{ rsync_command } #{ src } #{ dest }")
+        end
+      ensure
+        remove_password_file
+      end
+
+      ##
+      # Other storages add an additional timestamp directory to this path.
+      # This is not desired here, since we need to transfer the package files
+      # to the same location each time.
+      def remote_path
+        @remote_path ||= begin
+          if host
+            path.sub(/^~\//, '').sub(/\/$/, '')
+          else
+            File.expand_path(path)
+          end
+        end
+      end
+
+      ##
+      # Runs a 'mkdir -p' command on the host (or locally) to ensure the
+      # dest_path exists. This is used because we're transferring a single
+      # file, and rsync won't attempt to create the intermediate directories.
+      #
+      # This is only applicable locally and in :ssh mode.
+      # In :ssh_daemon and :rsync_daemon modes the `path` would include a
+      # module name that must define a path on the remote that already exists.
+      def create_remote_path
+        if host
+          run("#{ utility(:ssh) } #{ ssh_transport_args } #{ host } " +
+                  %Q["mkdir -p '#{ remote_path }'"]) if mode == :ssh
+        else
+          FileUtils.mkdir_p(remote_path)
+        end
+      end
+
+      def host_options
+        @host_options ||= begin
+          if !host
+            ''
+          elsif mode == :ssh
+            "#{ host }:"
+          else
+            user = "#{ rsync_user }@" if rsync_user
+            "#{ user }#{ host }::"
+          end
+        end
+      end
+
+      def rsync_command
+        @rsync_command ||= begin
+          cmd = utility(:rsync) << ' --archive' <<
+              " #{ Array(additional_rsync_options).join(' ') }".rstrip
+          cmd << compress_option << password_option << transport_options if host
+          cmd
+        end
+      end
+
+      def compress_option
+        compress ? ' --compress' : ''
+      end
+
+      def password_option
+        return '' if mode == :ssh
+
+        path = @password_file ? @password_file.path : rsync_password_file
+        path ? " --password-file='#{ File.expand_path(path) }'" : ''
+      end
+
+      def transport_options
+        if mode == :rsync_daemon
+          " --port #{ port }"
+        else
+          %Q[ -e "#{ utility(:ssh) } #{ ssh_transport_args }"]
+        end
+      end
+
+      def ssh_transport_args
+        args = "-p #{ port } "
+        args << "-l #{ ssh_user } " if ssh_user
+        args << Array(additional_ssh_options).join(' ')
+        args.rstrip
+      end
+
+      def write_password_file
+        return unless host && rsync_password && mode != :ssh
+
+        @password_file = Tempfile.new('backup-rsync-password')
+        @password_file.write(rsync_password)
+        @password_file.close
+      end
+
+      def remove_password_file
+        @password_file.delete if @password_file
+      end
+
+    end
+  end
+end