backupii 0.1.0.pre.alpha.1

data/lib/backup/notifier/slack.rb

@@ -0,0 +1,149 @@
+# frozen_string_literal: true
+
+require "uri"
+require "json"
+
+module Backup
+  module Notifier
+    class Slack < Base
+      ##
+      # The incoming webhook url
+      attr_accessor :webhook_url
+
+      ##
+      # The channel to send messages to
+      attr_accessor :channel
+
+      ##
+      # The username to display along with the notification
+      attr_accessor :username
+
+      ##
+      # The emoji icon to display along with the notification
+      #
+      # See http://www.emoji-cheat-sheet.com for a list of icons.
+      #
+      # Default: :floppy_disk:
+      attr_accessor :icon_emoji
+
+      ##
+      # Array of statuses for which the log file should be attached.
+      #
+      # Available statuses are: `:success`, `:warning` and `:failure`.
+      # Default: [:warning, :failure]
+      attr_accessor :send_log_on
+
+      def initialize(model, &block)
+        super
+        instance_eval(&block) if block_given?
+
+        @send_log_on ||= [:warning, :failure]
+        @icon_emoji  ||= ":floppy_disk:"
+      end
+
+      private
+
+      ##
+      # Notify the user of the backup operation results.
+      #
+      # `status` indicates one of the following:
+      #
+      # `:success`
+      # : The backup completed successfully.
+      # : Notification will be sent if `on_success` is `true`.
+      #
+      # `:warning`
+      # : The backup completed successfully, but warnings were logged.
+      # : Notification will be sent if `on_warning` or `on_success` is `true`.
+      #
+      # `:failure`
+      # : The backup operation failed.
+      # : Notification will be sent if `on_warning` or `on_success` is `true`.
+      #
+      def notify!(status)
+        data = {
+          text: message.call(model, status: status_data_for(status)),
+          attachments: [attachment(status)]
+        }
+        [:channel, :username, :icon_emoji].each do |param|
+          val = send(param)
+          data.merge!(param => val) if val
+        end
+
+        options = {
+          headers: { "Content-Type" => "application/x-www-form-urlencoded" },
+          body: URI.encode_www_form(payload: JSON.dump(data))
+        }
+        options[:expects] = 200 # raise error if unsuccessful
+        Excon.post(uri, options)
+      end
+
+      def attachment(status)
+        {
+          fallback: "#{title(status)} - Job: #{model.label} (#{model.trigger})",
+          text: title(status),
+          color: color(status),
+          fields: [
+            {
+              title: "Job",
+              value: "#{model.label} (#{model.trigger})",
+              short: false
+            },
+            {
+              title: "Started",
+              value: model.started_at,
+              short: true
+            },
+            {
+              title: "Finished",
+              value: model.finished_at,
+              short: true
+            },
+            {
+              title: "Duration",
+              value: model.duration,
+              short: true
+            },
+            {
+              title: "Version",
+              value: "Backup v#{Backup::VERSION}\nRuby: #{RUBY_DESCRIPTION}",
+              short: false
+            },
+            log_field(status)
+          ].compact
+        }
+      end
+
+      def log_field(status)
+        send_log = send_log_on.include?(status)
+        return unless send_log
+
+        {
+          title: "Detailed Backup Log",
+          value: Logger.messages.map(&:formatted_lines).flatten.join("\n"),
+          short: false
+        }
+      end
+
+      def color(status)
+        case status
+        when :success then "good"
+        when :failure then "danger"
+        when :warning then "warning"
+        end
+      end
+
+      def title(status)
+        case status
+        when :success then "Backup Completed Successfully!"
+        when :failure then "Backup Failed!"
+        when :warning then "Backup Completed Successfully (with Warnings)!"
+        end
+      end
+
+      def uri
+        @uri ||= webhook_url
+      end
+    end
+  end
+end

data/lib/backup/notifier/twitter.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require "twitter"
+
+module Backup
+  module Notifier
+    class Twitter < Base
+      ##
+      # Twitter consumer key credentials
+      attr_accessor :consumer_key, :consumer_secret
+
+      ##
+      # OAuth credentials
+      attr_accessor :oauth_token, :oauth_token_secret
+
+      def initialize(model, &block)
+        super
+        instance_eval(&block) if block_given?
+      end
+
+      private
+
+      ##
+      # Notify the user of the backup operation results.
+      #
+      # `status` indicates one of the following:
+      #
+      # `:success`
+      # : The backup completed successfully.
+      # : Notification will be sent if `on_success` is `true`.
+      #
+      # `:warning`
+      # : The backup completed successfully, but warnings were logged.
+      # : Notification will be sent if `on_warning` or `on_success` is `true`.
+      #
+      # `:failure`
+      # : The backup operation failed.
+      # : Notification will be sent if `on_warning` or `on_success` is `true`.
+      #
+      def notify!(status)
+        send_message(message.call(model, status: status_data_for(status)))
+      end
+
+      # Twitter::Client will raise an error if unsuccessful.
+      def send_message(message)
+        client = ::Twitter::REST::Client.new do |config|
+          config.consumer_key        = @consumer_key
+          config.consumer_secret     = @consumer_secret
+          config.access_token        = @oauth_token
+          config.access_token_secret = @oauth_token_secret
+        end
+
+        client.update(message)
+      end
+    end
+  end
+end

data/lib/backup/notifier/zabbix.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module Backup
+  module Notifier
+    class Zabbix < Base
+      attr_accessor :zabbix_host
+
+      attr_accessor :zabbix_port
+
+      attr_accessor :service_name
+
+      attr_accessor :service_host
+
+      attr_accessor :item_key
+
+      def initialize(model, &block)
+        super
+        instance_eval(&block) if block_given?
+
+        @zabbix_host  ||= Config.hostname
+        @zabbix_port  ||= 10_051
+        @service_name ||= "Backup #{model.trigger}"
+        @service_host ||= Config.hostname
+        @item_key     ||= "backup_status"
+      end
+
+      private
+
+      ##
+      # Notify the user of the backup operation results.
+      #
+      # `status` indicates one of the following:
+      #
+      # `:success`
+      # : The backup completed successfully.
+      # : Notification will be sent if `on_success` is `true`.
+      #
+      # `:warning`
+      # : The backup completed successfully, but warnings were logged.
+      # : Notification will be sent if `on_warning` or `on_success` is `true`.
+      #
+      # `:failure`
+      # : The backup operation failed.
+      # : Notification will be sent if `on_warning` or `on_success` is `true`.
+      #
+      def notify!(status)
+        send_message(message.call(model, status: status_data_for(status)))
+      end
+
+      def send_message(message)
+        msg = [service_host, service_name, model.exit_status, message].join("\t")
+        cmd = utility(:zabbix_sender).to_s +
+          " -z '#{zabbix_host}'" \
+          " -p '#{zabbix_port}'" \
+          " -s #{service_host}"  \
+          " -k #{item_key}"      \
+          " -o '#{msg}'"
+        run("echo '#{msg}' | #{cmd}")
+      end
+    end
+  end
+end

data/lib/backup/package.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Backup
+  class Package
+    ##
+    # The time when the backup initiated (in format: 2011.02.20.03.29.59)
+    attr_accessor :time
+
+    ##
+    # The trigger which initiated the backup process
+    attr_reader :trigger
+
+    ##
+    # Extension for the final archive file(s)
+    attr_accessor :extension
+
+    ##
+    # Set by the Splitter if the final archive was "chunked"
+    attr_accessor :chunk_suffixes
+
+    ##
+    # If true, the Cycler will not attempt to remove the package when Cycling.
+    attr_accessor :no_cycle
+
+    ##
+    # The version of Backup used to create the package
+    attr_reader :version
+
+    def initialize(model)
+      @trigger = model.trigger.dup
+      @extension = "tar".dup
+      @chunk_suffixes = []
+      @no_cycle = false
+      @version = VERSION
+    end
+
+    def filenames
+      if chunk_suffixes.empty?
+        [basename]
+      else
+        chunk_suffixes.map { |suffix| "#{basename}-#{suffix}" }
+      end
+    end
+
+    def basename
+      "#{trigger}.#{extension}"
+    end
+
+    def time_as_object
+      Time.strptime(time, "%Y.%m.%d.%H.%M.%S")
+    end
+  end
+end

data/lib/backup/packager.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+module Backup
+  module Packager
+    class Error < Backup::Error; end
+
+    class << self
+      include Utilities::Helpers
+
+      ##
+      # Build the final package for the backup model.
+      def package!(model)
+        @package   = model.package
+        @encryptor = model.encryptor
+        @splitter  = model.splitter
+        @pipeline  = Pipeline.new
+
+        Logger.info "Packaging the backup files..."
+        procedure.call
+
+        if @pipeline.success?
+          Logger.info "Packaging Complete!"
+        else
+          raise Error, "Failed to Create Backup Package\n" +
+            @pipeline.error_messages
+        end
+      end
+
+      private
+
+      ##
+      # Builds a chain of nested Procs which adds each command to a Pipeline
+      # needed to package the final command to package the backup.
+      # This is done so that the Encryptor and Splitter have the ability
+      # to perform actions before and after the final command is executed.
+      # No Encryptors currently utilize this, however the Splitter does.
+      def procedure
+        stack = []
+
+        ##
+        # Initial `tar` command to package the temporary backup folder.
+        # The command's output will then be either piped to the Encryptor
+        # or the Splitter (if no Encryptor), or through `cat` into the final
+        # output file if neither are configured.
+        @pipeline.add(
+          "#{utility(:tar)} -cf - " \
+          "-C '#{Config.tmp_path}' '#{@package.trigger}'",
+          tar_success_codes
+        )
+
+        ##
+        # If an Encryptor was configured, it will be called first
+        # to add the encryption utility command to be piped through,
+        # and amend the final package extension.
+        # It's output will then be either piped into a Splitter,
+        # or through `cat` into the final output file.
+        if @encryptor
+          stack << lambda do
+            @encryptor.encrypt_with do |command, ext|
+              @pipeline << command
+              @package.extension << ext
+              stack.shift.call
+            end
+          end
+        end
+
+        ##
+        # If a Splitter was configured, the `split` utility command will be
+        # added to the Pipeline to split the final output into multiple files.
+        # Once the Proc executing the Pipeline has completed and returns back
+        # to the Splitter, it will check the final output files to determine
+        # if the backup was indeed split.
+        # If so, it will set the package's chunk_suffixes. If not, it will
+        # remove the '-aa' suffix from the only file created by `split`.
+        #
+        # If no Splitter was configured, the final file output will be
+        # piped through `cat` into the final output file.
+        stack <<
+          if @splitter
+            lambda do
+              @splitter.split_with do |command|
+                @pipeline << command
+                stack.shift.call
+              end
+            end
+          else
+            lambda do
+              outfile = File.join(Config.tmp_path, @package.basename)
+              @pipeline << "#{utility(:cat)} > #{outfile}"
+              stack.shift.call
+            end
+          end
+
+        ##
+        # Last Proc to be called runs the Pipeline the procedure built.
+        # Once complete, the call stack will unwind back through the
+        # preceeding Procs in the stack (if any)
+        stack << -> { @pipeline.run }
+
+        stack.shift
+      end
+
+      def tar_success_codes
+        gnu_tar? ? [0, 1] : [0]
+      end
+    end
+  end
+end

data/lib/backup/pipeline.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+module Backup
+  class Pipeline
+    class Error < Backup::Error; end
+
+    include Utilities::Helpers
+
+    attr_reader :stderr, :errors
+
+    def initialize
+      @commands = []
+      @success_codes = []
+      @errors = []
+      @stderr = ""
+    end
+
+    ##
+    # Adds a command to be executed in the pipeline.
+    # Each command will be run in the order in which it was added,
+    # with it's output being piped to the next command.
+    #
+    # +success_codes+ must be an Array of Integer exit codes that will
+    # be considered successful for the +command+.
+    def add(command, success_codes)
+      @commands << command
+      @success_codes << success_codes
+    end
+
+    ##
+    # Commands added using this method will only be considered successful
+    # if their exit status is 0.
+    #
+    # Use #add if successful exit status codes need to be specified.
+    def <<(command)
+      add(command, [0])
+    end
+
+    ##
+    # Runs the command line from `#pipeline` and collects STDOUT/STDERR.
+    # STDOUT is then parsed to determine the exit status of each command.
+    # For each command with a non-zero exit status, a SystemCallError is
+    # created and added to @errors. All STDERR output is set in @stderr.
+    #
+    # Note that there is no accumulated STDOUT from the commands themselves.
+    # Also, the last command should not attempt to write to STDOUT.
+    # Any output on STDOUT from the final command will be sent to STDERR.
+    # This in itself will not cause #run to fail, but will log warnings
+    # when all commands exit with non-zero status.
+    #
+    # Use `#success?` to determine if all commands in the pipeline succeeded. If
+    # `#success?` returns `false`, use `#error_messages` to get an error report.
+    def run
+      Open4.popen4(pipeline) do |_pid, _stdin, stdout, stderr|
+        pipestatus = stdout.read.delete("\n").split(":").sort
+        pipestatus.each do |status|
+          index, exitstatus = status.split("|").map(&:to_i)
+          next if @success_codes[index].include?(exitstatus)
+
+          command = command_name(@commands[index])
+          @errors << SystemCallError.new(
+            "'#{command}' returned exit code: #{exitstatus}", exitstatus
+          )
+        end
+        @stderr = stderr.read.strip
+      end
+      Logger.warn(stderr_messages) if success? && stderr_messages
+    rescue Exception => err
+      raise Error.wrap(err, "Pipeline failed to execute")
+    end
+
+    def success?
+      @errors.empty?
+    end
+
+    ##
+    # Returns a multi-line String, reporting all STDERR messages received
+    # from the commands in the pipeline (if any), along with the SystemCallError
+    # (Errno) message for each command which had a non-zero exit status.
+    def error_messages
+      @error_messages ||= (stderr_messages || "") +
+        "The following system errors were returned:\n" +
+        @errors.map { |err| "#{err.class}: #{err.message}" }.join("\n")
+    end
+
+    private
+
+    ##
+    # Each command is added as part of the pipeline, grouped with an `echo`
+    # command to pass along the command's index in @commands and it's exit
+    # status. The command's STDERR is redirected to FD#4, and the `echo` command
+    # to report the "index|exit status" is redirected to FD#3. Each command's
+    # STDOUT will be connected to the STDIN of the next subshell. The entire
+    # pipeline is run within a container group, which redirects FD#3 to STDOUT
+    # and FD#4 to STDERR so these can be collected. FD#1 is redirected to STDERR
+    # so that any output from the final command on STDOUT will generate
+    # warnings, since the final command should not attempt to write to STDOUT,
+    # as this would interfere with collecting the exit statuses.
+    #
+    # There is no guarantee as to the order of this output, which is why the
+    # command's index in @commands is passed along with it's exit status. And,
+    # if multiple commands output messages on STDERR, those messages may be
+    # interleaved. Interleaving of the "index|exit status" outputs should not be
+    # an issue, given the small byte size of the data being written.
+    def pipeline
+      parts = []
+      @commands.each_with_index do |command, index|
+        parts << %({ #{command} 2>&4 ; echo "#{index}|$?:" >&3 ; })
+      end
+      %({ #{parts.join(" | ")} } 3>&1 1>&2 4>&2)
+    end
+
+    def stderr_messages
+      @stderr_messages ||= @stderr.empty? ? false : <<-EOS.gsub(%r{^ +}, "  ")
+        Pipeline STDERR Messages:
+        (Note: may be interleaved if multiple commands returned error messages)
+
+        #{@stderr}
+      EOS
+    end
+  end
+end

data/lib/backup/splitter.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module Backup
+  class Splitter
+    include Utilities::Helpers
+
+    attr_reader :package, :chunk_size, :suffix_length
+
+    def initialize(model, chunk_size, suffix_length)
+      @package = model.package
+      @chunk_size = chunk_size
+      @suffix_length = suffix_length
+    end
+
+    ##
+    # This is called as part of the procedure used to build the final
+    # backup package file(s). It yields it's portion of the command line
+    # for this procedure, which will split the data being piped into it
+    # into multiple files, based on the @chunk_size, using a suffix length as
+    # specified by @suffix_length.
+    # Once the packaging procedure is complete, it will return and
+    # @package.chunk_suffixes will be set based on the resulting files.
+    def split_with
+      Logger.info "Splitter configured with a chunk size of #{chunk_size}MB " \
+                  "and suffix length of #{suffix_length}."
+      yield split_command
+      after_packaging
+    end
+
+    private
+
+    ##
+    # The `split` command reads from $stdin and will store it's output in
+    # multiple files, based on @chunk_size and @suffix_length, using the full
+    # path to the final @package.basename, plus a '-' separator as the `prefix`.
+    def split_command
+      "#{utility(:split)} -a #{suffix_length} -b #{chunk_size}m - " \
+          "'#{File.join(Config.tmp_path, package.basename + "-")}'"
+    end
+
+    ##
+    # Finds the resulting files from the packaging procedure
+    # and stores an Array of suffixes used in @package.chunk_suffixes.
+    # If the @chunk_size was never reached and only one file
+    # was written, that file will be suffixed with '-aa' (or -a; -aaa; etc
+    # depending upon suffix_length). In which case, it will simply
+    # remove the suffix from the filename.
+    def after_packaging
+      suffixes = chunk_suffixes
+      first_suffix = "a" * suffix_length
+      if suffixes == [first_suffix]
+        FileUtils.mv(
+          File.join(Config.tmp_path, "#{package.basename}-#{first_suffix}"),
+          File.join(Config.tmp_path, package.basename)
+        )
+      else
+        package.chunk_suffixes = suffixes
+      end
+    end
+
+    ##
+    # Returns an array of suffixes for each chunk, in alphabetical order.
+    # For example: [aa, ab, ac, ad, ae] or [aaa, aab, aac aad]
+    def chunk_suffixes
+      chunks.map { |chunk| File.extname(chunk).split("-").last }.sort
+    end
+
+    ##
+    # Returns an array of full paths to the backup chunks.
+    # Chunks are sorted in alphabetical order.
+    def chunks
+      Dir[File.join(Config.tmp_path, package.basename + "-*")].sort
+    end
+  end
+end

data/lib/backup/storage/base.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module Backup
+  module Storage
+    class Base
+      include Config::Helpers
+
+      ##
+      # Base path on the remote where backup package files will be stored.
+      attr_reader :path
+
+      ##
+      # Number of backups to keep or time until which to keep.
+      #
+      # If an Integer is given it sets the limit to how many backups to keep in
+      # the remote location. If exceeded, the oldest will be removed to make
+      # room for the newest.
+      #
+      # If a Time object is given it will remove backups _older_ than the given
+      # date.
+      #
+      # @!attribute [rw] keep
+      #   @param [Integer|Time]
+      #   @return [Integer|Time]
+      attr_accessor :keep
+
+      attr_reader :model, :package, :storage_id
+
+      ##
+      # +storage_id+ is a user-defined string used to uniquely identify
+      # multiple storages of the same type. If multiple storages of the same
+      # type are added to a single backup model, this identifier must be set.
+      # This will be appended to the YAML storage file used for cycling backups.
+      def initialize(model, storage_id = nil, &block)
+        @model = model
+        @package = model.package
+        @storage_id = storage_id.to_s.gsub(%r{\W}, "_") if storage_id
+
+        load_defaults!
+        instance_eval(&block) if block_given?
+      end
+
+      def path=(value)
+        @path = value.dup
+      end
+
+      def perform!
+        Logger.info "#{storage_name} Started..."
+        transfer!
+        if respond_to?(:cycle!, true) && (keep.to_i > 0 || keep.is_a?(Time))
+          cycle!
+        end
+        Logger.info "#{storage_name} Finished!"
+      end
+
+      private
+
+      ##
+      # Return the remote path for the current or given package.
+      def remote_path(pkg = package)
+        path.empty? ? File.join(pkg.trigger, pkg.time) :
+                      File.join(path, pkg.trigger, pkg.time)
+      end
+      alias remote_path_for remote_path
+
+      def storage_name
+        @storage_name ||= self.class.to_s.sub("Backup::", "") +
+          (storage_id ? " (#{storage_id})" : "")
+      end
+    end
+  end
+end