backups-cli 1.0.9

data/lib/backups/ext/string.rb

@@ -0,0 +1,20 @@
+unless String.method_defined?(:to_bool)
+  class String
+    def to_bool
+      return true if self == true || self =~ (/^(true|t|yes|y|1)$/i)
+      return false if self == false || self.blank? || self =~ (/^(false|f|no|n|0)$/i)
+      raise ArgumentError.new("invalid value for Boolean: \"#{self}\"")
+    end
+  end
+end
+
+unless String.method_defined?(:squish!)
+  class String
+    def squish!
+      gsub!(/\A[[:space:]]+/, '')
+      gsub!(/[[:space:]]+\z/, '')
+      gsub!(/[[:space:]]+/, ' ')
+      self
+    end
+  end
+end

data/lib/backups/listener.rb

@@ -0,0 +1,19 @@
+module Backups
+  class Listener
+
+    def self.listen *names, &block
+      names.each do |name|
+        if block_given?
+          Events.on name do |params|
+            block.call(params)
+          end
+        else
+          Events.on name do |params|
+            notify params
+          end
+        end
+      end
+    end
+
+  end
+end

data/lib/backups/listeners/notify/datadog.rb

@@ -0,0 +1,44 @@
+require "dogapi"
+
+module Backups
+  module Listeners
+    module Notify
+      class Datadog < Listener
+
+        def initialize config
+          @config = config
+          api_key = @config.fetch("api_key")
+          app_key = @config.fetch("app_key")
+
+          @dog = Dogapi::Client.new(api_key, app_key)
+
+          $LOGGER.debug "Datadog listener started"
+          Events::on :start, :done, :error do |params|
+            _notify params
+          end
+        end
+
+        private
+        def _notify params
+          metric = @config.fetch("metric", "monitoring.backups")
+          tags   = _flatten(params.fetch(:config, {}).fetch("tags", {}))
+          tags << "type:#{params[:event]}"
+          tags << "job:#{params[:job]}"
+
+          status, response = @dog.emit_point(metric, 1, tags: tags)
+          abort "Failed to send metric to Datadog." unless status == "202"
+        end
+
+        def _flatten tags
+          values = []
+          tags.each do |k, v|
+            next if v == nil
+            values << "#{k}:#{v}"
+          end
+          values
+        end
+      end
+
+    end
+  end
+end

data/lib/backups/listeners/notify/slack.rb

@@ -0,0 +1,121 @@
+require "slack-notifier"
+
+module Backups
+  module Listeners
+    module Notify
+      class Slack < Listener
+
+        def initialize config
+          @config = config
+
+          webhook  = config.fetch("webhook")
+          channel  = config.fetch("channel", "backups")
+          username = config.fetch("username", "backups-cli")
+          events   = config.fetch("events", "all")
+          active   = config.fetch("active", true)
+
+          unless active
+            $LOGGER.debug "Slack listener not active"
+            return
+          end
+
+          @slack = ::Slack::Notifier.new webhook, channel: channel, username: username
+          $LOGGER.debug "Slack listener initialized"
+
+          if events == "all" or events.include? "start"
+            $LOGGER.debug "Slack listening to start events"
+            Events::on :start do |params|
+              _start params
+            end
+          end
+
+          if events == "all" or events.include? "complete"
+            $LOGGER.debug "Slack listening to complete events"
+            Events::on :done do |params|
+              _done params
+            end
+          end
+
+          if events == "all" or events.include? "error"
+            $LOGGER.debug "Slack listening to error events"
+            Events::on :error do |params|
+              _error params
+            end
+          end
+        end
+
+        private
+        def _tags attachment, params
+          params.fetch(:config, {}).fetch("tags", {}).each do |k,v|
+            next if v == nil
+            attachment[:fields] << {
+              title: "Tag: #{k}",
+              value: v,
+              short: true,
+            }
+          end
+        end
+
+        def _done params
+          details = params.fetch(:details)
+          notes   = {
+            color:  "good",
+            fields: [
+              {
+                title: "Size",
+                value: "#{details[:size].to_filesize} (#{details[:size]} bytes)",
+                short: true,
+              },
+              {
+                title: "Took",
+                value: "#{(details[:completed] - details[:started]).round(2)} seconds",
+                short: true,
+              },
+              {
+                title: "File",
+                value: "<#{details[:view]}|#{details[:url]}>",
+              },
+            ],
+          }
+
+          _tags notes, params
+          _send "Backup job `#{params[:job]}` is complete.", [notes]
+        end
+
+        def _error params
+          error = params.fetch(:error, "Something went wrong but I don't know what.")
+          notes = {
+            color:    "danger",
+            fallback: error,
+            fields:   [
+              {
+                title: "Error",
+                value: error,
+              },
+            ],
+          }
+
+          _tags notes, params
+          _send "Backup job `#{params[:job]}` failed.", [notes]
+        end
+
+        def _start params
+          notes = { fields: [] }
+          msg = "Backup job `#{params[:job]}` has started.".encode!("UTF-8", {undef: :replace})
+
+          _tags notes, params
+          _send msg, [notes]
+        end
+
+        def _send message, attachments
+          res = @slack.ping message, attachments: attachments
+          $LOGGER.debug "Event sent to Slack: #{message}"
+          # if res.code
+          #   $LOGGER.debug "Slack result: #{res.code} #{res.message}"
+          # end
+        end
+
+      end
+    end
+  end
+end

data/lib/backups/loader.rb

@@ -0,0 +1,55 @@
+require "yaml"
+
+module Backups
+  module Loader
+
+    CONFIG_ENV    = "BACKUPS_CONFIG_DIR"
+    CONFIG_USER   = "~/.backups-cli"
+    CONFIG_SYSTEM = "/etc/backups-cli"
+
+    def load_configs
+      config_dir = find_dir()
+      load_files config_dir
+    end
+
+    private
+    def load_files dir
+      $GLOBAL = {
+        "backups"  => {},
+        "defaults" => {},
+        "jobs"     => {},
+      }
+
+      # First, we dep merge all config into a single config
+      Dir["#{dir}/**/*.yaml"].each do |file|
+        $GLOBAL = $GLOBAL.deep_merge(YAML.load_file(file))
+      end
+
+      # Second, we apply the defaults config to all jobs
+      $GLOBAL["jobs"].each do |name, config|
+        $GLOBAL["jobs"][name] = $GLOBAL["defaults"].deep_merge(config)
+        $GLOBAL["jobs"][name]["_name"] = name
+      end
+
+      File.write("#{dir}/merged.compiled-yaml", $GLOBAL.to_yaml)
+    end
+
+    def find_dir
+      dirs = []
+      dirs << ENV.fetch(CONFIG_ENV) if ENV[CONFIG_ENV]
+      dirs << File.expand_path(CONFIG_USER) if ENV["HOME"]
+      dirs << CONFIG_SYSTEM
+
+      dirs.each do |dir|
+        return File.realpath(dir) if File.directory? dir
+      end
+
+      raise RuntimeError, <<-EOS.squish!
+        The config directory could not be found. You need to either set
+        the BACKUPS_CONFIG_DIR env var to a valid directory or create either a
+        #{CONFIG_USER} or a #{CONFIG_SYSTEM} directory.
+      EOS
+    end
+
+  end
+end

data/lib/backups/logger.rb

@@ -0,0 +1,8 @@
+module Backups
+  class Logger
+
+    def initialize
+    end
+
+  end
+end

data/lib/backups/runner.rb

@@ -0,0 +1,137 @@
+module Backups
+  class Runner
+
+    ADAPTER_PREFIX = "Backups::Adapter::"
+    VERIFY_PREFIX  = "Backups::Verify::"
+
+    include System
+    include Loader
+
+    def initialize
+      load_configs
+      load_listeners
+    end
+
+    def start_all
+      $LOGGER.info "Starting all jobs sequentially"
+      $GLOBAL["jobs"].each do |name, config|
+        start name
+      end
+    end
+
+    def verify_all
+      $LOGGER.info "Verifying all jobs sequentially"
+      $GLOBAL["jobs"].each do |name, config|
+        verify name
+      end
+    end
+
+    def start job
+      $LOGGER.progname = job
+      $LOGGER.info "Started backup"
+
+      config = _load_config(job)
+
+      Events.fire :start, {
+        job:    job,
+        config: config,
+      }
+
+      details = _backup(job, config)
+
+      Events.fire :done, {
+        type:    :backup,
+        job:     job,
+        config:  config,
+        details: details,
+      }
+
+      $LOGGER.info "Completed backup"
+    end
+
+    def verify job
+      $LOGGER.progname = job
+      $LOGGER.info "Started verification"
+
+      config = _load_config(job)
+
+      Events.fire :start, {
+        job:    job,
+        config: config,
+      }
+
+      details = _verify(job, config)
+
+      Events.fire :done, {
+        type:    :verify,
+        job:     job,
+        config:  config,
+        details: details,
+      }
+
+      $LOGGER.info "Completed verification"
+    end
+
+    def show job = nil
+      return _load_config(job) if job
+      $GLOBAL
+    end
+
+    private
+    def _load_config job
+      config = $GLOBAL["jobs"][job]
+      raise "Job #{job} is not defined." unless config
+      raise "Job #{job} is not enabled." unless config.fetch("enabled", true)
+      raise "Job #{job} has no type."    unless config["type"]
+
+      config
+    end
+
+    def _backup job, config
+      type  = config["type"].capitalize
+      klass = class_for_name("#{ADAPTER_PREFIX}#{type}")
+      raise RuntimeError, "Could not load the #{type} adapter." unless klass
+      adapter = klass.new(config)
+      details = adapter.run()
+
+      ### Test the verify step immediately after
+      # details["verify"] = _verify(job, config)
+
+      details
+    end
+
+    def _verify job, config
+      type  = config["type"].capitalize
+      klass = class_for_name("#{VERIFY_PREFIX}#{type}")
+      raise RuntimeError, "Could not load the #{type} verify." unless klass
+      adapter = klass.new(config)
+      adapter.verify()
+    end
+
+    def class_for_name name
+      name.split("::").inject(Object) { |o, c| o.const_get c }
+    end
+
+    def load_listeners
+      Dir["#{File.dirname(__FILE__)}/listeners/**/*.rb"].each do |file|
+        require file
+
+        name = file.gsub("#{File.dirname(__FILE__)}/listeners/", "")
+        name = name[0..-4]
+        full = ""
+        name.split("/").each do |v|
+          full += "::#{v.capitalize}"
+        end
+
+        listeners = $GLOBAL["backups"].fetch("listeners", {})
+        # search    = name.gsub("/", ".")
+        search    = File.basename(name)
+        config    = listeners.fetch(search, {})
+
+        klass = class_for_name("Backups::Listeners#{full}")
+        listn = klass.new(config)
+      end
+    end
+
+  end
+end

data/lib/backups/stats/mysql.rb

@@ -0,0 +1,80 @@
+module Backups
+  module Stats
+    module Mysql
+
+      include ::Backups::Driver::Mysql
+
+      def get_server_tables(databases = nil)
+        query = <<-EOS
+          SELECT
+                  TABLE_SCHEMA `database`,
+                  TABLE_NAME `table`
+          FROM
+                  INFORMATION_SCHEMA.TABLES
+          WHERE
+                  TABLE_SCHEMA NOT IN ('#{get_excluded_schemas.join("', '")}')
+        EOS
+
+        query += "AND TABLE_SCHEMA IN ('#{databases.join("', '")}')" if databases
+        query += "ORDER BY TABLE_SCHEMA ASC"
+
+        get_results(query)
+      end
+
+      def get_database_tables(database)
+        query = <<-EOS
+          SELECT
+                  TABLE_NAME `table`
+          FROM
+                  INFORMATION_SCHEMA.TABLES
+          WHERE
+                  TABLE_SCHEMA = '#{database}'
+          EOS
+
+        get_results(query)
+      end
+
+      def get_database_stats(database)
+        tables = get_database_tables(database)
+        stats  = {}
+        tables.each do |item|
+          table = item['table']
+
+          stats[table] = get_table_stats(database, table)
+          $LOGGER.debug "Writing stats for table #{database}.#{table}: #{stats[table]}"
+        end
+
+        return stats
+      end
+
+      def get_excluded_schemas
+        ["information_schema", "performance_schema", "mysql"]
+      end
+
+      def get_database_names
+        query = <<-EOS
+          SELECT
+                  SCHEMA_NAME `database`
+          FROM
+                  INFORMATION_SCHEMA.SCHEMATA
+          WHERE
+                  SCHEMA_NAME NOT IN ('#{get_excluded_schemas.join("', '")}')
+        EOS
+
+        get_results(query)
+      end
+
+      def get_table_stats(database, table)
+        query = <<-EOS
+          SELECT
+                  COUNT(*) `rows`
+          FROM
+                  `#{database}`.`#{table}`
+        EOS
+
+        get_result(query)
+      end
+
+    end
+  end
+end