backups-cli 1.0.9

data/lib/backups/base.rb

@@ -0,0 +1,70 @@
+module Backups
+  class Base
+
+    ALL_DATABASES = "all-databases"
+
+    include System
+    include Loader
+
+    def initialize config
+      @config = config
+    end
+
+    def get_timestamp
+      now = Time.new
+      now = now.utc if now.utc?
+      now.iso8601.gsub(':', '-')
+    end
+
+    def get_date_path
+      Time.now.strftime('%Y/%m/%d')
+    end
+
+    def get_prefix
+      return @options['s3']['prefix'] if @options['s3']['prefix']
+      return @config["_name"]
+    end
+
+    def compress source, dest, secret
+      if source.kind_of? Array
+        dir  = File.dirname(source[0])
+        base = source.map {|v| File.basename(v)}.join(' ')
+      else
+        dir  = File.dirname(source)
+        base = File.basename(source)
+      end
+
+      commands  = []
+      commands << "cd #{dir} && zip"
+      commands << "--password #{secret}" if secret
+      commands << "-r #{dest} #{base}"
+
+      exec commands.join(' ')
+    end
+
+    def send_to_s3 bucket, path, filename, options = nil
+      dest = "s3://#{bucket}/#{path}"
+      $LOGGER.info "Sending to #{dest}"
+      exec "aws s3 cp #{filename} #{dest}/"
+
+      return unless options
+      tags = options.fetch('tags', {})
+
+      return unless tags.size > 0
+      key = "#{path}/#{File.basename(filename)}"
+      tag_s3_object bucket, key, tags
+    end
+
+    def tag_s3_object bucket, key, tags
+      tagSet  = tags.map{ |k, v| "{Key=#{k},Value=#{v}}" }.join(",")
+      $LOGGER.info "Tagging s3://#{bucket}/#{key}"
+      $LOGGER.debug "Tagset: #{tagSet}"
+
+      exec "aws s3api put-object-tagging \
+        --bucket #{bucket} \
+        --key #{key} \
+        --tagging 'TagSet=[#{tagSet}]'"
+    end
+
+  end
+end

data/lib/backups/cli.rb

@@ -0,0 +1,51 @@
+require "thor"
+
+module Backups
+  class Cli < Thor
+
+    desc "version", "Show current version"
+    def version
+      puts "v#{VERSION}"
+    end
+
+    desc "ls", "Lists all the configured jobs"
+    def ls
+      Crontab.new.list()
+    end
+
+    desc "show [JOB]", "Shows the merged config (for a JOB or them all)"
+    def show job = nil
+      data = Runner.new.show(job)
+      puts data.to_json
+    end
+
+    desc "start [JOB]", "Starts a backup JOB or all of them"
+    def start job = nil
+      if job
+        Runner.new.start job
+      else
+        Runner.new.start_all
+      end
+    end
+
+    desc "verify [JOB]", "Restores and verifies a backup JOB or all of them"
+    def verify job = nil
+      if job
+        Runner.new.verify job
+      else
+        Runner.new.verify_all
+      end
+    end
+
+    desc "install", "Sets up the crontab for all jobs"
+    def install
+      Crontab.new.install
+    end
+
+    desc "crontab", "Shows the crontab config"
+    def crontab
+      puts Crontab.new.show()
+    end
+
+  end
+end

data/lib/backups/crontab.rb

@@ -0,0 +1,200 @@
+require "tablelize"
+
+module Backups
+  class Crontab
+
+    include System
+    include Loader
+
+    PADDING = 10
+
+    def initialize
+      load_configs
+      @base = File.expand_path('../../..', __FILE__)
+    end
+
+    def list
+      paddings = {
+        'job'    => 20,
+        'backup' => 10,
+        'verify' => 10,
+      }
+
+      rows = []
+      rows << ["JOB", "CRONTAB", "INSTALL", "ENABLED"]
+
+      $GLOBAL["jobs"].each do |job, config|
+        backup   = config.fetch('backup',  {})
+        crontab  = backup.fetch('crontab', {})
+        crontime = get_crontime(crontab)
+        install  = crontab.fetch('install', true)
+        enabled  = config.fetch('enabled', true)
+
+        rows << [job, crontime, install, enabled]
+      end
+
+      Tablelize::table rows
+    end
+
+    def show
+      get_install_lines()
+    end
+
+    def install
+      last_log_active  = $LOG_ACTIVE
+      $LOG_ACTIVE = 0
+
+      install_mysql_groups
+
+      content = get_install_lines()
+      cronfile = "/tmp/crontab.new"
+      write cronfile, content
+      exec "crontab #{cronfile}"
+      exec "rm #{cronfile}"
+      exec "crontab -l"
+
+      $LOG_ACTIVE = last_log_active
+    end
+
+    private
+    def get_install_lines
+      # We dont want passwords in the main log and write does that
+
+      start    = "# BEGIN SECTION Backups\n"
+      finish   = "# END SECTION Backups\n"
+      previous = `crontab -l 2>/dev/null`
+
+      backups = $GLOBAL.fetch("backups", {})
+      crontab = backups.fetch("crontab", {})
+      header  = crontab.fetch("header", "")
+      backup  = crontab.fetch("backup", {})
+      verify  = crontab.fetch("verify", {})
+
+      backup_all = backup.fetch("run-all", false)
+      verify_all = verify.fetch("run-all", false)
+
+      content = "#{header}\n#{previous}" \
+        if header and not previous.include? header
+
+      content << start
+      content << "# Generated at #{Time.now}\n"
+
+      if backup_all
+        content << get_all_backup(backup) + "\n"
+      else
+        content << get_jobs_crontab() + "\n"
+      end
+
+      if verify_all
+        content << get_all_verify(verify) + "\n"
+      end
+
+      content << finish
+    end
+
+    def get_all_backup crontab
+      line     = "#{@base}/bin/backups start"
+      minute   = crontab.fetch("minute", 0)
+      hour     = crontab.fetch("hour", "*")
+      crontime = get_crontime(crontab)
+      prefix   = crontab.fetch("prefix", "")
+      postfix  = crontab.fetch("postfix", "")
+
+      line = "#{prefix} #{line}"  if prefix
+      line = "#{line} #{postfix}" if postfix
+
+      "#{crontime} #{line}"
+    end
+
+    def get_all_verify crontab
+      line     = "#{@base}/bin/backups verify"
+      minute   = crontab.fetch("minute", 0)
+      hour     = crontab.fetch("hour", "*")
+      crontime = get_crontime(crontab)
+      prefix   = crontab.fetch("prefix", "")
+      postfix  = crontab.fetch("postfix", "")
+
+      line = "#{prefix} #{line}"  if prefix
+      line = "#{line} #{postfix}" if postfix
+
+      "#{crontime} #{line}"
+    end
+
+    def get_jobs_crontab
+      contents = []
+
+      $GLOBAL['jobs'].each do |job, config|
+        backup   = config.fetch('backup', {})
+        crontab  = backup.fetch('crontab', {})
+        crontime = get_cronjob(job, 'start', crontab)
+        contents << crontime
+      end
+
+      contents.join("\n")
+    end
+
+    def get_cronjob job, command, crontab
+      return unless crontab.fetch('install', true)
+      line = "#{@base}/bin/backups #{command} #{job}"
+      time = get_crontime(crontab)
+
+      prefix  = crontab.fetch("prefix", "")
+      postfix = crontab.fetch("postfix", "")
+      line = "#{prefix} #{line}" if prefix
+      line = "#{line} #{postfix}" if postfix
+
+      "#{time} #{line}" if not time.nil?
+    end
+
+    def get_crontime crontab
+      time = "#{crontab.fetch('minute',    '0')}"
+      time << " #{crontab.fetch('hour',    '4')}"
+      time << " #{crontab.fetch('day',     '*')}"
+      time << " #{crontab.fetch('month',   '*')}"
+      time << " #{crontab.fetch('weekday', '*')}"
+
+      time.strip()
+    end
+
+    def install_mysql_groups
+      $GLOBAL["jobs"].each do |job, config|
+        install_mysql_group job, config if config["type"].downcase === "mysql"
+      end
+    end
+
+    def install_mysql_group job, config
+      group = "client_" + job.gsub("-", "_")
+      username = config["backup"]["connection"]["username"]
+      password = config["backup"]["connection"]["password"]
+
+      contents = []
+      contents << "[" + group + "]"
+      contents << "user = " + username if username
+      contents << "password = " + password if password
+      replace_mysql_group group, contents.join("\n")
+    end
+
+    def replace_mysql_group group, content
+      path  = File.expand_path("~/.my.cnf")
+      lines = []
+      if File.exist? path
+        found = false
+        File.readlines(path).each do |line|
+          if line === "[#{group}]\n"
+            found = true
+          elsif line.match(/^\[.*\]\n/)
+            found = false
+          end
+          lines << line if not found
+        end
+      end
+
+      contents = lines.join()
+      contents += "\n" if contents.size > 0 and contents[contents.size-1..-1] != "\n"
+      contents += content
+
+      write path, contents
+    end
+
+  end
+end

data/lib/backups/driver/mysql.rb

@@ -0,0 +1,35 @@
+require "mysql2"
+
+module Backups
+  module Driver
+    module Mysql
+
+      def connect
+        @connection[flags: Mysql2::Client::MULTI_STATEMENTS]
+        @mysql = Mysql2::Client.new(@connection)
+      end
+
+      def exec_query(sql)
+        return puts sql if $dry_run
+        connect if not @mysql
+        @mysql.query sql
+      end
+
+      def get_results(sql)
+        return [] if not rset = exec_query(sql)
+        rows = []
+        rset.each do |row|
+          rows << row
+        end
+
+        return rows
+      end
+
+      def get_result(sql)
+        rows = get_results(sql)
+        return rows[0]
+      end
+
+    end
+  end
+end

data/lib/backups/events.rb

@@ -0,0 +1,40 @@
+module Backups
+  class Events
+
+    @@events = {}
+
+    def self.on *events, &block
+      names = []
+      if events.kind_of? Array
+        names = events
+      else
+        names << events
+      end
+
+      names.each do |name|
+        @@events[name] ||= []
+        @@events[name] << block
+      end
+    end
+
+    def self.fire event, params
+      params[:event] = event
+      names = []
+      if event.kind_of? Array
+        names = event
+      else
+        names << event
+      end
+
+      names.each do |name|
+        if @@events.has_key? name
+          @@events[name].each do |cb|
+            res = cb.call(params)
+            break if res == false
+          end
+        end
+      end
+    end
+
+  end
+end

data/lib/backups/ext/fixnum.rb

@@ -0,0 +1,21 @@
+unless Fixnum.method_defined?(:european)
+  class ::Fixnum
+    def european
+      self.to_s.reverse.gsub(/...(?=.)/,'\& ').reverse
+    end
+  end
+end
+
+unless Fixnum.method_defined?(:to_filesize)
+  class ::Fixnum
+    def to_filesize
+      {
+        'B'  => 1024,
+        'KB' => 1024 * 1024,
+        'MB' => 1024 * 1024 * 1024,
+        'GB' => 1024 * 1024 * 1024 * 1024,
+        'TB' => 1024 * 1024 * 1024 * 1024 * 1024
+      }.each_pair {|e, s| return "#{(self.to_f / (s/1024)).round(2)} #{e}" if self < s }
+    end
+  end
+end

data/lib/backups/ext/hash.rb

@@ -0,0 +1,8 @@
+unless Hash.method_defined?(:deep_merge)
+  class ::Hash
+    def deep_merge(second)
+      merger = proc { |key, v1, v2| Hash === v1 && Hash === v2 ? v1.merge(v2, &merger) : v2 }
+      self.merge(second, &merger)
+    end
+  end
+end

data/lib/backups/ext/nil_class.rb

@@ -0,0 +1,7 @@
+unless NilClass.method_defined?(:[])
+  class ::NilClass
+    def [](*args)
+      nil
+    end
+  end
+end

data/lib/backups/ext/ordered_hash.rb

@@ -0,0 +1,11 @@
+unless BSON::OrderedHash.method_defined?(:to_json)
+  class BSON::OrderedHash
+    def to_h
+      inject({}) { |acc, element| k,v = element; acc[k] = (if v.class == BSON::OrderedHash then v.to_h else v end); acc }
+    end
+
+    def to_json
+      to_h.to_json
+    end
+  end
+end