backups-cli 1.0.9

data/lib/backups/system.rb

@@ -0,0 +1,54 @@
+module Backups
+  module System
+
+    def exec command
+      # $LOGGER.debug "Running #{command}" if $LOG_ACTIVE == 1
+      return if $DRY_RUN
+
+      output = `#{command}`
+      if $?.exitstatus != 0
+        raise RuntimeError, \
+          "Command '#{command}' failed with exit code #{$?.exitstatus}."
+      end
+
+      return output.chomp()
+    end
+
+    def delete file
+      exec "rm #{file}"
+    end
+
+    def delete_dir start, stop = nil
+      return exec "rmdir #{start}" unless stop
+      stop = stop.chomp("/")
+      loop do
+        exec "rmdir #{start}" if File.directory? start
+        break if start == stop
+        start = File.dirname(start)
+      end
+    end
+
+    def nuke_dir dir
+      exec "rm -fr #{dir}"
+    end
+
+    def mkdir dirname
+      exec "mkdir -p #{dirname}"
+    end
+
+    def write filename, contents
+      commands = []
+      commands << "cat << CONTENTS > #{filename}"
+      commands << contents
+      commands << "CONTENTS"
+
+      exec commands.join("\n")
+    end
+
+    def get_latest_s3 path
+      # $LOGGER.debug  "aws s3 ls #{path}/|awk '{ print $4 }'|tail -n 1"
+      exec "aws s3 ls #{path}/|awk '{ print $4 }'|tail -n 1"
+    end
+
+  end
+end

data/lib/backups/verify/mysql.rb

@@ -0,0 +1,180 @@
+require "yaml"
+
+module Backups
+  module Verify
+    class Mysql < Base
+
+      VERIFY_PREFIX = "__verify__"
+
+      include ::Backups::Stats::Mysql
+
+      def verify
+        setup
+        download
+        import
+        clean_up
+        report
+
+        return @details
+      end
+
+      private
+      def setup
+        load_configs
+
+        @compared   = 0
+        @failures   = 0
+        @compsize   = nil
+        @datedir    = get_date_path()
+        @backups    = $GLOBAL.fetch("backups", {})
+        @name       = @config["_name"]
+        @verify     = @config.fetch("verify", {})
+        @connection = @verify.fetch("connection", {})
+        @secret     = @config.fetch("encryption", {}).fetch("secret", "")
+        @paths      = @backups.fetch("paths", {})
+        @verifydir  = @paths.fetch("verify", "/tmp/verify")
+        @jobdir     = "#{@verifydir}/#{@name}"
+        @s3bucket   = "#{@config["s3"]["bucket"]}"
+        @s3path     = "#{@config["s3"]["path"]}"
+        @s3fullpath = "s3://#{@s3bucket}/#{@s3path}/#{@name}/#{@datedir}"
+        @package    = get_latest_s3(@s3fullpath)
+
+        raise RuntimeError, "Error: Could not found the latest package in #{@s3fullpath}." \
+          unless @package.length > 1
+
+        @s3file     = "#{@s3fullpath}/#{@package}"
+        @basename   = File.basename(@package).gsub(".zip", "")
+        @download   = "#{@jobdir}/#{@package}"
+        @timedir    = @download.gsub(".zip", "")
+
+        @details    = {
+          started:   Time.now,
+          completed: nil,
+          file:      nil,
+          size:      nil,
+          report:    nil,
+        }
+
+        $LOGGER.info "Preparing #{@jobdir}"
+        # We clean up previous imports as we are less strict with them clearing
+        # up cleanly.
+        exec "rm -fr #{@jobdir}"
+        exec "mkdir -p #{@jobdir}"
+      end
+
+      def download
+        $LOGGER.info "Downloading #{@s3file}"
+        exec "aws s3 cp #{@s3file} #{@download}"
+        @compsize = File.size(@download) if not $DRY_RUN
+        exec "cd #{@jobdir} && unzip -P #{@secret} #{@download}"
+        $LOGGER.info "Downloaded #{@compsize} bytes"
+      end
+
+      def import
+        files = get_import_files()
+
+        if files == []
+          $LOGGER.fatal "Error: There were no files in #{@timedir} to import."
+          return
+        end
+
+        # Are we importing the whole server?
+        if File.basename(files[0], ".sql") == ALL_DATABASES
+          file = files[0]
+          import_server file
+          check_stats file
+        else
+          files.each do |file|
+            db = File.basename(file, ".sql")
+            dbname = "#{VERIFY_PREFIX}#{db}"
+            import_database dbname, file
+            check_database_stats dbname, file
+            drop_database dbname
+          end
+        end
+      end
+
+      def import_server file
+        $LOGGER.info "Importing server from #{file}"
+        exec "mysql < #{file}"
+      end
+
+      def import_database db, file
+        $LOGGER.info "Importing database #{db} from #{file}"
+        exec "mysql -e 'DROP DATABASE IF EXISTS #{db}'"
+        exec "mysql -e 'CREATE DATABASE #{db}'"
+        exec "mysql #{db} < #{file}"
+      end
+
+      def get_import_files
+        Dir["#{@timedir}/*.sql"]
+      end
+
+      def check_database_stats db, file
+        # return if db == "mysql"
+
+        file = file.gsub(".sql", "-stats.yaml")
+        data = YAML.load_file(file)
+
+        data.each do |table, stats|
+          object = "#{db}.#{table}"
+          check_table db, table
+
+          imported = get_table_count(db, table)
+          saved = stats["rows"]
+
+          @compared = @compared + 1
+          if saved.to_i == imported.to_i
+            $LOGGER.debug "Row count match for #{object} matches"
+          else
+            $LOGGER.warn "Row count failed for #{object}. Saved #{saved} vs imported #{imported}"
+          end
+        end
+      end
+
+      def get_table_count db, table
+        sql = "SELECT COUNT(1) FROM #{db}.#{table}"
+        exec "mysql -e \"#{sql}\"|awk 'NR>1'"
+      end
+
+      def drop_database db
+        $LOGGER.debug "Dropping database #{db}"
+        sql = "DROP DATABASE #{db}"
+        exec "mysql -e '#{sql}'"
+      end
+
+      def check_table database, table
+        @compared = @compared + 1
+        rs = get_result("CHECK TABLE #{database}.#{table}")
+        if rs["Msg_text"] != "OK"
+          @failures += 1
+          $LOGGER.warn "Check table failed for #{database}.#{table}"
+        else
+          $LOGGER.debug "Check table passed for #{database}.#{table}"
+        end
+      end
+
+      def clean_up
+        $LOGGER.info "Cleaning #{@jobdir}"
+        @compsize = File.size(@download)
+        exec "rm -fr #{@jobdir}"
+      end
+
+      def report
+        $LOGGER.info "Reporting that #{@compared} checks were compared"
+        if @failures > 0
+          $LOGGER.warn "#{@failures} checks failed"
+        else
+          $LOGGER.info "All checks passed"
+        end
+
+        @details[:file]      = @download
+        @details[:size]      = @compsize
+        @details[:view]      = "https://console.aws.amazon.com/s3/home?region=eu-west-1#&bucket=#{@s3bucket}&prefix=#{@s3path}"
+        @details[:report]    = "#{@compared} stats compared with #{@failures} failures"
+        @details[:completed] = Time.now
+      end
+
+    end
+  end
+end

data/lib/backups/version.rb

@@ -0,0 +1,4 @@
+
+module Backups
+  VERSION = "1.0.9"
+end

metadata

@@ -0,0 +1,173 @@
+--- !ruby/object:Gem::Specification
+name: backups-cli
+version: !ruby/object:Gem::Version
+  version: 1.0.9
+platform: ruby
+authors:
+- Schibsted
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-03-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: mysql2
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: dogapi
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: slack-notifier
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: tablelize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This tool backups different data sources to S3.
+email:
+- spt@schibsted.com
+executables:
+- backups
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- backups-cli.gemspec
+- bin/backups
+- lib/backups.rb
+- lib/backups/adapter/mysql.rb
+- lib/backups/base.rb
+- lib/backups/cli.rb
+- lib/backups/crontab.rb
+- lib/backups/driver/mysql.rb
+- lib/backups/events.rb
+- lib/backups/ext/fixnum.rb
+- lib/backups/ext/hash.rb
+- lib/backups/ext/nil_class.rb
+- lib/backups/ext/ordered_hash.rb
+- lib/backups/ext/string.rb
+- lib/backups/listener.rb
+- lib/backups/listeners/notify/datadog.rb
+- lib/backups/listeners/notify/slack.rb
+- lib/backups/loader.rb
+- lib/backups/logger.rb
+- lib/backups/runner.rb
+- lib/backups/stats/mysql.rb
+- lib/backups/system.rb
+- lib/backups/verify/mysql.rb
+- lib/backups/version.rb
+homepage: https://github.com/schibsted/backups-cli
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.2.1
+signing_key: 
+specification_version: 4
+summary: This tool backups different data sources to S3
+test_files: []